diff options
916 files changed, 18684 insertions, 16983 deletions
@@ -213,6 +213,7 @@ Kees Cook <keescook@chromium.org> <kees@ubuntu.com> Keith Busch <kbusch@kernel.org> <keith.busch@intel.com> Keith Busch <kbusch@kernel.org> <keith.busch@linux.intel.com> Kenneth W Chen <kenneth.w.chen@intel.com> +Kirill Tkhai <kirill.tkhai@openvz.org> <ktkhai@virtuozzo.com> Konstantin Khlebnikov <koct9i@gmail.com> <khlebnikov@yandex-team.ru> Konstantin Khlebnikov <koct9i@gmail.com> <k.khlebnikov@samsung.com> Koushik <raghavendra.koushik@neterion.com> diff --git a/Documentation/PCI/pci.rst b/Documentation/PCI/pci.rst index 87c6f4a6ca32..67a850b55617 100644 --- a/Documentation/PCI/pci.rst +++ b/Documentation/PCI/pci.rst @@ -278,20 +278,20 @@ appropriate parameters. In general this allows more efficient DMA on systems where System RAM exists above 4G _physical_ address. Drivers for all PCI-X and PCIe compliant devices must call -pci_set_dma_mask() as they are 64-bit DMA devices. +set_dma_mask() as they are 64-bit DMA devices. Similarly, drivers must also "register" this capability if the device -can directly address "consistent memory" in System RAM above 4G physical -address by calling pci_set_consistent_dma_mask(). +can directly address "coherent memory" in System RAM above 4G physical +address by calling dma_set_coherent_mask(). Again, this includes drivers for all PCI-X and PCIe compliant devices. Many 64-bit "PCI" devices (before PCI-X) and some PCI-X devices are 64-bit DMA capable for payload ("streaming") data but not control -("consistent") data. +("coherent") data. Setup shared control data ------------------------- -Once the DMA masks are set, the driver can allocate "consistent" (a.k.a. shared) +Once the DMA masks are set, the driver can allocate "coherent" (a.k.a. shared) memory. See Documentation/core-api/dma-api.rst for a full description of the DMA APIs. This section is just a reminder that it needs to be done before enabling DMA on the device. @@ -367,7 +367,7 @@ steps need to be performed: - Disable the device from generating IRQs - Release the IRQ (free_irq()) - Stop all DMA activity - - Release DMA buffers (both streaming and consistent) + - Release DMA buffers (both streaming and coherent) - Unregister from other subsystems (e.g. scsi or netdev) - Disable device from responding to MMIO/IO Port addresses - Release MMIO/IO Port resource(s) @@ -420,7 +420,7 @@ Once DMA is stopped, clean up streaming DMA first. I.e. unmap data buffers and return buffers to "upstream" owners if there is one. -Then clean up "consistent" buffers which contain the control data. +Then clean up "coherent" buffers which contain the control data. See Documentation/core-api/dma-api.rst for details on unmapping interfaces. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b7ccaa2ea867..3f1cc5e317ed 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4427,6 +4427,12 @@ fully seed the kernel's CRNG. Default is controlled by CONFIG_RANDOM_TRUST_CPU. + random.trust_bootloader={on,off} + [KNL] Enable or disable trusting the use of a + seed passed by the bootloader (if available) to + fully seed the kernel's CRNG. Default is controlled + by CONFIG_RANDOM_TRUST_BOOTLOADER. + randomize_kstack_offset= [KNL] Enable or disable kernel stack offset randomization, which provides roughly 5 bits of diff --git a/Documentation/core-api/xarray.rst b/Documentation/core-api/xarray.rst index a137a0e6d068..77e0ece2b1d6 100644 --- a/Documentation/core-api/xarray.rst +++ b/Documentation/core-api/xarray.rst @@ -315,11 +315,15 @@ indeed the normal API is implemented in terms of the advanced API. The advanced API is only available to modules with a GPL-compatible license. The advanced API is based around the xa_state. This is an opaque data -structure which you declare on the stack using the XA_STATE() -macro. This macro initialises the xa_state ready to start walking -around the XArray. It is used as a cursor to maintain the position -in the XArray and let you compose various operations together without -having to restart from the top every time. +structure which you declare on the stack using the XA_STATE() macro. +This macro initialises the xa_state ready to start walking around the +XArray. It is used as a cursor to maintain the position in the XArray +and let you compose various operations together without having to restart +from the top every time. The contents of the xa_state are protected by +the rcu_read_lock() or the xas_lock(). If you need to drop whichever of +those locks is protecting your state and tree, you must call xas_pause() +so that future calls do not rely on the parts of the state which were +left unprotected. The xa_state is also used to store errors. You can call xas_error() to retrieve the error. All operations check whether diff --git a/Documentation/dev-tools/kunit/architecture.rst b/Documentation/dev-tools/kunit/architecture.rst index aa2cea821e25..ff9c85a0bff2 100644 --- a/Documentation/dev-tools/kunit/architecture.rst +++ b/Documentation/dev-tools/kunit/architecture.rst @@ -26,10 +26,7 @@ The fundamental unit in KUnit is the test case. The KUnit test cases are grouped into KUnit suites. A KUnit test case is a function with type signature ``void (*)(struct kunit *test)``. These test case functions are wrapped in a struct called -``struct kunit_case``. For code, see: - -.. kernel-doc:: include/kunit/test.h - :identifiers: kunit_case +struct kunit_case. .. note: ``generate_params`` is optional for non-parameterized tests. @@ -152,18 +149,12 @@ Parameterized Tests Each KUnit parameterized test is associated with a collection of parameters. The test is invoked multiple times, once for each parameter value and the parameter is stored in the ``param_value`` field. -The test case includes a ``KUNIT_CASE_PARAM()`` macro that accepts a +The test case includes a KUNIT_CASE_PARAM() macro that accepts a generator function. The generator function is passed the previous parameter and returns the next parameter. It also provides a macro to generate common-case generators based on arrays. -For code, see: - -.. kernel-doc:: include/kunit/test.h - :identifiers: KUNIT_ARRAY_PARAM - - kunit_tool (Command Line Test Harness) ====================================== diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-pcie-mirror.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-pcie-mirror.yaml new file mode 100644 index 000000000000..9fbeb626ab23 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-pcie-mirror.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/arm/mediatek/mediatek,mt7622-pcie-mirror.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: MediaTek PCIE Mirror Controller for MT7622 + +maintainers: + - Lorenzo Bianconi <lorenzo@kernel.org> + - Felix Fietkau <nbd@nbd.name> + +description: + The mediatek PCIE mirror provides a configuration interface for PCIE + controller on MT7622 soc. + +properties: + compatible: + items: + - enum: + - mediatek,mt7622-pcie-mirror + - const: syscon + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + pcie_mirror: pcie-mirror@10000400 { + compatible = "mediatek,mt7622-pcie-mirror", "syscon"; + reg = <0 0x10000400 0 0x10>; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml new file mode 100644 index 000000000000..787d6673f952 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/arm/mediatek/mediatek,mt7622-wed.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: MediaTek Wireless Ethernet Dispatch Controller for MT7622 + +maintainers: + - Lorenzo Bianconi <lorenzo@kernel.org> + - Felix Fietkau <nbd@nbd.name> + +description: + The mediatek wireless ethernet dispatch controller can be configured to + intercept and handle access to the WLAN DMA queues and PCIe interrupts + and implement hardware flow offloading from ethernet to WLAN. + +properties: + compatible: + items: + - enum: + - mediatek,mt7622-wed + - const: syscon + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/interrupt-controller/irq.h> + soc { + #address-cells = <2>; + #size-cells = <2>; + wed0: wed@1020a000 { + compatible = "mediatek,mt7622-wed","syscon"; + reg = <0 0x1020a000 0 0x1000>; + interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_LOW>; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt b/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt index 6ce0b212ec6d..606b4b1b709d 100644 --- a/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt +++ b/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt @@ -81,4 +81,4 @@ Example: }; }; -[1]. Documentation/devicetree/bindings/arm/idle-states.yaml +[1]. Documentation/devicetree/bindings/cpu/idle-states.yaml diff --git a/Documentation/devicetree/bindings/arm/psci.yaml b/Documentation/devicetree/bindings/arm/psci.yaml index 8b77cf83a095..dd83ef278af0 100644 --- a/Documentation/devicetree/bindings/arm/psci.yaml +++ b/Documentation/devicetree/bindings/arm/psci.yaml @@ -101,7 +101,7 @@ properties: bindings in [1]) must specify this property. [1] Kernel documentation - ARM idle states bindings - Documentation/devicetree/bindings/arm/idle-states.yaml + Documentation/devicetree/bindings/cpu/idle-states.yaml patternProperties: "^power-domain-": diff --git a/Documentation/devicetree/bindings/arm/idle-states.yaml b/Documentation/devicetree/bindings/cpu/idle-states.yaml index 4d381fa1ee57..fa4d4142ac93 100644 --- a/Documentation/devicetree/bindings/arm/idle-states.yaml +++ b/Documentation/devicetree/bindings/cpu/idle-states.yaml @@ -1,25 +1,30 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/arm/idle-states.yaml# +$id: http://devicetree.org/schemas/cpu/idle-states.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM idle states binding description +title: Idle states binding description maintainers: - Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> + - Anup Patel <anup@brainfault.org> description: |+ ========================================== 1 - Introduction ========================================== - ARM systems contain HW capable of managing power consumption dynamically, - where cores can be put in different low-power states (ranging from simple wfi - to power gating) according to OS PM policies. The CPU states representing the - range of dynamic idle states that a processor can enter at run-time, can be - specified through device tree bindings representing the parameters required to - enter/exit specific idle states on a given processor. + ARM and RISC-V systems contain HW capable of managing power consumption + dynamically, where cores can be put in different low-power states (ranging + from simple wfi to power gating) according to OS PM policies. The CPU states + representing the range of dynamic idle states that a processor can enter at + run-time, can be specified through device tree bindings representing the + parameters required to enter/exit specific idle states on a given processor. + + ========================================== + 2 - ARM idle states + ========================================== According to the Server Base System Architecture document (SBSA, [3]), the power states an ARM CPU can be put into are identified by the following list: @@ -43,8 +48,23 @@ description: |+ The device tree binding definition for ARM idle states is the subject of this document. + ========================================== + 3 - RISC-V idle states + ========================================== + + On RISC-V systems, the HARTs (or CPUs) [6] can be put in platform specific + suspend (or idle) states (ranging from simple WFI, power gating, etc). The + RISC-V SBI v0.3 (or higher) [7] hart state management extension provides a + standard mechanism for OS to request HART state transitions. + + The platform specific suspend (or idle) states of a hart can be either + retentive or non-rententive in nature. A retentive suspend state will + preserve HART registers and CSR values for all privilege modes whereas + a non-retentive suspend state will not preserve HART registers and CSR + values. + =========================================== - 2 - idle-states definitions + 4 - idle-states definitions =========================================== Idle states are characterized for a specific system through a set of @@ -211,10 +231,10 @@ description: |+ properties specification that is the subject of the following sections. =========================================== - 3 - idle-states node + 5 - idle-states node =========================================== - ARM processor idle states are defined within the idle-states node, which is + The processor idle states are defined within the idle-states node, which is a direct child of the cpus node [1] and provides a container where the processor idle states, defined as device tree nodes, are listed. @@ -223,7 +243,7 @@ description: |+ just supports idle_standby, an idle-states node is not required. =========================================== - 4 - References + 6 - References =========================================== [1] ARM Linux Kernel documentation - CPUs bindings @@ -238,9 +258,15 @@ description: |+ [4] ARM Architecture Reference Manuals http://infocenter.arm.com/help/index.jsp - [6] ARM Linux Kernel documentation - Booting AArch64 Linux + [5] ARM Linux Kernel documentation - Booting AArch64 Linux Documentation/arm64/booting.rst + [6] RISC-V Linux Kernel documentation - CPUs bindings + Documentation/devicetree/bindings/riscv/cpus.yaml + + [7] RISC-V Supervisor Binary Interface (SBI) + http://github.com/riscv/riscv-sbi-doc/riscv-sbi.adoc + properties: $nodename: const: idle-states @@ -253,7 +279,7 @@ properties: On ARM 32-bit systems this property is optional This assumes that the "enable-method" property is set to "psci" in the cpu - node[6] that is responsible for setting up CPU idle management in the OS + node[5] that is responsible for setting up CPU idle management in the OS implementation. const: psci @@ -265,8 +291,8 @@ patternProperties: as follows. The idle state entered by executing the wfi instruction (idle_standby - SBSA,[3][4]) is considered standard on all ARM platforms and therefore - must not be listed. + SBSA,[3][4]) is considered standard on all ARM and RISC-V platforms and + therefore must not be listed. In addition to the properties listed above, a state node may require additional properties specific to the entry-method defined in the @@ -275,7 +301,27 @@ patternProperties: properties: compatible: - const: arm,idle-state + enum: + - arm,idle-state + - riscv,idle-state + + arm,psci-suspend-param: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + power_state parameter to pass to the ARM PSCI suspend call. + + Device tree nodes that require usage of PSCI CPU_SUSPEND function + (i.e. idle states node with entry-method property is set to "psci") + must specify this property. + + riscv,sbi-suspend-param: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + suspend_type parameter to pass to the RISC-V SBI HSM suspend call. + + This property is required in idle state nodes of device tree meant + for RISC-V systems. For more details on the suspend_type parameter + refer the SBI specifiation v0.3 (or higher) [7]. local-timer-stop: description: @@ -317,6 +363,8 @@ patternProperties: description: A string used as a descriptive name for the idle state. + additionalProperties: false + required: - compatible - entry-latency-us @@ -658,4 +706,150 @@ examples: }; }; + - | + // Example 3 (RISC-V 64-bit, 4-cpu systems, two clusters): + + cpus { + #size-cells = <0>; + #address-cells = <1>; + + cpu@0 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x0>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_0_0>, <&CPU_NONRET_0_0>, + <&CLUSTER_RET_0>, <&CLUSTER_NONRET_0>; + + cpu_intc0: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x1>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_0_0>, <&CPU_NONRET_0_0>, + <&CLUSTER_RET_0>, <&CLUSTER_NONRET_0>; + + cpu_intc1: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@10 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x10>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_1_0>, <&CPU_NONRET_1_0>, + <&CLUSTER_RET_1>, <&CLUSTER_NONRET_1>; + + cpu_intc10: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@11 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x11>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_1_0>, <&CPU_NONRET_1_0>, + <&CLUSTER_RET_1>, <&CLUSTER_NONRET_1>; + + cpu_intc11: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + idle-states { + CPU_RET_0_0: cpu-retentive-0-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x10000000>; + entry-latency-us = <20>; + exit-latency-us = <40>; + min-residency-us = <80>; + }; + + CPU_NONRET_0_0: cpu-nonretentive-0-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x90000000>; + entry-latency-us = <250>; + exit-latency-us = <500>; + min-residency-us = <950>; + }; + + CLUSTER_RET_0: cluster-retentive-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x11000000>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <250>; + wakeup-latency-us = <130>; + }; + + CLUSTER_NONRET_0: cluster-nonretentive-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x91000000>; + local-timer-stop; + entry-latency-us = <600>; + exit-latency-us = <1100>; + min-residency-us = <2700>; + wakeup-latency-us = <1500>; + }; + + CPU_RET_1_0: cpu-retentive-1-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x10000010>; + entry-latency-us = <20>; + exit-latency-us = <40>; + min-residency-us = <80>; + }; + + CPU_NONRET_1_0: cpu-nonretentive-1-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x90000010>; + entry-latency-us = <250>; + exit-latency-us = <500>; + min-residency-us = <950>; + }; + + CLUSTER_RET_1: cluster-retentive-1 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x11000010>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <250>; + wakeup-latency-us = <130>; + }; + + CLUSTER_NONRET_1: cluster-nonretentive-1 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x91000010>; + local-timer-stop; + entry-latency-us = <600>; + exit-latency-us = <1100>; + min-residency-us = <2700>; + wakeup-latency-us = <1500>; + }; + }; + }; + ... diff --git a/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml new file mode 100644 index 000000000000..b1770640f94b --- /dev/null +++ b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/mediatek,mt6779-keypad.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mediatek's Keypad Controller device tree bindings + +maintainers: + - Fengping Yu <fengping.yu@mediatek.com> + +allOf: + - $ref: "/schemas/input/matrix-keymap.yaml#" + +description: | + Mediatek's Keypad controller is used to interface a SoC with a matrix-type + keypad device. The keypad controller supports multiple row and column lines. + A key can be placed at each intersection of a unique row and a unique column. + The keypad controller can sense a key-press and key-release and report the + event using a interrupt to the cpu. + +properties: + compatible: + oneOf: + - const: mediatek,mt6779-keypad + - items: + - enum: + - mediatek,mt6873-keypad + - const: mediatek,mt6779-keypad + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: kpd + + wakeup-source: + description: use any event on keypad as wakeup event + type: boolean + + debounce-delay-ms: + maximum: 256 + default: 16 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include <dt-bindings/input/input.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + soc { + #address-cells = <2>; + #size-cells = <2>; + + keyboard@10010000 { + compatible = "mediatek,mt6779-keypad"; + reg = <0 0x10010000 0 0x1000>; + interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_FALLING>; + clocks = <&clk26m>; + clock-names = "kpd"; + }; + }; diff --git a/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt b/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt index 535d92885372..9d00f2a8e13a 100644 --- a/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt +++ b/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt @@ -9,7 +9,10 @@ For MT6397/MT6323 MFD bindings see: Documentation/devicetree/bindings/mfd/mt6397.txt Required properties: -- compatible: "mediatek,mt6397-keys" or "mediatek,mt6323-keys" +- compatible: Should be one of: + - "mediatek,mt6397-keys" + - "mediatek,mt6323-keys" + - "mediatek,mt6358-keys" - linux,keycodes: See Documentation/devicetree/bindings/input/input.yaml Optional Properties: diff --git a/Documentation/devicetree/bindings/input/touchscreen/imagis,ist3038c.yaml b/Documentation/devicetree/bindings/input/touchscreen/imagis,ist3038c.yaml new file mode 100644 index 000000000000..e3a2b871e50c --- /dev/null +++ b/Documentation/devicetree/bindings/input/touchscreen/imagis,ist3038c.yaml @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/touchscreen/imagis,ist3038c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Imagis IST30XXC family touchscreen controller bindings + +maintainers: + - Markuss Broks <markuss.broks@gmail.com> + +allOf: + - $ref: touchscreen.yaml# + +properties: + $nodename: + pattern: "^touchscreen@[0-9a-f]+$" + + compatible: + enum: + - imagis,ist3038c + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: + description: Power supply regulator for the chip + + vddio-supply: + description: Power supply regulator for the I2C bus + + touchscreen-size-x: true + touchscreen-size-y: true + touchscreen-fuzz-x: true + touchscreen-fuzz-y: true + touchscreen-inverted-x: true + touchscreen-inverted-y: true + touchscreen-swapped-x-y: true + +additionalProperties: false + +required: + - compatible + - reg + - interrupts + - touchscreen-size-x + - touchscreen-size-y + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + i2c { + #address-cells = <1>; + #size-cells = <0>; + touchscreen@50 { + compatible = "imagis,ist3038c"; + reg = <0x50>; + interrupt-parent = <&gpio>; + interrupts = <13 IRQ_TYPE_EDGE_FALLING>; + vdd-supply = <&ldo1_reg>; + vddio-supply = <&ldo2_reg>; + touchscreen-size-x = <720>; + touchscreen-size-y = <1280>; + touchscreen-fuzz-x = <10>; + touchscreen-fuzz-y = <10>; + touchscreen-inverted-x; + touchscreen-inverted-y; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml index 817794e56227..4f15463611f8 100644 --- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml @@ -106,6 +106,12 @@ properties: phy-mode: $ref: "#/properties/phy-connection-type" + pcs-handle: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Specifies a reference to a node representing a PCS PHY device on a MDIO + bus to link with an external PHY (phy-handle) if exists. + phy-handle: $ref: /schemas/types.yaml#/definitions/phandle description: diff --git a/Documentation/devicetree/bindings/net/mediatek-net.txt b/Documentation/devicetree/bindings/net/mediatek-net.txt index 72d03e07cf7c..f18d70189375 100644 --- a/Documentation/devicetree/bindings/net/mediatek-net.txt +++ b/Documentation/devicetree/bindings/net/mediatek-net.txt @@ -41,6 +41,16 @@ Required properties: - mediatek,pctl: phandle to the syscon node that handles the ports slew rate and driver current: only for MT2701 and MT7623 SoC +Optional properties: +- dma-coherent: present if dma operations are coherent +- mediatek,cci-control: phandle to the cache coherent interconnect node +- mediatek,hifsys: phandle to the mediatek hifsys controller used to provide + various clocks and reset to the system. +- mediatek,wed: a list of phandles to wireless ethernet dispatch nodes for + MT7622 SoC. +- mediatek,pcie-mirror: phandle to the mediatek pcie-mirror controller for + MT7622 SoC. + * Ethernet MAC node Required properties: diff --git a/Documentation/devicetree/bindings/net/micrel.txt b/Documentation/devicetree/bindings/net/micrel.txt index c5ab62c39133..8d157f0295a5 100644 --- a/Documentation/devicetree/bindings/net/micrel.txt +++ b/Documentation/devicetree/bindings/net/micrel.txt @@ -45,20 +45,3 @@ Optional properties: In fiber mode, auto-negotiation is disabled and the PHY can only work in 100base-fx (full and half duplex) modes. - - - lan8814,ignore-ts: If present the PHY will not support timestamping. - - This option acts as check whether Timestamping is supported by - hardware or not. LAN8814 phy support hardware tmestamping. - - - lan8814,latency_rx_10: Configures Latency value of phy in ingress at 10 Mbps. - - - lan8814,latency_tx_10: Configures Latency value of phy in egress at 10 Mbps. - - - lan8814,latency_rx_100: Configures Latency value of phy in ingress at 100 Mbps. - - - lan8814,latency_tx_100: Configures Latency value of phy in egress at 100 Mbps. - - - lan8814,latency_rx_1000: Configures Latency value of phy in ingress at 1000 Mbps. - - - lan8814,latency_tx_1000: Configures Latency value of phy in egress at 1000 Mbps. diff --git a/Documentation/devicetree/bindings/net/mscc,miim.yaml b/Documentation/devicetree/bindings/net/mscc,miim.yaml new file mode 100644 index 000000000000..2c451cfa4e0b --- /dev/null +++ b/Documentation/devicetree/bindings/net/mscc,miim.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/mscc,miim.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microsemi MII Management Controller (MIIM) + +maintainers: + - Alexandre Belloni <alexandre.belloni@bootlin.com> + +allOf: + - $ref: "mdio.yaml#" + +properties: + compatible: + enum: + - mscc,ocelot-miim + - microchip,lan966x-miim + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + reg: + items: + - description: base address + - description: associated reset register for internal PHYs + minItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-frequency: true + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + +unevaluatedProperties: false + +examples: + - | + mdio@107009c { + compatible = "mscc,ocelot-miim"; + reg = <0x107009c 0x36>, <0x10700f0 0x8>; + interrupts = <14>; + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@0 { + reg = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/net/mscc-miim.txt b/Documentation/devicetree/bindings/net/mscc-miim.txt deleted file mode 100644 index 70e0cb1ee485..000000000000 --- a/Documentation/devicetree/bindings/net/mscc-miim.txt +++ /dev/null @@ -1,26 +0,0 @@ -Microsemi MII Management Controller (MIIM) / MDIO -================================================= - -Properties: -- compatible: must be "mscc,ocelot-miim" or "microchip,lan966x-miim" -- reg: The base address of the MDIO bus controller register bank. Optionally, a - second register bank can be defined if there is an associated reset register - for internal PHYs -- #address-cells: Must be <1>. -- #size-cells: Must be <0>. MDIO addresses have no size component. -- interrupts: interrupt specifier (refer to the interrupt binding) - -Typically an MDIO bus might have several children. - -Example: - mdio@107009c { - #address-cells = <1>; - #size-cells = <0>; - compatible = "mscc,ocelot-miim"; - reg = <0x107009c 0x36>, <0x10700f0 0x8>; - interrupts = <14>; - - phy0: ethernet-phy@0 { - reg = <0>; - }; - }; diff --git a/Documentation/devicetree/bindings/net/xilinx_axienet.txt b/Documentation/devicetree/bindings/net/xilinx_axienet.txt index b8e4894bc634..1aa4c6006cd0 100644 --- a/Documentation/devicetree/bindings/net/xilinx_axienet.txt +++ b/Documentation/devicetree/bindings/net/xilinx_axienet.txt @@ -26,7 +26,8 @@ Required properties: specified, the TX/RX DMA interrupts should be on that node instead, and only the Ethernet core interrupt is optionally specified here. -- phy-handle : Should point to the external phy device. +- phy-handle : Should point to the external phy device if exists. Pointing + this to the PCS/PMA PHY is deprecated and should be avoided. See ethernet.txt file in the same directory. - xlnx,rxmem : Set to allocated memory buffer for Rx/Tx in the hardware @@ -68,6 +69,11 @@ Optional properties: required through the core's MDIO interface (i.e. always, unless the PHY is accessed through a different bus). + - pcs-handle: Phandle to the internal PCS/PMA PHY in SGMII or 1000Base-X + modes, where "pcs-handle" should be used to point + to the PCS/PMA PHY, and "phy-handle" should point to an + external PHY if exists. + Example: axi_ethernet_eth: ethernet@40c00000 { compatible = "xlnx,axi-ethernet-1.00.a"; diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index aa5fb64d57eb..d632ac76532e 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -99,6 +99,14 @@ properties: - compatible - interrupt-controller + cpu-idle-states: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + items: + maxItems: 1 + description: | + List of phandles to idle state nodes supported + by this hart (see ./idle-states.yaml). + required: - riscv,isa - interrupt-controller diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml index beeb90e55727..0b767fec39d8 100644 --- a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml @@ -16,16 +16,22 @@ properties: compatible: oneOf: - - const: allwinner,sun6i-a31-rtc - - const: allwinner,sun8i-a23-rtc - - const: allwinner,sun8i-h3-rtc - - const: allwinner,sun8i-r40-rtc - - const: allwinner,sun8i-v3-rtc - - const: allwinner,sun50i-h5-rtc + - enum: + - allwinner,sun6i-a31-rtc + - allwinner,sun8i-a23-rtc + - allwinner,sun8i-h3-rtc + - allwinner,sun8i-r40-rtc + - allwinner,sun8i-v3-rtc + - allwinner,sun50i-h5-rtc + - allwinner,sun50i-h6-rtc + - allwinner,sun50i-h616-rtc + - allwinner,sun50i-r329-rtc - items: - const: allwinner,sun50i-a64-rtc - const: allwinner,sun8i-h3-rtc - - const: allwinner,sun50i-h6-rtc + - items: + - const: allwinner,sun20i-d1-rtc + - const: allwinner,sun50i-r329-rtc reg: maxItems: 1 @@ -37,7 +43,12 @@ properties: - description: RTC Alarm 1 clocks: - maxItems: 1 + minItems: 1 + maxItems: 4 + + clock-names: + minItems: 1 + maxItems: 4 clock-output-names: minItems: 1 @@ -85,6 +96,7 @@ allOf: enum: - allwinner,sun8i-h3-rtc - allwinner,sun50i-h5-rtc + - allwinner,sun50i-h6-rtc then: properties: @@ -96,19 +108,68 @@ allOf: properties: compatible: contains: - const: allwinner,sun50i-h6-rtc + const: allwinner,sun50i-h616-rtc then: properties: - clock-output-names: + clocks: minItems: 3 maxItems: 3 + items: + - description: Bus clock for register access + - description: 24 MHz oscillator + - description: 32 kHz clock from the CCU + + clock-names: + minItems: 3 + maxItems: 3 + items: + - const: bus + - const: hosc + - const: pll-32k + + required: + - clocks + - clock-names - if: properties: compatible: contains: - const: allwinner,sun8i-r40-rtc + const: allwinner,sun50i-r329-rtc + + then: + properties: + clocks: + minItems: 3 + maxItems: 4 + items: + - description: Bus clock for register access + - description: 24 MHz oscillator + - description: AHB parent for internal SPI clock + - description: External 32768 Hz oscillator + + clock-names: + minItems: 3 + maxItems: 4 + items: + - const: bus + - const: hosc + - const: ahb + - const: ext-osc32k + + required: + - clocks + - clock-names + + - if: + properties: + compatible: + contains: + enum: + - allwinner,sun8i-r40-rtc + - allwinner,sun50i-h616-rtc + - allwinner,sun50i-r329-rtc then: properties: @@ -127,7 +188,6 @@ required: - compatible - reg - interrupts - - clock-output-names additionalProperties: false diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91sam9-rtc.txt b/Documentation/devicetree/bindings/rtc/atmel,at91sam9-rtc.txt deleted file mode 100644 index 3f0e2a5950eb..000000000000 --- a/Documentation/devicetree/bindings/rtc/atmel,at91sam9-rtc.txt +++ /dev/null @@ -1,25 +0,0 @@ -Atmel AT91SAM9260 Real Time Timer - -Required properties: -- compatible: should be one of the following: - - "atmel,at91sam9260-rtt" - - "microchip,sam9x60-rtt", "atmel,at91sam9260-rtt" -- reg: should encode the memory region of the RTT controller -- interrupts: rtt alarm/event interrupt -- clocks: should contain the 32 KHz slow clk that will drive the RTT block. -- atmel,rtt-rtc-time-reg: should encode the GPBR register used to store - the time base when the RTT is used as an RTC. - The first cell should point to the GPBR node and the second one - encode the offset within the GPBR block (or in other words, the - GPBR register used to store the time base). - - -Example: - -rtt@fffffd20 { - compatible = "atmel,at91sam9260-rtt"; - reg = <0xfffffd20 0x10>; - interrupts = <1 4 7>; - clocks = <&clk32k>; - atmel,rtt-rtc-time-reg = <&gpbr 0x0>; -}; diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml b/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml new file mode 100644 index 000000000000..0ef1b7ff4a77 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/atmel,at91sam9260-rtt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel AT91 RTT Device Tree Bindings + +allOf: + - $ref: "rtc.yaml#" + +maintainers: + - Alexandre Belloni <alexandre.belloni@bootlin.com> + +properties: + compatible: + oneOf: + - items: + - const: atmel,at91sam9260-rtt + - items: + - const: microchip,sam9x60-rtt + - const: atmel,at91sam9260-rtt + - items: + - const: microchip,sama7g5-rtt + - const: microchip,sam9x60-rtt + - const: atmel,at91sam9260-rtt + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + atmel,rtt-rtc-time-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle to the GPBR node. + - description: Offset within the GPBR block. + description: + Should encode the GPBR register used to store the time base when the + RTT is used as an RTC. The first cell should point to the GPBR node + and the second one encodes the offset within the GPBR block (or in + other words, the GPBR register used to store the time base). + +required: + - compatible + - reg + - interrupts + - clocks + - atmel,rtt-rtc-time-reg + +unevaluatedProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + + rtc@fffffd20 { + compatible = "atmel,at91sam9260-rtt"; + reg = <0xfffffd20 0x10>; + interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&clk32k>; + atmel,rtt-rtc-time-reg = <&gpbr 0x0>; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 8fe2d934f949..01430973ecec 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -560,6 +560,8 @@ patternProperties: description: Ingenieurburo Fur Ic-Technologie (I/F/I) "^ilitek,.*": description: ILI Technology Corporation (ILITEK) + "^imagis,.*": + description: Imagis Technologies Co., Ltd. "^img,.*": description: Imagination Technologies Ltd. "^imi,.*": diff --git a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml index 91a98ccd4226..d060438e1402 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml @@ -55,6 +55,11 @@ properties: - renesas,r8a779a0-wdt # R-Car V3U - const: renesas,rcar-gen3-wdt # R-Car Gen3 and RZ/G2 + - items: + - enum: + - renesas,r8a779f0-wdt # R-Car S4-8 + - const: renesas,rcar-gen4-wdt # R-Car Gen4 + reg: maxItems: 1 diff --git a/Documentation/filesystems/cifs/ksmbd.rst b/Documentation/filesystems/cifs/ksmbd.rst index b0d354fd8066..1af600db2e70 100644 --- a/Documentation/filesystems/cifs/ksmbd.rst +++ b/Documentation/filesystems/cifs/ksmbd.rst @@ -82,10 +82,10 @@ Signing Update Supported. Pre-authentication integrity Supported. SMB3 encryption(CCM, GCM) Supported. (CCM and GCM128 supported, GCM256 in progress) -SMB direct(RDMA) Partially Supported. SMB3 Multi-channel is - required to connect to Windows client. +SMB direct(RDMA) Supported. SMB3 Multi-channel Partially Supported. Planned to implement replay/retry mechanisms for future. +Receive Side Scaling mode Supported. SMB3.1.1 POSIX extension Supported. ACLs Partially Supported. only DACLs available, SACLs (auditing) is planned for the future. For diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 1d831e3cbcb3..8cc536d08f51 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -549,7 +549,7 @@ Pagecache ~~~~~~~~~ For filesystems using Linux's pagecache, the ``->readpage()`` and -``->readpages()`` methods must be modified to verify pages before they +``->readahead()`` methods must be modified to verify pages before they are marked Uptodate. Merely hooking ``->read_iter()`` would be insufficient, since ``->read_iter()`` is not used for memory maps. @@ -611,7 +611,7 @@ workqueue, and then the workqueue work does the decryption or verification. Finally, pages where no decryption or verity error occurred are marked Uptodate, and the pages are unlocked. -Files on ext4 and f2fs may contain holes. Normally, ``->readpages()`` +Files on ext4 and f2fs may contain holes. Normally, ``->readahead()`` simply zeroes holes and sets the corresponding pages Uptodate; no bios are issued. To prevent this case from bypassing fs-verity, these filesystems use fsverity_verify_page() to verify hole pages. @@ -778,7 +778,7 @@ weren't already directly answered in other parts of this document. - To prevent bypassing verification, pages must not be marked Uptodate until they've been verified. Currently, each filesystem is responsible for marking pages Uptodate via - ``->readpages()``. Therefore, currently it's not possible for + ``->readahead()``. Therefore, currently it's not possible for the VFS to do the verification on its own. Changing this would require significant changes to the VFS and all filesystems. diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 2998cec9af4b..c26d854275a0 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -241,8 +241,6 @@ prototypes:: int (*writepages)(struct address_space *, struct writeback_control *); bool (*dirty_folio)(struct address_space *, struct folio *folio); void (*readahead)(struct readahead_control *); - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); int (*write_begin)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); @@ -274,7 +272,6 @@ readpage: yes, unlocks shared writepages: dirty_folio maybe readahead: yes, unlocks shared -readpages: no shared write_begin: locks the page exclusive write_end: yes, unlocks exclusive bmap: @@ -300,9 +297,6 @@ completion. ->readahead() unlocks the pages that I/O is attempted on like ->readpage(). -->readpages() populates the pagecache with the passed pages and starts -I/O against them. They come unlocked upon I/O completion. - ->writepage() is used for two purposes: for "memory cleansing" and for "sync". These are quite different operations and the behaviour may differ depending upon the mode. diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst index 4f373a8ec47b..69f00179fdfe 100644 --- a/Documentation/filesystems/netfs_library.rst +++ b/Documentation/filesystems/netfs_library.rst @@ -7,6 +7,8 @@ Network Filesystem Helper Library .. Contents: - Overview. + - Per-inode context. + - Inode context helper functions. - Buffered read helpers. - Read helper functions. - Read helper structures. @@ -28,6 +30,69 @@ Note that the library module doesn't link against local caching directly, so access must be provided by the netfs. +Per-Inode Context +================= + +The network filesystem helper library needs a place to store a bit of state for +its use on each netfs inode it is helping to manage. To this end, a context +structure is defined:: + + struct netfs_i_context { + const struct netfs_request_ops *ops; + struct fscache_cookie *cache; + }; + +A network filesystem that wants to use netfs lib must place one of these +directly after the VFS ``struct inode`` it allocates, usually as part of its +own struct. This can be done in a way similar to the following:: + + struct my_inode { + struct { + /* These must be contiguous */ + struct inode vfs_inode; + struct netfs_i_context netfs_ctx; + }; + ... + }; + +This allows netfslib to find its state by simple offset from the inode pointer, +thereby allowing the netfslib helper functions to be pointed to directly by the +VFS/VM operation tables. + +The structure contains the following fields: + + * ``ops`` + + The set of operations provided by the network filesystem to netfslib. + + * ``cache`` + + Local caching cookie, or NULL if no caching is enabled. This field does not + exist if fscache is disabled. + + +Inode Context Helper Functions +------------------------------ + +To help deal with the per-inode context, a number helper functions are +provided. Firstly, a function to perform basic initialisation on a context and +set the operations table pointer:: + + void netfs_i_context_init(struct inode *inode, + const struct netfs_request_ops *ops); + +then two functions to cast between the VFS inode structure and the netfs +context:: + + struct netfs_i_context *netfs_i_context(struct inode *inode); + struct inode *netfs_inode(struct netfs_i_context *ctx); + +and finally, a function to get the cache cookie pointer from the context +attached to an inode (or NULL if fscache is disabled):: + + struct fscache_cookie *netfs_i_cookie(struct inode *inode); + + Buffered Read Helpers ===================== @@ -70,38 +135,22 @@ Read Helper Functions Three read helpers are provided:: - void netfs_readahead(struct readahead_control *ractl, - const struct netfs_read_request_ops *ops, - void *netfs_priv); + void netfs_readahead(struct readahead_control *ractl); int netfs_readpage(struct file *file, - struct folio *folio, - const struct netfs_read_request_ops *ops, - void *netfs_priv); + struct page *page); int netfs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned int len, unsigned int flags, struct folio **_folio, - void **_fsdata, - const struct netfs_read_request_ops *ops, - void *netfs_priv); - -Each corresponds to a VM operation, with the addition of a couple of parameters -for the use of the read helpers: + void **_fsdata); - * ``ops`` - - A table of operations through which the helpers can talk to the filesystem. - - * ``netfs_priv`` +Each corresponds to a VM address space operation. These operations use the +state in the per-inode context. - Filesystem private data (can be NULL). - -Both of these values will be stored into the read request structure. - -For ->readahead() and ->readpage(), the network filesystem should just jump -into the corresponding read helper; whereas for ->write_begin(), it may be a +For ->readahead() and ->readpage(), the network filesystem just point directly +at the corresponding read helper; whereas for ->write_begin(), it may be a little more complicated as the network filesystem might want to flush conflicting writes or track dirty data and needs to put the acquired folio if an error occurs after calling the helper. @@ -116,7 +165,7 @@ occurs, the request will get partially completed if sufficient data is read. Additionally, there is:: - * void netfs_subreq_terminated(struct netfs_read_subrequest *subreq, + * void netfs_subreq_terminated(struct netfs_io_subrequest *subreq, ssize_t transferred_or_error, bool was_async); @@ -132,7 +181,7 @@ Read Helper Structures The read helpers make use of a couple of structures to maintain the state of the read. The first is a structure that manages a read request as a whole:: - struct netfs_read_request { + struct netfs_io_request { struct inode *inode; struct address_space *mapping; struct netfs_cache_resources cache_resources; @@ -140,7 +189,7 @@ the read. The first is a structure that manages a read request as a whole:: loff_t start; size_t len; loff_t i_size; - const struct netfs_read_request_ops *netfs_ops; + const struct netfs_request_ops *netfs_ops; unsigned int debug_id; ... }; @@ -187,8 +236,8 @@ The above fields are the ones the netfs can use. They are: The second structure is used to manage individual slices of the overall read request:: - struct netfs_read_subrequest { - struct netfs_read_request *rreq; + struct netfs_io_subrequest { + struct netfs_io_request *rreq; loff_t start; size_t len; size_t transferred; @@ -244,32 +293,26 @@ Read Helper Operations The network filesystem must provide the read helpers with a table of operations through which it can issue requests and negotiate:: - struct netfs_read_request_ops { - void (*init_rreq)(struct netfs_read_request *rreq, struct file *file); - bool (*is_cache_enabled)(struct inode *inode); - int (*begin_cache_operation)(struct netfs_read_request *rreq); - void (*expand_readahead)(struct netfs_read_request *rreq); - bool (*clamp_length)(struct netfs_read_subrequest *subreq); - void (*issue_op)(struct netfs_read_subrequest *subreq); - bool (*is_still_valid)(struct netfs_read_request *rreq); + struct netfs_request_ops { + void (*init_request)(struct netfs_io_request *rreq, struct file *file); + int (*begin_cache_operation)(struct netfs_io_request *rreq); + void (*expand_readahead)(struct netfs_io_request *rreq); + bool (*clamp_length)(struct netfs_io_subrequest *subreq); + void (*issue_read)(struct netfs_io_subrequest *subreq); + bool (*is_still_valid)(struct netfs_io_request *rreq); int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, struct folio *folio, void **_fsdata); - void (*done)(struct netfs_read_request *rreq); + void (*done)(struct netfs_io_request *rreq); void (*cleanup)(struct address_space *mapping, void *netfs_priv); }; The operations are as follows: - * ``init_rreq()`` + * ``init_request()`` [Optional] This is called to initialise the request structure. It is given the file for reference and can modify the ->netfs_priv value. - * ``is_cache_enabled()`` - - [Required] This is called by netfs_write_begin() to ask if the file is being - cached. It should return true if it is being cached and false otherwise. - * ``begin_cache_operation()`` [Optional] This is called to ask the network filesystem to call into the @@ -305,7 +348,7 @@ The operations are as follows: This should return 0 on success and an error code on error. - * ``issue_op()`` + * ``issue_read()`` [Required] The helpers use this to dispatch a subrequest to the server for reading. In the subrequest, ->start, ->len and ->transferred indicate what @@ -420,12 +463,12 @@ The network filesystem's ->begin_cache_operation() method is called to set up a cache and this must call into the cache to do the work. If using fscache, for example, the cache would call:: - int fscache_begin_read_operation(struct netfs_read_request *rreq, + int fscache_begin_read_operation(struct netfs_io_request *rreq, struct fscache_cookie *cookie); passing in the request pointer and the cookie corresponding to the file. -The netfs_read_request object contains a place for the cache to hang its +The netfs_io_request object contains a place for the cache to hang its state:: struct netfs_cache_resources { @@ -443,7 +486,7 @@ operation table looks like the following:: void (*expand_readahead)(struct netfs_cache_resources *cres, loff_t *_start, size_t *_len, loff_t i_size); - enum netfs_read_source (*prepare_read)(struct netfs_read_subrequest *subreq, + enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq, loff_t i_size); int (*read)(struct netfs_cache_resources *cres, @@ -562,4 +605,5 @@ API Function Reference ====================== .. kernel-doc:: include/linux/netfs.h -.. kernel-doc:: fs/netfs/read_helper.c +.. kernel-doc:: fs/netfs/buffered_read.c +.. kernel-doc:: fs/netfs/io.c diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 4f14edf93941..794bd1a66bfb 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -726,8 +726,6 @@ cache in your filesystem. The following members are defined: int (*writepages)(struct address_space *, struct writeback_control *); bool (*dirty_folio)(struct address_space *, struct folio *); void (*readahead)(struct readahead_control *); - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); int (*write_begin)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); @@ -817,15 +815,6 @@ cache in your filesystem. The following members are defined: completes successfully. Setting PageError on any page will be ignored; simply unlock the page if an I/O error occurs. -``readpages`` - called by the VM to read pages associated with the address_space - object. This is essentially just a vector version of readpage. - Instead of just one page, several pages are requested. - readpages is only used for read-ahead, so read errors are - ignored. If anything goes wrong, feel free to give up. - This interface is deprecated and will be removed by the end of - 2020; implement readahead instead. - ``write_begin`` Called by the generic buffered write code to ask the filesystem to prepare to write len bytes at the given offset in the file. diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst index 2d1fc03d346e..ef19b9c13523 100644 --- a/Documentation/kbuild/kbuild.rst +++ b/Documentation/kbuild/kbuild.rst @@ -77,6 +77,17 @@ HOSTLDLIBS ---------- Additional libraries to link against when building host programs. +.. _userkbuildflags: + +USERCFLAGS +---------- +Additional options used for $(CC) when compiling userprogs. + +USERLDFLAGS +----------- +Additional options used for $(LD) when linking userprogs. userprogs are linked +with CC, so $(USERLDFLAGS) should include "-Wl," prefix as applicable. + KBUILD_KCONFIG -------------- Set the top-level Kconfig file to the value of this environment diff --git a/Documentation/kbuild/llvm.rst b/Documentation/kbuild/llvm.rst index d32616891dcf..b854bb413164 100644 --- a/Documentation/kbuild/llvm.rst +++ b/Documentation/kbuild/llvm.rst @@ -49,17 +49,36 @@ example: :: LLVM Utilities -------------- -LLVM has substitutes for GNU binutils utilities. Kbuild supports ``LLVM=1`` -to enable them. :: - - make LLVM=1 - -They can be enabled individually. The full list of the parameters: :: +LLVM has substitutes for GNU binutils utilities. They can be enabled individually. +The full list of supported make variables:: make CC=clang LD=ld.lld AR=llvm-ar NM=llvm-nm STRIP=llvm-strip \ OBJCOPY=llvm-objcopy OBJDUMP=llvm-objdump READELF=llvm-readelf \ HOSTCC=clang HOSTCXX=clang++ HOSTAR=llvm-ar HOSTLD=ld.lld +To simplify the above command, Kbuild supports the ``LLVM`` variable:: + + make LLVM=1 + +If your LLVM tools are not available in your PATH, you can supply their +location using the LLVM variable with a trailing slash:: + + make LLVM=/path/to/llvm/ + +which will use ``/path/to/llvm/clang``, ``/path/to/llvm/ld.lld``, etc. + +If your LLVM tools have a version suffix and you want to test with that +explicit version rather than the unsuffixed executables like ``LLVM=1``, you +can pass the suffix using the ``LLVM`` variable:: + + make LLVM=-14 + +which will use ``clang-14``, ``ld.lld-14``, etc. + +``LLVM=0`` is not the same as omitting ``LLVM`` altogether, it will behave like +``LLVM=1``. If you only wish to use certain LLVM utilities, use their respective +make variables. + The integrated assembler is enabled by default. You can pass ``LLVM_IAS=0`` to disable it. diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index b008b90b92c9..11a296e52d68 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -982,6 +982,8 @@ The syntax is quite similar. The difference is to use "userprogs" instead of When linking bpfilter_umh, it will be passed the extra option -static. + From command line, :ref:`USERCFLAGS and USERLDFLAGS <userkbuildflags>` will also be used. + 5.4 When userspace programs are actually built ---------------------------------------------- diff --git a/Documentation/locking/locktypes.rst b/Documentation/locking/locktypes.rst index bfa75ea1b66a..9933faad4771 100644 --- a/Documentation/locking/locktypes.rst +++ b/Documentation/locking/locktypes.rst @@ -211,9 +211,6 @@ raw_spinlock_t and spinlock_t raw_spinlock_t -------------- -raw_spinlock_t is a strict spinning lock implementation regardless of the -kernel configuration including PREEMPT_RT enabled kernels. - raw_spinlock_t is a strict spinning lock implementation in all kernels, including PREEMPT_RT kernels. Use raw_spinlock_t only in real critical core code, low-level interrupt handling and places where disabling diff --git a/Documentation/maintainer/index.rst b/Documentation/maintainer/index.rst index f0a60435b124..3e03283c144e 100644 --- a/Documentation/maintainer/index.rst +++ b/Documentation/maintainer/index.rst @@ -12,6 +12,7 @@ additions to this manual. configure-git rebasing-and-merging pull-requests + messy-diffstat maintainer-entry-profile modifying-patches diff --git a/Documentation/maintainer/messy-diffstat.rst b/Documentation/maintainer/messy-diffstat.rst new file mode 100644 index 000000000000..c015f66d7621 --- /dev/null +++ b/Documentation/maintainer/messy-diffstat.rst @@ -0,0 +1,96 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================================== +Handling messy pull-request diffstats +===================================== + +Subsystem maintainers routinely use ``git request-pull`` as part of the +process of sending work upstream. Normally, the result includes a nice +diffstat that shows which files will be touched and how much of each will +be changed. Occasionally, though, a repository with a relatively +complicated development history will yield a massive diffstat containing a +great deal of unrelated work. The result looks ugly and obscures what the +pull request is actually doing. This document describes what is happening +and how to fix things up; it is derived from The Wisdom of Linus Torvalds, +found in Linus1_ and Linus2_. + +.. _Linus1: https://lore.kernel.org/lkml/CAHk-=wg3wXH2JNxkQi+eLZkpuxqV+wPiHhw_Jf7ViH33Sw7PHA@mail.gmail.com/ +.. _Linus2: https://lore.kernel.org/lkml/CAHk-=wgXbSa8yq8Dht8at+gxb_idnJ7X5qWZQWRBN4_CUPr=eQ@mail.gmail.com/ + +A Git development history proceeds as a series of commits. In a simplified +manner, mainline kernel development looks like this:: + + ... vM --- vN-rc1 --- vN-rc2 --- vN-rc3 --- ... --- vN-rc7 --- vN + +If one wants to see what has changed between two points, a command like +this will do the job:: + + $ git diff --stat --summary vN-rc2..vN-rc3 + +Here, there are two clear points in the history; Git will essentially +"subtract" the beginning point from the end point and display the resulting +differences. The requested operation is unambiguous and easy enough to +understand. + +When a subsystem maintainer creates a branch and commits changes to it, the +result in the simplest case is a history that looks like:: + + ... vM --- vN-rc1 --- vN-rc2 --- vN-rc3 --- ... --- vN-rc7 --- vN + | + +-- c1 --- c2 --- ... --- cN + +If that maintainer now uses ``git diff`` to see what has changed between +the mainline branch (let's call it "linus") and cN, there are still two +clear endpoints, and the result is as expected. So a pull request +generated with ``git request-pull`` will also be as expected. But now +consider a slightly more complex development history:: + + ... vM --- vN-rc1 --- vN-rc2 --- vN-rc3 --- ... --- vN-rc7 --- vN + | | + | +-- c1 --- c2 --- ... --- cN + | / + +-- x1 --- x2 --- x3 + +Our maintainer has created one branch at vN-rc1 and another at vN-rc2; the +two were then subsequently merged into c2. Now a pull request generated +for cN may end up being messy indeed, and developers often end up wondering +why. + +What is happening here is that there are no longer two clear end points for +the ``git diff`` operation to use. The development culminating in cN +started in two different places; to generate the diffstat, ``git diff`` +ends up having pick one of them and hoping for the best. If the diffstat +starts at vN-rc1, it may end up including all of the changes between there +and the second origin end point (vN-rc2), which is certainly not what our +maintainer had in mind. With all of that extra junk in the diffstat, it +may be impossible to tell what actually happened in the changes leading up +to cN. + +Maintainers often try to resolve this problem by, for example, rebasing the +branch or performing another merge with the linus branch, then recreating +the pull request. This approach tends not to lead to joy at the receiving +end of that pull request; rebasing and/or merging just before pushing +upstream is a well-known way to get a grumpy response. + +So what is to be done? The best response when confronted with this +situation is to indeed to do a merge with the branch you intend your work +to be pulled into, but to do it privately, as if it were the source of +shame. Create a new, throwaway branch and do the merge there:: + + ... vM --- vN-rc1 --- vN-rc2 --- vN-rc3 --- ... --- vN-rc7 --- vN + | | | + | +-- c1 --- c2 --- ... --- cN | + | / | | + +-- x1 --- x2 --- x3 +------------+-- TEMP + +The merge operation resolves all of the complications resulting from the +multiple beginning points, yielding a coherent result that contains only +the differences from the mainline branch. Now it will be possible to +generate a diffstat with the desired information:: + + $ git diff -C --stat --summary linus..TEMP + +Save the output from this command, then simply delete the TEMP branch; +definitely do not expose it to the outside world. Take the saved diffstat +output and edit it into the messy pull request, yielding a result that +shows what is really going on. That request can then be sent upstream. diff --git a/Documentation/networking/dsa/dsa.rst b/Documentation/networking/dsa/dsa.rst index 89bb4fa4c362..ddc1dd039337 100644 --- a/Documentation/networking/dsa/dsa.rst +++ b/Documentation/networking/dsa/dsa.rst @@ -10,21 +10,21 @@ in joining the effort. Design principles ================= -The Distributed Switch Architecture is a subsystem which was primarily designed -to support Marvell Ethernet switches (MV88E6xxx, a.k.a Linkstreet product line) -using Linux, but has since evolved to support other vendors as well. +The Distributed Switch Architecture subsystem was primarily designed to +support Marvell Ethernet switches (MV88E6xxx, a.k.a. Link Street product +line) using Linux, but has since evolved to support other vendors as well. The original philosophy behind this design was to be able to use unmodified Linux tools such as bridge, iproute2, ifconfig to work transparently whether they configured/queried a switch port network device or a regular network device. -An Ethernet switch is typically comprised of multiple front-panel ports, and one -or more CPU or management port. The DSA subsystem currently relies on the +An Ethernet switch typically comprises multiple front-panel ports and one +or more CPU or management ports. The DSA subsystem currently relies on the presence of a management port connected to an Ethernet controller capable of receiving Ethernet frames from the switch. This is a very common setup for all kinds of Ethernet switches found in Small Home and Office products: routers, -gateways, or even top-of-the rack switches. This host Ethernet controller will +gateways, or even top-of-rack switches. This host Ethernet controller will be later referred to as "master" and "cpu" in DSA terminology and code. The D in DSA stands for Distributed, because the subsystem has been designed @@ -33,14 +33,14 @@ using upstream and downstream Ethernet links between switches. These specific ports are referred to as "dsa" ports in DSA terminology and code. A collection of multiple switches connected to each other is called a "switch tree". -For each front-panel port, DSA will create specialized network devices which are +For each front-panel port, DSA creates specialized network devices which are used as controlling and data-flowing endpoints for use by the Linux networking stack. These specialized network interfaces are referred to as "slave" network interfaces in DSA terminology and code. The ideal case for using DSA is when an Ethernet switch supports a "switch tag" which is a hardware feature making the switch insert a specific tag for each -Ethernet frames it received to/from specific ports to help the management +Ethernet frame it receives to/from specific ports to help the management interface figure out: - what port is this frame coming from @@ -125,7 +125,7 @@ other switches from the same fabric, and in this case, the outermost switch ports must decapsulate the packet. Note that in certain cases, it might be the case that the tagging format used -by a leaf switch (not connected directly to the CPU) to not be the same as what +by a leaf switch (not connected directly to the CPU) is not the same as what the network stack sees. This can be seen with Marvell switch trees, where the CPU port can be configured to use either the DSA or the Ethertype DSA (EDSA) format, but the DSA links are configured to use the shorter (without Ethertype) @@ -270,21 +270,21 @@ These interfaces are specialized in order to: to/from specific switch ports - query the switch for ethtool operations: statistics, link state, Wake-on-LAN, register dumps... -- external/internal PHY management: link, auto-negotiation etc. +- manage external/internal PHY: link, auto-negotiation, etc. These slave network devices have custom net_device_ops and ethtool_ops function pointers which allow DSA to introduce a level of layering between the networking -stack/ethtool, and the switch driver implementation. +stack/ethtool and the switch driver implementation. Upon frame transmission from these slave network devices, DSA will look up which -switch tagging protocol is currently registered with these network devices, and +switch tagging protocol is currently registered with these network devices and invoke a specific transmit routine which takes care of adding the relevant switch tag in the Ethernet frames. These frames are then queued for transmission using the master network device -``ndo_start_xmit()`` function, since they contain the appropriate switch tag, the +``ndo_start_xmit()`` function. Since they contain the appropriate switch tag, the Ethernet switch will be able to process these incoming frames from the -management interface and delivers these frames to the physical switch port. +management interface and deliver them to the physical switch port. Graphical representation ------------------------ @@ -330,9 +330,9 @@ MDIO reads/writes towards specific PHY addresses. In most MDIO-connected switches, these functions would utilize direct or indirect PHY addressing mode to return standard MII registers from the switch builtin PHYs, allowing the PHY library and/or to return link status, link partner pages, auto-negotiation -results etc.. +results, etc. -For Ethernet switches which have both external and internal MDIO busses, the +For Ethernet switches which have both external and internal MDIO buses, the slave MII bus can be utilized to mux/demux MDIO reads and writes towards either internal or external MDIO devices this switch might be connected to: internal PHYs, external PHYs, or even external switches. @@ -349,7 +349,7 @@ DSA data structures are defined in ``include/net/dsa.h`` as well as table indication (when cascading switches) - ``dsa_platform_data``: platform device configuration data which can reference - a collection of dsa_chip_data structure if multiples switches are cascaded, + a collection of dsa_chip_data structures if multiple switches are cascaded, the master network device this switch tree is attached to needs to be referenced @@ -426,7 +426,7 @@ logic basically looks like this: "phy-handle" property, if found, this PHY device is created and registered using ``of_phy_connect()`` -- if Device Tree is used, and the PHY device is "fixed", that is, conforms to +- if Device Tree is used and the PHY device is "fixed", that is, conforms to the definition of a non-MDIO managed PHY as defined in ``Documentation/devicetree/bindings/net/fixed-link.txt``, the PHY is registered and connected transparently using the special fixed MDIO bus driver @@ -481,7 +481,7 @@ Device Tree DSA features a standardized binding which is documented in ``Documentation/devicetree/bindings/net/dsa/dsa.txt``. PHY/MDIO library helper functions such as ``of_get_phy_mode()``, ``of_phy_connect()`` are also used to query -per-port PHY specific details: interface connection, MDIO bus location etc.. +per-port PHY specific details: interface connection, MDIO bus location, etc. Driver development ================== @@ -509,7 +509,7 @@ Switch configuration - ``setup``: setup function for the switch, this function is responsible for setting up the ``dsa_switch_ops`` private structure with all it needs: register maps, - interrupts, mutexes, locks etc.. This function is also expected to properly + interrupts, mutexes, locks, etc. This function is also expected to properly configure the switch to separate all network interfaces from each other, that is, they should be isolated by the switch hardware itself, typically by creating a Port-based VLAN ID for each port and allowing only the CPU port and the @@ -526,13 +526,13 @@ PHY devices and link management - ``get_phy_flags``: Some switches are interfaced to various kinds of Ethernet PHYs, if the PHY library PHY driver needs to know about information it cannot obtain on its own (e.g.: coming from switch memory mapped registers), this function - should return a 32-bits bitmask of "flags", that is private between the switch + should return a 32-bit bitmask of "flags" that is private between the switch driver and the Ethernet PHY driver in ``drivers/net/phy/\*``. - ``phy_read``: Function invoked by the DSA slave MDIO bus when attempting to read the switch port MDIO registers. If unavailable, return 0xffff for each read. For builtin switch Ethernet PHYs, this function should allow reading the link - status, auto-negotiation results, link partner pages etc.. + status, auto-negotiation results, link partner pages, etc. - ``phy_write``: Function invoked by the DSA slave MDIO bus when attempting to write to the switch port MDIO registers. If unavailable return a negative error @@ -554,7 +554,7 @@ Ethtool operations ------------------ - ``get_strings``: ethtool function used to query the driver's strings, will - typically return statistics strings, private flags strings etc. + typically return statistics strings, private flags strings, etc. - ``get_ethtool_stats``: ethtool function used to query per-port statistics and return their values. DSA overlays slave network devices general statistics: @@ -564,7 +564,7 @@ Ethtool operations - ``get_sset_count``: ethtool function used to query the number of statistics items - ``get_wol``: ethtool function used to obtain Wake-on-LAN settings per-port, this - function may, for certain implementations also query the master network device + function may for certain implementations also query the master network device Wake-on-LAN settings if this interface needs to participate in Wake-on-LAN - ``set_wol``: ethtool function used to configure Wake-on-LAN settings per-port, @@ -607,14 +607,14 @@ Power management in a fully active state - ``port_enable``: function invoked by the DSA slave network device ndo_open - function when a port is administratively brought up, this function should be - fully enabling a given switch port. DSA takes care of marking the port with + function when a port is administratively brought up, this function should + fully enable a given switch port. DSA takes care of marking the port with ``BR_STATE_BLOCKING`` if the port is a bridge member, or ``BR_STATE_FORWARDING`` if it was not, and propagating these changes down to the hardware - ``port_disable``: function invoked by the DSA slave network device ndo_close - function when a port is administratively brought down, this function should be - fully disabling a given switch port. DSA takes care of marking the port with + function when a port is administratively brought down, this function should + fully disable a given switch port. DSA takes care of marking the port with ``BR_STATE_DISABLED`` and propagating changes to the hardware if this port is disabled while being a bridge member @@ -622,12 +622,12 @@ Bridge layer ------------ - ``port_bridge_join``: bridge layer function invoked when a given switch port is - added to a bridge, this function should be doing the necessary at the switch - level to permit the joining port from being added to the relevant logical + added to a bridge, this function should do what's necessary at the switch + level to permit the joining port to be added to the relevant logical domain for it to ingress/egress traffic with other members of the bridge. - ``port_bridge_leave``: bridge layer function invoked when a given switch port is - removed from a bridge, this function should be doing the necessary at the + removed from a bridge, this function should do what's necessary at the switch level to deny the leaving port from ingress/egress traffic from the remaining bridge members. When the port leaves the bridge, it should be aged out at the switch hardware for the switch to (re) learn MAC addresses behind @@ -663,7 +663,7 @@ Bridge layer point for drivers that need to configure the hardware for enabling this feature. -- ``port_bridge_tx_fwd_unoffload``: bridge layer function invoken when a driver +- ``port_bridge_tx_fwd_unoffload``: bridge layer function invoked when a driver leaves a bridge port which had the TX forwarding offload feature enabled. Bridge VLAN filtering diff --git a/Documentation/riscv/index.rst b/Documentation/riscv/index.rst index ea915c196048..e23b876ad6eb 100644 --- a/Documentation/riscv/index.rst +++ b/Documentation/riscv/index.rst @@ -7,7 +7,6 @@ RISC-V architecture boot-image-header vm-layout - pmu patch-acceptance features diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index 4392b3cb4020..b5feb5b1d905 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -128,6 +128,7 @@ class KernelCmd(Directive): return out def nestedParse(self, lines, fname): + env = self.state.document.settings.env content = ViewList() node = nodes.section() @@ -137,7 +138,7 @@ class KernelCmd(Directive): code_block += "\n " + l lines = code_block + "\n\n" - line_regex = re.compile("^#define LINENO (\S+)\#([0-9]+)$") + line_regex = re.compile("^\.\. LINENO (\S+)\#([0-9]+)$") ln = 0 n = 0 f = fname @@ -154,6 +155,9 @@ class KernelCmd(Directive): self.do_parse(content, node) content = ViewList() + # Add the file to Sphinx build dependencies + env.note_dependency(os.path.abspath(f)) + f = new_f # sphinx counts lines from 0 diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index 8138d69a6987..27b701ed3681 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -33,6 +33,7 @@ u""" import codecs import os +import re import subprocess import sys @@ -82,7 +83,7 @@ class KernelFeat(Directive): env = doc.settings.env cwd = path.dirname(doc.current_source) - cmd = "get_feat.pl rest --dir " + cmd = "get_feat.pl rest --enable-fname --dir " cmd += self.arguments[0] if len(self.arguments) > 1: @@ -102,7 +103,22 @@ class KernelFeat(Directive): shell_env["srctree"] = srctree lines = self.runCmd(cmd, shell=True, cwd=cwd, env=shell_env) - nodeList = self.nestedParse(lines, fname) + + line_regex = re.compile("^\.\. FILE (\S+)$") + + out_lines = "" + + for line in lines.split("\n"): + match = line_regex.search(line) + if match: + fname = match.group(1) + + # Add the file to Sphinx build dependencies + env.note_dependency(os.path.abspath(fname)) + else: + out_lines += line + "\n" + + nodeList = self.nestedParse(out_lines, fname) return nodeList def runCmd(self, cmd, **kwargs): diff --git a/Documentation/sphinx/kernel_include.py b/Documentation/sphinx/kernel_include.py index f523aa68a36b..abe768088377 100755 --- a/Documentation/sphinx/kernel_include.py +++ b/Documentation/sphinx/kernel_include.py @@ -59,6 +59,7 @@ class KernelInclude(Include): u"""KernelInclude (``kernel-include``) directive""" def run(self): + env = self.state.document.settings.env path = os.path.realpath( os.path.expandvars(self.arguments[0])) @@ -70,6 +71,8 @@ class KernelInclude(Include): self.arguments[0] = path + env.note_dependency(os.path.abspath(path)) + #return super(KernelInclude, self).run() # won't work, see HINTs in _run() return self._run() diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index 8189c33b9dda..9395892c7ba3 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -130,7 +130,7 @@ class KernelDocDirective(Directive): result = ViewList() lineoffset = 0; - line_regex = re.compile("^#define LINENO ([0-9]+)$") + line_regex = re.compile("^\.\. LINENO ([0-9]+)$") for line in lines: match = line_regex.search(line) if match: diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py index 24d2b2addcce..cefdbb7e7523 100644 --- a/Documentation/sphinx/kfigure.py +++ b/Documentation/sphinx/kfigure.py @@ -212,7 +212,7 @@ def setupTools(app): if convert_cmd: kernellog.verbose(app, "use convert(1) from: " + convert_cmd) else: - kernellog.warn(app, + kernellog.verbose(app, "Neither inkscape(1) nor convert(1) found.\n" "For SVG to PDF conversion, " "install either Inkscape (https://inkscape.org/) (preferred) or\n" @@ -296,8 +296,10 @@ def convert_image(img_node, translator, src_fname=None): if translator.builder.format == 'latex': if not inkscape_cmd and convert_cmd is None: - kernellog.verbose(app, - "no SVG to PDF conversion available / include SVG raw.") + kernellog.warn(app, + "no SVG to PDF conversion available / include SVG raw." + "\nIncluding large raw SVGs can cause xelatex error." + "\nInstall Inkscape (preferred) or ImageMagick.") img_node.replace_self(file2literal(src_fname)) else: dst_fname = path.join(translator.builder.outdir, fname + '.pdf') diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt index 9a35f50798a6..2c573541ab71 100644 --- a/Documentation/sphinx/requirements.txt +++ b/Documentation/sphinx/requirements.txt @@ -1,2 +1,4 @@ +# jinja2>=3.1 is not compatible with Sphinx<4.0 +jinja2<3.1 sphinx_rtd_theme Sphinx==2.4.4 diff --git a/Documentation/trace/user_events.rst b/Documentation/trace/user_events.rst index bddedabaca80..c180936f49fc 100644 --- a/Documentation/trace/user_events.rst +++ b/Documentation/trace/user_events.rst @@ -7,7 +7,7 @@ user_events: User-based Event Tracing Overview -------- User based trace events allow user processes to create events and trace data -that can be viewed via existing tools, such as ftrace, perf and eBPF. +that can be viewed via existing tools, such as ftrace and perf. To enable this feature, build your kernel with CONFIG_USER_EVENTS=y. Programs can view status of the events via @@ -67,8 +67,7 @@ The command string format is as follows:: Supported Flags ^^^^^^^^^^^^^^^ -**BPF_ITER** - EBPF programs attached to this event will get the raw iovec -struct instead of any data copies for max performance. +None yet Field Format ^^^^^^^^^^^^ @@ -160,7 +159,7 @@ The following values are defined to aid in checking what has been attached: **EVENT_STATUS_FTRACE** - Bit set if ftrace has been attached (Bit 0). -**EVENT_STATUS_PERF** - Bit set if perf/eBPF has been attached (Bit 1). +**EVENT_STATUS_PERF** - Bit set if perf has been attached (Bit 1). Writing Data ------------ @@ -204,13 +203,6 @@ It's advised for user programs to do the following:: **NOTE:** *The write_index is not emitted out into the trace being recorded.* -EBPF ----- -EBPF programs that attach to a user-based event tracepoint are given a pointer -to a struct user_bpf_context. The bpf context contains the data type (which can -be a user or kernel buffer, or can be a pointer to the iovec) and the data -length that was emitted (minus the write_index). - Example Code ------------ See sample code in samples/user_events. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 07a45474abe9..d13fa6600467 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -151,12 +151,6 @@ In order to create user controlled virtual machines on S390, check KVM_CAP_S390_UCONTROL and use the flag KVM_VM_S390_UCONTROL as privileged user (CAP_SYS_ADMIN). -To use hardware assisted virtualization on MIPS (VZ ASE) rather than -the default trap & emulate implementation (which changes the virtual -memory layout to fit in user mode), check KVM_CAP_MIPS_VZ and use the -flag KVM_VM_MIPS_VZ. - - On arm64, the physical address size for a VM (IPA Size limit) is limited to 40bits by default. The limit can be configured if the host supports the extension KVM_CAP_ARM_VM_IPA_SIZE. When supported, use @@ -4081,6 +4075,11 @@ x2APIC MSRs are always allowed, independent of the ``default_allow`` setting, and their behavior depends on the ``X2APIC_ENABLE`` bit of the APIC base register. +.. warning:: + MSR accesses coming from nested vmentry/vmexit are not filtered. + This includes both writes to individual VMCS fields and reads/writes + through the MSR lists pointed to by the VMCS. + If a bit is within one of the defined ranges, read and write accesses are guarded by the bitmap's value for the MSR index if the kind of access is included in the ``struct kvm_msr_filter_range`` flags. If no range @@ -5293,6 +5292,10 @@ type values: KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO Sets the guest physical address of the vcpu_info for a given vCPU. + As with the shared_info page for the VM, the corresponding page may be + dirtied at any time if event channel interrupt delivery is enabled, so + userspace should always assume that the page is dirty without relying + on dirty logging. KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO Sets the guest physical address of an additional pvclock structure @@ -7719,3 +7722,49 @@ only be invoked on a VM prior to the creation of VCPUs. At this time, KVM_PMU_CAP_DISABLE is the only capability. Setting this capability will disable PMU virtualization for that VM. Usermode should adjust CPUID leaf 0xA to reflect that the PMU is disabled. + +9. Known KVM API problems +========================= + +In some cases, KVM's API has some inconsistencies or common pitfalls +that userspace need to be aware of. This section details some of +these issues. + +Most of them are architecture specific, so the section is split by +architecture. + +9.1. x86 +-------- + +``KVM_GET_SUPPORTED_CPUID`` issues +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In general, ``KVM_GET_SUPPORTED_CPUID`` is designed so that it is possible +to take its result and pass it directly to ``KVM_SET_CPUID2``. This section +documents some cases in which that requires some care. + +Local APIC features +~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=1]:ECX[21] (X2APIC) is reported by ``KVM_GET_SUPPORTED_CPUID``, +but it can only be enabled if ``KVM_CREATE_IRQCHIP`` or +``KVM_ENABLE_CAP(KVM_CAP_IRQCHIP_SPLIT)`` are used to enable in-kernel emulation of +the local APIC. + +The same is true for the ``KVM_FEATURE_PV_UNHALT`` paravirtualized feature. + +CPU[EAX=1]:ECX[24] (TSC_DEADLINE) is not reported by ``KVM_GET_SUPPORTED_CPUID``. +It can be enabled if ``KVM_CAP_TSC_DEADLINE_TIMER`` is present and the kernel +has enabled in-kernel emulation of the local APIC. + +Obsolete ioctls and capabilities +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +KVM_CAP_DISABLE_QUIRKS does not let userspace know which quirks are actually +available. Use ``KVM_CHECK_EXTENSION(KVM_CAP_DISABLE_QUIRKS2)`` instead if +available. + +Ordering of KVM_GET_*/KVM_SET_* ioctls +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +TBD diff --git a/Documentation/virt/kvm/index.rst b/Documentation/virt/kvm/index.rst index b6833c7bb474..e0a2c74e1043 100644 --- a/Documentation/virt/kvm/index.rst +++ b/Documentation/virt/kvm/index.rst @@ -8,25 +8,13 @@ KVM :maxdepth: 2 api - amd-memory-encryption - cpuid - halt-polling - hypercalls - locking - mmu - msr - nested-vmx - ppc-pv - s390-diag - s390-pv - s390-pv-boot - timekeeping - vcpu-requests - - review-checklist + devices/index arm/index + s390/index + ppc-pv + x86/index - devices/index - - running-nested-guests + locking + vcpu-requests + review-checklist diff --git a/Documentation/virt/kvm/locking.rst b/Documentation/virt/kvm/locking.rst index 5d27da356836..845a561629f1 100644 --- a/Documentation/virt/kvm/locking.rst +++ b/Documentation/virt/kvm/locking.rst @@ -210,32 +210,47 @@ time it will be set using the Dirty tracking mechanism described above. 3. Reference ------------ -:Name: kvm_lock +``kvm_lock`` +^^^^^^^^^^^^ + :Type: mutex :Arch: any :Protects: - vm_list -:Name: kvm_count_lock +``kvm_count_lock`` +^^^^^^^^^^^^^^^^^^ + :Type: raw_spinlock_t :Arch: any :Protects: - hardware virtualization enable/disable :Comment: 'raw' because hardware enabling/disabling must be atomic /wrt migration. -:Name: kvm_arch::tsc_write_lock -:Type: raw_spinlock +``kvm->mn_invalidate_lock`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:Type: spinlock_t +:Arch: any +:Protects: mn_active_invalidate_count, mn_memslots_update_rcuwait + +``kvm_arch::tsc_write_lock`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:Type: raw_spinlock_t :Arch: x86 :Protects: - kvm_arch::{last_tsc_write,last_tsc_nsec,last_tsc_offset} - tsc offset in vmcb :Comment: 'raw' because updating the tsc offsets must not be preempted. -:Name: kvm->mmu_lock -:Type: spinlock_t +``kvm->mmu_lock`` +^^^^^^^^^^^^^^^^^ +:Type: spinlock_t or rwlock_t :Arch: any :Protects: -shadow page/shadow tlb entry :Comment: it is a spinlock since it is used in mmu notifier. -:Name: kvm->srcu +``kvm->srcu`` +^^^^^^^^^^^^^ :Type: srcu lock :Arch: any :Protects: - kvm->memslots @@ -246,10 +261,20 @@ time it will be set using the Dirty tracking mechanism described above. The srcu index can be stored in kvm_vcpu->srcu_idx per vcpu if it is needed by multiple functions. -:Name: blocked_vcpu_on_cpu_lock +``kvm->slots_arch_lock`` +^^^^^^^^^^^^^^^^^^^^^^^^ +:Type: mutex +:Arch: any (only needed on x86 though) +:Protects: any arch-specific fields of memslots that have to be modified + in a ``kvm->srcu`` read-side critical section. +:Comment: must be held before reading the pointer to the current memslots, + until after all changes to the memslots are complete + +``wakeup_vcpus_on_cpu_lock`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :Type: spinlock_t :Arch: x86 -:Protects: blocked_vcpu_on_cpu +:Protects: wakeup_vcpus_on_cpu :Comment: This is a per-CPU lock and it is used for VT-d posted-interrupts. When VT-d posted-interrupts is supported and the VM has assigned devices, we put the blocked vCPU on the list blocked_vcpu_on_cpu diff --git a/Documentation/virt/kvm/s390/index.rst b/Documentation/virt/kvm/s390/index.rst new file mode 100644 index 000000000000..605f488f0cc5 --- /dev/null +++ b/Documentation/virt/kvm/s390/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================== +KVM for s390 systems +==================== + +.. toctree:: + :maxdepth: 2 + + s390-diag + s390-pv + s390-pv-boot diff --git a/Documentation/virt/kvm/s390-diag.rst b/Documentation/virt/kvm/s390/s390-diag.rst index ca85f030eb0b..ca85f030eb0b 100644 --- a/Documentation/virt/kvm/s390-diag.rst +++ b/Documentation/virt/kvm/s390/s390-diag.rst diff --git a/Documentation/virt/kvm/s390-pv-boot.rst b/Documentation/virt/kvm/s390/s390-pv-boot.rst index 73a6083cb5e7..73a6083cb5e7 100644 --- a/Documentation/virt/kvm/s390-pv-boot.rst +++ b/Documentation/virt/kvm/s390/s390-pv-boot.rst diff --git a/Documentation/virt/kvm/s390-pv.rst b/Documentation/virt/kvm/s390/s390-pv.rst index 8e41a3b63fa5..8e41a3b63fa5 100644 --- a/Documentation/virt/kvm/s390-pv.rst +++ b/Documentation/virt/kvm/s390/s390-pv.rst diff --git a/Documentation/virt/kvm/vcpu-requests.rst b/Documentation/virt/kvm/vcpu-requests.rst index b61d48aec36c..db43ee571f5a 100644 --- a/Documentation/virt/kvm/vcpu-requests.rst +++ b/Documentation/virt/kvm/vcpu-requests.rst @@ -135,6 +135,16 @@ KVM_REQ_UNHALT such as a pending signal, which does not indicate the VCPU's halt emulation should stop, and therefore does not make the request. +KVM_REQ_OUTSIDE_GUEST_MODE + + This "request" ensures the target vCPU has exited guest mode prior to the + sender of the request continuing on. No action needs be taken by the target, + and so no request is actually logged for the target. This request is similar + to a "kick", but unlike a kick it guarantees the vCPU has actually exited + guest mode. A kick only guarantees the vCPU will exit at some point in the + future, e.g. a previous kick may have started the process, but there's no + guarantee the to-be-kicked vCPU has fully exited guest mode. + KVM_REQUEST_MASK ---------------- diff --git a/Documentation/virt/kvm/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index 1c6847fff304..1c6847fff304 100644 --- a/Documentation/virt/kvm/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst diff --git a/Documentation/virt/kvm/cpuid.rst b/Documentation/virt/kvm/x86/cpuid.rst index bda3e3e737d7..bda3e3e737d7 100644 --- a/Documentation/virt/kvm/cpuid.rst +++ b/Documentation/virt/kvm/x86/cpuid.rst diff --git a/Documentation/virt/kvm/x86/errata.rst b/Documentation/virt/kvm/x86/errata.rst new file mode 100644 index 000000000000..806f049b6975 --- /dev/null +++ b/Documentation/virt/kvm/x86/errata.rst @@ -0,0 +1,39 @@ + +======================================= +Known limitations of CPU virtualization +======================================= + +Whenever perfect emulation of a CPU feature is impossible or too hard, KVM +has to choose between not implementing the feature at all or introducing +behavioral differences between virtual machines and bare metal systems. + +This file documents some of the known limitations that KVM has in +virtualizing CPU features. + +x86 +=== + +``KVM_GET_SUPPORTED_CPUID`` issues +---------------------------------- + +x87 features +~~~~~~~~~~~~ + +Unlike most other CPUID feature bits, CPUID[EAX=7,ECX=0]:EBX[6] +(FDP_EXCPTN_ONLY) and CPUID[EAX=7,ECX=0]:EBX]13] (ZERO_FCS_FDS) are +clear if the features are present and set if the features are not present. + +Clearing these bits in CPUID has no effect on the operation of the guest; +if these bits are set on hardware, the features will not be present on +any virtual machine that runs on that hardware. + +**Workaround:** It is recommended to always set these bits in guest CPUID. +Note however that any software (e.g ``WIN87EM.DLL``) expecting these features +to be present likely predates these CPUID feature bits, and therefore +doesn't know to check for them anyway. + +Nested virtualization features +------------------------------ + +TBD + diff --git a/Documentation/virt/kvm/halt-polling.rst b/Documentation/virt/kvm/x86/halt-polling.rst index 4922e4a15f18..4922e4a15f18 100644 --- a/Documentation/virt/kvm/halt-polling.rst +++ b/Documentation/virt/kvm/x86/halt-polling.rst diff --git a/Documentation/virt/kvm/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index e56fa8b9cfca..e56fa8b9cfca 100644 --- a/Documentation/virt/kvm/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst diff --git a/Documentation/virt/kvm/x86/index.rst b/Documentation/virt/kvm/x86/index.rst new file mode 100644 index 000000000000..7ff588826b9f --- /dev/null +++ b/Documentation/virt/kvm/x86/index.rst @@ -0,0 +1,19 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================== +KVM for x86 systems +=================== + +.. toctree:: + :maxdepth: 2 + + amd-memory-encryption + cpuid + errata + halt-polling + hypercalls + mmu + msr + nested-vmx + running-nested-guests + timekeeping diff --git a/Documentation/virt/kvm/mmu.rst b/Documentation/virt/kvm/x86/mmu.rst index 5b1ebad24c77..5b1ebad24c77 100644 --- a/Documentation/virt/kvm/mmu.rst +++ b/Documentation/virt/kvm/x86/mmu.rst diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/x86/msr.rst index 9315fc385fb0..9315fc385fb0 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/x86/msr.rst diff --git a/Documentation/virt/kvm/nested-vmx.rst b/Documentation/virt/kvm/x86/nested-vmx.rst index ac2095d41f02..ac2095d41f02 100644 --- a/Documentation/virt/kvm/nested-vmx.rst +++ b/Documentation/virt/kvm/x86/nested-vmx.rst diff --git a/Documentation/virt/kvm/running-nested-guests.rst b/Documentation/virt/kvm/x86/running-nested-guests.rst index bd70c69468ae..bd70c69468ae 100644 --- a/Documentation/virt/kvm/running-nested-guests.rst +++ b/Documentation/virt/kvm/x86/running-nested-guests.rst diff --git a/Documentation/virt/kvm/timekeeping.rst b/Documentation/virt/kvm/x86/timekeeping.rst index 21ae7efa29ba..21ae7efa29ba 100644 --- a/Documentation/virt/kvm/timekeeping.rst +++ b/Documentation/virt/kvm/x86/timekeeping.rst diff --git a/Documentation/virt/uml/user_mode_linux_howto_v2.rst b/Documentation/virt/uml/user_mode_linux_howto_v2.rst index d5ad96c795f6..863f67b72c05 100644 --- a/Documentation/virt/uml/user_mode_linux_howto_v2.rst +++ b/Documentation/virt/uml/user_mode_linux_howto_v2.rst @@ -1193,6 +1193,26 @@ E.g. ``os_close_file()`` is just a wrapper around ``close()`` which ensures that the userspace function close does not clash with similarly named function(s) in the kernel part. +Using UML as a Test Platform +============================ + +UML is an excellent test platform for device driver development. As +with most things UML, "some user assembly may be required". It is +up to the user to build their emulation environment. UML at present +provides only the kernel infrastructure. + +Part of this infrastructure is the ability to load and parse fdt +device tree blobs as used in Arm or Open Firmware platforms. These +are supplied as an optional extra argument to the kernel command +line:: + + dtb=filename + +The device tree is loaded and parsed at boottime and is accessible by +drivers which query it. At this moment in time this facility is +intended solely for development purposes. UML's own devices do not +query the device tree. + Security Considerations ----------------------- diff --git a/Documentation/vm/page_owner.rst b/Documentation/vm/page_owner.rst index c4de6f8dabe9..65204d7f004f 100644 --- a/Documentation/vm/page_owner.rst +++ b/Documentation/vm/page_owner.rst @@ -125,7 +125,6 @@ Usage additional function: Cull: - -c Cull by comparing stacktrace instead of total block. --cull <rules> Specify culling rules.Culling syntax is key[,key[,...]].Choose a multi-letter key from the **STANDARD FORMAT SPECIFIERS** section. diff --git a/Documentation/vm/unevictable-lru.rst b/Documentation/vm/unevictable-lru.rst index eae3af17f2d9..b280367d6a44 100644 --- a/Documentation/vm/unevictable-lru.rst +++ b/Documentation/vm/unevictable-lru.rst @@ -52,8 +52,13 @@ The infrastructure may also be able to handle other conditions that make pages unevictable, either by definition or by circumstance, in the future. -The Unevictable Page List -------------------------- +The Unevictable LRU Page List +----------------------------- + +The Unevictable LRU page list is a lie. It was never an LRU-ordered list, but a +companion to the LRU-ordered anonymous and file, active and inactive page lists; +and now it is not even a page list. But following familiar convention, here in +this document and in the source, we often imagine it as a fifth LRU page list. The Unevictable LRU infrastructure consists of an additional, per-node, LRU list called the "unevictable" list and an associated page flag, PG_unevictable, to @@ -63,8 +68,8 @@ The PG_unevictable flag is analogous to, and mutually exclusive with, the PG_active flag in that it indicates on which LRU list a page resides when PG_lru is set. -The Unevictable LRU infrastructure maintains unevictable pages on an additional -LRU list for a few reasons: +The Unevictable LRU infrastructure maintains unevictable pages as if they were +on an additional LRU list for a few reasons: (1) We get to "treat unevictable pages just like we treat other pages in the system - which means we get to use the same code to manipulate them, the @@ -72,13 +77,11 @@ LRU list for a few reasons: of the statistics, etc..." [Rik van Riel] (2) We want to be able to migrate unevictable pages between nodes for memory - defragmentation, workload management and memory hotplug. The linux kernel + defragmentation, workload management and memory hotplug. The Linux kernel can only migrate pages that it can successfully isolate from the LRU - lists. If we were to maintain pages elsewhere than on an LRU-like list, - where they can be found by isolate_lru_page(), we would prevent their - migration, unless we reworked migration code to find the unevictable pages - itself. - + lists (or "Movable" pages: outside of consideration here). If we were to + maintain pages elsewhere than on an LRU-like list, where they can be + detected by isolate_lru_page(), we would prevent their migration. The unevictable list does not differentiate between file-backed and anonymous, swap-backed pages. This differentiation is only important while the pages are, @@ -92,8 +95,8 @@ Memory Control Group Interaction -------------------------------- The unevictable LRU facility interacts with the memory control group [aka -memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by extending the -lru_list enum. +memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by +extending the lru_list enum. The memory controller data structure automatically gets a per-node unevictable list as a result of the "arrayification" of the per-node LRU lists (one per @@ -143,7 +146,6 @@ These are currently used in three places in the kernel: and this mark remains for the life of the inode. (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called. - Note that SHM_LOCK is not required to page in the locked pages if they're swapped out; the application must touch the pages manually if it wants to ensure they're in memory. @@ -156,19 +158,19 @@ These are currently used in three places in the kernel: Detecting Unevictable Pages --------------------------- -The function page_evictable() in vmscan.c determines whether a page is +The function page_evictable() in mm/internal.h determines whether a page is evictable or not using the query function outlined above [see section :ref:`Marking address spaces unevictable <mark_addr_space_unevict>`] to check the AS_UNEVICTABLE flag. For address spaces that are so marked after being populated (as SHM regions -might be), the lock action (eg: SHM_LOCK) can be lazy, and need not populate +might be), the lock action (e.g. SHM_LOCK) can be lazy, and need not populate the page tables for the region as does, for example, mlock(), nor need it make any special effort to push any pages in the SHM_LOCK'd area to the unevictable list. Instead, vmscan will do this if and when it encounters the pages during a reclamation scan. -On an unlock action (such as SHM_UNLOCK), the unlocker (eg: shmctl()) must scan +On an unlock action (such as SHM_UNLOCK), the unlocker (e.g. shmctl()) must scan the pages in the region and "rescue" them from the unevictable list if no other condition is keeping them unevictable. If an unevictable region is destroyed, the pages are also "rescued" from the unevictable list in the process of @@ -176,7 +178,7 @@ freeing them. page_evictable() also checks for mlocked pages by testing an additional page flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is -faulted into a VM_LOCKED vma, or found in a vma being VM_LOCKED. +faulted into a VM_LOCKED VMA, or found in a VMA being VM_LOCKED. Vmscan's Handling of Unevictable Pages @@ -186,28 +188,23 @@ If unevictable pages are culled in the fault path, or moved to the unevictable list at mlock() or mmap() time, vmscan will not encounter the pages until they have become evictable again (via munlock() for example) and have been "rescued" from the unevictable list. However, there may be situations where we decide, -for the sake of expediency, to leave a unevictable page on one of the regular +for the sake of expediency, to leave an unevictable page on one of the regular active/inactive LRU lists for vmscan to deal with. vmscan checks for such pages in all of the shrink_{active|inactive|page}_list() functions and will "cull" such pages that it encounters: that is, it diverts those pages to the -unevictable list for the node being scanned. +unevictable list for the memory cgroup and node being scanned. There may be situations where a page is mapped into a VM_LOCKED VMA, but the page is not marked as PG_mlocked. Such pages will make it all the way to -shrink_page_list() where they will be detected when vmscan walks the reverse -map in try_to_unmap(). If try_to_unmap() returns SWAP_MLOCK, -shrink_page_list() will cull the page at that point. +shrink_active_list() or shrink_page_list() where they will be detected when +vmscan walks the reverse map in page_referenced() or try_to_unmap(). The page +is culled to the unevictable list when it is released by the shrinker. To "cull" an unevictable page, vmscan simply puts the page back on the LRU list using putback_lru_page() - the inverse operation to isolate_lru_page() - after dropping the page lock. Because the condition which makes the page unevictable -may change once the page is unlocked, putback_lru_page() will recheck the -unevictable state of a page that it places on the unevictable list. If the -page has become unevictable, putback_lru_page() removes it from the list and -retries, including the page_unevictable() test. Because such a race is a rare -event and movement of pages onto the unevictable list should be rare, these -extra evictabilty checks should not occur in the majority of calls to -putback_lru_page(). +may change once the page is unlocked, __pagevec_lru_add_fn() will recheck the +unevictable state of a page before placing it on the unevictable list. MLOCKED Pages @@ -227,16 +224,25 @@ Nick posted his patch as an alternative to a patch posted by Christoph Lameter to achieve the same objective: hiding mlocked pages from vmscan. In Nick's patch, he used one of the struct page LRU list link fields as a count -of VM_LOCKED VMAs that map the page. This use of the link field for a count -prevented the management of the pages on an LRU list, and thus mlocked pages -were not migratable as isolate_lru_page() could not find them, and the LRU list -link field was not available to the migration subsystem. +of VM_LOCKED VMAs that map the page (Rik van Riel had the same idea three years +earlier). But this use of the link field for a count prevented the management +of the pages on an LRU list, and thus mlocked pages were not migratable as +isolate_lru_page() could not detect them, and the LRU list link field was not +available to the migration subsystem. -Nick resolved this by putting mlocked pages back on the lru list before +Nick resolved this by putting mlocked pages back on the LRU list before attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs. When Nick's patch was integrated with the Unevictable LRU work, the count was -replaced by walking the reverse map to determine whether any VM_LOCKED VMAs -mapped the page. More on this below. +replaced by walking the reverse map when munlocking, to determine whether any +other VM_LOCKED VMAs still mapped the page. + +However, walking the reverse map for each page when munlocking was ugly and +inefficient, and could lead to catastrophic contention on a file's rmap lock, +when many processes which had it mlocked were trying to exit. In 5.18, the +idea of keeping mlock_count in Unevictable LRU list link field was revived and +put to work, without preventing the migration of mlocked pages. This is why +the "Unevictable LRU list" cannot be a linked list of pages now; but there was +no use for that linked list anyway - though its size is maintained for meminfo. Basic Management @@ -250,22 +256,18 @@ PageMlocked() functions. A PG_mlocked page will be placed on the unevictable list when it is added to the LRU. Such pages can be "noticed" by memory management in several places: - (1) in the mlock()/mlockall() system call handlers; + (1) in the mlock()/mlock2()/mlockall() system call handlers; (2) in the mmap() system call handler when mmapping a region with the MAP_LOCKED flag; (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE - flag + flag; - (4) in the fault path, if mlocked pages are "culled" in the fault path, - and when a VM_LOCKED stack segment is expanded; or + (4) in the fault path and when a VM_LOCKED stack segment is expanded; or (5) as mentioned above, in vmscan:shrink_page_list() when attempting to - reclaim a page in a VM_LOCKED VMA via try_to_unmap() - -all of which result in the VM_LOCKED flag being set for the VMA if it doesn't -already have it set. + reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap(). mlocked pages become unlocked and rescued from the unevictable list when: @@ -280,51 +282,53 @@ mlocked pages become unlocked and rescued from the unevictable list when: (4) before a page is COW'd in a VM_LOCKED VMA. -mlock()/mlockall() System Call Handling ---------------------------------------- +mlock()/mlock2()/mlockall() System Call Handling +------------------------------------------------ -Both [do\_]mlock() and [do\_]mlockall() system call handlers call mlock_fixup() +mlock(), mlock2() and mlockall() system call handlers proceed to mlock_fixup() for each VMA in the range specified by the call. In the case of mlockall(), this is the entire active address space of the task. Note that mlock_fixup() is used for both mlocking and munlocking a range of memory. A call to mlock() -an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED is -treated as a no-op, and mlock_fixup() simply returns. +an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED, is +treated as a no-op and mlock_fixup() simply returns. -If the VMA passes some filtering as described in "Filtering Special Vmas" +If the VMA passes some filtering as described in "Filtering Special VMAs" below, mlock_fixup() will attempt to merge the VMA with its neighbors or split -off a subset of the VMA if the range does not cover the entire VMA. Once the -VMA has been merged or split or neither, mlock_fixup() will call -populate_vma_page_range() to fault in the pages via get_user_pages() and to -mark the pages as mlocked via mlock_vma_page(). +off a subset of the VMA if the range does not cover the entire VMA. Any pages +already present in the VMA are then marked as mlocked by mlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range(). + +Before returning from the system call, do_mlock() or mlockall() will call +__mm_populate() to fault in the remaining pages via get_user_pages() and to +mark those pages as mlocked as they are faulted. Note that the VMA being mlocked might be mapped with PROT_NONE. In this case, get_user_pages() will be unable to fault in the pages. That's okay. If pages -do end up getting faulted into this VM_LOCKED VMA, we'll handle them in the -fault path or in vmscan. - -Also note that a page returned by get_user_pages() could be truncated or -migrated out from under us, while we're trying to mlock it. To detect this, -populate_vma_page_range() checks page_mapping() after acquiring the page lock. -If the page is still associated with its mapping, we'll go ahead and call -mlock_vma_page(). If the mapping is gone, we just unlock the page and move on. -In the worst case, this will result in a page mapped in a VM_LOCKED VMA -remaining on a normal LRU list without being PageMlocked(). Again, vmscan will -detect and cull such pages. - -mlock_vma_page() will call TestSetPageMlocked() for each page returned by -get_user_pages(). We use TestSetPageMlocked() because the page might already -be mlocked by another task/VMA and we don't want to do extra work. We -especially do not want to count an mlocked page more than once in the -statistics. If the page was already mlocked, mlock_vma_page() need do nothing -more. - -If the page was NOT already mlocked, mlock_vma_page() attempts to isolate the -page from the LRU, as it is likely on the appropriate active or inactive list -at that time. If the isolate_lru_page() succeeds, mlock_vma_page() will put -back the page - by calling putback_lru_page() - which will notice that the page -is now mlocked and divert the page to the node's unevictable list. If -mlock_vma_page() is unable to isolate the page from the LRU, vmscan will handle -it later if and when it attempts to reclaim the page. +do end up getting faulted into this VM_LOCKED VMA, they will be handled in the +fault path - which is also how mlock2()'s MLOCK_ONFAULT areas are handled. + +For each PTE (or PMD) being faulted into a VMA, the page add rmap function +calls mlock_vma_page(), which calls mlock_page() when the VMA is VM_LOCKED +(unless it is a PTE mapping of a part of a transparent huge page). Or when +it is a newly allocated anonymous page, lru_cache_add_inactive_or_unevictable() +calls mlock_new_page() instead: similar to mlock_page(), but can make better +judgments, since this page is held exclusively and known not to be on LRU yet. + +mlock_page() sets PageMlocked immediately, then places the page on the CPU's +mlock pagevec, to batch up the rest of the work to be done under lru_lock by +__mlock_page(). __mlock_page() sets PageUnevictable, initializes mlock_count +and moves the page to unevictable state ("the unevictable LRU", but with +mlock_count in place of LRU threading). Or if the page was already PageLRU +and PageUnevictable and PageMlocked, it simply increments the mlock_count. + +But in practice that may not work ideally: the page may not yet be on an LRU, or +it may have been temporarily isolated from LRU. In such cases the mlock_count +field cannot be touched, but will be set to 0 later when __pagevec_lru_add_fn() +returns the page to "LRU". Races prohibit mlock_count from being set to 1 then: +rather than risk stranding a page indefinitely as unevictable, always err with +mlock_count on the low side, so that when munlocked the page will be rescued to +an evictable LRU, then perhaps be mlocked again later if vmscan finds it in a +VM_LOCKED VMA. Filtering Special VMAs @@ -339,68 +343,48 @@ mlock_fixup() filters several classes of "special" VMAs: so there is no sense in attempting to visit them. 2) VMAs mapping hugetlbfs page are already effectively pinned into memory. We - neither need nor want to mlock() these pages. However, to preserve the - prior behavior of mlock() - before the unevictable/mlock changes - - mlock_fixup() will call make_pages_present() in the hugetlbfs VMA range to - allocate the huge pages and populate the ptes. + neither need nor want to mlock() these pages. But __mm_populate() includes + hugetlbfs ranges, allocating the huge pages and populating the PTEs. 3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages, - such as the VDSO page, relay channel pages, etc. These pages - are inherently unevictable and are not managed on the LRU lists. - mlock_fixup() treats these VMAs the same as hugetlbfs VMAs. It calls - make_pages_present() to populate the ptes. + such as the VDSO page, relay channel pages, etc. These pages are inherently + unevictable and are not managed on the LRU lists. __mm_populate() includes + these ranges, populating the PTEs if not already populated. + +4) VMAs with VM_MIXEDMAP set are not marked VM_LOCKED, but __mm_populate() + includes these ranges, populating the PTEs if not already populated. Note that for all of these special VMAs, mlock_fixup() does not set the VM_LOCKED flag. Therefore, we won't have to deal with them later during munlock(), munmap() or task exit. Neither does mlock_fixup() account these VMAs against the task's "locked_vm". -.. _munlock_munlockall_handling: munlock()/munlockall() System Call Handling ------------------------------------------- -The munlock() and munlockall() system calls are handled by the same functions - -do_mlock[all]() - as the mlock() and mlockall() system calls with the unlock vs -lock operation indicated by an argument. So, these system calls are also -handled by mlock_fixup(). Again, if called for an already munlocked VMA, -mlock_fixup() simply returns. Because of the VMA filtering discussed above, -VM_LOCKED will not be set in any "special" VMAs. So, these VMAs will be -ignored for munlock. +The munlock() and munlockall() system calls are handled by the same +mlock_fixup() function as mlock(), mlock2() and mlockall() system calls are. +If called to munlock an already munlocked VMA, mlock_fixup() simply returns. +Because of the VMA filtering discussed above, VM_LOCKED will not be set in +any "special" VMAs. So, those VMAs will be ignored for munlock. If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the -specified range. The range is then munlocked via the function -populate_vma_page_range() - the same function used to mlock a VMA range - -passing a flag to indicate that munlock() is being performed. - -Because the VMA access protections could have been changed to PROT_NONE after -faulting in and mlocking pages, get_user_pages() was unreliable for visiting -these pages for munlocking. Because we don't want to leave pages mlocked, -get_user_pages() was enhanced to accept a flag to ignore the permissions when -fetching the pages - all of which should be resident as a result of previous -mlocking. - -For munlock(), populate_vma_page_range() unlocks individual pages by calling -munlock_vma_page(). munlock_vma_page() unconditionally clears the PG_mlocked -flag using TestClearPageMlocked(). As with mlock_vma_page(), -munlock_vma_page() use the Test*PageMlocked() function to handle the case where -the page might have already been unlocked by another task. If the page was -mlocked, munlock_vma_page() updates that zone statistics for the number of -mlocked pages. Note, however, that at this point we haven't checked whether -the page is mapped by other VM_LOCKED VMAs. - -We can't call page_mlock(), the function that walks the reverse map to -check for other VM_LOCKED VMAs, without first isolating the page from the LRU. -page_mlock() is a variant of try_to_unmap() and thus requires that the page -not be on an LRU list [more on these below]. However, the call to -isolate_lru_page() could fail, in which case we can't call page_mlock(). So, -we go ahead and clear PG_mlocked up front, as this might be the only chance we -have. If we can successfully isolate the page, we go ahead and call -page_mlock(), which will restore the PG_mlocked flag and update the zone -page statistics if it finds another VMA holding the page mlocked. If we fail -to isolate the page, we'll have left a potentially mlocked page on the LRU. -This is fine, because we'll catch it later if and if vmscan tries to reclaim -the page. This should be relatively rare. +specified range. All pages in the VMA are then munlocked by munlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range() - the same +function used when mlocking a VMA range, with new flags for the VMA indicating +that it is munlock() being performed. + +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. + +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. Migrating MLOCKED Pages @@ -410,33 +394,38 @@ A page that is being migrated has been isolated from the LRU lists and is held locked across unmapping of the page, updating the page's address space entry and copying the contents and state, until the page table entry has been replaced with an entry that refers to the new page. Linux supports migration -of mlocked pages and other unevictable pages. This involves simply moving the -PG_mlocked and PG_unevictable states from the old page to the new page. +of mlocked pages and other unevictable pages. PG_mlocked is cleared from the +the old page when it is unmapped from the last VM_LOCKED VMA, and set when the +new page is mapped in place of migration entry in a VM_LOCKED VMA. If the page +was unevictable because mlocked, PG_unevictable follows PG_mlocked; but if the +page was unevictable for other reasons, PG_unevictable is copied explicitly. Note that page migration can race with mlocking or munlocking of the same page. -This has been discussed from the mlock/munlock perspective in the respective -sections above. Both processes (migration and m[un]locking) hold the page -locked. This provides the first level of synchronization. Page migration -zeros out the page_mapping of the old page before unlocking it, so m[un]lock -can skip these pages by testing the page mapping under page lock. +There is mostly no problem since page migration requires unmapping all PTEs of +the old page (including munlock where VM_LOCKED), then mapping in the new page +(including mlock where VM_LOCKED). The page table locks provide sufficient +synchronization. -To complete page migration, we place the new and old pages back onto the LRU -after dropping the page lock. The "unneeded" page - old page on success, new -page on failure - will be freed when the reference count held by the migration -process is released. To ensure that we don't strand pages on the unevictable -list because of a race between munlock and migration, page migration uses the -putback_lru_page() function to add migrated pages back to the LRU. +However, since mlock_vma_pages_range() starts by setting VM_LOCKED on a VMA, +before mlocking any pages already present, if one of those pages were migrated +before mlock_pte_range() reached it, it would get counted twice in mlock_count. +To prevent that, mlock_vma_pages_range() temporarily marks the VMA as VM_IO, +so that mlock_vma_page() will skip it. + +To complete page migration, we place the old and new pages back onto the LRU +afterwards. The "unneeded" page - old page on success, new page on failure - +is freed when the reference count held by the migration process is released. Compacting MLOCKED Pages ------------------------ -The unevictable LRU can be scanned for compactable regions and the default -behavior is to do so. /proc/sys/vm/compact_unevictable_allowed controls -this behavior (see Documentation/admin-guide/sysctl/vm.rst). Once scanning of the -unevictable LRU is enabled, the work of compaction is mostly handled by -the page migration code and the same work flow as described in MIGRATING -MLOCKED PAGES will apply. +The memory map can be scanned for compactable regions and the default behavior +is to let unevictable pages be moved. /proc/sys/vm/compact_unevictable_allowed +controls this behavior (see Documentation/admin-guide/sysctl/vm.rst). The work +of compaction is mostly handled by the page migration code and the same work +flow as described in Migrating MLOCKED Pages will apply. + MLOCKING Transparent Huge Pages ------------------------------- @@ -445,51 +434,44 @@ A transparent huge page is represented by a single entry on an LRU list. Therefore, we can only make unevictable an entire compound page, not individual subpages. -If a user tries to mlock() part of a huge page, we want the rest of the -page to be reclaimable. +If a user tries to mlock() part of a huge page, and no user mlock()s the +whole of the huge page, we want the rest of the page to be reclaimable. We cannot just split the page on partial mlock() as split_huge_page() can -fail and new intermittent failure mode for the syscall is undesirable. +fail and a new intermittent failure mode for the syscall is undesirable. -We handle this by keeping PTE-mapped huge pages on normal LRU lists: the -PMD on border of VM_LOCKED VMA will be split into PTE table. +We handle this by keeping PTE-mlocked huge pages on evictable LRU lists: +the PMD on the border of a VM_LOCKED VMA will be split into a PTE table. -This way the huge page is accessible for vmscan. Under memory pressure the +This way the huge page is accessible for vmscan. Under memory pressure the page will be split, subpages which belong to VM_LOCKED VMAs will be moved -to unevictable LRU and the rest can be reclaimed. +to the unevictable LRU and the rest can be reclaimed. + +/proc/meminfo's Unevictable and Mlocked amounts do not include those parts +of a transparent huge page which are mapped only by PTEs in VM_LOCKED VMAs. -See also comment in follow_trans_huge_pmd(). mmap(MAP_LOCKED) System Call Handling ------------------------------------- -In addition the mlock()/mlockall() system calls, an application can request -that a region of memory be mlocked supplying the MAP_LOCKED flag to the mmap() -call. There is one important and subtle difference here, though. mmap() + mlock() -will fail if the range cannot be faulted in (e.g. because mm_populate fails) -and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. The mmaped -area will still have properties of the locked area - aka. pages will not get -swapped out - but major page faults to fault memory in might still happen. +In addition to the mlock(), mlock2() and mlockall() system calls, an application +can request that a region of memory be mlocked by supplying the MAP_LOCKED flag +to the mmap() call. There is one important and subtle difference here, though. +mmap() + mlock() will fail if the range cannot be faulted in (e.g. because +mm_populate fails) and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. +The mmaped area will still have properties of the locked area - pages will not +get swapped out - but major page faults to fault memory in might still happen. -Furthermore, any mmap() call or brk() call that expands the heap by a -task that has previously called mlockall() with the MCL_FUTURE flag will result +Furthermore, any mmap() call or brk() call that expands the heap by a task +that has previously called mlockall() with the MCL_FUTURE flag will result in the newly mapped memory being mlocked. Before the unevictable/mlock -changes, the kernel simply called make_pages_present() to allocate pages and -populate the page table. +changes, the kernel simply called make_pages_present() to allocate pages +and populate the page table. -To mlock a range of memory under the unevictable/mlock infrastructure, the -mmap() handler and task address space expansion functions call +To mlock a range of memory under the unevictable/mlock infrastructure, +the mmap() handler and task address space expansion functions call populate_vma_page_range() specifying the vma and the address range to mlock. -The callers of populate_vma_page_range() will have already added the memory range -to be mlocked to the task's "locked_vm". To account for filtered VMAs, -populate_vma_page_range() returns the number of pages NOT mlocked. All of the -callers then subtract a non-negative return value from the task's locked_vm. A -negative return value represent an error - for example, from get_user_pages() -attempting to fault in a VMA with PROT_NONE access. In this case, we leave the -memory range accounted as locked_vm, as the protections could be changed later -and pages allocated into that region. - munmap()/exit()/exec() System Call Handling ------------------------------------------- @@ -500,81 +482,53 @@ munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages. Before the unevictable/mlock changes, mlocking did not mark the pages in any way, so unmapping them required no processing. -To munlock a range of memory under the unevictable/mlock infrastructure, the -munmap() handler and task address space call tear down function -munlock_vma_pages_all(). The name reflects the observation that one always -specifies the entire VMA range when munlock()ing during unmap of a region. -Because of the VMA filtering when mlocking() regions, only "normal" VMAs that -actually contain mlocked pages will be passed to munlock_vma_pages_all(). - -munlock_vma_pages_all() clears the VM_LOCKED VMA flag and, like mlock_fixup() -for the munlock case, calls __munlock_vma_pages_range() to walk the page table -for the VMA's memory range and munlock_vma_page() each resident page mapped by -the VMA. This effectively munlocks the page, only if this is the last -VM_LOCKED VMA that maps the page. - - -try_to_unmap() --------------- - -Pages can, of course, be mapped into multiple VMAs. Some of these VMAs may -have VM_LOCKED flag set. It is possible for a page mapped into one or more -VM_LOCKED VMAs not to have the PG_mlocked flag set and therefore reside on one -of the active or inactive LRU lists. This could happen if, for example, a task -in the process of munlocking the page could not isolate the page from the LRU. -As a result, vmscan/shrink_page_list() might encounter such a page as described -in section "vmscan's handling of unevictable pages". To handle this situation, -try_to_unmap() checks for VM_LOCKED VMAs while it is walking a page's reverse -map. - -try_to_unmap() is always called, by either vmscan for reclaim or for page -migration, with the argument page locked and isolated from the LRU. Separate -functions handle anonymous and mapped file and KSM pages, as these types of -pages have different reverse map lookup mechanisms, with different locking. -In each case, whether rmap_walk_anon() or rmap_walk_file() or rmap_walk_ksm(), -it will call try_to_unmap_one() for every VMA which might contain the page. - -When trying to reclaim, if try_to_unmap_one() finds the page in a VM_LOCKED -VMA, it will then mlock the page via mlock_vma_page() instead of unmapping it, -and return SWAP_MLOCK to indicate that the page is unevictable: and the scan -stops there. - -mlock_vma_page() is called while holding the page table's lock (in addition -to the page lock, and the rmap lock): to serialize against concurrent mlock or -munlock or munmap system calls, mm teardown (munlock_vma_pages_all), reclaim, -holepunching, and truncation of file pages and their anonymous COWed pages. - - -page_mlock() Reverse Map Scan ---------------------------------- - -When munlock_vma_page() [see section :ref:`munlock()/munlockall() System Call -Handling <munlock_munlockall_handling>` above] tries to munlock a -page, it needs to determine whether or not the page is mapped by any -VM_LOCKED VMA without actually attempting to unmap all PTEs from the -page. For this purpose, the unevictable/mlock infrastructure -introduced a variant of try_to_unmap() called page_mlock(). - -page_mlock() walks the respective reverse maps looking for VM_LOCKED VMAs. When -such a VMA is found the page is mlocked via mlock_vma_page(). This undoes the -pre-clearing of the page's PG_mlocked done by munlock_vma_page. - -Note that page_mlock()'s reverse map walk must visit every VMA in a page's -reverse map to determine that a page is NOT mapped into any VM_LOCKED VMA. -However, the scan can terminate when it encounters a VM_LOCKED VMA. -Although page_mlock() might be called a great many times when munlocking a -large region or tearing down a large address space that has been mlocked via -mlockall(), overall this is a fairly rare event. +For each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). + +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. + +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. + + +Truncating MLOCKED Pages +------------------------ + +File truncation or hole punching forcibly unmaps the deleted pages from +userspace; truncation even unmaps and deletes any private anonymous pages +which had been Copied-On-Write from the file pages now being truncated. + +Mlocked pages can be munlocked and deleted in this way: like with munmap(), +for each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). + +However, if there is a racing munlock(), since mlock_vma_pages_range() starts +munlocking by clearing VM_LOCKED from a VMA, before munlocking all the pages +present, if one of those pages were unmapped by truncation or hole punch before +mlock_pte_range() reached it, it would not be recognized as mlocked by this VMA, +and would not be counted out of mlock_count. In this rare case, a page may +still appear as PageMlocked after it has been fully unmapped: and it is left to +release_pages() (or __page_cache_release()) to clear it and update statistics +before freeing (this event is counted in /proc/vmstat unevictable_pgs_cleared, +which is usually 0). Page Reclaim in shrink_*_list() ------------------------------- -shrink_active_list() culls any obviously unevictable pages - i.e. -!page_evictable(page) - diverting these to the unevictable list. +vmscan's shrink_active_list() culls any obviously unevictable pages - +i.e. !page_evictable(page) pages - diverting those to the unevictable list. However, shrink_active_list() only sees unevictable pages that made it onto the -active/inactive lru lists. Note that these pages do not have PageUnevictable -set - otherwise they would be on the unevictable list and shrink_active_list +active/inactive LRU lists. Note that these pages do not have PageUnevictable +set - otherwise they would be on the unevictable list and shrink_active_list() would never see them. Some examples of these unevictable pages on the LRU lists are: @@ -586,20 +540,15 @@ Some examples of these unevictable pages on the LRU lists are: when an application accesses the page the first time after SHM_LOCK'ing the segment. - (3) mlocked pages that could not be isolated from the LRU and moved to the - unevictable list in mlock_vma_page(). - -shrink_inactive_list() also diverts any unevictable pages that it finds on the -inactive lists to the appropriate node's unevictable list. + (3) pages still mapped into VM_LOCKED VMAs, which should be marked mlocked, + but events left mlock_count too low, so they were munlocked too early. -shrink_inactive_list() should only see SHM_LOCK'd pages that became SHM_LOCK'd -after shrink_active_list() had moved them to the inactive list, or pages mapped -into VM_LOCKED VMAs that munlock_vma_page() couldn't isolate from the LRU to -recheck via page_mlock(). shrink_inactive_list() won't notice the latter, -but will pass on to shrink_page_list(). +vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously +unevictable pages found on the inactive lists to the appropriate memory cgroup +and node unevictable list. -shrink_page_list() again culls obviously unevictable pages that it could -encounter for similar reason to shrink_inactive_list(). Pages mapped into -VM_LOCKED VMAs but without PG_mlocked set will make it all the way to -try_to_unmap(). shrink_page_list() will divert them to the unevictable list -when try_to_unmap() returns SWAP_MLOCK, as discussed above. +rmap's page_referenced_one(), called via vmscan's shrink_active_list() or +shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(), +check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page() +to correct them. Such pages are culled to the unevictable list when released +by the shrinker. diff --git a/MAINTAINERS b/MAINTAINERS index cf896438366b..fd768d43e048 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4640,6 +4640,7 @@ F: drivers/input/touchscreen/chipone_icn8505.c CHROME HARDWARE PLATFORM SUPPORT M: Benson Leung <bleung@chromium.org> +L: chrome-platform@lists.linux.dev S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git F: drivers/platform/chrome/ @@ -4648,6 +4649,7 @@ CHROMEOS EC CODEC DRIVER M: Cheng-Yi Chiang <cychiang@chromium.org> M: Tzung-Bi Shih <tzungbi@google.com> R: Guenter Roeck <groeck@chromium.org> +L: chrome-platform@lists.linux.dev S: Maintained F: Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml F: sound/soc/codecs/cros_ec_codec.* @@ -4655,6 +4657,7 @@ F: sound/soc/codecs/cros_ec_codec.* CHROMEOS EC SUBDRIVERS M: Benson Leung <bleung@chromium.org> R: Guenter Roeck <groeck@chromium.org> +L: chrome-platform@lists.linux.dev S: Maintained F: drivers/power/supply/cros_usbpd-charger.c N: cros_ec @@ -4662,11 +4665,13 @@ N: cros-ec CHROMEOS EC USB TYPE-C DRIVER M: Prashant Malani <pmalani@chromium.org> +L: chrome-platform@lists.linux.dev S: Maintained F: drivers/platform/chrome/cros_ec_typec.c CHROMEOS EC USB PD NOTIFY DRIVER M: Prashant Malani <pmalani@chromium.org> +L: chrome-platform@lists.linux.dev S: Maintained F: drivers/platform/chrome/cros_usbpd_notify.c F: include/linux/platform_data/cros_usbpd_notify.h @@ -5157,6 +5162,20 @@ S: Supported F: drivers/cpuidle/cpuidle-psci.h F: drivers/cpuidle/cpuidle-psci-domain.c +CPUIDLE DRIVER - DT IDLE PM DOMAIN +M: Ulf Hansson <ulf.hansson@linaro.org> +L: linux-pm@vger.kernel.org +S: Supported +F: drivers/cpuidle/dt_idle_genpd.c +F: drivers/cpuidle/dt_idle_genpd.h + +CPUIDLE DRIVER - RISC-V SBI +M: Anup Patel <anup@brainfault.org> +L: linux-pm@vger.kernel.org +L: linux-riscv@lists.infradead.org +S: Maintained +F: drivers/cpuidle/cpuidle-riscv-sbi.c + CRAMFS FILESYSTEM M: Nicolas Pitre <nico@fluxnic.net> S: Maintained @@ -6038,6 +6057,7 @@ F: drivers/scsi/dpt/ DRBD DRIVER M: Philipp Reisner <philipp.reisner@linbit.com> M: Lars Ellenberg <lars.ellenberg@linbit.com> +M: Christoph Böhmwalder <christoph.boehmwalder@linbit.com> L: drbd-dev@lists.linbit.com S: Supported W: http://www.drbd.org @@ -9517,6 +9537,12 @@ M: Stanislaw Gruszka <stf_xl@wp.pl> S: Maintained F: drivers/usb/atm/ueagle-atm.c +IMAGIS TOUCHSCREEN DRIVER +M: Markuss Broks <markuss.broks@gmail.com> +S: Maintained +F: Documentation/devicetree/bindings/input/touchscreen/imagis,ist3038c.yaml +F: drivers/input/touchscreen/imagis.c + IMGTEC ASCII LCD DRIVER M: Paul Burton <paulburton@kernel.org> S: Maintained @@ -10648,9 +10674,9 @@ F: tools/testing/selftests/ KERNEL SMB3 SERVER (KSMBD) M: Namjae Jeon <linkinjeon@kernel.org> -M: Sergey Senozhatsky <senozhatsky@chromium.org> M: Steve French <sfrench@samba.org> M: Hyunchul Lee <hyc.lee@gmail.com> +R: Sergey Senozhatsky <senozhatsky@chromium.org> L: linux-cifs@vger.kernel.org S: Maintained T: git git://git.samba.org/ksmbd.git @@ -14622,6 +14648,12 @@ L: op-tee@lists.trustedfirmware.org S: Maintained F: drivers/char/hw_random/optee-rng.c +OP-TEE RTC DRIVER +M: Clément Léger <clement.leger@bootlin.com> +L: linux-rtc@vger.kernel.org +S: Maintained +F: drivers/rtc/rtc-optee.c + OPA-VNIC DRIVER M: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com> M: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com> @@ -20539,14 +20571,15 @@ F: Documentation/admin-guide/media/zr364xx* F: drivers/media/usb/zr364xx/ USER-MODE LINUX (UML) -M: Jeff Dike <jdike@addtoit.com> M: Richard Weinberger <richard@nod.at> M: Anton Ivanov <anton.ivanov@cambridgegreys.com> +M: Johannes Berg <johannes@sipsolutions.net> L: linux-um@lists.infradead.org S: Maintained W: http://user-mode-linux.sourceforge.net Q: https://patchwork.ozlabs.org/project/linux-um/list/ -T: git git://git.kernel.org/pub/scm/linux/kernel/git/rw/uml.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git next +T: git git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git fixes F: Documentation/virt/uml/ F: arch/um/ F: arch/x86/um/ @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 -PATCHLEVEL = 17 +PATCHLEVEL = 18 SUBLEVEL = 0 -EXTRAVERSION = +EXTRAVERSION = -rc1 NAME = Superb Owl # *DOCUMENTATION* @@ -424,19 +424,26 @@ HOST_LFS_LDFLAGS := $(shell getconf LFS_LDFLAGS 2>/dev/null) HOST_LFS_LIBS := $(shell getconf LFS_LIBS 2>/dev/null) ifneq ($(LLVM),) -HOSTCC = clang -HOSTCXX = clang++ +ifneq ($(filter %/,$(LLVM)),) +LLVM_PREFIX := $(LLVM) +else ifneq ($(filter -%,$(LLVM)),) +LLVM_SUFFIX := $(LLVM) +endif + +HOSTCC = $(LLVM_PREFIX)clang$(LLVM_SUFFIX) +HOSTCXX = $(LLVM_PREFIX)clang++$(LLVM_SUFFIX) else HOSTCC = gcc HOSTCXX = g++ endif -export KBUILD_USERCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ - -O2 -fomit-frame-pointer -std=gnu11 \ - -Wdeclaration-after-statement -export KBUILD_USERLDFLAGS := +KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ + -O2 -fomit-frame-pointer -std=gnu11 \ + -Wdeclaration-after-statement +KBUILD_USERCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(USERCFLAGS) +KBUILD_USERLDFLAGS := $(USERLDFLAGS) -KBUILD_HOSTCFLAGS := $(KBUILD_USERCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS) +KBUILD_HOSTCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS) KBUILD_HOSTCXXFLAGS := -Wall -O2 $(HOST_LFS_CFLAGS) $(HOSTCXXFLAGS) KBUILD_HOSTLDFLAGS := $(HOST_LFS_LDFLAGS) $(HOSTLDFLAGS) KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) @@ -444,14 +451,14 @@ KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) # Make variables (CC, etc...) CPP = $(CC) -E ifneq ($(LLVM),) -CC = clang -LD = ld.lld -AR = llvm-ar -NM = llvm-nm -OBJCOPY = llvm-objcopy -OBJDUMP = llvm-objdump -READELF = llvm-readelf -STRIP = llvm-strip +CC = $(LLVM_PREFIX)clang$(LLVM_SUFFIX) +LD = $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) +AR = $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) +NM = $(LLVM_PREFIX)llvm-nm$(LLVM_SUFFIX) +OBJCOPY = $(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX) +OBJDUMP = $(LLVM_PREFIX)llvm-objdump$(LLVM_SUFFIX) +READELF = $(LLVM_PREFIX)llvm-readelf$(LLVM_SUFFIX) +STRIP = $(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX) else CC = $(CROSS_COMPILE)gcc LD = $(CROSS_COMPILE)ld @@ -531,6 +538,7 @@ export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AW export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE +export KBUILD_USERCFLAGS KBUILD_USERLDFLAGS export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE @@ -785,10 +793,6 @@ ifdef CONFIG_CC_IS_CLANG KBUILD_CPPFLAGS += -Qunused-arguments # The kernel builds with '-std=gnu11' so use of GNU extensions is acceptable. KBUILD_CFLAGS += -Wno-gnu -# CLANG uses a _MergedGlobals as optimization, but this breaks modpost, as the -# source of a reference will be _MergedGlobals and not on of the whitelisted names. -# See modpost pattern 2 -KBUILD_CFLAGS += -mno-global-merge else # gcc inanely warns about local variables called 'main' @@ -1237,8 +1241,8 @@ define filechk_version.h echo \#define LINUX_VERSION_SUBLEVEL $(SUBLEVEL) endef -$(version_h): PATCHLEVEL := $(if $(PATCHLEVEL), $(PATCHLEVEL), 0) -$(version_h): SUBLEVEL := $(if $(SUBLEVEL), $(SUBLEVEL), 0) +$(version_h): PATCHLEVEL := $(or $(PATCHLEVEL), 0) +$(version_h): SUBLEVEL := $(or $(SUBLEVEL), 0) $(version_h): FORCE $(call filechk,version.h) @@ -1621,7 +1625,7 @@ help: @$(MAKE) -f $(srctree)/Documentation/Makefile dochelp @echo '' @echo 'Architecture specific targets ($(SRCARCH)):' - @$(if $(archhelp),$(archhelp),\ + @$(or $(archhelp),\ echo ' No architecture specific help defined for $(SRCARCH)') @echo '' @$(if $(boards), \ @@ -1838,7 +1842,7 @@ $(clean-dirs): clean: $(clean-dirs) $(call cmd,rmfiles) - @find $(if $(KBUILD_EXTMOD), $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ + @find $(or $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ -o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ diff --git a/arch/alpha/include/asm/user.h b/arch/alpha/include/asm/user.h index 3df37492c7b7..c9f525a6aab8 100644 --- a/arch/alpha/include/asm/user.h +++ b/arch/alpha/include/asm/user.h @@ -45,10 +45,4 @@ struct user { char u_comm[32]; /* user command name */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_DATA_START_ADDR (u.start_data) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) - #endif /* _ALPHA_USER_H */ diff --git a/arch/alpha/kernel/syscalls/Makefile b/arch/alpha/kernel/syscalls/Makefile index 6713c65a25e1..b265e4bc16c2 100644 --- a/arch/alpha/kernel/syscalls/Makefile +++ b/arch/alpha/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 187a187706cb..41bcbb460fac 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -92,6 +92,8 @@ ifeq ($(CONFIG_USE_OF),y) OBJS += $(libfdt_objs) fdt_check_mem_start.o endif +OBJS += lib1funcs.o ashldi3.o bswapsdi2.o + targets := vmlinux vmlinux.lds piggy_data piggy.o \ head.o $(OBJS) @@ -126,8 +128,6 @@ endif # Next argument is a linker script LDFLAGS_vmlinux += -T -OBJS += lib1funcs.o ashldi3.o bswapsdi2.o - # We need to prevent any GOTOFF relocs being used with references # to symbols in the .bss section since we cannot relocate them # independently from the rest at run time. This can be achieved by diff --git a/arch/arm/boot/dts/spear1340.dtsi b/arch/arm/boot/dts/spear1340.dtsi index 827e887afbda..13e1bdb3ddbf 100644 --- a/arch/arm/boot/dts/spear1340.dtsi +++ b/arch/arm/boot/dts/spear1340.dtsi @@ -134,9 +134,9 @@ reg = <0xb4100000 0x1000>; interrupts = <0 105 0x4>; status = "disabled"; - dmas = <&dwdma0 12 0 1>, - <&dwdma0 13 1 0>; - dma-names = "tx", "rx"; + dmas = <&dwdma0 13 0 1>, + <&dwdma0 12 1 0>; + dma-names = "rx", "tx"; }; thermal@e07008c4 { diff --git a/arch/arm/boot/dts/spear13xx.dtsi b/arch/arm/boot/dts/spear13xx.dtsi index c87b881b2c8b..913553367687 100644 --- a/arch/arm/boot/dts/spear13xx.dtsi +++ b/arch/arm/boot/dts/spear13xx.dtsi @@ -284,9 +284,9 @@ #size-cells = <0>; interrupts = <0 31 0x4>; status = "disabled"; - dmas = <&dwdma0 4 0 0>, - <&dwdma0 5 0 0>; - dma-names = "tx", "rx"; + dmas = <&dwdma0 5 0 0>, + <&dwdma0 4 0 0>; + dma-names = "rx", "tx"; }; rtc@e0580000 { diff --git a/arch/arm/include/asm/user.h b/arch/arm/include/asm/user.h index c799a3c49342..167d44b550f4 100644 --- a/arch/arm/include/asm/user.h +++ b/arch/arm/include/asm/user.h @@ -77,10 +77,6 @@ struct user{ struct user_fp_struct * u_fp0;/* Used by gdb to help find the values for */ /* the FP registers. */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) /* * User specific VFP registers. If only VFPv2 is present, registers 16 to 31 diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 039feb7cd590..1e8a50a97edf 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1004,7 +1004,8 @@ static void __init reserve_crashkernel(void) total_mem = get_total_mem(); ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base); - if (ret) + /* invalid value specified or crashkernel=0 */ + if (ret || !crash_size) return; if (crash_base <= 0) { diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index b5efecb3d730..d0fa2037460a 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -54,17 +54,17 @@ int notrace unwind_frame(struct stackframe *frame) return -EINVAL; frame->sp = frame->fp; - frame->fp = *(unsigned long *)(fp); - frame->pc = *(unsigned long *)(fp + 4); + frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); + frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4)); #else /* check current frame pointer is within bounds */ if (fp < low + 12 || fp > high - 4) return -EINVAL; /* restore the registers from the stack frame */ - frame->fp = *(unsigned long *)(fp - 12); - frame->sp = *(unsigned long *)(fp - 8); - frame->pc = *(unsigned long *)(fp - 4); + frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 12)); + frame->sp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 8)); + frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 4)); #endif #ifdef CONFIG_KRETPROBES if (is_kretprobe_trampoline(frame->pc)) diff --git a/arch/arm/mach-omap2/omap-secure.c b/arch/arm/mach-omap2/omap-secure.c index 0659ab4cb0af..11677fc2968f 100644 --- a/arch/arm/mach-omap2/omap-secure.c +++ b/arch/arm/mach-omap2/omap-secure.c @@ -59,8 +59,13 @@ static void __init omap_optee_init_check(void) u32 omap_secure_dispatcher(u32 idx, u32 flag, u32 nargs, u32 arg1, u32 arg2, u32 arg3, u32 arg4) { + static u32 buf[NR_CPUS][5]; + u32 *param; + int cpu; u32 ret; - u32 param[5]; + + cpu = get_cpu(); + param = buf[cpu]; param[0] = nargs; param[1] = arg1; @@ -76,6 +81,8 @@ u32 omap_secure_dispatcher(u32 idx, u32 flag, u32 nargs, u32 arg1, u32 arg2, outer_clean_range(__pa(param), __pa(param + 5)); ret = omap_smc2(idx, flag, __pa(param)); + put_cpu(); + return ret; } @@ -119,8 +126,8 @@ phys_addr_t omap_secure_ram_mempool_base(void) #if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_PM) u32 omap3_save_secure_ram(void __iomem *addr, int size) { + static u32 param[5]; u32 ret; - u32 param[5]; if (size != OMAP3_SAVE_SECURE_RAM_SZ) return OMAP3_SAVE_SECURE_RAM_SZ; @@ -153,8 +160,8 @@ u32 omap3_save_secure_ram(void __iomem *addr, int size) u32 rx51_secure_dispatcher(u32 idx, u32 process, u32 flag, u32 nargs, u32 arg1, u32 arg2, u32 arg3, u32 arg4) { + static u32 param[5]; u32 ret; - u32 param[5]; param[0] = nargs+1; /* RX-51 needs number of arguments + 1 */ param[1] = arg1; diff --git a/arch/arm/mach-s3c/mach-jive.c b/arch/arm/mach-s3c/mach-jive.c index 285e1f0f4145..0d7d408c3729 100644 --- a/arch/arm/mach-s3c/mach-jive.c +++ b/arch/arm/mach-s3c/mach-jive.c @@ -236,11 +236,11 @@ static int __init jive_mtdset(char *options) unsigned long set; if (options == NULL || options[0] == '\0') - return 0; + return 1; if (kstrtoul(options, 10, &set)) { printk(KERN_ERR "failed to parse mtdset=%s\n", options); - return 0; + return 1; } switch (set) { @@ -256,7 +256,7 @@ static int __init jive_mtdset(char *options) "using default.", set); } - return 0; + return 1; } /* parse the mtdset= option given to the kernel command line */ diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 4b61541853ea..82ffac621854 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -381,6 +381,7 @@ out: */ postcore_initcall(atomic_pool_init); +#ifdef CONFIG_CMA_AREAS struct dma_contig_early_reserve { phys_addr_t base; unsigned long size; @@ -435,6 +436,7 @@ void __init dma_contiguous_remap(void) iotable_init(&map, 1); } } +#endif static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data) { diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 9ff683612f2a..d7ffccb7fea7 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -88,6 +88,10 @@ extern phys_addr_t arm_lowmem_limit; void __init bootmem_init(void); void arm_mm_memblock_reserve(void); +#ifdef CONFIG_CMA_AREAS void dma_contiguous_remap(void); +#else +static inline void dma_contiguous_remap(void) { } +#endif unsigned long __clear_cr(unsigned long mask); diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile index 4a5c50f67ced..81f13bdf32f2 100644 --- a/arch/arm/tools/Makefile +++ b/arch/arm/tools/Makefile @@ -29,8 +29,7 @@ kapi: $(kapi-hdrs-y) $(gen-y) uapi: $(uapi-hdrs-y) # Create output directory if not already present -_dummy := $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') \ - $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') +$(shell mkdir -p $(kapi) $(uapi)) quiet_cmd_gen_mach = GEN $@ cmd_gen_mach = $(AWK) -f $(real-prereqs) > $@ diff --git a/arch/arm64/boot/dts/amd/Makefile b/arch/arm64/boot/dts/amd/Makefile index 6a6093064a32..68103a8b0ef5 100644 --- a/arch/arm64/boot/dts/amd/Makefile +++ b/arch/arm64/boot/dts/amd/Makefile @@ -1,4 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive.dtb \ - amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb \ - husky.dtb +dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb diff --git a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts index 8e341be9a399..c290d1ce2b03 100644 --- a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts +++ b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts @@ -9,6 +9,7 @@ /dts-v1/; /include/ "amd-seattle-soc.dtsi" +/include/ "amd-seattle-cpus.dtsi" / { model = "AMD Seattle (Rev.B0) Development Board (Overdrive)"; @@ -36,14 +37,6 @@ status = "ok"; }; -&gpio2 { - status = "ok"; -}; - -&gpio3 { - status = "ok"; -}; - &gpio4 { status = "ok"; }; @@ -79,10 +72,6 @@ }; }; -&ipmi_kcs { - status = "ok"; -}; - &smb0 { /include/ "amd-seattle-xgbe-b.dtsi" }; diff --git a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts index 92cef05c6b74..e0926f6bb7c3 100644 --- a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts +++ b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts @@ -9,6 +9,7 @@ /dts-v1/; /include/ "amd-seattle-soc.dtsi" +/include/ "amd-seattle-cpus.dtsi" / { model = "AMD Seattle (Rev.B1) Development Board (Overdrive)"; diff --git a/arch/arm64/boot/dts/amd/amd-overdrive.dts b/arch/arm64/boot/dts/amd/amd-overdrive.dts deleted file mode 100644 index 41b3a6c0993d..000000000000 --- a/arch/arm64/boot/dts/amd/amd-overdrive.dts +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DTS file for AMD Seattle Overdrive Development Board - * - * Copyright (C) 2014 Advanced Micro Devices, Inc. - */ - -/dts-v1/; - -/include/ "amd-seattle-soc.dtsi" - -/ { - model = "AMD Seattle Development Board (Overdrive)"; - compatible = "amd,seattle-overdrive", "amd,seattle"; - - chosen { - stdout-path = &serial0; - }; -}; - -&ccp0 { - status = "ok"; -}; - -&gpio0 { - status = "ok"; -}; - -&gpio1 { - status = "ok"; -}; - -&i2c0 { - status = "ok"; -}; - -&pcie0 { - status = "ok"; -}; - -&spi0 { - status = "ok"; -}; - -&spi1 { - status = "ok"; - sdcard0: sdcard@0 { - compatible = "mmc-spi-slot"; - reg = <0>; - spi-max-frequency = <20000000>; - voltage-ranges = <3200 3400>; - gpios = <&gpio0 7 0>; - interrupt-parent = <&gpio0>; - interrupts = <7 3>; - pl022,hierarchy = <0>; - pl022,interface = <0>; - pl022,com-mode = <0x0>; - pl022,rx-level-trig = <0>; - pl022,tx-level-trig = <0>; - }; -}; - -&v2m0 { - arm,msi-base-spi = <64>; - arm,msi-num-spis = <256>; -}; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi new file mode 100644 index 000000000000..93688a0b6820 --- /dev/null +++ b/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0 + +/ { + cpus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&CPU0>; + }; + core1 { + cpu = <&CPU1>; + }; + }; + cluster1 { + core0 { + cpu = <&CPU2>; + }; + core1 { + cpu = <&CPU3>; + }; + }; + cluster2 { + core0 { + cpu = <&CPU4>; + }; + core1 { + cpu = <&CPU5>; + }; + }; + cluster3 { + core0 { + cpu = <&CPU6>; + }; + core1 { + cpu = <&CPU7>; + }; + }; + }; + + CPU0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_0>; + + }; + + CPU1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x1>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_0>; + }; + + CPU2: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x100>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_1>; + }; + + CPU3: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x101>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_1>; + }; + + CPU4: cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x200>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_2>; + }; + + CPU5: cpu@201 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x201>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_2>; + }; + + CPU6: cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x300>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_3>; + }; + + CPU7: cpu@301 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x301>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_3>; + }; + }; + + L2_0: l2-cache0 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_1: l2-cache1 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_2: l2-cache2 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_3: l2-cache3 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L3: l3-cache { + cache-level = <3>; + cache-size = <0x800000>; + cache-line-size = <64>; + cache-sets = <8192>; + cache-unified; + }; + + pmu { + compatible = "arm,cortex-a57-pmu"; + interrupts = <0x0 0x7 0x4>, + <0x0 0x8 0x4>, + <0x0 0x9 0x4>, + <0x0 0xa 0x4>, + <0x0 0xb 0x4>, + <0x0 0xc 0x4>, + <0x0 0xd 0x4>, + <0x0 0xe 0x4>; + interrupt-affinity = <&CPU0>, + <&CPU1>, + <&CPU2>, + <&CPU3>, + <&CPU4>, + <&CPU5>, + <&CPU6>, + <&CPU7>; + }; +}; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi index b664e7af74eb..690020589d41 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -38,18 +38,6 @@ <1 10 0xff04>; }; - pmu { - compatible = "arm,armv8-pmuv3"; - interrupts = <0 7 4>, - <0 8 4>, - <0 9 4>, - <0 10 4>, - <0 11 4>, - <0 12 4>, - <0 13 4>, - <0 14 4>; - }; - smb0: smb { compatible = "simple-bus"; #address-cells = <2>; @@ -70,6 +58,7 @@ reg = <0 0xe0300000 0 0xf0000>; interrupts = <0 355 4>; clocks = <&sataclk_333mhz>; + iommus = <&sata0_smmu 0x0 0x1f>; dma-coherent; }; @@ -80,6 +69,27 @@ reg = <0 0xe0d00000 0 0xf0000>; interrupts = <0 354 4>; clocks = <&sataclk_333mhz>; + iommus = <&sata1_smmu 0x0e>, + <&sata1_smmu 0x0f>, + <&sata1_smmu 0x1e>; + dma-coherent; + }; + + sata0_smmu: iommu@e0200000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0200000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 332 4>, <0 332 4>; + #iommu-cells = <2>; + dma-coherent; + }; + + sata1_smmu: iommu@e0c00000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0c00000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 331 4>, <0 331 4>; + #iommu-cells = <1>; dma-coherent; }; @@ -201,6 +211,10 @@ reg = <0 0xe0100000 0 0x10000>; interrupts = <0 3 4>; dma-coherent; + iommus = <&sata1_smmu 0x00>, + <&sata1_smmu 0x02>, + <&sata1_smmu 0x40>, + <&sata1_smmu 0x42>; }; pcie0: pcie@f0000000 { @@ -213,12 +227,22 @@ msi-parent = <&v2m0>; reg = <0 0xf0000000 0 0x10000000>; - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; + interrupt-map-mask = <0xff00 0x0 0x0 0x7>; interrupt-map = - <0x1000 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x120 0x1>, - <0x1000 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x121 0x1>, - <0x1000 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x122 0x1>, - <0x1000 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x123 0x1>; + <0x1100 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x120 0x1>, + <0x1100 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x121 0x1>, + <0x1100 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x122 0x1>, + <0x1100 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x123 0x1>, + + <0x1200 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x124 0x1>, + <0x1200 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x125 0x1>, + <0x1200 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x126 0x1>, + <0x1200 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x127 0x1>, + + <0x1300 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x128 0x1>, + <0x1300 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x129 0x1>, + <0x1300 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x12a 0x1>, + <0x1300 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x12b 0x1>; dma-coherent; dma-ranges = <0x43000000 0x0 0x0 0x0 0x0 0x100 0x0>; @@ -227,8 +251,18 @@ <0x01000000 0x00 0x00000000 0x00 0xefff0000 0x00 0x00010000>, /* 32-bit MMIO (size=2G) */ <0x02000000 0x00 0x40000000 0x00 0x40000000 0x00 0x80000000>, - /* 64-bit MMIO (size= 124G) */ + /* 64-bit MMIO (size= 508G) */ <0x03000000 0x01 0x00000000 0x01 0x00000000 0x7f 0x00000000>; + iommu-map = <0x0 &pcie_smmu 0x0 0x10000>; + }; + + pcie_smmu: iommu@e0a00000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0a00000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 333 4>, <0 333 4>; + #iommu-cells = <1>; + dma-coherent; }; /* Perf CCN504 PMU */ diff --git a/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi index d97498361ce3..9259e547e2e8 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi @@ -55,7 +55,7 @@ clocks = <&xgmacclk0_dma_250mhz>, <&xgmacclk0_ptp_250mhz>; clock-names = "dma_clk", "ptp_clk"; phy-mode = "xgmii"; - #stream-id-cells = <16>; + iommus = <&xgmac0_smmu 0x00 0x17>; /* 0-7, 16-23 */ dma-coherent; }; @@ -81,11 +81,11 @@ clocks = <&xgmacclk1_dma_250mhz>, <&xgmacclk1_ptp_250mhz>; clock-names = "dma_clk", "ptp_clk"; phy-mode = "xgmii"; - #stream-id-cells = <16>; + iommus = <&xgmac1_smmu 0x00 0x17>; /* 0-7, 16-23 */ dma-coherent; }; - xgmac0_smmu: smmu@e0600000 { + xgmac0_smmu: iommu@e0600000 { compatible = "arm,mmu-401"; reg = <0 0xe0600000 0 0x10000>; #global-interrupts = <1>; @@ -94,14 +94,11 @@ */ <0 336 4>, <0 336 4>; - - mmu-masters = <&xgmac0 - 0 1 2 3 4 5 6 7 - 16 17 18 19 20 21 22 23 - >; + #iommu-cells = <2>; + dma-coherent; }; - xgmac1_smmu: smmu@e0800000 { + xgmac1_smmu: iommu@e0800000 { compatible = "arm,mmu-401"; reg = <0 0xe0800000 0 0x10000>; #global-interrupts = <1>; @@ -110,9 +107,6 @@ */ <0 335 4>, <0 335 4>; - - mmu-masters = <&xgmac1 - 0 1 2 3 4 5 6 7 - 16 17 18 19 20 21 22 23 - >; + #iommu-cells = <2>; + dma-coherent; }; diff --git a/arch/arm64/boot/dts/amd/husky.dts b/arch/arm64/boot/dts/amd/husky.dts deleted file mode 100644 index 7acde34772cb..000000000000 --- a/arch/arm64/boot/dts/amd/husky.dts +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DTS file for AMD/Linaro 96Boards Enterprise Edition Server (Husky) Board - * Note: Based-on AMD Seattle Rev.B0 - * - * Copyright (C) 2015 Advanced Micro Devices, Inc. - */ - -/dts-v1/; - -/include/ "amd-seattle-soc.dtsi" - -/ { - model = "Linaro 96Boards Enterprise Edition Server (Husky) Board"; - compatible = "amd,seattle-overdrive", "amd,seattle"; - - chosen { - stdout-path = &serial0; - }; - - psci { - compatible = "arm,psci-0.2"; - method = "smc"; - }; -}; - -&ccp0 { - status = "ok"; - amd,zlib-support = <1>; -}; - -/** - * NOTE: In Rev.B, gpio0 is reserved. - */ -&gpio1 { - status = "ok"; -}; - -&gpio2 { - status = "ok"; -}; - -&gpio3 { - status = "ok"; -}; - -&gpio4 { - status = "ok"; -}; - -&i2c0 { - status = "ok"; -}; - -&i2c1 { - status = "ok"; -}; - -&pcie0 { - status = "ok"; -}; - -&spi0 { - status = "ok"; -}; - -&spi1 { - status = "ok"; - sdcard0: sdcard@0 { - compatible = "mmc-spi-slot"; - reg = <0>; - spi-max-frequency = <20000000>; - voltage-ranges = <3200 3400>; - pl022,hierarchy = <0>; - pl022,interface = <0>; - pl022,com-mode = <0x0>; - pl022,rx-level-trig = <0>; - pl022,tx-level-trig = <0>; - }; -}; - -&smb0 { - /include/ "amd-seattle-xgbe-b.dtsi" -}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi index 01b01e320411..35d1939e690b 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi @@ -536,9 +536,9 @@ clock-names = "i2c"; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(1)>; - dmas = <&edma0 1 39>, - <&edma0 1 38>; - dma-names = "tx", "rx"; + dmas = <&edma0 1 38>, + <&edma0 1 39>; + dma-names = "rx", "tx"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi index 687fea6d8afa..4e7bd04d9798 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi @@ -499,9 +499,9 @@ interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(2)>; - dmas = <&edma0 1 39>, - <&edma0 1 38>; - dma-names = "tx", "rx"; + dmas = <&edma0 1 38>, + <&edma0 1 39>; + dma-names = "rx", "tx"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt7622.dtsi b/arch/arm64/boot/dts/mediatek/mt7622.dtsi index 6f8cb3ad1e84..47d223e28f8d 100644 --- a/arch/arm64/boot/dts/mediatek/mt7622.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7622.dtsi @@ -357,7 +357,7 @@ }; cci_control2: slave-if@5000 { - compatible = "arm,cci-400-ctrl-if"; + compatible = "arm,cci-400-ctrl-if", "syscon"; interface-type = "ace"; reg = <0x5000 0x1000>; }; @@ -901,6 +901,11 @@ }; }; + hifsys: syscon@1af00000 { + compatible = "mediatek,mt7622-hifsys", "syscon"; + reg = <0 0x1af00000 0 0x70>; + }; + ethsys: syscon@1b000000 { compatible = "mediatek,mt7622-ethsys", "syscon"; @@ -919,6 +924,26 @@ #dma-cells = <1>; }; + pcie_mirror: pcie-mirror@10000400 { + compatible = "mediatek,mt7622-pcie-mirror", + "syscon"; + reg = <0 0x10000400 0 0x10>; + }; + + wed0: wed@1020a000 { + compatible = "mediatek,mt7622-wed", + "syscon"; + reg = <0 0x1020a000 0 0x1000>; + interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_LOW>; + }; + + wed1: wed@1020b000 { + compatible = "mediatek,mt7622-wed", + "syscon"; + reg = <0 0x1020b000 0 0x1000>; + interrupts = <GIC_SPI 215 IRQ_TYPE_LEVEL_LOW>; + }; + eth: ethernet@1b100000 { compatible = "mediatek,mt7622-eth", "mediatek,mt2701-eth", @@ -945,6 +970,11 @@ power-domains = <&scpsys MT7622_POWER_DOMAIN_ETHSYS>; mediatek,ethsys = <ðsys>; mediatek,sgmiisys = <&sgmiisys>; + mediatek,cci-control = <&cci_control2>; + mediatek,wed = <&wed0>, <&wed1>; + mediatek,pcie-mirror = <&pcie_mirror>; + mediatek,hifsys = <&hifsys>; + dma-coherent; #address-cells = <1>; #size-cells = <0>; status = "disabled"; diff --git a/arch/h8300/include/asm/user.h b/arch/h8300/include/asm/user.h index 2298909f24c6..161653d84b34 100644 --- a/arch/h8300/include/asm/user.h +++ b/arch/h8300/include/asm/user.h @@ -67,9 +67,5 @@ struct user { unsigned long magic; /* To uniquely identify a core file */ char u_comm[32]; /* User command that was responsible */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif diff --git a/arch/ia64/include/asm/user.h b/arch/ia64/include/asm/user.h index 0ba486651b7c..ec03d3ab8715 100644 --- a/arch/ia64/include/asm/user.h +++ b/arch/ia64/include/asm/user.h @@ -50,10 +50,4 @@ struct user { char u_comm[32]; /* user command name */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_DATA_START_ADDR (u.start_data) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) - #endif /* _ASM_IA64_USER_H */ diff --git a/arch/ia64/kernel/syscalls/Makefile b/arch/ia64/kernel/syscalls/Makefile index 14f40ecf8b65..d009f927a048 100644 --- a/arch/ia64/kernel/syscalls/Makefile +++ b/arch/ia64/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/m68k/include/asm/user.h b/arch/m68k/include/asm/user.h index 509d555977c8..61413bff613a 100644 --- a/arch/m68k/include/asm/user.h +++ b/arch/m68k/include/asm/user.h @@ -79,9 +79,5 @@ struct user{ unsigned long magic; /* To uniquely identify a core file */ char u_comm[32]; /* User command that was responsible */ }; -#define NBPG 4096 -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif diff --git a/arch/m68k/kernel/syscalls/Makefile b/arch/m68k/kernel/syscalls/Makefile index 6713c65a25e1..b265e4bc16c2 100644 --- a/arch/m68k/kernel/syscalls/Makefile +++ b/arch/m68k/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/microblaze/boot/Makefile b/arch/microblaze/boot/Makefile index cff570a71946..2b42c370d574 100644 --- a/arch/microblaze/boot/Makefile +++ b/arch/microblaze/boot/Makefile @@ -29,7 +29,7 @@ $(obj)/simpleImage.$(DTB).ub: $(obj)/simpleImage.$(DTB) FORCE $(call if_changed,uimage) $(obj)/simpleImage.$(DTB).unstrip: vmlinux FORCE - $(call if_changed,shipped) + $(call if_changed,copy) $(obj)/simpleImage.$(DTB).strip: vmlinux FORCE $(call if_changed,strip) diff --git a/arch/microblaze/boot/dts/Makefile b/arch/microblaze/boot/dts/Makefile index ef00dd30d19a..b84e2cbb20ee 100644 --- a/arch/microblaze/boot/dts/Makefile +++ b/arch/microblaze/boot/dts/Makefile @@ -12,7 +12,7 @@ $(obj)/linked_dtb.o: $(obj)/system.dtb # Generate system.dtb from $(DTB).dtb ifneq ($(DTB),system) $(obj)/system.dtb: $(obj)/$(DTB).dtb - $(call if_changed,shipped) + $(call if_changed,copy) endif endif diff --git a/arch/microblaze/kernel/syscalls/Makefile b/arch/microblaze/kernel/syscalls/Makefile index 6713c65a25e1..b265e4bc16c2 100644 --- a/arch/microblaze/kernel/syscalls/Makefile +++ b/arch/microblaze/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index 5cb91509bb7c..b0489437621a 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -182,7 +182,6 @@ CONFIG_ATM_FIRESTREAM=m CONFIG_ATM_ZATM=m CONFIG_ATM_NICSTAR=m CONFIG_ATM_IDT77252=m -CONFIG_ATM_AMBASSADOR=m CONFIG_ATM_HORIZON=m CONFIG_ATM_IA=m CONFIG_ATM_FORE200E=m @@ -214,7 +213,6 @@ CONFIG_ATH_DEBUG=y CONFIG_ATH5K=y CONFIG_ATH5K_DEBUG=y CONFIG_WAN=y -CONFIG_LANMEDIA=m CONFIG_HDLC=m CONFIG_HDLC_RAW=m CONFIG_HDLC_RAW_ETH=m diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig index 205d3b34528c..c98099f0b354 100644 --- a/arch/mips/configs/mtx1_defconfig +++ b/arch/mips/configs/mtx1_defconfig @@ -259,7 +259,6 @@ CONFIG_ATM_FIRESTREAM=m CONFIG_ATM_ZATM=m CONFIG_ATM_NICSTAR=m CONFIG_ATM_IDT77252=m -CONFIG_ATM_AMBASSADOR=m CONFIG_ATM_HORIZON=m CONFIG_ATM_IA=m CONFIG_ATM_FORE200E=m @@ -363,7 +362,6 @@ CONFIG_USB_AN2720=y CONFIG_USB_EPSON2888=y CONFIG_USB_SIERRA_NET=m CONFIG_WAN=y -CONFIG_LANMEDIA=m CONFIG_HDLC=m CONFIG_HDLC_RAW=m CONFIG_HDLC_RAW_ETH=m diff --git a/arch/mips/crypto/crc32-mips.c b/arch/mips/crypto/crc32-mips.c index 0a03529cf317..3e4f5ba104f8 100644 --- a/arch/mips/crypto/crc32-mips.c +++ b/arch/mips/crypto/crc32-mips.c @@ -28,7 +28,7 @@ enum crc_type { }; #ifndef TOOLCHAIN_SUPPORTS_CRC -#define _ASM_MACRO_CRC32(OP, SZ, TYPE) \ +#define _ASM_SET_CRC(OP, SZ, TYPE) \ _ASM_MACRO_3R(OP, rt, rs, rt2, \ ".ifnc \\rt, \\rt2\n\t" \ ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \ @@ -37,30 +37,36 @@ _ASM_MACRO_3R(OP, rt, rs, rt2, \ ((SZ) << 6) | ((TYPE) << 8)) \ _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \ ((SZ) << 14) | ((TYPE) << 3))) -_ASM_MACRO_CRC32(crc32b, 0, 0); -_ASM_MACRO_CRC32(crc32h, 1, 0); -_ASM_MACRO_CRC32(crc32w, 2, 0); -_ASM_MACRO_CRC32(crc32d, 3, 0); -_ASM_MACRO_CRC32(crc32cb, 0, 1); -_ASM_MACRO_CRC32(crc32ch, 1, 1); -_ASM_MACRO_CRC32(crc32cw, 2, 1); -_ASM_MACRO_CRC32(crc32cd, 3, 1); -#define _ASM_SET_CRC "" +#define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t" #else /* !TOOLCHAIN_SUPPORTS_CRC */ -#define _ASM_SET_CRC ".set\tcrc\n\t" +#define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t" +#define _ASM_UNSET_CRC(op, SZ, TYPE) #endif -#define _CRC32(crc, value, size, type) \ -do { \ - __asm__ __volatile__( \ - ".set push\n\t" \ - _ASM_SET_CRC \ - #type #size " %0, %1, %0\n\t" \ - ".set pop" \ - : "+r" (crc) \ - : "r" (value)); \ +#define __CRC32(crc, value, op, SZ, TYPE) \ +do { \ + __asm__ __volatile__( \ + ".set push\n\t" \ + _ASM_SET_CRC(op, SZ, TYPE) \ + #op " %0, %1, %0\n\t" \ + _ASM_UNSET_CRC(op, SZ, TYPE) \ + ".set pop" \ + : "+r" (crc) \ + : "r" (value)); \ } while (0) +#define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0) +#define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0) +#define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0) +#define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0) +#define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1) +#define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1) +#define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1) +#define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1) + +#define _CRC32(crc, value, size, op) \ + _CRC32_##op##size(crc, value) + #define CRC32(crc, value, size) \ _CRC32(crc, value, size, crc32) diff --git a/arch/mips/include/asm/mach-rc32434/rb.h b/arch/mips/include/asm/mach-rc32434/rb.h index 34d179ca020b..dd9d4b026e62 100644 --- a/arch/mips/include/asm/mach-rc32434/rb.h +++ b/arch/mips/include/asm/mach-rc32434/rb.h @@ -29,15 +29,6 @@ #define DEV3TC 0x01003C #define BTCS 0x010040 #define BTCOMPARE 0x010044 -#define GPIOBASE 0x050000 -/* Offsets relative to GPIOBASE */ -#define GPIOFUNC 0x00 -#define GPIOCFG 0x04 -#define GPIOD 0x08 -#define GPIOILEVEL 0x0C -#define GPIOISTAT 0x10 -#define GPIONMIEN 0x14 -#define IMASK6 0x38 #define LO_WPX (1 << 0) #define LO_ALE (1 << 1) #define LO_CLE (1 << 2) diff --git a/arch/mips/kernel/syscalls/Makefile b/arch/mips/kernel/syscalls/Makefile index 10bf90dc02c0..e6b21de65cca 100644 --- a/arch/mips/kernel/syscalls/Makefile +++ b/arch/mips/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syshdr := $(srctree)/scripts/syscallhdr.sh sysnr := $(srctree)/$(src)/syscallnr.sh diff --git a/arch/mips/lantiq/falcon/sysctrl.c b/arch/mips/lantiq/falcon/sysctrl.c index 64726c670ca6..5204fc6d6d50 100644 --- a/arch/mips/lantiq/falcon/sysctrl.c +++ b/arch/mips/lantiq/falcon/sysctrl.c @@ -167,6 +167,8 @@ static inline void clkdev_add_sys(const char *dev, unsigned int module, { struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL); + if (!clk) + return; clk->cl.dev_id = dev; clk->cl.con_id = NULL; clk->cl.clk = clk; diff --git a/arch/mips/lantiq/xway/gptu.c b/arch/mips/lantiq/xway/gptu.c index 3d5683e75cf1..200fe9ff641d 100644 --- a/arch/mips/lantiq/xway/gptu.c +++ b/arch/mips/lantiq/xway/gptu.c @@ -122,6 +122,8 @@ static inline void clkdev_add_gptu(struct device *dev, const char *con, { struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL); + if (!clk) + return; clk->cl.dev_id = dev_name(dev); clk->cl.con_id = con; clk->cl.clk = clk; diff --git a/arch/mips/lantiq/xway/sysctrl.c b/arch/mips/lantiq/xway/sysctrl.c index 917fac1636b7..084f6caba5f2 100644 --- a/arch/mips/lantiq/xway/sysctrl.c +++ b/arch/mips/lantiq/xway/sysctrl.c @@ -315,6 +315,8 @@ static void clkdev_add_pmu(const char *dev, const char *con, bool deactivate, { struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL); + if (!clk) + return; clk->cl.dev_id = dev; clk->cl.con_id = con; clk->cl.clk = clk; @@ -338,6 +340,8 @@ static void clkdev_add_cgu(const char *dev, const char *con, { struct clk *clk = kzalloc(sizeof(struct clk), GFP_KERNEL); + if (!clk) + return; clk->cl.dev_id = dev; clk->cl.con_id = con; clk->cl.clk = clk; @@ -356,24 +360,28 @@ static void clkdev_add_pci(void) struct clk *clk_ext = kzalloc(sizeof(struct clk), GFP_KERNEL); /* main pci clock */ - clk->cl.dev_id = "17000000.pci"; - clk->cl.con_id = NULL; - clk->cl.clk = clk; - clk->rate = CLOCK_33M; - clk->rates = valid_pci_rates; - clk->enable = pci_enable; - clk->disable = pmu_disable; - clk->module = 0; - clk->bits = PMU_PCI; - clkdev_add(&clk->cl); + if (clk) { + clk->cl.dev_id = "17000000.pci"; + clk->cl.con_id = NULL; + clk->cl.clk = clk; + clk->rate = CLOCK_33M; + clk->rates = valid_pci_rates; + clk->enable = pci_enable; + clk->disable = pmu_disable; + clk->module = 0; + clk->bits = PMU_PCI; + clkdev_add(&clk->cl); + } /* use internal/external bus clock */ - clk_ext->cl.dev_id = "17000000.pci"; - clk_ext->cl.con_id = "external"; - clk_ext->cl.clk = clk_ext; - clk_ext->enable = pci_ext_enable; - clk_ext->disable = pci_ext_disable; - clkdev_add(&clk_ext->cl); + if (clk_ext) { + clk_ext->cl.dev_id = "17000000.pci"; + clk_ext->cl.con_id = "external"; + clk_ext->cl.clk = clk_ext; + clk_ext->enable = pci_ext_enable; + clk_ext->disable = pci_ext_disable; + clkdev_add(&clk_ext->cl); + } } /* xway socs can generate clocks on gpio pins */ @@ -393,9 +401,15 @@ static void clkdev_add_clkout(void) char *name; name = kzalloc(sizeof("clkout0"), GFP_KERNEL); + if (!name) + continue; sprintf(name, "clkout%d", i); clk = kzalloc(sizeof(struct clk), GFP_KERNEL); + if (!clk) { + kfree(name); + continue; + } clk->cl.dev_id = "1f103000.cgu"; clk->cl.con_id = name; clk->cl.clk = clk; diff --git a/arch/mips/rb532/gpio.c b/arch/mips/rb532/gpio.c index 94f02ada4082..29c21b9d42da 100644 --- a/arch/mips/rb532/gpio.c +++ b/arch/mips/rb532/gpio.c @@ -37,6 +37,16 @@ #include <asm/mach-rc32434/rb.h> #include <asm/mach-rc32434/gpio.h> +#define GPIOBASE 0x050000 +/* Offsets relative to GPIOBASE */ +#define GPIOFUNC 0x00 +#define GPIOCFG 0x04 +#define GPIOD 0x08 +#define GPIOILEVEL 0x0C +#define GPIOISTAT 0x10 +#define GPIONMIEN 0x14 +#define IMASK6 0x38 + struct rb532_gpio_chip { struct gpio_chip chip; void __iomem *regbase; diff --git a/arch/mips/sgi-ip22/ip22-gio.c b/arch/mips/sgi-ip22/ip22-gio.c index dfc52f661ad0..38d12f417e48 100644 --- a/arch/mips/sgi-ip22/ip22-gio.c +++ b/arch/mips/sgi-ip22/ip22-gio.c @@ -363,6 +363,8 @@ static void ip22_check_gio(int slotno, unsigned long addr, int irq) printk(KERN_INFO "GIO: slot %d : %s (id %x)\n", slotno, name, id); gio_dev = kzalloc(sizeof *gio_dev, GFP_KERNEL); + if (!gio_dev) + return; gio_dev->name = name; gio_dev->slotno = slotno; gio_dev->id.id = id; diff --git a/arch/parisc/kernel/syscalls/Makefile b/arch/parisc/kernel/syscalls/Makefile index d63f18dd058d..8440c16dfb22 100644 --- a/arch/parisc/kernel/syscalls/Makefile +++ b/arch/parisc/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/powerpc/include/asm/user.h b/arch/powerpc/include/asm/user.h index 99443b8594e7..7fae7e597ba4 100644 --- a/arch/powerpc/include/asm/user.h +++ b/arch/powerpc/include/asm/user.h @@ -44,9 +44,4 @@ struct user { char u_comm[32]; /* user command name */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_DATA_START_ADDR (u.start_data) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif /* _ASM_POWERPC_USER_H */ diff --git a/arch/powerpc/kernel/syscalls/Makefile b/arch/powerpc/kernel/syscalls/Makefile index 5476f62eb80f..9d7bd81510b8 100644 --- a/arch/powerpc/kernel/syscalls/Makefile +++ b/arch/powerpc/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ea8ec8a960bd..00fd9c548f26 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -16,6 +16,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX @@ -47,6 +48,7 @@ config RISCV select CLONE_BACKWARDS select CLINT_TIMER if !MMU select COMMON_CLK + select CPU_PM if CPU_IDLE select EDAC_SUPPORT select GENERIC_ARCH_TOPOLOGY if SMP select GENERIC_ATOMIC64 if !64BIT @@ -533,4 +535,10 @@ source "kernel/power/Kconfig" endmenu +menu "CPU Power Management" + +source "drivers/cpuidle/Kconfig" + +endmenu + source "arch/riscv/kvm/Kconfig" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index c112ab2a9052..34592d00dde8 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -36,6 +36,9 @@ config SOC_VIRT select GOLDFISH select RTC_DRV_GOLDFISH if RTC_CLASS select SIFIVE_PLIC + select PM_GENERIC_DOMAINS if PM + select PM_GENERIC_DOMAINS_OF if PM && OF + select RISCV_SBI_CPUIDLE if CPU_IDLE help This enables support for QEMU Virt Machine. diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts index 984872f3d3a9..b9e30df127fe 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts @@ -203,6 +203,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts index 7ba99b4da304..8d23401b0bbb 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts @@ -205,6 +205,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts index be9b12c9b374..24fd83b43d9d 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts @@ -213,6 +213,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts index 031c0c28f819..25341f38292a 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts @@ -178,6 +178,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 7cd10ded7bf8..30e3017f22bc 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -15,11 +15,14 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set +CONFIG_PROFILING=y CONFIG_SOC_MICROCHIP_POLARFIRE=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_PM=y +CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m CONFIG_JUMP_LABEL=y @@ -64,8 +67,6 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_EARLYCON_RISCV_SBI=y -CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 3f42ed87dde8..2438fa39f8ae 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -21,7 +21,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index af64b95e88cc..9a133e63ae5b 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -13,7 +13,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index e1c9864b6237..5269fbb6b4fc 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -19,7 +19,6 @@ CONFIG_EXPERT=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index e0e5c7c09ab8..7e5efdc3829d 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -15,11 +15,14 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set +CONFIG_PROFILING=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_PM=y +CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m CONFIG_JUMP_LABEL=y @@ -62,8 +65,6 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_EARLYCON_RISCV_SBI=y -CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 618d7c5af1a2..8c2549b16ac0 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -67,4 +67,30 @@ #error "Unexpected __SIZEOF_SHORT__" #endif +#ifdef __ASSEMBLY__ + +/* Common assembly source macros */ + +#ifdef CONFIG_XIP_KERNEL +.macro XIP_FIXUP_OFFSET reg + REG_L t0, _xip_fixup + add \reg, \reg, t0 +.endm +.macro XIP_FIXUP_FLASH_OFFSET reg + la t1, __data_loc + REG_L t1, _xip_phys_offset + sub \reg, \reg, t1 + add \reg, \reg, t0 +.endm +_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET +_xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET +#else +.macro XIP_FIXUP_OFFSET reg +.endm +.macro XIP_FIXUP_FLASH_OFFSET reg +.endm +#endif /* CONFIG_XIP_KERNEL */ + +#endif /* __ASSEMBLY__ */ + #endif /* _ASM_RISCV_ASM_H */ diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h new file mode 100644 index 000000000000..71fdc607d4bc --- /dev/null +++ b/arch/riscv/include/asm/cpuidle.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 Allwinner Ltd + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + */ + +#ifndef _ASM_RISCV_CPUIDLE_H +#define _ASM_RISCV_CPUIDLE_H + +#include <asm/barrier.h> +#include <asm/processor.h> + +static inline void cpu_do_idle(void) +{ + /* + * Add mb() here to ensure that all + * IO/MEM accesses are completed prior + * to entering WFI. + */ + mb(); + wait_for_interrupt(); +} + +#endif diff --git a/arch/riscv/include/asm/current.h b/arch/riscv/include/asm/current.h index 1de233d8e8de..21774d868c65 100644 --- a/arch/riscv/include/asm/current.h +++ b/arch/riscv/include/asm/current.h @@ -33,6 +33,8 @@ static __always_inline struct task_struct *get_current(void) #define current get_current() +register unsigned long current_stack_pointer __asm__("sp"); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_CURRENT_H */ diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h index 4254ff2ff049..1075beae1ac6 100644 --- a/arch/riscv/include/asm/module.lds.h +++ b/arch/riscv/include/asm/module.lds.h @@ -2,8 +2,8 @@ /* Copyright (C) 2017 Andes Technology Corporation */ #ifdef CONFIG_MODULE_SECTIONS SECTIONS { - .plt (NOLOAD) : { BYTE(0) } - .got (NOLOAD) : { BYTE(0) } - .got.plt (NOLOAD) : { BYTE(0) } + .plt : { BYTE(0) } + .got : { BYTE(0) } + .got.plt : { BYTE(0) } } #endif diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h new file mode 100644 index 000000000000..8be391c2aecb --- /dev/null +++ b/arch/riscv/include/asm/suspend.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _ASM_RISCV_SUSPEND_H +#define _ASM_RISCV_SUSPEND_H + +#include <asm/ptrace.h> + +struct suspend_context { + /* Saved and restored by low-level functions */ + struct pt_regs regs; + /* Saved and restored by high-level functions */ + unsigned long scratch; + unsigned long tvec; + unsigned long ie; +#ifdef CONFIG_MMU + unsigned long satp; +#endif +}; + +/* Low-level CPU suspend entry function */ +int __cpu_suspend_enter(struct suspend_context *context); + +/* High-level CPU suspend which will save context and call finish() */ +int cpu_suspend(unsigned long arg, + int (*finish)(unsigned long arg, + unsigned long entry, + unsigned long context)); + +/* Low-level CPU resume entry function */ +int __cpu_resume_enter(unsigned long hartid, unsigned long context); + +#endif diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 60da0dcacf14..74d888c8d631 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -11,11 +11,17 @@ #include <asm/page.h> #include <linux/const.h> +#ifdef CONFIG_KASAN +#define KASAN_STACK_ORDER 1 +#else +#define KASAN_STACK_ORDER 0 +#endif + /* thread information allocation */ #ifdef CONFIG_64BIT -#define THREAD_SIZE_ORDER (2) +#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER) #else -#define THREAD_SIZE_ORDER (1) +#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER) #endif #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index e0133d113216..87adbe47bc15 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -48,6 +48,8 @@ obj-$(CONFIG_RISCV_BOOT_SPINWAIT) += cpu_ops_spinwait.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o +obj-$(CONFIG_CPU_PM) += suspend_entry.o suspend.o + obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index df0519a64eaf..df9444397908 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -13,6 +13,7 @@ #include <asm/thread_info.h> #include <asm/ptrace.h> #include <asm/cpu_ops_sbi.h> +#include <asm/suspend.h> void asm_offsets(void); @@ -113,6 +114,8 @@ void asm_offsets(void) OFFSET(PT_BADADDR, pt_regs, badaddr); OFFSET(PT_CAUSE, pt_regs, cause); + OFFSET(SUSPEND_CONTEXT_REGS, suspend_context, regs); + OFFSET(KVM_ARCH_GUEST_ZERO, kvm_vcpu_arch, guest_context.zero); OFFSET(KVM_ARCH_GUEST_RA, kvm_vcpu_arch, guest_context.ra); OFFSET(KVM_ARCH_GUEST_SP, kvm_vcpu_arch, guest_context.sp); diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index d2a936195295..ccb617791e56 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -69,11 +69,11 @@ int riscv_of_parent_hartid(struct device_node *node) .uprop = #UPROP, \ .isa_ext_id = EXTID, \ } -/** +/* * Here are the ordering rules of extension naming defined by RISC-V * specification : * 1. All extensions should be separated from other multi-letter extensions - * from other multi-letter extensions by an underscore. + * by an underscore. * 2. The first letter following the 'Z' conventionally indicates the most * closely related alphabetical extension category, IMAFDQLCBKJTPVH. * If multiple 'Z' extensions are named, they should be ordered first @@ -110,7 +110,7 @@ static void print_isa_ext(struct seq_file *f) } } -/** +/* * These are the only valid base (single letter) ISA extensions as per the spec. * It also specifies the canonical order in which it appears in the spec. * Some of the extension may just be a place holder for now (B, K, P, J). diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index 2e16f6732cdf..4f5a6f84e2a4 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -21,7 +21,7 @@ const struct cpu_operations cpu_ops_sbi; * be invoked from multiple threads in parallel. Define a per cpu data * to handle that. */ -DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); +static DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, unsigned long priv) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index ec07f991866a..893b8bb69391 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -16,26 +16,6 @@ #include <asm/image.h> #include "efi-header.S" -#ifdef CONFIG_XIP_KERNEL -.macro XIP_FIXUP_OFFSET reg - REG_L t0, _xip_fixup - add \reg, \reg, t0 -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg - la t0, __data_loc - REG_L t1, _xip_phys_offset - sub \reg, \reg, t1 - add \reg, \reg, t0 -.endm -_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET -_xip_phys_offset: .dword CONFIG_XIP_PHYS_ADDR + XIP_OFFSET -#else -.macro XIP_FIXUP_OFFSET reg -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg -.endm -#endif /* CONFIG_XIP_KERNEL */ - __HEAD ENTRY(_start) /* @@ -89,7 +69,8 @@ pe_head_start: .align 2 #ifdef CONFIG_MMU -relocate: + .global relocate_enable_mmu +relocate_enable_mmu: /* Relocate return address */ la a1, kernel_map XIP_FIXUP_OFFSET a1 @@ -184,7 +165,7 @@ secondary_start_sbi: /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir XIP_FIXUP_OFFSET a0 - call relocate + call relocate_enable_mmu #endif call setup_trap_vector tail smp_callin @@ -328,7 +309,7 @@ clear_bss_done: #ifdef CONFIG_MMU la a0, early_pg_dir XIP_FIXUP_OFFSET a0 - call relocate + call relocate_enable_mmu #endif /* CONFIG_MMU */ call setup_trap_vector diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 4a48287513c3..c29cef90d1dd 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -69,7 +69,7 @@ static int apply_r_riscv_jal_rela(struct module *me, u32 *location, return 0; } -static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, +static int apply_r_riscv_rvc_branch_rela(struct module *me, u32 *location, Elf_Addr v) { ptrdiff_t offset = (void *)v - (void *)location; @@ -301,7 +301,7 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, [R_RISCV_64] = apply_r_riscv_64_rela, [R_RISCV_BRANCH] = apply_r_riscv_branch_rela, [R_RISCV_JAL] = apply_r_riscv_jal_rela, - [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela, + [R_RISCV_RVC_BRANCH] = apply_r_riscv_rvc_branch_rela, [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela, [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela, [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela, diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 55faa4991b87..3348a61de7d9 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -68,7 +68,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, static bool fill_callchain(void *entry, unsigned long pc) { - return perf_callchain_store(entry, pc); + return perf_callchain_store(entry, pc) == 0; } void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 03ac3aa611f5..504b496787aa 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -23,6 +23,7 @@ #include <asm/string.h> #include <asm/switch_to.h> #include <asm/thread_info.h> +#include <asm/cpuidle.h> register unsigned long gp_in_global __asm__("gp"); @@ -37,7 +38,7 @@ extern asmlinkage void ret_from_kernel_thread(void); void arch_cpu_idle(void) { - wait_for_interrupt(); + cpu_do_idle(); raw_local_irq_enable(); } diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 14d2b53ec322..08d11a53f39e 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -14,8 +14,6 @@ #include <asm/stacktrace.h> -register unsigned long sp_in_global __asm__("sp"); - #ifdef CONFIG_FRAME_POINTER void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, @@ -30,7 +28,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, pc = instruction_pointer(regs); } else if (task == NULL || task == current) { fp = (unsigned long)__builtin_frame_address(0); - sp = sp_in_global; + sp = current_stack_pointer; pc = (unsigned long)walk_stackframe; } else { /* task blocked in __switch_to */ @@ -78,7 +76,7 @@ void notrace walk_stackframe(struct task_struct *task, sp = user_stack_pointer(regs); pc = instruction_pointer(regs); } else if (task == NULL || task == current) { - sp = sp_in_global; + sp = current_stack_pointer; pc = (unsigned long)walk_stackframe; } else { /* task blocked in __switch_to */ diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c new file mode 100644 index 000000000000..9ba24fb8cc93 --- /dev/null +++ b/arch/riscv/kernel/suspend.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/ftrace.h> +#include <asm/csr.h> +#include <asm/suspend.h> + +static void suspend_save_csrs(struct suspend_context *context) +{ + context->scratch = csr_read(CSR_SCRATCH); + context->tvec = csr_read(CSR_TVEC); + context->ie = csr_read(CSR_IE); + + /* + * No need to save/restore IP CSR (i.e. MIP or SIP) because: + * + * 1. For no-MMU (M-mode) kernel, the bits in MIP are set by + * external devices (such as interrupt controller, timer, etc). + * 2. For MMU (S-mode) kernel, the bits in SIP are set by + * M-mode firmware and external devices (such as interrupt + * controller, etc). + */ + +#ifdef CONFIG_MMU + context->satp = csr_read(CSR_SATP); +#endif +} + +static void suspend_restore_csrs(struct suspend_context *context) +{ + csr_write(CSR_SCRATCH, context->scratch); + csr_write(CSR_TVEC, context->tvec); + csr_write(CSR_IE, context->ie); + +#ifdef CONFIG_MMU + csr_write(CSR_SATP, context->satp); +#endif +} + +int cpu_suspend(unsigned long arg, + int (*finish)(unsigned long arg, + unsigned long entry, + unsigned long context)) +{ + int rc = 0; + struct suspend_context context = { 0 }; + + /* Finisher should be non-NULL */ + if (!finish) + return -EINVAL; + + /* Save additional CSRs*/ + suspend_save_csrs(&context); + + /* + * Function graph tracer state gets incosistent when the kernel + * calls functions that never return (aka finishers) hence disable + * graph tracing during their execution. + */ + pause_graph_tracing(); + + /* Save context on stack */ + if (__cpu_suspend_enter(&context)) { + /* Call the finisher */ + rc = finish(arg, __pa_symbol(__cpu_resume_enter), + (ulong)&context); + + /* + * Should never reach here, unless the suspend finisher + * fails. Successful cpu_suspend() should return from + * __cpu_resume_entry() + */ + if (!rc) + rc = -EOPNOTSUPP; + } + + /* Enable function graph tracer */ + unpause_graph_tracing(); + + /* Restore additional CSRs */ + suspend_restore_csrs(&context); + + return rc; +} diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S new file mode 100644 index 000000000000..4b07b809a2b8 --- /dev/null +++ b/arch/riscv/kernel/suspend_entry.S @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm/asm-offsets.h> +#include <asm/csr.h> + + .text + .altmacro + .option norelax + +ENTRY(__cpu_suspend_enter) + /* Save registers (except A0 and T0-T6) */ + REG_S ra, (SUSPEND_CONTEXT_REGS + PT_RA)(a0) + REG_S sp, (SUSPEND_CONTEXT_REGS + PT_SP)(a0) + REG_S gp, (SUSPEND_CONTEXT_REGS + PT_GP)(a0) + REG_S tp, (SUSPEND_CONTEXT_REGS + PT_TP)(a0) + REG_S s0, (SUSPEND_CONTEXT_REGS + PT_S0)(a0) + REG_S s1, (SUSPEND_CONTEXT_REGS + PT_S1)(a0) + REG_S a1, (SUSPEND_CONTEXT_REGS + PT_A1)(a0) + REG_S a2, (SUSPEND_CONTEXT_REGS + PT_A2)(a0) + REG_S a3, (SUSPEND_CONTEXT_REGS + PT_A3)(a0) + REG_S a4, (SUSPEND_CONTEXT_REGS + PT_A4)(a0) + REG_S a5, (SUSPEND_CONTEXT_REGS + PT_A5)(a0) + REG_S a6, (SUSPEND_CONTEXT_REGS + PT_A6)(a0) + REG_S a7, (SUSPEND_CONTEXT_REGS + PT_A7)(a0) + REG_S s2, (SUSPEND_CONTEXT_REGS + PT_S2)(a0) + REG_S s3, (SUSPEND_CONTEXT_REGS + PT_S3)(a0) + REG_S s4, (SUSPEND_CONTEXT_REGS + PT_S4)(a0) + REG_S s5, (SUSPEND_CONTEXT_REGS + PT_S5)(a0) + REG_S s6, (SUSPEND_CONTEXT_REGS + PT_S6)(a0) + REG_S s7, (SUSPEND_CONTEXT_REGS + PT_S7)(a0) + REG_S s8, (SUSPEND_CONTEXT_REGS + PT_S8)(a0) + REG_S s9, (SUSPEND_CONTEXT_REGS + PT_S9)(a0) + REG_S s10, (SUSPEND_CONTEXT_REGS + PT_S10)(a0) + REG_S s11, (SUSPEND_CONTEXT_REGS + PT_S11)(a0) + + /* Save CSRs */ + csrr t0, CSR_EPC + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_EPC)(a0) + csrr t0, CSR_STATUS + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_STATUS)(a0) + csrr t0, CSR_TVAL + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_BADADDR)(a0) + csrr t0, CSR_CAUSE + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_CAUSE)(a0) + + /* Return non-zero value */ + li a0, 1 + + /* Return to C code */ + ret +END(__cpu_suspend_enter) + +ENTRY(__cpu_resume_enter) + /* Load the global pointer */ + .option push + .option norelax + la gp, __global_pointer$ + .option pop + +#ifdef CONFIG_MMU + /* Save A0 and A1 */ + add t0, a0, zero + add t1, a1, zero + + /* Enable MMU */ + la a0, swapper_pg_dir + XIP_FIXUP_OFFSET a0 + call relocate_enable_mmu + + /* Restore A0 and A1 */ + add a0, t0, zero + add a1, t1, zero +#endif + + /* Make A0 point to suspend context */ + add a0, a1, zero + + /* Restore CSRs */ + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_EPC)(a0) + csrw CSR_EPC, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_STATUS)(a0) + csrw CSR_STATUS, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_BADADDR)(a0) + csrw CSR_TVAL, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_CAUSE)(a0) + csrw CSR_CAUSE, t0 + + /* Restore registers (except A0 and T0-T6) */ + REG_L ra, (SUSPEND_CONTEXT_REGS + PT_RA)(a0) + REG_L sp, (SUSPEND_CONTEXT_REGS + PT_SP)(a0) + REG_L gp, (SUSPEND_CONTEXT_REGS + PT_GP)(a0) + REG_L tp, (SUSPEND_CONTEXT_REGS + PT_TP)(a0) + REG_L s0, (SUSPEND_CONTEXT_REGS + PT_S0)(a0) + REG_L s1, (SUSPEND_CONTEXT_REGS + PT_S1)(a0) + REG_L a1, (SUSPEND_CONTEXT_REGS + PT_A1)(a0) + REG_L a2, (SUSPEND_CONTEXT_REGS + PT_A2)(a0) + REG_L a3, (SUSPEND_CONTEXT_REGS + PT_A3)(a0) + REG_L a4, (SUSPEND_CONTEXT_REGS + PT_A4)(a0) + REG_L a5, (SUSPEND_CONTEXT_REGS + PT_A5)(a0) + REG_L a6, (SUSPEND_CONTEXT_REGS + PT_A6)(a0) + REG_L a7, (SUSPEND_CONTEXT_REGS + PT_A7)(a0) + REG_L s2, (SUSPEND_CONTEXT_REGS + PT_S2)(a0) + REG_L s3, (SUSPEND_CONTEXT_REGS + PT_S3)(a0) + REG_L s4, (SUSPEND_CONTEXT_REGS + PT_S4)(a0) + REG_L s5, (SUSPEND_CONTEXT_REGS + PT_S5)(a0) + REG_L s6, (SUSPEND_CONTEXT_REGS + PT_S6)(a0) + REG_L s7, (SUSPEND_CONTEXT_REGS + PT_S7)(a0) + REG_L s8, (SUSPEND_CONTEXT_REGS + PT_S8)(a0) + REG_L s9, (SUSPEND_CONTEXT_REGS + PT_S9)(a0) + REG_L s10, (SUSPEND_CONTEXT_REGS + PT_S10)(a0) + REG_L s11, (SUSPEND_CONTEXT_REGS + PT_S11)(a0) + + /* Return zero value */ + add a0, zero, zero + + /* Return to C code */ + ret +END(__cpu_resume_enter) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 9b80e8bed3f6..77b5a03de13a 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -58,6 +58,7 @@ config S390 select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_USTAT_F_TINODE select ARCH_BINFMT_ELF_STATE + select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 diff --git a/arch/s390/include/asm/alternative-asm.h b/arch/s390/include/asm/alternative-asm.h index 955d620db23e..bb3837d7387c 100644 --- a/arch/s390/include/asm/alternative-asm.h +++ b/arch/s390/include/asm/alternative-asm.h @@ -37,9 +37,15 @@ * a 2-byte nop if the size of the area is not divisible by 6. */ .macro alt_pad_fill bytes - .fill ( \bytes ) / 6, 6, 0xc0040000 - .fill ( \bytes ) % 6 / 4, 4, 0x47000000 - .fill ( \bytes ) % 6 % 4 / 2, 2, 0x0700 + .rept ( \bytes ) / 6 + brcl 0,0 + .endr + .rept ( \bytes ) % 6 / 4 + nop + .endr + .rept ( \bytes ) % 6 % 4 / 2 + nopr + .endr .endm /* diff --git a/arch/s390/include/asm/alternative.h b/arch/s390/include/asm/alternative.h index d3880ca764ee..3f2856ed6808 100644 --- a/arch/s390/include/asm/alternative.h +++ b/arch/s390/include/asm/alternative.h @@ -71,11 +71,18 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end); ".if " oldinstr_pad_len(num) " > 6\n" \ "\tjg " e_oldinstr_pad_end "f\n" \ "6620:\n" \ - "\t.fill (" oldinstr_pad_len(num) " - (6620b-662b)) / 2, 2, 0x0700\n" \ + "\t.rept (" oldinstr_pad_len(num) " - (6620b-662b)) / 2\n" \ + "\tnopr\n" \ ".else\n" \ - "\t.fill " oldinstr_pad_len(num) " / 6, 6, 0xc0040000\n" \ - "\t.fill " oldinstr_pad_len(num) " %% 6 / 4, 4, 0x47000000\n" \ - "\t.fill " oldinstr_pad_len(num) " %% 6 %% 4 / 2, 2, 0x0700\n" \ + "\t.rept " oldinstr_pad_len(num) " / 6\n" \ + "\t.brcl 0,0\n" \ + "\t.endr\n" \ + "\t.rept " oldinstr_pad_len(num) " %% 6 / 4\n" \ + "\tnop\n" \ + "\t.endr\n" \ + "\t.rept " oldinstr_pad_len(num) " %% 6 %% 4 / 2\n" \ + "\tnopr\n" \ + ".endr\n" \ ".endif\n" #define OLDINSTR(oldinstr, num) \ diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index ae75da592ccb..b515cfa62bd9 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -60,11 +60,11 @@ static inline bool ap_instructions_available(void) unsigned long reg1 = 0; asm volatile( - " lgr 0,%[reg0]\n" /* qid into gr0 */ - " lghi 1,0\n" /* 0 into gr1 */ - " lghi 2,0\n" /* 0 into gr2 */ - " .long 0xb2af0000\n" /* PQAP(TAPQ) */ - "0: la %[reg1],1\n" /* 1 into reg1 */ + " lgr 0,%[reg0]\n" /* qid into gr0 */ + " lghi 1,0\n" /* 0 into gr1 */ + " lghi 2,0\n" /* 0 into gr2 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ + "0: la %[reg1],1\n" /* 1 into reg1 */ "1:\n" EX_TABLE(0b, 1b) : [reg1] "+&d" (reg1) @@ -86,11 +86,11 @@ static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info) unsigned long reg2; asm volatile( - " lgr 0,%[qid]\n" /* qid into gr0 */ - " lghi 2,0\n" /* 0 into gr2 */ - " .long 0xb2af0000\n" /* PQAP(TAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ - " lgr %[reg2],2\n" /* gr2 into reg2 */ + " lgr 0,%[qid]\n" /* qid into gr0 */ + " lghi 2,0\n" /* 0 into gr2 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg2],2\n" /* gr2 into reg2 */ : [reg1] "=&d" (reg1), [reg2] "=&d" (reg2) : [qid] "d" (qid) : "cc", "0", "1", "2"); @@ -128,9 +128,9 @@ static inline struct ap_queue_status ap_rapq(ap_qid_t qid) struct ap_queue_status reg1; asm volatile( - " lgr 0,%[reg0]\n" /* qid arg into gr0 */ - " .long 0xb2af0000\n" /* PQAP(RAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr 0,%[reg0]\n" /* qid arg into gr0 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(RAPQ) */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1"); @@ -149,9 +149,9 @@ static inline struct ap_queue_status ap_zapq(ap_qid_t qid) struct ap_queue_status reg1; asm volatile( - " lgr 0,%[reg0]\n" /* qid arg into gr0 */ - " .long 0xb2af0000\n" /* PQAP(ZAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr 0,%[reg0]\n" /* qid arg into gr0 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(ZAPQ) */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1"); @@ -190,10 +190,10 @@ static inline int ap_qci(struct ap_config_info *config) struct ap_config_info *reg2 = config; asm volatile( - " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ - " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ - " .long 0xb2af0000\n" /* PQAP(QCI) */ - "0: la %[reg1],0\n" /* good case, QCI fc available */ + " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ + " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ + "0: la %[reg1],0\n" /* good case, QCI fc available */ "1:\n" EX_TABLE(0b, 1b) : [reg1] "+&d" (reg1) @@ -246,11 +246,11 @@ static inline struct ap_queue_status ap_aqic(ap_qid_t qid, reg1.qirqctrl = qirqctrl; asm volatile( - " lgr 0,%[reg0]\n" /* qid param into gr0 */ - " lgr 1,%[reg1]\n" /* irq ctrl into gr1 */ - " lgr 2,%[reg2]\n" /* ni addr into gr2 */ - " .long 0xb2af0000\n" /* PQAP(AQIC) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr 0,%[reg0]\n" /* qid param into gr0 */ + " lgr 1,%[reg1]\n" /* irq ctrl into gr1 */ + " lgr 2,%[reg2]\n" /* ni addr into gr2 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(AQIC) */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "+&d" (reg1) : [reg0] "d" (reg0), [reg2] "d" (reg2) : "cc", "0", "1", "2"); @@ -297,11 +297,11 @@ static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit, reg1.value = apinfo->val; asm volatile( - " lgr 0,%[reg0]\n" /* qid param into gr0 */ - " lgr 1,%[reg1]\n" /* qact in info into gr1 */ - " .long 0xb2af0000\n" /* PQAP(QACT) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ - " lgr %[reg2],2\n" /* qact out info into reg2 */ + " lgr 0,%[reg0]\n" /* qid param into gr0 */ + " lgr 1,%[reg1]\n" /* qact in info into gr1 */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(QACT) */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg2],2\n" /* qact out info into reg2 */ : [reg1] "+&d" (reg1), [reg2] "=&d" (reg2) : [reg0] "d" (reg0) : "cc", "0", "1", "2"); diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index c800199a376b..82388da3f95f 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -74,8 +74,17 @@ static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) __ctl_load(reg, cr, cr); } -void smp_ctl_set_bit(int cr, int bit); -void smp_ctl_clear_bit(int cr, int bit); +void smp_ctl_set_clear_bit(int cr, int bit, bool set); + +static inline void ctl_set_bit(int cr, int bit) +{ + smp_ctl_set_clear_bit(cr, bit, true); +} + +static inline void ctl_clear_bit(int cr, int bit) +{ + smp_ctl_set_clear_bit(cr, bit, false); +} union ctlreg0 { unsigned long val; @@ -130,8 +139,5 @@ union ctlreg15 { }; }; -#define ctl_set_bit(cr, bit) smp_ctl_set_bit(cr, bit) -#define ctl_clear_bit(cr, bit) smp_ctl_clear_bit(cr, bit) - #endif /* __ASSEMBLY__ */ #endif /* __ASM_CTL_REG_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 84ec63145325..eee8d96fb38e 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -319,11 +319,18 @@ extern void (*s390_base_pgm_handler_fn)(struct pt_regs *regs); extern int memcpy_real(void *, unsigned long, size_t); extern void memcpy_absolute(void *, void *, size_t); -#define mem_assign_absolute(dest, val) do { \ - __typeof__(dest) __tmp = (val); \ - \ - BUILD_BUG_ON(sizeof(__tmp) != sizeof(val)); \ - memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \ +#define put_abs_lowcore(member, x) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) __tmp = (x); \ + \ + memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp)); \ +} while (0) + +#define get_abs_lowcore(x, member) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) *__ptr = &(x); \ + \ + memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr)); \ } while (0) extern int s390_isolate_bp(void); diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 888a2f1c9ee3..24a54443c865 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -78,7 +78,7 @@ static inline void arch_spin_unlock(arch_spinlock_t *lp) { typecheck(int, lp->lock); asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0070", 49) /* NIAI 7 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,7,0", 49) /* NIAI 7 */ " sth %1,%0\n" : "=R" (((unsigned short *) &lp->lock)[1]) : "d" (0) : "cc", "memory"); diff --git a/arch/s390/include/asm/syscall_wrapper.h b/arch/s390/include/asm/syscall_wrapper.h index ad2c996e7e93..fde7e6b1df48 100644 --- a/arch/s390/include/asm/syscall_wrapper.h +++ b/arch/s390/include/asm/syscall_wrapper.h @@ -162,4 +162,4 @@ __diag_pop(); \ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) -#endif /* _ASM_X86_SYSCALL_WRAPPER_H */ +#endif /* _ASM_S390_SYSCALL_WRAPPER_H */ diff --git a/arch/s390/include/asm/unwind.h b/arch/s390/include/asm/unwind.h index 5ebf534ef753..0bf06f1682d8 100644 --- a/arch/s390/include/asm/unwind.h +++ b/arch/s390/include/asm/unwind.h @@ -4,6 +4,8 @@ #include <linux/sched.h> #include <linux/ftrace.h> +#include <linux/kprobes.h> +#include <linux/llist.h> #include <asm/ptrace.h> #include <asm/stacktrace.h> @@ -36,10 +38,21 @@ struct unwind_state { struct pt_regs *regs; unsigned long sp, ip; int graph_idx; + struct llist_node *kr_cur; bool reliable; bool error; }; +/* Recover the return address modified by kretprobe and ftrace_graph. */ +static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, + unsigned long ip) +{ + ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, NULL); + if (is_kretprobe_trampoline(ip)) + ip = kretprobe_find_ret_addr(state->task, (void *)state->sp, &state->kr_cur); + return ip; +} + void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long first_frame); bool unwind_next_frame(struct unwind_state *state); diff --git a/arch/s390/include/asm/user.h b/arch/s390/include/asm/user.h index 0ca572ced21b..8e8aaf48582e 100644 --- a/arch/s390/include/asm/user.h +++ b/arch/s390/include/asm/user.h @@ -67,9 +67,5 @@ struct user { unsigned long magic; /* To uniquely identify a core file */ char u_comm[32]; /* User command that was responsible */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif /* _S390_USER_H */ diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index a601a518b569..59b69c8ab5e1 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -121,22 +121,22 @@ _LPP_OFFSET = __LC_LPP .endm .macro BPOFF - ALTERNATIVE "", ".long 0xb2e8c000", 82 + ALTERNATIVE "", ".insn rrf,0xb2e80000,0,0,12,0", 82 .endm .macro BPON - ALTERNATIVE "", ".long 0xb2e8d000", 82 + ALTERNATIVE "", ".insn rrf,0xb2e80000,0,0,13,0", 82 .endm .macro BPENTER tif_ptr,tif_mask - ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ + ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \ "", 82 .endm .macro BPEXIT tif_ptr,tif_mask TSTMSK \tif_ptr,\tif_mask - ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ - "jnz .+8; .long 0xb2e8d000", 82 + ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \ + "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82 .endm /* diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 28ae7df26c4a..1cc85b8ff42e 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1646,8 +1646,8 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); - mem_assign_absolute(S390_lowcore.ipib, ipib); - mem_assign_absolute(S390_lowcore.ipib_checksum, csum); + put_abs_lowcore(ipib, ipib); + put_abs_lowcore(ipib_checksum, csum); dump_run(trigger); } diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index e32c14fd1282..0032bdbe8e3f 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -284,11 +284,11 @@ NOKPROBE_SYMBOL(pop_kprobe); void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { - ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14]; - ri->fp = NULL; + ri->ret_addr = (kprobe_opcode_t *)regs->gprs[14]; + ri->fp = (void *)regs->gprs[15]; /* Replace the return addr with trampoline addr */ - regs->gprs[14] = (unsigned long) &__kretprobe_trampoline; + regs->gprs[14] = (unsigned long)&__kretprobe_trampoline; } NOKPROBE_SYMBOL(arch_prepare_kretprobe); @@ -385,7 +385,7 @@ NOKPROBE_SYMBOL(arch_kretprobe_fixup_return); */ void trampoline_probe_handler(struct pt_regs *regs) { - kretprobe_trampoline_handler(regs, NULL); + kretprobe_trampoline_handler(regs, (void *)regs->gprs[15]); } NOKPROBE_SYMBOL(trampoline_probe_handler); diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 088d57a3083f..b2ef014a9287 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -226,7 +226,7 @@ void arch_crash_save_vmcoreinfo(void) vmcoreinfo_append_str("SAMODE31=%lx\n", __samode31); vmcoreinfo_append_str("EAMODE31=%lx\n", __eamode31); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - mem_assign_absolute(S390_lowcore.vmcore_info, paddr_vmcoreinfo_note()); + put_abs_lowcore(vmcore_info, paddr_vmcoreinfo_note()); } void machine_shutdown(void) diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 6b5b64e67eee..1acc2e05d70f 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -63,7 +63,7 @@ void __init os_info_init(void) os_info.version_minor = OS_INFO_VERSION_MINOR; os_info.magic = OS_INFO_MAGIC; os_info.csum = os_info_csum(&os_info); - mem_assign_absolute(S390_lowcore.os_info, __pa(ptr)); + put_abs_lowcore(os_info, __pa(ptr)); } #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 84e23fcc1106..d860ac300919 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -481,11 +481,11 @@ static void __init setup_lowcore_dat_off(void) lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET; /* Setup absolute zero lowcore */ - mem_assign_absolute(S390_lowcore.restart_stack, lc->restart_stack); - mem_assign_absolute(S390_lowcore.restart_fn, lc->restart_fn); - mem_assign_absolute(S390_lowcore.restart_data, lc->restart_data); - mem_assign_absolute(S390_lowcore.restart_source, lc->restart_source); - mem_assign_absolute(S390_lowcore.restart_psw, lc->restart_psw); + put_abs_lowcore(restart_stack, lc->restart_stack); + put_abs_lowcore(restart_fn, lc->restart_fn); + put_abs_lowcore(restart_data, lc->restart_data); + put_abs_lowcore(restart_source, lc->restart_source); + put_abs_lowcore(restart_psw, lc->restart_psw); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; @@ -501,6 +501,7 @@ static void __init setup_lowcore_dat_off(void) static void __init setup_lowcore_dat_on(void) { struct lowcore *lc = lowcore_ptr[0]; + int cr; __ctl_clear_bit(0, 28); S390_lowcore.external_new_psw.mask |= PSW_MASK_DAT; @@ -509,10 +510,10 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; __ctl_store(S390_lowcore.cregs_save_area, 0, 15); __ctl_set_bit(0, 28); - mem_assign_absolute(S390_lowcore.restart_flags, RESTART_FLAG_CTLREGS); - mem_assign_absolute(S390_lowcore.program_new_psw, lc->program_new_psw); - memcpy_absolute(&S390_lowcore.cregs_save_area, lc->cregs_save_area, - sizeof(S390_lowcore.cregs_save_area)); + put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); + put_abs_lowcore(program_new_psw, lc->program_new_psw); + for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) + put_abs_lowcore(cregs_save_area[cr], lc->cregs_save_area[cr]); } static struct resource code_resource = { diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 127da1850b06..30c91d565933 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -213,7 +213,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) if (nmi_alloc_mcesa(&lc->mcesad)) goto out; lowcore_ptr[cpu] = lc; - pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc); + pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc)); return 0; out: @@ -326,10 +326,17 @@ static void pcpu_delegate(struct pcpu *pcpu, /* Stop target cpu (if func returns this stops the current cpu). */ pcpu_sigp_retry(pcpu, SIGP_STOP, 0); /* Restart func on the target cpu and stop the current cpu. */ - mem_assign_absolute(lc->restart_stack, stack); - mem_assign_absolute(lc->restart_fn, (unsigned long) func); - mem_assign_absolute(lc->restart_data, (unsigned long) data); - mem_assign_absolute(lc->restart_source, source_cpu); + if (lc) { + lc->restart_stack = stack; + lc->restart_fn = (unsigned long)func; + lc->restart_data = (unsigned long)data; + lc->restart_source = source_cpu; + } else { + put_abs_lowcore(restart_stack, stack); + put_abs_lowcore(restart_fn, (unsigned long)func); + put_abs_lowcore(restart_data, (unsigned long)data); + put_abs_lowcore(restart_source, source_cpu); + } __bpon(); asm volatile( "0: sigp 0,%0,%2 # sigp restart to target cpu\n" @@ -570,39 +577,27 @@ static void smp_ctl_bit_callback(void *info) } static DEFINE_SPINLOCK(ctl_lock); -static unsigned long ctlreg; -/* - * Set a bit in a control register of all cpus - */ -void smp_ctl_set_bit(int cr, int bit) +void smp_ctl_set_clear_bit(int cr, int bit, bool set) { - struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr }; - - spin_lock(&ctl_lock); - memcpy_absolute(&ctlreg, &S390_lowcore.cregs_save_area[cr], sizeof(ctlreg)); - __set_bit(bit, &ctlreg); - memcpy_absolute(&S390_lowcore.cregs_save_area[cr], &ctlreg, sizeof(ctlreg)); - spin_unlock(&ctl_lock); - on_each_cpu(smp_ctl_bit_callback, &parms, 1); -} -EXPORT_SYMBOL(smp_ctl_set_bit); - -/* - * Clear a bit in a control register of all cpus - */ -void smp_ctl_clear_bit(int cr, int bit) -{ - struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr }; + struct ec_creg_mask_parms parms = { .cr = cr, }; + u64 ctlreg; + if (set) { + parms.orval = 1UL << bit; + parms.andval = -1UL; + } else { + parms.orval = 0; + parms.andval = ~(1UL << bit); + } spin_lock(&ctl_lock); - memcpy_absolute(&ctlreg, &S390_lowcore.cregs_save_area[cr], sizeof(ctlreg)); - __clear_bit(bit, &ctlreg); - memcpy_absolute(&S390_lowcore.cregs_save_area[cr], &ctlreg, sizeof(ctlreg)); + get_abs_lowcore(ctlreg, cregs_save_area[cr]); + ctlreg = (ctlreg & parms.andval) | parms.orval; + put_abs_lowcore(cregs_save_area[cr], ctlreg); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } -EXPORT_SYMBOL(smp_ctl_clear_bit); +EXPORT_SYMBOL(smp_ctl_set_clear_bit); #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/syscalls/Makefile b/arch/s390/kernel/syscalls/Makefile index b98f25029b8e..fb85e797946d 100644 --- a/arch/s390/kernel/syscalls/Makefile +++ b/arch/s390/kernel/syscalls/Makefile @@ -21,8 +21,7 @@ uapi: $(uapi-hdrs-y) # Create output directory if not already present -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) filechk_syshdr = $(CONFIG_SHELL) '$(systbl)' -H -a $(syshdr_abi_$(basetarget)) -f "$2" < $< diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 674c65019434..1d2aa448d103 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -141,10 +141,10 @@ static inline void do_fp_trap(struct pt_regs *regs, __u32 fpc) do_trap(regs, SIGFPE, si_code, "floating point exception"); } -static void translation_exception(struct pt_regs *regs) +static void translation_specification_exception(struct pt_regs *regs) { /* May never happen. */ - panic("Translation exception"); + panic("Translation-Specification Exception"); } static void illegal_op(struct pt_regs *regs) @@ -368,7 +368,7 @@ static void (*pgm_check_table[128])(struct pt_regs *regs) = { [0x0f] = hfp_divide_exception, [0x10] = do_dat_exception, [0x11] = do_dat_exception, - [0x12] = translation_exception, + [0x12] = translation_specification_exception, [0x13] = special_op_exception, [0x14] = default_trap_handler, [0x15] = operand_exception, diff --git a/arch/s390/kernel/unwind_bc.c b/arch/s390/kernel/unwind_bc.c index 707fd99f6734..0ece156fdd7c 100644 --- a/arch/s390/kernel/unwind_bc.c +++ b/arch/s390/kernel/unwind_bc.c @@ -64,8 +64,8 @@ bool unwind_next_frame(struct unwind_state *state) ip = READ_ONCE_NOCHECK(sf->gprs[8]); reliable = false; regs = NULL; - if (!__kernel_text_address(ip)) { - /* skip bogus %r14 */ + /* skip bogus %r14 or if is the same as regs->psw.addr */ + if (!__kernel_text_address(ip) || state->ip == unwind_recover_ret_addr(state, ip)) { state->regs = NULL; return unwind_next_frame(state); } @@ -103,13 +103,11 @@ bool unwind_next_frame(struct unwind_state *state) if (sp & 0x7) goto out_err; - ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, (void *) sp); - /* Update unwind state */ state->sp = sp; - state->ip = ip; state->regs = regs; state->reliable = reliable; + state->ip = unwind_recover_ret_addr(state, ip); return true; out_err: @@ -161,12 +159,10 @@ void __unwind_start(struct unwind_state *state, struct task_struct *task, ip = READ_ONCE_NOCHECK(sf->gprs[8]); } - ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, NULL); - /* Update unwind state */ state->sp = sp; - state->ip = ip; state->reliable = true; + state->ip = unwind_recover_ret_addr(state, ip); if (!first_frame) return; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ab73d99483fe..156d1c25a3c1 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3462,7 +3462,7 @@ void exit_sie(struct kvm_vcpu *vcpu) /* Kick a guest cpu out of SIE to process a request synchronously */ void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu) { - kvm_make_request(req, vcpu); + __kvm_make_request(req, vcpu); kvm_s390_vcpu_request(vcpu); } diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 692dc84cd19c..5e7ea8b111e8 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -75,7 +75,7 @@ static inline int arch_load_niai4(int *lock) int owner; asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0040", 49) /* NIAI 4 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,4,0", 49) /* NIAI 4 */ " l %0,%1\n" : "=d" (owner) : "Q" (*lock) : "memory"); return owner; @@ -86,7 +86,7 @@ static inline int arch_cmpxchg_niai8(int *lock, int old, int new) int expected = old; asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0080", 49) /* NIAI 8 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,8,0", 49) /* NIAI 8 */ " cs %0,%3,%1\n" : "=d" (old), "=Q" (*lock) : "0" (old), "d" (new), "Q" (*lock) diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c index c01f02887de4..9bb067321ab4 100644 --- a/arch/s390/lib/test_unwind.c +++ b/arch/s390/lib/test_unwind.c @@ -47,7 +47,7 @@ static void print_backtrace(char *bt) static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, unsigned long sp) { - int frame_count, prev_is_func2, seen_func2_func1; + int frame_count, prev_is_func2, seen_func2_func1, seen_kretprobe_trampoline; const int max_frames = 128; struct unwind_state state; size_t bt_pos = 0; @@ -63,6 +63,7 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, frame_count = 0; prev_is_func2 = 0; seen_func2_func1 = 0; + seen_kretprobe_trampoline = 0; unwind_for_each_frame(&state, task, regs, sp) { unsigned long addr = unwind_get_return_address(&state); char sym[KSYM_SYMBOL_LEN]; @@ -88,6 +89,8 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, if (prev_is_func2 && str_has_prefix(sym, "unwindme_func1")) seen_func2_func1 = 1; prev_is_func2 = str_has_prefix(sym, "unwindme_func2"); + if (str_has_prefix(sym, "__kretprobe_trampoline+0x0/")) + seen_kretprobe_trampoline = 1; } /* Check the results. */ @@ -103,6 +106,10 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, kunit_err(current_test, "Maximum number of frames exceeded\n"); ret = -EINVAL; } + if (seen_kretprobe_trampoline) { + kunit_err(current_test, "__kretprobe_trampoline+0x0 in unwinding results\n"); + ret = -EINVAL; + } if (ret || force_bt) print_backtrace(bt); kfree(bt); @@ -132,36 +139,50 @@ static struct unwindme *unwindme; #define UWM_PGM 0x40 /* Unwind from program check handler */ #define UWM_KPROBE_ON_FTRACE 0x80 /* Unwind from kprobe handler called via ftrace. */ #define UWM_FTRACE 0x100 /* Unwind from ftrace handler. */ -#define UWM_KRETPROBE 0x200 /* Unwind kretprobe handlers. */ +#define UWM_KRETPROBE 0x200 /* Unwind through kretprobed function. */ +#define UWM_KRETPROBE_HANDLER 0x400 /* Unwind from kretprobe handler. */ -static __always_inline unsigned long get_psw_addr(void) +static __always_inline struct pt_regs fake_pt_regs(void) { - unsigned long psw_addr; + struct pt_regs regs; + + memset(®s, 0, sizeof(regs)); + regs.gprs[15] = current_stack_pointer(); asm volatile( "basr %[psw_addr],0\n" - : [psw_addr] "=d" (psw_addr)); - return psw_addr; + : [psw_addr] "=d" (regs.psw.addr)); + return regs; } static int kretprobe_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) { struct unwindme *u = unwindme; + if (!(u->flags & UWM_KRETPROBE_HANDLER)) + return 0; + u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL, (u->flags & UWM_SP) ? u->sp : 0); return 0; } -static noinline notrace void test_unwind_kretprobed_func(void) +static noinline notrace int test_unwind_kretprobed_func(struct unwindme *u) { - asm volatile(" nop\n"); + struct pt_regs regs; + + if (!(u->flags & UWM_KRETPROBE)) + return 0; + + regs = fake_pt_regs(); + return test_unwind(NULL, (u->flags & UWM_REGS) ? ®s : NULL, + (u->flags & UWM_SP) ? u->sp : 0); } -static noinline void test_unwind_kretprobed_func_caller(void) +static noinline int test_unwind_kretprobed_func_caller(struct unwindme *u) { - test_unwind_kretprobed_func(); + return test_unwind_kretprobed_func(u); } static int test_unwind_kretprobe(struct unwindme *u) @@ -187,10 +208,12 @@ static int test_unwind_kretprobe(struct unwindme *u) return -EINVAL; } - test_unwind_kretprobed_func_caller(); + ret = test_unwind_kretprobed_func_caller(u); unregister_kretprobe(&my_kretprobe); unwindme = NULL; - return u->ret; + if (u->flags & UWM_KRETPROBE_HANDLER) + ret = u->ret; + return ret; } static int kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) @@ -304,16 +327,13 @@ static noinline int unwindme_func4(struct unwindme *u) return 0; } else if (u->flags & (UWM_PGM | UWM_KPROBE_ON_FTRACE)) { return test_unwind_kprobe(u); - } else if (u->flags & (UWM_KRETPROBE)) { + } else if (u->flags & (UWM_KRETPROBE | UWM_KRETPROBE_HANDLER)) { return test_unwind_kretprobe(u); } else if (u->flags & UWM_FTRACE) { return test_unwind_ftrace(u); } else { - struct pt_regs regs; + struct pt_regs regs = fake_pt_regs(); - memset(®s, 0, sizeof(regs)); - regs.psw.addr = get_psw_addr(); - regs.gprs[15] = current_stack_pointer(); return test_unwind(NULL, (u->flags & UWM_REGS) ? ®s : NULL, (u->flags & UWM_SP) ? u->sp : 0); @@ -452,6 +472,10 @@ static const struct test_params param_list[] = { TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP), TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_REGS), TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP | UWM_REGS), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_REGS), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP | UWM_REGS), }; /* diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 792f8e0f2178..e563cb65c0c4 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -69,6 +69,7 @@ struct zpci_dev *get_zdev_by_fid(u32 fid) list_for_each_entry(tmp, &zpci_list, entry) { if (tmp->fid == fid) { zdev = tmp; + zpci_zdev_get(zdev); break; } } @@ -399,7 +400,7 @@ EXPORT_SYMBOL(pci_iounmap); static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { - struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn); + struct zpci_dev *zdev = zdev_from_bus(bus, devfn); return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; } @@ -407,7 +408,7 @@ static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { - struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn); + struct zpci_dev *zdev = zdev_from_bus(bus, devfn); return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; } diff --git a/arch/s390/pci/pci_bus.h b/arch/s390/pci/pci_bus.h index e359d2686178..e96c9860e064 100644 --- a/arch/s390/pci/pci_bus.h +++ b/arch/s390/pci/pci_bus.h @@ -19,7 +19,8 @@ void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error); void zpci_release_device(struct kref *kref); static inline void zpci_zdev_put(struct zpci_dev *zdev) { - kref_put(&zdev->kref, zpci_release_device); + if (zdev) + kref_put(&zdev->kref, zpci_release_device); } static inline void zpci_zdev_get(struct zpci_dev *zdev) @@ -32,8 +33,8 @@ void zpci_free_domain(int domain); int zpci_setup_bus_resources(struct zpci_dev *zdev, struct list_head *resources); -static inline struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus, - unsigned int devfn) +static inline struct zpci_dev *zdev_from_bus(struct pci_bus *bus, + unsigned int devfn) { struct zpci_bus *zbus = bus->sysdata; diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 63f3e057c168..1057d7af4a55 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -23,6 +23,8 @@ #include <asm/clp.h> #include <uapi/asm/clp.h> +#include "pci_bus.h" + bool zpci_unique_uid; void update_uid_checking(bool new) @@ -404,8 +406,11 @@ static void __clp_add(struct clp_fh_list_entry *entry, void *data) return; zdev = get_zdev_by_fid(entry->fid); - if (!zdev) - zpci_create_device(entry->fid, entry->fh, entry->config_state); + if (zdev) { + zpci_zdev_put(zdev); + return; + } + zpci_create_device(entry->fid, entry->fh, entry->config_state); } int clp_scan_pci_devices(void) diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 2e3e5b278925..ea9db5cea64e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -269,7 +269,7 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); if (!pdev) - return; + goto no_pdev; switch (ccdf->pec) { case 0x003a: /* Service Action or Error Recovery Successful */ @@ -286,6 +286,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) break; } pci_dev_put(pdev); +no_pdev: + zpci_zdev_put(zdev); } void zpci_event_error(void *data) @@ -314,6 +316,7 @@ static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh) static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) { struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); + bool existing_zdev = !!zdev; enum zpci_state state; zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", @@ -378,6 +381,8 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) default: break; } + if (existing_zdev) + zpci_zdev_put(zdev); } void zpci_event_availability(void *data) diff --git a/arch/sh/include/asm/user.h b/arch/sh/include/asm/user.h index 7dfd3f6461e6..12ea0f3f4419 100644 --- a/arch/sh/include/asm/user.h +++ b/arch/sh/include/asm/user.h @@ -52,10 +52,4 @@ struct user { char u_comm[32]; /* user command name */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_DATA_START_ADDR (u.start_data) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) - #endif /* __ASM_SH_USER_H */ diff --git a/arch/sh/kernel/syscalls/Makefile b/arch/sh/kernel/syscalls/Makefile index 6713c65a25e1..b265e4bc16c2 100644 --- a/arch/sh/kernel/syscalls/Makefile +++ b/arch/sh/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/sparc/kernel/syscalls/Makefile b/arch/sparc/kernel/syscalls/Makefile index d63f18dd058d..8440c16dfb22 100644 --- a/arch/sparc/kernel/syscalls/Makefile +++ b/arch/sparc/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c index 6ead1e240457..8ca67a692683 100644 --- a/arch/um/drivers/mconsole_kern.c +++ b/arch/um/drivers/mconsole_kern.c @@ -224,7 +224,7 @@ void mconsole_go(struct mc_request *req) void mconsole_stop(struct mc_request *req) { - deactivate_fd(req->originating_fd, MCONSOLE_IRQ); + block_signals(); os_set_fd_block(req->originating_fd, 1); mconsole_reply(req, "stopped", 0, 0); for (;;) { @@ -247,6 +247,7 @@ void mconsole_stop(struct mc_request *req) } os_set_fd_block(req->originating_fd, 0); mconsole_reply(req, "", 0, 0); + unblock_signals(); } static DEFINE_SPINLOCK(mc_devices_lock); diff --git a/arch/um/drivers/port_user.c b/arch/um/drivers/port_user.c index 5b5b64cb1071..3c62ae81df62 100644 --- a/arch/um/drivers/port_user.c +++ b/arch/um/drivers/port_user.c @@ -5,6 +5,7 @@ #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <errno.h> #include <termios.h> #include <unistd.h> @@ -167,14 +168,29 @@ static void port_pre_exec(void *arg) int port_connection(int fd, int *socket, int *pid_out) { int new, err; - char *argv[] = { "/usr/sbin/in.telnetd", "-L", + char *env; + char *argv[] = { "in.telnetd", "-L", OS_LIB_PATH "/uml/port-helper", NULL }; struct port_pre_exec_data data; + if ((env = getenv("UML_PORT_HELPER"))) + argv[2] = env; + new = accept(fd, NULL, 0); if (new < 0) return -errno; + err = os_access(argv[2], X_OK); + if (err < 0) { + printk(UM_KERN_ERR "port_connection : error accessing port-helper " + "executable at %s: %s\n", argv[2], strerror(-err)); + if (env == NULL) + printk(UM_KERN_ERR "Set UML_PORT_HELPER environment " + "variable to path to uml-utilities port-helper " + "binary\n"); + goto out_close; + } + err = os_pipe(socket, 0, 0); if (err < 0) goto out_close; diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index 69d2d0049a61..b03269faef71 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -1526,13 +1526,19 @@ static void do_io(struct io_thread_req *req, struct io_desc *desc) } break; case REQ_OP_DISCARD: - case REQ_OP_WRITE_ZEROES: n = os_falloc_punch(req->fds[bit], off, len); if (n) { req->error = map_error(-n); return; } break; + case REQ_OP_WRITE_ZEROES: + n = os_falloc_zeroes(req->fds[bit], off, len); + if (n) { + req->error = map_error(-n); + return; + } + break; default: WARN_ON_ONCE(1); req->error = BLK_STS_NOTSUPP; diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 4fc1a5d70dcf..1d6f6a66766c 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -67,6 +67,7 @@ static LIST_HEAD(vector_devices); static int driver_registered; static void vector_eth_configure(int n, struct arglist *def); +static int vector_mmsg_rx(struct vector_private *vp, int budget); /* Argument accessors to set variables (and/or set default values) * mtu, buffer sizing, default headroom, etc @@ -77,7 +78,6 @@ static void vector_eth_configure(int n, struct arglist *def); #define DEFAULT_VECTOR_SIZE 64 #define TX_SMALL_PACKET 128 #define MAX_IOV_SIZE (MAX_SKB_FRAGS + 1) -#define MAX_ITERATIONS 64 static const struct { const char string[ETH_GSTRING_LEN]; @@ -458,7 +458,6 @@ static int vector_send(struct vector_queue *qi) vp->estats.tx_queue_running_average = (vp->estats.tx_queue_running_average + result) >> 1; } - netif_trans_update(qi->dev); netif_wake_queue(qi->dev); /* if TX is busy, break out of the send loop, * poll write IRQ will reschedule xmit for us @@ -470,8 +469,6 @@ static int vector_send(struct vector_queue *qi) } } spin_unlock(&qi->head_lock); - } else { - tasklet_schedule(&vp->tx_poll); } return queue_depth; } @@ -608,7 +605,7 @@ out_fail: /* * We do not use the RX queue as a proper wraparound queue for now - * This is not necessary because the consumption via netif_rx() + * This is not necessary because the consumption via napi_gro_receive() * happens in-line. While we can try using the return code of * netif_rx() for flow control there are no drivers doing this today. * For this RX specific use we ignore the tail/head locks and @@ -896,7 +893,7 @@ static int vector_legacy_rx(struct vector_private *vp) skb->protocol = eth_type_trans(skb, skb->dev); vp->dev->stats.rx_bytes += skb->len; vp->dev->stats.rx_packets++; - netif_rx(skb); + napi_gro_receive(&vp->napi, skb); } else { dev_kfree_skb_irq(skb); } @@ -955,7 +952,7 @@ drop: * mmsg vector matched to an skb vector which we prepared earlier. */ -static int vector_mmsg_rx(struct vector_private *vp) +static int vector_mmsg_rx(struct vector_private *vp, int budget) { int packet_count, i; struct vector_queue *qi = vp->rx_queue; @@ -972,6 +969,9 @@ static int vector_mmsg_rx(struct vector_private *vp) /* Fire the Lazy Gun - get as many packets as we can in one go. */ + if (budget > qi->max_depth) + budget = qi->max_depth; + packet_count = uml_vector_recvmmsg( vp->fds->rx_fd, qi->mmsg_vector, qi->max_depth, 0); @@ -1021,7 +1021,7 @@ static int vector_mmsg_rx(struct vector_private *vp) */ vp->dev->stats.rx_bytes += skb->len; vp->dev->stats.rx_packets++; - netif_rx(skb); + napi_gro_receive(&vp->napi, skb); } else { /* Overlay header too short to do anything - discard. * We can actually keep this skb and reuse it, @@ -1044,23 +1044,6 @@ static int vector_mmsg_rx(struct vector_private *vp) return packet_count; } -static void vector_rx(struct vector_private *vp) -{ - int err; - int iter = 0; - - if ((vp->options & VECTOR_RX) > 0) - while (((err = vector_mmsg_rx(vp)) > 0) && (iter < MAX_ITERATIONS)) - iter++; - else - while (((err = vector_legacy_rx(vp)) > 0) && (iter < MAX_ITERATIONS)) - iter++; - if ((err != 0) && net_ratelimit()) - netdev_err(vp->dev, "vector_rx: error(%d)\n", err); - if (iter == MAX_ITERATIONS) - netdev_err(vp->dev, "vector_rx: device stuck, remote end may have closed the connection\n"); -} - static int vector_net_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct vector_private *vp = netdev_priv(dev); @@ -1085,25 +1068,15 @@ static int vector_net_start_xmit(struct sk_buff *skb, struct net_device *dev) netdev_sent_queue(vp->dev, skb->len); queue_depth = vector_enqueue(vp->tx_queue, skb); - /* if the device queue is full, stop the upper layers and - * flush it. - */ - - if (queue_depth >= vp->tx_queue->max_depth - 1) { - vp->estats.tx_kicks++; - netif_stop_queue(dev); - vector_send(vp->tx_queue); - return NETDEV_TX_OK; - } - if (netdev_xmit_more()) { + if (queue_depth < vp->tx_queue->max_depth && netdev_xmit_more()) { mod_timer(&vp->tl, vp->coalesce); return NETDEV_TX_OK; + } else { + queue_depth = vector_send(vp->tx_queue); + if (queue_depth > 0) + napi_schedule(&vp->napi); } - if (skb->len < TX_SMALL_PACKET) { - vp->estats.tx_kicks++; - vector_send(vp->tx_queue); - } else - tasklet_schedule(&vp->tx_poll); + return NETDEV_TX_OK; } @@ -1114,7 +1087,7 @@ static irqreturn_t vector_rx_interrupt(int irq, void *dev_id) if (!netif_running(dev)) return IRQ_NONE; - vector_rx(vp); + napi_schedule(&vp->napi); return IRQ_HANDLED; } @@ -1133,8 +1106,7 @@ static irqreturn_t vector_tx_interrupt(int irq, void *dev_id) * tweaking the IRQ mask less costly */ - if (vp->in_write_poll) - tasklet_schedule(&vp->tx_poll); + napi_schedule(&vp->napi); return IRQ_HANDLED; } @@ -1161,7 +1133,8 @@ static int vector_net_close(struct net_device *dev) um_free_irq(vp->tx_irq, dev); vp->tx_irq = 0; } - tasklet_kill(&vp->tx_poll); + napi_disable(&vp->napi); + netif_napi_del(&vp->napi); if (vp->fds->rx_fd > 0) { if (vp->bpf) uml_vector_detach_bpf(vp->fds->rx_fd, vp->bpf); @@ -1193,15 +1166,32 @@ static int vector_net_close(struct net_device *dev) return 0; } -/* TX tasklet */ - -static void vector_tx_poll(struct tasklet_struct *t) +static int vector_poll(struct napi_struct *napi, int budget) { - struct vector_private *vp = from_tasklet(vp, t, tx_poll); + struct vector_private *vp = container_of(napi, struct vector_private, napi); + int work_done = 0; + int err; + bool tx_enqueued = false; - vp->estats.tx_kicks++; - vector_send(vp->tx_queue); + if ((vp->options & VECTOR_TX) != 0) + tx_enqueued = (vector_send(vp->tx_queue) > 0); + if ((vp->options & VECTOR_RX) > 0) + err = vector_mmsg_rx(vp, budget); + else { + err = vector_legacy_rx(vp); + if (err > 0) + err = 1; + } + if (err > 0) + work_done += err; + + if (tx_enqueued || err > 0) + napi_schedule(napi); + if (work_done < budget) + napi_complete_done(napi, work_done); + return work_done; } + static void vector_reset_tx(struct work_struct *work) { struct vector_private *vp = @@ -1265,6 +1255,9 @@ static int vector_net_open(struct net_device *dev) goto out_close; } + netif_napi_add(vp->dev, &vp->napi, vector_poll, get_depth(vp->parsed)); + napi_enable(&vp->napi); + /* READ IRQ */ err = um_request_irq( irq_rr + VECTOR_BASE_IRQ, vp->fds->rx_fd, @@ -1306,15 +1299,15 @@ static int vector_net_open(struct net_device *dev) uml_vector_attach_bpf(vp->fds->rx_fd, vp->bpf); netif_start_queue(dev); + vector_reset_stats(vp); /* clear buffer - it can happen that the host side of the interface * is full when we get here. In this case, new data is never queued, * SIGIOs never arrive, and the net never works. */ - vector_rx(vp); + napi_schedule(&vp->napi); - vector_reset_stats(vp); vdevice = find_device(vp->unit); vdevice->opened = 1; @@ -1543,15 +1536,16 @@ static const struct net_device_ops vector_netdev_ops = { #endif }; - static void vector_timer_expire(struct timer_list *t) { struct vector_private *vp = from_timer(vp, t, tl); vp->estats.tx_kicks++; - vector_send(vp->tx_queue); + napi_schedule(&vp->napi); } + + static void vector_eth_configure( int n, struct arglist *def @@ -1634,7 +1628,6 @@ static void vector_eth_configure( }); dev->features = dev->hw_features = (NETIF_F_SG | NETIF_F_FRAGLIST); - tasklet_setup(&vp->tx_poll, vector_tx_poll); INIT_WORK(&vp->reset_tx, vector_reset_tx); timer_setup(&vp->tl, vector_timer_expire, 0); diff --git a/arch/um/drivers/vector_kern.h b/arch/um/drivers/vector_kern.h index 8fff93a75a92..2a1fa8e0f3e1 100644 --- a/arch/um/drivers/vector_kern.h +++ b/arch/um/drivers/vector_kern.h @@ -14,6 +14,7 @@ #include <linux/ctype.h> #include <linux/workqueue.h> #include <linux/interrupt.h> + #include "vector_user.h" /* Queue structure specially adapted for multiple enqueue/dequeue @@ -72,6 +73,7 @@ struct vector_private { struct list_head list; spinlock_t lock; struct net_device *dev; + struct napi_struct napi ____cacheline_aligned; int unit; @@ -115,7 +117,6 @@ struct vector_private { spinlock_t stats_lock; - struct tasklet_struct tx_poll; bool rexmit_scheduled; bool opened; bool in_write_poll; diff --git a/arch/um/drivers/vector_user.c b/arch/um/drivers/vector_user.c index e4ffeb9a1fa4..c650e428432b 100644 --- a/arch/um/drivers/vector_user.c +++ b/arch/um/drivers/vector_user.c @@ -771,7 +771,7 @@ int uml_vector_detach_bpf(int fd, void *bpf) printk(KERN_ERR BPF_DETACH_FAIL, prog->len, prog->filter, fd, -errno); return err; } -void *uml_vector_default_bpf(void *mac) +void *uml_vector_default_bpf(const void *mac) { struct sock_filter *bpf; uint32_t *mac1 = (uint32_t *)(mac + 2); diff --git a/arch/um/drivers/vector_user.h b/arch/um/drivers/vector_user.h index d29d5fdd98fa..3a73d17a0161 100644 --- a/arch/um/drivers/vector_user.h +++ b/arch/um/drivers/vector_user.h @@ -97,7 +97,7 @@ extern int uml_vector_recvmmsg( unsigned int vlen, unsigned int flags ); -extern void *uml_vector_default_bpf(void *mac); +extern void *uml_vector_default_bpf(const void *mac); extern void *uml_vector_user_bpf(char *filename); extern int uml_vector_attach_bpf(int fd, void *bpf); extern int uml_vector_detach_bpf(int fd, void *bpf); diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index b08bd2966253..f1f3f52f1e9c 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -24,7 +24,6 @@ generic-y += softirq_stack.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h -generic-y += word-at-a-time.h generic-y += kprobes.h generic-y += mm_hooks.h generic-y += vga.h diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h index f512704a9ec7..22b39de73c24 100644 --- a/arch/um/include/asm/xor.h +++ b/arch/um/include/asm/xor.h @@ -4,8 +4,10 @@ #ifdef CONFIG_64BIT #undef CONFIG_X86_32 +#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_sse_pf64)) #else #define CONFIG_X86_32 1 +#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_8regs)) #endif #include <asm/cpufeature.h> @@ -16,7 +18,7 @@ #undef XOR_SELECT_TEMPLATE /* pick an arbitrary one - measuring isn't possible with inf-cpu */ #define XOR_SELECT_TEMPLATE(x) \ - (time_travel_mode == TT_MODE_INFCPU ? &xor_block_8regs : NULL) + (time_travel_mode == TT_MODE_INFCPU ? TT_CPU_INF_XOR_DEFAULT : x)) #endif #endif diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h index 00214059d9ec..fafde1d5416e 100644 --- a/arch/um/include/shared/os.h +++ b/arch/um/include/shared/os.h @@ -168,6 +168,7 @@ extern unsigned os_major(unsigned long long dev); extern unsigned os_minor(unsigned long long dev); extern unsigned long long os_makedev(unsigned major, unsigned minor); extern int os_falloc_punch(int fd, unsigned long long offset, int count); +extern int os_falloc_zeroes(int fd, unsigned long long offset, int count); extern int os_eventfd(unsigned int initval, int flags); extern int os_sendmsg_fds(int fd, const void *buf, unsigned int len, const int *fds, unsigned int fds_num); diff --git a/arch/um/kernel/dtb.c b/arch/um/kernel/dtb.c index ca69d72025f3..484141b06938 100644 --- a/arch/um/kernel/dtb.c +++ b/arch/um/kernel/dtb.c @@ -25,8 +25,8 @@ void uml_dtb_init(void) return; } - unflatten_device_tree(); early_init_fdt_scan_reserved_mem(); + unflatten_device_tree(); } static int __init uml_dtb_setup(char *line, int *add) diff --git a/arch/um/os-Linux/file.c b/arch/um/os-Linux/file.c index e4421dbc4c36..fc4450db59bd 100644 --- a/arch/um/os-Linux/file.c +++ b/arch/um/os-Linux/file.c @@ -625,6 +625,15 @@ int os_falloc_punch(int fd, unsigned long long offset, int len) return n; } +int os_falloc_zeroes(int fd, unsigned long long offset, int len) +{ + int n = fallocate(fd, FALLOC_FL_ZERO_RANGE|FALLOC_FL_KEEP_SIZE, offset, len); + + if (n < 0) + return -errno; + return n; +} + int os_eventfd(unsigned int initval, int flags) { int fd = eventfd(initval, flags); diff --git a/arch/um/os-Linux/helper.c b/arch/um/os-Linux/helper.c index 32e88baf18dd..b459745f52e2 100644 --- a/arch/um/os-Linux/helper.c +++ b/arch/um/os-Linux/helper.c @@ -4,6 +4,7 @@ */ #include <stdlib.h> +#include <string.h> #include <unistd.h> #include <errno.h> #include <sched.h> @@ -99,6 +100,10 @@ int run_helper(void (*pre_exec)(void *), void *pre_data, char **argv) CATCH_EINTR(waitpid(pid, NULL, __WALL)); } + if (ret < 0) + printk(UM_KERN_ERR "run_helper : failed to exec %s on host: %s\n", + argv[0], strerror(-ret)); + out_free2: kfree(data.buf); out_close: diff --git a/arch/um/os-Linux/time.c b/arch/um/os-Linux/time.c index 6c5041c5560b..4d5591d96d8c 100644 --- a/arch/um/os-Linux/time.c +++ b/arch/um/os-Linux/time.c @@ -18,12 +18,6 @@ static timer_t event_high_res_timer = 0; -static inline long long timeval_to_ns(const struct timeval *tv) -{ - return ((long long) tv->tv_sec * UM_NSEC_PER_SEC) + - tv->tv_usec * UM_NSEC_PER_USEC; -} - static inline long long timespec_to_ns(const struct timespec *ts) { return ((long long) ts->tv_sec * UM_NSEC_PER_SEC) + ts->tv_nsec; diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index ff45a27fc29c..b0142e01002e 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -122,7 +122,6 @@ config X86 select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE select ARCH_WANT_LD_ORPHAN_WARN - select ARCH_WANTS_RT_DELAYED_SIGNALS select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select BUILDTIME_TABLE_SORT diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 71124cf8630c..98a4852ed6a0 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -1,5 +1,7 @@ +CONFIG_WERROR=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y +CONFIG_USELIB=y CONFIG_AUDIT=y CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y @@ -11,23 +13,30 @@ CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y CONFIG_LOG_BUF_SHIFT=18 CONFIG_CGROUPS=y +CONFIG_BLK_CGROUP=y CONFIG_CGROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_MISC=y +CONFIG_CGROUP_DEBUG=y CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y # CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y -# CONFIG_64BIT is not set CONFIG_SMP=y -CONFIG_X86_GENERIC=y -CONFIG_HPET_TIMER=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT=y +CONFIG_NR_CPUS=8 CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y -CONFIG_X86_REBOOTFIXUPS=y CONFIG_MICROCODE_AMD=y CONFIG_X86_MSR=y CONFIG_X86_CPUID=y -CONFIG_HIGHPTE=y CONFIG_X86_CHECK_BIOS_CORRUPTION=y # CONFIG_MTRR_SANITIZER is not set CONFIG_EFI=y @@ -43,12 +52,15 @@ CONFIG_ACPI_BGRT=y CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_ONDEMAND=y CONFIG_X86_ACPI_CPUFREQ=y -CONFIG_EFI_VARS=y CONFIG_KPROBES=y CONFIG_JUMP_LABEL=y +CONFIG_COMPAT_32BIT_TIME=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_BLK_CGROUP_IOLATENCY=y +CONFIG_BLK_CGROUP_IOCOST=y +CONFIG_BLK_CGROUP_IOPRIO=y CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y @@ -103,12 +115,16 @@ CONFIG_IP6_NF_FILTER=y CONFIG_IP6_NF_TARGET_REJECT=y CONFIG_IP6_NF_MANGLE=y CONFIG_NET_SCHED=y +CONFIG_NET_CLS_CGROUP=y CONFIG_NET_EMATCH=y CONFIG_NET_CLS_ACT=y +CONFIG_CGROUP_NET_PRIO=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_LEDS=y CONFIG_RFKILL=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_MSI=y @@ -119,13 +135,16 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_DEBUG_DEVRES=y CONFIG_CONNECTOR=y +CONFIG_EFI_VARS=y +CONFIG_EFI_CAPSULE_LOADER=y CONFIG_BLK_DEV_LOOP=y +CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=y CONFIG_SCSI_CONSTANTS=y CONFIG_SCSI_SPI_ATTRS=y -# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_ATA_PIIX=y @@ -143,6 +162,7 @@ CONFIG_MACINTOSH_DRIVERS=y CONFIG_MAC_EMUMOUSEBTN=y CONFIG_NETDEVICES=y CONFIG_NETCONSOLE=y +CONFIG_VIRTIO_NET=y CONFIG_BNX2=y CONFIG_TIGON3=y CONFIG_NET_TULIP=y @@ -170,6 +190,7 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y CONFIG_SERIAL_8250_DETECT_IRQ=y CONFIG_SERIAL_8250_RSA=y CONFIG_SERIAL_NONSTANDARD=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_NVRAM=y CONFIG_HPET=y @@ -181,12 +202,7 @@ CONFIG_AGP_AMD64=y CONFIG_AGP_INTEL=y CONFIG_DRM=y CONFIG_DRM_I915=y -CONFIG_FB_MODE_HELPERS=y -CONFIG_FB_TILEBLITTING=y -CONFIG_FB_EFI=y -CONFIG_LOGO=y -# CONFIG_LOGO_LINUX_MONO is not set -# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_DRM_VIRTIO_GPU=y CONFIG_SOUND=y CONFIG_SND=y CONFIG_SND_HRTIMER=y @@ -219,6 +235,8 @@ CONFIG_USB_STORAGE=y CONFIG_RTC_CLASS=y # CONFIG_RTC_HCTOSYS is not set CONFIG_DMADEVICES=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_INPUT=y CONFIG_EEEPC_LAPTOP=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y @@ -240,6 +258,7 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y +CONFIG_9P_FS=y CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=y @@ -251,14 +270,15 @@ CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y CONFIG_PRINTK_TIME=y +CONFIG_FRAME_WARN=1024 CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_STACK_USAGE=y -CONFIG_DEBUG_STACKOVERFLOW=y # CONFIG_SCHED_DEBUG is not set CONFIG_SCHEDSTATS=y CONFIG_BLK_DEV_IO_TRACE=y CONFIG_PROVIDE_OHCI1394_DMA_INIT=y CONFIG_EARLY_PRINTK_DBGP=y CONFIG_DEBUG_BOOT_PARAMS=y -CONFIG_KALLSYMS_ALL=y +CONFIG_UNWINDER_FRAME_POINTER=y +# CONFIG_64BIT is not set diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 92b1169ec90b..69784505a7a8 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -1,3 +1,4 @@ +CONFIG_WERROR=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y @@ -11,14 +12,25 @@ CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y CONFIG_LOG_BUF_SHIFT=18 CONFIG_CGROUPS=y +CONFIG_BLK_CGROUP=y CONFIG_CGROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_MISC=y +CONFIG_CGROUP_DEBUG=y CONFIG_BLK_DEV_INITRD=y +CONFIG_KALLSYMS_ALL=y # CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_SMP=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT=y CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y CONFIG_MICROCODE_AMD=y CONFIG_X86_MSR=y @@ -41,12 +53,14 @@ CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_ONDEMAND=y CONFIG_X86_ACPI_CPUFREQ=y CONFIG_IA32_EMULATION=y -CONFIG_EFI_VARS=y CONFIG_KPROBES=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_BLK_CGROUP_IOLATENCY=y +CONFIG_BLK_CGROUP_IOCOST=y +CONFIG_BLK_CGROUP_IOPRIO=y CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y @@ -101,12 +115,16 @@ CONFIG_IP6_NF_FILTER=y CONFIG_IP6_NF_TARGET_REJECT=y CONFIG_IP6_NF_MANGLE=y CONFIG_NET_SCHED=y +CONFIG_NET_CLS_CGROUP=y CONFIG_NET_EMATCH=y CONFIG_NET_CLS_ACT=y +CONFIG_CGROUP_NET_PRIO=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_LEDS=y CONFIG_RFKILL=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_HOTPLUG_PCI=y @@ -116,13 +134,15 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_DEBUG_DEVRES=y CONFIG_CONNECTOR=y +CONFIG_EFI_VARS=y CONFIG_BLK_DEV_LOOP=y +CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=y CONFIG_SCSI_CONSTANTS=y CONFIG_SCSI_SPI_ATTRS=y -# CONFIG_SCSI_LOWLEVEL is not set +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_ATA_PIIX=y @@ -138,6 +158,7 @@ CONFIG_MACINTOSH_DRIVERS=y CONFIG_MAC_EMUMOUSEBTN=y CONFIG_NETDEVICES=y CONFIG_NETCONSOLE=y +CONFIG_VIRTIO_NET=y CONFIG_TIGON3=y CONFIG_NET_TULIP=y CONFIG_E100=y @@ -162,6 +183,7 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y CONFIG_SERIAL_8250_DETECT_IRQ=y CONFIG_SERIAL_8250_RSA=y CONFIG_SERIAL_NONSTANDARD=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y # CONFIG_HW_RANDOM_INTEL is not set # CONFIG_HW_RANDOM_AMD is not set @@ -175,12 +197,7 @@ CONFIG_AGP_AMD64=y CONFIG_AGP_INTEL=y CONFIG_DRM=y CONFIG_DRM_I915=y -CONFIG_FB_MODE_HELPERS=y -CONFIG_FB_TILEBLITTING=y -CONFIG_FB_EFI=y -CONFIG_LOGO=y -# CONFIG_LOGO_LINUX_MONO is not set -# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_DRM_VIRTIO_GPU=y CONFIG_SOUND=y CONFIG_SND=y CONFIG_SND_HRTIMER=y @@ -213,6 +230,8 @@ CONFIG_USB_STORAGE=y CONFIG_RTC_CLASS=y # CONFIG_RTC_HCTOSYS is not set CONFIG_DMADEVICES=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_INPUT=y CONFIG_EEEPC_LAPTOP=y CONFIG_AMD_IOMMU=y CONFIG_INTEL_IOMMU=y @@ -237,6 +256,7 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y +CONFIG_9P_FS=y CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=y @@ -257,4 +277,3 @@ CONFIG_BLK_DEV_IO_TRACE=y CONFIG_PROVIDE_OHCI1394_DMA_INIT=y CONFIG_EARLY_PRINTK_DBGP=y CONFIG_DEBUG_BOOT_PARAMS=y -CONFIG_KALLSYMS_ALL=y diff --git a/arch/x86/entry/syscalls/Makefile b/arch/x86/entry/syscalls/Makefile index 7f3886eeb2ff..eca5d6eff132 100644 --- a/arch/x86/entry/syscalls/Makefile +++ b/arch/x86/entry/syscalls/Makefile @@ -3,8 +3,7 @@ out := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm # Create output directory if not already present -_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \ - $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') +$(shell mkdir -p $(out) $(uapi)) syscall32 := $(src)/syscall_32.tbl syscall64 := $(src)/syscall_64.tbl diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 4138939532c6..d23e80a56eb8 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -249,6 +249,7 @@ enum x86_intercept_stage; #define PFERR_SGX_BIT 15 #define PFERR_GUEST_FINAL_BIT 32 #define PFERR_GUEST_PAGE_BIT 33 +#define PFERR_IMPLICIT_ACCESS_BIT 48 #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT) #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT) @@ -259,6 +260,7 @@ enum x86_intercept_stage; #define PFERR_SGX_MASK (1U << PFERR_SGX_BIT) #define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT) #define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT) +#define PFERR_IMPLICIT_ACCESS (1ULL << PFERR_IMPLICIT_ACCESS_BIT) #define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \ PFERR_WRITE_MASK | \ @@ -430,7 +432,7 @@ struct kvm_mmu { void (*inject_page_fault)(struct kvm_vcpu *vcpu, struct x86_exception *fault); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gpa_t gva_or_gpa, u32 access, + gpa_t gva_or_gpa, u64 access, struct x86_exception *exception); int (*sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp); @@ -512,6 +514,7 @@ struct kvm_pmu { u64 global_ctrl_mask; u64 global_ovf_ctrl_mask; u64 reserved_bits; + u64 raw_event_mask; u8 version; struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC]; struct kvm_pmc fixed_counters[KVM_PMC_MAX_FIXED]; @@ -1040,14 +1043,16 @@ struct kvm_x86_msr_filter { struct msr_bitmap_range ranges[16]; }; -#define APICV_INHIBIT_REASON_DISABLE 0 -#define APICV_INHIBIT_REASON_HYPERV 1 -#define APICV_INHIBIT_REASON_NESTED 2 -#define APICV_INHIBIT_REASON_IRQWIN 3 -#define APICV_INHIBIT_REASON_PIT_REINJ 4 -#define APICV_INHIBIT_REASON_X2APIC 5 -#define APICV_INHIBIT_REASON_BLOCKIRQ 6 -#define APICV_INHIBIT_REASON_ABSENT 7 +enum kvm_apicv_inhibit { + APICV_INHIBIT_REASON_DISABLE, + APICV_INHIBIT_REASON_HYPERV, + APICV_INHIBIT_REASON_NESTED, + APICV_INHIBIT_REASON_IRQWIN, + APICV_INHIBIT_REASON_PIT_REINJ, + APICV_INHIBIT_REASON_X2APIC, + APICV_INHIBIT_REASON_BLOCKIRQ, + APICV_INHIBIT_REASON_ABSENT, +}; struct kvm_arch { unsigned long n_used_mmu_pages; @@ -1401,7 +1406,7 @@ struct kvm_x86_ops { void (*enable_nmi_window)(struct kvm_vcpu *vcpu); void (*enable_irq_window)(struct kvm_vcpu *vcpu); void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr); - bool (*check_apicv_inhibit_reasons)(ulong bit); + bool (*check_apicv_inhibit_reasons)(enum kvm_apicv_inhibit reason); void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); @@ -1585,7 +1590,7 @@ void kvm_mmu_module_exit(void); void kvm_mmu_destroy(struct kvm_vcpu *vcpu); int kvm_mmu_create(struct kvm_vcpu *vcpu); -void kvm_mmu_init_vm(struct kvm *kvm); +int kvm_mmu_init_vm(struct kvm *kvm); void kvm_mmu_uninit_vm(struct kvm *kvm); void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu); @@ -1795,11 +1800,22 @@ gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, bool kvm_apicv_activated(struct kvm *kvm); void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu); -void kvm_request_apicv_update(struct kvm *kvm, bool activate, - unsigned long bit); +void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason, bool set); +void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason, bool set); + +static inline void kvm_set_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason) +{ + kvm_set_or_clear_apicv_inhibit(kvm, reason, true); +} -void __kvm_request_apicv_update(struct kvm *kvm, bool activate, - unsigned long bit); +static inline void kvm_clear_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason) +{ + kvm_set_or_clear_apicv_inhibit(kvm, reason, false); +} int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 7eb2df5417fb..f70a5108d464 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -221,8 +221,14 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_NESTED_CTL_SEV_ES_ENABLE BIT(2) +#define SVM_TSC_RATIO_RSVD 0xffffff0000000000ULL +#define SVM_TSC_RATIO_MIN 0x0000000000000001ULL +#define SVM_TSC_RATIO_MAX 0x000000ffffffffffULL +#define SVM_TSC_RATIO_DEFAULT 0x0100000000ULL + + /* AVIC */ -#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) +#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFFULL) #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) @@ -230,9 +236,11 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) -#define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK (0xFF) +#define AVIC_PHYSICAL_ID_TABLE_SIZE_MASK (0xFFULL) + +#define AVIC_DOORBELL_PHYSICAL_ID_MASK GENMASK_ULL(11, 0) -#define AVIC_DOORBELL_PHYSICAL_ID_MASK (0xFF) +#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL #define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 diff --git a/arch/x86/include/asm/user_32.h b/arch/x86/include/asm/user_32.h index d72c3d66e94f..8963915e533f 100644 --- a/arch/x86/include/asm/user_32.h +++ b/arch/x86/include/asm/user_32.h @@ -124,9 +124,5 @@ struct user{ char u_comm[32]; /* User command that was responsible */ int u_debugreg[8]; }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif /* _ASM_X86_USER_32_H */ diff --git a/arch/x86/include/asm/user_64.h b/arch/x86/include/asm/user_64.h index db909923611c..1dd10f07ccd6 100644 --- a/arch/x86/include/asm/user_64.h +++ b/arch/x86/include/asm/user_64.h @@ -130,9 +130,5 @@ struct user { unsigned long error_code; /* CPU error code or 0 */ unsigned long fault_address; /* CR3 or 0 */ }; -#define NBPG PAGE_SIZE -#define UPAGES 1 -#define HOST_TEXT_START_ADDR (u.start_code) -#define HOST_STACK_END_ADDR (u.start_stack + u.u_ssize * NBPG) #endif /* _ASM_X86_USER_64_H */ diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 19821f027cb3..c049561f373a 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -415,9 +415,6 @@ int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, xpkru = get_xsave_addr(&kstate->regs.xsave, XFEATURE_PKRU); *vpkru = xpkru->pkru; } - - /* Ensure that XCOMP_BV is set up for XSAVES */ - xstate_init_xcomp_bv(&kstate->regs.xsave, kstate->xfeatures); return 0; } EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate); diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 7c7824ae7862..39e1c8626ab9 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -81,10 +81,10 @@ static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; -static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init = - { [ 0 ... XFEATURE_MAX - 1] = -1}; -static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init = - { [ 0 ... XFEATURE_MAX - 1] = -1}; +static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init; + +#define XSTATE_FLAG_SUPERVISOR BIT(0) +#define XSTATE_FLAG_ALIGNED64 BIT(1) /* * Return whether the system supports a given xfeature. @@ -124,17 +124,41 @@ int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) } EXPORT_SYMBOL_GPL(cpu_has_xfeatures); +static bool xfeature_is_aligned64(int xfeature_nr) +{ + return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64; +} + static bool xfeature_is_supervisor(int xfeature_nr) { + return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR; +} + +static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature) +{ + unsigned int offs, i; + /* - * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1) - * returns ECX[0] set to (1) for a supervisor state, and cleared (0) - * for a user state. + * Non-compacted format and legacy features use the cached fixed + * offsets. */ - u32 eax, ebx, ecx, edx; + if (!cpu_feature_enabled(X86_FEATURE_XSAVES) || xfeature <= XFEATURE_SSE) + return xstate_offsets[xfeature]; - cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); - return ecx & 1; + /* + * Compacted format offsets depend on the actual content of the + * compacted xsave area which is determined by the xcomp_bv header + * field. + */ + offs = FXSAVE_SIZE + XSAVE_HDR_SIZE; + for_each_extended_xfeature(i, xcomp_bv) { + if (xfeature_is_aligned64(i)) + offs = ALIGN(offs, 64); + if (i == xfeature) + break; + offs += xstate_sizes[i]; + } + return offs; } /* @@ -182,7 +206,7 @@ static bool xfeature_enabled(enum xfeature xfeature) * Record the offsets and sizes of various xstates contained * in the XSAVE state memory layout. */ -static void __init setup_xstate_features(void) +static void __init setup_xstate_cache(void) { u32 eax, ebx, ecx, edx, i; /* start at the beginning of the "extended state" */ @@ -205,6 +229,7 @@ static void __init setup_xstate_features(void) cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); xstate_sizes[i] = eax; + xstate_flags[i] = ecx; /* * If an xfeature is supervisor state, the offset in EBX is @@ -264,94 +289,6 @@ static void __init print_xstate_features(void) } while (0) /* - * We could cache this like xstate_size[], but we only use - * it here, so it would be a waste of space. - */ -static int xfeature_is_aligned(int xfeature_nr) -{ - u32 eax, ebx, ecx, edx; - - CHECK_XFEATURE(xfeature_nr); - - if (!xfeature_enabled(xfeature_nr)) { - WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n", - xfeature_nr); - return 0; - } - - cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); - /* - * The value returned by ECX[1] indicates the alignment - * of state component 'i' when the compacted format - * of the extended region of an XSAVE area is used: - */ - return !!(ecx & 2); -} - -/* - * This function sets up offsets and sizes of all extended states in - * xsave area. This supports both standard format and compacted format - * of the xsave area. - */ -static void __init setup_xstate_comp_offsets(void) -{ - unsigned int next_offset; - int i; - - /* - * The FP xstates and SSE xstates are legacy states. They are always - * in the fixed offsets in the xsave area in either compacted form - * or standard form. - */ - xstate_comp_offsets[XFEATURE_FP] = 0; - xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state, - xmm_space); - - if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) { - for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) - xstate_comp_offsets[i] = xstate_offsets[i]; - return; - } - - next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; - - for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { - if (xfeature_is_aligned(i)) - next_offset = ALIGN(next_offset, 64); - - xstate_comp_offsets[i] = next_offset; - next_offset += xstate_sizes[i]; - } -} - -/* - * Setup offsets of a supervisor-state-only XSAVES buffer: - * - * The offsets stored in xstate_comp_offsets[] only work for one specific - * value of the Requested Feature BitMap (RFBM). In cases where a different - * RFBM value is used, a different set of offsets is required. This set of - * offsets is for when RFBM=xfeatures_mask_supervisor(). - */ -static void __init setup_supervisor_only_offsets(void) -{ - unsigned int next_offset; - int i; - - next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE; - - for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { - if (!xfeature_is_supervisor(i)) - continue; - - if (xfeature_is_aligned(i)) - next_offset = ALIGN(next_offset, 64); - - xstate_supervisor_only_offsets[i] = next_offset; - next_offset += xstate_sizes[i]; - } -} - -/* * Print out xstate component offsets and sizes */ static void __init print_xstate_offset_size(void) @@ -360,7 +297,8 @@ static void __init print_xstate_offset_size(void) for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", - i, xstate_comp_offsets[i], i, xstate_sizes[i]); + i, xfeature_get_offset(fpu_kernel_cfg.max_features, i), + i, xstate_sizes[i]); } } @@ -419,7 +357,6 @@ static void __init setup_init_fpu_buf(void) if (!boot_cpu_has(X86_FEATURE_XSAVE)) return; - setup_xstate_features(); print_xstate_features(); xstate_init_xcomp_bv(&init_fpstate.regs.xsave, fpu_kernel_cfg.max_features); @@ -448,25 +385,6 @@ static void __init setup_init_fpu_buf(void) fxsave(&init_fpstate.regs.fxsave); } -static int xfeature_uncompacted_offset(int xfeature_nr) -{ - u32 eax, ebx, ecx, edx; - - /* - * Only XSAVES supports supervisor states and it uses compacted - * format. Checking a supervisor state's uncompacted offset is - * an error. - */ - if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) { - WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr); - return -1; - } - - CHECK_XFEATURE(xfeature_nr); - cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); - return ebx; -} - int xfeature_size(int xfeature_nr) { u32 eax, ebx, ecx, edx; @@ -644,29 +562,15 @@ static bool __init check_xstate_against_struct(int nr) static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) { - unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE; - int i; + unsigned int topmost = fls64(xfeatures) - 1; + unsigned int offset = xstate_offsets[topmost]; - for_each_extended_xfeature(i, xfeatures) { - /* Align from the end of the previous feature */ - if (xfeature_is_aligned(i)) - size = ALIGN(size, 64); - /* - * In compacted format the enabled features are packed, - * i.e. disabled features do not occupy space. - * - * In non-compacted format the offsets are fixed and - * disabled states still occupy space in the memory buffer. - */ - if (!compacted) - size = xfeature_uncompacted_offset(i); - /* - * Add the feature size even for non-compacted format - * to make the end result correct - */ - size += xfeature_size(i); - } - return size; + if (topmost <= XFEATURE_SSE) + return sizeof(struct xregs_state); + + if (compacted) + offset = xfeature_get_offset(xfeatures, topmost); + return offset + xstate_sizes[topmost]; } /* @@ -935,6 +839,10 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); + + /* Cache size, offset and flags for initialization */ + setup_xstate_cache(); + err = init_xstate_size(); if (err) goto out_disable; @@ -950,8 +858,6 @@ void __init fpu__init_system_xstate(unsigned int legacy_size) fpu_user_cfg.max_features); setup_init_fpu_buf(); - setup_xstate_comp_offsets(); - setup_supervisor_only_offsets(); /* * Paranoia check whether something in the setup modified the @@ -1006,13 +912,19 @@ void fpu__resume_cpu(void) */ static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) { - if (!xfeature_enabled(xfeature_nr)) { - WARN_ON_FPU(1); + u64 xcomp_bv = xsave->header.xcomp_bv; + + if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) return NULL; + + if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { + if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr)))) + return NULL; } - return (void *)xsave + xstate_comp_offsets[xfeature_nr]; + return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr); } + /* * Given the xsave area and a state inside, this function returns the * address of the state. @@ -1043,8 +955,9 @@ void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) * We should not ever be requesting features that we * have not enabled. */ - WARN_ONCE(!(fpu_kernel_cfg.max_features & BIT_ULL(xfeature_nr)), - "get of unsupported state"); + if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) + return NULL; + /* * This assumes the last 'xsave*' instruction to * have requested that 'xfeature_nr' be saved. @@ -1625,6 +1538,9 @@ static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) /* Calculate the resulting kernel state size */ mask = permitted | requested; + /* Take supervisor states into account on the host */ + if (!guest) + mask |= xfeatures_mask_supervisor(); ksize = xstate_calculate_size(mask, compacted); /* Calculate the resulting user state size */ @@ -1639,7 +1555,7 @@ static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) perm = guest ? &fpu->guest_perm : &fpu->perm; /* Pairs with the READ_ONCE() in xstate_get_group_perm() */ - WRITE_ONCE(perm->__state_perm, requested); + WRITE_ONCE(perm->__state_perm, mask); /* Protected by sighand lock */ perm->__state_size = ksize; perm->__user_state_size = usize; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 79e0b8d63ffa..a22deb58f86d 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -517,7 +517,7 @@ static void __send_ipi_mask(const struct cpumask *mask, int vector) } else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) { ipi_bitmap <<= min - apic_id; min = apic_id; - } else if (apic_id < min + KVM_IPI_CLUSTER_SIZE) { + } else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) { max = apic_id < max ? max : apic_id; } else { ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap, diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index a00cd97b2623..b24ca7f4ed7c 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -735,6 +735,7 @@ static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array, if (function > READ_ONCE(max_cpuid_80000000)) return entry; } + break; default: break; diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index f9c00c89829b..89b11e7dca8a 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3540,8 +3540,10 @@ static int em_rdpid(struct x86_emulate_ctxt *ctxt) { u64 tsc_aux = 0; - if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux)) + if (!ctxt->ops->guest_has_rdpid(ctxt)) return emulate_ud(ctxt); + + ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux); ctxt->dst.val = tsc_aux; return X86EMUL_CONTINUE; } @@ -3642,7 +3644,7 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt) msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX) | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32); - r = ctxt->ops->set_msr(ctxt, msr_index, msr_data); + r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data); if (r == X86EMUL_IO_NEEDED) return r; @@ -3659,7 +3661,7 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt) u64 msr_data; int r; - r = ctxt->ops->get_msr(ctxt, msr_index, &msr_data); + r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data); if (r == X86EMUL_IO_NEEDED) return r; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index a32f54ab84a2..123b677111c5 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -122,9 +122,13 @@ static void synic_update_vector(struct kvm_vcpu_hv_synic *synic, else hv->synic_auto_eoi_used--; - __kvm_request_apicv_update(vcpu->kvm, - !hv->synic_auto_eoi_used, - APICV_INHIBIT_REASON_HYPERV); + /* + * Inhibit APICv if any vCPU is using SynIC's AutoEOI, which relies on + * the hypervisor to manually inject IRQs. + */ + __kvm_set_or_clear_apicv_inhibit(vcpu->kvm, + APICV_INHIBIT_REASON_HYPERV, + !!hv->synic_auto_eoi_used); up_write(&vcpu->kvm->arch.apicv_update_lock); } @@ -239,7 +243,7 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic); int ret; - if (!synic->active && !host) + if (!synic->active && (!host || data)) return 1; trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host); @@ -285,6 +289,9 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, case HV_X64_MSR_EOM: { int i; + if (!synic->active) + break; + for (i = 0; i < ARRAY_SIZE(synic->sint); i++) kvm_hv_notify_acked_sint(vcpu, i); break; @@ -449,6 +456,9 @@ static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint) struct kvm_lapic_irq irq; int ret, vector; + if (KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) + return -EINVAL; + if (sint >= ARRAY_SIZE(synic->sint)) return -EINVAL; @@ -661,7 +671,7 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); - if (!synic->active && !host) + if (!synic->active && (!host || config)) return 1; if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && @@ -690,7 +700,7 @@ static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count, struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); - if (!synic->active && !host) + if (!synic->active && (!host || count)) return 1; trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id, diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index 0b65a764ed3a..1c83076091af 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -305,15 +305,13 @@ void kvm_pit_set_reinject(struct kvm_pit *pit, bool reinject) * So, deactivate APICv when PIT is in reinject mode. */ if (reinject) { - kvm_request_apicv_update(kvm, false, - APICV_INHIBIT_REASON_PIT_REINJ); + kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PIT_REINJ); /* The initial state is preserved while ps->reinject == 0. */ kvm_pit_reset_reinject(pit); kvm_register_irq_ack_notifier(kvm, &ps->irq_ack_notifier); kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier); } else { - kvm_request_apicv_update(kvm, true, - APICV_INHIBIT_REASON_PIT_REINJ); + kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_PIT_REINJ); kvm_unregister_irq_ack_notifier(kvm, &ps->irq_ack_notifier); kvm_unregister_irq_mask_notifier(kvm, 0, &pit->mask_notifier); } diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 840ddb4a9302..8dff25d267b7 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -210,6 +210,8 @@ struct x86_emulate_ops { int (*set_dr)(struct x86_emulate_ctxt *ctxt, int dr, ulong value); u64 (*get_smbase)(struct x86_emulate_ctxt *ctxt); void (*set_smbase)(struct x86_emulate_ctxt *ctxt, u64 smbase); + int (*set_msr_with_filter)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 data); + int (*get_msr_with_filter)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 *pdata); int (*set_msr)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 data); int (*get_msr)(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 *pdata); int (*check_pmc)(struct x86_emulate_ctxt *ctxt, u32 pmc); @@ -226,6 +228,7 @@ struct x86_emulate_ops { bool (*guest_has_long_mode)(struct x86_emulate_ctxt *ctxt); bool (*guest_has_movbe)(struct x86_emulate_ctxt *ctxt); bool (*guest_has_fxsr)(struct x86_emulate_ctxt *ctxt); + bool (*guest_has_rdpid)(struct x86_emulate_ctxt *ctxt); void (*set_nmi_mask)(struct x86_emulate_ctxt *ctxt, bool masked); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 80a2020c4db4..66b0eb0bda94 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1024,6 +1024,10 @@ bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, *r = -1; if (irq->shorthand == APIC_DEST_SELF) { + if (KVM_BUG_ON(!src, kvm)) { + *r = 0; + return true; + } *r = kvm_apic_set_irq(src->vcpu, irq, dest_map); return true; } diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index bf8dbc4bb12a..e6cae6f22683 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -214,27 +214,27 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, */ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned pte_access, unsigned pte_pkey, - unsigned pfec) + u64 access) { - int cpl = static_call(kvm_x86_get_cpl)(vcpu); + /* strip nested paging fault error codes */ + unsigned int pfec = access; unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu); /* - * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1. + * For explicit supervisor accesses, SMAP is disabled if EFLAGS.AC = 1. + * For implicit supervisor accesses, SMAP cannot be overridden. * - * If CPL = 3, SMAP applies to all supervisor-mode data accesses - * (these are implicit supervisor accesses) regardless of the value - * of EFLAGS.AC. + * SMAP works on supervisor accesses only, and not_smap can + * be set or not set when user access with neither has any bearing + * on the result. * - * This computes (cpl < 3) && (rflags & X86_EFLAGS_AC), leaving - * the result in X86_EFLAGS_AC. We then insert it in place of - * the PFERR_RSVD_MASK bit; this bit will always be zero in pfec, - * but it will be one in index if SMAP checks are being overridden. - * It is important to keep this branchless. + * We put the SMAP checking bit in place of the PFERR_RSVD_MASK bit; + * this bit will always be zero in pfec, but it will be one in index + * if SMAP checks are being disabled. */ - unsigned long smap = (cpl - 3) & (rflags & X86_EFLAGS_AC); - int index = (pfec >> 1) + - (smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1)); + u64 implicit_access = access & PFERR_IMPLICIT_ACCESS; + bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC; + int index = (pfec + (not_smap << PFERR_RSVD_BIT)) >> 1; bool fault = (mmu->permissions[index] >> pte_access) & 1; u32 errcode = PFERR_PRESENT_MASK; @@ -317,12 +317,12 @@ static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count) atomic64_add(count, &kvm->stat.pages[level - 1]); } -gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, +gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access, struct x86_exception *exception); static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gpa_t gpa, u32 access, + gpa_t gpa, u64 access, struct x86_exception *exception) { if (mmu != &vcpu->arch.nested_mmu) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 51671cb34fb6..8f19ea752704 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2696,8 +2696,8 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, if (*sptep == spte) { ret = RET_PF_SPURIOUS; } else { - trace_kvm_mmu_set_spte(level, gfn, sptep); flush |= mmu_spte_update(sptep, spte); + trace_kvm_mmu_set_spte(level, gfn, sptep); } if (wrprot) { @@ -3703,7 +3703,7 @@ void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu) } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gpa_t vaddr, u32 access, + gpa_t vaddr, u64 access, struct x86_exception *exception) { if (exception) @@ -4591,11 +4591,11 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept) * - X86_CR4_SMAP is set in CR4 * - A user page is accessed * - The access is not a fetch - * - Page fault in kernel mode - * - if CPL = 3 or X86_EFLAGS_AC is clear + * - The access is supervisor mode + * - If implicit supervisor access or X86_EFLAGS_AC is clear * - * Here, we cover the first three conditions. - * The fourth is computed dynamically in permission_fault(); + * Here, we cover the first four conditions. + * The fifth is computed dynamically in permission_fault(); * PFERR_RSVD_MASK bit will be set in PFEC if the access is * *not* subject to SMAP restrictions. */ @@ -5768,17 +5768,24 @@ static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, kvm_mmu_zap_all_fast(kvm); } -void kvm_mmu_init_vm(struct kvm *kvm) +int kvm_mmu_init_vm(struct kvm *kvm) { struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; + int r; + INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); + INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages); + INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages); spin_lock_init(&kvm->arch.mmu_unsync_pages_lock); - kvm_mmu_init_tdp_mmu(kvm); + r = kvm_mmu_init_tdp_mmu(kvm); + if (r < 0) + return r; node->track_write = kvm_mmu_pte_write; node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; kvm_page_track_register_notifier(kvm, node); + return 0; } void kvm_mmu_uninit_vm(struct kvm *kvm) @@ -5842,8 +5849,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (is_tdp_mmu_enabled(kvm)) { for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) - flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start, - gfn_end, flush); + flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start, + gfn_end, true, flush); } if (flush) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 252c77805eb9..01fee5f67ac3 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -34,9 +34,8 @@ #define PT_HAVE_ACCESSED_DIRTY(mmu) true #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL - #define CMPXCHG cmpxchg + #define CMPXCHG "cmpxchgq" #else - #define CMPXCHG cmpxchg64 #define PT_MAX_FULL_LEVELS 2 #endif #elif PTTYPE == 32 @@ -52,7 +51,7 @@ #define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT #define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT #define PT_HAVE_ACCESSED_DIRTY(mmu) true - #define CMPXCHG cmpxchg + #define CMPXCHG "cmpxchgl" #elif PTTYPE == PTTYPE_EPT #define pt_element_t u64 #define guest_walker guest_walkerEPT @@ -65,7 +64,9 @@ #define PT_GUEST_DIRTY_SHIFT 9 #define PT_GUEST_ACCESSED_SHIFT 8 #define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad) - #define CMPXCHG cmpxchg64 + #ifdef CONFIG_X86_64 + #define CMPXCHG "cmpxchgq" + #endif #define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL #else #error Invalid PTTYPE value @@ -147,43 +148,36 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, pt_element_t __user *ptep_user, unsigned index, pt_element_t orig_pte, pt_element_t new_pte) { - int npages; - pt_element_t ret; - pt_element_t *table; - struct page *page; - - npages = get_user_pages_fast((unsigned long)ptep_user, 1, FOLL_WRITE, &page); - if (likely(npages == 1)) { - table = kmap_atomic(page); - ret = CMPXCHG(&table[index], orig_pte, new_pte); - kunmap_atomic(table); - - kvm_release_page_dirty(page); - } else { - struct vm_area_struct *vma; - unsigned long vaddr = (unsigned long)ptep_user & PAGE_MASK; - unsigned long pfn; - unsigned long paddr; - - mmap_read_lock(current->mm); - vma = find_vma_intersection(current->mm, vaddr, vaddr + PAGE_SIZE); - if (!vma || !(vma->vm_flags & VM_PFNMAP)) { - mmap_read_unlock(current->mm); - return -EFAULT; - } - pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - paddr = pfn << PAGE_SHIFT; - table = memremap(paddr, PAGE_SIZE, MEMREMAP_WB); - if (!table) { - mmap_read_unlock(current->mm); - return -EFAULT; - } - ret = CMPXCHG(&table[index], orig_pte, new_pte); - memunmap(table); - mmap_read_unlock(current->mm); - } + signed char r; - return (ret != orig_pte); + if (!user_access_begin(ptep_user, sizeof(pt_element_t))) + return -EFAULT; + +#ifdef CMPXCHG + asm volatile("1:" LOCK_PREFIX CMPXCHG " %[new], %[ptr]\n" + "setnz %b[r]\n" + "2:" + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %k[r]) + : [ptr] "+m" (*ptep_user), + [old] "+a" (orig_pte), + [r] "=q" (r) + : [new] "r" (new_pte) + : "memory"); +#else + asm volatile("1:" LOCK_PREFIX "cmpxchg8b %[ptr]\n" + "setnz %b[r]\n" + "2:" + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %k[r]) + : [ptr] "+m" (*ptep_user), + [old] "+A" (orig_pte), + [r] "=q" (r) + : [new_lo] "b" ((u32)new_pte), + [new_hi] "c" ((u32)(new_pte >> 32)) + : "memory"); +#endif + + user_access_end(); + return r; } static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, @@ -339,7 +333,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu, */ static int FNAME(walk_addr_generic)(struct guest_walker *walker, struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gpa_t addr, u32 access) + gpa_t addr, u64 access) { int ret; pt_element_t pte; @@ -347,7 +341,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, gfn_t table_gfn; u64 pt_access, pte_access; unsigned index, accessed_dirty, pte_pkey; - unsigned nested_access; + u64 nested_access; gpa_t pte_gpa; bool have_ad; int offset; @@ -540,7 +534,7 @@ error: } static int FNAME(walk_addr)(struct guest_walker *walker, - struct kvm_vcpu *vcpu, gpa_t addr, u32 access) + struct kvm_vcpu *vcpu, gpa_t addr, u64 access) { return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr, access); @@ -988,7 +982,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - gpa_t addr, u32 access, + gpa_t addr, u64 access, struct x86_exception *exception) { struct guest_walker walker; diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index e7e7876251b3..d71d177ae6b8 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -14,21 +14,24 @@ static bool __read_mostly tdp_mmu_enabled = true; module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644); /* Initializes the TDP MMU for the VM, if enabled. */ -bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) +int kvm_mmu_init_tdp_mmu(struct kvm *kvm) { + struct workqueue_struct *wq; + if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled)) - return false; + return 0; + + wq = alloc_workqueue("kvm", WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 0); + if (!wq) + return -ENOMEM; /* This should not be changed for the lifetime of the VM. */ kvm->arch.tdp_mmu_enabled = true; - INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages); - kvm->arch.tdp_mmu_zap_wq = - alloc_workqueue("kvm", WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 0); - - return true; + kvm->arch.tdp_mmu_zap_wq = wq; + return 1; } /* Arbitrarily returns true so that this may be used in if statements. */ @@ -906,10 +909,8 @@ bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp) } /* - * Tears down the mappings for the range of gfns, [start, end), and frees the - * non-root pages mapping GFNs strictly within that range. Returns true if - * SPTEs have been cleared and a TLB flush is needed before releasing the - * MMU lock. + * Zap leafs SPTEs for the range of gfns, [start, end). Returns true if SPTEs + * have been cleared and a TLB flush is needed before releasing the MMU lock. * * If can_yield is true, will release the MMU lock and reschedule if the * scheduler needs the CPU or there is contention on the MMU lock. If this @@ -917,42 +918,25 @@ bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp) * the caller must ensure it does not supply too large a GFN range, or the * operation can cause a soft lockup. */ -static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t start, gfn_t end, bool can_yield, bool flush) +static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root, + gfn_t start, gfn_t end, bool can_yield, bool flush) { - bool zap_all = (start == 0 && end >= tdp_mmu_max_gfn_host()); struct tdp_iter iter; - /* - * No need to try to step down in the iterator when zapping all SPTEs, - * zapping the top-level non-leaf SPTEs will recurse on their children. - */ - int min_level = zap_all ? root->role.level : PG_LEVEL_4K; - end = min(end, tdp_mmu_max_gfn_host()); lockdep_assert_held_write(&kvm->mmu_lock); rcu_read_lock(); - for_each_tdp_pte_min_level(iter, root, min_level, start, end) { + for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) { if (can_yield && tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) { flush = false; continue; } - if (!is_shadow_present_pte(iter.old_spte)) - continue; - - /* - * If this is a non-last-level SPTE that covers a larger range - * than should be zapped, continue, and zap the mappings at a - * lower level, except when zapping all SPTEs. - */ - if (!zap_all && - (iter.gfn < start || - iter.gfn + KVM_PAGES_PER_HPAGE(iter.level) > end) && + if (!is_shadow_present_pte(iter.old_spte) || !is_last_spte(iter.old_spte, iter.level)) continue; @@ -960,17 +944,13 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, flush = true; } - /* - * Need to flush before releasing RCU. TODO: do it only if intermediate - * page tables were zapped; there is no need to flush under RCU protection - * if no 'struct kvm_mmu_page' is freed. - */ - if (flush) - kvm_flush_remote_tlbs_with_address(kvm, start, end - start); - rcu_read_unlock(); - return false; + /* + * Because this flow zaps _only_ leaf SPTEs, the caller doesn't need + * to provide RCU protection as no 'struct kvm_mmu_page' will be freed. + */ + return flush; } /* @@ -979,13 +959,13 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, * SPTEs have been cleared and a TLB flush is needed before releasing the * MMU lock. */ -bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, gfn_t start, - gfn_t end, bool can_yield, bool flush) +bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, + bool can_yield, bool flush) { struct kvm_mmu_page *root; for_each_tdp_mmu_root_yield_safe(kvm, root, as_id) - flush = zap_gfn_range(kvm, root, start, end, can_yield, flush); + flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush); return flush; } @@ -1233,8 +1213,8 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range, bool flush) { - return __kvm_tdp_mmu_zap_gfn_range(kvm, range->slot->as_id, range->start, - range->end, range->may_block, flush); + return kvm_tdp_mmu_zap_leafs(kvm, range->slot->as_id, range->start, + range->end, range->may_block, flush); } typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter, diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index 5e5ef2576c81..c163f7cc23ca 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -15,14 +15,8 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root) void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root, bool shared); -bool __kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, gfn_t start, +bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, bool can_yield, bool flush); -static inline bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, - gfn_t start, gfn_t end, bool flush) -{ - return __kvm_tdp_mmu_zap_gfn_range(kvm, as_id, start, end, true, flush); -} - bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp); void kvm_tdp_mmu_zap_all(struct kvm *kvm); void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm); @@ -72,7 +66,7 @@ u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, u64 addr, u64 *spte); #ifdef CONFIG_X86_64 -bool kvm_mmu_init_tdp_mmu(struct kvm *kvm); +int kvm_mmu_init_tdp_mmu(struct kvm *kvm); void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm); static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; } @@ -93,7 +87,7 @@ static inline bool is_tdp_mmu(struct kvm_mmu *mmu) return sp && is_tdp_mmu_page(sp) && sp->root_count; } #else -static inline bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) { return false; } +static inline int kvm_mmu_init_tdp_mmu(struct kvm *kvm) { return 0; } static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {} static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; } static inline bool is_tdp_mmu(struct kvm_mmu *mmu) { return false; } diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index b1a02993782b..eca39f56c231 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -96,8 +96,7 @@ static void kvm_perf_overflow(struct perf_event *perf_event, static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, bool exclude_user, - bool exclude_kernel, bool intr, - bool in_tx, bool in_tx_cp) + bool exclude_kernel, bool intr) { struct perf_event *event; struct perf_event_attr attr = { @@ -116,16 +115,14 @@ static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, attr.sample_period = get_sample_period(pmc, pmc->counter); - if (in_tx) - attr.config |= HSW_IN_TX; - if (in_tx_cp) { + if ((attr.config & HSW_IN_TX_CHECKPOINTED) && + guest_cpuid_is_intel(pmc->vcpu)) { /* * HSW_IN_TX_CHECKPOINTED is not supported with nonzero * period. Just clear the sample period so at least * allocating the counter doesn't fail. */ attr.sample_period = 0; - attr.config |= HSW_IN_TX_CHECKPOINTED; } event = perf_event_create_kernel_counter(&attr, -1, current, @@ -185,6 +182,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) u32 type = PERF_TYPE_RAW; struct kvm *kvm = pmc->vcpu->kvm; struct kvm_pmu_event_filter *filter; + struct kvm_pmu *pmu = vcpu_to_pmu(pmc->vcpu); bool allow_event = true; if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) @@ -221,7 +219,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) } if (type == PERF_TYPE_RAW) - config = eventsel & AMD64_RAW_EVENT_MASK; + config = eventsel & pmu->raw_event_mask; if (pmc->current_config == eventsel && pmc_resume_counter(pmc)) return; @@ -232,9 +230,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) pmc_reprogram_counter(pmc, type, config, !(eventsel & ARCH_PERFMON_EVENTSEL_USR), !(eventsel & ARCH_PERFMON_EVENTSEL_OS), - eventsel & ARCH_PERFMON_EVENTSEL_INT, - (eventsel & HSW_IN_TX), - (eventsel & HSW_IN_TX_CHECKPOINTED)); + eventsel & ARCH_PERFMON_EVENTSEL_INT); } EXPORT_SYMBOL_GPL(reprogram_gp_counter); @@ -270,7 +266,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ - pmi, false, false); + pmi); } EXPORT_SYMBOL_GPL(reprogram_fixed_counter); diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index b37b353ec086..a1cf9c31273b 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -726,7 +726,7 @@ int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, { struct kvm_kernel_irq_routing_entry *e; struct kvm_irq_routing_table *irq_rt; - int idx, ret = -EINVAL; + int idx, ret = 0; if (!kvm_arch_has_assigned_device(kvm) || !irq_remapping_cap(IRQ_POSTING_CAP)) @@ -737,7 +737,13 @@ int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, idx = srcu_read_lock(&kvm->irq_srcu); irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); - WARN_ON(guest_irq >= irq_rt->nr_rt_entries); + + if (guest_irq >= irq_rt->nr_rt_entries || + hlist_empty(&irq_rt->map[guest_irq])) { + pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", + guest_irq, irq_rt->nr_rt_entries); + goto out; + } hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { struct vcpu_data vcpu_info; @@ -822,7 +828,7 @@ out: return ret; } -bool avic_check_apicv_inhibit_reasons(ulong bit) +bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) { ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | BIT(APICV_INHIBIT_REASON_ABSENT) | @@ -833,7 +839,7 @@ bool avic_check_apicv_inhibit_reasons(ulong bit) BIT(APICV_INHIBIT_REASON_X2APIC) | BIT(APICV_INHIBIT_REASON_BLOCKIRQ); - return supported & BIT(bit); + return supported & BIT(reason); } diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index d4de52409335..24eb935b6f85 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -262,12 +262,10 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) /* MSR_EVNTSELn */ pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_EVNTSEL); if (pmc) { - if (data == pmc->eventsel) - return 0; - if (!(data & pmu->reserved_bits)) { + data &= ~pmu->reserved_bits; + if (data != pmc->eventsel) reprogram_gp_counter(pmc, data); - return 0; - } + return 0; } return 1; @@ -284,6 +282,7 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu) pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << 48) - 1; pmu->reserved_bits = 0xfffffff000280000ull; + pmu->raw_event_mask = AMD64_RAW_EVENT_MASK; pmu->version = 1; /* not applicable to AMD; but clean them to prevent any fall out */ pmu->counter_bitmask[KVM_PMC_FIXED] = 0; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0884c3414a1b..bd4c64b362d2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -62,20 +62,8 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); #define SEG_TYPE_LDT 2 #define SEG_TYPE_BUSY_TSS16 3 -#define SVM_FEATURE_LBRV (1 << 1) -#define SVM_FEATURE_SVML (1 << 2) -#define SVM_FEATURE_TSC_RATE (1 << 4) -#define SVM_FEATURE_VMCB_CLEAN (1 << 5) -#define SVM_FEATURE_FLUSH_ASID (1 << 6) -#define SVM_FEATURE_DECODE_ASSIST (1 << 7) -#define SVM_FEATURE_PAUSE_FILTER (1 << 10) - #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) -#define TSC_RATIO_RSVD 0xffffff0000000000ULL -#define TSC_RATIO_MIN 0x0000000000000001ULL -#define TSC_RATIO_MAX 0x000000ffffffffffULL - static bool erratum_383_found __read_mostly; u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; @@ -87,7 +75,6 @@ u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; static uint64_t osvw_len = 4, osvw_status; static DEFINE_PER_CPU(u64, current_tsc_ratio); -#define TSC_RATIO_DEFAULT 0x0100000000ULL static const struct svm_direct_access_msrs { u32 index; /* Index of the MSR */ @@ -480,7 +467,7 @@ static void svm_hardware_disable(void) { /* Make sure we clean up behind us */ if (tsc_scaling) - wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); + wrmsrl(MSR_AMD64_TSC_RATIO, SVM_TSC_RATIO_DEFAULT); cpu_svm_disable(); @@ -526,8 +513,8 @@ static int svm_hardware_enable(void) * Set the default value, even if we don't use TSC scaling * to avoid having stale value in the msr */ - wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT); - __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT); + wrmsrl(MSR_AMD64_TSC_RATIO, SVM_TSC_RATIO_DEFAULT); + __this_cpu_write(current_tsc_ratio, SVM_TSC_RATIO_DEFAULT); } @@ -2723,7 +2710,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) break; } - if (data & TSC_RATIO_RSVD) + if (data & SVM_TSC_RATIO_RSVD) return 1; svm->tsc_ratio_msr = data; @@ -2918,7 +2905,7 @@ static int interrupt_window_interception(struct kvm_vcpu *vcpu) * In this case AVIC was temporarily disabled for * requesting the IRQ window and we have to re-enable it. */ - kvm_request_apicv_update(vcpu->kvm, true, APICV_INHIBIT_REASON_IRQWIN); + kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); ++vcpu->stat.irq_window_exits; return 1; @@ -3516,7 +3503,7 @@ static void svm_enable_irq_window(struct kvm_vcpu *vcpu) * via AVIC. In such case, we need to temporarily disable AVIC, * and fallback to injecting IRQ via V_IRQ. */ - kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_IRQWIN); + kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); svm_set_vintr(svm); } } @@ -3948,6 +3935,7 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_cpuid_entry2 *best; + struct kvm *kvm = vcpu->kvm; vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && boot_cpu_has(X86_FEATURE_XSAVE) && @@ -3974,16 +3962,14 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) * is exposed to the guest, disable AVIC. */ if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) - kvm_request_apicv_update(vcpu->kvm, false, - APICV_INHIBIT_REASON_X2APIC); + kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_X2APIC); /* * Currently, AVIC does not work with nested virtualization. * So, we disable AVIC when cpuid for SVM is set in the L1 guest. */ if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) - kvm_request_apicv_update(vcpu->kvm, false, - APICV_INHIBIT_REASON_NESTED); + kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_NESTED); } init_vmcb_after_set_cpuid(vcpu); } @@ -4766,10 +4752,10 @@ static __init int svm_hardware_setup(void) } else { pr_info("TSC scaling supported\n"); kvm_has_tsc_control = true; - kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX; - kvm_tsc_scaling_ratio_frac_bits = 32; } } + kvm_max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX; + kvm_tsc_scaling_ratio_frac_bits = 32; tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e37bb3508cfa..f77a7d2d39dd 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -22,6 +22,8 @@ #include <asm/svm.h> #include <asm/sev-common.h> +#include "kvm_cache_regs.h" + #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT) #define IOPM_SIZE PAGE_SIZE * 3 @@ -569,17 +571,6 @@ extern struct kvm_x86_nested_ops svm_nested_ops; /* avic.c */ -#define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) -#define AVIC_LOGICAL_ID_ENTRY_VALID_BIT 31 -#define AVIC_LOGICAL_ID_ENTRY_VALID_MASK (1 << 31) - -#define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK GENMASK_ULL(11, 0) -#define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK (0xFFFFFFFFFFULL << 12) -#define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK (1ULL << 62) -#define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK (1ULL << 63) - -#define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL - int avic_ga_log_notifier(u32 ga_tag); void avic_vm_destroy(struct kvm *kvm); int avic_vm_init(struct kvm *kvm); @@ -592,7 +583,7 @@ void __avic_vcpu_put(struct kvm_vcpu *vcpu); void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu); void avic_set_virtual_apic_mode(struct kvm_vcpu *vcpu); void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu); -bool avic_check_apicv_inhibit_reasons(ulong bit); +bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason); void avic_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr); void avic_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr); bool avic_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/svm_onhyperv.c b/arch/x86/kvm/svm/svm_onhyperv.c index 98aa981c04ec..8cdc62c74a96 100644 --- a/arch/x86/kvm/svm/svm_onhyperv.c +++ b/arch/x86/kvm/svm/svm_onhyperv.c @@ -4,7 +4,6 @@ */ #include <linux/kvm_host.h> -#include "kvm_cache_regs.h" #include <asm/mshyperv.h> diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 193f5ba930d1..e3a24b8f04be 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -1339,23 +1339,25 @@ TRACE_EVENT(kvm_hv_stimer_cleanup, __entry->vcpu_id, __entry->timer_index) ); -TRACE_EVENT(kvm_apicv_update_request, - TP_PROTO(bool activate, unsigned long bit), - TP_ARGS(activate, bit), +TRACE_EVENT(kvm_apicv_inhibit_changed, + TP_PROTO(int reason, bool set, unsigned long inhibits), + TP_ARGS(reason, set, inhibits), TP_STRUCT__entry( - __field(bool, activate) - __field(unsigned long, bit) + __field(int, reason) + __field(bool, set) + __field(unsigned long, inhibits) ), TP_fast_assign( - __entry->activate = activate; - __entry->bit = bit; + __entry->reason = reason; + __entry->set = set; + __entry->inhibits = inhibits; ), - TP_printk("%s bit=%lu", - __entry->activate ? "activate" : "deactivate", - __entry->bit) + TP_printk("%s reason=%u, inhibits=0x%lx", + __entry->set ? "set" : "cleared", + __entry->reason, __entry->inhibits) ); TRACE_EVENT(kvm_apicv_accept_irq, diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 0684e519181b..bc3f8512bb64 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -389,6 +389,7 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) struct kvm_pmc *pmc; u32 msr = msr_info->index; u64 data = msr_info->data; + u64 reserved_bits; switch (msr) { case MSR_CORE_PERF_FIXED_CTR_CTRL: @@ -443,7 +444,11 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { if (data == pmc->eventsel) return 0; - if (!(data & pmu->reserved_bits)) { + reserved_bits = pmu->reserved_bits; + if ((pmc->idx == 2) && + (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED)) + reserved_bits ^= HSW_IN_TX_CHECKPOINTED; + if (!(data & reserved_bits)) { reprogram_gp_counter(pmc, data); return 0; } @@ -485,6 +490,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->counter_bitmask[KVM_PMC_FIXED] = 0; pmu->version = 0; pmu->reserved_bits = 0xffffffff00200000ull; + pmu->raw_event_mask = X86_RAW_EVENT_MASK; entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); if (!entry || !vcpu->kvm->arch.enable_pmu) @@ -533,8 +539,10 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) entry = kvm_find_cpuid_entry(vcpu, 7, 0); if (entry && (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && - (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) - pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED; + (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) { + pmu->reserved_bits ^= HSW_IN_TX; + pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED); + } bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e8963f5af618..04d170c4b61e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2866,21 +2866,17 @@ static void enter_rmode(struct kvm_vcpu *vcpu) int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) { struct vcpu_vmx *vmx = to_vmx(vcpu); - struct vmx_uret_msr *msr = vmx_find_uret_msr(vmx, MSR_EFER); /* Nothing to do if hardware doesn't support EFER. */ - if (!msr) + if (!vmx_find_uret_msr(vmx, MSR_EFER)) return 0; vcpu->arch.efer = efer; - if (efer & EFER_LMA) { - vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); - msr->data = efer; - } else { - vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); + if (efer & EFER_LMA) + vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); + else + vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); - msr->data = efer & ~EFER_LME; - } vmx_setup_uret_msrs(vmx); return 0; } @@ -2906,7 +2902,6 @@ static void enter_lmode(struct kvm_vcpu *vcpu) static void exit_lmode(struct kvm_vcpu *vcpu) { - vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE); vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); } @@ -7705,14 +7700,14 @@ static void vmx_hardware_unsetup(void) free_kvm_area(); } -static bool vmx_check_apicv_inhibit_reasons(ulong bit) +static bool vmx_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason) { ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | BIT(APICV_INHIBIT_REASON_ABSENT) | BIT(APICV_INHIBIT_REASON_HYPERV) | BIT(APICV_INHIBIT_REASON_BLOCKIRQ); - return supported & BIT(bit); + return supported & BIT(reason); } static struct kvm_x86_ops vmx_x86_ops __initdata = { @@ -7980,12 +7975,11 @@ static __init int hardware_setup(void) if (!enable_apicv) vmx_x86_ops.sync_pir_to_irr = NULL; - if (cpu_has_vmx_tsc_scaling()) { + if (cpu_has_vmx_tsc_scaling()) kvm_has_tsc_control = true; - kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; - kvm_tsc_scaling_ratio_frac_bits = 48; - } + kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; + kvm_tsc_scaling_ratio_frac_bits = 48; kvm_has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 02cf0a7e1d14..0c0ca599a353 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1748,9 +1748,6 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, { struct msr_data msr; - if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) - return KVM_MSR_RET_FILTERED; - switch (index) { case MSR_FS_BASE: case MSR_GS_BASE: @@ -1832,9 +1829,6 @@ int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, struct msr_data msr; int ret; - if (!host_initiated && !kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) - return KVM_MSR_RET_FILTERED; - switch (index) { case MSR_TSC_AUX: if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) @@ -1871,6 +1865,20 @@ static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, return ret; } +static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data) +{ + if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) + return KVM_MSR_RET_FILTERED; + return kvm_get_msr_ignored_check(vcpu, index, data, false); +} + +static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data) +{ + if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) + return KVM_MSR_RET_FILTERED; + return kvm_set_msr_ignored_check(vcpu, index, data, false); +} + int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data) { return kvm_get_msr_ignored_check(vcpu, index, data, false); @@ -1953,7 +1961,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) u64 data; int r; - r = kvm_get_msr(vcpu, ecx, &data); + r = kvm_get_msr_with_filter(vcpu, ecx, &data); if (!r) { trace_kvm_msr_read(ecx, data); @@ -1978,7 +1986,7 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) u64 data = kvm_read_edx_eax(vcpu); int r; - r = kvm_set_msr(vcpu, ecx, data); + r = kvm_set_msr_with_filter(vcpu, ecx, data); if (!r) { trace_kvm_msr_write(ecx, data); @@ -5938,7 +5946,7 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, smp_wmb(); kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT; kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; - kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT); + kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); r = 0; split_irqchip_unlock: mutex_unlock(&kvm->lock); @@ -6335,7 +6343,7 @@ set_identity_unlock: /* Write kvm->irq_routing before enabling irqchip_in_kernel. */ smp_wmb(); kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL; - kvm_request_apicv_update(kvm, true, APICV_INHIBIT_REASON_ABSENT); + kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); create_irqchip_unlock: mutex_unlock(&kvm->lock); break; @@ -6726,7 +6734,7 @@ void kvm_get_segment(struct kvm_vcpu *vcpu, static_call(kvm_x86_get_segment)(vcpu, var, seg); } -gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, +gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access, struct x86_exception *exception) { struct kvm_mmu *mmu = vcpu->arch.mmu; @@ -6746,7 +6754,7 @@ gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; + u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read); @@ -6756,7 +6764,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read); { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; + u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_FETCH_MASK; return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } @@ -6766,7 +6774,7 @@ gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; + u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; access |= PFERR_WRITE_MASK; return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); } @@ -6782,7 +6790,7 @@ gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, } static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 access, + struct kvm_vcpu *vcpu, u64 access, struct x86_exception *exception) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; @@ -6819,7 +6827,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; + u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; unsigned offset; int ret; @@ -6844,7 +6852,7 @@ int kvm_read_guest_virt(struct kvm_vcpu *vcpu, gva_t addr, void *val, unsigned int bytes, struct x86_exception *exception) { - u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; + u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; /* * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED @@ -6863,9 +6871,11 @@ static int emulator_read_std(struct x86_emulate_ctxt *ctxt, struct x86_exception *exception, bool system) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - u32 access = 0; + u64 access = 0; - if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3) + if (system) + access |= PFERR_IMPLICIT_ACCESS; + else if (static_call(kvm_x86_get_cpl)(vcpu) == 3) access |= PFERR_USER_MASK; return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); @@ -6881,7 +6891,7 @@ static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt, } static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, - struct kvm_vcpu *vcpu, u32 access, + struct kvm_vcpu *vcpu, u64 access, struct x86_exception *exception) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; @@ -6915,9 +6925,11 @@ static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *v bool system) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - u32 access = PFERR_WRITE_MASK; + u64 access = PFERR_WRITE_MASK; - if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3) + if (system) + access |= PFERR_IMPLICIT_ACCESS; + else if (static_call(kvm_x86_get_cpl)(vcpu) == 3) access |= PFERR_USER_MASK; return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, @@ -6984,7 +6996,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, bool write) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - u32 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) + u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) | (write ? PFERR_WRITE_MASK : 0); /* @@ -7627,13 +7639,13 @@ static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, return; } -static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, - u32 msr_index, u64 *pdata) +static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, + u32 msr_index, u64 *pdata) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); int r; - r = kvm_get_msr(vcpu, msr_index, pdata); + r = kvm_get_msr_with_filter(vcpu, msr_index, pdata); if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, complete_emulated_rdmsr, r)) { @@ -7644,13 +7656,13 @@ static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, return r; } -static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, - u32 msr_index, u64 data) +static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, + u32 msr_index, u64 data) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); int r; - r = kvm_set_msr(vcpu, msr_index, data); + r = kvm_set_msr_with_filter(vcpu, msr_index, data); if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, complete_emulated_msr_access, r)) { @@ -7661,6 +7673,18 @@ static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, return r; } +static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, + u32 msr_index, u64 *pdata) +{ + return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); +} + +static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, + u32 msr_index, u64 data) +{ + return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data); +} + static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); @@ -7724,6 +7748,11 @@ static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt) return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR); } +static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt) +{ + return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID); +} + static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) { return kvm_register_read_raw(emul_to_vcpu(ctxt), reg); @@ -7794,6 +7823,8 @@ static const struct x86_emulate_ops emulate_ops = { .set_dr = emulator_set_dr, .get_smbase = emulator_get_smbase, .set_smbase = emulator_set_smbase, + .set_msr_with_filter = emulator_set_msr_with_filter, + .get_msr_with_filter = emulator_get_msr_with_filter, .set_msr = emulator_set_msr, .get_msr = emulator_get_msr, .check_pmc = emulator_check_pmc, @@ -7806,6 +7837,7 @@ static const struct x86_emulate_ops emulate_ops = { .guest_has_long_mode = emulator_guest_has_long_mode, .guest_has_movbe = emulator_guest_has_movbe, .guest_has_fxsr = emulator_guest_has_fxsr, + .guest_has_rdpid = emulator_guest_has_rdpid, .set_nmi_mask = emulator_set_nmi_mask, .get_hflags = emulator_get_hflags, .exiting_smm = emulator_exiting_smm, @@ -9058,15 +9090,29 @@ bool kvm_apicv_activated(struct kvm *kvm) } EXPORT_SYMBOL_GPL(kvm_apicv_activated); + +static void set_or_clear_apicv_inhibit(unsigned long *inhibits, + enum kvm_apicv_inhibit reason, bool set) +{ + if (set) + __set_bit(reason, inhibits); + else + __clear_bit(reason, inhibits); + + trace_kvm_apicv_inhibit_changed(reason, set, *inhibits); +} + static void kvm_apicv_init(struct kvm *kvm) { + unsigned long *inhibits = &kvm->arch.apicv_inhibit_reasons; + init_rwsem(&kvm->arch.apicv_update_lock); - set_bit(APICV_INHIBIT_REASON_ABSENT, - &kvm->arch.apicv_inhibit_reasons); + set_or_clear_apicv_inhibit(inhibits, APICV_INHIBIT_REASON_ABSENT, true); + if (!enable_apicv) - set_bit(APICV_INHIBIT_REASON_DISABLE, - &kvm->arch.apicv_inhibit_reasons); + set_or_clear_apicv_inhibit(inhibits, + APICV_INHIBIT_REASON_ABSENT, true); } static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) @@ -9740,24 +9786,21 @@ out: } EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv); -void __kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) +void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason, bool set) { unsigned long old, new; lockdep_assert_held_write(&kvm->arch.apicv_update_lock); - if (!static_call(kvm_x86_check_apicv_inhibit_reasons)(bit)) + if (!static_call(kvm_x86_check_apicv_inhibit_reasons)(reason)) return; old = new = kvm->arch.apicv_inhibit_reasons; - if (activate) - __clear_bit(bit, &new); - else - __set_bit(bit, &new); + set_or_clear_apicv_inhibit(&new, reason, set); if (!!old != !!new) { - trace_kvm_apicv_update_request(activate, bit); /* * Kick all vCPUs before setting apicv_inhibit_reasons to avoid * false positives in the sanity check WARN in svm_vcpu_run(). @@ -9776,20 +9819,22 @@ void __kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); kvm_zap_gfn_range(kvm, gfn, gfn+1); } - } else + } else { kvm->arch.apicv_inhibit_reasons = new; + } } -void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) +void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, + enum kvm_apicv_inhibit reason, bool set) { if (!enable_apicv) return; down_write(&kvm->arch.apicv_update_lock); - __kvm_request_apicv_update(kvm, activate, bit); + __kvm_set_or_clear_apicv_inhibit(kvm, reason, set); up_write(&kvm->arch.apicv_update_lock); } -EXPORT_SYMBOL_GPL(kvm_request_apicv_update); +EXPORT_SYMBOL_GPL(kvm_set_or_clear_apicv_inhibit); static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) { @@ -10937,7 +10982,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) { - bool inhibit = false; + bool set = false; struct kvm_vcpu *vcpu; unsigned long i; @@ -10945,11 +10990,11 @@ static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { - inhibit = true; + set = true; break; } } - __kvm_request_apicv_update(kvm, !inhibit, APICV_INHIBIT_REASON_BLOCKIRQ); + __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set); up_write(&kvm->arch.apicv_update_lock); } @@ -11557,10 +11602,8 @@ int kvm_arch_hardware_setup(void *opaque) u64 max = min(0x7fffffffULL, __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz)); kvm_max_guest_tsc_khz = max; - - kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits; } - + kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits; kvm_init_msr_list(); return 0; } @@ -11629,12 +11672,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) ret = kvm_page_track_init(kvm); if (ret) - return ret; + goto out; + + ret = kvm_mmu_init_vm(kvm); + if (ret) + goto out_page_track; INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list); - INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); - INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages); - INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages); INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); atomic_set(&kvm->arch.noncoherent_dma_count, 0); @@ -11666,10 +11710,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm_apicv_init(kvm); kvm_hv_init_vm(kvm); - kvm_mmu_init_vm(kvm); kvm_xen_init_vm(kvm); return static_call(kvm_x86_vm_init)(kvm); + +out_page_track: + kvm_page_track_cleanup(kvm); +out: + return ret; } int kvm_arch_post_init_vm(struct kvm *kvm) @@ -12593,7 +12641,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; struct x86_exception fault; - u32 access = error_code & + u64 access = error_code & (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK); if (!(error_code & PFERR_PRESENT_MASK) || @@ -12933,7 +12981,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); -EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 4aa0f2b31665..bf6cc25eee76 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -39,8 +39,8 @@ static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn) } do { - ret = kvm_gfn_to_pfn_cache_init(kvm, gpc, NULL, false, true, - gpa, PAGE_SIZE, false); + ret = kvm_gfn_to_pfn_cache_init(kvm, gpc, NULL, KVM_HOST_USES_PFN, + gpa, PAGE_SIZE); if (ret) goto out; @@ -1025,8 +1025,7 @@ static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm break; idx = srcu_read_lock(&kvm->srcu); - rc = kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpc->gpa, - PAGE_SIZE, false); + rc = kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpc->gpa, PAGE_SIZE); srcu_read_unlock(&kvm->srcu, idx); } while(!rc); diff --git a/arch/x86/lib/csum-partial_64.c b/arch/x86/lib/csum-partial_64.c index 1f8a8f895173..50734a23034c 100644 --- a/arch/x86/lib/csum-partial_64.c +++ b/arch/x86/lib/csum-partial_64.c @@ -93,7 +93,6 @@ __wsum csum_partial(const void *buff, int len, __wsum sum) buff += 8; } if (len & 7) { -#ifdef CONFIG_DCACHE_WORD_ACCESS unsigned int shift = (8 - (len & 7)) * 8; unsigned long trail; @@ -103,31 +102,6 @@ __wsum csum_partial(const void *buff, int len, __wsum sum) "adcq $0,%[res]" : [res] "+r" (temp64) : [trail] "r" (trail)); -#else - if (len & 4) { - asm("addq %[val],%[res]\n\t" - "adcq $0,%[res]" - : [res] "+r" (temp64) - : [val] "r" ((u64)*(u32 *)buff) - : "memory"); - buff += 4; - } - if (len & 2) { - asm("addq %[val],%[res]\n\t" - "adcq $0,%[res]" - : [res] "+r" (temp64) - : [val] "r" ((u64)*(u16 *)buff) - : "memory"); - buff += 2; - } - if (len & 1) { - asm("addq %[val],%[res]\n\t" - "adcq $0,%[res]" - : [res] "+r" (temp64) - : [val] "r" ((u64)*(u8 *)buff) - : "memory"); - } -#endif } result = add32_with_carry(temp64 >> 32, temp64 & 0xffffffff); if (unlikely(odd)) { diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c index df50451d94ef..3e2f33fc33de 100644 --- a/arch/x86/lib/iomem.c +++ b/arch/x86/lib/iomem.c @@ -22,7 +22,7 @@ static __always_inline void rep_movs(void *to, const void *from, size_t n) : "memory"); } -void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +static void string_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) { if (unlikely(!n)) return; @@ -38,9 +38,8 @@ void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) } rep_movs(to, (const void *)from, n); } -EXPORT_SYMBOL(memcpy_fromio); -void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) { if (unlikely(!n)) return; @@ -56,14 +55,64 @@ void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) } rep_movs((void *)to, (const void *) from, n); } + +static void unrolled_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +{ + const volatile char __iomem *in = from; + char *out = to; + int i; + + for (i = 0; i < n; ++i) + out[i] = readb(&in[i]); +} + +static void unrolled_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +{ + volatile char __iomem *out = to; + const char *in = from; + int i; + + for (i = 0; i < n; ++i) + writeb(in[i], &out[i]); +} + +static void unrolled_memset_io(volatile void __iomem *a, int b, size_t c) +{ + volatile char __iomem *mem = a; + int i; + + for (i = 0; i < c; ++i) + writeb(b, &mem[i]); +} + +void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +{ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) + unrolled_memcpy_fromio(to, from, n); + else + string_memcpy_fromio(to, from, n); +} +EXPORT_SYMBOL(memcpy_fromio); + +void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +{ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) + unrolled_memcpy_toio(to, from, n); + else + string_memcpy_toio(to, from, n); +} EXPORT_SYMBOL(memcpy_toio); void memset_io(volatile void __iomem *a, int b, size_t c) { - /* - * TODO: memset can mangle the IO patterns quite a bit. - * perhaps it would be better to use a dumb one: - */ - memset((void *)a, b, c); + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { + unrolled_memset_io(a, b, c); + } else { + /* + * TODO: memset can mangle the IO patterns quite a bit. + * perhaps it would be better to use a dumb one: + */ + memset((void *)a, b, c); + } } EXPORT_SYMBOL(memset_io); diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 9f2b251e83c5..3822666fb73d 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -40,7 +40,8 @@ static void msr_save_context(struct saved_context *ctxt) struct saved_msr *end = msr + ctxt->saved_msrs.num; while (msr < end) { - msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q); + if (msr->valid) + rdmsrl(msr->info.msr_no, msr->info.reg.q); msr++; } } @@ -424,8 +425,10 @@ static int msr_build_context(const u32 *msr_id, const int num) } for (i = saved_msrs->num, j = 0; i < total_num; i++, j++) { + u64 dummy; + msr_array[i].info.msr_no = msr_id[j]; - msr_array[i].valid = false; + msr_array[i].valid = !rdmsrl_safe(msr_id[j], &dummy); msr_array[i].info.reg.q = 0; } saved_msrs->num = total_num; @@ -500,10 +503,24 @@ static int pm_cpu_check(const struct x86_cpu_id *c) return ret; } +static void pm_save_spec_msr(void) +{ + u32 spec_msr_id[] = { + MSR_IA32_SPEC_CTRL, + MSR_IA32_TSX_CTRL, + MSR_TSX_FORCE_ABORT, + MSR_IA32_MCU_OPT_CTRL, + MSR_AMD64_LS_CFG, + }; + + msr_build_context(spec_msr_id, ARRAY_SIZE(spec_msr_id)); +} + static int pm_check_save_msr(void) { dmi_check_system(msr_save_dmi_table); pm_cpu_check(msr_save_cpu_table); + pm_save_spec_msr(); return 0; } diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig index ead7e5b3a975..1bcd42c53039 100644 --- a/arch/x86/um/Kconfig +++ b/arch/x86/um/Kconfig @@ -9,6 +9,7 @@ endmenu config UML_X86 def_bool y select ARCH_BINFMT_ELF_EXTRA_PHDRS if X86_32 + select DCACHE_WORD_ACCESS config 64BIT bool "64-bit kernel" if "$(SUBARCH)" = "x86" diff --git a/arch/x86/um/shared/sysdep/syscalls_64.h b/arch/x86/um/shared/sysdep/syscalls_64.h index 48d6cd12f8a5..b6b997225841 100644 --- a/arch/x86/um/shared/sysdep/syscalls_64.h +++ b/arch/x86/um/shared/sysdep/syscalls_64.h @@ -10,13 +10,12 @@ #include <linux/msg.h> #include <linux/shm.h> -typedef long syscall_handler_t(void); +typedef long syscall_handler_t(long, long, long, long, long, long); extern syscall_handler_t *sys_call_table[]; #define EXECUTE_SYSCALL(syscall, regs) \ - (((long (*)(long, long, long, long, long, long)) \ - (*sys_call_table[syscall]))(UPT_SYSCALL_ARG1(®s->regs), \ + (((*sys_call_table[syscall]))(UPT_SYSCALL_ARG1(®s->regs), \ UPT_SYSCALL_ARG2(®s->regs), \ UPT_SYSCALL_ARG3(®s->regs), \ UPT_SYSCALL_ARG4(®s->regs), \ diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c index fe5323f0c42d..27b29ae6c471 100644 --- a/arch/x86/um/syscalls_64.c +++ b/arch/x86/um/syscalls_64.c @@ -12,7 +12,6 @@ #include <asm/prctl.h> /* XXX This should get the constants from libc */ #include <registers.h> #include <os.h> -#include <registers.h> long arch_prctl(struct task_struct *task, int option, unsigned long __user *arg2) diff --git a/arch/xtensa/kernel/syscalls/Makefile b/arch/xtensa/kernel/syscalls/Makefile index 6713c65a25e1..b265e4bc16c2 100644 --- a/arch/xtensa/kernel/syscalls/Makefile +++ b/arch/xtensa/kernel/syscalls/Makefile @@ -2,8 +2,7 @@ kapi := arch/$(SRCARCH)/include/generated/asm uapi := arch/$(SRCARCH)/include/generated/uapi/asm -_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ - $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') +$(shell mkdir -p $(uapi) $(kapi)) syscall := $(src)/syscall.tbl syshdr := $(srctree)/scripts/syscallhdr.sh diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 0430926426fe..8dfe62786cd5 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -65,19 +65,12 @@ static bool blkcg_policy_enabled(struct request_queue *q, return pol && test_bit(pol->plid, q->blkcg_pols); } -/** - * blkg_free - free a blkg - * @blkg: blkg to free - * - * Free @blkg which may be partially allocated. - */ -static void blkg_free(struct blkcg_gq *blkg) +static void blkg_free_workfn(struct work_struct *work) { + struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, + free_work); int i; - if (!blkg) - return; - for (i = 0; i < BLKCG_MAX_POLS; i++) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); @@ -89,6 +82,25 @@ static void blkg_free(struct blkcg_gq *blkg) kfree(blkg); } +/** + * blkg_free - free a blkg + * @blkg: blkg to free + * + * Free @blkg which may be partially allocated. + */ +static void blkg_free(struct blkcg_gq *blkg) +{ + if (!blkg) + return; + + /* + * Both ->pd_free_fn() and request queue's release handler may + * sleep, so free us by scheduling one work func + */ + INIT_WORK(&blkg->free_work, blkg_free_workfn); + schedule_work(&blkg->free_work); +} + static void __blkg_release(struct rcu_head *rcu) { struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 11f49f78db32..df9cfe4ca532 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -280,7 +280,6 @@ int set_task_ioprio(struct task_struct *task, int ioprio) task_lock(task); if (task->flags & PF_EXITING) { - err = -ESRCH; kmem_cache_free(iocontext_cachep, ioc); goto out; } @@ -292,7 +291,7 @@ int set_task_ioprio(struct task_struct *task, int ioprio) task->io_context->ioprio = ioprio; out: task_unlock(task); - return err; + return 0; } EXPORT_SYMBOL_GPL(set_task_ioprio); diff --git a/block/blk-mq.c b/block/blk-mq.c index e6f24fa4a4c2..ed3ed86f7dd2 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4462,21 +4462,28 @@ static bool blk_mq_elv_switch_none(struct list_head *head, return true; } -static void blk_mq_elv_switch_back(struct list_head *head, - struct request_queue *q) +static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head, + struct request_queue *q) { struct blk_mq_qe_pair *qe; - struct elevator_type *t = NULL; list_for_each_entry(qe, head, node) - if (qe->q == q) { - t = qe->type; - break; - } + if (qe->q == q) + return qe; - if (!t) - return; + return NULL; +} +static void blk_mq_elv_switch_back(struct list_head *head, + struct request_queue *q) +{ + struct blk_mq_qe_pair *qe; + struct elevator_type *t; + + qe = blk_lookup_qe_pair(head, q); + if (!qe) + return; + t = qe->type; list_del(&qe->node); kfree(qe); diff --git a/block/blk-wbt.h b/block/blk-wbt.h index 2eb01becde8c..7e44eccc676d 100644 --- a/block/blk-wbt.h +++ b/block/blk-wbt.h @@ -101,9 +101,6 @@ u64 wbt_default_latency_nsec(struct request_queue *); #else -static inline void wbt_track(struct request *rq, enum wbt_flags flags) -{ -} static inline int wbt_init(struct request_queue *q) { return -EINVAL; diff --git a/block/genhd.c b/block/genhd.c index c9a4fc90d3e9..b8b6759d670f 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -335,7 +335,7 @@ int blk_alloc_ext_minor(void) { int idx; - idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT, GFP_KERNEL); + idx = ida_alloc_range(&ext_devt_ida, 0, NR_EXT_DEVT - 1, GFP_KERNEL); if (idx == -ENOSPC) return -EBUSY; return idx; diff --git a/certs/Makefile b/certs/Makefile index 3ea7fe60823f..d8443cfb1c40 100644 --- a/certs/Makefile +++ b/certs/Makefile @@ -13,34 +13,20 @@ obj-$(CONFIG_SYSTEM_BLACKLIST_KEYRING) += blacklist_nohashes.o endif quiet_cmd_extract_certs = CERT $@ - cmd_extract_certs = $(obj)/extract-cert $(2) $@ + cmd_extract_certs = $(obj)/extract-cert $(extract-cert-in) $@ +extract-cert-in = $(or $(filter-out $(obj)/extract-cert, $(real-prereqs)),"") $(obj)/system_certificates.o: $(obj)/x509_certificate_list $(obj)/x509_certificate_list: $(CONFIG_SYSTEM_TRUSTED_KEYS) $(obj)/extract-cert FORCE - $(call if_changed,extract_certs,$(if $(CONFIG_SYSTEM_TRUSTED_KEYS),$<,"")) + $(call if_changed,extract_certs) targets += x509_certificate_list -ifeq ($(CONFIG_MODULE_SIG),y) - SIGN_KEY = y -endif - -ifeq ($(CONFIG_IMA_APPRAISE_MODSIG),y) -ifeq ($(CONFIG_MODULES),y) - SIGN_KEY = y -endif -endif - -ifdef SIGN_KEY -############################################################################### -# # If module signing is requested, say by allyesconfig, but a key has not been # supplied, then one will need to be generated to make sure the build does not # fail and that the kernel may be used afterwards. # -############################################################################### - # We do it this way rather than having a boolean option for enabling an # external private key, because 'make randconfig' might enable such a # boolean option and we unfortunately can't make it depend on !RANDCONFIG. @@ -67,23 +53,22 @@ $(obj)/x509.genkey: endif # CONFIG_MODULE_SIG_KEY -# If CONFIG_MODULE_SIG_KEY isn't a PKCS#11 URI, depend on it -ifneq ($(filter-out pkcs11:%, $(CONFIG_MODULE_SIG_KEY)),) -X509_DEP := $(CONFIG_MODULE_SIG_KEY) -endif - $(obj)/system_certificates.o: $(obj)/signing_key.x509 -$(obj)/signing_key.x509: $(X509_DEP) $(obj)/extract-cert FORCE - $(call if_changed,extract_certs,$(if $(CONFIG_MODULE_SIG_KEY),$(if $(X509_DEP),$<,$(CONFIG_MODULE_SIG_KEY)),"")) -endif # CONFIG_MODULE_SIG +PKCS11_URI := $(filter pkcs11:%, $(CONFIG_MODULE_SIG_KEY)) +ifdef PKCS11_URI +$(obj)/signing_key.x509: extract-cert-in := $(PKCS11_URI) +endif + +$(obj)/signing_key.x509: $(filter-out $(PKCS11_URI),$(CONFIG_MODULE_SIG_KEY)) $(obj)/extract-cert FORCE + $(call if_changed,extract_certs) targets += signing_key.x509 $(obj)/revocation_certificates.o: $(obj)/x509_revocation_list $(obj)/x509_revocation_list: $(CONFIG_SYSTEM_REVOCATION_KEYS) $(obj)/extract-cert FORCE - $(call if_changed,extract_certs,$(if $(CONFIG_SYSTEM_REVOCATION_KEYS),$<,"")) + $(call if_changed,extract_certs) targets += x509_revocation_list diff --git a/certs/system_certificates.S b/certs/system_certificates.S index e1645e6f4d97..003e25d4a17e 100644 --- a/certs/system_certificates.S +++ b/certs/system_certificates.S @@ -9,10 +9,7 @@ system_certificate_list: __cert_list_start: __module_cert_start: -#if defined(CONFIG_MODULE_SIG) || (defined(CONFIG_IMA_APPRAISE_MODSIG) \ - && defined(CONFIG_MODULES)) .incbin "certs/signing_key.x509" -#endif __module_cert_end: .incbin "certs/x509_certificate_list" __cert_list_end: diff --git a/drivers/acpi/acpi_ipmi.c b/drivers/acpi/acpi_ipmi.c index a5fe2926bf50..0555f68c2dfd 100644 --- a/drivers/acpi/acpi_ipmi.c +++ b/drivers/acpi/acpi_ipmi.c @@ -353,29 +353,27 @@ static void ipmi_flush_tx_msg(struct acpi_ipmi_device *ipmi) static void ipmi_cancel_tx_msg(struct acpi_ipmi_device *ipmi, struct acpi_ipmi_msg *msg) { - struct acpi_ipmi_msg *tx_msg, *temp; - bool msg_found = false; + struct acpi_ipmi_msg *tx_msg = NULL, *iter, *temp; unsigned long flags; spin_lock_irqsave(&ipmi->tx_msg_lock, flags); - list_for_each_entry_safe(tx_msg, temp, &ipmi->tx_msg_list, head) { - if (msg == tx_msg) { - msg_found = true; - list_del(&tx_msg->head); + list_for_each_entry_safe(iter, temp, &ipmi->tx_msg_list, head) { + if (msg == iter) { + tx_msg = iter; + list_del(&iter->head); break; } } spin_unlock_irqrestore(&ipmi->tx_msg_lock, flags); - if (msg_found) + if (tx_msg) acpi_ipmi_msg_put(tx_msg); } static void ipmi_msg_handler(struct ipmi_recv_msg *msg, void *user_msg_data) { struct acpi_ipmi_device *ipmi_device = user_msg_data; - bool msg_found = false; - struct acpi_ipmi_msg *tx_msg, *temp; + struct acpi_ipmi_msg *tx_msg = NULL, *iter, *temp; struct device *dev = ipmi_device->dev; unsigned long flags; @@ -387,16 +385,16 @@ static void ipmi_msg_handler(struct ipmi_recv_msg *msg, void *user_msg_data) } spin_lock_irqsave(&ipmi_device->tx_msg_lock, flags); - list_for_each_entry_safe(tx_msg, temp, &ipmi_device->tx_msg_list, head) { - if (msg->msgid == tx_msg->tx_msgid) { - msg_found = true; - list_del(&tx_msg->head); + list_for_each_entry_safe(iter, temp, &ipmi_device->tx_msg_list, head) { + if (msg->msgid == iter->tx_msgid) { + tx_msg = iter; + list_del(&iter->head); break; } } spin_unlock_irqrestore(&ipmi_device->tx_msg_lock, flags); - if (!msg_found) { + if (!tx_msg) { dev_warn(dev, "Unexpected response (msg id %ld) is returned.\n", msg->msgid); @@ -482,15 +480,14 @@ err_ref: static void ipmi_bmc_gone(int iface) { - struct acpi_ipmi_device *ipmi_device, *temp; - bool dev_found = false; + struct acpi_ipmi_device *ipmi_device = NULL, *iter, *temp; mutex_lock(&driver_data.ipmi_lock); - list_for_each_entry_safe(ipmi_device, temp, + list_for_each_entry_safe(iter, temp, &driver_data.ipmi_devices, head) { - if (ipmi_device->ipmi_ifnum != iface) { - dev_found = true; - __ipmi_dev_kill(ipmi_device); + if (iter->ipmi_ifnum != iface) { + ipmi_device = iter; + __ipmi_dev_kill(iter); break; } } @@ -500,7 +497,7 @@ static void ipmi_bmc_gone(int iface) struct acpi_ipmi_device, head); mutex_unlock(&driver_data.ipmi_lock); - if (dev_found) { + if (ipmi_device) { ipmi_flush_tx_msg(ipmi_device); acpi_ipmi_dev_put(ipmi_device); } diff --git a/drivers/acpi/apei/apei-base.c b/drivers/acpi/apei/apei-base.c index c7fdb12c3310..33b7fbbeda82 100644 --- a/drivers/acpi/apei/apei-base.c +++ b/drivers/acpi/apei/apei-base.c @@ -319,7 +319,7 @@ repeat: if (res_ins) list_add(&res_ins->list, res_list); else { - res_ins = kmalloc(sizeof(*res), GFP_KERNEL); + res_ins = kmalloc(sizeof(*res_ins), GFP_KERNEL); if (!res_ins) return -ENOMEM; res_ins->start = start; diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index d418449194ee..bc1454789a06 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -654,7 +654,7 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) unsigned int num_ent, i, cpc_rev; int pcc_subspace_id = -1; acpi_status status; - int ret = -EFAULT; + int ret = -ENODATA; if (osc_sb_cppc_not_supported) return -ENODEV; @@ -679,9 +679,14 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) cpc_obj = &out_obj->package.elements[0]; if (cpc_obj->type == ACPI_TYPE_INTEGER) { num_ent = cpc_obj->integer.value; + if (num_ent <= 1) { + pr_debug("Unexpected _CPC NumEntries value (%d) for CPU:%d\n", + num_ent, pr->id); + goto out_free; + } } else { - pr_debug("Unexpected entry type(%d) for NumEntries\n", - cpc_obj->type); + pr_debug("Unexpected _CPC NumEntries entry type (%d) for CPU:%d\n", + cpc_obj->type, pr->id); goto out_free; } cpc_ptr->num_entries = num_ent; @@ -691,8 +696,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) if (cpc_obj->type == ACPI_TYPE_INTEGER) { cpc_rev = cpc_obj->integer.value; } else { - pr_debug("Unexpected entry type(%d) for Revision\n", - cpc_obj->type); + pr_debug("Unexpected _CPC Revision entry type (%d) for CPU:%d\n", + cpc_obj->type, pr->id); goto out_free; } cpc_ptr->version = cpc_rev; @@ -723,7 +728,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) if (pcc_data_alloc(pcc_subspace_id)) goto out_free; } else if (pcc_subspace_id != gas_t->access_width) { - pr_debug("Mismatched PCC ids.\n"); + pr_debug("Mismatched PCC ids in _CPC for CPU:%d\n", + pr->id); goto out_free; } } else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { @@ -742,20 +748,21 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) * SystemIO doesn't implement 64-bit * registers. */ - pr_debug("Invalid access width %d for SystemIO register\n", - gas_t->access_width); + pr_debug("Invalid access width %d for SystemIO register in _CPC\n", + gas_t->access_width); goto out_free; } if (gas_t->address & OVER_16BTS_MASK) { /* SystemIO registers use 16-bit integer addresses */ - pr_debug("Invalid IO port %llu for SystemIO register\n", - gas_t->address); + pr_debug("Invalid IO port %llu for SystemIO register in _CPC\n", + gas_t->address); goto out_free; } } else { if (gas_t->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE || !cpc_ffh_supported()) { /* Support only PCC, SystemMemory, SystemIO, and FFH type regs. */ - pr_debug("Unsupported register type: %d\n", gas_t->space_id); + pr_debug("Unsupported register type (%d) in _CPC\n", + gas_t->space_id); goto out_free; } } @@ -763,7 +770,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) cpc_ptr->cpc_regs[i-2].type = ACPI_TYPE_BUFFER; memcpy(&cpc_ptr->cpc_regs[i-2].cpc_entry.reg, gas_t, sizeof(*gas_t)); } else { - pr_debug("Err in entry:%d in CPC table of CPU:%d\n", i, pr->id); + pr_debug("Invalid entry type (%d) in _CPC for CPU:%d\n", + i, pr->id); goto out_free; } } diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index ceee808f7f2a..47ec11d4c68e 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -151,8 +151,8 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) { struct acpi_madt_local_apic_override *p = (struct acpi_madt_local_apic_override *)header; - pr_info("LAPIC_ADDR_OVR (address[%p])\n", - (void *)(unsigned long)p->address); + pr_info("LAPIC_ADDR_OVR (address[0x%llx])\n", + p->address); } break; diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index e5641e6c52ee..bb45a9c00514 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -115,14 +115,16 @@ config SATA_AHCI If unsure, say N. -config SATA_LPM_POLICY +config SATA_MOBILE_LPM_POLICY int "Default SATA Link Power Management policy for low power chipsets" range 0 4 default 0 depends on SATA_AHCI help Select the Default SATA Link Power Management (LPM) policy to use - for chipsets / "South Bridges" designated as supporting low power. + for chipsets / "South Bridges" supporting low-power modes. Such + chipsets are typically found on most laptops but desktops and + servers now also widely use chipsets supporting low power modes. The value set has the following meanings: 0 => Keep firmware settings diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 84456c05e845..397dfd27c90d 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1595,7 +1595,7 @@ static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports, static void ahci_update_initial_lpm_policy(struct ata_port *ap, struct ahci_host_priv *hpriv) { - int policy = CONFIG_SATA_LPM_POLICY; + int policy = CONFIG_SATA_MOBILE_LPM_POLICY; /* Ignore processing for chipsets that don't use policy */ diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index 6ead58c1b6e5..ad11a4c52fbe 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -236,7 +236,7 @@ enum { AHCI_HFLAG_NO_WRITE_TO_RO = (1 << 24), /* don't write to read only registers */ AHCI_HFLAG_USE_LPM_POLICY = (1 << 25), /* chipset that should use - SATA_LPM_POLICY + SATA_MOBILE_LPM_POLICY as default lpm_policy */ AHCI_HFLAG_SUSPEND_PHYS = (1 << 26), /* handle PHYs during suspend/resume */ diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index cceedde51126..ca64837641be 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -4014,6 +4014,9 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { ATA_HORKAGE_ZERO_AFTER_TRIM, }, { "Crucial_CT*MX100*", "MU01", ATA_HORKAGE_NO_NCQ_TRIM | ATA_HORKAGE_ZERO_AFTER_TRIM, }, + { "Samsung SSD 840 EVO*", NULL, ATA_HORKAGE_NO_NCQ_TRIM | + ATA_HORKAGE_NO_DMA_LOG | + ATA_HORKAGE_ZERO_AFTER_TRIM, }, { "Samsung SSD 840*", NULL, ATA_HORKAGE_NO_NCQ_TRIM | ATA_HORKAGE_ZERO_AFTER_TRIM, }, { "Samsung SSD 850*", NULL, ATA_HORKAGE_NO_NCQ_TRIM | diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index b3be7a8f5bea..b1666adc1c3a 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -1634,7 +1634,7 @@ EXPORT_SYMBOL_GPL(ata_sff_interrupt); void ata_sff_lost_interrupt(struct ata_port *ap) { - u8 status; + u8 status = 0; struct ata_queued_cmd *qc; /* Only one outstanding command per SFF channel */ diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c index bec33d781ae0..e3263e961045 100644 --- a/drivers/ata/sata_dwc_460ex.c +++ b/drivers/ata/sata_dwc_460ex.c @@ -137,7 +137,11 @@ struct sata_dwc_device { #endif }; -#define SATA_DWC_QCMD_MAX 32 +/* + * Allow one extra special slot for commands and DMA management + * to account for libata internal commands. + */ +#define SATA_DWC_QCMD_MAX (ATA_MAX_QUEUE + 1) struct sata_dwc_device_port { struct sata_dwc_device *hsdev; diff --git a/drivers/atm/Kconfig b/drivers/atm/Kconfig index b9370bbca828..7be08e24955c 100644 --- a/drivers/atm/Kconfig +++ b/drivers/atm/Kconfig @@ -244,31 +244,6 @@ config ATM_IDT77252_USE_SUNI depends on ATM_IDT77252 default y -config ATM_AMBASSADOR - tristate "Madge Ambassador (Collage PCI 155 Server)" - depends on PCI && VIRT_TO_BUS - select BITREVERSE - help - This is a driver for ATMizer based ATM card produced by Madge - Networks Ltd. Say Y (or M to compile as a module named ambassador) - here if you have one of these cards. - -config ATM_AMBASSADOR_DEBUG - bool "Enable debugging messages" - depends on ATM_AMBASSADOR - help - Somewhat useful debugging messages are available. The choice of - messages is controlled by a bitmap. This may be specified as a - module argument (kernel command line argument as well?), changed - dynamically using an ioctl (not yet) or changed by sending the - string "Dxxxx" to VCI 1023 (where x is a hex digit). See the file - <file:drivers/atm/ambassador.h> for the meanings of the bits in the - mask. - - When active, these messages can have a significant impact on the - speed of the driver, and the size of your syslog files! When - inactive, they will have only a modest impact on performance. - config ATM_HORIZON tristate "Madge Horizon [Ultra] (Collage PCI 25 and Collage PCI 155 Client)" depends on PCI && VIRT_TO_BUS diff --git a/drivers/atm/Makefile b/drivers/atm/Makefile index aa191616a72e..99ecbc280643 100644 --- a/drivers/atm/Makefile +++ b/drivers/atm/Makefile @@ -7,7 +7,6 @@ fore_200e-y := fore200e.o obj-$(CONFIG_ATM_ZATM) += zatm.o uPD98402.o obj-$(CONFIG_ATM_NICSTAR) += nicstar.o -obj-$(CONFIG_ATM_AMBASSADOR) += ambassador.o obj-$(CONFIG_ATM_HORIZON) += horizon.o obj-$(CONFIG_ATM_IA) += iphase.o suni.o obj-$(CONFIG_ATM_FORE200E) += fore_200e.o diff --git a/drivers/atm/ambassador.c b/drivers/atm/ambassador.c deleted file mode 100644 index c039b8a4fefe..000000000000 --- a/drivers/atm/ambassador.c +++ /dev/null @@ -1,2400 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - Madge Ambassador ATM Adapter driver. - Copyright (C) 1995-1999 Madge Networks Ltd. - -*/ - -/* * dedicated to the memory of Graham Gordon 1971-1998 * */ - -#include <linux/module.h> -#include <linux/types.h> -#include <linux/pci.h> -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/ioport.h> -#include <linux/atmdev.h> -#include <linux/delay.h> -#include <linux/interrupt.h> -#include <linux/poison.h> -#include <linux/bitrev.h> -#include <linux/mutex.h> -#include <linux/firmware.h> -#include <linux/ihex.h> -#include <linux/slab.h> - -#include <linux/atomic.h> -#include <asm/io.h> -#include <asm/byteorder.h> - -#include "ambassador.h" - -#define maintainer_string "Giuliano Procida at Madge Networks <gprocida@madge.com>" -#define description_string "Madge ATM Ambassador driver" -#define version_string "1.2.4" - -static inline void __init show_version (void) { - printk ("%s version %s\n", description_string, version_string); -} - -/* - - Theory of Operation - - I Hardware, detection, initialisation and shutdown. - - 1. Supported Hardware - - This driver is for the PCI ATMizer-based Ambassador card (except - very early versions). It is not suitable for the similar EISA "TR7" - card. Commercially, both cards are known as Collage Server ATM - adapters. - - The loader supports image transfer to the card, image start and few - other miscellaneous commands. - - Only AAL5 is supported with vpi = 0 and vci in the range 0 to 1023. - - The cards are big-endian. - - 2. Detection - - Standard PCI stuff, the early cards are detected and rejected. - - 3. Initialisation - - The cards are reset and the self-test results are checked. The - microcode image is then transferred and started. This waits for a - pointer to a descriptor containing details of the host-based queues - and buffers and various parameters etc. Once they are processed - normal operations may begin. The BIA is read using a microcode - command. - - 4. Shutdown - - This may be accomplished either by a card reset or via the microcode - shutdown command. Further investigation required. - - 5. Persistent state - - The card reset does not affect PCI configuration (good) or the - contents of several other "shared run-time registers" (bad) which - include doorbell and interrupt control as well as EEPROM and PCI - control. The driver must be careful when modifying these registers - not to touch bits it does not use and to undo any changes at exit. - - II Driver software - - 0. Generalities - - The adapter is quite intelligent (fast) and has a simple interface - (few features). VPI is always zero, 1024 VCIs are supported. There - is limited cell rate support. UBR channels can be capped and ABR - (explicit rate, but not EFCI) is supported. There is no CBR or VBR - support. - - 1. Driver <-> Adapter Communication - - Apart from the basic loader commands, the driver communicates - through three entities: the command queue (CQ), the transmit queue - pair (TXQ) and the receive queue pairs (RXQ). These three entities - are set up by the host and passed to the microcode just after it has - been started. - - All queues are host-based circular queues. They are contiguous and - (due to hardware limitations) have some restrictions as to their - locations in (bus) memory. They are of the "full means the same as - empty so don't do that" variety since the adapter uses pointers - internally. - - The queue pairs work as follows: one queue is for supply to the - adapter, items in it are pending and are owned by the adapter; the - other is the queue for return from the adapter, items in it have - been dealt with by the adapter. The host adds items to the supply - (TX descriptors and free RX buffer descriptors) and removes items - from the return (TX and RX completions). The adapter deals with out - of order completions. - - Interrupts (card to host) and the doorbell (host to card) are used - for signalling. - - 1. CQ - - This is to communicate "open VC", "close VC", "get stats" etc. to - the adapter. At most one command is retired every millisecond by the - card. There is no out of order completion or notification. The - driver needs to check the return code of the command, waiting as - appropriate. - - 2. TXQ - - TX supply items are of variable length (scatter gather support) and - so the queue items are (more or less) pointers to the real thing. - Each TX supply item contains a unique, host-supplied handle (the skb - bus address seems most sensible as this works for Alphas as well, - there is no need to do any endian conversions on the handles). - - TX return items consist of just the handles above. - - 3. RXQ (up to 4 of these with different lengths and buffer sizes) - - RX supply items consist of a unique, host-supplied handle (the skb - bus address again) and a pointer to the buffer data area. - - RX return items consist of the handle above, the VC, length and a - status word. This just screams "oh so easy" doesn't it? - - Note on RX pool sizes: - - Each pool should have enough buffers to handle a back-to-back stream - of minimum sized frames on a single VC. For example: - - frame spacing = 3us (about right) - - delay = IRQ lat + RX handling + RX buffer replenish = 20 (us) (a guess) - - min number of buffers for one VC = 1 + delay/spacing (buffers) - - delay/spacing = latency = (20+2)/3 = 7 (buffers) (rounding up) - - The 20us delay assumes that there is no need to sleep; if we need to - sleep to get buffers we are going to drop frames anyway. - - In fact, each pool should have enough buffers to support the - simultaneous reassembly of a separate frame on each VC and cope with - the case in which frames complete in round robin cell fashion on - each VC. - - Only one frame can complete at each cell arrival, so if "n" VCs are - open, the worst case is to have them all complete frames together - followed by all starting new frames together. - - desired number of buffers = n + delay/spacing - - These are the extreme requirements, however, they are "n+k" for some - "k" so we have only the constant to choose. This is the argument - rx_lats which current defaults to 7. - - Actually, "n ? n+k : 0" is better and this is what is implemented, - subject to the limit given by the pool size. - - 4. Driver locking - - Simple spinlocks are used around the TX and RX queue mechanisms. - Anyone with a faster, working method is welcome to implement it. - - The adapter command queue is protected with a spinlock. We always - wait for commands to complete. - - A more complex form of locking is used around parts of the VC open - and close functions. There are three reasons for a lock: 1. we need - to do atomic rate reservation and release (not used yet), 2. Opening - sometimes involves two adapter commands which must not be separated - by another command on the same VC, 3. the changes to RX pool size - must be atomic. The lock needs to work over context switches, so we - use a semaphore. - - III Hardware Features and Microcode Bugs - - 1. Byte Ordering - - *%^"$&%^$*&^"$(%^$#&^%$(&#%$*(&^#%!"!"!*! - - 2. Memory access - - All structures that are not accessed using DMA must be 4-byte - aligned (not a problem) and must not cross 4MB boundaries. - - There is a DMA memory hole at E0000000-E00000FF (groan). - - TX fragments (DMA read) must not cross 4MB boundaries (would be 16MB - but for a hardware bug). - - RX buffers (DMA write) must not cross 16MB boundaries and must - include spare trailing bytes up to the next 4-byte boundary; they - will be written with rubbish. - - The PLX likes to prefetch; if reading up to 4 u32 past the end of - each TX fragment is not a problem, then TX can be made to go a - little faster by passing a flag at init that disables a prefetch - workaround. We do not pass this flag. (new microcode only) - - Now we: - . Note that alloc_skb rounds up size to a 16byte boundary. - . Ensure all areas do not traverse 4MB boundaries. - . Ensure all areas do not start at a E00000xx bus address. - (I cannot be certain, but this may always hold with Linux) - . Make all failures cause a loud message. - . Discard non-conforming SKBs (causes TX failure or RX fill delay). - . Discard non-conforming TX fragment descriptors (the TX fails). - In the future we could: - . Allow RX areas that traverse 4MB (but not 16MB) boundaries. - . Segment TX areas into some/more fragments, when necessary. - . Relax checks for non-DMA items (ignore hole). - . Give scatter-gather (iovec) requirements using ???. (?) - - 3. VC close is broken (only for new microcode) - - The VC close adapter microcode command fails to do anything if any - frames have been received on the VC but none have been transmitted. - Frames continue to be reassembled and passed (with IRQ) to the - driver. - - IV To Do List - - . Fix bugs! - - . Timer code may be broken. - - . Deal with buggy VC close (somehow) in microcode 12. - - . Handle interrupted and/or non-blocking writes - is this a job for - the protocol layer? - - . Add code to break up TX fragments when they span 4MB boundaries. - - . Add SUNI phy layer (need to know where SUNI lives on card). - - . Implement a tx_alloc fn to (a) satisfy TX alignment etc. and (b) - leave extra headroom space for Ambassador TX descriptors. - - . Understand these elements of struct atm_vcc: recvq (proto?), - sleep, callback, listenq, backlog_quota, reply and user_back. - - . Adjust TX/RX skb allocation to favour IP with LANE/CLIP (configurable). - - . Impose a TX-pending limit (2?) on each VC, help avoid TX q overflow. - - . Decide whether RX buffer recycling is or can be made completely safe; - turn it back on. It looks like Werner is going to axe this. - - . Implement QoS changes on open VCs (involves extracting parts of VC open - and close into separate functions and using them to make changes). - - . Hack on command queue so that someone can issue multiple commands and wait - on the last one (OR only "no-op" or "wait" commands are waited for). - - . Eliminate need for while-schedule around do_command. - -*/ - -static void do_housekeeping (struct timer_list *t); -/********** globals **********/ - -static unsigned short debug = 0; -static unsigned int cmds = 8; -static unsigned int txs = 32; -static unsigned int rxs[NUM_RX_POOLS] = { 64, 64, 64, 64 }; -static unsigned int rxs_bs[NUM_RX_POOLS] = { 4080, 12240, 36720, 65535 }; -static unsigned int rx_lats = 7; -static unsigned char pci_lat = 0; - -static const unsigned long onegigmask = -1 << 30; - -/********** access to adapter **********/ - -static inline void wr_plain (const amb_dev * dev, size_t addr, u32 data) { - PRINTD (DBG_FLOW|DBG_REGS, "wr: %08zx <- %08x", addr, data); -#ifdef AMB_MMIO - dev->membase[addr / sizeof(u32)] = data; -#else - outl (data, dev->iobase + addr); -#endif -} - -static inline u32 rd_plain (const amb_dev * dev, size_t addr) { -#ifdef AMB_MMIO - u32 data = dev->membase[addr / sizeof(u32)]; -#else - u32 data = inl (dev->iobase + addr); -#endif - PRINTD (DBG_FLOW|DBG_REGS, "rd: %08zx -> %08x", addr, data); - return data; -} - -static inline void wr_mem (const amb_dev * dev, size_t addr, u32 data) { - __be32 be = cpu_to_be32 (data); - PRINTD (DBG_FLOW|DBG_REGS, "wr: %08zx <- %08x b[%08x]", addr, data, be); -#ifdef AMB_MMIO - dev->membase[addr / sizeof(u32)] = be; -#else - outl (be, dev->iobase + addr); -#endif -} - -static inline u32 rd_mem (const amb_dev * dev, size_t addr) { -#ifdef AMB_MMIO - __be32 be = dev->membase[addr / sizeof(u32)]; -#else - __be32 be = inl (dev->iobase + addr); -#endif - u32 data = be32_to_cpu (be); - PRINTD (DBG_FLOW|DBG_REGS, "rd: %08zx -> %08x b[%08x]", addr, data, be); - return data; -} - -/********** dump routines **********/ - -static inline void dump_registers (const amb_dev * dev) { -#ifdef DEBUG_AMBASSADOR - if (debug & DBG_REGS) { - size_t i; - PRINTD (DBG_REGS, "reading PLX control: "); - for (i = 0x00; i < 0x30; i += sizeof(u32)) - rd_mem (dev, i); - PRINTD (DBG_REGS, "reading mailboxes: "); - for (i = 0x40; i < 0x60; i += sizeof(u32)) - rd_mem (dev, i); - PRINTD (DBG_REGS, "reading doorb irqev irqen reset:"); - for (i = 0x60; i < 0x70; i += sizeof(u32)) - rd_mem (dev, i); - } -#else - (void) dev; -#endif - return; -} - -static inline void dump_loader_block (volatile loader_block * lb) { -#ifdef DEBUG_AMBASSADOR - unsigned int i; - PRINTDB (DBG_LOAD, "lb @ %p; res: %d, cmd: %d, pay:", - lb, be32_to_cpu (lb->result), be32_to_cpu (lb->command)); - for (i = 0; i < MAX_COMMAND_DATA; ++i) - PRINTDM (DBG_LOAD, " %08x", be32_to_cpu (lb->payload.data[i])); - PRINTDE (DBG_LOAD, ", vld: %08x", be32_to_cpu (lb->valid)); -#else - (void) lb; -#endif - return; -} - -static inline void dump_command (command * cmd) { -#ifdef DEBUG_AMBASSADOR - unsigned int i; - PRINTDB (DBG_CMD, "cmd @ %p, req: %08x, pars:", - cmd, /*be32_to_cpu*/ (cmd->request)); - for (i = 0; i < 3; ++i) - PRINTDM (DBG_CMD, " %08x", /*be32_to_cpu*/ (cmd->args.par[i])); - PRINTDE (DBG_CMD, ""); -#else - (void) cmd; -#endif - return; -} - -static inline void dump_skb (char * prefix, unsigned int vc, struct sk_buff * skb) { -#ifdef DEBUG_AMBASSADOR - unsigned int i; - unsigned char * data = skb->data; - PRINTDB (DBG_DATA, "%s(%u) ", prefix, vc); - for (i=0; i<skb->len && i < 256;i++) - PRINTDM (DBG_DATA, "%02x ", data[i]); - PRINTDE (DBG_DATA,""); -#else - (void) prefix; - (void) vc; - (void) skb; -#endif - return; -} - -/********** check memory areas for use by Ambassador **********/ - -/* see limitations under Hardware Features */ - -static int check_area (void * start, size_t length) { - // assumes length > 0 - const u32 fourmegmask = -1 << 22; - const u32 twofivesixmask = -1 << 8; - const u32 starthole = 0xE0000000; - u32 startaddress = virt_to_bus (start); - u32 lastaddress = startaddress+length-1; - if ((startaddress ^ lastaddress) & fourmegmask || - (startaddress & twofivesixmask) == starthole) { - PRINTK (KERN_ERR, "check_area failure: [%x,%x] - mail maintainer!", - startaddress, lastaddress); - return -1; - } else { - return 0; - } -} - -/********** free an skb (as per ATM device driver documentation) **********/ - -static void amb_kfree_skb (struct sk_buff * skb) { - if (ATM_SKB(skb)->vcc->pop) { - ATM_SKB(skb)->vcc->pop (ATM_SKB(skb)->vcc, skb); - } else { - dev_kfree_skb_any (skb); - } -} - -/********** TX completion **********/ - -static void tx_complete (amb_dev * dev, tx_out * tx) { - tx_simple * tx_descr = bus_to_virt (tx->handle); - struct sk_buff * skb = tx_descr->skb; - - PRINTD (DBG_FLOW|DBG_TX, "tx_complete %p %p", dev, tx); - - // VC layer stats - atomic_inc(&ATM_SKB(skb)->vcc->stats->tx); - - // free the descriptor - kfree (tx_descr); - - // free the skb - amb_kfree_skb (skb); - - dev->stats.tx_ok++; - return; -} - -/********** RX completion **********/ - -static void rx_complete (amb_dev * dev, rx_out * rx) { - struct sk_buff * skb = bus_to_virt (rx->handle); - u16 vc = be16_to_cpu (rx->vc); - // unused: u16 lec_id = be16_to_cpu (rx->lec_id); - u16 status = be16_to_cpu (rx->status); - u16 rx_len = be16_to_cpu (rx->length); - - PRINTD (DBG_FLOW|DBG_RX, "rx_complete %p %p (len=%hu)", dev, rx, rx_len); - - // XXX move this in and add to VC stats ??? - if (!status) { - struct atm_vcc * atm_vcc = dev->rxer[vc]; - dev->stats.rx.ok++; - - if (atm_vcc) { - - if (rx_len <= atm_vcc->qos.rxtp.max_sdu) { - - if (atm_charge (atm_vcc, skb->truesize)) { - - // prepare socket buffer - ATM_SKB(skb)->vcc = atm_vcc; - skb_put (skb, rx_len); - - dump_skb ("<<<", vc, skb); - - // VC layer stats - atomic_inc(&atm_vcc->stats->rx); - __net_timestamp(skb); - // end of our responsibility - atm_vcc->push (atm_vcc, skb); - return; - - } else { - // someone fix this (message), please! - PRINTD (DBG_INFO|DBG_RX, "dropped thanks to atm_charge (vc %hu, truesize %u)", vc, skb->truesize); - // drop stats incremented in atm_charge - } - - } else { - PRINTK (KERN_INFO, "dropped over-size frame"); - // should we count this? - atomic_inc(&atm_vcc->stats->rx_drop); - } - - } else { - PRINTD (DBG_WARN|DBG_RX, "got frame but RX closed for channel %hu", vc); - // this is an adapter bug, only in new version of microcode - } - - } else { - dev->stats.rx.error++; - if (status & CRC_ERR) - dev->stats.rx.badcrc++; - if (status & LEN_ERR) - dev->stats.rx.toolong++; - if (status & ABORT_ERR) - dev->stats.rx.aborted++; - if (status & UNUSED_ERR) - dev->stats.rx.unused++; - } - - dev_kfree_skb_any (skb); - return; -} - -/* - - Note on queue handling. - - Here "give" and "take" refer to queue entries and a queue (pair) - rather than frames to or from the host or adapter. Empty frame - buffers are given to the RX queue pair and returned unused or - containing RX frames. TX frames (well, pointers to TX fragment - lists) are given to the TX queue pair, completions are returned. - -*/ - -/********** command queue **********/ - -// I really don't like this, but it's the best I can do at the moment - -// also, the callers are responsible for byte order as the microcode -// sometimes does 16-bit accesses (yuk yuk yuk) - -static int command_do (amb_dev * dev, command * cmd) { - amb_cq * cq = &dev->cq; - volatile amb_cq_ptrs * ptrs = &cq->ptrs; - command * my_slot; - - PRINTD (DBG_FLOW|DBG_CMD, "command_do %p", dev); - - if (test_bit (dead, &dev->flags)) - return 0; - - spin_lock (&cq->lock); - - // if not full... - if (cq->pending < cq->maximum) { - // remember my slot for later - my_slot = ptrs->in; - PRINTD (DBG_CMD, "command in slot %p", my_slot); - - dump_command (cmd); - - // copy command in - *ptrs->in = *cmd; - cq->pending++; - ptrs->in = NEXTQ (ptrs->in, ptrs->start, ptrs->limit); - - // mail the command - wr_mem (dev, offsetof(amb_mem, mb.adapter.cmd_address), virt_to_bus (ptrs->in)); - - if (cq->pending > cq->high) - cq->high = cq->pending; - spin_unlock (&cq->lock); - - // these comments were in a while-loop before, msleep removes the loop - // go to sleep - // PRINTD (DBG_CMD, "wait: sleeping %lu for command", timeout); - msleep(cq->pending); - - // wait for my slot to be reached (all waiters are here or above, until...) - while (ptrs->out != my_slot) { - PRINTD (DBG_CMD, "wait: command slot (now at %p)", ptrs->out); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - } - - // wait on my slot (... one gets to its slot, and... ) - while (ptrs->out->request != cpu_to_be32 (SRB_COMPLETE)) { - PRINTD (DBG_CMD, "wait: command slot completion"); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - } - - PRINTD (DBG_CMD, "command complete"); - // update queue (... moves the queue along to the next slot) - spin_lock (&cq->lock); - cq->pending--; - // copy command out - *cmd = *ptrs->out; - ptrs->out = NEXTQ (ptrs->out, ptrs->start, ptrs->limit); - spin_unlock (&cq->lock); - - return 0; - } else { - cq->filled++; - spin_unlock (&cq->lock); - return -EAGAIN; - } - -} - -/********** TX queue pair **********/ - -static int tx_give (amb_dev * dev, tx_in * tx) { - amb_txq * txq = &dev->txq; - unsigned long flags; - - PRINTD (DBG_FLOW|DBG_TX, "tx_give %p", dev); - - if (test_bit (dead, &dev->flags)) - return 0; - - spin_lock_irqsave (&txq->lock, flags); - - if (txq->pending < txq->maximum) { - PRINTD (DBG_TX, "TX in slot %p", txq->in.ptr); - - *txq->in.ptr = *tx; - txq->pending++; - txq->in.ptr = NEXTQ (txq->in.ptr, txq->in.start, txq->in.limit); - // hand over the TX and ring the bell - wr_mem (dev, offsetof(amb_mem, mb.adapter.tx_address), virt_to_bus (txq->in.ptr)); - wr_mem (dev, offsetof(amb_mem, doorbell), TX_FRAME); - - if (txq->pending > txq->high) - txq->high = txq->pending; - spin_unlock_irqrestore (&txq->lock, flags); - return 0; - } else { - txq->filled++; - spin_unlock_irqrestore (&txq->lock, flags); - return -EAGAIN; - } -} - -static int tx_take (amb_dev * dev) { - amb_txq * txq = &dev->txq; - unsigned long flags; - - PRINTD (DBG_FLOW|DBG_TX, "tx_take %p", dev); - - spin_lock_irqsave (&txq->lock, flags); - - if (txq->pending && txq->out.ptr->handle) { - // deal with TX completion - tx_complete (dev, txq->out.ptr); - // mark unused again - txq->out.ptr->handle = 0; - // remove item - txq->pending--; - txq->out.ptr = NEXTQ (txq->out.ptr, txq->out.start, txq->out.limit); - - spin_unlock_irqrestore (&txq->lock, flags); - return 0; - } else { - - spin_unlock_irqrestore (&txq->lock, flags); - return -1; - } -} - -/********** RX queue pairs **********/ - -static int rx_give (amb_dev * dev, rx_in * rx, unsigned char pool) { - amb_rxq * rxq = &dev->rxq[pool]; - unsigned long flags; - - PRINTD (DBG_FLOW|DBG_RX, "rx_give %p[%hu]", dev, pool); - - spin_lock_irqsave (&rxq->lock, flags); - - if (rxq->pending < rxq->maximum) { - PRINTD (DBG_RX, "RX in slot %p", rxq->in.ptr); - - *rxq->in.ptr = *rx; - rxq->pending++; - rxq->in.ptr = NEXTQ (rxq->in.ptr, rxq->in.start, rxq->in.limit); - // hand over the RX buffer - wr_mem (dev, offsetof(amb_mem, mb.adapter.rx_address[pool]), virt_to_bus (rxq->in.ptr)); - - spin_unlock_irqrestore (&rxq->lock, flags); - return 0; - } else { - spin_unlock_irqrestore (&rxq->lock, flags); - return -1; - } -} - -static int rx_take (amb_dev * dev, unsigned char pool) { - amb_rxq * rxq = &dev->rxq[pool]; - unsigned long flags; - - PRINTD (DBG_FLOW|DBG_RX, "rx_take %p[%hu]", dev, pool); - - spin_lock_irqsave (&rxq->lock, flags); - - if (rxq->pending && (rxq->out.ptr->status || rxq->out.ptr->length)) { - // deal with RX completion - rx_complete (dev, rxq->out.ptr); - // mark unused again - rxq->out.ptr->status = 0; - rxq->out.ptr->length = 0; - // remove item - rxq->pending--; - rxq->out.ptr = NEXTQ (rxq->out.ptr, rxq->out.start, rxq->out.limit); - - if (rxq->pending < rxq->low) - rxq->low = rxq->pending; - spin_unlock_irqrestore (&rxq->lock, flags); - return 0; - } else { - if (!rxq->pending && rxq->buffers_wanted) - rxq->emptied++; - spin_unlock_irqrestore (&rxq->lock, flags); - return -1; - } -} - -/********** RX Pool handling **********/ - -/* pre: buffers_wanted = 0, post: pending = 0 */ -static void drain_rx_pool (amb_dev * dev, unsigned char pool) { - amb_rxq * rxq = &dev->rxq[pool]; - - PRINTD (DBG_FLOW|DBG_POOL, "drain_rx_pool %p %hu", dev, pool); - - if (test_bit (dead, &dev->flags)) - return; - - /* we are not quite like the fill pool routines as we cannot just - remove one buffer, we have to remove all of them, but we might as - well pretend... */ - if (rxq->pending > rxq->buffers_wanted) { - command cmd; - cmd.request = cpu_to_be32 (SRB_FLUSH_BUFFER_Q); - cmd.args.flush.flags = cpu_to_be32 (pool << SRB_POOL_SHIFT); - while (command_do (dev, &cmd)) - schedule(); - /* the pool may also be emptied via the interrupt handler */ - while (rxq->pending > rxq->buffers_wanted) - if (rx_take (dev, pool)) - schedule(); - } - - return; -} - -static void drain_rx_pools (amb_dev * dev) { - unsigned char pool; - - PRINTD (DBG_FLOW|DBG_POOL, "drain_rx_pools %p", dev); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - drain_rx_pool (dev, pool); -} - -static void fill_rx_pool (amb_dev * dev, unsigned char pool, - gfp_t priority) -{ - rx_in rx; - amb_rxq * rxq; - - PRINTD (DBG_FLOW|DBG_POOL, "fill_rx_pool %p %hu %x", dev, pool, priority); - - if (test_bit (dead, &dev->flags)) - return; - - rxq = &dev->rxq[pool]; - while (rxq->pending < rxq->maximum && rxq->pending < rxq->buffers_wanted) { - - struct sk_buff * skb = alloc_skb (rxq->buffer_size, priority); - if (!skb) { - PRINTD (DBG_SKB|DBG_POOL, "failed to allocate skb for RX pool %hu", pool); - return; - } - if (check_area (skb->data, skb->truesize)) { - dev_kfree_skb_any (skb); - return; - } - // cast needed as there is no %? for pointer differences - PRINTD (DBG_SKB, "allocated skb at %p, head %p, area %li", - skb, skb->head, (long) skb_end_offset(skb)); - rx.handle = virt_to_bus (skb); - rx.host_address = cpu_to_be32 (virt_to_bus (skb->data)); - if (rx_give (dev, &rx, pool)) - dev_kfree_skb_any (skb); - - } - - return; -} - -// top up all RX pools -static void fill_rx_pools (amb_dev * dev) { - unsigned char pool; - - PRINTD (DBG_FLOW|DBG_POOL, "fill_rx_pools %p", dev); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - fill_rx_pool (dev, pool, GFP_ATOMIC); - - return; -} - -/********** enable host interrupts **********/ - -static void interrupts_on (amb_dev * dev) { - wr_plain (dev, offsetof(amb_mem, interrupt_control), - rd_plain (dev, offsetof(amb_mem, interrupt_control)) - | AMB_INTERRUPT_BITS); -} - -/********** disable host interrupts **********/ - -static void interrupts_off (amb_dev * dev) { - wr_plain (dev, offsetof(amb_mem, interrupt_control), - rd_plain (dev, offsetof(amb_mem, interrupt_control)) - &~ AMB_INTERRUPT_BITS); -} - -/********** interrupt handling **********/ - -static irqreturn_t interrupt_handler(int irq, void *dev_id) { - amb_dev * dev = dev_id; - - PRINTD (DBG_IRQ|DBG_FLOW, "interrupt_handler: %p", dev_id); - - { - u32 interrupt = rd_plain (dev, offsetof(amb_mem, interrupt)); - - // for us or someone else sharing the same interrupt - if (!interrupt) { - PRINTD (DBG_IRQ, "irq not for me: %d", irq); - return IRQ_NONE; - } - - // definitely for us - PRINTD (DBG_IRQ, "FYI: interrupt was %08x", interrupt); - wr_plain (dev, offsetof(amb_mem, interrupt), -1); - } - - { - unsigned int irq_work = 0; - unsigned char pool; - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - while (!rx_take (dev, pool)) - ++irq_work; - while (!tx_take (dev)) - ++irq_work; - - if (irq_work) { - fill_rx_pools (dev); - - PRINTD (DBG_IRQ, "work done: %u", irq_work); - } else { - PRINTD (DBG_IRQ|DBG_WARN, "no work done"); - } - } - - PRINTD (DBG_IRQ|DBG_FLOW, "interrupt_handler done: %p", dev_id); - return IRQ_HANDLED; -} - -/********** make rate (not quite as much fun as Horizon) **********/ - -static int make_rate (unsigned int rate, rounding r, - u16 * bits, unsigned int * actual) { - unsigned char exp = -1; // hush gcc - unsigned int man = -1; // hush gcc - - PRINTD (DBG_FLOW|DBG_QOS, "make_rate %u", rate); - - // rates in cells per second, ITU format (nasty 16-bit floating-point) - // given 5-bit e and 9-bit m: - // rate = EITHER (1+m/2^9)*2^e OR 0 - // bits = EITHER 1<<14 | e<<9 | m OR 0 - // (bit 15 is "reserved", bit 14 "non-zero") - // smallest rate is 0 (special representation) - // largest rate is (1+511/512)*2^31 = 4290772992 (< 2^32-1) - // smallest non-zero rate is (1+0/512)*2^0 = 1 (> 0) - // simple algorithm: - // find position of top bit, this gives e - // remove top bit and shift (rounding if feeling clever) by 9-e - - // ucode bug: please don't set bit 14! so 0 rate not representable - - if (rate > 0xffc00000U) { - // larger than largest representable rate - - if (r == round_up) { - return -EINVAL; - } else { - exp = 31; - man = 511; - } - - } else if (rate) { - // representable rate - - exp = 31; - man = rate; - - // invariant: rate = man*2^(exp-31) - while (!(man & (1<<31))) { - exp = exp - 1; - man = man<<1; - } - - // man has top bit set - // rate = (2^31+(man-2^31))*2^(exp-31) - // rate = (1+(man-2^31)/2^31)*2^exp - man = man<<1; - man &= 0xffffffffU; // a nop on 32-bit systems - // rate = (1+man/2^32)*2^exp - - // exp is in the range 0 to 31, man is in the range 0 to 2^32-1 - // time to lose significance... we want m in the range 0 to 2^9-1 - // rounding presents a minor problem... we first decide which way - // we are rounding (based on given rounding direction and possibly - // the bits of the mantissa that are to be discarded). - - switch (r) { - case round_down: { - // just truncate - man = man>>(32-9); - break; - } - case round_up: { - // check all bits that we are discarding - if (man & (~0U>>9)) { - man = (man>>(32-9)) + 1; - if (man == (1<<9)) { - // no need to check for round up outside of range - man = 0; - exp += 1; - } - } else { - man = (man>>(32-9)); - } - break; - } - case round_nearest: { - // check msb that we are discarding - if (man & (1<<(32-9-1))) { - man = (man>>(32-9)) + 1; - if (man == (1<<9)) { - // no need to check for round up outside of range - man = 0; - exp += 1; - } - } else { - man = (man>>(32-9)); - } - break; - } - } - - } else { - // zero rate - not representable - - if (r == round_down) { - return -EINVAL; - } else { - exp = 0; - man = 0; - } - - } - - PRINTD (DBG_QOS, "rate: man=%u, exp=%hu", man, exp); - - if (bits) - *bits = /* (1<<14) | */ (exp<<9) | man; - - if (actual) - *actual = (exp >= 9) - ? (1 << exp) + (man << (exp-9)) - : (1 << exp) + ((man + (1<<(9-exp-1))) >> (9-exp)); - - return 0; -} - -/********** Linux ATM Operations **********/ - -// some are not yet implemented while others do not make sense for -// this device - -/********** Open a VC **********/ - -static int amb_open (struct atm_vcc * atm_vcc) -{ - int error; - - struct atm_qos * qos; - struct atm_trafprm * txtp; - struct atm_trafprm * rxtp; - u16 tx_rate_bits = -1; // hush gcc - u16 tx_vc_bits = -1; // hush gcc - u16 tx_frame_bits = -1; // hush gcc - - amb_dev * dev = AMB_DEV(atm_vcc->dev); - amb_vcc * vcc; - unsigned char pool = -1; // hush gcc - short vpi = atm_vcc->vpi; - int vci = atm_vcc->vci; - - PRINTD (DBG_FLOW|DBG_VCC, "amb_open %x %x", vpi, vci); - -#ifdef ATM_VPI_UNSPEC - // UNSPEC is deprecated, remove this code eventually - if (vpi == ATM_VPI_UNSPEC || vci == ATM_VCI_UNSPEC) { - PRINTK (KERN_WARNING, "rejecting open with unspecified VPI/VCI (deprecated)"); - return -EINVAL; - } -#endif - - if (!(0 <= vpi && vpi < (1<<NUM_VPI_BITS) && - 0 <= vci && vci < (1<<NUM_VCI_BITS))) { - PRINTD (DBG_WARN|DBG_VCC, "VPI/VCI out of range: %hd/%d", vpi, vci); - return -EINVAL; - } - - qos = &atm_vcc->qos; - - if (qos->aal != ATM_AAL5) { - PRINTD (DBG_QOS, "AAL not supported"); - return -EINVAL; - } - - // traffic parameters - - PRINTD (DBG_QOS, "TX:"); - txtp = &qos->txtp; - if (txtp->traffic_class != ATM_NONE) { - switch (txtp->traffic_class) { - case ATM_UBR: { - // we take "the PCR" as a rate-cap - int pcr = atm_pcr_goal (txtp); - if (!pcr) { - // no rate cap - tx_rate_bits = 0; - tx_vc_bits = TX_UBR; - tx_frame_bits = TX_FRAME_NOTCAP; - } else { - rounding r; - if (pcr < 0) { - r = round_down; - pcr = -pcr; - } else { - r = round_up; - } - error = make_rate (pcr, r, &tx_rate_bits, NULL); - if (error) - return error; - tx_vc_bits = TX_UBR_CAPPED; - tx_frame_bits = TX_FRAME_CAPPED; - } - break; - } -#if 0 - case ATM_ABR: { - pcr = atm_pcr_goal (txtp); - PRINTD (DBG_QOS, "pcr goal = %d", pcr); - break; - } -#endif - default: { - // PRINTD (DBG_QOS, "request for non-UBR/ABR denied"); - PRINTD (DBG_QOS, "request for non-UBR denied"); - return -EINVAL; - } - } - PRINTD (DBG_QOS, "tx_rate_bits=%hx, tx_vc_bits=%hx", - tx_rate_bits, tx_vc_bits); - } - - PRINTD (DBG_QOS, "RX:"); - rxtp = &qos->rxtp; - if (rxtp->traffic_class == ATM_NONE) { - // do nothing - } else { - // choose an RX pool (arranged in increasing size) - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - if ((unsigned int) rxtp->max_sdu <= dev->rxq[pool].buffer_size) { - PRINTD (DBG_VCC|DBG_QOS|DBG_POOL, "chose pool %hu (max_sdu %u <= %u)", - pool, rxtp->max_sdu, dev->rxq[pool].buffer_size); - break; - } - if (pool == NUM_RX_POOLS) { - PRINTD (DBG_WARN|DBG_VCC|DBG_QOS|DBG_POOL, - "no pool suitable for VC (RX max_sdu %d is too large)", - rxtp->max_sdu); - return -EINVAL; - } - - switch (rxtp->traffic_class) { - case ATM_UBR: { - break; - } -#if 0 - case ATM_ABR: { - pcr = atm_pcr_goal (rxtp); - PRINTD (DBG_QOS, "pcr goal = %d", pcr); - break; - } -#endif - default: { - // PRINTD (DBG_QOS, "request for non-UBR/ABR denied"); - PRINTD (DBG_QOS, "request for non-UBR denied"); - return -EINVAL; - } - } - } - - // get space for our vcc stuff - vcc = kmalloc (sizeof(amb_vcc), GFP_KERNEL); - if (!vcc) { - PRINTK (KERN_ERR, "out of memory!"); - return -ENOMEM; - } - atm_vcc->dev_data = (void *) vcc; - - // no failures beyond this point - - // we are not really "immediately before allocating the connection - // identifier in hardware", but it will just have to do! - set_bit(ATM_VF_ADDR,&atm_vcc->flags); - - if (txtp->traffic_class != ATM_NONE) { - command cmd; - - vcc->tx_frame_bits = tx_frame_bits; - - mutex_lock(&dev->vcc_sf); - if (dev->rxer[vci]) { - // RXer on the channel already, just modify rate... - cmd.request = cpu_to_be32 (SRB_MODIFY_VC_RATE); - cmd.args.modify_rate.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.modify_rate.rate = cpu_to_be32 (tx_rate_bits << SRB_RATE_SHIFT); - while (command_do (dev, &cmd)) - schedule(); - // ... and TX flags, preserving the RX pool - cmd.request = cpu_to_be32 (SRB_MODIFY_VC_FLAGS); - cmd.args.modify_flags.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.modify_flags.flags = cpu_to_be32 - ( (AMB_VCC(dev->rxer[vci])->rx_info.pool << SRB_POOL_SHIFT) - | (tx_vc_bits << SRB_FLAGS_SHIFT) ); - while (command_do (dev, &cmd)) - schedule(); - } else { - // no RXer on the channel, just open (with pool zero) - cmd.request = cpu_to_be32 (SRB_OPEN_VC); - cmd.args.open.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.open.flags = cpu_to_be32 (tx_vc_bits << SRB_FLAGS_SHIFT); - cmd.args.open.rate = cpu_to_be32 (tx_rate_bits << SRB_RATE_SHIFT); - while (command_do (dev, &cmd)) - schedule(); - } - dev->txer[vci].tx_present = 1; - mutex_unlock(&dev->vcc_sf); - } - - if (rxtp->traffic_class != ATM_NONE) { - command cmd; - - vcc->rx_info.pool = pool; - - mutex_lock(&dev->vcc_sf); - /* grow RX buffer pool */ - if (!dev->rxq[pool].buffers_wanted) - dev->rxq[pool].buffers_wanted = rx_lats; - dev->rxq[pool].buffers_wanted += 1; - fill_rx_pool (dev, pool, GFP_KERNEL); - - if (dev->txer[vci].tx_present) { - // TXer on the channel already - // switch (from pool zero) to this pool, preserving the TX bits - cmd.request = cpu_to_be32 (SRB_MODIFY_VC_FLAGS); - cmd.args.modify_flags.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.modify_flags.flags = cpu_to_be32 - ( (pool << SRB_POOL_SHIFT) - | (dev->txer[vci].tx_vc_bits << SRB_FLAGS_SHIFT) ); - } else { - // no TXer on the channel, open the VC (with no rate info) - cmd.request = cpu_to_be32 (SRB_OPEN_VC); - cmd.args.open.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.open.flags = cpu_to_be32 (pool << SRB_POOL_SHIFT); - cmd.args.open.rate = cpu_to_be32 (0); - } - while (command_do (dev, &cmd)) - schedule(); - // this link allows RX frames through - dev->rxer[vci] = atm_vcc; - mutex_unlock(&dev->vcc_sf); - } - - // indicate readiness - set_bit(ATM_VF_READY,&atm_vcc->flags); - - return 0; -} - -/********** Close a VC **********/ - -static void amb_close (struct atm_vcc * atm_vcc) { - amb_dev * dev = AMB_DEV (atm_vcc->dev); - amb_vcc * vcc = AMB_VCC (atm_vcc); - u16 vci = atm_vcc->vci; - - PRINTD (DBG_VCC|DBG_FLOW, "amb_close"); - - // indicate unreadiness - clear_bit(ATM_VF_READY,&atm_vcc->flags); - - // disable TXing - if (atm_vcc->qos.txtp.traffic_class != ATM_NONE) { - command cmd; - - mutex_lock(&dev->vcc_sf); - if (dev->rxer[vci]) { - // RXer still on the channel, just modify rate... XXX not really needed - cmd.request = cpu_to_be32 (SRB_MODIFY_VC_RATE); - cmd.args.modify_rate.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.modify_rate.rate = cpu_to_be32 (0); - // ... and clear TX rate flags (XXX to stop RM cell output?), preserving RX pool - } else { - // no RXer on the channel, close channel - cmd.request = cpu_to_be32 (SRB_CLOSE_VC); - cmd.args.close.vc = cpu_to_be32 (vci); // vpi 0 - } - dev->txer[vci].tx_present = 0; - while (command_do (dev, &cmd)) - schedule(); - mutex_unlock(&dev->vcc_sf); - } - - // disable RXing - if (atm_vcc->qos.rxtp.traffic_class != ATM_NONE) { - command cmd; - - // this is (the?) one reason why we need the amb_vcc struct - unsigned char pool = vcc->rx_info.pool; - - mutex_lock(&dev->vcc_sf); - if (dev->txer[vci].tx_present) { - // TXer still on the channel, just go to pool zero XXX not really needed - cmd.request = cpu_to_be32 (SRB_MODIFY_VC_FLAGS); - cmd.args.modify_flags.vc = cpu_to_be32 (vci); // vpi 0 - cmd.args.modify_flags.flags = cpu_to_be32 - (dev->txer[vci].tx_vc_bits << SRB_FLAGS_SHIFT); - } else { - // no TXer on the channel, close the VC - cmd.request = cpu_to_be32 (SRB_CLOSE_VC); - cmd.args.close.vc = cpu_to_be32 (vci); // vpi 0 - } - // forget the rxer - no more skbs will be pushed - if (atm_vcc != dev->rxer[vci]) - PRINTK (KERN_ERR, "%s vcc=%p rxer[vci]=%p", - "arghhh! we're going to die!", - vcc, dev->rxer[vci]); - dev->rxer[vci] = NULL; - while (command_do (dev, &cmd)) - schedule(); - - /* shrink RX buffer pool */ - dev->rxq[pool].buffers_wanted -= 1; - if (dev->rxq[pool].buffers_wanted == rx_lats) { - dev->rxq[pool].buffers_wanted = 0; - drain_rx_pool (dev, pool); - } - mutex_unlock(&dev->vcc_sf); - } - - // free our structure - kfree (vcc); - - // say the VPI/VCI is free again - clear_bit(ATM_VF_ADDR,&atm_vcc->flags); - - return; -} - -/********** Send **********/ - -static int amb_send (struct atm_vcc * atm_vcc, struct sk_buff * skb) { - amb_dev * dev = AMB_DEV(atm_vcc->dev); - amb_vcc * vcc = AMB_VCC(atm_vcc); - u16 vc = atm_vcc->vci; - unsigned int tx_len = skb->len; - unsigned char * tx_data = skb->data; - tx_simple * tx_descr; - tx_in tx; - - if (test_bit (dead, &dev->flags)) - return -EIO; - - PRINTD (DBG_FLOW|DBG_TX, "amb_send vc %x data %p len %u", - vc, tx_data, tx_len); - - dump_skb (">>>", vc, skb); - - if (!dev->txer[vc].tx_present) { - PRINTK (KERN_ERR, "attempt to send on RX-only VC %x", vc); - return -EBADFD; - } - - // this is a driver private field so we have to set it ourselves, - // despite the fact that we are _required_ to use it to check for a - // pop function - ATM_SKB(skb)->vcc = atm_vcc; - - if (skb->len > (size_t) atm_vcc->qos.txtp.max_sdu) { - PRINTK (KERN_ERR, "sk_buff length greater than agreed max_sdu, dropping..."); - return -EIO; - } - - if (check_area (skb->data, skb->len)) { - atomic_inc(&atm_vcc->stats->tx_err); - return -ENOMEM; // ? - } - - // allocate memory for fragments - tx_descr = kmalloc (sizeof(tx_simple), GFP_KERNEL); - if (!tx_descr) { - PRINTK (KERN_ERR, "could not allocate TX descriptor"); - return -ENOMEM; - } - if (check_area (tx_descr, sizeof(tx_simple))) { - kfree (tx_descr); - return -ENOMEM; - } - PRINTD (DBG_TX, "fragment list allocated at %p", tx_descr); - - tx_descr->skb = skb; - - tx_descr->tx_frag.bytes = cpu_to_be32 (tx_len); - tx_descr->tx_frag.address = cpu_to_be32 (virt_to_bus (tx_data)); - - tx_descr->tx_frag_end.handle = virt_to_bus (tx_descr); - tx_descr->tx_frag_end.vc = 0; - tx_descr->tx_frag_end.next_descriptor_length = 0; - tx_descr->tx_frag_end.next_descriptor = 0; -#ifdef AMB_NEW_MICROCODE - tx_descr->tx_frag_end.cpcs_uu = 0; - tx_descr->tx_frag_end.cpi = 0; - tx_descr->tx_frag_end.pad = 0; -#endif - - tx.vc = cpu_to_be16 (vcc->tx_frame_bits | vc); - tx.tx_descr_length = cpu_to_be16 (sizeof(tx_frag)+sizeof(tx_frag_end)); - tx.tx_descr_addr = cpu_to_be32 (virt_to_bus (&tx_descr->tx_frag)); - - while (tx_give (dev, &tx)) - schedule(); - return 0; -} - -/********** Change QoS on a VC **********/ - -// int amb_change_qos (struct atm_vcc * atm_vcc, struct atm_qos * qos, int flags); - -/********** Free RX Socket Buffer **********/ - -#if 0 -static void amb_free_rx_skb (struct atm_vcc * atm_vcc, struct sk_buff * skb) { - amb_dev * dev = AMB_DEV (atm_vcc->dev); - amb_vcc * vcc = AMB_VCC (atm_vcc); - unsigned char pool = vcc->rx_info.pool; - rx_in rx; - - // This may be unsafe for various reasons that I cannot really guess - // at. However, I note that the ATM layer calls kfree_skb rather - // than dev_kfree_skb at this point so we are least covered as far - // as buffer locking goes. There may be bugs if pcap clones RX skbs. - - PRINTD (DBG_FLOW|DBG_SKB, "amb_rx_free skb %p (atm_vcc %p, vcc %p)", - skb, atm_vcc, vcc); - - rx.handle = virt_to_bus (skb); - rx.host_address = cpu_to_be32 (virt_to_bus (skb->data)); - - skb->data = skb->head; - skb_reset_tail_pointer(skb); - skb->len = 0; - - if (!rx_give (dev, &rx, pool)) { - // success - PRINTD (DBG_SKB|DBG_POOL, "recycled skb for pool %hu", pool); - return; - } - - // just do what the ATM layer would have done - dev_kfree_skb_any (skb); - - return; -} -#endif - -/********** Proc File Output **********/ - -static int amb_proc_read (struct atm_dev * atm_dev, loff_t * pos, char * page) { - amb_dev * dev = AMB_DEV (atm_dev); - int left = *pos; - unsigned char pool; - - PRINTD (DBG_FLOW, "amb_proc_read"); - - /* more diagnostics here? */ - - if (!left--) { - amb_stats * s = &dev->stats; - return sprintf (page, - "frames: TX OK %lu, RX OK %lu, RX bad %lu " - "(CRC %lu, long %lu, aborted %lu, unused %lu).\n", - s->tx_ok, s->rx.ok, s->rx.error, - s->rx.badcrc, s->rx.toolong, - s->rx.aborted, s->rx.unused); - } - - if (!left--) { - amb_cq * c = &dev->cq; - return sprintf (page, "cmd queue [cur/hi/max]: %u/%u/%u. ", - c->pending, c->high, c->maximum); - } - - if (!left--) { - amb_txq * t = &dev->txq; - return sprintf (page, "TX queue [cur/max high full]: %u/%u %u %u.\n", - t->pending, t->maximum, t->high, t->filled); - } - - if (!left--) { - unsigned int count = sprintf (page, "RX queues [cur/max/req low empty]:"); - for (pool = 0; pool < NUM_RX_POOLS; ++pool) { - amb_rxq * r = &dev->rxq[pool]; - count += sprintf (page+count, " %u/%u/%u %u %u", - r->pending, r->maximum, r->buffers_wanted, r->low, r->emptied); - } - count += sprintf (page+count, ".\n"); - return count; - } - - if (!left--) { - unsigned int count = sprintf (page, "RX buffer sizes:"); - for (pool = 0; pool < NUM_RX_POOLS; ++pool) { - amb_rxq * r = &dev->rxq[pool]; - count += sprintf (page+count, " %u", r->buffer_size); - } - count += sprintf (page+count, ".\n"); - return count; - } - -#if 0 - if (!left--) { - // suni block etc? - } -#endif - - return 0; -} - -/********** Operation Structure **********/ - -static const struct atmdev_ops amb_ops = { - .open = amb_open, - .close = amb_close, - .send = amb_send, - .proc_read = amb_proc_read, - .owner = THIS_MODULE, -}; - -/********** housekeeping **********/ -static void do_housekeeping (struct timer_list *t) { - amb_dev * dev = from_timer(dev, t, housekeeping); - - // could collect device-specific (not driver/atm-linux) stats here - - // last resort refill once every ten seconds - fill_rx_pools (dev); - mod_timer(&dev->housekeeping, jiffies + 10*HZ); - - return; -} - -/********** creation of communication queues **********/ - -static int create_queues(amb_dev *dev, unsigned int cmds, unsigned int txs, - unsigned int *rxs, unsigned int *rx_buffer_sizes) -{ - unsigned char pool; - size_t total = 0; - void * memory; - void * limit; - - PRINTD (DBG_FLOW, "create_queues %p", dev); - - total += cmds * sizeof(command); - - total += txs * (sizeof(tx_in) + sizeof(tx_out)); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - total += rxs[pool] * (sizeof(rx_in) + sizeof(rx_out)); - - memory = kmalloc (total, GFP_KERNEL); - if (!memory) { - PRINTK (KERN_ERR, "could not allocate queues"); - return -ENOMEM; - } - if (check_area (memory, total)) { - PRINTK (KERN_ERR, "queues allocated in nasty area"); - kfree (memory); - return -ENOMEM; - } - - limit = memory + total; - PRINTD (DBG_INIT, "queues from %p to %p", memory, limit); - - PRINTD (DBG_CMD, "command queue at %p", memory); - - { - command * cmd = memory; - amb_cq * cq = &dev->cq; - - cq->pending = 0; - cq->high = 0; - cq->maximum = cmds - 1; - - cq->ptrs.start = cmd; - cq->ptrs.in = cmd; - cq->ptrs.out = cmd; - cq->ptrs.limit = cmd + cmds; - - memory = cq->ptrs.limit; - } - - PRINTD (DBG_TX, "TX queue pair at %p", memory); - - { - tx_in * in = memory; - tx_out * out; - amb_txq * txq = &dev->txq; - - txq->pending = 0; - txq->high = 0; - txq->filled = 0; - txq->maximum = txs - 1; - - txq->in.start = in; - txq->in.ptr = in; - txq->in.limit = in + txs; - - memory = txq->in.limit; - out = memory; - - txq->out.start = out; - txq->out.ptr = out; - txq->out.limit = out + txs; - - memory = txq->out.limit; - } - - PRINTD (DBG_RX, "RX queue pairs at %p", memory); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) { - rx_in * in = memory; - rx_out * out; - amb_rxq * rxq = &dev->rxq[pool]; - - rxq->buffer_size = rx_buffer_sizes[pool]; - rxq->buffers_wanted = 0; - - rxq->pending = 0; - rxq->low = rxs[pool] - 1; - rxq->emptied = 0; - rxq->maximum = rxs[pool] - 1; - - rxq->in.start = in; - rxq->in.ptr = in; - rxq->in.limit = in + rxs[pool]; - - memory = rxq->in.limit; - out = memory; - - rxq->out.start = out; - rxq->out.ptr = out; - rxq->out.limit = out + rxs[pool]; - - memory = rxq->out.limit; - } - - if (memory == limit) { - return 0; - } else { - PRINTK (KERN_ERR, "bad queue alloc %p != %p (tell maintainer)", memory, limit); - kfree (limit - total); - return -ENOMEM; - } - -} - -/********** destruction of communication queues **********/ - -static void destroy_queues (amb_dev * dev) { - // all queues assumed empty - void * memory = dev->cq.ptrs.start; - // includes txq.in, txq.out, rxq[].in and rxq[].out - - PRINTD (DBG_FLOW, "destroy_queues %p", dev); - - PRINTD (DBG_INIT, "freeing queues at %p", memory); - kfree (memory); - - return; -} - -/********** basic loader commands and error handling **********/ -// centisecond timeouts - guessing away here -static unsigned int command_timeouts [] = { - [host_memory_test] = 15, - [read_adapter_memory] = 2, - [write_adapter_memory] = 2, - [adapter_start] = 50, - [get_version_number] = 10, - [interrupt_host] = 1, - [flash_erase_sector] = 1, - [adap_download_block] = 1, - [adap_erase_flash] = 1, - [adap_run_in_iram] = 1, - [adap_end_download] = 1 -}; - - -static unsigned int command_successes [] = { - [host_memory_test] = COMMAND_PASSED_TEST, - [read_adapter_memory] = COMMAND_READ_DATA_OK, - [write_adapter_memory] = COMMAND_WRITE_DATA_OK, - [adapter_start] = COMMAND_COMPLETE, - [get_version_number] = COMMAND_COMPLETE, - [interrupt_host] = COMMAND_COMPLETE, - [flash_erase_sector] = COMMAND_COMPLETE, - [adap_download_block] = COMMAND_COMPLETE, - [adap_erase_flash] = COMMAND_COMPLETE, - [adap_run_in_iram] = COMMAND_COMPLETE, - [adap_end_download] = COMMAND_COMPLETE -}; - -static int decode_loader_result (loader_command cmd, u32 result) -{ - int res; - const char *msg; - - if (result == command_successes[cmd]) - return 0; - - switch (result) { - case BAD_COMMAND: - res = -EINVAL; - msg = "bad command"; - break; - case COMMAND_IN_PROGRESS: - res = -ETIMEDOUT; - msg = "command in progress"; - break; - case COMMAND_PASSED_TEST: - res = 0; - msg = "command passed test"; - break; - case COMMAND_FAILED_TEST: - res = -EIO; - msg = "command failed test"; - break; - case COMMAND_READ_DATA_OK: - res = 0; - msg = "command read data ok"; - break; - case COMMAND_READ_BAD_ADDRESS: - res = -EINVAL; - msg = "command read bad address"; - break; - case COMMAND_WRITE_DATA_OK: - res = 0; - msg = "command write data ok"; - break; - case COMMAND_WRITE_BAD_ADDRESS: - res = -EINVAL; - msg = "command write bad address"; - break; - case COMMAND_WRITE_FLASH_FAILURE: - res = -EIO; - msg = "command write flash failure"; - break; - case COMMAND_COMPLETE: - res = 0; - msg = "command complete"; - break; - case COMMAND_FLASH_ERASE_FAILURE: - res = -EIO; - msg = "command flash erase failure"; - break; - case COMMAND_WRITE_BAD_DATA: - res = -EINVAL; - msg = "command write bad data"; - break; - default: - res = -EINVAL; - msg = "unknown error"; - PRINTD (DBG_LOAD|DBG_ERR, - "decode_loader_result got %d=%x !", - result, result); - break; - } - - PRINTK (KERN_ERR, "%s", msg); - return res; -} - -static int do_loader_command(volatile loader_block *lb, const amb_dev *dev, - loader_command cmd) -{ - - unsigned long timeout; - - PRINTD (DBG_FLOW|DBG_LOAD, "do_loader_command"); - - /* do a command - - Set the return value to zero, set the command type and set the - valid entry to the right magic value. The payload is already - correctly byte-ordered so we leave it alone. Hit the doorbell - with the bus address of this structure. - - */ - - lb->result = 0; - lb->command = cpu_to_be32 (cmd); - lb->valid = cpu_to_be32 (DMA_VALID); - // dump_registers (dev); - // dump_loader_block (lb); - wr_mem (dev, offsetof(amb_mem, doorbell), virt_to_bus (lb) & ~onegigmask); - - timeout = command_timeouts[cmd] * 10; - - while (!lb->result || lb->result == cpu_to_be32 (COMMAND_IN_PROGRESS)) - if (timeout) { - timeout = msleep_interruptible(timeout); - } else { - PRINTD (DBG_LOAD|DBG_ERR, "command %d timed out", cmd); - dump_registers (dev); - dump_loader_block (lb); - return -ETIMEDOUT; - } - - if (cmd == adapter_start) { - // wait for start command to acknowledge... - timeout = 100; - while (rd_plain (dev, offsetof(amb_mem, doorbell))) - if (timeout) { - timeout = msleep_interruptible(timeout); - } else { - PRINTD (DBG_LOAD|DBG_ERR, "start command did not clear doorbell, res=%08x", - be32_to_cpu (lb->result)); - dump_registers (dev); - return -ETIMEDOUT; - } - return 0; - } else { - return decode_loader_result (cmd, be32_to_cpu (lb->result)); - } - -} - -/* loader: determine loader version */ - -static int get_loader_version(loader_block *lb, const amb_dev *dev, - u32 *version) -{ - int res; - - PRINTD (DBG_FLOW|DBG_LOAD, "get_loader_version"); - - res = do_loader_command (lb, dev, get_version_number); - if (res) - return res; - if (version) - *version = be32_to_cpu (lb->payload.version); - return 0; -} - -/* loader: write memory data blocks */ - -static int loader_write(loader_block *lb, const amb_dev *dev, - const struct ihex_binrec *rec) -{ - transfer_block * tb = &lb->payload.transfer; - - PRINTD (DBG_FLOW|DBG_LOAD, "loader_write"); - - tb->address = rec->addr; - tb->count = cpu_to_be32(be16_to_cpu(rec->len) / 4); - memcpy(tb->data, rec->data, be16_to_cpu(rec->len)); - return do_loader_command (lb, dev, write_adapter_memory); -} - -/* loader: verify memory data blocks */ - -static int loader_verify(loader_block *lb, const amb_dev *dev, - const struct ihex_binrec *rec) -{ - transfer_block * tb = &lb->payload.transfer; - int res; - - PRINTD (DBG_FLOW|DBG_LOAD, "loader_verify"); - - tb->address = rec->addr; - tb->count = cpu_to_be32(be16_to_cpu(rec->len) / 4); - res = do_loader_command (lb, dev, read_adapter_memory); - if (!res && memcmp(tb->data, rec->data, be16_to_cpu(rec->len))) - res = -EINVAL; - return res; -} - -/* loader: start microcode */ - -static int loader_start(loader_block *lb, const amb_dev *dev, u32 address) -{ - PRINTD (DBG_FLOW|DBG_LOAD, "loader_start"); - - lb->payload.start = cpu_to_be32 (address); - return do_loader_command (lb, dev, adapter_start); -} - -/********** reset card **********/ - -static inline void sf (const char * msg) -{ - PRINTK (KERN_ERR, "self-test failed: %s", msg); -} - -static int amb_reset (amb_dev * dev, int diags) { - u32 word; - - PRINTD (DBG_FLOW|DBG_LOAD, "amb_reset"); - - word = rd_plain (dev, offsetof(amb_mem, reset_control)); - // put card into reset state - wr_plain (dev, offsetof(amb_mem, reset_control), word | AMB_RESET_BITS); - // wait a short while - udelay (10); -#if 1 - // put card into known good state - wr_plain (dev, offsetof(amb_mem, interrupt_control), AMB_DOORBELL_BITS); - // clear all interrupts just in case - wr_plain (dev, offsetof(amb_mem, interrupt), -1); -#endif - // clear self-test done flag - wr_plain (dev, offsetof(amb_mem, mb.loader.ready), 0); - // take card out of reset state - wr_plain (dev, offsetof(amb_mem, reset_control), word &~ AMB_RESET_BITS); - - if (diags) { - unsigned long timeout; - // 4.2 second wait - msleep(4200); - // half second time-out - timeout = 500; - while (!rd_plain (dev, offsetof(amb_mem, mb.loader.ready))) - if (timeout) { - timeout = msleep_interruptible(timeout); - } else { - PRINTD (DBG_LOAD|DBG_ERR, "reset timed out"); - return -ETIMEDOUT; - } - - // get results of self-test - // XXX double check byte-order - word = rd_mem (dev, offsetof(amb_mem, mb.loader.result)); - if (word & SELF_TEST_FAILURE) { - if (word & GPINT_TST_FAILURE) - sf ("interrupt"); - if (word & SUNI_DATA_PATTERN_FAILURE) - sf ("SUNI data pattern"); - if (word & SUNI_DATA_BITS_FAILURE) - sf ("SUNI data bits"); - if (word & SUNI_UTOPIA_FAILURE) - sf ("SUNI UTOPIA interface"); - if (word & SUNI_FIFO_FAILURE) - sf ("SUNI cell buffer FIFO"); - if (word & SRAM_FAILURE) - sf ("bad SRAM"); - // better return value? - return -EIO; - } - - } - return 0; -} - -/********** transfer and start the microcode **********/ - -static int ucode_init(loader_block *lb, amb_dev *dev) -{ - const struct firmware *fw; - unsigned long start_address; - const struct ihex_binrec *rec; - const char *errmsg = NULL; - int res; - - res = request_ihex_firmware(&fw, "atmsar11.fw", &dev->pci_dev->dev); - if (res) { - PRINTK (KERN_ERR, "Cannot load microcode data"); - return res; - } - - /* First record contains just the start address */ - rec = (const struct ihex_binrec *)fw->data; - if (be16_to_cpu(rec->len) != sizeof(__be32) || be32_to_cpu(rec->addr)) { - errmsg = "no start record"; - goto fail; - } - start_address = be32_to_cpup((__be32 *)rec->data); - - rec = ihex_next_binrec(rec); - - PRINTD (DBG_FLOW|DBG_LOAD, "ucode_init"); - - while (rec) { - PRINTD (DBG_LOAD, "starting region (%x, %u)", be32_to_cpu(rec->addr), - be16_to_cpu(rec->len)); - if (be16_to_cpu(rec->len) > 4 * MAX_TRANSFER_DATA) { - errmsg = "record too long"; - goto fail; - } - if (be16_to_cpu(rec->len) & 3) { - errmsg = "odd number of bytes"; - goto fail; - } - res = loader_write(lb, dev, rec); - if (res) - break; - - res = loader_verify(lb, dev, rec); - if (res) - break; - rec = ihex_next_binrec(rec); - } - release_firmware(fw); - if (!res) - res = loader_start(lb, dev, start_address); - - return res; -fail: - release_firmware(fw); - PRINTK(KERN_ERR, "Bad microcode data (%s)", errmsg); - return -EINVAL; -} - -/********** give adapter parameters **********/ - -static inline __be32 bus_addr(void * addr) { - return cpu_to_be32 (virt_to_bus (addr)); -} - -static int amb_talk(amb_dev *dev) -{ - adap_talk_block a; - unsigned char pool; - unsigned long timeout; - - PRINTD (DBG_FLOW, "amb_talk %p", dev); - - a.command_start = bus_addr (dev->cq.ptrs.start); - a.command_end = bus_addr (dev->cq.ptrs.limit); - a.tx_start = bus_addr (dev->txq.in.start); - a.tx_end = bus_addr (dev->txq.in.limit); - a.txcom_start = bus_addr (dev->txq.out.start); - a.txcom_end = bus_addr (dev->txq.out.limit); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) { - // the other "a" items are set up by the adapter - a.rec_struct[pool].buffer_start = bus_addr (dev->rxq[pool].in.start); - a.rec_struct[pool].buffer_end = bus_addr (dev->rxq[pool].in.limit); - a.rec_struct[pool].rx_start = bus_addr (dev->rxq[pool].out.start); - a.rec_struct[pool].rx_end = bus_addr (dev->rxq[pool].out.limit); - a.rec_struct[pool].buffer_size = cpu_to_be32 (dev->rxq[pool].buffer_size); - } - -#ifdef AMB_NEW_MICROCODE - // disable fast PLX prefetching - a.init_flags = 0; -#endif - - // pass the structure - wr_mem (dev, offsetof(amb_mem, doorbell), virt_to_bus (&a)); - - // 2.2 second wait (must not touch doorbell during 2 second DMA test) - msleep(2200); - // give the adapter another half second? - timeout = 500; - while (rd_plain (dev, offsetof(amb_mem, doorbell))) - if (timeout) { - timeout = msleep_interruptible(timeout); - } else { - PRINTD (DBG_INIT|DBG_ERR, "adapter init timed out"); - return -ETIMEDOUT; - } - - return 0; -} - -// get microcode version -static void amb_ucode_version(amb_dev *dev) -{ - u32 major; - u32 minor; - command cmd; - cmd.request = cpu_to_be32 (SRB_GET_VERSION); - while (command_do (dev, &cmd)) { - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - } - major = be32_to_cpu (cmd.args.version.major); - minor = be32_to_cpu (cmd.args.version.minor); - PRINTK (KERN_INFO, "microcode version is %u.%u", major, minor); -} - -// get end station address -static void amb_esi(amb_dev *dev, u8 *esi) -{ - u32 lower4; - u16 upper2; - command cmd; - - cmd.request = cpu_to_be32 (SRB_GET_BIA); - while (command_do (dev, &cmd)) { - set_current_state(TASK_UNINTERRUPTIBLE); - schedule(); - } - lower4 = be32_to_cpu (cmd.args.bia.lower4); - upper2 = be32_to_cpu (cmd.args.bia.upper2); - PRINTD (DBG_LOAD, "BIA: lower4: %08x, upper2 %04x", lower4, upper2); - - if (esi) { - unsigned int i; - - PRINTDB (DBG_INIT, "ESI:"); - for (i = 0; i < ESI_LEN; ++i) { - if (i < 4) - esi[i] = bitrev8(lower4>>(8*i)); - else - esi[i] = bitrev8(upper2>>(8*(i-4))); - PRINTDM (DBG_INIT, " %02x", esi[i]); - } - - PRINTDE (DBG_INIT, ""); - } - - return; -} - -static void fixup_plx_window (amb_dev *dev, loader_block *lb) -{ - // fix up the PLX-mapped window base address to match the block - unsigned long blb; - u32 mapreg; - blb = virt_to_bus(lb); - // the kernel stack had better not ever cross a 1Gb boundary! - mapreg = rd_plain (dev, offsetof(amb_mem, stuff[10])); - mapreg &= ~onegigmask; - mapreg |= blb & onegigmask; - wr_plain (dev, offsetof(amb_mem, stuff[10]), mapreg); - return; -} - -static int amb_init(amb_dev *dev) -{ - loader_block lb; - - u32 version; - - if (amb_reset (dev, 1)) { - PRINTK (KERN_ERR, "card reset failed!"); - } else { - fixup_plx_window (dev, &lb); - - if (get_loader_version (&lb, dev, &version)) { - PRINTK (KERN_INFO, "failed to get loader version"); - } else { - PRINTK (KERN_INFO, "loader version is %08x", version); - - if (ucode_init (&lb, dev)) { - PRINTK (KERN_ERR, "microcode failure"); - } else if (create_queues (dev, cmds, txs, rxs, rxs_bs)) { - PRINTK (KERN_ERR, "failed to get memory for queues"); - } else { - - if (amb_talk (dev)) { - PRINTK (KERN_ERR, "adapter did not accept queues"); - } else { - - amb_ucode_version (dev); - return 0; - - } /* amb_talk */ - - destroy_queues (dev); - } /* create_queues, ucode_init */ - - amb_reset (dev, 0); - } /* get_loader_version */ - - } /* amb_reset */ - - return -EINVAL; -} - -static void setup_dev(amb_dev *dev, struct pci_dev *pci_dev) -{ - unsigned char pool; - - // set up known dev items straight away - dev->pci_dev = pci_dev; - pci_set_drvdata(pci_dev, dev); - - dev->iobase = pci_resource_start (pci_dev, 1); - dev->irq = pci_dev->irq; - dev->membase = bus_to_virt(pci_resource_start(pci_dev, 0)); - - // flags (currently only dead) - dev->flags = 0; - - // Allocate cell rates (fibre) - // ATM_OC3_PCR = 1555200000/8/270*260/53 - 29/53 - // to be really pedantic, this should be ATM_OC3c_PCR - dev->tx_avail = ATM_OC3_PCR; - dev->rx_avail = ATM_OC3_PCR; - - // semaphore for txer/rxer modifications - we cannot use a - // spinlock as the critical region needs to switch processes - mutex_init(&dev->vcc_sf); - // queue manipulation spinlocks; we want atomic reads and - // writes to the queue descriptors (handles IRQ and SMP) - // consider replacing "int pending" -> "atomic_t available" - // => problem related to who gets to move queue pointers - spin_lock_init (&dev->cq.lock); - spin_lock_init (&dev->txq.lock); - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - spin_lock_init (&dev->rxq[pool].lock); -} - -static void setup_pci_dev(struct pci_dev *pci_dev) -{ - unsigned char lat; - - // enable bus master accesses - pci_set_master(pci_dev); - - // frobnicate latency (upwards, usually) - pci_read_config_byte (pci_dev, PCI_LATENCY_TIMER, &lat); - - if (!pci_lat) - pci_lat = (lat < MIN_PCI_LATENCY) ? MIN_PCI_LATENCY : lat; - - if (lat != pci_lat) { - PRINTK (KERN_INFO, "Changing PCI latency timer from %hu to %hu", - lat, pci_lat); - pci_write_config_byte(pci_dev, PCI_LATENCY_TIMER, pci_lat); - } -} - -static int amb_probe(struct pci_dev *pci_dev, - const struct pci_device_id *pci_ent) -{ - amb_dev * dev; - int err; - unsigned int irq; - - err = pci_enable_device(pci_dev); - if (err < 0) { - PRINTK (KERN_ERR, "skipped broken (PLX rev 2) card"); - goto out; - } - - // read resources from PCI configuration space - irq = pci_dev->irq; - - if (pci_dev->device == PCI_DEVICE_ID_MADGE_AMBASSADOR_BAD) { - PRINTK (KERN_ERR, "skipped broken (PLX rev 2) card"); - err = -EINVAL; - goto out_disable; - } - - PRINTD (DBG_INFO, "found Madge ATM adapter (amb) at" - " IO %llx, IRQ %u, MEM %p", - (unsigned long long)pci_resource_start(pci_dev, 1), - irq, bus_to_virt(pci_resource_start(pci_dev, 0))); - - // check IO region - err = pci_request_region(pci_dev, 1, DEV_LABEL); - if (err < 0) { - PRINTK (KERN_ERR, "IO range already in use!"); - goto out_disable; - } - - dev = kzalloc(sizeof(amb_dev), GFP_KERNEL); - if (!dev) { - PRINTK (KERN_ERR, "out of memory!"); - err = -ENOMEM; - goto out_release; - } - - setup_dev(dev, pci_dev); - - err = amb_init(dev); - if (err < 0) { - PRINTK (KERN_ERR, "adapter initialisation failure"); - goto out_free; - } - - setup_pci_dev(pci_dev); - - // grab (but share) IRQ and install handler - err = request_irq(irq, interrupt_handler, IRQF_SHARED, DEV_LABEL, dev); - if (err < 0) { - PRINTK (KERN_ERR, "request IRQ failed!"); - goto out_reset; - } - - dev->atm_dev = atm_dev_register (DEV_LABEL, &pci_dev->dev, &amb_ops, -1, - NULL); - if (!dev->atm_dev) { - PRINTD (DBG_ERR, "failed to register Madge ATM adapter"); - err = -EINVAL; - goto out_free_irq; - } - - PRINTD (DBG_INFO, "registered Madge ATM adapter (no. %d) (%p) at %p", - dev->atm_dev->number, dev, dev->atm_dev); - dev->atm_dev->dev_data = (void *) dev; - - // register our address - amb_esi (dev, dev->atm_dev->esi); - - // 0 bits for vpi, 10 bits for vci - dev->atm_dev->ci_range.vpi_bits = NUM_VPI_BITS; - dev->atm_dev->ci_range.vci_bits = NUM_VCI_BITS; - - timer_setup(&dev->housekeeping, do_housekeeping, 0); - mod_timer(&dev->housekeeping, jiffies); - - // enable host interrupts - interrupts_on (dev); - -out: - return err; - -out_free_irq: - free_irq(irq, dev); -out_reset: - amb_reset(dev, 0); -out_free: - kfree(dev); -out_release: - pci_release_region(pci_dev, 1); -out_disable: - pci_disable_device(pci_dev); - goto out; -} - - -static void amb_remove_one(struct pci_dev *pci_dev) -{ - struct amb_dev *dev; - - dev = pci_get_drvdata(pci_dev); - - PRINTD(DBG_INFO|DBG_INIT, "closing %p (atm_dev = %p)", dev, dev->atm_dev); - del_timer_sync(&dev->housekeeping); - // the drain should not be necessary - drain_rx_pools(dev); - interrupts_off(dev); - amb_reset(dev, 0); - free_irq(dev->irq, dev); - pci_disable_device(pci_dev); - destroy_queues(dev); - atm_dev_deregister(dev->atm_dev); - kfree(dev); - pci_release_region(pci_dev, 1); -} - -static void __init amb_check_args (void) { - unsigned char pool; - unsigned int max_rx_size; - -#ifdef DEBUG_AMBASSADOR - PRINTK (KERN_NOTICE, "debug bitmap is %hx", debug &= DBG_MASK); -#else - if (debug) - PRINTK (KERN_NOTICE, "no debugging support"); -#endif - - if (cmds < MIN_QUEUE_SIZE) - PRINTK (KERN_NOTICE, "cmds has been raised to %u", - cmds = MIN_QUEUE_SIZE); - - if (txs < MIN_QUEUE_SIZE) - PRINTK (KERN_NOTICE, "txs has been raised to %u", - txs = MIN_QUEUE_SIZE); - - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - if (rxs[pool] < MIN_QUEUE_SIZE) - PRINTK (KERN_NOTICE, "rxs[%hu] has been raised to %u", - pool, rxs[pool] = MIN_QUEUE_SIZE); - - // buffers sizes should be greater than zero and strictly increasing - max_rx_size = 0; - for (pool = 0; pool < NUM_RX_POOLS; ++pool) - if (rxs_bs[pool] <= max_rx_size) - PRINTK (KERN_NOTICE, "useless pool (rxs_bs[%hu] = %u)", - pool, rxs_bs[pool]); - else - max_rx_size = rxs_bs[pool]; - - if (rx_lats < MIN_RX_BUFFERS) - PRINTK (KERN_NOTICE, "rx_lats has been raised to %u", - rx_lats = MIN_RX_BUFFERS); - - return; -} - -/********** module stuff **********/ - -MODULE_AUTHOR(maintainer_string); -MODULE_DESCRIPTION(description_string); -MODULE_LICENSE("GPL"); -MODULE_FIRMWARE("atmsar11.fw"); -module_param(debug, ushort, 0644); -module_param(cmds, uint, 0); -module_param(txs, uint, 0); -module_param_array(rxs, uint, NULL, 0); -module_param_array(rxs_bs, uint, NULL, 0); -module_param(rx_lats, uint, 0); -module_param(pci_lat, byte, 0); -MODULE_PARM_DESC(debug, "debug bitmap, see .h file"); -MODULE_PARM_DESC(cmds, "number of command queue entries"); -MODULE_PARM_DESC(txs, "number of TX queue entries"); -MODULE_PARM_DESC(rxs, "number of RX queue entries [" __MODULE_STRING(NUM_RX_POOLS) "]"); -MODULE_PARM_DESC(rxs_bs, "size of RX buffers [" __MODULE_STRING(NUM_RX_POOLS) "]"); -MODULE_PARM_DESC(rx_lats, "number of extra buffers to cope with RX latencies"); -MODULE_PARM_DESC(pci_lat, "PCI latency in bus cycles"); - -/********** module entry **********/ - -static const struct pci_device_id amb_pci_tbl[] = { - { PCI_VDEVICE(MADGE, PCI_DEVICE_ID_MADGE_AMBASSADOR), 0 }, - { PCI_VDEVICE(MADGE, PCI_DEVICE_ID_MADGE_AMBASSADOR_BAD), 0 }, - { 0, } -}; - -MODULE_DEVICE_TABLE(pci, amb_pci_tbl); - -static struct pci_driver amb_driver = { - .name = "amb", - .probe = amb_probe, - .remove = amb_remove_one, - .id_table = amb_pci_tbl, -}; - -static int __init amb_module_init (void) -{ - PRINTD (DBG_FLOW|DBG_INIT, "init_module"); - - BUILD_BUG_ON(sizeof(amb_mem) != 4*16 + 4*12); - - show_version(); - - amb_check_args(); - - // get the juice - return pci_register_driver(&amb_driver); -} - -/********** module exit **********/ - -static void __exit amb_module_exit (void) -{ - PRINTD (DBG_FLOW|DBG_INIT, "cleanup_module"); - - pci_unregister_driver(&amb_driver); -} - -module_init(amb_module_init); -module_exit(amb_module_exit); diff --git a/drivers/atm/ambassador.h b/drivers/atm/ambassador.h deleted file mode 100644 index 086ceb8568dc..000000000000 --- a/drivers/atm/ambassador.h +++ /dev/null @@ -1,648 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - Madge Ambassador ATM Adapter driver. - Copyright (C) 1995-1999 Madge Networks Ltd. - -*/ - -#ifndef AMBASSADOR_H -#define AMBASSADOR_H - - -#ifdef CONFIG_ATM_AMBASSADOR_DEBUG -#define DEBUG_AMBASSADOR -#endif - -#define DEV_LABEL "amb" - -#ifndef PCI_VENDOR_ID_MADGE -#define PCI_VENDOR_ID_MADGE 0x10B6 -#endif -#ifndef PCI_VENDOR_ID_MADGE_AMBASSADOR -#define PCI_DEVICE_ID_MADGE_AMBASSADOR 0x1001 -#endif -#ifndef PCI_VENDOR_ID_MADGE_AMBASSADOR_BAD -#define PCI_DEVICE_ID_MADGE_AMBASSADOR_BAD 0x1002 -#endif - -// diagnostic output - -#define PRINTK(severity,format,args...) \ - printk(severity DEV_LABEL ": " format "\n" , ## args) - -#ifdef DEBUG_AMBASSADOR - -#define DBG_ERR 0x0001 -#define DBG_WARN 0x0002 -#define DBG_INFO 0x0004 -#define DBG_INIT 0x0008 -#define DBG_LOAD 0x0010 -#define DBG_VCC 0x0020 -#define DBG_QOS 0x0040 -#define DBG_CMD 0x0080 -#define DBG_TX 0x0100 -#define DBG_RX 0x0200 -#define DBG_SKB 0x0400 -#define DBG_POOL 0x0800 -#define DBG_IRQ 0x1000 -#define DBG_FLOW 0x2000 -#define DBG_REGS 0x4000 -#define DBG_DATA 0x8000 -#define DBG_MASK 0xffff - -/* the ## prevents the annoying double expansion of the macro arguments */ -/* KERN_INFO is used since KERN_DEBUG often does not make it to the console */ -#define PRINTDB(bits,format,args...) \ - ( (debug & (bits)) ? printk (KERN_INFO DEV_LABEL ": " format , ## args) : 1 ) -#define PRINTDM(bits,format,args...) \ - ( (debug & (bits)) ? printk (format , ## args) : 1 ) -#define PRINTDE(bits,format,args...) \ - ( (debug & (bits)) ? printk (format "\n" , ## args) : 1 ) -#define PRINTD(bits,format,args...) \ - ( (debug & (bits)) ? printk (KERN_INFO DEV_LABEL ": " format "\n" , ## args) : 1 ) - -#else - -#define PRINTD(bits,format,args...) -#define PRINTDB(bits,format,args...) -#define PRINTDM(bits,format,args...) -#define PRINTDE(bits,format,args...) - -#endif - -#define PRINTDD(bits,format,args...) -#define PRINTDDB(sec,fmt,args...) -#define PRINTDDM(sec,fmt,args...) -#define PRINTDDE(sec,fmt,args...) - -// tunable values (?) - -/* MUST be powers of two -- why ? */ -#define COM_Q_ENTRIES 8 -#define TX_Q_ENTRIES 32 -#define RX_Q_ENTRIES 64 - -// fixed values - -// guessing -#define AMB_EXTENT 0x80 - -// Minimum allowed size for an Ambassador queue -#define MIN_QUEUE_SIZE 2 - -// Ambassador microcode allows 1 to 4 pools, we use 4 (simpler) -#define NUM_RX_POOLS 4 - -// minimum RX buffers required to cope with replenishing delay -#define MIN_RX_BUFFERS 1 - -// minimum PCI latency we will tolerate (32 IS TOO SMALL) -#define MIN_PCI_LATENCY 64 // 255 - -// VCs supported by card (VPI always 0) -#define NUM_VPI_BITS 0 -#define NUM_VCI_BITS 10 -#define NUM_VCS 1024 - -/* The status field bits defined so far. */ -#define RX_ERR 0x8000 // always present if there is an error (hmm) -#define CRC_ERR 0x4000 // AAL5 CRC error -#define LEN_ERR 0x2000 // overlength frame -#define ABORT_ERR 0x1000 // zero length field in received frame -#define UNUSED_ERR 0x0800 // buffer returned unused - -// Adaptor commands - -#define SRB_OPEN_VC 0 -/* par_0: dwordswap(VC_number) */ -/* par_1: dwordswap(flags<<16) or wordswap(flags)*/ -/* flags: */ - -/* LANE: 0x0004 */ -/* NOT_UBR: 0x0008 */ -/* ABR: 0x0010 */ - -/* RxPool0: 0x0000 */ -/* RxPool1: 0x0020 */ -/* RxPool2: 0x0040 */ -/* RxPool3: 0x0060 */ - -/* par_2: dwordswap(fp_rate<<16) or wordswap(fp_rate) */ - -#define SRB_CLOSE_VC 1 -/* par_0: dwordswap(VC_number) */ - -#define SRB_GET_BIA 2 -/* returns */ -/* par_0: dwordswap(half BIA) */ -/* par_1: dwordswap(half BIA) */ - -#define SRB_GET_SUNI_STATS 3 -/* par_0: dwordswap(physical_host_address) */ - -#define SRB_SET_BITS_8 4 -#define SRB_SET_BITS_16 5 -#define SRB_SET_BITS_32 6 -#define SRB_CLEAR_BITS_8 7 -#define SRB_CLEAR_BITS_16 8 -#define SRB_CLEAR_BITS_32 9 -/* par_0: dwordswap(ATMizer address) */ -/* par_1: dwordswap(mask) */ - -#define SRB_SET_8 10 -#define SRB_SET_16 11 -#define SRB_SET_32 12 -/* par_0: dwordswap(ATMizer address) */ -/* par_1: dwordswap(data) */ - -#define SRB_GET_32 13 -/* par_0: dwordswap(ATMizer address) */ -/* returns */ -/* par_1: dwordswap(ATMizer data) */ - -#define SRB_GET_VERSION 14 -/* returns */ -/* par_0: dwordswap(Major Version) */ -/* par_1: dwordswap(Minor Version) */ - -#define SRB_FLUSH_BUFFER_Q 15 -/* Only flags to define which buffer pool; all others must be zero */ -/* par_0: dwordswap(flags<<16) or wordswap(flags)*/ - -#define SRB_GET_DMA_SPEEDS 16 -/* returns */ -/* par_0: dwordswap(Read speed (bytes/sec)) */ -/* par_1: dwordswap(Write speed (bytes/sec)) */ - -#define SRB_MODIFY_VC_RATE 17 -/* par_0: dwordswap(VC_number) */ -/* par_1: dwordswap(fp_rate<<16) or wordswap(fp_rate) */ - -#define SRB_MODIFY_VC_FLAGS 18 -/* par_0: dwordswap(VC_number) */ -/* par_1: dwordswap(flags<<16) or wordswap(flags)*/ - -/* flags: */ - -/* LANE: 0x0004 */ -/* NOT_UBR: 0x0008 */ -/* ABR: 0x0010 */ - -/* RxPool0: 0x0000 */ -/* RxPool1: 0x0020 */ -/* RxPool2: 0x0040 */ -/* RxPool3: 0x0060 */ - -#define SRB_RATE_SHIFT 16 -#define SRB_POOL_SHIFT (SRB_FLAGS_SHIFT+5) -#define SRB_FLAGS_SHIFT 16 - -#define SRB_STOP_TASKING 19 -#define SRB_START_TASKING 20 -#define SRB_SHUT_DOWN 21 -#define MAX_SRB 21 - -#define SRB_COMPLETE 0xffffffff - -#define TX_FRAME 0x80000000 - -// number of types of SRB MUST be a power of two -- why? -#define NUM_OF_SRB 32 - -// number of bits of period info for rate -#define MAX_RATE_BITS 6 - -#define TX_UBR 0x0000 -#define TX_UBR_CAPPED 0x0008 -#define TX_ABR 0x0018 -#define TX_FRAME_NOTCAP 0x0000 -#define TX_FRAME_CAPPED 0x8000 - -#define FP_155_RATE 0x24b1 -#define FP_25_RATE 0x1f9d - -/* #define VERSION_NUMBER 0x01000000 // initial release */ -/* #define VERSION_NUMBER 0x01010000 // fixed startup probs PLX MB0 not cleared */ -/* #define VERSION_NUMBER 0x01020000 // changed SUNI reset timings; allowed r/w onchip */ - -/* #define VERSION_NUMBER 0x01030000 // clear local doorbell int reg on reset */ -/* #define VERSION_NUMBER 0x01040000 // PLX bug work around version PLUS */ -/* remove race conditions on basic interface */ -/* indicate to the host that diagnostics */ -/* have finished; if failed, how and what */ -/* failed */ -/* fix host memory test to fix PLX bug */ -/* allow flash upgrade and BIA upgrade directly */ -/* */ -#define VERSION_NUMBER 0x01050025 /* Jason's first hacked version. */ -/* Change in download algorithm */ - -#define DMA_VALID 0xb728e149 /* completely random */ - -#define FLASH_BASE 0xa0c00000 -#define FLASH_SIZE 0x00020000 /* 128K */ -#define BIA_BASE (FLASH_BASE+0x0001c000) /* Flash Sector 7 */ -#define BIA_ADDRESS ((void *)0xa0c1c000) -#define PLX_BASE 0xe0000000 - -typedef enum { - host_memory_test = 1, - read_adapter_memory, - write_adapter_memory, - adapter_start, - get_version_number, - interrupt_host, - flash_erase_sector, - adap_download_block = 0x20, - adap_erase_flash, - adap_run_in_iram, - adap_end_download -} loader_command; - -#define BAD_COMMAND (-1) -#define COMMAND_IN_PROGRESS 1 -#define COMMAND_PASSED_TEST 2 -#define COMMAND_FAILED_TEST 3 -#define COMMAND_READ_DATA_OK 4 -#define COMMAND_READ_BAD_ADDRESS 5 -#define COMMAND_WRITE_DATA_OK 6 -#define COMMAND_WRITE_BAD_ADDRESS 7 -#define COMMAND_WRITE_FLASH_FAILURE 8 -#define COMMAND_COMPLETE 9 -#define COMMAND_FLASH_ERASE_FAILURE 10 -#define COMMAND_WRITE_BAD_DATA 11 - -/* bit fields for mailbox[0] return values */ - -#define GPINT_TST_FAILURE 0x00000001 -#define SUNI_DATA_PATTERN_FAILURE 0x00000002 -#define SUNI_DATA_BITS_FAILURE 0x00000004 -#define SUNI_UTOPIA_FAILURE 0x00000008 -#define SUNI_FIFO_FAILURE 0x00000010 -#define SRAM_FAILURE 0x00000020 -#define SELF_TEST_FAILURE 0x0000003f - -/* mailbox[1] = 0 in progress, -1 on completion */ -/* mailbox[2] = current test 00 00 test(8 bit) phase(8 bit) */ -/* mailbox[3] = last failure, 00 00 test(8 bit) phase(8 bit) */ -/* mailbox[4],mailbox[5],mailbox[6] random failure values */ - -/* PLX/etc. memory map including command structure */ - -/* These registers may also be memory mapped in PCI memory */ - -#define UNUSED_LOADER_MAILBOXES 6 - -typedef struct { - u32 stuff[16]; - union { - struct { - u32 result; - u32 ready; - u32 stuff[UNUSED_LOADER_MAILBOXES]; - } loader; - struct { - u32 cmd_address; - u32 tx_address; - u32 rx_address[NUM_RX_POOLS]; - u32 gen_counter; - u32 spare; - } adapter; - } mb; - u32 doorbell; - u32 interrupt; - u32 interrupt_control; - u32 reset_control; -} amb_mem; - -/* RESET bit, IRQ (card to host) and doorbell (host to card) enable bits */ -#define AMB_RESET_BITS 0x40000000 -#define AMB_INTERRUPT_BITS 0x00000300 -#define AMB_DOORBELL_BITS 0x00030000 - -/* loader commands */ - -#define MAX_COMMAND_DATA 13 -#define MAX_TRANSFER_DATA 11 - -typedef struct { - __be32 address; - __be32 count; - __be32 data[MAX_TRANSFER_DATA]; -} transfer_block; - -typedef struct { - __be32 result; - __be32 command; - union { - transfer_block transfer; - __be32 version; - __be32 start; - __be32 data[MAX_COMMAND_DATA]; - } payload; - __be32 valid; -} loader_block; - -/* command queue */ - -/* Again all data are BIG ENDIAN */ - -typedef struct { - union { - struct { - __be32 vc; - __be32 flags; - __be32 rate; - } open; - struct { - __be32 vc; - __be32 rate; - } modify_rate; - struct { - __be32 vc; - __be32 flags; - } modify_flags; - struct { - __be32 vc; - } close; - struct { - __be32 lower4; - __be32 upper2; - } bia; - struct { - __be32 address; - } suni; - struct { - __be32 major; - __be32 minor; - } version; - struct { - __be32 read; - __be32 write; - } speed; - struct { - __be32 flags; - } flush; - struct { - __be32 address; - __be32 data; - } memory; - __be32 par[3]; - } args; - __be32 request; -} command; - -/* transmit queues and associated structures */ - -/* The hosts transmit structure. All BIG ENDIAN; host address - restricted to first 1GByte, but address passed to the card must - have the top MS bit or'ed in. -- check this */ - -/* TX is described by 1+ tx_frags followed by a tx_frag_end */ - -typedef struct { - __be32 bytes; - __be32 address; -} tx_frag; - -/* apart from handle the fields here are for the adapter to play with - and should be set to zero */ - -typedef struct { - u32 handle; - u16 vc; - u16 next_descriptor_length; - u32 next_descriptor; -#ifdef AMB_NEW_MICROCODE - u8 cpcs_uu; - u8 cpi; - u16 pad; -#endif -} tx_frag_end; - -typedef struct { - tx_frag tx_frag; - tx_frag_end tx_frag_end; - struct sk_buff * skb; -} tx_simple; - -#if 0 -typedef union { - tx_frag fragment; - tx_frag_end end_of_list; -} tx_descr; -#endif - -/* this "points" to the sequence of fragments and trailer */ - -typedef struct { - __be16 vc; - __be16 tx_descr_length; - __be32 tx_descr_addr; -} tx_in; - -/* handle is the handle from tx_in */ - -typedef struct { - u32 handle; -} tx_out; - -/* receive frame structure */ - -/* All BIG ENDIAN; handle is as passed from host; length is zero for - aborted frames, and frames with errors. Header is actually VC - number, lec-id is NOT yet supported. */ - -typedef struct { - u32 handle; - __be16 vc; - __be16 lec_id; // unused - __be16 status; - __be16 length; -} rx_out; - -/* buffer supply structure */ - -typedef struct { - u32 handle; - __be32 host_address; -} rx_in; - -/* This first structure is the area in host memory where the adapter - writes its pointer values. These pointer values are BIG ENDIAN and - reside in the same 4MB 'page' as this structure. The host gives the - adapter the address of this block by sending a doorbell interrupt - to the adapter after downloading the code and setting it going. The - addresses have the top 10 bits set to 1010000010b -- really? - - The host must initialise these before handing the block to the - adapter. */ - -typedef struct { - __be32 command_start; /* SRB commands completions */ - __be32 command_end; /* SRB commands completions */ - __be32 tx_start; - __be32 tx_end; - __be32 txcom_start; /* tx completions */ - __be32 txcom_end; /* tx completions */ - struct { - __be32 buffer_start; - __be32 buffer_end; - u32 buffer_q_get; - u32 buffer_q_end; - u32 buffer_aptr; - __be32 rx_start; /* rx completions */ - __be32 rx_end; - u32 rx_ptr; - __be32 buffer_size; /* size of host buffer */ - } rec_struct[NUM_RX_POOLS]; -#ifdef AMB_NEW_MICROCODE - u16 init_flags; - u16 talk_block_spare; -#endif -} adap_talk_block; - -/* This structure must be kept in line with the vcr image in sarmain.h - - This is the structure in the host filled in by the adapter by - GET_SUNI_STATS */ - -typedef struct { - u8 racp_chcs; - u8 racp_uhcs; - u16 spare; - u32 racp_rcell; - u32 tacp_tcell; - u32 flags; - u32 dropped_cells; - u32 dropped_frames; -} suni_stats; - -typedef enum { - dead -} amb_flags; - -#define NEXTQ(current,start,limit) \ - ( (current)+1 < (limit) ? (current)+1 : (start) ) - -typedef struct { - command * start; - command * in; - command * out; - command * limit; -} amb_cq_ptrs; - -typedef struct { - spinlock_t lock; - unsigned int pending; - unsigned int high; - unsigned int filled; - unsigned int maximum; // size - 1 (q implementation) - amb_cq_ptrs ptrs; -} amb_cq; - -typedef struct { - spinlock_t lock; - unsigned int pending; - unsigned int high; - unsigned int filled; - unsigned int maximum; // size - 1 (q implementation) - struct { - tx_in * start; - tx_in * ptr; - tx_in * limit; - } in; - struct { - tx_out * start; - tx_out * ptr; - tx_out * limit; - } out; -} amb_txq; - -typedef struct { - spinlock_t lock; - unsigned int pending; - unsigned int low; - unsigned int emptied; - unsigned int maximum; // size - 1 (q implementation) - struct { - rx_in * start; - rx_in * ptr; - rx_in * limit; - } in; - struct { - rx_out * start; - rx_out * ptr; - rx_out * limit; - } out; - unsigned int buffers_wanted; - unsigned int buffer_size; -} amb_rxq; - -typedef struct { - unsigned long tx_ok; - struct { - unsigned long ok; - unsigned long error; - unsigned long badcrc; - unsigned long toolong; - unsigned long aborted; - unsigned long unused; - } rx; -} amb_stats; - -// a single struct pointed to by atm_vcc->dev_data - -typedef struct { - u8 tx_vc_bits:7; - u8 tx_present:1; -} amb_tx_info; - -typedef struct { - unsigned char pool; -} amb_rx_info; - -typedef struct { - amb_rx_info rx_info; - u16 tx_frame_bits; - unsigned int tx_rate; - unsigned int rx_rate; -} amb_vcc; - -struct amb_dev { - u8 irq; - unsigned long flags; - u32 iobase; - u32 * membase; - - amb_cq cq; - amb_txq txq; - amb_rxq rxq[NUM_RX_POOLS]; - - struct mutex vcc_sf; - amb_tx_info txer[NUM_VCS]; - struct atm_vcc * rxer[NUM_VCS]; - unsigned int tx_avail; - unsigned int rx_avail; - - amb_stats stats; - - struct atm_dev * atm_dev; - struct pci_dev * pci_dev; - struct timer_list housekeeping; -}; - -typedef struct amb_dev amb_dev; - -#define AMB_DEV(atm_dev) ((amb_dev *) (atm_dev)->dev_data) -#define AMB_VCC(atm_vcc) ((amb_vcc *) (atm_vcc)->dev_data) - -/* rate rounding */ - -typedef enum { - round_up, - round_down, - round_nearest -} rounding; - -#endif diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c index 2578b2d45439..e465108d9998 100644 --- a/drivers/auxdisplay/lcd2s.c +++ b/drivers/auxdisplay/lcd2s.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* - * console driver for LCD2S 4x20 character displays connected through i2c. - * The display also has a spi interface, but the driver does not support + * Console driver for LCD2S 4x20 character displays connected through i2c. + * The display also has a SPI interface, but the driver does not support * this yet. * - * This is a driver allowing you to use a LCD2S 4x20 from modtronix + * This is a driver allowing you to use a LCD2S 4x20 from Modtronix * engineering as auxdisplay character device. * * (C) 2019 by Lemonage Software GmbH @@ -12,7 +12,9 @@ * All rights reserved. */ #include <linux/kernel.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> +#include <linux/property.h> #include <linux/slab.h> #include <linux/i2c.h> #include <linux/delay.h> @@ -104,7 +106,7 @@ static int lcd2s_print(struct charlcd *lcd, int c) static int lcd2s_gotoxy(struct charlcd *lcd, unsigned int x, unsigned int y) { struct lcd2s_data *lcd2s = lcd->drvdata; - u8 buf[] = { LCD2S_CMD_CUR_POS, y + 1, x + 1}; + u8 buf[3] = { LCD2S_CMD_CUR_POS, y + 1, x + 1 }; lcd2s_i2c_master_send(lcd2s->i2c, buf, sizeof(buf)); @@ -214,16 +216,15 @@ static int lcd2s_lines(struct charlcd *lcd, enum charlcd_lines lines) return 0; } +/* + * Generator: LGcxxxxx...xx; must have <c> between '0' and '7', + * representing the numerical ASCII code of the redefined character, + * and <xx...xx> a sequence of 16 hex digits representing 8 bytes + * for each character. Most LCDs will only use 5 lower bits of + * the 7 first bytes. + */ static int lcd2s_redefine_char(struct charlcd *lcd, char *esc) { - /* Generator : LGcxxxxx...xx; must have <c> between '0' - * and '7', representing the numerical ASCII code of the - * redefined character, and <xx...xx> a sequence of 16 - * hex digits representing 8 bytes for each character. - * Most LCDs will only use 5 lower bits of the 7 first - * bytes. - */ - struct lcd2s_data *lcd2s = lcd->drvdata; u8 buf[LCD2S_CHARACTER_SIZE + 2] = { LCD2S_CMD_DEF_CUSTOM_CHAR }; u8 value; @@ -286,8 +287,7 @@ static const struct charlcd_ops lcd2s_ops = { .redefine_char = lcd2s_redefine_char, }; -static int lcd2s_i2c_probe(struct i2c_client *i2c, - const struct i2c_device_id *id) +static int lcd2s_i2c_probe(struct i2c_client *i2c) { struct charlcd *lcd; struct lcd2s_data *lcd2s; @@ -355,43 +355,22 @@ static const struct i2c_device_id lcd2s_i2c_id[] = { }; MODULE_DEVICE_TABLE(i2c, lcd2s_i2c_id); -#ifdef CONFIG_OF static const struct of_device_id lcd2s_of_table[] = { { .compatible = "modtronix,lcd2s" }, { } }; MODULE_DEVICE_TABLE(of, lcd2s_of_table); -#endif static struct i2c_driver lcd2s_i2c_driver = { .driver = { .name = "lcd2s", -#ifdef CONFIG_OF - .of_match_table = of_match_ptr(lcd2s_of_table), -#endif + .of_match_table = lcd2s_of_table, }, - .probe = lcd2s_i2c_probe, + .probe_new = lcd2s_i2c_probe, .remove = lcd2s_i2c_remove, .id_table = lcd2s_i2c_id, }; - -static int __init lcd2s_modinit(void) -{ - int ret = 0; - - ret = i2c_add_driver(&lcd2s_i2c_driver); - if (ret != 0) - pr_err("Failed to register lcd2s driver\n"); - - return ret; -} -module_init(lcd2s_modinit) - -static void __exit lcd2s_exit(void) -{ - i2c_del_driver(&lcd2s_i2c_driver); -} -module_exit(lcd2s_exit) +module_i2c_driver(lcd2s_i2c_driver); MODULE_DESCRIPTION("LCD2S character display driver"); MODULE_AUTHOR("Lars Poeschel"); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 96881d5babd9..9676a1d214bc 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -171,7 +171,7 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr, unsigned int set_size) { struct drbd_request *r; - struct drbd_request *req = NULL; + struct drbd_request *req = NULL, *tmp = NULL; int expect_epoch = 0; int expect_size = 0; @@ -225,8 +225,11 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr, * to catch requests being barrier-acked "unexpectedly". * It usually should find the same req again, or some READ preceding it. */ list_for_each_entry(req, &connection->transfer_log, tl_requests) - if (req->epoch == expect_epoch) + if (req->epoch == expect_epoch) { + tmp = req; break; + } + req = list_prepare_entry(tmp, &connection->transfer_log, tl_requests); list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) { if (req->epoch != expect_epoch) break; diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index c04394518b07..75be0e16770a 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -180,7 +180,8 @@ void start_new_tl_epoch(struct drbd_connection *connection) void complete_master_bio(struct drbd_device *device, struct bio_and_error *m) { - m->bio->bi_status = errno_to_blk_status(m->error); + if (unlikely(m->error)) + m->bio->bi_status = errno_to_blk_status(m->error); bio_endio(m->bio); dec_ap_bio(device); } @@ -332,17 +333,21 @@ static void set_if_null_req_next(struct drbd_peer_device *peer_device, struct dr static void advance_conn_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_next != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if (s & RQ_NET_QUEUED) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if (s & RQ_NET_QUEUED) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_next = req; } @@ -358,17 +363,21 @@ static void set_if_null_req_ack_pending(struct drbd_peer_device *peer_device, st static void advance_conn_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_ack_pending != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING)) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING)) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_ack_pending = req; } @@ -384,17 +393,21 @@ static void set_if_null_req_not_net_done(struct drbd_peer_device *peer_device, s static void advance_conn_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req) { struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; + struct drbd_request *iter = req; if (!connection) return; if (connection->req_not_net_done != req) return; - list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) { - const unsigned s = req->rq_state; - if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE)) + + req = NULL; + list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { + const unsigned int s = iter->rq_state; + + if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE)) { + req = iter; break; + } } - if (&req->tl_requests == &connection->transfer_log) - req = NULL; connection->req_not_net_done = req; } diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 3e636a75c83a..a58595f5ee2c 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1591,6 +1591,7 @@ struct compat_loop_info { compat_ulong_t lo_inode; /* ioctl r/o */ compat_dev_t lo_rdevice; /* ioctl r/o */ compat_int_t lo_offset; + compat_int_t lo_encrypt_type; /* obsolete, ignored */ compat_int_t lo_encrypt_key_size; /* ioctl w/o */ compat_int_t lo_flags; /* ioctl r/o */ char lo_name[LO_NAME_SIZE]; diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c index 4db9a8c244af..e094d2b8b5a9 100644 --- a/drivers/block/n64cart.c +++ b/drivers/block/n64cart.c @@ -88,7 +88,7 @@ static void n64cart_submit_bio(struct bio *bio) { struct bio_vec bvec; struct bvec_iter iter; - struct device *dev = bio->bi_disk->private_data; + struct device *dev = bio->bi_bdev->bd_disk->private_data; u32 pos = bio->bi_iter.bi_sector << SECTOR_SHIFT; bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index d1e26461a64e..de42458195bc 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -931,7 +931,7 @@ static int xen_blkbk_parse_indirect(struct blkif_request *req, if (rc) goto unmap; - for (n = 0, i = 0; n < nseg; n++) { + for (n = 0; n < nseg; n++) { uint8_t first_sect, last_sect; if ((n % SEGS_PER_INDIRECT_FRAME) == 0) { diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 378262ec47ae..003056d4f7f5 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -576,7 +576,7 @@ struct setup_rw_req { struct blkif_request *ring_req; grant_ref_t gref_head; unsigned int id; - /* Only used when persistent grant is used and it's a read request */ + /* Only used when persistent grant is used and it's a write request */ bool need_copy; unsigned int bvec_off; char *bvec_data; diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 740811893c57..55f48375e3fe 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -449,6 +449,7 @@ config RANDOM_TRUST_BOOTLOADER device randomness. Say Y here to assume the entropy provided by the booloader is trustworthy so it will be added to the kernel's entropy pool. Otherwise, say N here so it will be regarded as device input that - only mixes the entropy pool. + only mixes the entropy pool. This can also be configured at boot with + "random.trust_bootloader=on/off". endmenu diff --git a/drivers/char/random.c b/drivers/char/random.c index 66ce7c03a142..e15063d61460 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -224,9 +224,10 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, void * * These interfaces will return the requested number of random bytes * into the given buffer or as a return value. This is equivalent to - * a read from /dev/urandom. The integer family of functions may be - * higher performance for one-off random integers, because they do a - * bit of buffering. + * a read from /dev/urandom. The u32, u64, int, and long family of + * functions may be higher performance for one-off random integers, + * because they do a bit of buffering and do not invoke reseeding + * until the buffer is emptied. * *********************************************************************/ @@ -436,11 +437,8 @@ static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS], * This shouldn't be set by functions like add_device_randomness(), * where we can't trust the buffer passed to it is guaranteed to be * unpredictable (so it might not have any entropy at all). - * - * Returns the number of bytes processed from input, which is bounded - * by CRNG_INIT_CNT_THRESH if account is true. */ -static size_t crng_pre_init_inject(const void *input, size_t len, bool account) +static void crng_pre_init_inject(const void *input, size_t len, bool account) { static int crng_init_cnt = 0; struct blake2s_state hash; @@ -451,18 +449,15 @@ static size_t crng_pre_init_inject(const void *input, size_t len, bool account) spin_lock_irqsave(&base_crng.lock, flags); if (crng_init != 0) { spin_unlock_irqrestore(&base_crng.lock, flags); - return 0; + return; } - if (account) - len = min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt); - blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); blake2s_update(&hash, input, len); blake2s_final(&hash, base_crng.key); if (account) { - crng_init_cnt += len; + crng_init_cnt += min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt); if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { ++base_crng.generation; crng_init = 1; @@ -473,8 +468,6 @@ static size_t crng_pre_init_inject(const void *input, size_t len, bool account) if (crng_init == 1) pr_notice("fast init done\n"); - - return len; } static void _get_random_bytes(void *buf, size_t nbytes) @@ -530,7 +523,6 @@ EXPORT_SYMBOL(get_random_bytes); static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes) { - bool large_request = nbytes > 256; ssize_t ret = 0; size_t len; u32 chacha_state[CHACHA_STATE_WORDS]; @@ -539,22 +531,23 @@ static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes) if (!nbytes) return 0; - len = min_t(size_t, 32, nbytes); - crng_make_state(chacha_state, output, len); - - if (copy_to_user(buf, output, len)) - return -EFAULT; - nbytes -= len; - buf += len; - ret += len; - - while (nbytes) { - if (large_request && need_resched()) { - if (signal_pending(current)) - break; - schedule(); - } + /* + * Immediately overwrite the ChaCha key at index 4 with random + * bytes, in case userspace causes copy_to_user() below to sleep + * forever, so that we still retain forward secrecy in that case. + */ + crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA_KEY_SIZE); + /* + * However, if we're doing a read of len <= 32, we don't need to + * use chacha_state after, so we can simply return those bytes to + * the user directly. + */ + if (nbytes <= CHACHA_KEY_SIZE) { + ret = copy_to_user(buf, &chacha_state[4], nbytes) ? -EFAULT : nbytes; + goto out_zero_chacha; + } + do { chacha20_block(chacha_state, output); if (unlikely(chacha_state[12] == 0)) ++chacha_state[13]; @@ -568,10 +561,18 @@ static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes) nbytes -= len; buf += len; ret += len; - } - memzero_explicit(chacha_state, sizeof(chacha_state)); + BUILD_BUG_ON(PAGE_SIZE % CHACHA_BLOCK_SIZE != 0); + if (!(ret % PAGE_SIZE) && nbytes) { + if (signal_pending(current)) + break; + cond_resched(); + } + } while (nbytes); + memzero_explicit(output, sizeof(output)); +out_zero_chacha: + memzero_explicit(chacha_state, sizeof(chacha_state)); return ret; } @@ -948,11 +949,17 @@ static bool drain_entropy(void *buf, size_t nbytes, bool force) **********************************************************************/ static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); +static bool trust_bootloader __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER); static int __init parse_trust_cpu(char *arg) { return kstrtobool(arg, &trust_cpu); } +static int __init parse_trust_bootloader(char *arg) +{ + return kstrtobool(arg, &trust_bootloader); +} early_param("random.trust_cpu", parse_trust_cpu); +early_param("random.trust_bootloader", parse_trust_bootloader); /* * The first collection of entropy occurs at system boot while interrupts @@ -968,6 +975,11 @@ int __init rand_initialize(void) bool arch_init = true; unsigned long rv; +#if defined(LATENT_ENTROPY_PLUGIN) + static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy; + _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed)); +#endif + for (i = 0; i < BLAKE2S_BLOCK_SIZE; i += sizeof(rv)) { if (!arch_get_random_seed_long_early(&rv) && !arch_get_random_long_early(&rv)) { @@ -1128,13 +1140,10 @@ void rand_initialize_disk(struct gendisk *disk) void add_hwgenerator_randomness(const void *buffer, size_t count, size_t entropy) { - if (unlikely(crng_init == 0)) { - size_t ret = crng_pre_init_inject(buffer, count, true); - mix_pool_bytes(buffer, ret); - count -= ret; - buffer += ret; - if (!count || crng_init == 0) - return; + if (unlikely(crng_init == 0 && entropy < POOL_MIN_BITS)) { + crng_pre_init_inject(buffer, count, true); + mix_pool_bytes(buffer, count); + return; } /* @@ -1160,7 +1169,7 @@ EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); */ void add_bootloader_randomness(const void *buf, size_t size) { - if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER)) + if (trust_bootloader) add_hwgenerator_randomness(buf, size, size * 8); else add_device_randomness(buf, size); @@ -1533,6 +1542,13 @@ static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, { static int maxwarn = 10; + /* + * Opportunistically attempt to initialize the RNG on platforms that + * have fast cycle counters, but don't (for now) require it to succeed. + */ + if (!crng_ready()) + try_to_generate_entropy(); + if (!crng_ready() && maxwarn > 0) { maxwarn--; if (__ratelimit(&urandom_warning)) diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 07a27b65b773..ed119182aa1b 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -2332,15 +2332,19 @@ int clk_set_rate_exclusive(struct clk *clk, unsigned long rate) } EXPORT_SYMBOL_GPL(clk_set_rate_exclusive); -static int clk_set_rate_range_nolock(struct clk *clk, - unsigned long min, - unsigned long max) +/** + * clk_set_rate_range - set a rate range for a clock source + * @clk: clock source + * @min: desired minimum clock rate in Hz, inclusive + * @max: desired maximum clock rate in Hz, inclusive + * + * Returns success (0) or negative errno. + */ +int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) { int ret = 0; unsigned long old_min, old_max, rate; - lockdep_assert_held(&prepare_lock); - if (!clk) return 0; @@ -2353,6 +2357,8 @@ static int clk_set_rate_range_nolock(struct clk *clk, return -EINVAL; } + clk_prepare_lock(); + if (clk->exclusive_count) clk_core_rate_unprotect(clk->core); @@ -2396,28 +2402,6 @@ out: if (clk->exclusive_count) clk_core_rate_protect(clk->core); - return ret; -} - -/** - * clk_set_rate_range - set a rate range for a clock source - * @clk: clock source - * @min: desired minimum clock rate in Hz, inclusive - * @max: desired maximum clock rate in Hz, inclusive - * - * Return: 0 for success or negative errno on failure. - */ -int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) -{ - int ret; - - if (!clk) - return 0; - - clk_prepare_lock(); - - ret = clk_set_rate_range_nolock(clk, min, max); - clk_prepare_unlock(); return ret; @@ -4419,7 +4403,9 @@ void __clk_put(struct clk *clk) } hlist_del(&clk->clks_node); - clk_set_rate_range_nolock(clk, 0, ULONG_MAX); + if (clk->min_rate > clk->core->req_rate || + clk->max_rate < clk->core->req_rate) + clk_core_set_rate_nolock(clk->core, clk->core->req_rate); owner = clk->core->owner; kref_put(&clk->core->ref, __clk_release); diff --git a/drivers/clk/clk_test.c b/drivers/clk/clk_test.c index fd2339cc5898..6731a822f4e3 100644 --- a/drivers/clk/clk_test.c +++ b/drivers/clk/clk_test.c @@ -760,65 +760,9 @@ static void clk_range_test_multiple_set_range_rate_maximized(struct kunit *test) clk_put(user1); } -/* - * Test that if we have several subsequent calls to - * clk_set_rate_range(), across multiple users, the core will reevaluate - * whether a new rate is needed, including when a user drop its clock. - * - * With clk_dummy_maximize_rate_ops, this means that the rate will - * trail along the maximum as it evolves. - */ -static void clk_range_test_multiple_set_range_rate_put_maximized(struct kunit *test) -{ - struct clk_dummy_context *ctx = test->priv; - struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; - struct clk *user1, *user2; - unsigned long rate; - - user1 = clk_hw_get_clk(hw, NULL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1); - - user2 = clk_hw_get_clk(hw, NULL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2); - - KUNIT_ASSERT_EQ(test, - clk_set_rate(clk, DUMMY_CLOCK_RATE_2 + 1000), - 0); - - KUNIT_ASSERT_EQ(test, - clk_set_rate_range(user1, - 0, - DUMMY_CLOCK_RATE_2), - 0); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); - - KUNIT_ASSERT_EQ(test, - clk_set_rate_range(user2, - 0, - DUMMY_CLOCK_RATE_1), - 0); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); - - clk_put(user2); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); - - clk_put(user1); -} - static struct kunit_case clk_range_maximize_test_cases[] = { KUNIT_CASE(clk_range_test_set_range_rate_maximized), KUNIT_CASE(clk_range_test_multiple_set_range_rate_maximized), - KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_maximized), {} }; @@ -933,61 +877,9 @@ static void clk_range_test_multiple_set_range_rate_minimized(struct kunit *test) clk_put(user1); } -/* - * Test that if we have several subsequent calls to - * clk_set_rate_range(), across multiple users, the core will reevaluate - * whether a new rate is needed, including when a user drop its clock. - * - * With clk_dummy_minimize_rate_ops, this means that the rate will - * trail along the minimum as it evolves. - */ -static void clk_range_test_multiple_set_range_rate_put_minimized(struct kunit *test) -{ - struct clk_dummy_context *ctx = test->priv; - struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; - struct clk *user1, *user2; - unsigned long rate; - - user1 = clk_hw_get_clk(hw, NULL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1); - - user2 = clk_hw_get_clk(hw, NULL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2); - - KUNIT_ASSERT_EQ(test, - clk_set_rate_range(user1, - DUMMY_CLOCK_RATE_1, - ULONG_MAX), - 0); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); - - KUNIT_ASSERT_EQ(test, - clk_set_rate_range(user2, - DUMMY_CLOCK_RATE_2, - ULONG_MAX), - 0); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); - - clk_put(user2); - - rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, rate, 0); - KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); - - clk_put(user1); -} - static struct kunit_case clk_range_minimize_test_cases[] = { KUNIT_CASE(clk_range_test_set_range_rate_minimized), KUNIT_CASE(clk_range_test_multiple_set_range_rate_minimized), - KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_minimized), {} }; diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index 68a94e5af8ed..461537679c04 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -69,6 +69,11 @@ config SUN6I_A31_CCU default MACH_SUN6I depends on MACH_SUN6I || COMPILE_TEST +config SUN6I_RTC_CCU + tristate "Support for the Allwinner H616/R329 RTC CCU" + default ARCH_SUNXI + depends on ARCH_SUNXI || COMPILE_TEST + config SUN8I_A23_CCU tristate "Support for the Allwinner A23 CCU" default MACH_SUN8I diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index ec931cb7aa14..6b3ae2b620db 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_SUN50I_H616_CCU) += sun50i-h616-ccu.o obj-$(CONFIG_SUN4I_A10_CCU) += sun4i-a10-ccu.o obj-$(CONFIG_SUN5I_CCU) += sun5i-ccu.o obj-$(CONFIG_SUN6I_A31_CCU) += sun6i-a31-ccu.o +obj-$(CONFIG_SUN6I_RTC_CCU) += sun6i-rtc-ccu.o obj-$(CONFIG_SUN8I_A23_CCU) += sun8i-a23-ccu.o obj-$(CONFIG_SUN8I_A33_CCU) += sun8i-a33-ccu.o obj-$(CONFIG_SUN8I_A83T_CCU) += sun8i-a83t-ccu.o @@ -60,6 +61,7 @@ sun50i-h616-ccu-y += ccu-sun50i-h616.o sun4i-a10-ccu-y += ccu-sun4i-a10.o sun5i-ccu-y += ccu-sun5i.o sun6i-a31-ccu-y += ccu-sun6i-a31.o +sun6i-rtc-ccu-y += ccu-sun6i-rtc.o sun8i-a23-ccu-y += ccu-sun8i-a23.o sun8i-a33-ccu-y += ccu-sun8i-a33.o sun8i-a83t-ccu-y += ccu-sun8i-a83t.o diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c new file mode 100644 index 000000000000..8a10bade7e0d --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright (c) 2021 Samuel Holland <samuel@sholland.org> +// + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_device.h> + +#include <linux/clk/sunxi-ng.h> + +#include "ccu_common.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mux.h" + +#include "ccu-sun6i-rtc.h" + +#define IOSC_ACCURACY 300000000 /* 30% */ +#define IOSC_RATE 16000000 + +#define LOSC_RATE 32768 +#define LOSC_RATE_SHIFT 15 + +#define LOSC_CTRL_REG 0x0 +#define LOSC_CTRL_KEY 0x16aa0000 + +#define IOSC_32K_CLK_DIV_REG 0x8 +#define IOSC_32K_CLK_DIV GENMASK(4, 0) +#define IOSC_32K_PRE_DIV 32 + +#define IOSC_CLK_CALI_REG 0xc +#define IOSC_CLK_CALI_DIV_ONES 22 +#define IOSC_CLK_CALI_EN BIT(1) +#define IOSC_CLK_CALI_SRC_SEL BIT(0) + +#define LOSC_OUT_GATING_REG 0x60 + +#define DCXO_CTRL_REG 0x160 +#define DCXO_CTRL_CLK16M_RC_EN BIT(0) + +struct sun6i_rtc_match_data { + bool have_ext_osc32k : 1; + bool have_iosc_calibration : 1; + bool rtc_32k_single_parent : 1; + const struct clk_parent_data *osc32k_fanout_parents; + u8 osc32k_fanout_nparents; +}; + +static bool have_iosc_calibration; + +static int ccu_iosc_enable(struct clk_hw *hw) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + + return ccu_gate_helper_enable(cm, DCXO_CTRL_CLK16M_RC_EN); +} + +static void ccu_iosc_disable(struct clk_hw *hw) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + + return ccu_gate_helper_disable(cm, DCXO_CTRL_CLK16M_RC_EN); +} + +static int ccu_iosc_is_enabled(struct clk_hw *hw) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + + return ccu_gate_helper_is_enabled(cm, DCXO_CTRL_CLK16M_RC_EN); +} + +static unsigned long ccu_iosc_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + + if (have_iosc_calibration) { + u32 reg = readl(cm->base + IOSC_CLK_CALI_REG); + + /* + * Recover the IOSC frequency by shifting the ones place of + * (fixed-point divider * 32768) into bit zero. + */ + if (reg & IOSC_CLK_CALI_EN) + return reg >> (IOSC_CLK_CALI_DIV_ONES - LOSC_RATE_SHIFT); + } + + return IOSC_RATE; +} + +static unsigned long ccu_iosc_recalc_accuracy(struct clk_hw *hw, + unsigned long parent_accuracy) +{ + return IOSC_ACCURACY; +} + +static const struct clk_ops ccu_iosc_ops = { + .enable = ccu_iosc_enable, + .disable = ccu_iosc_disable, + .is_enabled = ccu_iosc_is_enabled, + .recalc_rate = ccu_iosc_recalc_rate, + .recalc_accuracy = ccu_iosc_recalc_accuracy, +}; + +static struct ccu_common iosc_clk = { + .reg = DCXO_CTRL_REG, + .hw.init = CLK_HW_INIT_NO_PARENT("iosc", &ccu_iosc_ops, + CLK_GET_RATE_NOCACHE), +}; + +static int ccu_iosc_32k_prepare(struct clk_hw *hw) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + u32 val; + + if (!have_iosc_calibration) + return 0; + + val = readl(cm->base + IOSC_CLK_CALI_REG); + writel(val | IOSC_CLK_CALI_EN | IOSC_CLK_CALI_SRC_SEL, + cm->base + IOSC_CLK_CALI_REG); + + return 0; +} + +static void ccu_iosc_32k_unprepare(struct clk_hw *hw) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + u32 val; + + if (!have_iosc_calibration) + return; + + val = readl(cm->base + IOSC_CLK_CALI_REG); + writel(val & ~(IOSC_CLK_CALI_EN | IOSC_CLK_CALI_SRC_SEL), + cm->base + IOSC_CLK_CALI_REG); +} + +static unsigned long ccu_iosc_32k_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + u32 val; + + if (have_iosc_calibration) { + val = readl(cm->base + IOSC_CLK_CALI_REG); + + /* Assume the calibrated 32k clock is accurate. */ + if (val & IOSC_CLK_CALI_SRC_SEL) + return LOSC_RATE; + } + + val = readl(cm->base + IOSC_32K_CLK_DIV_REG) & IOSC_32K_CLK_DIV; + + return parent_rate / IOSC_32K_PRE_DIV / (val + 1); +} + +static unsigned long ccu_iosc_32k_recalc_accuracy(struct clk_hw *hw, + unsigned long parent_accuracy) +{ + struct ccu_common *cm = hw_to_ccu_common(hw); + u32 val; + + if (have_iosc_calibration) { + val = readl(cm->base + IOSC_CLK_CALI_REG); + + /* Assume the calibrated 32k clock is accurate. */ + if (val & IOSC_CLK_CALI_SRC_SEL) + return 0; + } + + return parent_accuracy; +} + +static const struct clk_ops ccu_iosc_32k_ops = { + .prepare = ccu_iosc_32k_prepare, + .unprepare = ccu_iosc_32k_unprepare, + .recalc_rate = ccu_iosc_32k_recalc_rate, + .recalc_accuracy = ccu_iosc_32k_recalc_accuracy, +}; + +static struct ccu_common iosc_32k_clk = { + .hw.init = CLK_HW_INIT_HW("iosc-32k", &iosc_clk.hw, + &ccu_iosc_32k_ops, + CLK_GET_RATE_NOCACHE), +}; + +static const struct clk_hw *ext_osc32k[] = { NULL }; /* updated during probe */ + +static SUNXI_CCU_GATE_HWS(ext_osc32k_gate_clk, "ext-osc32k-gate", + ext_osc32k, 0x0, BIT(4), 0); + +static const struct clk_hw *osc32k_parents[] = { + &iosc_32k_clk.hw, + &ext_osc32k_gate_clk.common.hw +}; + +static struct clk_init_data osc32k_init_data = { + .name = "osc32k", + .ops = &ccu_mux_ops, + .parent_hws = osc32k_parents, + .num_parents = ARRAY_SIZE(osc32k_parents), /* updated during probe */ +}; + +static struct ccu_mux osc32k_clk = { + .mux = _SUNXI_CCU_MUX(0, 1), + .common = { + .reg = LOSC_CTRL_REG, + .features = CCU_FEATURE_KEY_FIELD, + .hw.init = &osc32k_init_data, + }, +}; + +/* This falls back to the global name for fwnodes without a named reference. */ +static const struct clk_parent_data osc24M[] = { + { .fw_name = "hosc", .name = "osc24M" } +}; + +static struct ccu_gate osc24M_32k_clk = { + .enable = BIT(16), + .common = { + .reg = LOSC_OUT_GATING_REG, + .prediv = 750, + .features = CCU_FEATURE_ALL_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS_DATA("osc24M-32k", osc24M, + &ccu_gate_ops, 0), + }, +}; + +static const struct clk_hw *rtc_32k_parents[] = { + &osc32k_clk.common.hw, + &osc24M_32k_clk.common.hw +}; + +static struct clk_init_data rtc_32k_init_data = { + .name = "rtc-32k", + .ops = &ccu_mux_ops, + .parent_hws = rtc_32k_parents, + .num_parents = ARRAY_SIZE(rtc_32k_parents), /* updated during probe */ +}; + +static struct ccu_mux rtc_32k_clk = { + .mux = _SUNXI_CCU_MUX(1, 1), + .common = { + .reg = LOSC_CTRL_REG, + .features = CCU_FEATURE_KEY_FIELD, + .hw.init = &rtc_32k_init_data, + }, +}; + +static struct clk_init_data osc32k_fanout_init_data = { + .name = "osc32k-fanout", + .ops = &ccu_mux_ops, + /* parents are set during probe */ +}; + +static struct ccu_mux osc32k_fanout_clk = { + .enable = BIT(0), + .mux = _SUNXI_CCU_MUX(1, 2), + .common = { + .reg = LOSC_OUT_GATING_REG, + .hw.init = &osc32k_fanout_init_data, + }, +}; + +static struct ccu_common *sun6i_rtc_ccu_clks[] = { + &iosc_clk, + &iosc_32k_clk, + &ext_osc32k_gate_clk.common, + &osc32k_clk.common, + &osc24M_32k_clk.common, + &rtc_32k_clk.common, + &osc32k_fanout_clk.common, +}; + +static struct clk_hw_onecell_data sun6i_rtc_ccu_hw_clks = { + .num = CLK_NUMBER, + .hws = { + [CLK_OSC32K] = &osc32k_clk.common.hw, + [CLK_OSC32K_FANOUT] = &osc32k_fanout_clk.common.hw, + [CLK_IOSC] = &iosc_clk.hw, + [CLK_IOSC_32K] = &iosc_32k_clk.hw, + [CLK_EXT_OSC32K_GATE] = &ext_osc32k_gate_clk.common.hw, + [CLK_OSC24M_32K] = &osc24M_32k_clk.common.hw, + [CLK_RTC_32K] = &rtc_32k_clk.common.hw, + }, +}; + +static const struct sunxi_ccu_desc sun6i_rtc_ccu_desc = { + .ccu_clks = sun6i_rtc_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun6i_rtc_ccu_clks), + + .hw_clks = &sun6i_rtc_ccu_hw_clks, +}; + +static const struct clk_parent_data sun50i_h6_osc32k_fanout_parents[] = { + { .hw = &osc32k_clk.common.hw }, +}; + +static const struct clk_parent_data sun50i_h616_osc32k_fanout_parents[] = { + { .hw = &osc32k_clk.common.hw }, + { .fw_name = "pll-32k" }, + { .hw = &osc24M_32k_clk.common.hw } +}; + +static const struct clk_parent_data sun50i_r329_osc32k_fanout_parents[] = { + { .hw = &osc32k_clk.common.hw }, + { .hw = &ext_osc32k_gate_clk.common.hw }, + { .hw = &osc24M_32k_clk.common.hw } +}; + +static const struct sun6i_rtc_match_data sun50i_h6_rtc_ccu_data = { + .have_ext_osc32k = true, + .have_iosc_calibration = true, + .osc32k_fanout_parents = sun50i_h6_osc32k_fanout_parents, + .osc32k_fanout_nparents = ARRAY_SIZE(sun50i_h6_osc32k_fanout_parents), +}; + +static const struct sun6i_rtc_match_data sun50i_h616_rtc_ccu_data = { + .have_iosc_calibration = true, + .rtc_32k_single_parent = true, + .osc32k_fanout_parents = sun50i_h616_osc32k_fanout_parents, + .osc32k_fanout_nparents = ARRAY_SIZE(sun50i_h616_osc32k_fanout_parents), +}; + +static const struct sun6i_rtc_match_data sun50i_r329_rtc_ccu_data = { + .have_ext_osc32k = true, + .osc32k_fanout_parents = sun50i_r329_osc32k_fanout_parents, + .osc32k_fanout_nparents = ARRAY_SIZE(sun50i_r329_osc32k_fanout_parents), +}; + +static const struct of_device_id sun6i_rtc_ccu_match[] = { + { + .compatible = "allwinner,sun50i-h6-rtc", + .data = &sun50i_h6_rtc_ccu_data, + }, + { + .compatible = "allwinner,sun50i-h616-rtc", + .data = &sun50i_h616_rtc_ccu_data, + }, + { + .compatible = "allwinner,sun50i-r329-rtc", + .data = &sun50i_r329_rtc_ccu_data, + }, +}; + +int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg) +{ + const struct sun6i_rtc_match_data *data; + struct clk *ext_osc32k_clk = NULL; + const struct of_device_id *match; + + /* This driver is only used for newer variants of the hardware. */ + match = of_match_device(sun6i_rtc_ccu_match, dev); + if (!match) + return 0; + + data = match->data; + have_iosc_calibration = data->have_iosc_calibration; + + if (data->have_ext_osc32k) { + const char *fw_name; + + /* ext-osc32k was the only input clock in the old binding. */ + fw_name = of_property_read_bool(dev->of_node, "clock-names") + ? "ext-osc32k" : NULL; + ext_osc32k_clk = devm_clk_get_optional(dev, fw_name); + if (IS_ERR(ext_osc32k_clk)) + return PTR_ERR(ext_osc32k_clk); + } + + if (ext_osc32k_clk) { + /* Link ext-osc32k-gate to its parent. */ + *ext_osc32k = __clk_get_hw(ext_osc32k_clk); + } else { + /* ext-osc32k-gate is an orphan, so do not register it. */ + sun6i_rtc_ccu_hw_clks.hws[CLK_EXT_OSC32K_GATE] = NULL; + osc32k_init_data.num_parents = 1; + } + + if (data->rtc_32k_single_parent) + rtc_32k_init_data.num_parents = 1; + + osc32k_fanout_init_data.parent_data = data->osc32k_fanout_parents; + osc32k_fanout_init_data.num_parents = data->osc32k_fanout_nparents; + + return devm_sunxi_ccu_probe(dev, reg, &sun6i_rtc_ccu_desc); +} + +MODULE_IMPORT_NS(SUNXI_CCU); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.h b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.h new file mode 100644 index 000000000000..9ae821fc2599 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _CCU_SUN6I_RTC_H +#define _CCU_SUN6I_RTC_H + +#include <dt-bindings/clock/sun6i-rtc.h> + +#define CLK_IOSC_32K 3 +#define CLK_EXT_OSC32K_GATE 4 +#define CLK_OSC24M_32K 5 +#define CLK_RTC_32K 6 + +#define CLK_NUMBER (CLK_RTC_32K + 1) + +#endif /* _CCU_SUN6I_RTC_H */ diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h index 98a1834b58bb..fbf16c6b896d 100644 --- a/drivers/clk/sunxi-ng/ccu_common.h +++ b/drivers/clk/sunxi-ng/ccu_common.h @@ -17,6 +17,7 @@ #define CCU_FEATURE_LOCK_REG BIT(5) #define CCU_FEATURE_MMC_TIMING_SWITCH BIT(6) #define CCU_FEATURE_SIGMA_DELTA_MOD BIT(7) +#define CCU_FEATURE_KEY_FIELD BIT(8) /* MMC timing mode switch bit */ #define CCU_MMC_NEW_TIMING_MODE BIT(30) diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c index 2306a1cd83e4..1d557e323169 100644 --- a/drivers/clk/sunxi-ng/ccu_mux.c +++ b/drivers/clk/sunxi-ng/ccu_mux.c @@ -12,6 +12,8 @@ #include "ccu_gate.h" #include "ccu_mux.h" +#define CCU_MUX_KEY_VALUE 0x16aa0000 + static u16 ccu_mux_get_prediv(struct ccu_common *common, struct ccu_mux_internal *cm, int parent_index) @@ -191,6 +193,11 @@ int ccu_mux_helper_set_parent(struct ccu_common *common, spin_lock_irqsave(common->lock, flags); reg = readl(common->base + common->reg); + + /* The key field always reads as zero. */ + if (common->features & CCU_FEATURE_KEY_FIELD) + reg |= CCU_MUX_KEY_VALUE; + reg &= ~GENMASK(cm->width + cm->shift - 1, cm->shift); writel(reg | (index << cm->shift), common->base + common->reg); diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index c0aeedd66f02..ff71dd662880 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -47,6 +47,10 @@ config CPU_IDLE_GOV_HALTPOLL config DT_IDLE_STATES bool +config DT_IDLE_GENPD + depends on PM_GENERIC_DOMAINS_OF + bool + menu "ARM CPU Idle Drivers" depends on ARM || ARM64 source "drivers/cpuidle/Kconfig.arm" @@ -62,6 +66,11 @@ depends on PPC source "drivers/cpuidle/Kconfig.powerpc" endmenu +menu "RISC-V CPU Idle Drivers" +depends on RISCV +source "drivers/cpuidle/Kconfig.riscv" +endmenu + config HALTPOLL_CPUIDLE tristate "Halt poll cpuidle driver" depends on X86 && KVM_GUEST diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index 15d6c46c0a47..be7f512109f7 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -27,6 +27,7 @@ config ARM_PSCI_CPUIDLE_DOMAIN bool "PSCI CPU idle Domain" depends on ARM_PSCI_CPUIDLE depends on PM_GENERIC_DOMAINS_OF + select DT_IDLE_GENPD default y help Select this to enable the PSCI based CPUidle driver to use PM domains, diff --git a/drivers/cpuidle/Kconfig.riscv b/drivers/cpuidle/Kconfig.riscv new file mode 100644 index 000000000000..78518c26af74 --- /dev/null +++ b/drivers/cpuidle/Kconfig.riscv @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# RISC-V CPU Idle drivers +# + +config RISCV_SBI_CPUIDLE + bool "RISC-V SBI CPU idle Driver" + depends on RISCV_SBI + select DT_IDLE_STATES + select CPU_IDLE_MULTIPLE_DRIVERS + select DT_IDLE_GENPD if PM_GENERIC_DOMAINS_OF + help + Select this option to enable RISC-V SBI firmware based CPU idle + driver for RISC-V systems. This drivers also supports hierarchical + DT based layout of the idle state. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 26bbc5e74123..d103342b7cfc 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -6,6 +6,7 @@ obj-y += cpuidle.o driver.o governor.o sysfs.o governors/ obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o obj-$(CONFIG_DT_IDLE_STATES) += dt_idle_states.o +obj-$(CONFIG_DT_IDLE_GENPD) += dt_idle_genpd.o obj-$(CONFIG_ARCH_HAS_CPU_RELAX) += poll_state.o obj-$(CONFIG_HALTPOLL_CPUIDLE) += cpuidle-haltpoll.o @@ -34,3 +35,7 @@ obj-$(CONFIG_MIPS_CPS_CPUIDLE) += cpuidle-cps.o # POWERPC drivers obj-$(CONFIG_PSERIES_CPUIDLE) += cpuidle-pseries.o obj-$(CONFIG_POWERNV_CPUIDLE) += cpuidle-powernv.o + +############################################################################### +# RISC-V drivers +obj-$(CONFIG_RISCV_SBI_CPUIDLE) += cpuidle-riscv-sbi.o diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index ff2c3f8e4668..755bbdfc5b82 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -47,73 +47,14 @@ static int psci_pd_power_off(struct generic_pm_domain *pd) return 0; } -static int psci_pd_parse_state_nodes(struct genpd_power_state *states, - int state_count) -{ - int i, ret; - u32 psci_state, *psci_state_buf; - - for (i = 0; i < state_count; i++) { - ret = psci_dt_parse_state_node(to_of_node(states[i].fwnode), - &psci_state); - if (ret) - goto free_state; - - psci_state_buf = kmalloc(sizeof(u32), GFP_KERNEL); - if (!psci_state_buf) { - ret = -ENOMEM; - goto free_state; - } - *psci_state_buf = psci_state; - states[i].data = psci_state_buf; - } - - return 0; - -free_state: - i--; - for (; i >= 0; i--) - kfree(states[i].data); - return ret; -} - -static int psci_pd_parse_states(struct device_node *np, - struct genpd_power_state **states, int *state_count) -{ - int ret; - - /* Parse the domain idle states. */ - ret = of_genpd_parse_idle_states(np, states, state_count); - if (ret) - return ret; - - /* Fill out the PSCI specifics for each found state. */ - ret = psci_pd_parse_state_nodes(*states, *state_count); - if (ret) - kfree(*states); - - return ret; -} - -static void psci_pd_free_states(struct genpd_power_state *states, - unsigned int state_count) -{ - int i; - - for (i = 0; i < state_count; i++) - kfree(states[i].data); - kfree(states); -} - static int psci_pd_init(struct device_node *np, bool use_osi) { struct generic_pm_domain *pd; struct psci_pd_provider *pd_provider; struct dev_power_governor *pd_gov; - struct genpd_power_state *states = NULL; int ret = -ENOMEM, state_count = 0; - pd = kzalloc(sizeof(*pd), GFP_KERNEL); + pd = dt_idle_pd_alloc(np, psci_dt_parse_state_node); if (!pd) goto out; @@ -121,22 +62,6 @@ static int psci_pd_init(struct device_node *np, bool use_osi) if (!pd_provider) goto free_pd; - pd->name = kasprintf(GFP_KERNEL, "%pOF", np); - if (!pd->name) - goto free_pd_prov; - - /* - * Parse the domain idle states and let genpd manage the state selection - * for those being compatible with "domain-idle-state". - */ - ret = psci_pd_parse_states(np, &states, &state_count); - if (ret) - goto free_name; - - pd->free_states = psci_pd_free_states; - pd->name = kbasename(pd->name); - pd->states = states; - pd->state_count = state_count; pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; /* Allow power off when OSI has been successfully enabled. */ @@ -149,10 +74,8 @@ static int psci_pd_init(struct device_node *np, bool use_osi) pd_gov = state_count > 0 ? &pm_domain_cpu_gov : NULL; ret = pm_genpd_init(pd, pd_gov, false); - if (ret) { - psci_pd_free_states(states, state_count); - goto free_name; - } + if (ret) + goto free_pd_prov; ret = of_genpd_add_provider_simple(np, pd); if (ret) @@ -166,12 +89,10 @@ static int psci_pd_init(struct device_node *np, bool use_osi) remove_pd: pm_genpd_remove(pd); -free_name: - kfree(pd->name); free_pd_prov: kfree(pd_provider); free_pd: - kfree(pd); + dt_idle_pd_free(pd); out: pr_err("failed to init PM domain ret=%d %pOF\n", ret, np); return ret; @@ -195,30 +116,6 @@ static void psci_pd_remove(void) } } -static int psci_pd_init_topology(struct device_node *np) -{ - struct device_node *node; - struct of_phandle_args child, parent; - int ret; - - for_each_child_of_node(np, node) { - if (of_parse_phandle_with_args(node, "power-domains", - "#power-domain-cells", 0, &parent)) - continue; - - child.np = node; - child.args_count = 0; - ret = of_genpd_add_subdomain(&parent, &child); - of_node_put(parent.np); - if (ret) { - of_node_put(node); - return ret; - } - } - - return 0; -} - static bool psci_pd_try_set_osi_mode(void) { int ret; @@ -282,7 +179,7 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) goto no_pd; /* Link genpd masters/subdomains to model the CPU topology. */ - ret = psci_pd_init_topology(np); + ret = dt_idle_pd_init_topology(np); if (ret) goto remove_pd; @@ -314,28 +211,3 @@ static int __init psci_idle_init_domains(void) return platform_driver_register(&psci_cpuidle_domain_driver); } subsys_initcall(psci_idle_init_domains); - -struct device *psci_dt_attach_cpu(int cpu) -{ - struct device *dev; - - dev = dev_pm_domain_attach_by_name(get_cpu_device(cpu), "psci"); - if (IS_ERR_OR_NULL(dev)) - return dev; - - pm_runtime_irq_safe(dev); - if (cpu_online(cpu)) - pm_runtime_get_sync(dev); - - dev_pm_syscore_device(dev, true); - - return dev; -} - -void psci_dt_detach_cpu(struct device *dev) -{ - if (IS_ERR_OR_NULL(dev)) - return; - - dev_pm_domain_detach(dev, false); -} diff --git a/drivers/cpuidle/cpuidle-psci.h b/drivers/cpuidle/cpuidle-psci.h index d8e925e84c27..4e132640ed64 100644 --- a/drivers/cpuidle/cpuidle-psci.h +++ b/drivers/cpuidle/cpuidle-psci.h @@ -10,8 +10,19 @@ void psci_set_domain_state(u32 state); int psci_dt_parse_state_node(struct device_node *np, u32 *state); #ifdef CONFIG_ARM_PSCI_CPUIDLE_DOMAIN -struct device *psci_dt_attach_cpu(int cpu); -void psci_dt_detach_cpu(struct device *dev); + +#include "dt_idle_genpd.h" + +static inline struct device *psci_dt_attach_cpu(int cpu) +{ + return dt_idle_attach_cpu(cpu, "psci"); +} + +static inline void psci_dt_detach_cpu(struct device *dev) +{ + dt_idle_detach_cpu(dev); +} + #else static inline struct device *psci_dt_attach_cpu(int cpu) { return NULL; } static inline void psci_dt_detach_cpu(struct device *dev) { } diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c new file mode 100644 index 000000000000..b459eda2cd37 --- /dev/null +++ b/drivers/cpuidle/cpuidle-riscv-sbi.c @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RISC-V SBI CPU idle driver. + * + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "cpuidle-riscv-sbi: " fmt + +#include <linux/cpuidle.h> +#include <linux/cpumask.h> +#include <linux/cpu_pm.h> +#include <linux/cpu_cooling.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/slab.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <asm/cpuidle.h> +#include <asm/sbi.h> +#include <asm/suspend.h> + +#include "dt_idle_states.h" +#include "dt_idle_genpd.h" + +struct sbi_cpuidle_data { + u32 *states; + struct device *dev; +}; + +struct sbi_domain_state { + bool available; + u32 state; +}; + +static DEFINE_PER_CPU_READ_MOSTLY(struct sbi_cpuidle_data, sbi_cpuidle_data); +static DEFINE_PER_CPU(struct sbi_domain_state, domain_state); +static bool sbi_cpuidle_use_osi; +static bool sbi_cpuidle_use_cpuhp; +static bool sbi_cpuidle_pd_allow_domain_state; + +static inline void sbi_set_domain_state(u32 state) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + data->available = true; + data->state = state; +} + +static inline u32 sbi_get_domain_state(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + return data->state; +} + +static inline void sbi_clear_domain_state(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + data->available = false; +} + +static inline bool sbi_is_domain_state_available(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + return data->available; +} + +static int sbi_suspend_finisher(unsigned long suspend_type, + unsigned long resume_addr, + unsigned long opaque) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND, + suspend_type, resume_addr, opaque, 0, 0, 0); + + return (ret.error) ? sbi_err_map_linux_errno(ret.error) : 0; +} + +static int sbi_suspend(u32 state) +{ + if (state & SBI_HSM_SUSP_NON_RET_BIT) + return cpu_suspend(state, sbi_suspend_finisher); + else + return sbi_suspend_finisher(state, 0, 0); +} + +static int sbi_cpuidle_enter_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx) +{ + u32 *states = __this_cpu_read(sbi_cpuidle_data.states); + + return CPU_PM_CPU_IDLE_ENTER_PARAM(sbi_suspend, idx, states[idx]); +} + +static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx, + bool s2idle) +{ + struct sbi_cpuidle_data *data = this_cpu_ptr(&sbi_cpuidle_data); + u32 *states = data->states; + struct device *pd_dev = data->dev; + u32 state; + int ret; + + ret = cpu_pm_enter(); + if (ret) + return -1; + + /* Do runtime PM to manage a hierarchical CPU toplogy. */ + rcu_irq_enter_irqson(); + if (s2idle) + dev_pm_genpd_suspend(pd_dev); + else + pm_runtime_put_sync_suspend(pd_dev); + rcu_irq_exit_irqson(); + + if (sbi_is_domain_state_available()) + state = sbi_get_domain_state(); + else + state = states[idx]; + + ret = sbi_suspend(state) ? -1 : idx; + + rcu_irq_enter_irqson(); + if (s2idle) + dev_pm_genpd_resume(pd_dev); + else + pm_runtime_get_sync(pd_dev); + rcu_irq_exit_irqson(); + + cpu_pm_exit(); + + /* Clear the domain state to start fresh when back from idle. */ + sbi_clear_domain_state(); + return ret; +} + +static int sbi_enter_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx) +{ + return __sbi_enter_domain_idle_state(dev, drv, idx, false); +} + +static int sbi_enter_s2idle_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, + int idx) +{ + return __sbi_enter_domain_idle_state(dev, drv, idx, true); +} + +static int sbi_cpuidle_cpuhp_up(unsigned int cpu) +{ + struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev); + + if (pd_dev) + pm_runtime_get_sync(pd_dev); + + return 0; +} + +static int sbi_cpuidle_cpuhp_down(unsigned int cpu) +{ + struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev); + + if (pd_dev) { + pm_runtime_put_sync(pd_dev); + /* Clear domain state to start fresh at next online. */ + sbi_clear_domain_state(); + } + + return 0; +} + +static void sbi_idle_init_cpuhp(void) +{ + int err; + + if (!sbi_cpuidle_use_cpuhp) + return; + + err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING, + "cpuidle/sbi:online", + sbi_cpuidle_cpuhp_up, + sbi_cpuidle_cpuhp_down); + if (err) + pr_warn("Failed %d while setup cpuhp state\n", err); +} + +static const struct of_device_id sbi_cpuidle_state_match[] = { + { .compatible = "riscv,idle-state", + .data = sbi_cpuidle_enter_state }, + { }, +}; + +static bool sbi_suspend_state_is_valid(u32 state) +{ + if (state > SBI_HSM_SUSPEND_RET_DEFAULT && + state < SBI_HSM_SUSPEND_RET_PLATFORM) + return false; + if (state > SBI_HSM_SUSPEND_NON_RET_DEFAULT && + state < SBI_HSM_SUSPEND_NON_RET_PLATFORM) + return false; + return true; +} + +static int sbi_dt_parse_state_node(struct device_node *np, u32 *state) +{ + int err = of_property_read_u32(np, "riscv,sbi-suspend-param", state); + + if (err) { + pr_warn("%pOF missing riscv,sbi-suspend-param property\n", np); + return err; + } + + if (!sbi_suspend_state_is_valid(*state)) { + pr_warn("Invalid SBI suspend state %#x\n", *state); + return -EINVAL; + } + + return 0; +} + +static int sbi_dt_cpu_init_topology(struct cpuidle_driver *drv, + struct sbi_cpuidle_data *data, + unsigned int state_count, int cpu) +{ + /* Currently limit the hierarchical topology to be used in OSI mode. */ + if (!sbi_cpuidle_use_osi) + return 0; + + data->dev = dt_idle_attach_cpu(cpu, "sbi"); + if (IS_ERR_OR_NULL(data->dev)) + return PTR_ERR_OR_ZERO(data->dev); + + /* + * Using the deepest state for the CPU to trigger a potential selection + * of a shared state for the domain, assumes the domain states are all + * deeper states. + */ + drv->states[state_count - 1].enter = sbi_enter_domain_idle_state; + drv->states[state_count - 1].enter_s2idle = + sbi_enter_s2idle_domain_idle_state; + sbi_cpuidle_use_cpuhp = true; + + return 0; +} + +static int sbi_cpuidle_dt_init_states(struct device *dev, + struct cpuidle_driver *drv, + unsigned int cpu, + unsigned int state_count) +{ + struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu); + struct device_node *state_node; + struct device_node *cpu_node; + u32 *states; + int i, ret; + + cpu_node = of_cpu_device_node_get(cpu); + if (!cpu_node) + return -ENODEV; + + states = devm_kcalloc(dev, state_count, sizeof(*states), GFP_KERNEL); + if (!states) { + ret = -ENOMEM; + goto fail; + } + + /* Parse SBI specific details from state DT nodes */ + for (i = 1; i < state_count; i++) { + state_node = of_get_cpu_state_node(cpu_node, i - 1); + if (!state_node) + break; + + ret = sbi_dt_parse_state_node(state_node, &states[i]); + of_node_put(state_node); + + if (ret) + return ret; + + pr_debug("sbi-state %#x index %d\n", states[i], i); + } + if (i != state_count) { + ret = -ENODEV; + goto fail; + } + + /* Initialize optional data, used for the hierarchical topology. */ + ret = sbi_dt_cpu_init_topology(drv, data, state_count, cpu); + if (ret < 0) + return ret; + + /* Store states in the per-cpu struct. */ + data->states = states; + +fail: + of_node_put(cpu_node); + + return ret; +} + +static void sbi_cpuidle_deinit_cpu(int cpu) +{ + struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu); + + dt_idle_detach_cpu(data->dev); + sbi_cpuidle_use_cpuhp = false; +} + +static int sbi_cpuidle_init_cpu(struct device *dev, int cpu) +{ + struct cpuidle_driver *drv; + unsigned int state_count = 0; + int ret = 0; + + drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL); + if (!drv) + return -ENOMEM; + + drv->name = "sbi_cpuidle"; + drv->owner = THIS_MODULE; + drv->cpumask = (struct cpumask *)cpumask_of(cpu); + + /* RISC-V architectural WFI to be represented as state index 0. */ + drv->states[0].enter = sbi_cpuidle_enter_state; + drv->states[0].exit_latency = 1; + drv->states[0].target_residency = 1; + drv->states[0].power_usage = UINT_MAX; + strcpy(drv->states[0].name, "WFI"); + strcpy(drv->states[0].desc, "RISC-V WFI"); + + /* + * If no DT idle states are detected (ret == 0) let the driver + * initialization fail accordingly since there is no reason to + * initialize the idle driver if only wfi is supported, the + * default archictectural back-end already executes wfi + * on idle entry. + */ + ret = dt_init_idle_driver(drv, sbi_cpuidle_state_match, 1); + if (ret <= 0) { + pr_debug("HART%ld: failed to parse DT idle states\n", + cpuid_to_hartid_map(cpu)); + return ret ? : -ENODEV; + } + state_count = ret + 1; /* Include WFI state as well */ + + /* Initialize idle states from DT. */ + ret = sbi_cpuidle_dt_init_states(dev, drv, cpu, state_count); + if (ret) { + pr_err("HART%ld: failed to init idle states\n", + cpuid_to_hartid_map(cpu)); + return ret; + } + + ret = cpuidle_register(drv, NULL); + if (ret) + goto deinit; + + cpuidle_cooling_register(drv); + + return 0; +deinit: + sbi_cpuidle_deinit_cpu(cpu); + return ret; +} + +static void sbi_cpuidle_domain_sync_state(struct device *dev) +{ + /* + * All devices have now been attached/probed to the PM domain + * topology, hence it's fine to allow domain states to be picked. + */ + sbi_cpuidle_pd_allow_domain_state = true; +} + +#ifdef CONFIG_DT_IDLE_GENPD + +static int sbi_cpuidle_pd_power_off(struct generic_pm_domain *pd) +{ + struct genpd_power_state *state = &pd->states[pd->state_idx]; + u32 *pd_state; + + if (!state->data) + return 0; + + if (!sbi_cpuidle_pd_allow_domain_state) + return -EBUSY; + + /* OSI mode is enabled, set the corresponding domain state. */ + pd_state = state->data; + sbi_set_domain_state(*pd_state); + + return 0; +} + +struct sbi_pd_provider { + struct list_head link; + struct device_node *node; +}; + +static LIST_HEAD(sbi_pd_providers); + +static int sbi_pd_init(struct device_node *np) +{ + struct generic_pm_domain *pd; + struct sbi_pd_provider *pd_provider; + struct dev_power_governor *pd_gov; + int ret = -ENOMEM, state_count = 0; + + pd = dt_idle_pd_alloc(np, sbi_dt_parse_state_node); + if (!pd) + goto out; + + pd_provider = kzalloc(sizeof(*pd_provider), GFP_KERNEL); + if (!pd_provider) + goto free_pd; + + pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; + + /* Allow power off when OSI is available. */ + if (sbi_cpuidle_use_osi) + pd->power_off = sbi_cpuidle_pd_power_off; + else + pd->flags |= GENPD_FLAG_ALWAYS_ON; + + /* Use governor for CPU PM domains if it has some states to manage. */ + pd_gov = state_count > 0 ? &pm_domain_cpu_gov : NULL; + + ret = pm_genpd_init(pd, pd_gov, false); + if (ret) + goto free_pd_prov; + + ret = of_genpd_add_provider_simple(np, pd); + if (ret) + goto remove_pd; + + pd_provider->node = of_node_get(np); + list_add(&pd_provider->link, &sbi_pd_providers); + + pr_debug("init PM domain %s\n", pd->name); + return 0; + +remove_pd: + pm_genpd_remove(pd); +free_pd_prov: + kfree(pd_provider); +free_pd: + dt_idle_pd_free(pd); +out: + pr_err("failed to init PM domain ret=%d %pOF\n", ret, np); + return ret; +} + +static void sbi_pd_remove(void) +{ + struct sbi_pd_provider *pd_provider, *it; + struct generic_pm_domain *genpd; + + list_for_each_entry_safe(pd_provider, it, &sbi_pd_providers, link) { + of_genpd_del_provider(pd_provider->node); + + genpd = of_genpd_remove_last(pd_provider->node); + if (!IS_ERR(genpd)) + kfree(genpd); + + of_node_put(pd_provider->node); + list_del(&pd_provider->link); + kfree(pd_provider); + } +} + +static int sbi_genpd_probe(struct device_node *np) +{ + struct device_node *node; + int ret = 0, pd_count = 0; + + if (!np) + return -ENODEV; + + /* + * Parse child nodes for the "#power-domain-cells" property and + * initialize a genpd/genpd-of-provider pair when it's found. + */ + for_each_child_of_node(np, node) { + if (!of_find_property(node, "#power-domain-cells", NULL)) + continue; + + ret = sbi_pd_init(node); + if (ret) + goto put_node; + + pd_count++; + } + + /* Bail out if not using the hierarchical CPU topology. */ + if (!pd_count) + goto no_pd; + + /* Link genpd masters/subdomains to model the CPU topology. */ + ret = dt_idle_pd_init_topology(np); + if (ret) + goto remove_pd; + + return 0; + +put_node: + of_node_put(node); +remove_pd: + sbi_pd_remove(); + pr_err("failed to create CPU PM domains ret=%d\n", ret); +no_pd: + return ret; +} + +#else + +static inline int sbi_genpd_probe(struct device_node *np) +{ + return 0; +} + +#endif + +static int sbi_cpuidle_probe(struct platform_device *pdev) +{ + int cpu, ret; + struct cpuidle_driver *drv; + struct cpuidle_device *dev; + struct device_node *np, *pds_node; + + /* Detect OSI support based on CPU DT nodes */ + sbi_cpuidle_use_osi = true; + for_each_possible_cpu(cpu) { + np = of_cpu_device_node_get(cpu); + if (np && + of_find_property(np, "power-domains", NULL) && + of_find_property(np, "power-domain-names", NULL)) { + continue; + } else { + sbi_cpuidle_use_osi = false; + break; + } + } + + /* Populate generic power domains from DT nodes */ + pds_node = of_find_node_by_path("/cpus/power-domains"); + if (pds_node) { + ret = sbi_genpd_probe(pds_node); + of_node_put(pds_node); + if (ret) + return ret; + } + + /* Initialize CPU idle driver for each CPU */ + for_each_possible_cpu(cpu) { + ret = sbi_cpuidle_init_cpu(&pdev->dev, cpu); + if (ret) { + pr_debug("HART%ld: idle driver init failed\n", + cpuid_to_hartid_map(cpu)); + goto out_fail; + } + } + + /* Setup CPU hotplut notifiers */ + sbi_idle_init_cpuhp(); + + pr_info("idle driver registered for all CPUs\n"); + + return 0; + +out_fail: + while (--cpu >= 0) { + dev = per_cpu(cpuidle_devices, cpu); + drv = cpuidle_get_cpu_driver(dev); + cpuidle_unregister(drv); + sbi_cpuidle_deinit_cpu(cpu); + } + + return ret; +} + +static struct platform_driver sbi_cpuidle_driver = { + .probe = sbi_cpuidle_probe, + .driver = { + .name = "sbi-cpuidle", + .sync_state = sbi_cpuidle_domain_sync_state, + }, +}; + +static int __init sbi_cpuidle_init(void) +{ + int ret; + struct platform_device *pdev; + + /* + * The SBI HSM suspend function is only available when: + * 1) SBI version is 0.3 or higher + * 2) SBI HSM extension is available + */ + if ((sbi_spec_version < sbi_mk_version(0, 3)) || + sbi_probe_extension(SBI_EXT_HSM) <= 0) { + pr_info("HSM suspend not available\n"); + return 0; + } + + ret = platform_driver_register(&sbi_cpuidle_driver); + if (ret) + return ret; + + pdev = platform_device_register_simple("sbi-cpuidle", + -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&sbi_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(sbi_cpuidle_init); diff --git a/drivers/cpuidle/dt_idle_genpd.c b/drivers/cpuidle/dt_idle_genpd.c new file mode 100644 index 000000000000..b37165514d4e --- /dev/null +++ b/drivers/cpuidle/dt_idle_genpd.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PM domains for CPUs via genpd. + * + * Copyright (C) 2019 Linaro Ltd. + * Author: Ulf Hansson <ulf.hansson@linaro.org> + * + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "dt-idle-genpd: " fmt + +#include <linux/cpu.h> +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "dt_idle_genpd.h" + +static int pd_parse_state_nodes( + int (*parse_state)(struct device_node *, u32 *), + struct genpd_power_state *states, int state_count) +{ + int i, ret; + u32 state, *state_buf; + + for (i = 0; i < state_count; i++) { + ret = parse_state(to_of_node(states[i].fwnode), &state); + if (ret) + goto free_state; + + state_buf = kmalloc(sizeof(u32), GFP_KERNEL); + if (!state_buf) { + ret = -ENOMEM; + goto free_state; + } + *state_buf = state; + states[i].data = state_buf; + } + + return 0; + +free_state: + i--; + for (; i >= 0; i--) + kfree(states[i].data); + return ret; +} + +static int pd_parse_states(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *), + struct genpd_power_state **states, + int *state_count) +{ + int ret; + + /* Parse the domain idle states. */ + ret = of_genpd_parse_idle_states(np, states, state_count); + if (ret) + return ret; + + /* Fill out the dt specifics for each found state. */ + ret = pd_parse_state_nodes(parse_state, *states, *state_count); + if (ret) + kfree(*states); + + return ret; +} + +static void pd_free_states(struct genpd_power_state *states, + unsigned int state_count) +{ + int i; + + for (i = 0; i < state_count; i++) + kfree(states[i].data); + kfree(states); +} + +void dt_idle_pd_free(struct generic_pm_domain *pd) +{ + pd_free_states(pd->states, pd->state_count); + kfree(pd->name); + kfree(pd); +} + +struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)) +{ + struct generic_pm_domain *pd; + struct genpd_power_state *states = NULL; + int ret, state_count = 0; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) + goto out; + + pd->name = kasprintf(GFP_KERNEL, "%pOF", np); + if (!pd->name) + goto free_pd; + + /* + * Parse the domain idle states and let genpd manage the state selection + * for those being compatible with "domain-idle-state". + */ + ret = pd_parse_states(np, parse_state, &states, &state_count); + if (ret) + goto free_name; + + pd->free_states = pd_free_states; + pd->name = kbasename(pd->name); + pd->states = states; + pd->state_count = state_count; + + pr_debug("alloc PM domain %s\n", pd->name); + return pd; + +free_name: + kfree(pd->name); +free_pd: + kfree(pd); +out: + pr_err("failed to alloc PM domain %pOF\n", np); + return NULL; +} + +int dt_idle_pd_init_topology(struct device_node *np) +{ + struct device_node *node; + struct of_phandle_args child, parent; + int ret; + + for_each_child_of_node(np, node) { + if (of_parse_phandle_with_args(node, "power-domains", + "#power-domain-cells", 0, &parent)) + continue; + + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) { + of_node_put(node); + return ret; + } + } + + return 0; +} + +struct device *dt_idle_attach_cpu(int cpu, const char *name) +{ + struct device *dev; + + dev = dev_pm_domain_attach_by_name(get_cpu_device(cpu), name); + if (IS_ERR_OR_NULL(dev)) + return dev; + + pm_runtime_irq_safe(dev); + if (cpu_online(cpu)) + pm_runtime_get_sync(dev); + + dev_pm_syscore_device(dev, true); + + return dev; +} + +void dt_idle_detach_cpu(struct device *dev) +{ + if (IS_ERR_OR_NULL(dev)) + return; + + dev_pm_domain_detach(dev, false); +} diff --git a/drivers/cpuidle/dt_idle_genpd.h b/drivers/cpuidle/dt_idle_genpd.h new file mode 100644 index 000000000000..a95483d08a02 --- /dev/null +++ b/drivers/cpuidle/dt_idle_genpd.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DT_IDLE_GENPD +#define __DT_IDLE_GENPD + +struct device_node; +struct generic_pm_domain; + +#ifdef CONFIG_DT_IDLE_GENPD + +void dt_idle_pd_free(struct generic_pm_domain *pd); + +struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)); + +int dt_idle_pd_init_topology(struct device_node *np); + +struct device *dt_idle_attach_cpu(int cpu, const char *name); + +void dt_idle_detach_cpu(struct device *dev); + +#else + +static inline void dt_idle_pd_free(struct generic_pm_domain *pd) +{ +} + +static inline struct generic_pm_domain *dt_idle_pd_alloc( + struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)) +{ + return NULL; +} + +static inline int dt_idle_pd_init_topology(struct device_node *np) +{ + return 0; +} + +static inline struct device *dt_idle_attach_cpu(int cpu, const char *name) +{ + return NULL; +} + +static inline void dt_idle_detach_cpu(struct device *dev) +{ +} + +#endif + +#endif diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig index b894e3a8be4f..5f8915f4a9ff 100644 --- a/drivers/crypto/virtio/Kconfig +++ b/drivers/crypto/virtio/Kconfig @@ -3,8 +3,11 @@ config CRYPTO_DEV_VIRTIO tristate "VirtIO crypto driver" depends on VIRTIO select CRYPTO_AEAD + select CRYPTO_AKCIPHER2 select CRYPTO_SKCIPHER select CRYPTO_ENGINE + select CRYPTO_RSA + select MPILIB help This driver provides support for virtio crypto device. If you choose 'M' here, this module will be called virtio_crypto. diff --git a/drivers/crypto/virtio/Makefile b/drivers/crypto/virtio/Makefile index cbfccccfa135..bfa6cbae342e 100644 --- a/drivers/crypto/virtio/Makefile +++ b/drivers/crypto/virtio/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o virtio_crypto-objs := \ - virtio_crypto_algs.o \ + virtio_crypto_skcipher_algs.o \ + virtio_crypto_akcipher_algs.o \ virtio_crypto_mgr.o \ virtio_crypto_core.o diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c new file mode 100644 index 000000000000..f3ec9420215e --- /dev/null +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + /* Asymmetric algorithms supported by virtio crypto device + * + * Authors: zhenwei pi <pizhenwei@bytedance.com> + * lei he <helei.sig11@bytedance.com> + * + * Copyright 2022 Bytedance CO., LTD. + */ + +#include <linux/mpi.h> +#include <linux/scatterlist.h> +#include <crypto/algapi.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/rsa.h> +#include <linux/err.h> +#include <crypto/scatterwalk.h> +#include <linux/atomic.h> + +#include <uapi/linux/virtio_crypto.h> +#include "virtio_crypto_common.h" + +struct virtio_crypto_rsa_ctx { + MPI n; +}; + +struct virtio_crypto_akcipher_ctx { + struct crypto_engine_ctx enginectx; + struct virtio_crypto *vcrypto; + struct crypto_akcipher *tfm; + bool session_valid; + __u64 session_id; + union { + struct virtio_crypto_rsa_ctx rsa_ctx; + }; +}; + +struct virtio_crypto_akcipher_request { + struct virtio_crypto_request base; + struct virtio_crypto_akcipher_ctx *akcipher_ctx; + struct akcipher_request *akcipher_req; + void *src_buf; + void *dst_buf; + uint32_t opcode; +}; + +struct virtio_crypto_akcipher_algo { + uint32_t algonum; + uint32_t service; + unsigned int active_devs; + struct akcipher_alg algo; +}; + +static DEFINE_MUTEX(algs_lock); + +static void virtio_crypto_akcipher_finalize_req( + struct virtio_crypto_akcipher_request *vc_akcipher_req, + struct akcipher_request *req, int err) +{ + virtcrypto_clear_request(&vc_akcipher_req->base); + + crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); +} + +static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len) +{ + struct virtio_crypto_akcipher_request *vc_akcipher_req = + container_of(vc_req, struct virtio_crypto_akcipher_request, base); + struct akcipher_request *akcipher_req; + int error; + + switch (vc_req->status) { + case VIRTIO_CRYPTO_OK: + error = 0; + break; + case VIRTIO_CRYPTO_INVSESS: + case VIRTIO_CRYPTO_ERR: + error = -EINVAL; + break; + case VIRTIO_CRYPTO_BADMSG: + error = -EBADMSG; + break; + + case VIRTIO_CRYPTO_KEY_REJECTED: + error = -EKEYREJECTED; + break; + + default: + error = -EIO; + break; + } + + akcipher_req = vc_akcipher_req->akcipher_req; + if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY) + sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), + vc_akcipher_req->dst_buf, akcipher_req->dst_len); + virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); +} + +static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx, + struct virtio_crypto_ctrl_header *header, void *para, + const uint8_t *key, unsigned int keylen) +{ + struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3]; + struct virtio_crypto *vcrypto = ctx->vcrypto; + uint8_t *pkey; + unsigned int inlen; + int err; + unsigned int num_out = 0, num_in = 0; + + pkey = kmemdup(key, keylen, GFP_ATOMIC); + if (!pkey) + return -ENOMEM; + + spin_lock(&vcrypto->ctrl_lock); + memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header)); + memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u)); + vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR); + + sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sgs[num_out++] = &outhdr_sg; + + sg_init_one(&key_sg, pkey, keylen); + sgs[num_out++] = &key_sg; + + sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input)); + sgs[num_out + num_in++] = &inhdr_sg; + + err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); + if (err < 0) + goto out; + + virtqueue_kick(vcrypto->ctrl_vq); + while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && + !virtqueue_is_broken(vcrypto->ctrl_vq)) + cpu_relax(); + + if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) { + err = -EINVAL; + goto out; + } + + ctx->session_id = le64_to_cpu(vcrypto->input.session_id); + ctx->session_valid = true; + err = 0; + +out: + spin_unlock(&vcrypto->ctrl_lock); + kfree_sensitive(pkey); + + if (err < 0) + pr_err("virtio_crypto: Create session failed status: %u\n", + le32_to_cpu(vcrypto->input.status)); + + return err; +} + +static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx) +{ + struct scatterlist outhdr_sg, inhdr_sg, *sgs[2]; + struct virtio_crypto_destroy_session_req *destroy_session; + struct virtio_crypto *vcrypto = ctx->vcrypto; + unsigned int num_out = 0, num_in = 0, inlen; + int err; + + spin_lock(&vcrypto->ctrl_lock); + if (!ctx->session_valid) { + err = 0; + goto out; + } + vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR; + vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); + vcrypto->ctrl.header.queue_id = 0; + + destroy_session = &vcrypto->ctrl.u.destroy_session; + destroy_session->session_id = cpu_to_le64(ctx->session_id); + + sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sgs[num_out++] = &outhdr_sg; + + sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status)); + sgs[num_out + num_in++] = &inhdr_sg; + + err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); + if (err < 0) + goto out; + + virtqueue_kick(vcrypto->ctrl_vq); + while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && + !virtqueue_is_broken(vcrypto->ctrl_vq)) + cpu_relax(); + + if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) { + err = -EINVAL; + goto out; + } + + err = 0; + ctx->session_valid = false; + +out: + spin_unlock(&vcrypto->ctrl_lock); + if (err < 0) { + pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", + vcrypto->ctrl_status.status, destroy_session->session_id); + } + + return err; +} + +static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req, + struct akcipher_request *req, struct data_queue *data_vq) +{ + struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx; + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + struct virtio_crypto_op_data_req *req_data = vc_req->req_data; + struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg; + void *src_buf = NULL, *dst_buf = NULL; + unsigned int num_out = 0, num_in = 0; + int node = dev_to_node(&vcrypto->vdev->dev); + unsigned long flags; + int ret = -ENOMEM; + bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY; + unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len; + + /* out header */ + sg_init_one(&outhdr_sg, req_data, sizeof(*req_data)); + sgs[num_out++] = &outhdr_sg; + + /* src data */ + src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node); + if (!src_buf) + goto err; + + if (verify) { + /* for verify operation, both src and dst data work as OUT direction */ + sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len); + sg_init_one(&srcdata_sg, src_buf, src_len); + sgs[num_out++] = &srcdata_sg; + } else { + sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len); + sg_init_one(&srcdata_sg, src_buf, src_len); + sgs[num_out++] = &srcdata_sg; + + /* dst data */ + dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node); + if (!dst_buf) + goto err; + + sg_init_one(&dstdata_sg, dst_buf, req->dst_len); + sgs[num_out + num_in++] = &dstdata_sg; + } + + vc_akcipher_req->src_buf = src_buf; + vc_akcipher_req->dst_buf = dst_buf; + + /* in header */ + sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status)); + sgs[num_out + num_in++] = &inhdr_sg; + + spin_lock_irqsave(&data_vq->lock, flags); + ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC); + virtqueue_kick(data_vq->vq); + spin_unlock_irqrestore(&data_vq->lock, flags); + if (ret) + goto err; + + return 0; + +err: + kfree(src_buf); + kfree(dst_buf); + + return -ENOMEM; +} + +static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq) +{ + struct akcipher_request *req = container_of(vreq, struct akcipher_request, base); + struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx; + struct virtio_crypto *vcrypto = ctx->vcrypto; + struct data_queue *data_vq = vc_req->dataq; + struct virtio_crypto_op_header *header; + struct virtio_crypto_akcipher_data_req *akcipher_req; + int ret; + + vc_req->sgs = NULL; + vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data), + GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev)); + if (!vc_req->req_data) + return -ENOMEM; + + /* build request header */ + header = &vc_req->req_data->header; + header->opcode = cpu_to_le32(vc_akcipher_req->opcode); + header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + header->session_id = cpu_to_le64(ctx->session_id); + + /* build request akcipher data */ + akcipher_req = &vc_req->req_data->u.akcipher_req; + akcipher_req->para.src_data_len = cpu_to_le32(req->src_len); + akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len); + + ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq); + if (ret < 0) { + kfree_sensitive(vc_req->req_data); + vc_req->req_data = NULL; + return ret; + } + + return 0; +} + +static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode) +{ + struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req); + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm); + struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_akcipher_req->base; + struct virtio_crypto *vcrypto = ctx->vcrypto; + /* Use the first data virtqueue as default */ + struct data_queue *data_vq = &vcrypto->data_vq[0]; + + vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback; + vc_akcipher_req->akcipher_ctx = ctx; + vc_akcipher_req->akcipher_req = req; + vc_akcipher_req->opcode = opcode; + + return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req); +} + +static int virtio_crypto_rsa_encrypt(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT); +} + +static int virtio_crypto_rsa_decrypt(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT); +} + +static int virtio_crypto_rsa_sign(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN); +} + +static int virtio_crypto_rsa_verify(struct akcipher_request *req) +{ + return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY); +} + +static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen, + bool private, + int padding_algo, + int hash_algo) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; + struct virtio_crypto *vcrypto; + struct virtio_crypto_ctrl_header header; + struct virtio_crypto_akcipher_session_para para; + struct rsa_key rsa_key = {0}; + int node = virtio_crypto_get_current_node(); + uint32_t keytype; + int ret; + + /* mpi_free will test n, just free it. */ + mpi_free(rsa_ctx->n); + rsa_ctx->n = NULL; + + if (private) { + keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE; + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + } else { + keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC; + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + } + + if (ret) + return ret; + + rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz); + if (!rsa_ctx->n) + return -ENOMEM; + + if (!ctx->vcrypto) { + vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER, + VIRTIO_CRYPTO_AKCIPHER_RSA); + if (!vcrypto) { + pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n"); + return -ENODEV; + } + + ctx->vcrypto = vcrypto; + } else { + virtio_crypto_alg_akcipher_close_session(ctx); + } + + /* set ctrl header */ + header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION); + header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + header.queue_id = 0; + + /* set RSA para */ + para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA); + para.keytype = cpu_to_le32(keytype); + para.keylen = cpu_to_le32(keylen); + para.u.rsa.padding_algo = cpu_to_le32(padding_algo); + para.u.rsa.hash_algo = cpu_to_le32(hash_algo); + + return virtio_crypto_alg_akcipher_init_session(ctx, &header, ¶, key, keylen); +} + +static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, + VIRTIO_CRYPTO_RSA_RAW_PADDING, + VIRTIO_CRYPTO_RSA_NO_HASH); +} + + +static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 1, + VIRTIO_CRYPTO_RSA_PKCS1_PADDING, + VIRTIO_CRYPTO_RSA_SHA1); +} + +static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, + VIRTIO_CRYPTO_RSA_RAW_PADDING, + VIRTIO_CRYPTO_RSA_NO_HASH); +} + +static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm, + const void *key, + unsigned int keylen) +{ + return virtio_crypto_rsa_set_key(tfm, key, keylen, 0, + VIRTIO_CRYPTO_RSA_PKCS1_PADDING, + VIRTIO_CRYPTO_RSA_SHA1); +} + +static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; + + return mpi_get_size(rsa_ctx->n); +} + +static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->tfm = tfm; + ctx->enginectx.op.do_one_request = virtio_crypto_rsa_do_req; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm); + struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx; + + virtio_crypto_alg_akcipher_close_session(ctx); + virtcrypto_dev_put(ctx->vcrypto); + mpi_free(rsa_ctx->n); + rsa_ctx->n = NULL; +} + +static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = { + { + .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, + .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, + .algo = { + .encrypt = virtio_crypto_rsa_encrypt, + .decrypt = virtio_crypto_rsa_decrypt, + .set_pub_key = virtio_crypto_rsa_raw_set_pub_key, + .set_priv_key = virtio_crypto_rsa_raw_set_priv_key, + .max_size = virtio_crypto_rsa_max_size, + .init = virtio_crypto_rsa_init_tfm, + .exit = virtio_crypto_rsa_exit_tfm, + .reqsize = sizeof(struct virtio_crypto_akcipher_request), + .base = { + .cra_name = "rsa", + .cra_driver_name = "virtio-crypto-rsa", + .cra_priority = 150, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), + }, + }, + }, + { + .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA, + .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER, + .algo = { + .encrypt = virtio_crypto_rsa_encrypt, + .decrypt = virtio_crypto_rsa_decrypt, + .sign = virtio_crypto_rsa_sign, + .verify = virtio_crypto_rsa_verify, + .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key, + .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key, + .max_size = virtio_crypto_rsa_max_size, + .init = virtio_crypto_rsa_init_tfm, + .exit = virtio_crypto_rsa_exit_tfm, + .reqsize = sizeof(struct virtio_crypto_akcipher_request), + .base = { + .cra_name = "pkcs1pad(rsa,sha1)", + .cra_driver_name = "virtio-pkcs1-rsa-with-sha1", + .cra_priority = 150, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), + }, + }, + }, +}; + +int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto) +{ + int ret = 0; + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { + uint32_t service = virtio_crypto_akcipher_algs[i].service; + uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; + + if (!virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_akcipher_algs[i].active_devs == 0) { + ret = crypto_register_akcipher(&virtio_crypto_akcipher_algs[i].algo); + if (ret) + goto unlock; + } + + virtio_crypto_akcipher_algs[i].active_devs++; + dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n", + virtio_crypto_akcipher_algs[i].algo.base.cra_name); + } + +unlock: + mutex_unlock(&algs_lock); + return ret; +} + +void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto) +{ + int i = 0; + + mutex_lock(&algs_lock); + + for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) { + uint32_t service = virtio_crypto_akcipher_algs[i].service; + uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum; + + if (virtio_crypto_akcipher_algs[i].active_devs == 0 || + !virtcrypto_algo_is_supported(vcrypto, service, algonum)) + continue; + + if (virtio_crypto_akcipher_algs[i].active_devs == 1) + crypto_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo); + + virtio_crypto_akcipher_algs[i].active_devs--; + } + + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index a24f85c589e7..e693d4ee83a6 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -56,6 +56,7 @@ struct virtio_crypto { u32 mac_algo_l; u32 mac_algo_h; u32 aead_algo; + u32 akcipher_algo; /* Maximum length of cipher key */ u32 max_cipher_key_len; @@ -129,7 +130,9 @@ static inline int virtio_crypto_get_current_node(void) return node; } -int virtio_crypto_algs_register(struct virtio_crypto *vcrypto); -void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto); +int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto); +void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto); +int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto); +void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto); #endif /* _VIRTIO_CRYPTO_COMMON_H */ diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 8e977b7627cb..c6f482db0bc0 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -297,6 +297,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) u32 mac_algo_l = 0; u32 mac_algo_h = 0; u32 aead_algo = 0; + u32 akcipher_algo = 0; u32 crypto_services = 0; if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) @@ -348,6 +349,9 @@ static int virtcrypto_probe(struct virtio_device *vdev) mac_algo_h, &mac_algo_h); virtio_cread_le(vdev, struct virtio_crypto_config, aead_algo, &aead_algo); + if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER)) + virtio_cread_le(vdev, struct virtio_crypto_config, + akcipher_algo, &akcipher_algo); /* Add virtio crypto device to global table */ err = virtcrypto_devmgr_add_dev(vcrypto); @@ -374,7 +378,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) vcrypto->mac_algo_h = mac_algo_h; vcrypto->hash_algo = hash_algo; vcrypto->aead_algo = aead_algo; - + vcrypto->akcipher_algo = akcipher_algo; dev_info(&vdev->dev, "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n", diff --git a/drivers/crypto/virtio/virtio_crypto_mgr.c b/drivers/crypto/virtio/virtio_crypto_mgr.c index 6860f8180c7c..70e778aac0f2 100644 --- a/drivers/crypto/virtio/virtio_crypto_mgr.c +++ b/drivers/crypto/virtio/virtio_crypto_mgr.c @@ -237,8 +237,14 @@ struct virtio_crypto *virtcrypto_get_dev_node(int node, uint32_t service, */ int virtcrypto_dev_start(struct virtio_crypto *vcrypto) { - if (virtio_crypto_algs_register(vcrypto)) { - pr_err("virtio_crypto: Failed to register crypto algs\n"); + if (virtio_crypto_skcipher_algs_register(vcrypto)) { + pr_err("virtio_crypto: Failed to register crypto skcipher algs\n"); + return -EFAULT; + } + + if (virtio_crypto_akcipher_algs_register(vcrypto)) { + pr_err("virtio_crypto: Failed to register crypto akcipher algs\n"); + virtio_crypto_skcipher_algs_unregister(vcrypto); return -EFAULT; } @@ -257,7 +263,8 @@ int virtcrypto_dev_start(struct virtio_crypto *vcrypto) */ void virtcrypto_dev_stop(struct virtio_crypto *vcrypto) { - virtio_crypto_algs_unregister(vcrypto); + virtio_crypto_skcipher_algs_unregister(vcrypto); + virtio_crypto_akcipher_algs_unregister(vcrypto); } /* @@ -312,6 +319,10 @@ bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto, case VIRTIO_CRYPTO_SERVICE_AEAD: algo_mask = vcrypto->aead_algo; break; + + case VIRTIO_CRYPTO_SERVICE_AKCIPHER: + algo_mask = vcrypto->akcipher_algo; + break; } if (!(algo_mask & (1u << algo))) diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 583c0b535d13..a618c46a52b8 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -613,7 +613,7 @@ static struct virtio_crypto_algo virtio_crypto_algs[] = { { }, } }; -int virtio_crypto_algs_register(struct virtio_crypto *vcrypto) +int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto) { int ret = 0; int i = 0; @@ -644,7 +644,7 @@ unlock: return ret; } -void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto) +void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto) { int i = 0; diff --git a/drivers/gpio/gpio-ts4900.c b/drivers/gpio/gpio-ts4900.c index 69854fd2382a..416725c26e94 100644 --- a/drivers/gpio/gpio-ts4900.c +++ b/drivers/gpio/gpio-ts4900.c @@ -47,8 +47,9 @@ static int ts4900_gpio_direction_input(struct gpio_chip *chip, { struct ts4900_gpio_priv *priv = gpiochip_get_data(chip); - /* Only clear the OE bit here, requires a RMW. Prevents potential issue - * with OE and data getting to the physical pin at different times. + /* + * Only clear the OE bit here, requires a RMW. Prevents a potential issue + * with OE and DAT getting to the physical pin at different times. */ return regmap_update_bits(priv->regmap, offset, TS4900_GPIO_OE, 0); } @@ -60,9 +61,10 @@ static int ts4900_gpio_direction_output(struct gpio_chip *chip, unsigned int reg; int ret; - /* If changing from an input to an output, we need to first set the - * proper data bit to what is requested and then set OE bit. This - * prevents a glitch that can occur on the IO line + /* + * If changing from an input to an output, we need to first set the + * GPIO's DAT bit to what is requested and then set the OE bit. This + * prevents a glitch that can occur on the IO line. */ regmap_read(priv->regmap, offset, ®); if (!(reg & TS4900_GPIO_OE)) { diff --git a/drivers/gpio/gpio-ts5500.c b/drivers/gpio/gpio-ts5500.c index b159e92a3612..8e03614c7a24 100644 --- a/drivers/gpio/gpio-ts5500.c +++ b/drivers/gpio/gpio-ts5500.c @@ -11,11 +11,11 @@ * Actually, the following platforms have DIO support: * * TS-5500: - * Documentation: http://wiki.embeddedarm.com/wiki/TS-5500 + * Documentation: https://docs.embeddedts.com/TS-5500 * Blocks: DIO1, DIO2 and LCD port. * * TS-5600: - * Documentation: http://wiki.embeddedarm.com/wiki/TS-5600 + * Documentation: https://docs.embeddedts.com/TS-5600 * Blocks: LCD port (identical to TS-5500 LCD). */ diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 7a674873d794..a95a7cbc4a59 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -405,14 +405,25 @@ config HOLTEK_FF Say Y here if you have a Holtek On Line Grip based game controller and want to have force feedback support for it. +config HID_VIVALDI_COMMON + tristate + help + ChromeOS Vivaldi HID parsing support library. This is a hidden + option so that drivers can use common code to parse the HID + descriptors for vivaldi function row keymap. + config HID_GOOGLE_HAMMER tristate "Google Hammer Keyboard" + select HID_VIVALDI_COMMON + select INPUT_VIVALDIFMAP depends on USB_HID && LEDS_CLASS && CROS_EC help Say Y here if you have a Google Hammer device. config HID_VIVALDI tristate "Vivaldi Keyboard" + select HID_VIVALDI_COMMON + select INPUT_VIVALDIFMAP depends on HID help Say Y here if you want to enable support for Vivaldi keyboards. diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index d5ce8d747b14..345ac5581bd8 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -50,6 +50,7 @@ obj-$(CONFIG_HID_FT260) += hid-ft260.o obj-$(CONFIG_HID_GEMBIRD) += hid-gembird.o obj-$(CONFIG_HID_GFRM) += hid-gfrm.o obj-$(CONFIG_HID_GLORIOUS) += hid-glorious.o +obj-$(CONFIG_HID_VIVALDI_COMMON) += hid-vivaldi-common.o obj-$(CONFIG_HID_GOOGLE_HAMMER) += hid-google-hammer.o obj-$(CONFIG_HID_VIVALDI) += hid-vivaldi.o obj-$(CONFIG_HID_GT683R) += hid-gt683r.o diff --git a/drivers/hid/hid-google-hammer.c b/drivers/hid/hid-google-hammer.c index ddbe0de177e2..ff40f1e55c21 100644 --- a/drivers/hid/hid-google-hammer.c +++ b/drivers/hid/hid-google-hammer.c @@ -15,6 +15,7 @@ #include <linux/acpi.h> #include <linux/hid.h> +#include <linux/input/vivaldi-fmap.h> #include <linux/leds.h> #include <linux/module.h> #include <linux/of.h> @@ -25,6 +26,7 @@ #include <asm/unaligned.h> #include "hid-ids.h" +#include "hid-vivaldi-common.h" /* * C(hrome)B(ase)A(ttached)S(witch) - switch exported by Chrome EC and reporting @@ -340,9 +342,9 @@ static int hammer_kbd_brightness_set_blocking(struct led_classdev *cdev, static int hammer_register_leds(struct hid_device *hdev) { struct hammer_kbd_leds *kbd_backlight; - int error; - kbd_backlight = kzalloc(sizeof(*kbd_backlight), GFP_KERNEL); + kbd_backlight = devm_kzalloc(&hdev->dev, sizeof(*kbd_backlight), + GFP_KERNEL); if (!kbd_backlight) return -ENOMEM; @@ -356,26 +358,7 @@ static int hammer_register_leds(struct hid_device *hdev) /* Set backlight to 0% initially. */ hammer_kbd_brightness_set_blocking(&kbd_backlight->cdev, 0); - error = led_classdev_register(&hdev->dev, &kbd_backlight->cdev); - if (error) - goto err_free_mem; - - hid_set_drvdata(hdev, kbd_backlight); - return 0; - -err_free_mem: - kfree(kbd_backlight); - return error; -} - -static void hammer_unregister_leds(struct hid_device *hdev) -{ - struct hammer_kbd_leds *kbd_backlight = hid_get_drvdata(hdev); - - if (kbd_backlight) { - led_classdev_unregister(&kbd_backlight->cdev); - kfree(kbd_backlight); - } + return devm_led_classdev_register(&hdev->dev, &kbd_backlight->cdev); } #define HID_UP_GOOGLEVENDOR 0xffd10000 @@ -512,11 +495,23 @@ out: kfree(buf); } +static void hammer_stop(void *hdev) +{ + hid_hw_stop(hdev); +} + static int hammer_probe(struct hid_device *hdev, const struct hid_device_id *id) { + struct vivaldi_data *vdata; int error; + vdata = devm_kzalloc(&hdev->dev, sizeof(*vdata), GFP_KERNEL); + if (!vdata) + return -ENOMEM; + + hid_set_drvdata(hdev, vdata); + error = hid_parse(hdev); if (error) return error; @@ -525,6 +520,10 @@ static int hammer_probe(struct hid_device *hdev, if (error) return error; + error = devm_add_action(&hdev->dev, hammer_stop, hdev); + if (error) + return error; + /* * We always want to poll for, and handle tablet mode events from * devices that have folded usage, even when nobody has opened the input @@ -577,15 +576,13 @@ static void hammer_remove(struct hid_device *hdev) spin_unlock_irqrestore(&cbas_ec_lock, flags); } - hammer_unregister_leds(hdev); - - hid_hw_stop(hdev); + /* Unregistering LEDs and stopping the hardware is done via devm */ } static const struct hid_device_id hammer_devices[] = { { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_DON) }, - { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, + { HID_DEVICE(BUS_USB, HID_GROUP_VIVALDI, USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_EEL) }, { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_HAMMER) }, @@ -610,6 +607,8 @@ static struct hid_driver hammer_driver = { .id_table = hammer_devices, .probe = hammer_probe, .remove = hammer_remove, + .feature_mapping = vivaldi_feature_mapping, + .input_configured = vivaldi_input_configured, .input_mapping = hammer_input_mapping, .event = hammer_event, }; diff --git a/drivers/hid/hid-vivaldi-common.c b/drivers/hid/hid-vivaldi-common.c new file mode 100644 index 000000000000..8b3e515d0f06 --- /dev/null +++ b/drivers/hid/hid-vivaldi-common.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helpers for ChromeOS HID Vivaldi keyboards + * + * Copyright (C) 2022 Google, Inc + */ + +#include <linux/export.h> +#include <linux/hid.h> +#include <linux/input/vivaldi-fmap.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/types.h> + +#include "hid-vivaldi-common.h" + +#define MIN_FN_ROW_KEY 1 +#define MAX_FN_ROW_KEY VIVALDI_MAX_FUNCTION_ROW_KEYS +#define HID_VD_FN_ROW_PHYSMAP 0x00000001 +#define HID_USAGE_FN_ROW_PHYSMAP (HID_UP_GOOGLEVENDOR | HID_VD_FN_ROW_PHYSMAP) + +/** + * vivaldi_feature_mapping - Fill out vivaldi keymap data exposed via HID + * @hdev: HID device to parse + * @field: HID field to parse + * @usage: HID usage to parse + * + * Note: this function assumes that driver data attached to @hdev contains an + * instance of &struct vivaldi_data at the very beginning. + */ +void vivaldi_feature_mapping(struct hid_device *hdev, + struct hid_field *field, struct hid_usage *usage) +{ + struct vivaldi_data *data = hid_get_drvdata(hdev); + struct hid_report *report = field->report; + u8 *report_data, *buf; + u32 report_len; + unsigned int fn_key; + int ret; + + if (field->logical != HID_USAGE_FN_ROW_PHYSMAP || + (usage->hid & HID_USAGE_PAGE) != HID_UP_ORDINAL) + return; + + fn_key = usage->hid & HID_USAGE; + if (fn_key < MIN_FN_ROW_KEY || fn_key > MAX_FN_ROW_KEY) + return; + + if (fn_key > data->num_function_row_keys) + data->num_function_row_keys = fn_key; + + report_data = buf = hid_alloc_report_buf(report, GFP_KERNEL); + if (!report_data) + return; + + report_len = hid_report_len(report); + if (!report->id) { + /* + * hid_hw_raw_request() will stuff report ID (which will be 0) + * into the first byte of the buffer even for unnumbered + * reports, so we need to account for this to avoid getting + * -EOVERFLOW in return. + * Note that hid_alloc_report_buf() adds 7 bytes to the size + * so we can safely say that we have space for an extra byte. + */ + report_len++; + } + + ret = hid_hw_raw_request(hdev, report->id, report_data, + report_len, HID_FEATURE_REPORT, + HID_REQ_GET_REPORT); + if (ret < 0) { + dev_warn(&hdev->dev, "failed to fetch feature %d\n", + field->report->id); + goto out; + } + + if (!report->id) { + /* + * Undo the damage from hid_hw_raw_request() for unnumbered + * reports. + */ + report_data++; + report_len--; + } + + ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, report_data, + report_len, 0); + if (ret) { + dev_warn(&hdev->dev, "failed to report feature %d\n", + field->report->id); + goto out; + } + + data->function_row_physmap[fn_key - MIN_FN_ROW_KEY] = + field->value[usage->usage_index]; + +out: + kfree(buf); +} +EXPORT_SYMBOL_GPL(vivaldi_feature_mapping); + +static ssize_t function_row_physmap_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct vivaldi_data *data = hid_get_drvdata(hdev); + + return vivaldi_function_row_physmap_show(data, buf); +} + +static DEVICE_ATTR_RO(function_row_physmap); +static struct attribute *vivaldi_sysfs_attrs[] = { + &dev_attr_function_row_physmap.attr, + NULL +}; + +static const struct attribute_group vivaldi_attribute_group = { + .attrs = vivaldi_sysfs_attrs, +}; + +/** + * vivaldi_input_configured - Complete initialization of device using vivaldi map + * @hdev: HID device to which vivaldi attributes should be attached + * @hidinput: HID input device (unused) + */ +int vivaldi_input_configured(struct hid_device *hdev, + struct hid_input *hidinput) +{ + struct vivaldi_data *data = hid_get_drvdata(hdev); + + if (!data->num_function_row_keys) + return 0; + + return devm_device_add_group(&hdev->dev, &vivaldi_attribute_group); +} +EXPORT_SYMBOL_GPL(vivaldi_input_configured); + +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-vivaldi-common.h b/drivers/hid/hid-vivaldi-common.h new file mode 100644 index 000000000000..d42e82d77825 --- /dev/null +++ b/drivers/hid/hid-vivaldi-common.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _HID_VIVALDI_COMMON_H +#define _HID_VIVALDI_COMMON_H + +struct hid_device; +struct hid_field; +struct hid_input; +struct hid_usage; + +void vivaldi_feature_mapping(struct hid_device *hdev, + struct hid_field *field, struct hid_usage *usage); + +int vivaldi_input_configured(struct hid_device *hdev, + struct hid_input *hidinput); + +#endif /* _HID_VIVALDI_COMMON_H */ diff --git a/drivers/hid/hid-vivaldi.c b/drivers/hid/hid-vivaldi.c index 42ceb2058a09..3a979123e7d3 100644 --- a/drivers/hid/hid-vivaldi.c +++ b/drivers/hid/hid-vivaldi.c @@ -8,48 +8,11 @@ #include <linux/device.h> #include <linux/hid.h> +#include <linux/input/vivaldi-fmap.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/sysfs.h> -#define MIN_FN_ROW_KEY 1 -#define MAX_FN_ROW_KEY 24 -#define HID_VD_FN_ROW_PHYSMAP 0x00000001 -#define HID_USAGE_FN_ROW_PHYSMAP (HID_UP_GOOGLEVENDOR | HID_VD_FN_ROW_PHYSMAP) - -struct vivaldi_data { - u32 function_row_physmap[MAX_FN_ROW_KEY - MIN_FN_ROW_KEY + 1]; - int max_function_row_key; -}; - -static ssize_t function_row_physmap_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct hid_device *hdev = to_hid_device(dev); - struct vivaldi_data *drvdata = hid_get_drvdata(hdev); - ssize_t size = 0; - int i; - - if (!drvdata->max_function_row_key) - return 0; - - for (i = 0; i < drvdata->max_function_row_key; i++) - size += sprintf(buf + size, "%02X ", - drvdata->function_row_physmap[i]); - size += sprintf(buf + size, "\n"); - return size; -} - -static DEVICE_ATTR_RO(function_row_physmap); -static struct attribute *sysfs_attrs[] = { - &dev_attr_function_row_physmap.attr, - NULL -}; - -static const struct attribute_group input_attribute_group = { - .attrs = sysfs_attrs -}; +#include "hid-vivaldi-common.h" static int vivaldi_probe(struct hid_device *hdev, const struct hid_device_id *id) @@ -70,86 +33,8 @@ static int vivaldi_probe(struct hid_device *hdev, return hid_hw_start(hdev, HID_CONNECT_DEFAULT); } -static void vivaldi_feature_mapping(struct hid_device *hdev, - struct hid_field *field, - struct hid_usage *usage) -{ - struct vivaldi_data *drvdata = hid_get_drvdata(hdev); - struct hid_report *report = field->report; - int fn_key; - int ret; - u32 report_len; - u8 *report_data, *buf; - - if (field->logical != HID_USAGE_FN_ROW_PHYSMAP || - (usage->hid & HID_USAGE_PAGE) != HID_UP_ORDINAL) - return; - - fn_key = (usage->hid & HID_USAGE); - if (fn_key < MIN_FN_ROW_KEY || fn_key > MAX_FN_ROW_KEY) - return; - if (fn_key > drvdata->max_function_row_key) - drvdata->max_function_row_key = fn_key; - - report_data = buf = hid_alloc_report_buf(report, GFP_KERNEL); - if (!report_data) - return; - - report_len = hid_report_len(report); - if (!report->id) { - /* - * hid_hw_raw_request() will stuff report ID (which will be 0) - * into the first byte of the buffer even for unnumbered - * reports, so we need to account for this to avoid getting - * -EOVERFLOW in return. - * Note that hid_alloc_report_buf() adds 7 bytes to the size - * so we can safely say that we have space for an extra byte. - */ - report_len++; - } - - ret = hid_hw_raw_request(hdev, report->id, report_data, - report_len, HID_FEATURE_REPORT, - HID_REQ_GET_REPORT); - if (ret < 0) { - dev_warn(&hdev->dev, "failed to fetch feature %d\n", - field->report->id); - goto out; - } - - if (!report->id) { - /* - * Undo the damage from hid_hw_raw_request() for unnumbered - * reports. - */ - report_data++; - report_len--; - } - - ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, report_data, - report_len, 0); - if (ret) { - dev_warn(&hdev->dev, "failed to report feature %d\n", - field->report->id); - goto out; - } - - drvdata->function_row_physmap[fn_key - MIN_FN_ROW_KEY] = - field->value[usage->usage_index]; - -out: - kfree(buf); -} - -static int vivaldi_input_configured(struct hid_device *hdev, - struct hid_input *hidinput) -{ - return devm_device_add_group(&hdev->dev, &input_attribute_group); -} - static const struct hid_device_id vivaldi_table[] = { - { HID_DEVICE(HID_BUS_ANY, HID_GROUP_VIVALDI, HID_ANY_ID, - HID_ANY_ID) }, + { HID_DEVICE(HID_BUS_ANY, HID_GROUP_VIVALDI, HID_ANY_ID, HID_ANY_ID) }, { } }; diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 26d269ba947c..85a2142c9384 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -380,7 +380,7 @@ void vmbus_channel_map_relid(struct vmbus_channel *channel) * execute: * * (a) In the "normal (i.e., not resuming from hibernation)" path, - * the full barrier in smp_store_mb() guarantees that the store + * the full barrier in virt_store_mb() guarantees that the store * is propagated to all CPUs before the add_channel_work work * is queued. In turn, add_channel_work is queued before the * channel's ring buffer is allocated/initialized and the @@ -392,14 +392,14 @@ void vmbus_channel_map_relid(struct vmbus_channel *channel) * recv_int_page before retrieving the channel pointer from the * array of channels. * - * (b) In the "resuming from hibernation" path, the smp_store_mb() + * (b) In the "resuming from hibernation" path, the virt_store_mb() * guarantees that the store is propagated to all CPUs before * the VMBus connection is marked as ready for the resume event * (cf. check_ready_for_resume_event()). The interrupt handler * of the VMBus driver and vmbus_chan_sched() can not run before * vmbus_bus_resume() has completed execution (cf. resume_noirq). */ - smp_store_mb( + virt_store_mb( vmbus_connection.channels[channel->offermsg.child_relid], channel); } diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index 439f99b8b5de..3248b48f37f6 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -17,6 +17,7 @@ #include <linux/slab.h> #include <linux/kthread.h> #include <linux/completion.h> +#include <linux/count_zeros.h> #include <linux/memory_hotplug.h> #include <linux/memory.h> #include <linux/notifier.h> @@ -1130,6 +1131,7 @@ static void post_status(struct hv_dynmem_device *dm) struct dm_status status; unsigned long now = jiffies; unsigned long last_post = last_post_time; + unsigned long num_pages_avail, num_pages_committed; if (pressure_report_delay > 0) { --pressure_report_delay; @@ -1154,16 +1156,21 @@ static void post_status(struct hv_dynmem_device *dm) * num_pages_onlined) as committed to the host, otherwise it can try * asking us to balloon them out. */ - status.num_avail = si_mem_available(); - status.num_committed = vm_memory_committed() + + num_pages_avail = si_mem_available(); + num_pages_committed = vm_memory_committed() + dm->num_pages_ballooned + (dm->num_pages_added > dm->num_pages_onlined ? dm->num_pages_added - dm->num_pages_onlined : 0) + compute_balloon_floor(); - trace_balloon_status(status.num_avail, status.num_committed, + trace_balloon_status(num_pages_avail, num_pages_committed, vm_memory_committed(), dm->num_pages_ballooned, dm->num_pages_added, dm->num_pages_onlined); + + /* Convert numbers of pages into numbers of HV_HYP_PAGEs. */ + status.num_avail = num_pages_avail * NR_HV_HYP_PAGES_IN_PAGE; + status.num_committed = num_pages_committed * NR_HV_HYP_PAGES_IN_PAGE; + /* * If our transaction ID is no longer current, just don't * send the status. This can happen if we were interrupted @@ -1653,6 +1660,38 @@ static void disable_page_reporting(void) } } +static int ballooning_enabled(void) +{ + /* + * Disable ballooning if the page size is not 4k (HV_HYP_PAGE_SIZE), + * since currently it's unclear to us whether an unballoon request can + * make sure all page ranges are guest page size aligned. + */ + if (PAGE_SIZE != HV_HYP_PAGE_SIZE) { + pr_info("Ballooning disabled because page size is not 4096 bytes\n"); + return 0; + } + + return 1; +} + +static int hot_add_enabled(void) +{ + /* + * Disable hot add on ARM64, because we currently rely on + * memory_add_physaddr_to_nid() to get a node id of a hot add range, + * however ARM64's memory_add_physaddr_to_nid() always return 0 and + * DM_MEM_HOT_ADD_REQUEST doesn't have the NUMA node information for + * add_memory(). + */ + if (IS_ENABLED(CONFIG_ARM64)) { + pr_info("Memory hot add disabled on ARM64\n"); + return 0; + } + + return 1; +} + static int balloon_connect_vsp(struct hv_device *dev) { struct dm_version_request version_req; @@ -1724,8 +1763,8 @@ static int balloon_connect_vsp(struct hv_device *dev) * currently still requires the bits to be set, so we have to add code * to fail the host's hot-add and balloon up/down requests, if any. */ - cap_msg.caps.cap_bits.balloon = 1; - cap_msg.caps.cap_bits.hot_add = 1; + cap_msg.caps.cap_bits.balloon = ballooning_enabled(); + cap_msg.caps.cap_bits.hot_add = hot_add_enabled(); /* * Specify our alignment requirements as it relates diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index c1dd21d0d7ef..ae68298c0dca 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -20,6 +20,7 @@ #include <linux/panic_notifier.h> #include <linux/ptrace.h> #include <linux/slab.h> +#include <linux/dma-map-ops.h> #include <asm/hyperv-tlfs.h> #include <asm/mshyperv.h> @@ -218,6 +219,16 @@ bool hv_query_ext_cap(u64 cap_query) } EXPORT_SYMBOL_GPL(hv_query_ext_cap); +void hv_setup_dma_ops(struct device *dev, bool coherent) +{ + /* + * Hyper-V does not offer a vIOMMU in the guest + * VM, so pass 0/NULL for the IOMMU settings + */ + arch_setup_dma_ops(dev, 0, 0, NULL, coherent); +} +EXPORT_SYMBOL_GPL(hv_setup_dma_ops); + bool hv_is_hibernation_supported(void) { return !hv_root_partition && acpi_sleep_state_supported(ACPI_STATE_S4); diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 71efacb90965..3d215d9dec43 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -439,7 +439,16 @@ int hv_ringbuffer_read(struct vmbus_channel *channel, static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) { u32 priv_read_loc = rbi->priv_read_index; - u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index); + u32 write_loc; + + /* + * The Hyper-V host writes the packet data, then uses + * store_release() to update the write_index. Use load_acquire() + * here to prevent loads of the packet data from being re-ordered + * before the read of the write_index and potentially getting + * stale data. + */ + write_loc = virt_load_acquire(&rbi->ring_buffer->write_index); if (write_loc >= priv_read_loc) return write_loc - priv_read_loc; diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 60ee8b329f9e..14de17087864 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -77,8 +77,8 @@ static int hyperv_panic_event(struct notifier_block *nb, unsigned long val, /* * Hyper-V should be notified only once about a panic. If we will be - * doing hyperv_report_panic_msg() later with kmsg data, don't do - * the notification here. + * doing hv_kmsg_dump() with kmsg data later, don't do the notification + * here. */ if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE && hyperv_report_reg()) { @@ -100,8 +100,8 @@ static int hyperv_die_event(struct notifier_block *nb, unsigned long val, /* * Hyper-V should be notified only once about a panic. If we will be - * doing hyperv_report_panic_msg() later with kmsg data, don't do - * the notification here. + * doing hv_kmsg_dump() with kmsg data later, don't do the notification + * here. */ if (hyperv_report_reg()) hyperv_report_panic(regs, val, true); @@ -921,6 +921,21 @@ static int vmbus_probe(struct device *child_device) } /* + * vmbus_dma_configure -- Configure DMA coherence for VMbus device + */ +static int vmbus_dma_configure(struct device *child_device) +{ + /* + * On ARM64, propagate the DMA coherence setting from the top level + * VMbus ACPI device to the child VMbus device being added here. + * On x86/x64 coherence is assumed and these calls have no effect. + */ + hv_setup_dma_ops(child_device, + device_get_dma_attr(&hv_acpi_dev->dev) == DEV_DMA_COHERENT); + return 0; +} + +/* * vmbus_remove - Remove a vmbus device */ static void vmbus_remove(struct device *child_device) @@ -1040,6 +1055,7 @@ static struct bus_type hv_bus = { .remove = vmbus_remove, .probe = vmbus_probe, .uevent = vmbus_uevent, + .dma_configure = vmbus_dma_configure, .dev_groups = vmbus_dev_groups, .drv_groups = vmbus_drv_groups, .bus_groups = vmbus_bus_groups, @@ -1546,14 +1562,20 @@ static int vmbus_bus_init(void) if (ret) goto err_connect; + if (hv_is_isolation_supported()) + sysctl_record_panic_msg = 0; + /* * Only register if the crash MSRs are available */ if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) { u64 hyperv_crash_ctl; /* - * Sysctl registration is not fatal, since by default - * reporting is enabled. + * Panic message recording (sysctl_record_panic_msg) + * is enabled by default in non-isolated guests and + * disabled by default in isolated guests; the panic + * message recording won't be available in isolated + * guests should the following registration fail. */ hv_ctl_table_hdr = register_sysctl_table(hv_root_table); if (!hv_ctl_table_hdr) @@ -2097,6 +2119,10 @@ int vmbus_device_register(struct hv_device *child_device_obj) child_device_obj->device.parent = &hv_acpi_dev->dev; child_device_obj->device.release = vmbus_device_release; + child_device_obj->device.dma_parms = &child_device_obj->dma_parms; + child_device_obj->device.dma_mask = &child_device_obj->dma_mask; + dma_set_mask(&child_device_obj->device, DMA_BIT_MASK(64)); + /* * Register with the LDM. This will kick off the driver/device * binding...which will eventually call vmbus_match() and vmbus_probe() @@ -2122,9 +2148,6 @@ int vmbus_device_register(struct hv_device *child_device_obj) } hv_debug_add_dev_dir(child_device_obj); - child_device_obj->device.dma_parms = &child_device_obj->dma_parms; - child_device_obj->device.dma_mask = &child_device_obj->dma_mask; - dma_set_mask(&child_device_obj->device, DMA_BIT_MASK(64)); return 0; err_kset_unregister: @@ -2428,6 +2451,21 @@ static int vmbus_acpi_add(struct acpi_device *device) hv_acpi_dev = device; + /* + * Older versions of Hyper-V for ARM64 fail to include the _CCA + * method on the top level VMbus device in the DSDT. But devices + * are hardware coherent in all current Hyper-V use cases, so fix + * up the ACPI device to behave as if _CCA is present and indicates + * hardware coherence. + */ + ACPI_COMPANION_SET(&device->dev, device); + if (IS_ENABLED(CONFIG_ACPI_CCA_REQUIRED) && + device_get_dma_attr(&device->dev) == DEV_DMA_NOT_SUPPORTED) { + pr_info("No ACPI _CCA found; assuming coherent device I/O\n"); + device->flags.cca_seen = true; + device->flags.coherent_dma = true; + } + result = acpi_walk_resources(device->handle, METHOD_NAME__CRS, vmbus_walk_resources, NULL); @@ -2780,10 +2818,15 @@ static void __exit vmbus_exit(void) if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) { kmsg_dump_unregister(&hv_kmsg_dumper); unregister_die_notifier(&hyperv_die_block); - atomic_notifier_chain_unregister(&panic_notifier_list, - &hyperv_panic_block); } + /* + * The panic notifier is always registered, hence we should + * also unconditionally unregister it here as well. + */ + atomic_notifier_chain_unregister(&panic_notifier_list, + &hyperv_panic_block); + free_page((unsigned long)hv_panic_page); unregister_sysctl_table(hv_ctl_table_hdr); hv_ctl_table_hdr = NULL; diff --git a/drivers/input/Kconfig b/drivers/input/Kconfig index 5baebf62df33..e2752f7364bc 100644 --- a/drivers/input/Kconfig +++ b/drivers/input/Kconfig @@ -77,6 +77,13 @@ config INPUT_MATRIXKMAP To compile this driver as a module, choose M here: the module will be called matrix-keymap. +config INPUT_VIVALDIFMAP + tristate + help + ChromeOS Vivaldi keymap support library. This is a hidden + option so that drivers can use common code to parse and + expose the vivaldi function row keymap. + comment "Userland interfaces" config INPUT_MOUSEDEV diff --git a/drivers/input/Makefile b/drivers/input/Makefile index 037cc595106c..2266c7d010ef 100644 --- a/drivers/input/Makefile +++ b/drivers/input/Makefile @@ -12,6 +12,7 @@ input-core-y += touchscreen.o obj-$(CONFIG_INPUT_FF_MEMLESS) += ff-memless.o obj-$(CONFIG_INPUT_SPARSEKMAP) += sparse-keymap.o obj-$(CONFIG_INPUT_MATRIXKMAP) += matrix-keymap.o +obj-$(CONFIG_INPUT_VIVALDIFMAP) += vivaldi-fmap.o obj-$(CONFIG_INPUT_LEDS) += input-leds.o obj-$(CONFIG_INPUT_MOUSEDEV) += mousedev.o diff --git a/drivers/input/input.c b/drivers/input/input.c index c3139bc2aa0d..e5a668ce884d 100644 --- a/drivers/input/input.c +++ b/drivers/input/input.c @@ -47,6 +47,17 @@ static DEFINE_MUTEX(input_mutex); static const struct input_value input_value_sync = { EV_SYN, SYN_REPORT, 1 }; +static const unsigned int input_max_code[EV_CNT] = { + [EV_KEY] = KEY_MAX, + [EV_REL] = REL_MAX, + [EV_ABS] = ABS_MAX, + [EV_MSC] = MSC_MAX, + [EV_SW] = SW_MAX, + [EV_LED] = LED_MAX, + [EV_SND] = SND_MAX, + [EV_FF] = FF_MAX, +}; + static inline int is_event_supported(unsigned int code, unsigned long *bm, unsigned int max) { @@ -511,6 +522,9 @@ void input_set_abs_params(struct input_dev *dev, unsigned int axis, { struct input_absinfo *absinfo; + __set_bit(EV_ABS, dev->evbit); + __set_bit(axis, dev->absbit); + input_alloc_absinfo(dev); if (!dev->absinfo) return; @@ -520,12 +534,45 @@ void input_set_abs_params(struct input_dev *dev, unsigned int axis, absinfo->maximum = max; absinfo->fuzz = fuzz; absinfo->flat = flat; - - __set_bit(EV_ABS, dev->evbit); - __set_bit(axis, dev->absbit); } EXPORT_SYMBOL(input_set_abs_params); +/** + * input_copy_abs - Copy absinfo from one input_dev to another + * @dst: Destination input device to copy the abs settings to + * @dst_axis: ABS_* value selecting the destination axis + * @src: Source input device to copy the abs settings from + * @src_axis: ABS_* value selecting the source axis + * + * Set absinfo for the selected destination axis by copying it from + * the specified source input device's source axis. + * This is useful to e.g. setup a pen/stylus input-device for combined + * touchscreen/pen hardware where the pen uses the same coordinates as + * the touchscreen. + */ +void input_copy_abs(struct input_dev *dst, unsigned int dst_axis, + const struct input_dev *src, unsigned int src_axis) +{ + /* src must have EV_ABS and src_axis set */ + if (WARN_ON(!(test_bit(EV_ABS, src->evbit) && + test_bit(src_axis, src->absbit)))) + return; + + /* + * input_alloc_absinfo() may have failed for the source. Our caller is + * expected to catch this when registering the input devices, which may + * happen after the input_copy_abs() call. + */ + if (!src->absinfo) + return; + + input_set_capability(dst, EV_ABS, dst_axis); + if (!dst->absinfo) + return; + + dst->absinfo[dst_axis] = src->absinfo[src_axis]; +} +EXPORT_SYMBOL(input_copy_abs); /** * input_grab_device - grabs device for exclusive use @@ -2074,6 +2121,14 @@ EXPORT_SYMBOL(input_get_timestamp); */ void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code) { + if (type < EV_CNT && input_max_code[type] && + code > input_max_code[type]) { + pr_err("%s: invalid code %u for type %u\n", __func__, code, + type); + dump_stack(); + return; + } + switch (type) { case EV_KEY: __set_bit(code, dev->keybit); @@ -2085,9 +2140,6 @@ void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int case EV_ABS: input_alloc_absinfo(dev); - if (!dev->absinfo) - return; - __set_bit(code, dev->absbit); break; @@ -2285,12 +2337,6 @@ int input_register_device(struct input_dev *dev) /* KEY_RESERVED is not supposed to be transmitted to userspace. */ __clear_bit(KEY_RESERVED, dev->keybit); - /* Buttonpads should not map BTN_RIGHT and/or BTN_MIDDLE. */ - if (test_bit(INPUT_PROP_BUTTONPAD, dev->propbit)) { - __clear_bit(BTN_RIGHT, dev->keybit); - __clear_bit(BTN_MIDDLE, dev->keybit); - } - /* Make sure that bitmasks not mentioned in dev->evbit are clean. */ input_cleanse_bitmasks(dev); diff --git a/drivers/input/joystick/adi.c b/drivers/input/joystick/adi.c index 592c95b87f54..e10d57bf1180 100644 --- a/drivers/input/joystick/adi.c +++ b/drivers/input/joystick/adi.c @@ -123,7 +123,7 @@ static void adi_read_packet(struct adi_port *port) { struct adi *adi = port->adi; struct gameport *gameport = port->gameport; - unsigned char u, v, w, x, z; + unsigned char u, v, w, x; int t[2], s[2], i; unsigned long flags; @@ -136,7 +136,7 @@ static void adi_read_packet(struct adi_port *port) local_irq_save(flags); gameport_trigger(gameport); - v = z = gameport_read(gameport); + v = gameport_read(gameport); do { u = v; diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 4c914f75a902..18190b529bca 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -131,7 +131,7 @@ static const struct xpad_device { { 0x045e, 0x02e3, "Microsoft X-Box One Elite pad", 0, XTYPE_XBOXONE }, { 0x045e, 0x02ea, "Microsoft X-Box One S pad", 0, XTYPE_XBOXONE }, { 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, - { 0x045e, 0x0b12, "Microsoft Xbox One X pad", MAP_SELECT_BUTTON, XTYPE_XBOXONE }, + { 0x045e, 0x0b12, "Microsoft Xbox Series S|X Controller", MAP_SELECT_BUTTON, XTYPE_XBOXONE }, { 0x046d, 0xc21d, "Logitech Gamepad F310", 0, XTYPE_XBOX360 }, { 0x046d, 0xc21e, "Logitech Gamepad F510", 0, XTYPE_XBOX360 }, { 0x046d, 0xc21f, "Logitech Gamepad F710", 0, XTYPE_XBOX360 }, diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig index 9417ee0b1eff..4ea79db8f134 100644 --- a/drivers/input/keyboard/Kconfig +++ b/drivers/input/keyboard/Kconfig @@ -103,6 +103,7 @@ config KEYBOARD_ATKBD select SERIO_LIBPS2 select SERIO_I8042 if ARCH_MIGHT_HAVE_PC_SERIO select SERIO_GSCPS2 if GSC + select INPUT_VIVALDIFMAP help Say Y here if you want to use a standard AT or PS/2 keyboard. Usually you'll need this, unless you have a different type keyboard (USB, ADB @@ -749,6 +750,7 @@ config KEYBOARD_XTKBD config KEYBOARD_CROS_EC tristate "ChromeOS EC keyboard" select INPUT_MATRIXKMAP + select INPUT_VIVALDIFMAP depends on CROS_EC help Say Y here to enable the matrix keyboard used by ChromeOS devices @@ -779,6 +781,18 @@ config KEYBOARD_BCM To compile this driver as a module, choose M here: the module will be called bcm-keypad. +config KEYBOARD_MT6779 + tristate "MediaTek Keypad Support" + depends on ARCH_MEDIATEK || COMPILE_TEST + select REGMAP_MMIO + select INPUT_MATRIXKMAP + help + Say Y here if you want to use the keypad on MediaTek SoCs. + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called mt6779-keypad. + config KEYBOARD_MTK_PMIC tristate "MediaTek PMIC keys support" depends on MFD_MT6397 diff --git a/drivers/input/keyboard/Makefile b/drivers/input/keyboard/Makefile index e3c8648f834e..721936e90290 100644 --- a/drivers/input/keyboard/Makefile +++ b/drivers/input/keyboard/Makefile @@ -44,6 +44,7 @@ obj-$(CONFIG_KEYBOARD_MATRIX) += matrix_keypad.o obj-$(CONFIG_KEYBOARD_MAX7359) += max7359_keypad.o obj-$(CONFIG_KEYBOARD_MCS) += mcs_touchkey.o obj-$(CONFIG_KEYBOARD_MPR121) += mpr121_touchkey.o +obj-$(CONFIG_KEYBOARD_MT6779) += mt6779-keypad.o obj-$(CONFIG_KEYBOARD_MTK_PMIC) += mtk-pmic-keys.o obj-$(CONFIG_KEYBOARD_NEWTON) += newtonkbd.o obj-$(CONFIG_KEYBOARD_NOMADIK) += nomadik-ske-keypad.o diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index fbdef95291e9..d4131236d18c 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -19,6 +19,7 @@ #include <linux/interrupt.h> #include <linux/init.h> #include <linux/input.h> +#include <linux/input/vivaldi-fmap.h> #include <linux/serio.h> #include <linux/workqueue.h> #include <linux/libps2.h> @@ -64,8 +65,6 @@ static bool atkbd_terminal; module_param_named(terminal, atkbd_terminal, bool, 0); MODULE_PARM_DESC(terminal, "Enable break codes on an IBM Terminal keyboard connected via AT/PS2"); -#define MAX_FUNCTION_ROW_KEYS 24 - #define SCANCODE(keymap) ((keymap >> 16) & 0xFFFF) #define KEYCODE(keymap) (keymap & 0xFFFF) @@ -237,8 +236,7 @@ struct atkbd { /* Serializes reconnect(), attr->set() and event work */ struct mutex mutex; - u32 function_row_physmap[MAX_FUNCTION_ROW_KEYS]; - int num_function_row_keys; + struct vivaldi_data vdata; }; /* @@ -308,17 +306,7 @@ static struct attribute *atkbd_attributes[] = { static ssize_t atkbd_show_function_row_physmap(struct atkbd *atkbd, char *buf) { - ssize_t size = 0; - int i; - - if (!atkbd->num_function_row_keys) - return 0; - - for (i = 0; i < atkbd->num_function_row_keys; i++) - size += scnprintf(buf + size, PAGE_SIZE - size, "%02X ", - atkbd->function_row_physmap[i]); - size += scnprintf(buf + size, PAGE_SIZE - size, "\n"); - return size; + return vivaldi_function_row_physmap_show(&atkbd->vdata, buf); } static umode_t atkbd_attr_is_visible(struct kobject *kobj, @@ -329,7 +317,7 @@ static umode_t atkbd_attr_is_visible(struct kobject *kobj, struct atkbd *atkbd = serio_get_drvdata(serio); if (attr == &atkbd_attr_function_row_physmap.attr && - !atkbd->num_function_row_keys) + !atkbd->vdata.num_function_row_keys) return 0; return attr->mode; @@ -1206,10 +1194,11 @@ static void atkbd_parse_fwnode_data(struct serio *serio) /* Parse "function-row-physmap" property */ n = device_property_count_u32(dev, "function-row-physmap"); - if (n > 0 && n <= MAX_FUNCTION_ROW_KEYS && + if (n > 0 && n <= VIVALDI_MAX_FUNCTION_ROW_KEYS && !device_property_read_u32_array(dev, "function-row-physmap", - atkbd->function_row_physmap, n)) { - atkbd->num_function_row_keys = n; + atkbd->vdata.function_row_physmap, + n)) { + atkbd->vdata.num_function_row_keys = n; dev_dbg(dev, "FW reported %d function-row key locations\n", n); } } diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c index fc02c540636e..6534dfca60b4 100644 --- a/drivers/input/keyboard/cros_ec_keyb.c +++ b/drivers/input/keyboard/cros_ec_keyb.c @@ -15,6 +15,7 @@ #include <linux/bitops.h> #include <linux/i2c.h> #include <linux/input.h> +#include <linux/input/vivaldi-fmap.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/notifier.h> @@ -27,8 +28,6 @@ #include <asm/unaligned.h> -#define MAX_NUM_TOP_ROW_KEYS 15 - /** * struct cros_ec_keyb - Structure representing EC keyboard device * @@ -44,9 +43,7 @@ * @idev: The input device for the matrix keys. * @bs_idev: The input device for non-matrix buttons and switches (or NULL). * @notifier: interrupt event notifier for transport devices - * @function_row_physmap: An array of the encoded rows/columns for the top - * row function keys, in an order from left to right - * @num_function_row_keys: The number of top row keys in a custom keyboard + * @vdata: vivaldi function row data */ struct cros_ec_keyb { unsigned int rows; @@ -64,8 +61,7 @@ struct cros_ec_keyb { struct input_dev *bs_idev; struct notifier_block notifier; - u16 function_row_physmap[MAX_NUM_TOP_ROW_KEYS]; - size_t num_function_row_keys; + struct vivaldi_data vdata; }; /** @@ -537,9 +533,9 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev) int err; struct property *prop; const __be32 *p; - u16 *physmap; + u32 *physmap; u32 key_pos; - int row, col; + unsigned int row, col, scancode, n_physmap; err = matrix_keypad_parse_properties(dev, &ckdev->rows, &ckdev->cols); if (err) @@ -591,20 +587,21 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev) ckdev->idev = idev; cros_ec_keyb_compute_valid_keys(ckdev); - physmap = ckdev->function_row_physmap; + physmap = ckdev->vdata.function_row_physmap; + n_physmap = 0; of_property_for_each_u32(dev->of_node, "function-row-physmap", prop, p, key_pos) { - if (ckdev->num_function_row_keys == MAX_NUM_TOP_ROW_KEYS) { + if (n_physmap == VIVALDI_MAX_FUNCTION_ROW_KEYS) { dev_warn(dev, "Only support up to %d top row keys\n", - MAX_NUM_TOP_ROW_KEYS); + VIVALDI_MAX_FUNCTION_ROW_KEYS); break; } row = KEY_ROW(key_pos); col = KEY_COL(key_pos); - *physmap = MATRIX_SCAN_CODE(row, col, ckdev->row_shift); - physmap++; - ckdev->num_function_row_keys++; + scancode = MATRIX_SCAN_CODE(row, col, ckdev->row_shift); + physmap[n_physmap++] = scancode; } + ckdev->vdata.num_function_row_keys = n_physmap; err = input_register_device(ckdev->idev); if (err) { @@ -619,18 +616,10 @@ static ssize_t function_row_physmap_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t size = 0; - int i; - struct cros_ec_keyb *ckdev = dev_get_drvdata(dev); - u16 *physmap = ckdev->function_row_physmap; - - for (i = 0; i < ckdev->num_function_row_keys; i++) - size += scnprintf(buf + size, PAGE_SIZE - size, - "%s%02X", size ? " " : "", physmap[i]); - if (size) - size += scnprintf(buf + size, PAGE_SIZE - size, "\n"); + const struct cros_ec_keyb *ckdev = dev_get_drvdata(dev); + const struct vivaldi_data *data = &ckdev->vdata; - return size; + return vivaldi_function_row_physmap_show(data, buf); } static DEVICE_ATTR_RO(function_row_physmap); @@ -648,7 +637,7 @@ static umode_t cros_ec_keyb_attr_is_visible(struct kobject *kobj, struct cros_ec_keyb *ckdev = dev_get_drvdata(dev); if (attr == &dev_attr_function_row_physmap.attr && - !ckdev->num_function_row_keys) + !ckdev->vdata.num_function_row_keys) return 0; return attr->mode; diff --git a/drivers/input/keyboard/mt6779-keypad.c b/drivers/input/keyboard/mt6779-keypad.c new file mode 100644 index 000000000000..0dbbddc7f298 --- /dev/null +++ b/drivers/input/keyboard/mt6779-keypad.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + * Author Fengping Yu <fengping.yu@mediatek.com> + */ +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/input/matrix_keypad.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/property.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#define MTK_KPD_NAME "mt6779-keypad" +#define MTK_KPD_MEM 0x0004 +#define MTK_KPD_DEBOUNCE 0x0018 +#define MTK_KPD_DEBOUNCE_MASK GENMASK(13, 0) +#define MTK_KPD_DEBOUNCE_MAX_MS 256 +#define MTK_KPD_NUM_MEMS 5 +#define MTK_KPD_NUM_BITS 136 /* 4*32+8 MEM5 only use 8 BITS */ + +struct mt6779_keypad { + struct regmap *regmap; + struct input_dev *input_dev; + struct clk *clk; + void __iomem *base; + u32 n_rows; + u32 n_cols; + DECLARE_BITMAP(keymap_state, MTK_KPD_NUM_BITS); +}; + +static const struct regmap_config mt6779_keypad_regmap_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = sizeof(u32), + .max_register = 36, +}; + +static irqreturn_t mt6779_keypad_irq_handler(int irq, void *dev_id) +{ + struct mt6779_keypad *keypad = dev_id; + const unsigned short *keycode = keypad->input_dev->keycode; + DECLARE_BITMAP(new_state, MTK_KPD_NUM_BITS); + DECLARE_BITMAP(change, MTK_KPD_NUM_BITS); + unsigned int bit_nr; + unsigned int row, col; + unsigned int scancode; + unsigned int row_shift = get_count_order(keypad->n_cols); + bool pressed; + + regmap_bulk_read(keypad->regmap, MTK_KPD_MEM, + new_state, MTK_KPD_NUM_MEMS); + + bitmap_xor(change, new_state, keypad->keymap_state, MTK_KPD_NUM_BITS); + + for_each_set_bit(bit_nr, change, MTK_KPD_NUM_BITS) { + /* + * Registers are 32bits, but only bits [15:0] are used to + * indicate key status. + */ + if (bit_nr % 32 >= 16) + continue; + + row = bit_nr / 32; + col = bit_nr % 32; + scancode = MATRIX_SCAN_CODE(row, col, row_shift); + /* 1: not pressed, 0: pressed */ + pressed = !test_bit(bit_nr, new_state); + dev_dbg(&keypad->input_dev->dev, "%s", + pressed ? "pressed" : "released"); + + input_event(keypad->input_dev, EV_MSC, MSC_SCAN, scancode); + input_report_key(keypad->input_dev, keycode[scancode], pressed); + input_sync(keypad->input_dev); + + dev_dbg(&keypad->input_dev->dev, + "report Linux keycode = %d\n", keycode[scancode]); + } + + bitmap_copy(keypad->keymap_state, new_state, MTK_KPD_NUM_BITS); + + return IRQ_HANDLED; +} + +static void mt6779_keypad_clk_disable(void *data) +{ + clk_disable_unprepare(data); +} + +static int mt6779_keypad_pdrv_probe(struct platform_device *pdev) +{ + struct mt6779_keypad *keypad; + int irq; + u32 debounce; + bool wakeup; + int error; + + keypad = devm_kzalloc(&pdev->dev, sizeof(*keypad), GFP_KERNEL); + if (!keypad) + return -ENOMEM; + + keypad->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(keypad->base)) + return PTR_ERR(keypad->base); + + keypad->regmap = devm_regmap_init_mmio(&pdev->dev, keypad->base, + &mt6779_keypad_regmap_cfg); + if (IS_ERR(keypad->regmap)) { + dev_err(&pdev->dev, + "regmap init failed:%pe\n", keypad->regmap); + return PTR_ERR(keypad->regmap); + } + + bitmap_fill(keypad->keymap_state, MTK_KPD_NUM_BITS); + + keypad->input_dev = devm_input_allocate_device(&pdev->dev); + if (!keypad->input_dev) { + dev_err(&pdev->dev, "Failed to allocate input dev\n"); + return -ENOMEM; + } + + keypad->input_dev->name = MTK_KPD_NAME; + keypad->input_dev->id.bustype = BUS_HOST; + + error = matrix_keypad_parse_properties(&pdev->dev, &keypad->n_rows, + &keypad->n_cols); + if (error) { + dev_err(&pdev->dev, "Failed to parse keypad params\n"); + return error; + } + + if (device_property_read_u32(&pdev->dev, "debounce-delay-ms", + &debounce)) + debounce = 16; + + if (debounce > MTK_KPD_DEBOUNCE_MAX_MS) { + dev_err(&pdev->dev, + "Debounce time exceeds the maximum allowed time %dms\n", + MTK_KPD_DEBOUNCE_MAX_MS); + return -EINVAL; + } + + wakeup = device_property_read_bool(&pdev->dev, "wakeup-source"); + + dev_dbg(&pdev->dev, "n_row=%d n_col=%d debounce=%d\n", + keypad->n_rows, keypad->n_cols, debounce); + + error = matrix_keypad_build_keymap(NULL, NULL, + keypad->n_rows, keypad->n_cols, + NULL, keypad->input_dev); + if (error) { + dev_err(&pdev->dev, "Failed to build keymap\n"); + return error; + } + + input_set_capability(keypad->input_dev, EV_MSC, MSC_SCAN); + + regmap_write(keypad->regmap, MTK_KPD_DEBOUNCE, + (debounce * (1 << 5)) & MTK_KPD_DEBOUNCE_MASK); + + keypad->clk = devm_clk_get(&pdev->dev, "kpd"); + if (IS_ERR(keypad->clk)) + return PTR_ERR(keypad->clk); + + error = clk_prepare_enable(keypad->clk); + if (error) { + dev_err(&pdev->dev, "cannot prepare/enable keypad clock\n"); + return error; + } + + error = devm_add_action_or_reset(&pdev->dev, mt6779_keypad_clk_disable, + keypad->clk); + if (error) + return error; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + error = devm_request_threaded_irq(&pdev->dev, irq, + NULL, mt6779_keypad_irq_handler, + IRQF_ONESHOT, MTK_KPD_NAME, keypad); + if (error) { + dev_err(&pdev->dev, "Failed to request IRQ#%d: %d\n", + irq, error); + return error; + } + + error = input_register_device(keypad->input_dev); + if (error) { + dev_err(&pdev->dev, "Failed to register device\n"); + return error; + } + + error = device_init_wakeup(&pdev->dev, wakeup); + if (error) + dev_warn(&pdev->dev, "device_init_wakeup() failed: %d\n", + error); + + return 0; +} + +static const struct of_device_id mt6779_keypad_of_match[] = { + { .compatible = "mediatek,mt6779-keypad" }, + { .compatible = "mediatek,mt6873-keypad" }, + { /* sentinel */ } +}; + +static struct platform_driver mt6779_keypad_pdrv = { + .probe = mt6779_keypad_pdrv_probe, + .driver = { + .name = MTK_KPD_NAME, + .of_match_table = mt6779_keypad_of_match, + }, +}; +module_platform_driver(mt6779_keypad_pdrv); + +MODULE_AUTHOR("Mediatek Corporation"); +MODULE_DESCRIPTION("MTK Keypad (KPD) Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/keyboard/mtk-pmic-keys.c b/drivers/input/keyboard/mtk-pmic-keys.c index 62391d6c7da6..c31ab4368388 100644 --- a/drivers/input/keyboard/mtk-pmic-keys.c +++ b/drivers/input/keyboard/mtk-pmic-keys.c @@ -9,6 +9,7 @@ #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/mfd/mt6323/registers.h> +#include <linux/mfd/mt6358/registers.h> #include <linux/mfd/mt6397/core.h> #include <linux/mfd/mt6397/registers.h> #include <linux/module.h> @@ -74,11 +75,22 @@ static const struct mtk_pmic_regs mt6323_regs = { .pmic_rst_reg = MT6323_TOP_RST_MISC, }; +static const struct mtk_pmic_regs mt6358_regs = { + .keys_regs[MTK_PMIC_PWRKEY_INDEX] = + MTK_PMIC_KEYS_REGS(MT6358_TOPSTATUS, + 0x2, MT6358_PSC_TOP_INT_CON0, 0x5), + .keys_regs[MTK_PMIC_HOMEKEY_INDEX] = + MTK_PMIC_KEYS_REGS(MT6358_TOPSTATUS, + 0x8, MT6358_PSC_TOP_INT_CON0, 0xa), + .pmic_rst_reg = MT6358_TOP_RST_MISC, +}; + struct mtk_pmic_keys_info { struct mtk_pmic_keys *keys; const struct mtk_pmic_keys_regs *regs; unsigned int keycode; int irq; + int irq_r; /* optional: release irq if different */ bool wakeup:1; }; @@ -188,6 +200,18 @@ static int mtk_pmic_key_setup(struct mtk_pmic_keys *keys, return ret; } + if (info->irq_r > 0) { + ret = devm_request_threaded_irq(keys->dev, info->irq_r, NULL, + mtk_pmic_keys_irq_handler_thread, + IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + "mtk-pmic-keys", info); + if (ret) { + dev_err(keys->dev, "Failed to request IRQ_r: %d: %d\n", + info->irq, ret); + return ret; + } + } + input_set_capability(keys->input_dev, EV_KEY, info->keycode); return 0; @@ -199,8 +223,11 @@ static int __maybe_unused mtk_pmic_keys_suspend(struct device *dev) int index; for (index = 0; index < MTK_PMIC_MAX_KEY_COUNT; index++) { - if (keys->keys[index].wakeup) + if (keys->keys[index].wakeup) { enable_irq_wake(keys->keys[index].irq); + if (keys->keys[index].irq_r > 0) + enable_irq_wake(keys->keys[index].irq_r); + } } return 0; @@ -212,8 +239,11 @@ static int __maybe_unused mtk_pmic_keys_resume(struct device *dev) int index; for (index = 0; index < MTK_PMIC_MAX_KEY_COUNT; index++) { - if (keys->keys[index].wakeup) + if (keys->keys[index].wakeup) { disable_irq_wake(keys->keys[index].irq); + if (keys->keys[index].irq_r > 0) + disable_irq_wake(keys->keys[index].irq_r); + } } return 0; @@ -230,6 +260,9 @@ static const struct of_device_id of_mtk_pmic_keys_match_tbl[] = { .compatible = "mediatek,mt6323-keys", .data = &mt6323_regs, }, { + .compatible = "mediatek,mt6358-keys", + .data = &mt6358_regs, + }, { /* sentinel */ } }; @@ -241,6 +274,8 @@ static int mtk_pmic_keys_probe(struct platform_device *pdev) unsigned int keycount; struct mt6397_chip *pmic_chip = dev_get_drvdata(pdev->dev.parent); struct device_node *node = pdev->dev.of_node, *child; + static const char *const irqnames[] = { "powerkey", "homekey" }; + static const char *const irqnames_r[] = { "powerkey_r", "homekey_r" }; struct mtk_pmic_keys *keys; const struct mtk_pmic_regs *mtk_pmic_regs; struct input_dev *input_dev; @@ -268,7 +303,8 @@ static int mtk_pmic_keys_probe(struct platform_device *pdev) input_dev->id.version = 0x0001; keycount = of_get_available_child_count(node); - if (keycount > MTK_PMIC_MAX_KEY_COUNT) { + if (keycount > MTK_PMIC_MAX_KEY_COUNT || + keycount > ARRAY_SIZE(irqnames)) { dev_err(keys->dev, "too many keys defined (%d)\n", keycount); return -EINVAL; } @@ -276,12 +312,23 @@ static int mtk_pmic_keys_probe(struct platform_device *pdev) for_each_child_of_node(node, child) { keys->keys[index].regs = &mtk_pmic_regs->keys_regs[index]; - keys->keys[index].irq = platform_get_irq(pdev, index); + keys->keys[index].irq = + platform_get_irq_byname(pdev, irqnames[index]); if (keys->keys[index].irq < 0) { of_node_put(child); return keys->keys[index].irq; } + if (of_device_is_compatible(node, "mediatek,mt6358-keys")) { + keys->keys[index].irq_r = platform_get_irq_byname(pdev, + irqnames_r[index]); + + if (keys->keys[index].irq_r < 0) { + of_node_put(child); + return keys->keys[index].irq_r; + } + } + error = of_property_read_u32(child, "linux,keycodes", &keys->keys[index].keycode); if (error) { diff --git a/drivers/input/misc/da9063_onkey.c b/drivers/input/misc/da9063_onkey.c index 79851923ee57..b14a389600c9 100644 --- a/drivers/input/misc/da9063_onkey.c +++ b/drivers/input/misc/da9063_onkey.c @@ -4,6 +4,7 @@ * Copyright (C) 2015 Dialog Semiconductor Ltd. */ +#include <linux/devm-helpers.h> #include <linux/module.h> #include <linux/errno.h> #include <linux/input.h> @@ -182,13 +183,6 @@ static irqreturn_t da9063_onkey_irq_handler(int irq, void *data) return IRQ_HANDLED; } -static void da9063_cancel_poll(void *data) -{ - struct da9063_onkey *onkey = data; - - cancel_delayed_work_sync(&onkey->work); -} - static int da9063_onkey_probe(struct platform_device *pdev) { struct da9063_onkey *onkey; @@ -234,9 +228,8 @@ static int da9063_onkey_probe(struct platform_device *pdev) input_set_capability(onkey->input, EV_KEY, KEY_POWER); - INIT_DELAYED_WORK(&onkey->work, da9063_poll_on); - - error = devm_add_action(&pdev->dev, da9063_cancel_poll, onkey); + error = devm_delayed_work_autocancel(&pdev->dev, &onkey->work, + da9063_poll_on); if (error) { dev_err(&pdev->dev, "Failed to add cancel poll action: %d\n", diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index ffad142801b3..434d48ae4b12 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -186,6 +186,7 @@ static const char * const smbus_pnp_ids[] = { "LEN2044", /* L470 */ "LEN2054", /* E480 */ "LEN2055", /* E580 */ + "LEN2064", /* T14 Gen 1 AMD / P14s Gen 1 AMD */ "LEN2068", /* T14 Gen 1 */ "SYN3052", /* HP EliteBook 840 G4 */ "SYN3221", /* HP 15-ay000 */ diff --git a/drivers/input/serio/ps2-gpio.c b/drivers/input/serio/ps2-gpio.c index 8970b49ea09a..9b02dd5dd2b9 100644 --- a/drivers/input/serio/ps2-gpio.c +++ b/drivers/input/serio/ps2-gpio.c @@ -19,6 +19,7 @@ #include <linux/of.h> #include <linux/jiffies.h> #include <linux/delay.h> +#include <linux/timekeeping.h> #define DRIVER_NAME "ps2-gpio" @@ -36,14 +37,37 @@ #define PS2_DATA_BIT7 8 #define PS2_PARITY_BIT 9 #define PS2_STOP_BIT 10 -#define PS2_TX_TIMEOUT 11 -#define PS2_ACK_BIT 12 +#define PS2_ACK_BIT 11 #define PS2_DEV_RET_ACK 0xfa #define PS2_DEV_RET_NACK 0xfe #define PS2_CMD_RESEND 0xfe +/* + * The PS2 protocol specifies a clock frequency between 10kHz and 16.7kHz, + * therefore the maximal interrupt interval should be 100us and the minimum + * interrupt interval should be ~60us. Let's allow +/- 20us for frequency + * deviations and interrupt latency. + * + * The data line must be samples after ~30us to 50us after the falling edge, + * since the device updates the data line at the rising edge. + * + * ___ ______ ______ ______ ___ + * \ / \ / \ / \ / + * \ / \ / \ / \ / + * \______/ \______/ \______/ \______/ + * + * |-----------------| |--------| + * 60us/100us 30us/50us + */ +#define PS2_CLK_FREQ_MIN_HZ 10000 +#define PS2_CLK_FREQ_MAX_HZ 16700 +#define PS2_CLK_MIN_INTERVAL_US ((1000 * 1000) / PS2_CLK_FREQ_MAX_HZ) +#define PS2_CLK_MAX_INTERVAL_US ((1000 * 1000) / PS2_CLK_FREQ_MIN_HZ) +#define PS2_IRQ_MIN_INTERVAL_US (PS2_CLK_MIN_INTERVAL_US - 20) +#define PS2_IRQ_MAX_INTERVAL_US (PS2_CLK_MAX_INTERVAL_US + 20) + struct ps2_gpio_data { struct device *dev; struct serio *serio; @@ -52,19 +76,30 @@ struct ps2_gpio_data { struct gpio_desc *gpio_data; bool write_enable; int irq; - unsigned char rx_cnt; - unsigned char rx_byte; - unsigned char tx_cnt; - unsigned char tx_byte; - struct completion tx_done; - struct mutex tx_mutex; - struct delayed_work tx_work; + ktime_t t_irq_now; + ktime_t t_irq_last; + struct { + unsigned char cnt; + unsigned char byte; + } rx; + struct { + unsigned char cnt; + unsigned char byte; + ktime_t t_xfer_start; + ktime_t t_xfer_end; + struct completion complete; + struct mutex mutex; + struct delayed_work work; + } tx; }; static int ps2_gpio_open(struct serio *serio) { struct ps2_gpio_data *drvdata = serio->port_data; + drvdata->t_irq_last = 0; + drvdata->tx.t_xfer_end = 0; + enable_irq(drvdata->irq); return 0; } @@ -73,7 +108,7 @@ static void ps2_gpio_close(struct serio *serio) { struct ps2_gpio_data *drvdata = serio->port_data; - flush_delayed_work(&drvdata->tx_work); + flush_delayed_work(&drvdata->tx.work); disable_irq(drvdata->irq); } @@ -85,9 +120,9 @@ static int __ps2_gpio_write(struct serio *serio, unsigned char val) gpiod_direction_output(drvdata->gpio_clk, 0); drvdata->mode = PS2_MODE_TX; - drvdata->tx_byte = val; + drvdata->tx.byte = val; - schedule_delayed_work(&drvdata->tx_work, usecs_to_jiffies(200)); + schedule_delayed_work(&drvdata->tx.work, usecs_to_jiffies(200)); return 0; } @@ -98,12 +133,12 @@ static int ps2_gpio_write(struct serio *serio, unsigned char val) int ret = 0; if (in_task()) { - mutex_lock(&drvdata->tx_mutex); + mutex_lock(&drvdata->tx.mutex); __ps2_gpio_write(serio, val); - if (!wait_for_completion_timeout(&drvdata->tx_done, + if (!wait_for_completion_timeout(&drvdata->tx.complete, msecs_to_jiffies(10000))) ret = SERIO_TIMEOUT; - mutex_unlock(&drvdata->tx_mutex); + mutex_unlock(&drvdata->tx.mutex); } else { __ps2_gpio_write(serio, val); } @@ -115,9 +150,10 @@ static void ps2_gpio_tx_work_fn(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); struct ps2_gpio_data *drvdata = container_of(dwork, - struct ps2_gpio_data, - tx_work); + struct ps2_gpio_data, + tx.work); + drvdata->tx.t_xfer_start = ktime_get(); enable_irq(drvdata->irq); gpiod_direction_output(drvdata->gpio_data, 0); gpiod_direction_input(drvdata->gpio_clk); @@ -128,20 +164,31 @@ static irqreturn_t ps2_gpio_irq_rx(struct ps2_gpio_data *drvdata) unsigned char byte, cnt; int data; int rxflags = 0; - static unsigned long old_jiffies; + s64 us_delta; - byte = drvdata->rx_byte; - cnt = drvdata->rx_cnt; + byte = drvdata->rx.byte; + cnt = drvdata->rx.cnt; - if (old_jiffies == 0) - old_jiffies = jiffies; + drvdata->t_irq_now = ktime_get(); + + /* + * We need to consider spurious interrupts happening right after + * a TX xfer finished. + */ + us_delta = ktime_us_delta(drvdata->t_irq_now, drvdata->tx.t_xfer_end); + if (unlikely(us_delta < PS2_IRQ_MIN_INTERVAL_US)) + goto end; - if ((jiffies - old_jiffies) > usecs_to_jiffies(100)) { + us_delta = ktime_us_delta(drvdata->t_irq_now, drvdata->t_irq_last); + if (us_delta > PS2_IRQ_MAX_INTERVAL_US && cnt) { dev_err(drvdata->dev, "RX: timeout, probably we missed an interrupt\n"); goto err; + } else if (unlikely(us_delta < PS2_IRQ_MIN_INTERVAL_US)) { + /* Ignore spurious IRQs. */ + goto end; } - old_jiffies = jiffies; + drvdata->t_irq_last = drvdata->t_irq_now; data = gpiod_get_value(drvdata->gpio_data); if (unlikely(data < 0)) { @@ -178,8 +225,16 @@ static irqreturn_t ps2_gpio_irq_rx(struct ps2_gpio_data *drvdata) if (!drvdata->write_enable) goto err; } + break; + case PS2_STOP_BIT: + /* stop bit should be high */ + if (unlikely(!data)) { + dev_err(drvdata->dev, "RX: stop bit should be high\n"); + goto err; + } - /* Do not send spurious ACK's and NACK's when write fn is + /* + * Do not send spurious ACK's and NACK's when write fn is * not provided. */ if (!drvdata->write_enable) { @@ -189,23 +244,11 @@ static irqreturn_t ps2_gpio_irq_rx(struct ps2_gpio_data *drvdata) break; } - /* Let's send the data without waiting for the stop bit to be - * sent. It may happen that we miss the stop bit. When this - * happens we have no way to recover from this, certainly - * missing the parity bit would be recognized when processing - * the stop bit. When missing both, data is lost. - */ serio_interrupt(drvdata->serio, byte, rxflags); dev_dbg(drvdata->dev, "RX: sending byte 0x%x\n", byte); - break; - case PS2_STOP_BIT: - /* stop bit should be high */ - if (unlikely(!data)) { - dev_err(drvdata->dev, "RX: stop bit should be high\n"); - goto err; - } + cnt = byte = 0; - old_jiffies = 0; + goto end; /* success */ default: dev_err(drvdata->dev, "RX: got out of sync with the device\n"); @@ -217,11 +260,10 @@ static irqreturn_t ps2_gpio_irq_rx(struct ps2_gpio_data *drvdata) err: cnt = byte = 0; - old_jiffies = 0; __ps2_gpio_write(drvdata->serio, PS2_CMD_RESEND); end: - drvdata->rx_cnt = cnt; - drvdata->rx_byte = byte; + drvdata->rx.cnt = cnt; + drvdata->rx.byte = byte; return IRQ_HANDLED; } @@ -229,20 +271,34 @@ static irqreturn_t ps2_gpio_irq_tx(struct ps2_gpio_data *drvdata) { unsigned char byte, cnt; int data; - static unsigned long old_jiffies; + s64 us_delta; + + cnt = drvdata->tx.cnt; + byte = drvdata->tx.byte; - cnt = drvdata->tx_cnt; - byte = drvdata->tx_byte; + drvdata->t_irq_now = ktime_get(); - if (old_jiffies == 0) - old_jiffies = jiffies; + /* + * There might be pending IRQs since we disabled IRQs in + * __ps2_gpio_write(). We can expect at least one clock period until + * the device generates the first falling edge after releasing the + * clock line. + */ + us_delta = ktime_us_delta(drvdata->t_irq_now, + drvdata->tx.t_xfer_start); + if (unlikely(us_delta < PS2_CLK_MIN_INTERVAL_US)) + goto end; - if ((jiffies - old_jiffies) > usecs_to_jiffies(100)) { + us_delta = ktime_us_delta(drvdata->t_irq_now, drvdata->t_irq_last); + if (us_delta > PS2_IRQ_MAX_INTERVAL_US && cnt > 1) { dev_err(drvdata->dev, "TX: timeout, probably we missed an interrupt\n"); goto err; + } else if (unlikely(us_delta < PS2_IRQ_MIN_INTERVAL_US)) { + /* Ignore spurious IRQs. */ + goto end; } - old_jiffies = jiffies; + drvdata->t_irq_last = drvdata->t_irq_now; switch (cnt) { case PS2_START_BIT: @@ -270,27 +326,22 @@ static irqreturn_t ps2_gpio_irq_tx(struct ps2_gpio_data *drvdata) /* release data line to generate stop bit */ gpiod_direction_input(drvdata->gpio_data); break; - case PS2_TX_TIMEOUT: - /* Devices generate one extra clock pulse before sending the - * acknowledgment. - */ - break; case PS2_ACK_BIT: - gpiod_direction_input(drvdata->gpio_data); data = gpiod_get_value(drvdata->gpio_data); if (data) { dev_warn(drvdata->dev, "TX: received NACK, retry\n"); goto err; } + drvdata->tx.t_xfer_end = ktime_get(); drvdata->mode = PS2_MODE_RX; - complete(&drvdata->tx_done); + complete(&drvdata->tx.complete); cnt = 1; - old_jiffies = 0; goto end; /* success */ default: - /* Probably we missed the stop bit. Therefore we release data + /* + * Probably we missed the stop bit. Therefore we release data * line and try again. */ gpiod_direction_input(drvdata->gpio_data); @@ -303,11 +354,10 @@ static irqreturn_t ps2_gpio_irq_tx(struct ps2_gpio_data *drvdata) err: cnt = 1; - old_jiffies = 0; gpiod_direction_input(drvdata->gpio_data); - __ps2_gpio_write(drvdata->serio, drvdata->tx_byte); + __ps2_gpio_write(drvdata->serio, drvdata->tx.byte); end: - drvdata->tx_cnt = cnt; + drvdata->tx.cnt = cnt; return IRQ_HANDLED; } @@ -322,14 +372,19 @@ static irqreturn_t ps2_gpio_irq(int irq, void *dev_id) static int ps2_gpio_get_props(struct device *dev, struct ps2_gpio_data *drvdata) { - drvdata->gpio_data = devm_gpiod_get(dev, "data", GPIOD_IN); + enum gpiod_flags gflags; + + /* Enforce open drain, since this is required by the PS/2 bus. */ + gflags = GPIOD_IN | GPIOD_FLAGS_BIT_OPEN_DRAIN; + + drvdata->gpio_data = devm_gpiod_get(dev, "data", gflags); if (IS_ERR(drvdata->gpio_data)) { dev_err(dev, "failed to request data gpio: %ld", PTR_ERR(drvdata->gpio_data)); return PTR_ERR(drvdata->gpio_data); } - drvdata->gpio_clk = devm_gpiod_get(dev, "clk", GPIOD_IN); + drvdata->gpio_clk = devm_gpiod_get(dev, "clk", gflags); if (IS_ERR(drvdata->gpio_clk)) { dev_err(dev, "failed to request clock gpio: %ld", PTR_ERR(drvdata->gpio_clk)); @@ -387,7 +442,8 @@ static int ps2_gpio_probe(struct platform_device *pdev) serio->id.type = SERIO_8042; serio->open = ps2_gpio_open; serio->close = ps2_gpio_close; - /* Write can be enabled in platform/dt data, but possibly it will not + /* + * Write can be enabled in platform/dt data, but possibly it will not * work because of the tough timings. */ serio->write = drvdata->write_enable ? ps2_gpio_write : NULL; @@ -400,14 +456,15 @@ static int ps2_gpio_probe(struct platform_device *pdev) drvdata->dev = dev; drvdata->mode = PS2_MODE_RX; - /* Tx count always starts at 1, as the start bit is sent implicitly by + /* + * Tx count always starts at 1, as the start bit is sent implicitly by * host-to-device communication initialization. */ - drvdata->tx_cnt = 1; + drvdata->tx.cnt = 1; - INIT_DELAYED_WORK(&drvdata->tx_work, ps2_gpio_tx_work_fn); - init_completion(&drvdata->tx_done); - mutex_init(&drvdata->tx_mutex); + INIT_DELAYED_WORK(&drvdata->tx.work, ps2_gpio_tx_work_fn); + init_completion(&drvdata->tx.complete); + mutex_init(&drvdata->tx.mutex); serio_register_port(serio); platform_set_drvdata(pdev, drvdata); diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig index ff7794cecf69..43c7d6e5bdc0 100644 --- a/drivers/input/touchscreen/Kconfig +++ b/drivers/input/touchscreen/Kconfig @@ -638,6 +638,16 @@ config TOUCHSCREEN_MTOUCH To compile this driver as a module, choose M here: the module will be called mtouch. +config TOUCHSCREEN_IMAGIS + tristate "Imagis touchscreen support" + depends on I2C + help + Say Y here if you have an Imagis IST30xxC touchscreen. + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called imagis. + config TOUCHSCREEN_IMX6UL_TSC tristate "Freescale i.MX6UL touchscreen controller" depends on ((OF && GPIOLIB) || COMPILE_TEST) && HAS_IOMEM diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile index 39a8127cf6a5..557f84fd2075 100644 --- a/drivers/input/touchscreen/Makefile +++ b/drivers/input/touchscreen/Makefile @@ -49,6 +49,7 @@ obj-$(CONFIG_TOUCHSCREEN_GOODIX) += goodix_ts.o obj-$(CONFIG_TOUCHSCREEN_HIDEEP) += hideep.o obj-$(CONFIG_TOUCHSCREEN_ILI210X) += ili210x.o obj-$(CONFIG_TOUCHSCREEN_ILITEK) += ilitek_ts_i2c.o +obj-$(CONFIG_TOUCHSCREEN_IMAGIS) += imagis.o obj-$(CONFIG_TOUCHSCREEN_IMX6UL_TSC) += imx6ul_tsc.o obj-$(CONFIG_TOUCHSCREEN_INEXIO) += inexio.o obj-$(CONFIG_TOUCHSCREEN_IPROC) += bcm_iproc_tsc.o diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 752e8ba4fecb..3ad9870db108 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -298,32 +298,17 @@ static int goodix_ts_read_input_report(struct goodix_ts_data *ts, u8 *data) return -ENOMSG; } -static struct input_dev *goodix_create_pen_input(struct goodix_ts_data *ts) +static int goodix_create_pen_input(struct goodix_ts_data *ts) { struct device *dev = &ts->client->dev; struct input_dev *input; input = devm_input_allocate_device(dev); if (!input) - return NULL; - - input_alloc_absinfo(input); - if (!input->absinfo) { - input_free_device(input); - return NULL; - } - - input->absinfo[ABS_X] = ts->input_dev->absinfo[ABS_MT_POSITION_X]; - input->absinfo[ABS_Y] = ts->input_dev->absinfo[ABS_MT_POSITION_Y]; - __set_bit(ABS_X, input->absbit); - __set_bit(ABS_Y, input->absbit); - input_set_abs_params(input, ABS_PRESSURE, 0, 255, 0, 0); + return -ENOMEM; - input_set_capability(input, EV_KEY, BTN_TOUCH); - input_set_capability(input, EV_KEY, BTN_TOOL_PEN); - input_set_capability(input, EV_KEY, BTN_STYLUS); - input_set_capability(input, EV_KEY, BTN_STYLUS2); - __set_bit(INPUT_PROP_DIRECT, input->propbit); + input_copy_abs(input, ABS_X, ts->input_dev, ABS_MT_POSITION_X); + input_copy_abs(input, ABS_Y, ts->input_dev, ABS_MT_POSITION_Y); /* * The resolution of these touchscreens is about 10 units/mm, the actual * resolution does not matter much since we set INPUT_PROP_DIRECT. @@ -331,6 +316,13 @@ static struct input_dev *goodix_create_pen_input(struct goodix_ts_data *ts) */ input_abs_set_res(input, ABS_X, 10); input_abs_set_res(input, ABS_Y, 10); + input_set_abs_params(input, ABS_PRESSURE, 0, 255, 0, 0); + + input_set_capability(input, EV_KEY, BTN_TOUCH); + input_set_capability(input, EV_KEY, BTN_TOOL_PEN); + input_set_capability(input, EV_KEY, BTN_STYLUS); + input_set_capability(input, EV_KEY, BTN_STYLUS2); + __set_bit(INPUT_PROP_DIRECT, input->propbit); input->name = "Goodix Active Pen"; input->phys = "input/pen"; @@ -340,25 +332,23 @@ static struct input_dev *goodix_create_pen_input(struct goodix_ts_data *ts) input->id.product = 0x1001; input->id.version = ts->version; - if (input_register_device(input) != 0) { - input_free_device(input); - return NULL; - } - - return input; + ts->input_pen = input; + return 0; } static void goodix_ts_report_pen_down(struct goodix_ts_data *ts, u8 *data) { - int input_x, input_y, input_w; + int input_x, input_y, input_w, error; u8 key_value; - if (!ts->input_pen) { - ts->input_pen = goodix_create_pen_input(ts); - if (!ts->input_pen) - return; + if (!ts->pen_input_registered) { + error = input_register_device(ts->input_pen); + ts->pen_input_registered = (error == 0) ? 1 : error; } + if (ts->pen_input_registered < 0) + return; + if (ts->contact_size == 9) { input_x = get_unaligned_le16(&data[4]); input_y = get_unaligned_le16(&data[6]); @@ -1215,6 +1205,17 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) return error; } + /* + * Create the input_pen device before goodix_request_irq() calls + * devm_request_threaded_irq() so that the devm framework frees + * it after disabling the irq. + * Unfortunately there is no way to detect if the touchscreen has pen + * support, so registering the dev is delayed till the first pen event. + */ + error = goodix_create_pen_input(ts); + if (error) + return error; + ts->irq_flags = goodix_irq_flags[ts->int_trigger_type] | IRQF_ONESHOT; error = goodix_request_irq(ts); if (error) { diff --git a/drivers/input/touchscreen/goodix.h b/drivers/input/touchscreen/goodix.h index fa8602e78a64..87797cc88b32 100644 --- a/drivers/input/touchscreen/goodix.h +++ b/drivers/input/touchscreen/goodix.h @@ -94,6 +94,7 @@ struct goodix_ts_data { u16 version; bool reset_controller_at_probe; bool load_cfg_from_disk; + int pen_input_registered; struct completion firmware_loading_complete; unsigned long irq_flags; enum goodix_irq_pin_access_method irq_pin_access_method; diff --git a/drivers/input/touchscreen/imagis.c b/drivers/input/touchscreen/imagis.c new file mode 100644 index 000000000000..e2697e6c6d2a --- /dev/null +++ b/drivers/input/touchscreen/imagis.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/bits.h> +#include <linux/delay.h> +#include <linux/i2c.h> +#include <linux/input.h> +#include <linux/input/mt.h> +#include <linux/input/touchscreen.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/property.h> +#include <linux/regulator/consumer.h> + +#define IST3038C_HIB_ACCESS (0x800B << 16) +#define IST3038C_DIRECT_ACCESS BIT(31) +#define IST3038C_REG_CHIPID 0x40001000 +#define IST3038C_REG_HIB_BASE 0x30000100 +#define IST3038C_REG_TOUCH_STATUS (IST3038C_REG_HIB_BASE | IST3038C_HIB_ACCESS) +#define IST3038C_REG_TOUCH_COORD (IST3038C_REG_HIB_BASE | IST3038C_HIB_ACCESS | 0x8) +#define IST3038C_REG_INTR_MESSAGE (IST3038C_REG_HIB_BASE | IST3038C_HIB_ACCESS | 0x4) +#define IST3038C_WHOAMI 0x38c +#define IST3038C_CHIP_ON_DELAY_MS 60 +#define IST3038C_I2C_RETRY_COUNT 3 +#define IST3038C_MAX_FINGER_NUM 10 +#define IST3038C_X_MASK GENMASK(23, 12) +#define IST3038C_X_SHIFT 12 +#define IST3038C_Y_MASK GENMASK(11, 0) +#define IST3038C_AREA_MASK GENMASK(27, 24) +#define IST3038C_AREA_SHIFT 24 +#define IST3038C_FINGER_COUNT_MASK GENMASK(15, 12) +#define IST3038C_FINGER_COUNT_SHIFT 12 +#define IST3038C_FINGER_STATUS_MASK GENMASK(9, 0) + +struct imagis_ts { + struct i2c_client *client; + struct input_dev *input_dev; + struct touchscreen_properties prop; + struct regulator_bulk_data supplies[2]; +}; + +static int imagis_i2c_read_reg(struct imagis_ts *ts, + unsigned int reg, u32 *data) +{ + __be32 ret_be; + __be32 reg_be = cpu_to_be32(reg); + struct i2c_msg msg[] = { + { + .addr = ts->client->addr, + .flags = 0, + .buf = (unsigned char *)®_be, + .len = sizeof(reg_be), + }, { + .addr = ts->client->addr, + .flags = I2C_M_RD, + .buf = (unsigned char *)&ret_be, + .len = sizeof(ret_be), + }, + }; + int ret, error; + int retry = IST3038C_I2C_RETRY_COUNT; + + /* Retry in case the controller fails to respond */ + do { + ret = i2c_transfer(ts->client->adapter, msg, ARRAY_SIZE(msg)); + if (ret == ARRAY_SIZE(msg)) { + *data = be32_to_cpu(ret_be); + return 0; + } + + error = ret < 0 ? ret : -EIO; + dev_err(&ts->client->dev, + "%s - i2c_transfer failed: %d (%d)\n", + __func__, error, ret); + } while (--retry); + + return error; +} + +static irqreturn_t imagis_interrupt(int irq, void *dev_id) +{ + struct imagis_ts *ts = dev_id; + u32 intr_message, finger_status; + unsigned int finger_count, finger_pressed; + int i; + int error; + + error = imagis_i2c_read_reg(ts, IST3038C_REG_INTR_MESSAGE, + &intr_message); + if (error) { + dev_err(&ts->client->dev, + "failed to read the interrupt message: %d\n", error); + goto out; + } + + finger_count = (intr_message & IST3038C_FINGER_COUNT_MASK) >> + IST3038C_FINGER_COUNT_SHIFT; + if (finger_count > IST3038C_MAX_FINGER_NUM) { + dev_err(&ts->client->dev, + "finger count %d is more than maximum supported\n", + finger_count); + goto out; + } + + finger_pressed = intr_message & IST3038C_FINGER_STATUS_MASK; + + for (i = 0; i < finger_count; i++) { + error = imagis_i2c_read_reg(ts, + IST3038C_REG_TOUCH_COORD + (i * 4), + &finger_status); + if (error) { + dev_err(&ts->client->dev, + "failed to read coordinates for finger %d: %d\n", + i, error); + goto out; + } + + input_mt_slot(ts->input_dev, i); + input_mt_report_slot_state(ts->input_dev, MT_TOOL_FINGER, + finger_pressed & BIT(i)); + touchscreen_report_pos(ts->input_dev, &ts->prop, + (finger_status & IST3038C_X_MASK) >> + IST3038C_X_SHIFT, + finger_status & IST3038C_Y_MASK, 1); + input_report_abs(ts->input_dev, ABS_MT_TOUCH_MAJOR, + (finger_status & IST3038C_AREA_MASK) >> + IST3038C_AREA_SHIFT); + } + + input_mt_sync_frame(ts->input_dev); + input_sync(ts->input_dev); + +out: + return IRQ_HANDLED; +} + +static void imagis_power_off(void *_ts) +{ + struct imagis_ts *ts = _ts; + + regulator_bulk_disable(ARRAY_SIZE(ts->supplies), ts->supplies); +} + +static int imagis_power_on(struct imagis_ts *ts) +{ + int error; + + error = regulator_bulk_enable(ARRAY_SIZE(ts->supplies), ts->supplies); + if (error) + return error; + + msleep(IST3038C_CHIP_ON_DELAY_MS); + + return 0; +} + +static int imagis_start(struct imagis_ts *ts) +{ + int error; + + error = imagis_power_on(ts); + if (error) + return error; + + enable_irq(ts->client->irq); + + return 0; +} + +static int imagis_stop(struct imagis_ts *ts) +{ + disable_irq(ts->client->irq); + + imagis_power_off(ts); + + return 0; +} + +static int imagis_input_open(struct input_dev *dev) +{ + struct imagis_ts *ts = input_get_drvdata(dev); + + return imagis_start(ts); +} + +static void imagis_input_close(struct input_dev *dev) +{ + struct imagis_ts *ts = input_get_drvdata(dev); + + imagis_stop(ts); +} + +static int imagis_init_input_dev(struct imagis_ts *ts) +{ + struct input_dev *input_dev; + int error; + + input_dev = devm_input_allocate_device(&ts->client->dev); + if (!input_dev) + return -ENOMEM; + + ts->input_dev = input_dev; + + input_dev->name = "Imagis capacitive touchscreen"; + input_dev->phys = "input/ts"; + input_dev->id.bustype = BUS_I2C; + input_dev->open = imagis_input_open; + input_dev->close = imagis_input_close; + + input_set_drvdata(input_dev, ts); + + input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_X); + input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_Y); + input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0); + + touchscreen_parse_properties(input_dev, true, &ts->prop); + if (!ts->prop.max_x || !ts->prop.max_y) { + dev_err(&ts->client->dev, + "Touchscreen-size-x and/or touchscreen-size-y not set in dts\n"); + return -EINVAL; + } + + error = input_mt_init_slots(input_dev, + IST3038C_MAX_FINGER_NUM, + INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED); + if (error) { + dev_err(&ts->client->dev, + "Failed to initialize MT slots: %d", error); + return error; + } + + error = input_register_device(input_dev); + if (error) { + dev_err(&ts->client->dev, + "Failed to register input device: %d", error); + return error; + } + + return 0; +} + +static int imagis_init_regulators(struct imagis_ts *ts) +{ + struct i2c_client *client = ts->client; + + ts->supplies[0].supply = "vdd"; + ts->supplies[1].supply = "vddio"; + return devm_regulator_bulk_get(&client->dev, + ARRAY_SIZE(ts->supplies), + ts->supplies); +} + +static int imagis_probe(struct i2c_client *i2c) +{ + struct device *dev = &i2c->dev; + struct imagis_ts *ts; + int chip_id, error; + + ts = devm_kzalloc(dev, sizeof(*ts), GFP_KERNEL); + if (!ts) + return -ENOMEM; + + ts->client = i2c; + + error = imagis_init_regulators(ts); + if (error) { + dev_err(dev, "regulator init error: %d\n", error); + return error; + } + + error = imagis_power_on(ts); + if (error) { + dev_err(dev, "failed to enable regulators: %d\n", error); + return error; + } + + error = devm_add_action_or_reset(dev, imagis_power_off, ts); + if (error) { + dev_err(dev, "failed to install poweroff action: %d\n", error); + return error; + } + + error = imagis_i2c_read_reg(ts, + IST3038C_REG_CHIPID | IST3038C_DIRECT_ACCESS, + &chip_id); + if (error) { + dev_err(dev, "chip ID read failure: %d\n", error); + return error; + } + + if (chip_id != IST3038C_WHOAMI) { + dev_err(dev, "unknown chip ID: 0x%x\n", chip_id); + return -EINVAL; + } + + error = devm_request_threaded_irq(dev, i2c->irq, + NULL, imagis_interrupt, + IRQF_ONESHOT | IRQF_NO_AUTOEN, + "imagis-touchscreen", ts); + if (error) { + dev_err(dev, "IRQ %d allocation failure: %d\n", + i2c->irq, error); + return error; + } + + error = imagis_init_input_dev(ts); + if (error) + return error; + + return 0; +} + +static int __maybe_unused imagis_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct imagis_ts *ts = i2c_get_clientdata(client); + int retval = 0; + + mutex_lock(&ts->input_dev->mutex); + + if (input_device_enabled(ts->input_dev)) + retval = imagis_stop(ts); + + mutex_unlock(&ts->input_dev->mutex); + + return retval; +} + +static int __maybe_unused imagis_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct imagis_ts *ts = i2c_get_clientdata(client); + int retval = 0; + + mutex_lock(&ts->input_dev->mutex); + + if (input_device_enabled(ts->input_dev)) + retval = imagis_start(ts); + + mutex_unlock(&ts->input_dev->mutex); + + return retval; +} + +static SIMPLE_DEV_PM_OPS(imagis_pm_ops, imagis_suspend, imagis_resume); + +#ifdef CONFIG_OF +static const struct of_device_id imagis_of_match[] = { + { .compatible = "imagis,ist3038c", }, + { }, +}; +MODULE_DEVICE_TABLE(of, imagis_of_match); +#endif + +static struct i2c_driver imagis_ts_driver = { + .driver = { + .name = "imagis-touchscreen", + .pm = &imagis_pm_ops, + .of_match_table = of_match_ptr(imagis_of_match), + }, + .probe_new = imagis_probe, +}; + +module_i2c_driver(imagis_ts_driver); + +MODULE_DESCRIPTION("Imagis IST3038C Touchscreen Driver"); +MODULE_AUTHOR("Markuss Broks <markuss.broks@gmail.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/touchscreen/iqs5xx.c b/drivers/input/touchscreen/iqs5xx.c index b3fa71213d60..34c4cca57d13 100644 --- a/drivers/input/touchscreen/iqs5xx.c +++ b/drivers/input/touchscreen/iqs5xx.c @@ -486,11 +486,11 @@ static int iqs5xx_axis_init(struct i2c_client *client) { struct iqs5xx_private *iqs5xx = i2c_get_clientdata(client); struct touchscreen_properties *prop = &iqs5xx->prop; - struct input_dev *input; + struct input_dev *input = iqs5xx->input; u16 max_x, max_y; int error; - if (!iqs5xx->input) { + if (!input) { input = devm_input_allocate_device(&client->dev); if (!input) return -ENOMEM; @@ -512,11 +512,11 @@ static int iqs5xx_axis_init(struct i2c_client *client) if (error) return error; - input_set_abs_params(iqs5xx->input, ABS_MT_POSITION_X, 0, max_x, 0, 0); - input_set_abs_params(iqs5xx->input, ABS_MT_POSITION_Y, 0, max_y, 0, 0); - input_set_abs_params(iqs5xx->input, ABS_MT_PRESSURE, 0, U16_MAX, 0, 0); + input_set_abs_params(input, ABS_MT_POSITION_X, 0, max_x, 0, 0); + input_set_abs_params(input, ABS_MT_POSITION_Y, 0, max_y, 0, 0); + input_set_abs_params(input, ABS_MT_PRESSURE, 0, U16_MAX, 0, 0); - touchscreen_parse_properties(iqs5xx->input, true, prop); + touchscreen_parse_properties(input, true, prop); /* * The device reserves 0xFFFF for coordinates that correspond to slots @@ -540,7 +540,7 @@ static int iqs5xx_axis_init(struct i2c_client *client) return error; } - error = input_mt_init_slots(iqs5xx->input, IQS5XX_NUM_CONTACTS, + error = input_mt_init_slots(input, IQS5XX_NUM_CONTACTS, INPUT_MT_DIRECT); if (error) dev_err(&client->dev, "Failed to initialize slots: %d\n", @@ -674,7 +674,7 @@ static irqreturn_t iqs5xx_irq(int irq, void *data) input_mt_slot(input, i); if (input_mt_report_slot_state(input, MT_TOOL_FINGER, pressure != 0)) { - touchscreen_report_pos(iqs5xx->input, &iqs5xx->prop, + touchscreen_report_pos(input, &iqs5xx->prop, be16_to_cpu(touch_data->abs_x), be16_to_cpu(touch_data->abs_y), true); diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c index bc11203c9cf7..72e0b767e1ba 100644 --- a/drivers/input/touchscreen/stmfts.c +++ b/drivers/input/touchscreen/stmfts.c @@ -339,11 +339,11 @@ static int stmfts_input_open(struct input_dev *dev) err = pm_runtime_get_sync(&sdata->client->dev); if (err < 0) - return err; + goto out; err = i2c_smbus_write_byte(sdata->client, STMFTS_MS_MT_SENSE_ON); if (err) - return err; + goto out; mutex_lock(&sdata->mutex); sdata->running = true; @@ -366,7 +366,9 @@ static int stmfts_input_open(struct input_dev *dev) "failed to enable touchkey\n"); } - return 0; +out: + pm_runtime_put_noidle(&sdata->client->dev); + return err; } static void stmfts_input_close(struct input_dev *dev) diff --git a/drivers/input/touchscreen/tsc200x-core.c b/drivers/input/touchscreen/tsc200x-core.c index 27810f6c69f6..72c7258b93a5 100644 --- a/drivers/input/touchscreen/tsc200x-core.c +++ b/drivers/input/touchscreen/tsc200x-core.c @@ -88,6 +88,8 @@ struct tsc200x { int in_z1; int in_z2; + struct touchscreen_properties prop; + spinlock_t lock; struct timer_list penup_timer; @@ -113,8 +115,7 @@ static void tsc200x_update_pen_state(struct tsc200x *ts, int x, int y, int pressure) { if (pressure) { - input_report_abs(ts->idev, ABS_X, x); - input_report_abs(ts->idev, ABS_Y, y); + touchscreen_report_pos(ts->idev, &ts->prop, x, y, false); input_report_abs(ts->idev, ABS_PRESSURE, pressure); if (!ts->pen_down) { input_report_key(ts->idev, BTN_TOUCH, !!pressure); @@ -533,7 +534,7 @@ int tsc200x_probe(struct device *dev, int irq, const struct input_id *tsc_id, input_set_abs_params(input_dev, ABS_PRESSURE, 0, MAX_12BIT, TSC200X_DEF_P_FUZZ, 0); - touchscreen_parse_properties(input_dev, false, NULL); + touchscreen_parse_properties(input_dev, false, &ts->prop); /* Ensure the touchscreen is off */ tsc200x_stop_scan(ts); diff --git a/drivers/input/vivaldi-fmap.c b/drivers/input/vivaldi-fmap.c new file mode 100644 index 000000000000..6dae83d96806 --- /dev/null +++ b/drivers/input/vivaldi-fmap.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helpers for ChromeOS Vivaldi keyboard function row mapping + * + * Copyright (C) 2022 Google, Inc + */ + +#include <linux/export.h> +#include <linux/input/vivaldi-fmap.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/types.h> + +/** + * vivaldi_function_row_physmap_show - Print vivaldi function row physmap attribute + * @data: The vivaldi function row map + * @buf: Buffer to print the function row phsymap to + */ +ssize_t vivaldi_function_row_physmap_show(const struct vivaldi_data *data, + char *buf) +{ + ssize_t size = 0; + int i; + const u32 *physmap = data->function_row_physmap; + + if (!data->num_function_row_keys) + return 0; + + for (i = 0; i < data->num_function_row_keys; i++) + size += scnprintf(buf + size, PAGE_SIZE - size, + "%s%02X", size ? " " : "", physmap[i]); + if (size) + size += scnprintf(buf + size, PAGE_SIZE - size, "\n"); + + return size; +} +EXPORT_SYMBOL_GPL(vivaldi_function_row_physmap_show); + +MODULE_LICENSE("GPL"); diff --git a/drivers/isdn/mISDN/socket.c b/drivers/isdn/mISDN/socket.c index a6606736d8c5..2776ca5fc33f 100644 --- a/drivers/isdn/mISDN/socket.c +++ b/drivers/isdn/mISDN/socket.c @@ -121,7 +121,7 @@ mISDN_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (sk->sk_state == MISDN_CLOSED) return 0; - skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) return err; diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index 4081cb6cf7b3..4277853c7535 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -210,7 +210,7 @@ struct dm_table { #define DM_TIO_MAGIC 28714 struct dm_target_io { unsigned short magic; - unsigned short flags; + blk_short_t flags; unsigned int target_bio_nr; struct dm_io *io; struct dm_target *ti; @@ -244,7 +244,7 @@ static inline void dm_tio_set_flag(struct dm_target_io *tio, unsigned int bit) #define DM_IO_MAGIC 19577 struct dm_io { unsigned short magic; - unsigned short flags; + blk_short_t flags; atomic_t io_count; struct mapped_device *md; struct bio *orig_bio; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index c58a5111cb57..ad2d5faa2ebb 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2472,9 +2472,11 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, dm_integrity_io_error(ic, "invalid sector in journal", -EIO); sec &= ~(sector_t)(ic->sectors_per_block - 1); } + if (unlikely(sec >= ic->provided_data_sectors)) { + journal_entry_set_unused(je); + continue; + } } - if (unlikely(sec >= ic->provided_data_sectors)) - continue; get_area_and_offset(ic, sec, &area, &offset); restore_last_bytes(ic, access_journal_data(ic, i, j), je); for (k = j + 1; k < ic->journal_section_entries; k++) { diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 901abd6dea41..87310fceb0d8 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -891,15 +891,21 @@ static struct hash_cell *__find_device_hash_cell(struct dm_ioctl *param) struct hash_cell *hc = NULL; if (*param->uuid) { - if (*param->name || param->dev) + if (*param->name || param->dev) { + DMERR("Invalid ioctl structure: uuid %s, name %s, dev %llx", + param->uuid, param->name, (unsigned long long)param->dev); return NULL; + } hc = __get_uuid_cell(param->uuid); if (!hc) return NULL; } else if (*param->name) { - if (param->dev) + if (param->dev) { + DMERR("Invalid ioctl structure: name %s, dev %llx", + param->name, (unsigned long long)param->dev); return NULL; + } hc = __get_name_cell(param->name); if (!hc) @@ -1851,8 +1857,11 @@ static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kern if (copy_from_user(param_kernel, user, minimum_data_size)) return -EFAULT; - if (param_kernel->data_size < minimum_data_size) + if (param_kernel->data_size < minimum_data_size) { + DMERR("Invalid data size in the ioctl structure: %u", + param_kernel->data_size); return -EINVAL; + } secure_data = param_kernel->flags & DM_SECURE_DATA_FLAG; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index ad2e0bbeb559..3c5fad7c4ee6 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -892,13 +892,19 @@ static void dm_io_complete(struct dm_io *io) if (unlikely(wq_has_sleeper(&md->wait))) wake_up(&md->wait); - if (io_error == BLK_STS_DM_REQUEUE) { - /* - * Upper layer won't help us poll split bio, io->orig_bio - * may only reflect a subset of the pre-split original, - * so clear REQ_POLLED in case of requeue - */ - bio->bi_opf &= ~REQ_POLLED; + if (io_error == BLK_STS_DM_REQUEUE || io_error == BLK_STS_AGAIN) { + if (bio->bi_opf & REQ_POLLED) { + /* + * Upper layer won't help us poll split bio (io->orig_bio + * may only reflect a subset of the pre-split original) + * so clear REQ_POLLED in case of requeue. + */ + bio->bi_opf &= ~REQ_POLLED; + if (io_error == BLK_STS_AGAIN) { + /* io_uring doesn't handle BLK_STS_AGAIN (yet) */ + queue_io(md, bio); + } + } return; } diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index a7e3eb9befb6..a32050fecabf 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -351,9 +351,6 @@ static ssize_t dev_attribute_show(struct device *dev, * we still can use 'ubi->ubi_num'. */ ubi = container_of(dev, struct ubi_device, dev); - ubi = ubi_get_device(ubi->ubi_num); - if (!ubi) - return -ENODEV; if (attr == &dev_eraseblock_size) ret = sprintf(buf, "%d\n", ubi->leb_size); @@ -382,7 +379,6 @@ static ssize_t dev_attribute_show(struct device *dev, else ret = -EINVAL; - ubi_put_device(ubi); return ret; } @@ -979,9 +975,6 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, goto out_detach; } - /* Make device "available" before it becomes accessible via sysfs */ - ubi_devices[ubi_num] = ubi; - err = uif_init(ubi); if (err) goto out_detach; @@ -1026,6 +1019,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, wake_up_process(ubi->bgt_thread); spin_unlock(&ubi->wl_lock); + ubi_devices[ubi_num] = ubi; ubi_notify_all(ubi, UBI_VOLUME_ADDED, NULL); return ubi_num; @@ -1034,7 +1028,6 @@ out_debugfs: out_uif: uif_close(ubi); out_detach: - ubi_devices[ubi_num] = NULL; ubi_wl_close(ubi); ubi_free_all_volumes(ubi); vfree(ubi->vtbl); diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c index 022af59906aa..6b5f1ffd961b 100644 --- a/drivers/mtd/ubi/fastmap.c +++ b/drivers/mtd/ubi/fastmap.c @@ -468,7 +468,9 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai, if (err == UBI_IO_FF_BITFLIPS) scrub = 1; - add_aeb(ai, free, pnum, ec, scrub); + ret = add_aeb(ai, free, pnum, ec, scrub); + if (ret) + goto out; continue; } else if (err == 0 || err == UBI_IO_BITFLIPS) { dbg_bld("Found non empty PEB:%i in pool", pnum); @@ -638,8 +640,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi, if (fm_pos >= fm_size) goto fail_bad; - add_aeb(ai, &ai->free, be32_to_cpu(fmec->pnum), - be32_to_cpu(fmec->ec), 0); + ret = add_aeb(ai, &ai->free, be32_to_cpu(fmec->pnum), + be32_to_cpu(fmec->ec), 0); + if (ret) + goto fail; } /* read EC values from used list */ @@ -649,8 +653,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi, if (fm_pos >= fm_size) goto fail_bad; - add_aeb(ai, &used, be32_to_cpu(fmec->pnum), - be32_to_cpu(fmec->ec), 0); + ret = add_aeb(ai, &used, be32_to_cpu(fmec->pnum), + be32_to_cpu(fmec->ec), 0); + if (ret) + goto fail; } /* read EC values from scrub list */ @@ -660,8 +666,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi, if (fm_pos >= fm_size) goto fail_bad; - add_aeb(ai, &used, be32_to_cpu(fmec->pnum), - be32_to_cpu(fmec->ec), 1); + ret = add_aeb(ai, &used, be32_to_cpu(fmec->pnum), + be32_to_cpu(fmec->ec), 1); + if (ret) + goto fail; } /* read EC values from erase list */ @@ -671,8 +679,10 @@ static int ubi_attach_fastmap(struct ubi_device *ubi, if (fm_pos >= fm_size) goto fail_bad; - add_aeb(ai, &ai->erase, be32_to_cpu(fmec->pnum), - be32_to_cpu(fmec->ec), 1); + ret = add_aeb(ai, &ai->erase, be32_to_cpu(fmec->pnum), + be32_to_cpu(fmec->ec), 1); + if (ret) + goto fail; } ai->mean_ec = div_u64(ai->ec_sum, ai->ec_count); diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c index 139ee132bfbc..1bc7b3a05604 100644 --- a/drivers/mtd/ubi/vmt.c +++ b/drivers/mtd/ubi/vmt.c @@ -56,16 +56,11 @@ static ssize_t vol_attribute_show(struct device *dev, { int ret; struct ubi_volume *vol = container_of(dev, struct ubi_volume, dev); - struct ubi_device *ubi; - - ubi = ubi_get_device(vol->ubi->ubi_num); - if (!ubi) - return -ENODEV; + struct ubi_device *ubi = vol->ubi; spin_lock(&ubi->volumes_lock); if (!ubi->volumes[vol->vol_id]) { spin_unlock(&ubi->volumes_lock); - ubi_put_device(ubi); return -ENODEV; } /* Take a reference to prevent volume removal */ @@ -103,7 +98,6 @@ static ssize_t vol_attribute_show(struct device *dev, vol->ref_count -= 1; ubi_assert(vol->ref_count >= 0); spin_unlock(&ubi->volumes_lock); - ubi_put_device(ubi); return ret; } diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h index 5caa75b41b73..881ac33fe914 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h @@ -6218,7 +6218,7 @@ #define AEU_INPUTS_ATTN_BITS_GPIO0_FUNCTION_0 (0x1<<2) #define AEU_INPUTS_ATTN_BITS_IGU_PARITY_ERROR (0x1<<12) #define AEU_INPUTS_ATTN_BITS_MCP_LATCHED_ROM_PARITY (0x1<<28) -#define AEU_INPUTS_ATTN_BITS_MCP_LATCHED_SCPAD_PARITY (0x1<<31) +#define AEU_INPUTS_ATTN_BITS_MCP_LATCHED_SCPAD_PARITY (0x1U<<31) #define AEU_INPUTS_ATTN_BITS_MCP_LATCHED_UMP_RX_PARITY (0x1<<29) #define AEU_INPUTS_ATTN_BITS_MCP_LATCHED_UMP_TX_PARITY (0x1<<30) #define AEU_INPUTS_ATTN_BITS_MISC_HW_INTERRUPT (0x1<<15) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 1c28495875cf..0489c1c2e7dd 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -56,6 +56,7 @@ #include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> #include <net/page_pool.h> +#include <linux/align.h> #include "bnxt_hsi.h" #include "bnxt.h" @@ -738,7 +739,6 @@ static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, page_pool_recycle_direct(rxr->page_pool, page); return NULL; } - *mapping += bp->rx_dma_offset; return page; } @@ -780,6 +780,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, if (!page) return -ENOMEM; + mapping += bp->rx_dma_offset; rx_buf->data = page; rx_buf->data_ptr = page_address(page) + bp->rx_offset; } else { @@ -840,33 +841,41 @@ static inline int bnxt_alloc_rx_page(struct bnxt *bp, u16 sw_prod = rxr->rx_sw_agg_prod; unsigned int offset = 0; - if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { - page = rxr->rx_page; - if (!page) { + if (BNXT_RX_PAGE_MODE(bp)) { + page = __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); + + if (!page) + return -ENOMEM; + + } else { + if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { + page = rxr->rx_page; + if (!page) { + page = alloc_page(gfp); + if (!page) + return -ENOMEM; + rxr->rx_page = page; + rxr->rx_page_offset = 0; + } + offset = rxr->rx_page_offset; + rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; + if (rxr->rx_page_offset == PAGE_SIZE) + rxr->rx_page = NULL; + else + get_page(page); + } else { page = alloc_page(gfp); if (!page) return -ENOMEM; - rxr->rx_page = page; - rxr->rx_page_offset = 0; } - offset = rxr->rx_page_offset; - rxr->rx_page_offset += BNXT_RX_PAGE_SIZE; - if (rxr->rx_page_offset == PAGE_SIZE) - rxr->rx_page = NULL; - else - get_page(page); - } else { - page = alloc_page(gfp); - if (!page) - return -ENOMEM; - } - mapping = dma_map_page_attrs(&pdev->dev, page, offset, - BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, - DMA_ATTR_WEAK_ORDERING); - if (dma_mapping_error(&pdev->dev, mapping)) { - __free_page(page); - return -EIO; + mapping = dma_map_page_attrs(&pdev->dev, page, offset, + BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, + DMA_ATTR_WEAK_ORDERING); + if (dma_mapping_error(&pdev->dev, mapping)) { + __free_page(page); + return -EIO; + } } if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) @@ -962,6 +971,39 @@ static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, rxr->rx_sw_agg_prod = sw_prod; } +static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, + u16 cons, void *data, u8 *data_ptr, + dma_addr_t dma_addr, + unsigned int offset_and_len) +{ + unsigned int len = offset_and_len & 0xffff; + struct page *page = data; + u16 prod = rxr->rx_prod; + struct sk_buff *skb; + int err; + + err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); + if (unlikely(err)) { + bnxt_reuse_rx_data(rxr, cons, data); + return NULL; + } + dma_addr -= bp->rx_dma_offset; + dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, + DMA_ATTR_WEAK_ORDERING); + skb = build_skb(page_address(page), BNXT_PAGE_MODE_BUF_SIZE + + bp->rx_dma_offset); + if (!skb) { + __free_page(page); + return NULL; + } + skb_mark_for_recycle(skb); + skb_reserve(skb, bp->rx_dma_offset); + __skb_put(skb, len); + + return skb; +} + static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, void *data, u8 *data_ptr, @@ -984,7 +1026,6 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, dma_addr -= bp->rx_dma_offset; dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); - page_pool_release_page(rxr->page_pool, page); if (unlikely(!payload)) payload = eth_get_headlen(bp->dev, data_ptr, len); @@ -995,6 +1036,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, return NULL; } + skb_mark_for_recycle(skb); off = (void *)data_ptr - page_address(page); skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE); memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, @@ -1038,22 +1080,24 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, return skb; } -static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, - struct sk_buff *skb, u16 idx, - u32 agg_bufs, bool tpa) +static u32 __bnxt_rx_agg_pages(struct bnxt *bp, + struct bnxt_cp_ring_info *cpr, + struct skb_shared_info *shinfo, + u16 idx, u32 agg_bufs, bool tpa, + struct xdp_buff *xdp) { struct bnxt_napi *bnapi = cpr->bnapi; struct pci_dev *pdev = bp->pdev; struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; u16 prod = rxr->rx_agg_prod; + u32 i, total_frag_len = 0; bool p5_tpa = false; - u32 i; if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa) p5_tpa = true; for (i = 0; i < agg_bufs; i++) { + skb_frag_t *frag = &shinfo->frags[i]; u16 cons, frag_len; struct rx_agg_cmp *agg; struct bnxt_sw_rx_agg_bd *cons_rx_buf; @@ -1069,8 +1113,10 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; cons_rx_buf = &rxr->rx_agg_ring[cons]; - skb_fill_page_desc(skb, i, cons_rx_buf->page, - cons_rx_buf->offset, frag_len); + skb_frag_off_set(frag, cons_rx_buf->offset); + skb_frag_size_set(frag, frag_len); + __skb_frag_set_page(frag, cons_rx_buf->page); + shinfo->nr_frags = i + 1; __clear_bit(cons, rxr->rx_agg_bmap); /* It is possible for bnxt_alloc_rx_page() to allocate @@ -1081,16 +1127,14 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, page = cons_rx_buf->page; cons_rx_buf->page = NULL; + if (xdp && page_is_pfmemalloc(page)) + xdp_buff_set_frag_pfmemalloc(xdp); + if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) { - struct skb_shared_info *shinfo; unsigned int nr_frags; - shinfo = skb_shinfo(skb); nr_frags = --shinfo->nr_frags; __skb_frag_set_page(&shinfo->frags[nr_frags], NULL); - - dev_kfree_skb(skb); - cons_rx_buf->page = page; /* Update prod since possibly some pages have been @@ -1098,23 +1142,62 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, */ rxr->rx_agg_prod = prod; bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); - return NULL; + return 0; } dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE, - DMA_FROM_DEVICE, + bp->rx_dir, DMA_ATTR_WEAK_ORDERING); - skb->data_len += frag_len; - skb->len += frag_len; - skb->truesize += PAGE_SIZE; - + total_frag_len += frag_len; prod = NEXT_RX_AGG(prod); } rxr->rx_agg_prod = prod; + return total_frag_len; +} + +static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp, + struct bnxt_cp_ring_info *cpr, + struct sk_buff *skb, u16 idx, + u32 agg_bufs, bool tpa) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + u32 total_frag_len = 0; + + total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx, + agg_bufs, tpa, NULL); + if (!total_frag_len) { + dev_kfree_skb(skb); + return NULL; + } + + skb->data_len += total_frag_len; + skb->len += total_frag_len; + skb->truesize += PAGE_SIZE * agg_bufs; return skb; } +static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp, + struct bnxt_cp_ring_info *cpr, + struct xdp_buff *xdp, u16 idx, + u32 agg_bufs, bool tpa) +{ + struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp); + u32 total_frag_len = 0; + + if (!xdp_buff_has_frags(xdp)) + shinfo->nr_frags = 0; + + total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, + idx, agg_bufs, tpa, xdp); + if (total_frag_len) { + xdp_buff_set_frags_flag(xdp); + shinfo->nr_frags = agg_bufs; + shinfo->xdp_frags_size = total_frag_len; + } + return total_frag_len; +} + static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, u8 agg_bufs, u32 *raw_cons) { @@ -1644,7 +1727,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, } if (agg_bufs) { - skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true); + skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true); if (!skb) { /* Page reuse already handled by bnxt_rx_pages(). */ cpr->sw_stats.rx.rx_oom_discards += 1; @@ -1729,8 +1812,10 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, struct bnxt_sw_rx_bd *rx_buf; unsigned int len; u8 *data_ptr, agg_bufs, cmp_type; + bool xdp_active = false; dma_addr_t dma_addr; struct sk_buff *skb; + struct xdp_buff xdp; u32 flags, misc; void *data; int rc = 0; @@ -1839,18 +1924,39 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, len = flags >> RX_CMP_LEN_SHIFT; dma_addr = rx_buf->mapping; - if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { - rc = 1; - goto next_rx; + if (bnxt_xdp_attached(bp, rxr)) { + bnxt_xdp_buff_init(bp, rxr, cons, &data_ptr, &len, &xdp); + if (agg_bufs) { + u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp, + cp_cons, agg_bufs, + false); + if (!frag_len) { + cpr->sw_stats.rx.rx_oom_discards += 1; + rc = -ENOMEM; + goto next_rx; + } + } + xdp_active = true; + } + + if (xdp_active) { + if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &len, event)) { + rc = 1; + goto next_rx; + } } if (len <= bp->rx_copy_thresh) { skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); bnxt_reuse_rx_data(rxr, cons, data); if (!skb) { - if (agg_bufs) - bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, - agg_bufs, false); + if (agg_bufs) { + if (!xdp_active) + bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, + agg_bufs, false); + else + bnxt_xdp_buff_frags_free(rxr, &xdp); + } cpr->sw_stats.rx.rx_oom_discards += 1; rc = -ENOMEM; goto next_rx; @@ -1872,11 +1978,22 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } if (agg_bufs) { - skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false); - if (!skb) { - cpr->sw_stats.rx.rx_oom_discards += 1; - rc = -ENOMEM; - goto next_rx; + if (!xdp_active) { + skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false); + if (!skb) { + cpr->sw_stats.rx.rx_oom_discards += 1; + rc = -ENOMEM; + goto next_rx; + } + } else { + skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); + if (!skb) { + /* we should be able to free the old skb here */ + bnxt_xdp_buff_frags_free(rxr, &xdp); + cpr->sw_stats.rx.rx_oom_discards += 1; + rc = -ENOMEM; + goto next_rx; + } } } @@ -2492,10 +2609,13 @@ static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi) if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; - if (bnapi->events & BNXT_AGG_EVENT) - bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); } + if (bnapi->events & BNXT_AGG_EVENT) { + struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + + bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); + } bnapi->events = 0; } @@ -2872,14 +2992,23 @@ skip_rx_buf_free: if (!page) continue; - dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, - BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, - DMA_ATTR_WEAK_ORDERING); + if (BNXT_RX_PAGE_MODE(bp)) { + dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, + BNXT_RX_PAGE_SIZE, bp->rx_dir, + DMA_ATTR_WEAK_ORDERING); + rx_agg_buf->page = NULL; + __clear_bit(i, rxr->rx_agg_bmap); - rx_agg_buf->page = NULL; - __clear_bit(i, rxr->rx_agg_bmap); + page_pool_recycle_direct(rxr->page_pool, page); + } else { + dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping, + BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE, + DMA_ATTR_WEAK_ORDERING); + rx_agg_buf->page = NULL; + __clear_bit(i, rxr->rx_agg_bmap); - __free_page(page); + __free_page(page); + } } skip_rx_agg_free: @@ -3253,6 +3382,7 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) } qidx = bp->tc_to_qidx[j]; ring->queue_id = bp->q_info[qidx].queue_id; + spin_lock_init(&txr->xdp_tx_lock); if (i < bp->tx_nr_rings_xdp) continue; if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1)) @@ -3792,7 +3922,7 @@ void bnxt_set_ring_params(struct bnxt *bp) /* 8 for CRC and VLAN */ rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); - rx_space = rx_size + NET_SKB_PAD + + rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); bp->rx_copy_thresh = BNXT_RX_COPY_THRESH; @@ -3833,9 +3963,15 @@ void bnxt_set_ring_params(struct bnxt *bp) } bp->rx_agg_ring_size = agg_ring_size; bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; - rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); - rx_space = rx_size + NET_SKB_PAD + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + + if (BNXT_RX_PAGE_MODE(bp)) { + rx_space = BNXT_PAGE_MODE_BUF_SIZE; + rx_size = BNXT_MAX_PAGE_MODE_MTU; + } else { + rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN); + rx_space = rx_size + NET_SKB_PAD + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + } } bp->rx_buf_use_size = rx_size; @@ -3876,14 +4012,21 @@ void bnxt_set_ring_params(struct bnxt *bp) int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) { if (page_mode) { - if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) - return -EOPNOTSUPP; - bp->dev->max_mtu = - min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); bp->flags &= ~BNXT_FLAG_AGG_RINGS; - bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE; + bp->flags |= BNXT_FLAG_RX_PAGE_MODE; + + if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { + bp->flags |= BNXT_FLAG_JUMBO; + bp->rx_skb_func = bnxt_rx_multi_page_skb; + bp->dev->max_mtu = + min_t(u16, bp->max_mtu, BNXT_MAX_MTU); + } else { + bp->flags |= BNXT_FLAG_NO_AGG_RINGS; + bp->rx_skb_func = bnxt_rx_page_skb; + bp->dev->max_mtu = + min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); + } bp->rx_dir = DMA_BIDIRECTIONAL; - bp->rx_skb_func = bnxt_rx_page_skb; /* Disable LRO or GRO_HW */ netdev_update_features(bp->dev); } else { @@ -5225,12 +5368,15 @@ static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id) if (rc) return rc; - req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | - VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | - VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); - req->enables = - cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | - VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); + req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT); + req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID); + + if (BNXT_RX_PAGE_MODE(bp) && !BNXT_RX_JUMBO_MODE(bp)) { + req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | + VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); + req->enables |= + cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); + } /* thresholds not implemented in firmware yet */ req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh); req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh); @@ -10338,6 +10484,12 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) if (irq_re_init) udp_tunnel_nic_reset_ntf(bp->dev); + if (bp->tx_nr_rings_xdp < num_possible_cpus()) { + if (!static_key_enabled(&bnxt_xdp_locking_key)) + static_branch_enable(&bnxt_xdp_locking_key); + } else if (static_key_enabled(&bnxt_xdp_locking_key)) { + static_branch_disable(&bnxt_xdp_locking_key); + } set_bit(BNXT_STATE_OPEN, &bp->state); bnxt_enable_int(bp); /* Enable TX queues */ @@ -11024,6 +11176,9 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev, if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); + if (!(bp->flags & BNXT_FLAG_TPA)) + features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); + if (!(features & NETIF_F_GRO)) features &= ~NETIF_F_GRO_HW; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 61aa3e8c5952..a498ee297946 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -591,9 +591,12 @@ struct nqe_cn { #define BNXT_RX_PAGE_SIZE (1 << BNXT_RX_PAGE_SHIFT) #define BNXT_MAX_MTU 9500 -#define BNXT_MAX_PAGE_MODE_MTU \ +#define BNXT_PAGE_MODE_BUF_SIZE \ ((unsigned int)PAGE_SIZE - VLAN_ETH_HLEN - NET_IP_ALIGN - \ XDP_PACKET_HEADROOM) +#define BNXT_MAX_PAGE_MODE_MTU \ + BNXT_PAGE_MODE_BUF_SIZE - \ + SKB_DATA_ALIGN((unsigned int)sizeof(struct skb_shared_info)) #define BNXT_MIN_PKT_SIZE 52 @@ -698,13 +701,12 @@ struct bnxt_sw_tx_bd { }; DEFINE_DMA_UNMAP_ADDR(mapping); DEFINE_DMA_UNMAP_LEN(len); + struct page *page; u8 is_gso; u8 is_push; u8 action; - union { - unsigned short nr_frags; - u16 rx_prod; - }; + unsigned short nr_frags; + u16 rx_prod; }; struct bnxt_sw_rx_bd { @@ -800,6 +802,8 @@ struct bnxt_tx_ring_info { u32 dev_state; struct bnxt_ring_struct tx_ring_struct; + /* Synchronize simultaneous xdp_xmit on same ring */ + spinlock_t xdp_tx_lock; }; #define BNXT_LEGACY_COAL_CMPL_PARAMS \ @@ -1814,6 +1818,7 @@ struct bnxt { #define BNXT_SUPPORTS_TPA(bp) (!BNXT_CHIP_TYPE_NITRO_A0(bp) && \ (!((bp)->flags & BNXT_FLAG_CHIP_P5) || \ (bp)->max_tpa_v2) && !is_kdump_kernel()) +#define BNXT_RX_JUMBO_MODE(bp) ((bp)->flags & BNXT_FLAG_JUMBO) #define BNXT_CHIP_SR2(bp) \ ((bp)->chip_num == CHIP_NUM_58818) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 22e965e18fbc..b3a48d6675fe 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -3491,7 +3491,7 @@ static int bnxt_run_loopback(struct bnxt *bp) dev_kfree_skb(skb); return -EIO; } - bnxt_xmit_bd(bp, txr, map, pkt_size); + bnxt_xmit_bd(bp, txr, map, pkt_size, NULL); /* Sync BD data before updating doorbell */ wmb(); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 52fad0fdeacf..f02fe906dedb 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -20,38 +20,95 @@ #include "bnxt.h" #include "bnxt_xdp.h" +DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); + struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len) + dma_addr_t mapping, u32 len, + struct xdp_buff *xdp) { - struct bnxt_sw_tx_bd *tx_buf; + struct skb_shared_info *sinfo; + struct bnxt_sw_tx_bd *tx_buf, *first_buf; struct tx_bd *txbd; + int num_frags = 0; u32 flags; u16 prod; + int i; + + if (xdp && xdp_buff_has_frags(xdp)) { + sinfo = xdp_get_shared_info_from_buff(xdp); + num_frags = sinfo->nr_frags; + } + /* fill up the first buffer */ prod = txr->tx_prod; tx_buf = &txr->tx_buf_ring[prod]; + first_buf = tx_buf; + tx_buf->nr_frags = num_frags; + if (xdp) + tx_buf->page = virt_to_head_page(xdp->data); txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; - flags = (len << TX_BD_LEN_SHIFT) | (1 << TX_BD_FLAGS_BD_CNT_SHIFT) | - TX_BD_FLAGS_PACKET_END | bnxt_lhint_arr[len >> 9]; + flags = ((len) << TX_BD_LEN_SHIFT) | ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT); txbd->tx_bd_len_flags_type = cpu_to_le32(flags); txbd->tx_bd_opaque = prod; txbd->tx_bd_haddr = cpu_to_le64(mapping); + /* now let us fill up the frags into the next buffers */ + for (i = 0; i < num_frags ; i++) { + skb_frag_t *frag = &sinfo->frags[i]; + struct bnxt_sw_tx_bd *frag_tx_buf; + struct pci_dev *pdev = bp->pdev; + dma_addr_t frag_mapping; + int frag_len; + + prod = NEXT_TX(prod); + txr->tx_prod = prod; + + /* first fill up the first buffer */ + frag_tx_buf = &txr->tx_buf_ring[prod]; + frag_tx_buf->page = skb_frag_page(frag); + + txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; + + frag_len = skb_frag_size(frag); + frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0, + frag_len, DMA_TO_DEVICE); + + if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping))) + return NULL; + + dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping); + + flags = frag_len << TX_BD_LEN_SHIFT; + txbd->tx_bd_len_flags_type = cpu_to_le32(flags); + txbd->tx_bd_opaque = prod; + txbd->tx_bd_haddr = cpu_to_le64(frag_mapping); + + len = frag_len; + } + + flags &= ~TX_BD_LEN; + txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags | + TX_BD_FLAGS_PACKET_END); + /* Sync TX BD */ + wmb(); prod = NEXT_TX(prod); txr->tx_prod = prod; - return tx_buf; + + return first_buf; } static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len, u16 rx_prod) + dma_addr_t mapping, u32 len, u16 rx_prod, + struct xdp_buff *xdp) { struct bnxt_sw_tx_bd *tx_buf; - tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); tx_buf->rx_prod = rx_prod; tx_buf->action = XDP_TX; + } static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, @@ -61,7 +118,7 @@ static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, { struct bnxt_sw_tx_bd *tx_buf; - tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL); tx_buf->action = XDP_REDIRECT; tx_buf->xdpf = xdpf; dma_unmap_addr_set(tx_buf, mapping, mapping); @@ -76,7 +133,7 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) struct bnxt_sw_tx_bd *tx_buf; u16 tx_cons = txr->tx_cons; u16 last_tx_cons = tx_cons; - int i; + int i, j, frags; for (i = 0; i < nr_pkts; i++) { tx_buf = &txr->tx_buf_ring[tx_cons]; @@ -94,6 +151,13 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) } else if (tx_buf->action == XDP_TX) { rx_doorbell_needed = true; last_tx_cons = tx_cons; + + frags = tx_buf->nr_frags; + for (j = 0; j < frags; j++) { + tx_cons = NEXT_TX(tx_cons); + tx_buf = &txr->tx_buf_ring[tx_cons]; + page_pool_recycle_direct(rxr->page_pool, tx_buf->page); + } } tx_cons = NEXT_TX(tx_cons); } @@ -101,22 +165,67 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) if (rx_doorbell_needed) { tx_buf = &txr->tx_buf_ring[last_tx_cons]; bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod); + } } +bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) +{ + struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); + + return !!xdp_prog; +} + +void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 cons, u8 **data_ptr, unsigned int *len, + struct xdp_buff *xdp) +{ + struct bnxt_sw_rx_bd *rx_buf; + struct pci_dev *pdev; + dma_addr_t mapping; + u32 offset; + + pdev = bp->pdev; + rx_buf = &rxr->rx_buf_ring[cons]; + offset = bp->rx_offset; + + mapping = rx_buf->mapping - bp->rx_dma_offset; + dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); + + xdp_init_buff(xdp, BNXT_PAGE_MODE_BUF_SIZE + offset, &rxr->xdp_rxq); + xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false); +} + +void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, + struct xdp_buff *xdp) +{ + struct skb_shared_info *shinfo; + int i; + + if (!xdp || !xdp_buff_has_frags(xdp)) + return; + shinfo = xdp_get_shared_info_from_buff(xdp); + for (i = 0; i < shinfo->nr_frags; i++) { + struct page *page = skb_frag_page(&shinfo->frags[i]); + + page_pool_recycle_direct(rxr->page_pool, page); + } + shinfo->nr_frags = 0; +} + /* returns the following: * true - packet consumed by XDP and new buffer is allocated. * false - packet should be passed to the stack. */ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, - struct page *page, u8 **data_ptr, unsigned int *len, u8 *event) + struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event) { struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); struct bnxt_tx_ring_info *txr; struct bnxt_sw_rx_bd *rx_buf; struct pci_dev *pdev; - struct xdp_buff xdp; dma_addr_t mapping; + u32 tx_needed = 1; void *orig_data; u32 tx_avail; u32 offset; @@ -126,16 +235,10 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, return false; pdev = bp->pdev; - rx_buf = &rxr->rx_buf_ring[cons]; offset = bp->rx_offset; - mapping = rx_buf->mapping - bp->rx_dma_offset; - dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); - txr = rxr->bnapi->tx_ring; /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */ - xdp_init_buff(&xdp, PAGE_SIZE, &rxr->xdp_rxq); - xdp_prepare_buff(&xdp, *data_ptr - offset, offset, *len, false); orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp); @@ -148,26 +251,38 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, *event &= ~BNXT_RX_EVENT; *len = xdp.data_end - xdp.data; - if (orig_data != xdp.data) { + if (orig_data != xdp.data) offset = xdp.data - xdp.data_hard_start; - *data_ptr = xdp.data_hard_start + offset; - } + switch (act) { case XDP_PASS: return false; case XDP_TX: - if (tx_avail < 1) { + rx_buf = &rxr->rx_buf_ring[cons]; + mapping = rx_buf->mapping - bp->rx_dma_offset; + *event = 0; + + if (unlikely(xdp_buff_has_frags(&xdp))) { + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp); + + tx_needed += sinfo->nr_frags; + *event = BNXT_AGG_EVENT; + } + + if (tx_avail < tx_needed) { trace_xdp_exception(bp->dev, xdp_prog, act); + bnxt_xdp_buff_frags_free(rxr, &xdp); bnxt_reuse_rx_data(rxr, cons, page); return true; } - *event = BNXT_TX_EVENT; dma_sync_single_for_device(&pdev->dev, mapping + offset, *len, bp->rx_dir); + + *event |= BNXT_TX_EVENT; __bnxt_xmit_xdp(bp, txr, mapping + offset, *len, - NEXT_RX(rxr->rx_prod)); + NEXT_RX(rxr->rx_prod), &xdp); bnxt_reuse_rx_data(rxr, cons, page); return true; case XDP_REDIRECT: @@ -175,6 +290,8 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, * redirect is coming from a frame received by the * bnxt_en driver. */ + rx_buf = &rxr->rx_buf_ring[cons]; + mapping = rx_buf->mapping - bp->rx_dma_offset; dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); @@ -182,6 +299,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, /* if we are unable to allocate a new buffer, abort and reuse */ if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) { trace_xdp_exception(bp->dev, xdp_prog, act); + bnxt_xdp_buff_frags_free(rxr, &xdp); bnxt_reuse_rx_data(rxr, cons, page); return true; } @@ -201,6 +319,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, trace_xdp_exception(bp->dev, xdp_prog, act); fallthrough; case XDP_DROP: + bnxt_xdp_buff_frags_free(rxr, &xdp); bnxt_reuse_rx_data(rxr, cons, page); break; } @@ -227,11 +346,16 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, ring = smp_processor_id() % bp->tx_nr_rings_xdp; txr = &bp->tx_ring[ring]; + if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING) + return -EINVAL; + + if (static_branch_unlikely(&bnxt_xdp_locking_key)) + spin_lock(&txr->xdp_tx_lock); + for (i = 0; i < num_frames; i++) { struct xdp_frame *xdp = frames[i]; - if (!txr || !bnxt_tx_avail(bp, txr) || - !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) + if (!bnxt_tx_avail(bp, txr)) break; mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, @@ -250,6 +374,9 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames, bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); } + if (static_branch_unlikely(&bnxt_xdp_locking_key)) + spin_unlock(&txr->xdp_tx_lock); + return nxmit; } @@ -260,8 +387,9 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) int tx_xdp = 0, rc, tc; struct bpf_prog *old; - if (prog && bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { - netdev_warn(dev, "MTU %d larger than largest XDP supported MTU %d.\n", + if (prog && !prog->aux->xdp_has_frags && + bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { + netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n", bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU); return -EOPNOTSUPP; } @@ -327,3 +455,26 @@ int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) } return rc; } + +struct sk_buff * +bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, + struct page_pool *pool, struct xdp_buff *xdp, + struct rx_cmp_ext *rxcmp1) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); + + if (!skb) + return NULL; + skb_checksum_none_assert(skb); + if (RX_CMP_L4_CS_OK(rxcmp1)) { + if (bp->dev->features & NETIF_F_RXCSUM) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = RX_CMP_ENCAP(rxcmp1); + } + } + xdp_update_skb_shared_info(skb, num_frags, + sinfo->xdp_frags_size, + PAGE_SIZE * sinfo->nr_frags, + xdp_buff_is_frag_pfmemalloc(xdp)); + return skb; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 0df40c3beb05..505911ae095d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -10,15 +10,29 @@ #ifndef BNXT_XDP_H #define BNXT_XDP_H +DECLARE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); + struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len); + dma_addr_t mapping, u32 len, + struct xdp_buff *xdp); void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, - struct page *page, u8 **data_ptr, unsigned int *len, + struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event); int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp); int bnxt_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags); +bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr); + +void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 cons, u8 **data_ptr, unsigned int *len, + struct xdp_buff *xdp); +void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, + struct xdp_buff *xdp); +struct sk_buff *bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, + u8 num_frags, struct page_pool *pool, + struct xdp_buff *xdp, + struct rx_cmp_ext *rxcmp1); #endif diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c index 5f5f8c53c4a0..c8cb541572ff 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c @@ -167,7 +167,7 @@ static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev) base = of_iomap(node, 0); if (!base) { err = -ENOMEM; - goto err_close; + goto err_put; } err = fsl_mc_allocate_irqs(mc_dev); @@ -210,6 +210,8 @@ err_free_mc_irq: fsl_mc_free_irqs(mc_dev); err_unmap: iounmap(base); +err_put: + of_node_put(node); err_close: dprtc_close(mc_dev->mc_io, 0, mc_dev->mc_handle); err_free_mcp: diff --git a/drivers/net/ethernet/fungible/funcore/fun_dev.c b/drivers/net/ethernet/fungible/funcore/fun_dev.c index 5d7aef73df61..fb5120d90f26 100644 --- a/drivers/net/ethernet/fungible/funcore/fun_dev.c +++ b/drivers/net/ethernet/fungible/funcore/fun_dev.c @@ -586,8 +586,8 @@ static int fun_get_dev_limits(struct fun_dev *fdev) /* Calculate the max QID based on SQ/CQ/doorbell counts. * SQ/CQ doorbells alternate. */ - num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) / - (fdev->db_stride * 4); + num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) >> + (2 + NVME_CAP_STRIDE(fdev->cap_reg)); fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1; fdev->kern_end_qid = fdev->max_qid + 1; return 0; diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index d4f1874df7d0..8ed3c9ab7ff7 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -301,7 +301,6 @@ enum ice_vsi_state { ICE_VSI_NETDEV_REGISTERED, ICE_VSI_UMAC_FLTR_CHANGED, ICE_VSI_MMAC_FLTR_CHANGED, - ICE_VSI_VLAN_FLTR_CHANGED, ICE_VSI_PROMISC_CHANGED, ICE_VSI_STATE_NBITS /* must be last */ }; @@ -672,7 +671,7 @@ static inline struct ice_pf *ice_netdev_to_pf(struct net_device *netdev) static inline bool ice_is_xdp_ena_vsi(struct ice_vsi *vsi) { - return !!vsi->xdp_prog; + return !!READ_ONCE(vsi->xdp_prog); } static inline void ice_set_ring_xdp(struct ice_tx_ring *ring) diff --git a/drivers/net/ethernet/intel/ice/ice_fltr.c b/drivers/net/ethernet/intel/ice/ice_fltr.c index af57eb114966..85a94483c2ed 100644 --- a/drivers/net/ethernet/intel/ice/ice_fltr.c +++ b/drivers/net/ethernet/intel/ice/ice_fltr.c @@ -58,7 +58,16 @@ int ice_fltr_set_vlan_vsi_promisc(struct ice_hw *hw, struct ice_vsi *vsi, u8 promisc_mask) { - return ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_mask, false); + struct ice_pf *pf = hw->back; + int result; + + result = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_mask, false); + if (result) + dev_err(ice_pf_to_dev(pf), + "Error setting promisc mode on VSI %i (rc=%d)\n", + vsi->vsi_num, result); + + return result; } /** @@ -73,7 +82,16 @@ int ice_fltr_clear_vlan_vsi_promisc(struct ice_hw *hw, struct ice_vsi *vsi, u8 promisc_mask) { - return ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_mask, true); + struct ice_pf *pf = hw->back; + int result; + + result = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_mask, true); + if (result) + dev_err(ice_pf_to_dev(pf), + "Error clearing promisc mode on VSI %i (rc=%d)\n", + vsi->vsi_num, result); + + return result; } /** @@ -87,7 +105,16 @@ int ice_fltr_clear_vsi_promisc(struct ice_hw *hw, u16 vsi_handle, u8 promisc_mask, u16 vid) { - return ice_clear_vsi_promisc(hw, vsi_handle, promisc_mask, vid); + struct ice_pf *pf = hw->back; + int result; + + result = ice_clear_vsi_promisc(hw, vsi_handle, promisc_mask, vid); + if (result) + dev_err(ice_pf_to_dev(pf), + "Error clearing promisc mode on VSI %i for VID %u (rc=%d)\n", + ice_get_hw_vsi_num(hw, vsi_handle), vid, result); + + return result; } /** @@ -101,7 +128,16 @@ int ice_fltr_set_vsi_promisc(struct ice_hw *hw, u16 vsi_handle, u8 promisc_mask, u16 vid) { - return ice_set_vsi_promisc(hw, vsi_handle, promisc_mask, vid); + struct ice_pf *pf = hw->back; + int result; + + result = ice_set_vsi_promisc(hw, vsi_handle, promisc_mask, vid); + if (result) + dev_err(ice_pf_to_dev(pf), + "Error setting promisc mode on VSI %i for VID %u (rc=%d)\n", + ice_get_hw_vsi_num(hw, vsi_handle), vid, result); + + return result; } /** diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index b897926f817d..2774cbd5b12a 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -1480,6 +1480,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->tx_tstamps = &pf->ptp.port.tx; ring->dev = dev; ring->count = vsi->num_tx_desc; + ring->txq_teid = ICE_INVAL_TEID; if (dvm_ena) ring->flags |= ICE_TX_FLAGS_RING_VLAN_L2TAG2; else @@ -2983,6 +2984,8 @@ int ice_vsi_release(struct ice_vsi *vsi) } } + if (ice_is_vsi_dflt_vsi(pf->first_sw, vsi)) + ice_clear_dflt_vsi(pf->first_sw); ice_fltr_remove_all(vsi); ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); err = ice_rm_vsi_rdma_cfg(vsi->port_info, vsi->idx); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index b588d7995631..d768925785ca 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -243,8 +243,7 @@ static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) { return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) || - test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state) || - test_bit(ICE_VSI_VLAN_FLTR_CHANGED, vsi->state); + test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); } /** @@ -260,10 +259,15 @@ static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m) if (vsi->type != ICE_VSI_PF) return 0; - if (ice_vsi_has_non_zero_vlans(vsi)) - status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi, promisc_m); - else - status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, promisc_m, 0); + if (ice_vsi_has_non_zero_vlans(vsi)) { + promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); + status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi, + promisc_m); + } else { + status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, + promisc_m, 0); + } + return status; } @@ -280,10 +284,15 @@ static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m) if (vsi->type != ICE_VSI_PF) return 0; - if (ice_vsi_has_non_zero_vlans(vsi)) - status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi, promisc_m); - else - status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, promisc_m, 0); + if (ice_vsi_has_non_zero_vlans(vsi)) { + promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); + status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi, + promisc_m); + } else { + status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, + promisc_m, 0); + } + return status; } @@ -302,7 +311,6 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; u32 changed_flags = 0; - u8 promisc_m; int err; if (!vsi->netdev) @@ -320,7 +328,6 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) if (ice_vsi_fltr_changed(vsi)) { clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); - clear_bit(ICE_VSI_VLAN_FLTR_CHANGED, vsi->state); /* grab the netdev's addr_list_lock */ netif_addr_lock_bh(netdev); @@ -369,29 +376,15 @@ static int ice_vsi_sync_fltr(struct ice_vsi *vsi) /* check for changes in promiscuous modes */ if (changed_flags & IFF_ALLMULTI) { if (vsi->current_netdev_flags & IFF_ALLMULTI) { - if (ice_vsi_has_non_zero_vlans(vsi)) - promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; - else - promisc_m = ICE_MCAST_PROMISC_BITS; - - err = ice_set_promisc(vsi, promisc_m); + err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS); if (err) { - netdev_err(netdev, "Error setting Multicast promiscuous mode on VSI %i\n", - vsi->vsi_num); vsi->current_netdev_flags &= ~IFF_ALLMULTI; goto out_promisc; } } else { /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ - if (ice_vsi_has_non_zero_vlans(vsi)) - promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; - else - promisc_m = ICE_MCAST_PROMISC_BITS; - - err = ice_clear_promisc(vsi, promisc_m); + err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS); if (err) { - netdev_err(netdev, "Error clearing Multicast promiscuous mode on VSI %i\n", - vsi->vsi_num); vsi->current_netdev_flags |= IFF_ALLMULTI; goto out_promisc; } @@ -2569,7 +2562,7 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) spin_lock_init(&xdp_ring->tx_lock); for (j = 0; j < xdp_ring->count; j++) { tx_desc = ICE_TX_DESC(xdp_ring, j); - tx_desc->cmd_type_offset_bsz = cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE); + tx_desc->cmd_type_offset_bsz = 0; } } @@ -2765,8 +2758,10 @@ free_qmap: ice_for_each_xdp_txq(vsi, i) if (vsi->xdp_rings[i]) { - if (vsi->xdp_rings[i]->desc) + if (vsi->xdp_rings[i]->desc) { + synchronize_rcu(); ice_free_tx_ring(vsi->xdp_rings[i]); + } kfree_rcu(vsi->xdp_rings[i], rcu); vsi->xdp_rings[i] = NULL; } @@ -3488,6 +3483,20 @@ ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) if (!vid) return 0; + while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) + usleep_range(1000, 2000); + + /* Add multicast promisc rule for the VLAN ID to be added if + * all-multicast is currently enabled. + */ + if (vsi->current_netdev_flags & IFF_ALLMULTI) { + ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_VLAN_PROMISC_BITS, + vid); + if (ret) + goto finish; + } + vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged @@ -3495,8 +3504,23 @@ ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) */ vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); ret = vlan_ops->add_vlan(vsi, &vlan); - if (!ret) - set_bit(ICE_VSI_VLAN_FLTR_CHANGED, vsi->state); + if (ret) + goto finish; + + /* If all-multicast is currently enabled and this VLAN ID is only one + * besides VLAN-0 we have to update look-up type of multicast promisc + * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN. + */ + if ((vsi->current_netdev_flags & IFF_ALLMULTI) && + ice_vsi_num_non_zero_vlans(vsi) == 1) { + ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_PROMISC_BITS, 0); + ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_VLAN_PROMISC_BITS, 0); + } + +finish: + clear_bit(ICE_CFG_BUSY, vsi->state); return ret; } @@ -3522,6 +3546,9 @@ ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) if (!vid) return 0; + while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) + usleep_range(1000, 2000); + vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); /* Make sure VLAN delete is successful before updating VLAN @@ -3530,10 +3557,33 @@ ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); ret = vlan_ops->del_vlan(vsi, &vlan); if (ret) - return ret; + goto finish; - set_bit(ICE_VSI_VLAN_FLTR_CHANGED, vsi->state); - return 0; + /* Remove multicast promisc rule for the removed VLAN ID if + * all-multicast is enabled. + */ + if (vsi->current_netdev_flags & IFF_ALLMULTI) + ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_VLAN_PROMISC_BITS, vid); + + if (!ice_vsi_has_non_zero_vlans(vsi)) { + /* Update look-up type of multicast promisc rule for VLAN 0 + * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when + * all-multicast is enabled and VLAN 0 is the only VLAN rule. + */ + if (vsi->current_netdev_flags & IFF_ALLMULTI) { + ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_VLAN_PROMISC_BITS, + 0); + ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, + ICE_MCAST_PROMISC_BITS, 0); + } + } + +finish: + clear_bit(ICE_CFG_BUSY, vsi->state); + + return ret; } /** @@ -5475,16 +5525,19 @@ static int ice_set_mac_address(struct net_device *netdev, void *pi) /* Add filter for new MAC. If filter exists, return success */ err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); - if (err == -EEXIST) + if (err == -EEXIST) { /* Although this MAC filter is already present in hardware it's * possible in some cases (e.g. bonding) that dev_addr was * modified outside of the driver and needs to be restored back * to this value. */ netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); - else if (err) + + return 0; + } else if (err) { /* error if the new filter addition failed */ err = -EADDRNOTAVAIL; + } err_update_filters: if (err) { diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 25b8f6f726eb..496250f9f8fc 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -30,12 +30,46 @@ static const u8 dummy_eth_header[DUMMY_ETH_HDR_LEN] = { 0x2, 0, 0, 0, 0, 0, 0x2, 0, 0, 0, 0, 0, 0x81, 0, 0, 0}; +enum { + ICE_PKT_VLAN = BIT(0), + ICE_PKT_OUTER_IPV6 = BIT(1), + ICE_PKT_TUN_GTPC = BIT(2), + ICE_PKT_TUN_GTPU = BIT(3), + ICE_PKT_TUN_NVGRE = BIT(4), + ICE_PKT_TUN_UDP = BIT(5), + ICE_PKT_INNER_IPV6 = BIT(6), + ICE_PKT_INNER_TCP = BIT(7), + ICE_PKT_INNER_UDP = BIT(8), + ICE_PKT_GTP_NOPAY = BIT(9), +}; + struct ice_dummy_pkt_offsets { enum ice_protocol_type type; u16 offset; /* ICE_PROTOCOL_LAST indicates end of list */ }; -static const struct ice_dummy_pkt_offsets dummy_gre_tcp_packet_offsets[] = { +struct ice_dummy_pkt_profile { + const struct ice_dummy_pkt_offsets *offsets; + const u8 *pkt; + u32 match; + u16 pkt_len; +}; + +#define ICE_DECLARE_PKT_OFFSETS(type) \ + static const struct ice_dummy_pkt_offsets \ + ice_dummy_##type##_packet_offsets[] + +#define ICE_DECLARE_PKT_TEMPLATE(type) \ + static const u8 ice_dummy_##type##_packet[] + +#define ICE_PKT_PROFILE(type, m) { \ + .match = (m), \ + .pkt = ice_dummy_##type##_packet, \ + .pkt_len = sizeof(ice_dummy_##type##_packet), \ + .offsets = ice_dummy_##type##_packet_offsets, \ +} + +ICE_DECLARE_PKT_OFFSETS(gre_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -47,7 +81,7 @@ static const struct ice_dummy_pkt_offsets dummy_gre_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_gre_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(gre_tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -82,7 +116,7 @@ static const u8 dummy_gre_tcp_packet[] = { 0x00, 0x00, 0x00, 0x00 }; -static const struct ice_dummy_pkt_offsets dummy_gre_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(gre_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -94,7 +128,7 @@ static const struct ice_dummy_pkt_offsets dummy_gre_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_gre_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(gre_udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -126,7 +160,7 @@ static const u8 dummy_gre_udp_packet[] = { 0x00, 0x08, 0x00, 0x00, }; -static const struct ice_dummy_pkt_offsets dummy_udp_tun_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp_tun_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -141,7 +175,7 @@ static const struct ice_dummy_pkt_offsets dummy_udp_tun_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_udp_tun_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp_tun_tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -179,7 +213,7 @@ static const u8 dummy_udp_tun_tcp_packet[] = { 0x00, 0x00, 0x00, 0x00 }; -static const struct ice_dummy_pkt_offsets dummy_udp_tun_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp_tun_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -194,7 +228,7 @@ static const struct ice_dummy_pkt_offsets dummy_udp_tun_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_udp_tun_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp_tun_udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -229,8 +263,7 @@ static const u8 dummy_udp_tun_udp_packet[] = { 0x00, 0x08, 0x00, 0x00, }; -static const struct ice_dummy_pkt_offsets -dummy_gre_ipv6_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(gre_ipv6_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -242,7 +275,7 @@ dummy_gre_ipv6_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_gre_ipv6_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(gre_ipv6_tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -282,8 +315,7 @@ static const u8 dummy_gre_ipv6_tcp_packet[] = { 0x00, 0x00, 0x00, 0x00 }; -static const struct ice_dummy_pkt_offsets -dummy_gre_ipv6_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(gre_ipv6_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -295,7 +327,7 @@ dummy_gre_ipv6_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_gre_ipv6_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(gre_ipv6_udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -332,8 +364,7 @@ static const u8 dummy_gre_ipv6_udp_packet[] = { 0x00, 0x08, 0x00, 0x00, }; -static const struct ice_dummy_pkt_offsets -dummy_udp_tun_ipv6_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp_tun_ipv6_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -348,7 +379,7 @@ dummy_udp_tun_ipv6_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_udp_tun_ipv6_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp_tun_ipv6_tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -391,8 +422,7 @@ static const u8 dummy_udp_tun_ipv6_tcp_packet[] = { 0x00, 0x00, 0x00, 0x00 }; -static const struct ice_dummy_pkt_offsets -dummy_udp_tun_ipv6_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp_tun_ipv6_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -407,7 +437,7 @@ dummy_udp_tun_ipv6_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_udp_tun_ipv6_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp_tun_ipv6_udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -448,7 +478,7 @@ static const u8 dummy_udp_tun_ipv6_udp_packet[] = { }; /* offset info for MAC + IPv4 + UDP dummy packet */ -static const struct ice_dummy_pkt_offsets dummy_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -457,7 +487,7 @@ static const struct ice_dummy_pkt_offsets dummy_udp_packet_offsets[] = { }; /* Dummy packet for MAC + IPv4 + UDP */ -static const u8 dummy_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -477,7 +507,7 @@ static const u8 dummy_udp_packet[] = { }; /* offset info for MAC + VLAN + IPv4 + UDP dummy packet */ -static const struct ice_dummy_pkt_offsets dummy_vlan_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(vlan_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_VLAN_OFOS, 12 }, { ICE_ETYPE_OL, 16 }, @@ -487,7 +517,7 @@ static const struct ice_dummy_pkt_offsets dummy_vlan_udp_packet_offsets[] = { }; /* C-tag (801.1Q), IPv4:UDP dummy packet */ -static const u8 dummy_vlan_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(vlan_udp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -509,7 +539,7 @@ static const u8 dummy_vlan_udp_packet[] = { }; /* offset info for MAC + IPv4 + TCP dummy packet */ -static const struct ice_dummy_pkt_offsets dummy_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV4_OFOS, 14 }, @@ -518,7 +548,7 @@ static const struct ice_dummy_pkt_offsets dummy_tcp_packet_offsets[] = { }; /* Dummy packet for MAC + IPv4 + TCP */ -static const u8 dummy_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -541,7 +571,7 @@ static const u8 dummy_tcp_packet[] = { }; /* offset info for MAC + VLAN (C-tag, 802.1Q) + IPv4 + TCP dummy packet */ -static const struct ice_dummy_pkt_offsets dummy_vlan_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(vlan_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_VLAN_OFOS, 12 }, { ICE_ETYPE_OL, 16 }, @@ -551,7 +581,7 @@ static const struct ice_dummy_pkt_offsets dummy_vlan_tcp_packet_offsets[] = { }; /* C-tag (801.1Q), IPv4:TCP dummy packet */ -static const u8 dummy_vlan_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(vlan_tcp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -575,7 +605,7 @@ static const u8 dummy_vlan_tcp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const struct ice_dummy_pkt_offsets dummy_tcp_ipv6_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(tcp_ipv6) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV6_OFOS, 14 }, @@ -583,7 +613,7 @@ static const struct ice_dummy_pkt_offsets dummy_tcp_ipv6_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_tcp_ipv6_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(tcp_ipv6) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -611,8 +641,7 @@ static const u8 dummy_tcp_ipv6_packet[] = { }; /* C-tag (802.1Q): IPv6 + TCP */ -static const struct ice_dummy_pkt_offsets -dummy_vlan_tcp_ipv6_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(vlan_tcp_ipv6) = { { ICE_MAC_OFOS, 0 }, { ICE_VLAN_OFOS, 12 }, { ICE_ETYPE_OL, 16 }, @@ -622,7 +651,7 @@ dummy_vlan_tcp_ipv6_packet_offsets[] = { }; /* C-tag (802.1Q), IPv6 + TCP dummy packet */ -static const u8 dummy_vlan_tcp_ipv6_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(vlan_tcp_ipv6) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -652,7 +681,7 @@ static const u8 dummy_vlan_tcp_ipv6_packet[] = { }; /* IPv6 + UDP */ -static const struct ice_dummy_pkt_offsets dummy_udp_ipv6_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(udp_ipv6) = { { ICE_MAC_OFOS, 0 }, { ICE_ETYPE_OL, 12 }, { ICE_IPV6_OFOS, 14 }, @@ -661,7 +690,7 @@ static const struct ice_dummy_pkt_offsets dummy_udp_ipv6_packet_offsets[] = { }; /* IPv6 + UDP dummy packet */ -static const u8 dummy_udp_ipv6_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(udp_ipv6) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -689,8 +718,7 @@ static const u8 dummy_udp_ipv6_packet[] = { }; /* C-tag (802.1Q): IPv6 + UDP */ -static const struct ice_dummy_pkt_offsets -dummy_vlan_udp_ipv6_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(vlan_udp_ipv6) = { { ICE_MAC_OFOS, 0 }, { ICE_VLAN_OFOS, 12 }, { ICE_ETYPE_OL, 16 }, @@ -700,7 +728,7 @@ dummy_vlan_udp_ipv6_packet_offsets[] = { }; /* C-tag (802.1Q), IPv6 + UDP dummy packet */ -static const u8 dummy_vlan_udp_ipv6_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(vlan_udp_ipv6) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -727,8 +755,7 @@ static const u8 dummy_vlan_udp_ipv6_packet[] = { }; /* Outer IPv4 + Outer UDP + GTP + Inner IPv4 + Inner TCP */ -static const -struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv4_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv4_gtpu_ipv4_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV4_OFOS, 14 }, { ICE_UDP_OF, 34 }, @@ -738,7 +765,7 @@ struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv4_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv4_gtpu_ipv4_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv4_gtpu_ipv4_tcp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -776,8 +803,7 @@ static const u8 dummy_ipv4_gtpu_ipv4_tcp_packet[] = { }; /* Outer IPv4 + Outer UDP + GTP + Inner IPv4 + Inner UDP */ -static const -struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv4_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv4_gtpu_ipv4_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV4_OFOS, 14 }, { ICE_UDP_OF, 34 }, @@ -787,7 +813,7 @@ struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv4_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv4_gtpu_ipv4_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv4_gtpu_ipv4_udp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -822,8 +848,7 @@ static const u8 dummy_ipv4_gtpu_ipv4_udp_packet[] = { }; /* Outer IPv6 + Outer UDP + GTP + Inner IPv4 + Inner TCP */ -static const -struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv6_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv4_gtpu_ipv6_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV4_OFOS, 14 }, { ICE_UDP_OF, 34 }, @@ -833,7 +858,7 @@ struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv6_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv4_gtpu_ipv6_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv4_gtpu_ipv6_tcp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -875,8 +900,7 @@ static const u8 dummy_ipv4_gtpu_ipv6_tcp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const -struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv6_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv4_gtpu_ipv6_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV4_OFOS, 14 }, { ICE_UDP_OF, 34 }, @@ -886,7 +910,7 @@ struct ice_dummy_pkt_offsets dummy_ipv4_gtpu_ipv6_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv4_gtpu_ipv6_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv4_gtpu_ipv6_udp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -925,8 +949,7 @@ static const u8 dummy_ipv4_gtpu_ipv6_udp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const -struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv4_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv6_gtpu_ipv4_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV6_OFOS, 14 }, { ICE_UDP_OF, 54 }, @@ -936,7 +959,7 @@ struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv4_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv6_gtpu_ipv4_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv6_gtpu_ipv4_tcp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -978,8 +1001,7 @@ static const u8 dummy_ipv6_gtpu_ipv4_tcp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const -struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv4_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv6_gtpu_ipv4_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV6_OFOS, 14 }, { ICE_UDP_OF, 54 }, @@ -989,7 +1011,7 @@ struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv4_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv6_gtpu_ipv4_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv6_gtpu_ipv4_udp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1028,8 +1050,7 @@ static const u8 dummy_ipv6_gtpu_ipv4_udp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const -struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv6_tcp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv6_gtpu_ipv6_tcp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV6_OFOS, 14 }, { ICE_UDP_OF, 54 }, @@ -1039,7 +1060,7 @@ struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv6_tcp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv6_gtpu_ipv6_tcp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv6_gtpu_ipv6_tcp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1086,8 +1107,7 @@ static const u8 dummy_ipv6_gtpu_ipv6_tcp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const -struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv6_udp_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv6_gtpu_ipv6_udp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV6_OFOS, 14 }, { ICE_UDP_OF, 54 }, @@ -1097,7 +1117,7 @@ struct ice_dummy_pkt_offsets dummy_ipv6_gtpu_ipv6_udp_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv6_gtpu_ipv6_udp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv6_gtpu_ipv6_udp) = { 0x00, 0x00, 0x00, 0x00, /* Ethernet 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1141,7 +1161,15 @@ static const u8 dummy_ipv6_gtpu_ipv6_udp_packet[] = { 0x00, 0x00, /* 2 bytes for 4 byte alignment */ }; -static const u8 dummy_ipv4_gtpu_ipv4_packet[] = { +ICE_DECLARE_PKT_OFFSETS(ipv4_gtpu_ipv4) = { + { ICE_MAC_OFOS, 0 }, + { ICE_IPV4_OFOS, 14 }, + { ICE_UDP_OF, 34 }, + { ICE_GTP_NO_PAY, 42 }, + { ICE_PROTOCOL_LAST, 0 }, +}; + +ICE_DECLARE_PKT_TEMPLATE(ipv4_gtpu_ipv4) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1171,17 +1199,7 @@ static const u8 dummy_ipv4_gtpu_ipv4_packet[] = { 0x00, 0x00, }; -static const -struct ice_dummy_pkt_offsets dummy_ipv4_gtp_no_pay_packet_offsets[] = { - { ICE_MAC_OFOS, 0 }, - { ICE_IPV4_OFOS, 14 }, - { ICE_UDP_OF, 34 }, - { ICE_GTP_NO_PAY, 42 }, - { ICE_PROTOCOL_LAST, 0 }, -}; - -static const -struct ice_dummy_pkt_offsets dummy_ipv6_gtp_no_pay_packet_offsets[] = { +ICE_DECLARE_PKT_OFFSETS(ipv6_gtp) = { { ICE_MAC_OFOS, 0 }, { ICE_IPV6_OFOS, 14 }, { ICE_UDP_OF, 54 }, @@ -1189,7 +1207,7 @@ struct ice_dummy_pkt_offsets dummy_ipv6_gtp_no_pay_packet_offsets[] = { { ICE_PROTOCOL_LAST, 0 }, }; -static const u8 dummy_ipv6_gtp_packet[] = { +ICE_DECLARE_PKT_TEMPLATE(ipv6_gtp) = { 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1215,6 +1233,55 @@ static const u8 dummy_ipv6_gtp_packet[] = { 0x00, 0x00, }; +static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = { + ICE_PKT_PROFILE(ipv6_gtp, ICE_PKT_TUN_GTPU | ICE_PKT_OUTER_IPV6 | + ICE_PKT_GTP_NOPAY), + ICE_PKT_PROFILE(ipv6_gtpu_ipv6_udp, ICE_PKT_TUN_GTPU | + ICE_PKT_OUTER_IPV6 | + ICE_PKT_INNER_IPV6 | + ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(ipv6_gtpu_ipv6_tcp, ICE_PKT_TUN_GTPU | + ICE_PKT_OUTER_IPV6 | + ICE_PKT_INNER_IPV6), + ICE_PKT_PROFILE(ipv6_gtpu_ipv4_udp, ICE_PKT_TUN_GTPU | + ICE_PKT_OUTER_IPV6 | + ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(ipv6_gtpu_ipv4_tcp, ICE_PKT_TUN_GTPU | + ICE_PKT_OUTER_IPV6), + ICE_PKT_PROFILE(ipv4_gtpu_ipv4, ICE_PKT_TUN_GTPU | ICE_PKT_GTP_NOPAY), + ICE_PKT_PROFILE(ipv4_gtpu_ipv6_udp, ICE_PKT_TUN_GTPU | + ICE_PKT_INNER_IPV6 | + ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(ipv4_gtpu_ipv6_tcp, ICE_PKT_TUN_GTPU | + ICE_PKT_INNER_IPV6), + ICE_PKT_PROFILE(ipv4_gtpu_ipv4_udp, ICE_PKT_TUN_GTPU | + ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(ipv4_gtpu_ipv4_tcp, ICE_PKT_TUN_GTPU), + ICE_PKT_PROFILE(ipv6_gtp, ICE_PKT_TUN_GTPC | ICE_PKT_OUTER_IPV6), + ICE_PKT_PROFILE(ipv4_gtpu_ipv4, ICE_PKT_TUN_GTPC), + ICE_PKT_PROFILE(gre_ipv6_tcp, ICE_PKT_TUN_NVGRE | ICE_PKT_INNER_IPV6 | + ICE_PKT_INNER_TCP), + ICE_PKT_PROFILE(gre_tcp, ICE_PKT_TUN_NVGRE | ICE_PKT_INNER_TCP), + ICE_PKT_PROFILE(gre_ipv6_udp, ICE_PKT_TUN_NVGRE | ICE_PKT_INNER_IPV6), + ICE_PKT_PROFILE(gre_udp, ICE_PKT_TUN_NVGRE), + ICE_PKT_PROFILE(udp_tun_ipv6_tcp, ICE_PKT_TUN_UDP | + ICE_PKT_INNER_IPV6 | + ICE_PKT_INNER_TCP), + ICE_PKT_PROFILE(udp_tun_tcp, ICE_PKT_TUN_UDP | ICE_PKT_INNER_TCP), + ICE_PKT_PROFILE(udp_tun_ipv6_udp, ICE_PKT_TUN_UDP | + ICE_PKT_INNER_IPV6), + ICE_PKT_PROFILE(udp_tun_udp, ICE_PKT_TUN_UDP), + ICE_PKT_PROFILE(vlan_udp_ipv6, ICE_PKT_OUTER_IPV6 | ICE_PKT_INNER_UDP | + ICE_PKT_VLAN), + ICE_PKT_PROFILE(udp_ipv6, ICE_PKT_OUTER_IPV6 | ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(vlan_udp, ICE_PKT_INNER_UDP | ICE_PKT_VLAN), + ICE_PKT_PROFILE(udp, ICE_PKT_INNER_UDP), + ICE_PKT_PROFILE(vlan_tcp_ipv6, ICE_PKT_OUTER_IPV6 | ICE_PKT_VLAN), + ICE_PKT_PROFILE(tcp_ipv6, ICE_PKT_OUTER_IPV6), + ICE_PKT_PROFILE(vlan_tcp, ICE_PKT_VLAN), + ICE_PKT_PROFILE(tcp, 0), +}; + #define ICE_SW_RULE_RX_TX_ETH_HDR_SIZE \ (offsetof(struct ice_aqc_sw_rules_elem, pdata.lkup_tx_rx.hdr) + \ (DUMMY_ETH_HDR_LEN * \ @@ -5501,212 +5568,66 @@ err_free_lkup_exts: * structure per protocol header * @lkups_cnt: number of protocols * @tun_type: tunnel type - * @pkt: dummy packet to fill according to filter match criteria - * @pkt_len: packet length of dummy packet - * @offsets: pointer to receive the pointer to the offsets for the packet + * + * Returns the &ice_dummy_pkt_profile corresponding to these lookup params. */ -static void +static const struct ice_dummy_pkt_profile * ice_find_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, - enum ice_sw_tunnel_type tun_type, - const u8 **pkt, u16 *pkt_len, - const struct ice_dummy_pkt_offsets **offsets) + enum ice_sw_tunnel_type tun_type) { - bool inner_tcp = false, inner_udp = false, outer_ipv6 = false; - bool vlan = false, inner_ipv6 = false, gtp_no_pay = false; + const struct ice_dummy_pkt_profile *ret = ice_dummy_pkt_profiles; + u32 match = 0; u16 i; + switch (tun_type) { + case ICE_SW_TUN_GTPC: + match |= ICE_PKT_TUN_GTPC; + break; + case ICE_SW_TUN_GTPU: + match |= ICE_PKT_TUN_GTPU; + break; + case ICE_SW_TUN_NVGRE: + match |= ICE_PKT_TUN_NVGRE; + break; + case ICE_SW_TUN_GENEVE: + case ICE_SW_TUN_VXLAN: + match |= ICE_PKT_TUN_UDP; + break; + default: + break; + } + for (i = 0; i < lkups_cnt; i++) { if (lkups[i].type == ICE_UDP_ILOS) - inner_udp = true; + match |= ICE_PKT_INNER_UDP; else if (lkups[i].type == ICE_TCP_IL) - inner_tcp = true; + match |= ICE_PKT_INNER_TCP; else if (lkups[i].type == ICE_IPV6_OFOS) - outer_ipv6 = true; + match |= ICE_PKT_OUTER_IPV6; else if (lkups[i].type == ICE_VLAN_OFOS) - vlan = true; + match |= ICE_PKT_VLAN; else if (lkups[i].type == ICE_ETYPE_OL && lkups[i].h_u.ethertype.ethtype_id == cpu_to_be16(ICE_IPV6_ETHER_ID) && lkups[i].m_u.ethertype.ethtype_id == cpu_to_be16(0xFFFF)) - outer_ipv6 = true; + match |= ICE_PKT_OUTER_IPV6; else if (lkups[i].type == ICE_ETYPE_IL && lkups[i].h_u.ethertype.ethtype_id == cpu_to_be16(ICE_IPV6_ETHER_ID) && lkups[i].m_u.ethertype.ethtype_id == cpu_to_be16(0xFFFF)) - inner_ipv6 = true; + match |= ICE_PKT_INNER_IPV6; else if (lkups[i].type == ICE_IPV6_IL) - inner_ipv6 = true; + match |= ICE_PKT_INNER_IPV6; else if (lkups[i].type == ICE_GTP_NO_PAY) - gtp_no_pay = true; + match |= ICE_PKT_GTP_NOPAY; } - if (tun_type == ICE_SW_TUN_GTPU) { - if (outer_ipv6) { - if (gtp_no_pay) { - *pkt = dummy_ipv6_gtp_packet; - *pkt_len = sizeof(dummy_ipv6_gtp_packet); - *offsets = dummy_ipv6_gtp_no_pay_packet_offsets; - } else if (inner_ipv6) { - if (inner_udp) { - *pkt = dummy_ipv6_gtpu_ipv6_udp_packet; - *pkt_len = sizeof(dummy_ipv6_gtpu_ipv6_udp_packet); - *offsets = dummy_ipv6_gtpu_ipv6_udp_packet_offsets; - } else { - *pkt = dummy_ipv6_gtpu_ipv6_tcp_packet; - *pkt_len = sizeof(dummy_ipv6_gtpu_ipv6_tcp_packet); - *offsets = dummy_ipv6_gtpu_ipv6_tcp_packet_offsets; - } - } else { - if (inner_udp) { - *pkt = dummy_ipv6_gtpu_ipv4_udp_packet; - *pkt_len = sizeof(dummy_ipv6_gtpu_ipv4_udp_packet); - *offsets = dummy_ipv6_gtpu_ipv4_udp_packet_offsets; - } else { - *pkt = dummy_ipv6_gtpu_ipv4_tcp_packet; - *pkt_len = sizeof(dummy_ipv6_gtpu_ipv4_tcp_packet); - *offsets = dummy_ipv6_gtpu_ipv4_tcp_packet_offsets; - } - } - } else { - if (gtp_no_pay) { - *pkt = dummy_ipv4_gtpu_ipv4_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv4_packet); - *offsets = dummy_ipv4_gtp_no_pay_packet_offsets; - } else if (inner_ipv6) { - if (inner_udp) { - *pkt = dummy_ipv4_gtpu_ipv6_udp_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv6_udp_packet); - *offsets = dummy_ipv4_gtpu_ipv6_udp_packet_offsets; - } else { - *pkt = dummy_ipv4_gtpu_ipv6_tcp_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv6_tcp_packet); - *offsets = dummy_ipv4_gtpu_ipv6_tcp_packet_offsets; - } - } else { - if (inner_udp) { - *pkt = dummy_ipv4_gtpu_ipv4_udp_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv4_udp_packet); - *offsets = dummy_ipv4_gtpu_ipv4_udp_packet_offsets; - } else { - *pkt = dummy_ipv4_gtpu_ipv4_tcp_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv4_tcp_packet); - *offsets = dummy_ipv4_gtpu_ipv4_tcp_packet_offsets; - } - } - } - return; - } - - if (tun_type == ICE_SW_TUN_GTPC) { - if (outer_ipv6) { - *pkt = dummy_ipv6_gtp_packet; - *pkt_len = sizeof(dummy_ipv6_gtp_packet); - *offsets = dummy_ipv6_gtp_no_pay_packet_offsets; - } else { - *pkt = dummy_ipv4_gtpu_ipv4_packet; - *pkt_len = sizeof(dummy_ipv4_gtpu_ipv4_packet); - *offsets = dummy_ipv4_gtp_no_pay_packet_offsets; - } - return; - } - - if (tun_type == ICE_SW_TUN_NVGRE) { - if (inner_tcp && inner_ipv6) { - *pkt = dummy_gre_ipv6_tcp_packet; - *pkt_len = sizeof(dummy_gre_ipv6_tcp_packet); - *offsets = dummy_gre_ipv6_tcp_packet_offsets; - return; - } - if (inner_tcp) { - *pkt = dummy_gre_tcp_packet; - *pkt_len = sizeof(dummy_gre_tcp_packet); - *offsets = dummy_gre_tcp_packet_offsets; - return; - } - if (inner_ipv6) { - *pkt = dummy_gre_ipv6_udp_packet; - *pkt_len = sizeof(dummy_gre_ipv6_udp_packet); - *offsets = dummy_gre_ipv6_udp_packet_offsets; - return; - } - *pkt = dummy_gre_udp_packet; - *pkt_len = sizeof(dummy_gre_udp_packet); - *offsets = dummy_gre_udp_packet_offsets; - return; - } - - if (tun_type == ICE_SW_TUN_VXLAN || - tun_type == ICE_SW_TUN_GENEVE) { - if (inner_tcp && inner_ipv6) { - *pkt = dummy_udp_tun_ipv6_tcp_packet; - *pkt_len = sizeof(dummy_udp_tun_ipv6_tcp_packet); - *offsets = dummy_udp_tun_ipv6_tcp_packet_offsets; - return; - } - if (inner_tcp) { - *pkt = dummy_udp_tun_tcp_packet; - *pkt_len = sizeof(dummy_udp_tun_tcp_packet); - *offsets = dummy_udp_tun_tcp_packet_offsets; - return; - } - if (inner_ipv6) { - *pkt = dummy_udp_tun_ipv6_udp_packet; - *pkt_len = sizeof(dummy_udp_tun_ipv6_udp_packet); - *offsets = dummy_udp_tun_ipv6_udp_packet_offsets; - return; - } - *pkt = dummy_udp_tun_udp_packet; - *pkt_len = sizeof(dummy_udp_tun_udp_packet); - *offsets = dummy_udp_tun_udp_packet_offsets; - return; - } + while (ret->match && (match & ret->match) != ret->match) + ret++; - if (inner_udp && !outer_ipv6) { - if (vlan) { - *pkt = dummy_vlan_udp_packet; - *pkt_len = sizeof(dummy_vlan_udp_packet); - *offsets = dummy_vlan_udp_packet_offsets; - return; - } - *pkt = dummy_udp_packet; - *pkt_len = sizeof(dummy_udp_packet); - *offsets = dummy_udp_packet_offsets; - return; - } else if (inner_udp && outer_ipv6) { - if (vlan) { - *pkt = dummy_vlan_udp_ipv6_packet; - *pkt_len = sizeof(dummy_vlan_udp_ipv6_packet); - *offsets = dummy_vlan_udp_ipv6_packet_offsets; - return; - } - *pkt = dummy_udp_ipv6_packet; - *pkt_len = sizeof(dummy_udp_ipv6_packet); - *offsets = dummy_udp_ipv6_packet_offsets; - return; - } else if ((inner_tcp && outer_ipv6) || outer_ipv6) { - if (vlan) { - *pkt = dummy_vlan_tcp_ipv6_packet; - *pkt_len = sizeof(dummy_vlan_tcp_ipv6_packet); - *offsets = dummy_vlan_tcp_ipv6_packet_offsets; - return; - } - *pkt = dummy_tcp_ipv6_packet; - *pkt_len = sizeof(dummy_tcp_ipv6_packet); - *offsets = dummy_tcp_ipv6_packet_offsets; - return; - } - - if (vlan) { - *pkt = dummy_vlan_tcp_packet; - *pkt_len = sizeof(dummy_vlan_tcp_packet); - *offsets = dummy_vlan_tcp_packet_offsets; - } else { - *pkt = dummy_tcp_packet; - *pkt_len = sizeof(dummy_tcp_packet); - *offsets = dummy_tcp_packet_offsets; - } + return ret; } /** @@ -5716,15 +5637,12 @@ ice_find_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, * structure per protocol header * @lkups_cnt: number of protocols * @s_rule: stores rule information from the match criteria - * @dummy_pkt: dummy packet to fill according to filter match criteria - * @pkt_len: packet length of dummy packet - * @offsets: offset info for the dummy packet + * @profile: dummy packet profile (the template, its size and header offsets) */ static int ice_fill_adv_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, struct ice_aqc_sw_rules_elem *s_rule, - const u8 *dummy_pkt, u16 pkt_len, - const struct ice_dummy_pkt_offsets *offsets) + const struct ice_dummy_pkt_profile *profile) { u8 *pkt; u16 i; @@ -5734,9 +5652,10 @@ ice_fill_adv_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, */ pkt = s_rule->pdata.lkup_tx_rx.hdr; - memcpy(pkt, dummy_pkt, pkt_len); + memcpy(pkt, profile->pkt, profile->pkt_len); for (i = 0; i < lkups_cnt; i++) { + const struct ice_dummy_pkt_offsets *offsets = profile->offsets; enum ice_protocol_type type; u16 offset = 0, len = 0, j; bool found = false; @@ -5810,16 +5729,18 @@ ice_fill_adv_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, * indicated by the mask to make sure we don't improperly write * over any significant packet data. */ - for (j = 0; j < len / sizeof(u16); j++) - if (((u16 *)&lkups[i].m_u)[j]) - ((u16 *)(pkt + offset))[j] = - (((u16 *)(pkt + offset))[j] & - ~((u16 *)&lkups[i].m_u)[j]) | - (((u16 *)&lkups[i].h_u)[j] & - ((u16 *)&lkups[i].m_u)[j]); + for (j = 0; j < len / sizeof(u16); j++) { + u16 *ptr = (u16 *)(pkt + offset); + u16 mask = lkups[i].m_raw[j]; + + if (!mask) + continue; + + ptr[j] = (ptr[j] & ~mask) | (lkups[i].h_raw[j] & mask); + } } - s_rule->pdata.lkup_tx_rx.hdr_len = cpu_to_le16(pkt_len); + s_rule->pdata.lkup_tx_rx.hdr_len = cpu_to_le16(profile->pkt_len); return 0; } @@ -6042,12 +5963,11 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, struct ice_rule_query_data *added_entry) { struct ice_adv_fltr_mgmt_list_entry *m_entry, *adv_fltr = NULL; - u16 rid = 0, i, pkt_len, rule_buf_sz, vsi_handle; - const struct ice_dummy_pkt_offsets *pkt_offsets; struct ice_aqc_sw_rules_elem *s_rule = NULL; + const struct ice_dummy_pkt_profile *profile; + u16 rid = 0, i, rule_buf_sz, vsi_handle; struct list_head *rule_head; struct ice_switch_info *sw; - const u8 *pkt = NULL; u16 word_cnt; u32 act = 0; int status; @@ -6065,24 +5985,18 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, /* get # of words we need to match */ word_cnt = 0; for (i = 0; i < lkups_cnt; i++) { - u16 j, *ptr; + u16 j; - ptr = (u16 *)&lkups[i].m_u; - for (j = 0; j < sizeof(lkups->m_u) / sizeof(u16); j++) - if (ptr[j] != 0) + for (j = 0; j < ARRAY_SIZE(lkups->m_raw); j++) + if (lkups[i].m_raw[j]) word_cnt++; } if (!word_cnt || word_cnt > ICE_MAX_CHAIN_WORDS) return -EINVAL; - /* make sure that we can locate a dummy packet */ - ice_find_dummy_packet(lkups, lkups_cnt, rinfo->tun_type, &pkt, &pkt_len, - &pkt_offsets); - if (!pkt) { - status = -EINVAL; - goto err_ice_add_adv_rule; - } + /* locate a dummy packet */ + profile = ice_find_dummy_packet(lkups, lkups_cnt, rinfo->tun_type); if (!(rinfo->sw_act.fltr_act == ICE_FWD_TO_VSI || rinfo->sw_act.fltr_act == ICE_FWD_TO_Q || @@ -6123,7 +6037,7 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, } return status; } - rule_buf_sz = ICE_SW_RULE_RX_TX_NO_HDR_SIZE + pkt_len; + rule_buf_sz = ICE_SW_RULE_RX_TX_NO_HDR_SIZE + profile->pkt_len; s_rule = kzalloc(rule_buf_sz, GFP_KERNEL); if (!s_rule) return -ENOMEM; @@ -6183,8 +6097,7 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, s_rule->pdata.lkup_tx_rx.recipe_id = cpu_to_le16(rid); s_rule->pdata.lkup_tx_rx.act = cpu_to_le32(act); - status = ice_fill_adv_dummy_packet(lkups, lkups_cnt, s_rule, pkt, - pkt_len, pkt_offsets); + status = ice_fill_adv_dummy_packet(lkups, lkups_cnt, s_rule, profile); if (status) goto err_ice_add_adv_rule; @@ -6192,7 +6105,7 @@ ice_add_adv_rule(struct ice_hw *hw, struct ice_adv_lkup_elem *lkups, rinfo->tun_type != ICE_SW_TUN_AND_NON_TUN) { status = ice_fill_adv_packet_tun(hw, rinfo->tun_type, s_rule->pdata.lkup_tx_rx.hdr, - pkt_offsets); + profile->offsets); if (status) goto err_ice_add_adv_rule; } diff --git a/drivers/net/ethernet/intel/ice/ice_switch.h b/drivers/net/ethernet/intel/ice/ice_switch.h index ed3d1d03befa..ecac75e71395 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.h +++ b/drivers/net/ethernet/intel/ice/ice_switch.h @@ -138,8 +138,16 @@ struct ice_update_recipe_lkup_idx_params { struct ice_adv_lkup_elem { enum ice_protocol_type type; - union ice_prot_hdr h_u; /* Header values */ - union ice_prot_hdr m_u; /* Mask of header values to match */ + union { + union ice_prot_hdr h_u; /* Header values */ + /* Used to iterate over the headers */ + u16 h_raw[sizeof(union ice_prot_hdr) / sizeof(u16)]; + }; + union { + union ice_prot_hdr m_u; /* Mask of header values to match */ + /* Used to iterate over header mask */ + u16 m_raw[sizeof(union ice_prot_hdr) / sizeof(u16)]; + }; }; struct ice_sw_act_ctrl { diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index 3f1a63815bac..69ff4b929772 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -1358,9 +1358,9 @@ static int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg) goto error_param; } - /* Skip queue if not enabled */ if (!test_bit(vf_q_id, vf->txq_ena)) - continue; + dev_dbg(ice_pf_to_dev(vsi->back), "Queue %u on VSI %u is not enabled, but stopping it anyway\n", + vf_q_id, vsi->vsi_num); ice_fill_txq_meta(vsi, ring, &txq_meta); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index dfbcaf08520e..866ee4df9671 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -41,8 +41,10 @@ static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) { ice_clean_tx_ring(vsi->tx_rings[q_idx]); - if (ice_is_xdp_ena_vsi(vsi)) + if (ice_is_xdp_ena_vsi(vsi)) { + synchronize_rcu(); ice_clean_tx_ring(vsi->xdp_rings[q_idx]); + } ice_clean_rx_ring(vsi->rx_rings[q_idx]); } @@ -918,7 +920,7 @@ ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, struct ice_vsi *vsi = np->vsi; struct ice_tx_ring *ring; - if (test_bit(ICE_DOWN, vsi->state)) + if (test_bit(ICE_VSI_DOWN, vsi->state)) return -ENETDOWN; if (!ice_is_xdp_ena_vsi(vsi)) diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 5f9ab1842d49..c18801490649 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -2751,7 +2751,7 @@ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev, } ret = of_get_mac_address(pnp, ppd.mac_addr); - if (ret) + if (ret == -EPROBE_DEFER) return ret; mv643xx_eth_property(pnp, "tx-queue-size", ppd.tx_queue_size); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c index 47c899c08951..e5627782fac6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -421,13 +421,6 @@ int prestera_acl_rule_add(struct prestera_switch *sw, rule->re_arg.vtcam_id = ruleset->vtcam_id; rule->re_key.prio = rule->priority; - /* setup counter */ - rule->re_arg.count.valid = true; - err = prestera_acl_chain_to_client(ruleset->ht_key.chain_index, - &rule->re_arg.count.client); - if (err) - goto err_rule_add; - rule->re = prestera_acl_rule_entry_find(sw->acl, &rule->re_key); err = WARN_ON(rule->re) ? -EEXIST : 0; if (err) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index 921959a980ee..c12b09ac6559 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -70,6 +70,24 @@ static int prestera_flower_parse_actions(struct prestera_flow_block *block, if (!flow_action_has_entries(flow_action)) return 0; + if (!flow_action_mixed_hw_stats_check(flow_action, extack)) + return -EOPNOTSUPP; + + act = flow_action_first_entry_get(flow_action); + if (act->hw_stats & FLOW_ACTION_HW_STATS_DISABLED) { + /* Nothing to do */ + } else if (act->hw_stats & FLOW_ACTION_HW_STATS_DELAYED) { + /* setup counter first */ + rule->re_arg.count.valid = true; + err = prestera_acl_chain_to_client(chain_index, + &rule->re_arg.count.client); + if (err) + return err; + } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported action HW stats type"); + return -EOPNOTSUPP; + } + flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_ACCEPT: diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig index 86d356b4388d..da4ec235d146 100644 --- a/drivers/net/ethernet/mediatek/Kconfig +++ b/drivers/net/ethernet/mediatek/Kconfig @@ -7,6 +7,10 @@ config NET_VENDOR_MEDIATEK if NET_VENDOR_MEDIATEK +config NET_MEDIATEK_SOC_WED + depends on ARCH_MEDIATEK || COMPILE_TEST + def_bool NET_MEDIATEK_SOC != n + config NET_MEDIATEK_SOC tristate "MediaTek SoC Gigabit Ethernet support" depends on NET_DSA || !NET_DSA diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile index 79d4cdbbcbf5..45ba0970504a 100644 --- a/drivers/net/ethernet/mediatek/Makefile +++ b/drivers/net/ethernet/mediatek/Makefile @@ -5,4 +5,9 @@ obj-$(CONFIG_NET_MEDIATEK_SOC) += mtk_eth.o mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o +mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed.o +ifdef CONFIG_DEBUG_FS +mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_debugfs.o +endif +obj-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_ops.o obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index f02d07ec5ccb..209d00f56f62 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -9,6 +9,7 @@ #include <linux/of_device.h> #include <linux/of_mdio.h> #include <linux/of_net.h> +#include <linux/of_address.h> #include <linux/mfd/syscon.h> #include <linux/regmap.h> #include <linux/clk.h> @@ -20,9 +21,11 @@ #include <linux/pinctrl/devinfo.h> #include <linux/phylink.h> #include <linux/jhash.h> +#include <linux/bitfield.h> #include <net/dsa.h> #include "mtk_eth_soc.h" +#include "mtk_wed.h" static int mtk_msg_level = -1; module_param_named(msg_level, mtk_msg_level, int, 0); @@ -786,7 +789,7 @@ static int mtk_init_fq_dma(struct mtk_eth *eth) dma_addr_t dma_addr; int i; - eth->scratch_ring = dma_alloc_coherent(eth->dev, + eth->scratch_ring = dma_alloc_coherent(eth->dma_dev, cnt * sizeof(struct mtk_tx_dma), ð->phy_scratch_ring, GFP_ATOMIC); @@ -798,10 +801,10 @@ static int mtk_init_fq_dma(struct mtk_eth *eth) if (unlikely(!eth->scratch_head)) return -ENOMEM; - dma_addr = dma_map_single(eth->dev, + dma_addr = dma_map_single(eth->dma_dev, eth->scratch_head, cnt * MTK_QDMA_PAGE_SIZE, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(eth->dev, dma_addr))) + if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) return -ENOMEM; phy_ring_tail = eth->phy_scratch_ring + @@ -855,26 +858,26 @@ static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, { if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) { if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) { - dma_unmap_single(eth->dev, + dma_unmap_single(eth->dma_dev, dma_unmap_addr(tx_buf, dma_addr0), dma_unmap_len(tx_buf, dma_len0), DMA_TO_DEVICE); } else if (tx_buf->flags & MTK_TX_FLAGS_PAGE0) { - dma_unmap_page(eth->dev, + dma_unmap_page(eth->dma_dev, dma_unmap_addr(tx_buf, dma_addr0), dma_unmap_len(tx_buf, dma_len0), DMA_TO_DEVICE); } } else { if (dma_unmap_len(tx_buf, dma_len0)) { - dma_unmap_page(eth->dev, + dma_unmap_page(eth->dma_dev, dma_unmap_addr(tx_buf, dma_addr0), dma_unmap_len(tx_buf, dma_len0), DMA_TO_DEVICE); } if (dma_unmap_len(tx_buf, dma_len1)) { - dma_unmap_page(eth->dev, + dma_unmap_page(eth->dma_dev, dma_unmap_addr(tx_buf, dma_addr1), dma_unmap_len(tx_buf, dma_len1), DMA_TO_DEVICE); @@ -952,9 +955,9 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, if (skb_vlan_tag_present(skb)) txd4 |= TX_DMA_INS_VLAN | skb_vlan_tag_get(skb); - mapped_addr = dma_map_single(eth->dev, skb->data, + mapped_addr = dma_map_single(eth->dma_dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(eth->dev, mapped_addr))) + if (unlikely(dma_mapping_error(eth->dma_dev, mapped_addr))) return -ENOMEM; WRITE_ONCE(itxd->txd1, mapped_addr); @@ -993,10 +996,10 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, frag_map_size = min(frag_size, MTK_TX_DMA_BUF_LEN); - mapped_addr = skb_frag_dma_map(eth->dev, frag, offset, + mapped_addr = skb_frag_dma_map(eth->dma_dev, frag, offset, frag_map_size, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(eth->dev, mapped_addr))) + if (unlikely(dma_mapping_error(eth->dma_dev, mapped_addr))) goto err_dma; if (i == nr_frags - 1 && @@ -1237,7 +1240,7 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, struct net_device *netdev; unsigned int pktlen; dma_addr_t dma_addr; - u32 hash; + u32 hash, reason; int mac; ring = mtk_get_rx_ring(eth); @@ -1274,18 +1277,18 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, netdev->stats.rx_dropped++; goto release_desc; } - dma_addr = dma_map_single(eth->dev, + dma_addr = dma_map_single(eth->dma_dev, new_data + NET_SKB_PAD + eth->ip_align, ring->buf_size, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(eth->dev, dma_addr))) { + if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) { skb_free_frag(new_data); netdev->stats.rx_dropped++; goto release_desc; } - dma_unmap_single(eth->dev, trxd.rxd1, + dma_unmap_single(eth->dma_dev, trxd.rxd1, ring->buf_size, DMA_FROM_DEVICE); /* receive data */ @@ -1313,6 +1316,11 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); } + reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4); + if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) + mtk_ppe_check_skb(eth->ppe, skb, + trxd.rxd4 & MTK_RXD4_FOE_ENTRY); + if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX && (trxd.rxd2 & RX_DMA_VTAG)) __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), @@ -1558,7 +1566,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth) if (!ring->buf) goto no_tx_mem; - ring->dma = dma_alloc_coherent(eth->dev, MTK_DMA_SIZE * sz, + ring->dma = dma_alloc_coherent(eth->dma_dev, MTK_DMA_SIZE * sz, &ring->phys, GFP_ATOMIC); if (!ring->dma) goto no_tx_mem; @@ -1576,7 +1584,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth) * descriptors in ring->dma_pdma. */ if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) { - ring->dma_pdma = dma_alloc_coherent(eth->dev, MTK_DMA_SIZE * sz, + ring->dma_pdma = dma_alloc_coherent(eth->dma_dev, MTK_DMA_SIZE * sz, &ring->phys_pdma, GFP_ATOMIC); if (!ring->dma_pdma) @@ -1635,7 +1643,7 @@ static void mtk_tx_clean(struct mtk_eth *eth) } if (ring->dma) { - dma_free_coherent(eth->dev, + dma_free_coherent(eth->dma_dev, MTK_DMA_SIZE * sizeof(*ring->dma), ring->dma, ring->phys); @@ -1643,7 +1651,7 @@ static void mtk_tx_clean(struct mtk_eth *eth) } if (ring->dma_pdma) { - dma_free_coherent(eth->dev, + dma_free_coherent(eth->dma_dev, MTK_DMA_SIZE * sizeof(*ring->dma_pdma), ring->dma_pdma, ring->phys_pdma); @@ -1688,18 +1696,18 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag) return -ENOMEM; } - ring->dma = dma_alloc_coherent(eth->dev, + ring->dma = dma_alloc_coherent(eth->dma_dev, rx_dma_size * sizeof(*ring->dma), &ring->phys, GFP_ATOMIC); if (!ring->dma) return -ENOMEM; for (i = 0; i < rx_dma_size; i++) { - dma_addr_t dma_addr = dma_map_single(eth->dev, + dma_addr_t dma_addr = dma_map_single(eth->dma_dev, ring->data[i] + NET_SKB_PAD + eth->ip_align, ring->buf_size, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(eth->dev, dma_addr))) + if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) return -ENOMEM; ring->dma[i].rxd1 = (unsigned int)dma_addr; @@ -1735,7 +1743,7 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring) continue; if (!ring->dma[i].rxd1) continue; - dma_unmap_single(eth->dev, + dma_unmap_single(eth->dma_dev, ring->dma[i].rxd1, ring->buf_size, DMA_FROM_DEVICE); @@ -1746,7 +1754,7 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring) } if (ring->dma) { - dma_free_coherent(eth->dev, + dma_free_coherent(eth->dma_dev, ring->dma_size * sizeof(*ring->dma), ring->dma, ring->phys); @@ -2099,7 +2107,7 @@ static void mtk_dma_free(struct mtk_eth *eth) if (eth->netdev[i]) netdev_reset_queue(eth->netdev[i]); if (eth->scratch_ring) { - dma_free_coherent(eth->dev, + dma_free_coherent(eth->dma_dev, MTK_DMA_SIZE * sizeof(struct mtk_tx_dma), eth->scratch_ring, eth->phy_scratch_ring); @@ -2267,7 +2275,7 @@ static int mtk_open(struct net_device *dev) if (err) return err; - if (eth->soc->offload_version && mtk_ppe_start(ð->ppe) == 0) + if (eth->soc->offload_version && mtk_ppe_start(eth->ppe) == 0) gdm_config = MTK_GDMA_TO_PPE; mtk_gdm_config(eth, gdm_config); @@ -2341,7 +2349,7 @@ static int mtk_stop(struct net_device *dev) mtk_dma_free(eth); if (eth->soc->offload_version) - mtk_ppe_stop(ð->ppe); + mtk_ppe_stop(eth->ppe); return 0; } @@ -2448,6 +2456,8 @@ static void mtk_dim_tx(struct work_struct *work) static int mtk_hw_init(struct mtk_eth *eth) { + u32 dma_mask = ETHSYS_DMA_AG_MAP_PDMA | ETHSYS_DMA_AG_MAP_QDMA | + ETHSYS_DMA_AG_MAP_PPE; int i, val, ret; if (test_and_set_bit(MTK_HW_INIT, ð->state)) @@ -2460,6 +2470,10 @@ static int mtk_hw_init(struct mtk_eth *eth) if (ret) goto err_disable_pm; + if (eth->ethsys) + regmap_update_bits(eth->ethsys, ETHSYS_DMA_AG_MAP, dma_mask, + of_dma_is_coherent(eth->dma_dev->of_node) * dma_mask); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { ret = device_reset(eth->dev); if (ret) { @@ -3040,6 +3054,35 @@ free_netdev: return err; } +void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev) +{ + struct net_device *dev, *tmp; + LIST_HEAD(dev_list); + int i; + + rtnl_lock(); + + for (i = 0; i < MTK_MAC_COUNT; i++) { + dev = eth->netdev[i]; + + if (!dev || !(dev->flags & IFF_UP)) + continue; + + list_add_tail(&dev->close_list, &dev_list); + } + + dev_close_many(&dev_list, false); + + eth->dma_dev = dma_dev; + + list_for_each_entry_safe(dev, tmp, &dev_list, close_list) { + list_del_init(&dev->close_list); + dev_open(dev, NULL); + } + + rtnl_unlock(); +} + static int mtk_probe(struct platform_device *pdev) { struct device_node *mac_np; @@ -3053,6 +3096,7 @@ static int mtk_probe(struct platform_device *pdev) eth->soc = of_device_get_match_data(&pdev->dev); eth->dev = &pdev->dev; + eth->dma_dev = &pdev->dev; eth->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(eth->base)) return PTR_ERR(eth->base); @@ -3101,6 +3145,16 @@ static int mtk_probe(struct platform_device *pdev) } } + if (of_dma_is_coherent(pdev->dev.of_node)) { + struct regmap *cci; + + cci = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, + "mediatek,cci-control"); + /* enable CPU/bus coherency */ + if (!IS_ERR(cci)) + regmap_write(cci, 0, 3); + } + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) { eth->sgmii = devm_kzalloc(eth->dev, sizeof(*eth->sgmii), GFP_KERNEL); @@ -3123,6 +3177,22 @@ static int mtk_probe(struct platform_device *pdev) } } + for (i = 0;; i++) { + struct device_node *np = of_parse_phandle(pdev->dev.of_node, + "mediatek,wed", i); + static const u32 wdma_regs[] = { + MTK_WDMA0_BASE, + MTK_WDMA1_BASE + }; + void __iomem *wdma; + + if (!np || i >= ARRAY_SIZE(wdma_regs)) + break; + + wdma = eth->base + wdma_regs[i]; + mtk_wed_add_hw(np, eth, wdma, i); + } + for (i = 0; i < 3; i++) { if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT) && i > 0) eth->irq[i] = eth->irq[0]; @@ -3198,10 +3268,11 @@ static int mtk_probe(struct platform_device *pdev) } if (eth->soc->offload_version) { - err = mtk_ppe_init(ð->ppe, eth->dev, - eth->base + MTK_ETH_PPE_BASE, 2); - if (err) + eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2); + if (!eth->ppe) { + err = -ENOMEM; goto err_free_dev; + } err = mtk_eth_offload_init(eth); if (err) diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index c9d42be314b5..c98c7ee42c6f 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -295,6 +295,9 @@ #define MTK_GDM1_TX_GPCNT 0x2438 #define MTK_STAT_OFFSET 0x40 +#define MTK_WDMA0_BASE 0x2800 +#define MTK_WDMA1_BASE 0x2c00 + /* QDMA descriptor txd4 */ #define TX_DMA_CHKSUM (0x7 << 29) #define TX_DMA_TSO BIT(28) @@ -465,6 +468,12 @@ #define RSTCTRL_FE BIT(6) #define RSTCTRL_PPE BIT(31) +/* ethernet dma channel agent map */ +#define ETHSYS_DMA_AG_MAP 0x408 +#define ETHSYS_DMA_AG_MAP_PDMA BIT(0) +#define ETHSYS_DMA_AG_MAP_QDMA BIT(1) +#define ETHSYS_DMA_AG_MAP_PPE BIT(2) + /* SGMII subsystem config registers */ /* Register to auto-negotiation restart */ #define SGMSYS_PCS_CONTROL_1 0x0 @@ -882,6 +891,7 @@ struct mtk_sgmii { /* struct mtk_eth - This is the main datasructure for holding the state * of the driver * @dev: The device pointer + * @dev: The device pointer used for dma mapping/alloc * @base: The mapped register i/o base * @page_lock: Make sure that register operations are atomic * @tx_irq__lock: Make sure that IRQ register operations are atomic @@ -925,6 +935,7 @@ struct mtk_sgmii { struct mtk_eth { struct device *dev; + struct device *dma_dev; void __iomem *base; spinlock_t page_lock; spinlock_t tx_irq_lock; @@ -974,7 +985,7 @@ struct mtk_eth { u32 rx_dma_l4_valid; int ip_align; - struct mtk_ppe ppe; + struct mtk_ppe *ppe; struct rhashtable flow_table; }; @@ -1023,6 +1034,7 @@ int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id); int mtk_eth_offload_init(struct mtk_eth *eth); int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data); +void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev); #endif /* MTK_ETH_H */ diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 3ad10c793308..282a1f34a88a 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -6,9 +6,22 @@ #include <linux/iopoll.h> #include <linux/etherdevice.h> #include <linux/platform_device.h> +#include <linux/if_ether.h> +#include <linux/if_vlan.h> +#include <net/dsa.h> +#include "mtk_eth_soc.h" #include "mtk_ppe.h" #include "mtk_ppe_regs.h" +static DEFINE_SPINLOCK(ppe_lock); + +static const struct rhashtable_params mtk_flow_l2_ht_params = { + .head_offset = offsetof(struct mtk_flow_entry, l2_node), + .key_offset = offsetof(struct mtk_flow_entry, data.bridge), + .key_len = offsetof(struct mtk_foe_bridge, key_end), + .automatic_shrinking = true, +}; + static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) { writel(val, ppe->base + reg); @@ -41,6 +54,11 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) return ppe_m32(ppe, reg, val, 0); } +static u32 mtk_eth_timestamp(struct mtk_eth *eth) +{ + return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP; +} + static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) { int ret; @@ -76,13 +94,6 @@ static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e) u32 hash; switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) { - case MTK_PPE_PKT_TYPE_BRIDGE: - hv1 = e->bridge.src_mac_lo; - hv1 ^= ((e->bridge.src_mac_hi & 0xffff) << 16); - hv2 = e->bridge.src_mac_hi >> 16; - hv2 ^= e->bridge.dest_mac_lo; - hv3 = e->bridge.dest_mac_hi; - break; case MTK_PPE_PKT_TYPE_IPV4_ROUTE: case MTK_PPE_PKT_TYPE_IPV4_HNAPT: hv1 = e->ipv4.orig.ports; @@ -122,6 +133,9 @@ mtk_foe_entry_l2(struct mtk_foe_entry *entry) { int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return &entry->bridge.l2; + if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) return &entry->ipv6.l2; @@ -133,6 +147,9 @@ mtk_foe_entry_ib2(struct mtk_foe_entry *entry) { int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return &entry->bridge.ib2; + if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) return &entry->ipv6.ib2; @@ -167,7 +184,12 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) entry->ipv6.ports = ports_pad; - if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { + if (type == MTK_PPE_PKT_TYPE_BRIDGE) { + ether_addr_copy(entry->bridge.src_mac, src_mac); + ether_addr_copy(entry->bridge.dest_mac, dest_mac); + entry->bridge.ib2 = val; + l2 = &entry->bridge.l2; + } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { entry->ipv6.ib2 = val; l2 = &entry->ipv6.l2; } else { @@ -329,32 +351,167 @@ int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid) return 0; } +int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq, + int bss, int wcid) +{ + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + u32 *ib2 = mtk_foe_entry_ib2(entry); + + *ib2 &= ~MTK_FOE_IB2_PORT_MG; + *ib2 |= MTK_FOE_IB2_WDMA_WINFO; + if (wdma_idx) + *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; + + l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); + + return 0; +} + static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) { return !(entry->ib1 & MTK_FOE_IB1_STATIC) && FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND; } -int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, - u16 timestamp) +static bool +mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data) +{ + int type, len; + + if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) + return false; + + type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1); + if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) + len = offsetof(struct mtk_foe_entry, ipv6._rsv); + else + len = offsetof(struct mtk_foe_entry, ipv4.ib2); + + return !memcmp(&entry->data.data, &data->data, len - 4); +} + +static void +__mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + struct hlist_head *head; + struct hlist_node *tmp; + + if (entry->type == MTK_FLOW_TYPE_L2) { + rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, + mtk_flow_l2_ht_params); + + head = &entry->l2_flows; + hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) + __mtk_foe_entry_clear(ppe, entry); + return; + } + + hlist_del_init(&entry->list); + if (entry->hash != 0xffff) { + ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE; + ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, + MTK_FOE_STATE_BIND); + dma_wmb(); + } + entry->hash = 0xffff; + + if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) + return; + + hlist_del_init(&entry->l2_data.list); + kfree(entry); +} + +static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) +{ + u16 timestamp; + u16 now; + + now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP; + timestamp = ib1 & MTK_FOE_IB1_BIND_TIMESTAMP; + + if (timestamp > now) + return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now; + else + return now - timestamp; +} + +static void +mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) { + struct mtk_flow_entry *cur; struct mtk_foe_entry *hwe; - u32 hash; + struct hlist_node *tmp; + int idle; + + idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); + hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { + int cur_idle; + u32 ib1; + + hwe = &ppe->foe_table[cur->hash]; + ib1 = READ_ONCE(hwe->ib1); + + if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { + cur->hash = 0xffff; + __mtk_foe_entry_clear(ppe, cur); + continue; + } + + cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); + if (cur_idle >= idle) + continue; + + idle = cur_idle; + entry->data.ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; + entry->data.ib1 |= hwe->ib1 & MTK_FOE_IB1_BIND_TIMESTAMP; + } +} + +static void +mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + struct mtk_foe_entry *hwe; + struct mtk_foe_entry foe; + + spin_lock_bh(&ppe_lock); + + if (entry->type == MTK_FLOW_TYPE_L2) { + mtk_flow_entry_update_l2(ppe, entry); + goto out; + } + + if (entry->hash == 0xffff) + goto out; + + hwe = &ppe->foe_table[entry->hash]; + memcpy(&foe, hwe, sizeof(foe)); + if (!mtk_flow_entry_match(entry, &foe)) { + entry->hash = 0xffff; + goto out; + } + + entry->data.ib1 = foe.ib1; + +out: + spin_unlock_bh(&ppe_lock); +} + +static void +__mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, + u16 hash) +{ + struct mtk_foe_entry *hwe; + u16 timestamp; + timestamp = mtk_eth_timestamp(ppe->eth); timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP; entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp); - hash = mtk_ppe_hash_entry(entry); hwe = &ppe->foe_table[hash]; - if (!mtk_foe_entry_usable(hwe)) { - hwe++; - hash++; - - if (!mtk_foe_entry_usable(hwe)) - return -ENOSPC; - } - memcpy(&hwe->data, &entry->data, sizeof(hwe->data)); wmb(); hwe->ib1 = entry->ib1; @@ -362,32 +519,194 @@ int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, dma_wmb(); mtk_ppe_cache_clear(ppe); +} - return hash; +void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + spin_lock_bh(&ppe_lock); + __mtk_foe_entry_clear(ppe, entry); + spin_unlock_bh(&ppe_lock); +} + +static int +mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + entry->type = MTK_FLOW_TYPE_L2; + + return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node, + mtk_flow_l2_ht_params); +} + +int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1); + u32 hash; + + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return mtk_foe_entry_commit_l2(ppe, entry); + + hash = mtk_ppe_hash_entry(&entry->data); + entry->hash = 0xffff; + spin_lock_bh(&ppe_lock); + hlist_add_head(&entry->list, &ppe->foe_flow[hash / 2]); + spin_unlock_bh(&ppe_lock); + + return 0; +} + +static void +mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, + u16 hash) +{ + struct mtk_flow_entry *flow_info; + struct mtk_foe_entry foe, *hwe; + struct mtk_foe_mac_info *l2; + u32 ib1_mask = MTK_FOE_IB1_PACKET_TYPE | MTK_FOE_IB1_UDP; + int type; + + flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), + GFP_ATOMIC); + if (!flow_info) + return; + + flow_info->l2_data.base_flow = entry; + flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; + flow_info->hash = hash; + hlist_add_head(&flow_info->list, &ppe->foe_flow[hash / 2]); + hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); + + hwe = &ppe->foe_table[hash]; + memcpy(&foe, hwe, sizeof(foe)); + foe.ib1 &= ib1_mask; + foe.ib1 |= entry->data.ib1 & ~ib1_mask; + + l2 = mtk_foe_entry_l2(&foe); + memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); + + type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, foe.ib1); + if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) + memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); + else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) + l2->etype = ETH_P_IPV6; + + *mtk_foe_entry_ib2(&foe) = entry->data.bridge.ib2; + + __mtk_foe_entry_commit(ppe, &foe, hash); } -int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base, +void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) +{ + struct hlist_head *head = &ppe->foe_flow[hash / 2]; + struct mtk_foe_entry *hwe = &ppe->foe_table[hash]; + struct mtk_flow_entry *entry; + struct mtk_foe_bridge key = {}; + struct ethhdr *eh; + bool found = false; + u8 *tag; + + spin_lock_bh(&ppe_lock); + + if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) + goto out; + + hlist_for_each_entry(entry, head, list) { + if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { + if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == + MTK_FOE_STATE_BIND)) + continue; + + entry->hash = 0xffff; + __mtk_foe_entry_clear(ppe, entry); + continue; + } + + if (found || !mtk_flow_entry_match(entry, hwe)) { + if (entry->hash != 0xffff) + entry->hash = 0xffff; + continue; + } + + entry->hash = hash; + __mtk_foe_entry_commit(ppe, &entry->data, hash); + found = true; + } + + if (found) + goto out; + + eh = eth_hdr(skb); + ether_addr_copy(key.dest_mac, eh->h_dest); + ether_addr_copy(key.src_mac, eh->h_source); + tag = skb->data - 2; + key.vlan = 0; + switch (skb->protocol) { +#if IS_ENABLED(CONFIG_NET_DSA) + case htons(ETH_P_XDSA): + if (!netdev_uses_dsa(skb->dev) || + skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) + goto out; + + tag += 4; + if (get_unaligned_be16(tag) != ETH_P_8021Q) + break; + + fallthrough; +#endif + case htons(ETH_P_8021Q): + key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; + break; + default: + break; + } + + entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); + if (!entry) + goto out; + + mtk_foe_entry_commit_subflow(ppe, entry, hash); + +out: + spin_unlock_bh(&ppe_lock); +} + +int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + mtk_flow_entry_update(ppe, entry); + + return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); +} + +struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version) { + struct device *dev = eth->dev; struct mtk_foe_entry *foe; + struct mtk_ppe *ppe; + + ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); + if (!ppe) + return NULL; + + rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); /* need to allocate a separate device, since it PPE DMA access is * not coherent. */ ppe->base = base; + ppe->eth = eth; ppe->dev = dev; ppe->version = version; foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe), &ppe->foe_phys, GFP_KERNEL); if (!foe) - return -ENOMEM; + return NULL; ppe->foe_table = foe; mtk_ppe_debugfs_init(ppe); - return 0; + return ppe; } static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) @@ -443,7 +762,6 @@ int mtk_ppe_start(struct mtk_ppe *ppe) MTK_PPE_FLOW_CFG_IP4_NAT | MTK_PPE_FLOW_CFG_IP4_NAPT | MTK_PPE_FLOW_CFG_IP4_DSLITE | - MTK_PPE_FLOW_CFG_L2_BRIDGE | MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index 242fb8f2ae65..1f5cf1c9a947 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -6,6 +6,7 @@ #include <linux/kernel.h> #include <linux/bitfield.h> +#include <linux/rhashtable.h> #define MTK_ETH_PPE_BASE 0xc00 @@ -48,9 +49,9 @@ enum { #define MTK_FOE_IB2_DEST_PORT GENMASK(7, 5) #define MTK_FOE_IB2_MULTICAST BIT(8) -#define MTK_FOE_IB2_WHNAT_QID2 GENMASK(13, 12) -#define MTK_FOE_IB2_WHNAT_DEVIDX BIT(16) -#define MTK_FOE_IB2_WHNAT_NAT BIT(17) +#define MTK_FOE_IB2_WDMA_QID2 GENMASK(13, 12) +#define MTK_FOE_IB2_WDMA_DEVIDX BIT(16) +#define MTK_FOE_IB2_WDMA_WINFO BIT(17) #define MTK_FOE_IB2_PORT_MG GENMASK(17, 12) @@ -58,9 +59,9 @@ enum { #define MTK_FOE_IB2_DSCP GENMASK(31, 24) -#define MTK_FOE_VLAN2_WHNAT_BSS GEMMASK(5, 0) -#define MTK_FOE_VLAN2_WHNAT_WCID GENMASK(13, 6) -#define MTK_FOE_VLAN2_WHNAT_RING GENMASK(15, 14) +#define MTK_FOE_VLAN2_WINFO_BSS GENMASK(5, 0) +#define MTK_FOE_VLAN2_WINFO_WCID GENMASK(13, 6) +#define MTK_FOE_VLAN2_WINFO_RING GENMASK(15, 14) enum { MTK_FOE_STATE_INVALID, @@ -84,19 +85,16 @@ struct mtk_foe_mac_info { u16 src_mac_lo; }; +/* software-only entry type */ struct mtk_foe_bridge { - u32 dest_mac_hi; + u8 dest_mac[ETH_ALEN]; + u8 src_mac[ETH_ALEN]; + u16 vlan; - u16 src_mac_lo; - u16 dest_mac_lo; - - u32 src_mac_hi; + struct {} key_end; u32 ib2; - u32 _rsv[5]; - - u32 udf_tsid; struct mtk_foe_mac_info l2; }; @@ -235,7 +233,37 @@ enum { MTK_PPE_CPU_REASON_INVALID = 0x1f, }; +enum { + MTK_FLOW_TYPE_L4, + MTK_FLOW_TYPE_L2, + MTK_FLOW_TYPE_L2_SUBFLOW, +}; + +struct mtk_flow_entry { + union { + struct hlist_node list; + struct { + struct rhash_head l2_node; + struct hlist_head l2_flows; + }; + }; + u8 type; + s8 wed_index; + u16 hash; + union { + struct mtk_foe_entry data; + struct { + struct mtk_flow_entry *base_flow; + struct hlist_node list; + struct {} end; + } l2_data; + }; + struct rhash_head node; + unsigned long cookie; +}; + struct mtk_ppe { + struct mtk_eth *eth; struct device *dev; void __iomem *base; int version; @@ -243,19 +271,35 @@ struct mtk_ppe { struct mtk_foe_entry *foe_table; dma_addr_t foe_phys; + u16 foe_check_time[MTK_PPE_ENTRIES]; + struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2]; + + struct rhashtable l2_flows; + void *acct_table; }; -int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base, - int version); +struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version); int mtk_ppe_start(struct mtk_ppe *ppe); int mtk_ppe_stop(struct mtk_ppe *ppe); +void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash); + static inline void -mtk_foe_entry_clear(struct mtk_ppe *ppe, u16 hash) +mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) { - ppe->foe_table[hash].ib1 = 0; - dma_wmb(); + u16 now, diff; + + if (!ppe) + return; + + now = (u16)jiffies; + diff = now - ppe->foe_check_time[hash]; + if (diff < HZ / 10) + return; + + ppe->foe_check_time[hash] = now; + __mtk_ppe_check_skb(ppe, skb, hash); } static inline int @@ -281,8 +325,11 @@ int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port); int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid); int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid); -int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, - u16 timestamp); +int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq, + int bss, int wcid); +int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); +void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); +int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); int mtk_ppe_debugfs_init(struct mtk_ppe *ppe); #endif diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c index d4b482340cb9..eb0b598f14e4 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c @@ -32,7 +32,6 @@ static const char *mtk_foe_pkt_type_str(int type) static const char * const type_str[] = { [MTK_PPE_PKT_TYPE_IPV4_HNAPT] = "IPv4 5T", [MTK_PPE_PKT_TYPE_IPV4_ROUTE] = "IPv4 3T", - [MTK_PPE_PKT_TYPE_BRIDGE] = "L2", [MTK_PPE_PKT_TYPE_IPV4_DSLITE] = "DS-LITE", [MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T] = "IPv6 3T", [MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T] = "IPv6 5T", diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index 7bb1f20002b5..1fe31058b0f2 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -6,10 +6,12 @@ #include <linux/if_ether.h> #include <linux/rhashtable.h> #include <linux/ip.h> +#include <linux/ipv6.h> #include <net/flow_offload.h> #include <net/pkt_cls.h> #include <net/dsa.h> #include "mtk_eth_soc.h" +#include "mtk_wed.h" struct mtk_flow_data { struct ethhdr eth; @@ -19,11 +21,18 @@ struct mtk_flow_data { __be32 src_addr; __be32 dst_addr; } v4; + + struct { + struct in6_addr src_addr; + struct in6_addr dst_addr; + } v6; }; __be16 src_port; __be16 dst_port; + u16 vlan_in; + struct { u16 id; __be16 proto; @@ -35,12 +44,6 @@ struct mtk_flow_data { } pppoe; }; -struct mtk_flow_entry { - struct rhash_head node; - unsigned long cookie; - u16 hash; -}; - static const struct rhashtable_params mtk_flow_ht_params = { .head_offset = offsetof(struct mtk_flow_entry, node), .key_offset = offsetof(struct mtk_flow_entry, cookie), @@ -48,12 +51,6 @@ static const struct rhashtable_params mtk_flow_ht_params = { .automatic_shrinking = true, }; -static u32 -mtk_eth_timestamp(struct mtk_eth *eth) -{ - return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP; -} - static int mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data, bool egress) @@ -63,6 +60,14 @@ mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data, data->v4.dst_addr, data->dst_port); } +static int +mtk_flow_set_ipv6_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data) +{ + return mtk_foe_entry_set_ipv6_tuple(foe, + data->v6.src_addr.s6_addr32, data->src_port, + data->v6.dst_addr.s6_addr32, data->dst_port); +} + static void mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth) { @@ -80,6 +85,35 @@ mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth) memcpy(dest, src, act->mangle.mask ? 2 : 4); } +static int +mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_info *info) +{ + struct net_device_path_ctx ctx = { + .dev = dev, + .daddr = addr, + }; + struct net_device_path path = {}; + + if (!IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED)) + return -1; + + if (!dev->netdev_ops->ndo_fill_forward_path) + return -1; + + if (dev->netdev_ops->ndo_fill_forward_path(&ctx, &path)) + return -1; + + if (path.type != DEV_PATH_MTK_WDMA) + return -1; + + info->wdma_idx = path.mtk_wdma.wdma_idx; + info->queue = path.mtk_wdma.queue; + info->bss = path.mtk_wdma.bss; + info->wcid = path.mtk_wdma.wcid; + + return 0; +} + static int mtk_flow_mangle_ports(const struct flow_action_entry *act, @@ -149,10 +183,20 @@ mtk_flow_get_dsa_port(struct net_device **dev) static int mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, - struct net_device *dev) + struct net_device *dev, const u8 *dest_mac, + int *wed_index) { + struct mtk_wdma_info info = {}; int pse_port, dsa_port; + if (mtk_flow_get_wdma_info(dev, dest_mac, &info) == 0) { + mtk_foe_entry_set_wdma(foe, info.wdma_idx, info.queue, info.bss, + info.wcid); + pse_port = 3; + *wed_index = info.wdma_idx; + goto out; + } + dsa_port = mtk_flow_get_dsa_port(&dev); if (dsa_port >= 0) mtk_foe_entry_set_dsa(foe, dsa_port); @@ -164,6 +208,7 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, else return -EOPNOTSUPP; +out: mtk_foe_entry_set_pse_port(foe, pse_port); return 0; @@ -179,11 +224,10 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) struct net_device *odev = NULL; struct mtk_flow_entry *entry; int offload_type = 0; + int wed_index = -1; u16 addr_type = 0; - u32 timestamp; u8 l4proto = 0; int err = 0; - int hash; int i; if (rhashtable_lookup(ð->flow_table, &f->cookie, mtk_flow_ht_params)) @@ -215,9 +259,45 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) return -EOPNOTSUPP; } + switch (addr_type) { + case 0: + offload_type = MTK_PPE_PKT_TYPE_BRIDGE; + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs match; + + flow_rule_match_eth_addrs(rule, &match); + memcpy(data.eth.h_dest, match.key->dst, ETH_ALEN); + memcpy(data.eth.h_source, match.key->src, ETH_ALEN); + } else { + return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match; + + flow_rule_match_vlan(rule, &match); + + if (match.key->vlan_tpid != cpu_to_be16(ETH_P_8021Q)) + return -EOPNOTSUPP; + + data.vlan_in = match.key->vlan_id; + } + break; + case FLOW_DISSECTOR_KEY_IPV4_ADDRS: + offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT; + break; + case FLOW_DISSECTOR_KEY_IPV6_ADDRS: + offload_type = MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T; + break; + default: + return -EOPNOTSUPP; + } + flow_action_for_each(i, act, &rule->action) { switch (act->id) { case FLOW_ACTION_MANGLE: + if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE) + return -EOPNOTSUPP; if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH) mtk_flow_offload_mangle_eth(act, &data.eth); break; @@ -249,14 +329,6 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) } } - switch (addr_type) { - case FLOW_DISSECTOR_KEY_IPV4_ADDRS: - offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT; - break; - default: - return -EOPNOTSUPP; - } - if (!is_valid_ether_addr(data.eth.h_source) || !is_valid_ether_addr(data.eth.h_dest)) return -EINVAL; @@ -270,10 +342,13 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { struct flow_match_ports ports; + if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE) + return -EOPNOTSUPP; + flow_rule_match_ports(rule, &ports); data.src_port = ports.key->src; data.dst_port = ports.key->dst; - } else { + } else if (offload_type != MTK_PPE_PKT_TYPE_BRIDGE) { return -EOPNOTSUPP; } @@ -288,10 +363,24 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) mtk_flow_set_ipv4_addr(&foe, &data, false); } + if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { + struct flow_match_ipv6_addrs addrs; + + flow_rule_match_ipv6_addrs(rule, &addrs); + + data.v6.src_addr = addrs.key->src; + data.v6.dst_addr = addrs.key->dst; + + mtk_flow_set_ipv6_addr(&foe, &data); + } + flow_action_for_each(i, act, &rule->action) { if (act->id != FLOW_ACTION_MANGLE) continue; + if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE) + return -EOPNOTSUPP; + switch (act->mangle.htype) { case FLOW_ACT_MANGLE_HDR_TYPE_TCP: case FLOW_ACT_MANGLE_HDR_TYPE_UDP: @@ -317,6 +406,9 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) return err; } + if (offload_type == MTK_PPE_PKT_TYPE_BRIDGE) + foe.bridge.vlan = data.vlan_in; + if (data.vlan.num == 1) { if (data.vlan.proto != htons(ETH_P_8021Q)) return -EOPNOTSUPP; @@ -326,33 +418,38 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) if (data.pppoe.num == 1) mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid); - err = mtk_flow_set_output_device(eth, &foe, odev); + err = mtk_flow_set_output_device(eth, &foe, odev, data.eth.h_dest, + &wed_index); if (err) return err; + if (wed_index >= 0 && (err = mtk_wed_flow_add(wed_index)) < 0) + return err; + entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; entry->cookie = f->cookie; - timestamp = mtk_eth_timestamp(eth); - hash = mtk_foe_entry_commit(ð->ppe, &foe, timestamp); - if (hash < 0) { - err = hash; + memcpy(&entry->data, &foe, sizeof(entry->data)); + entry->wed_index = wed_index; + + if (mtk_foe_entry_commit(eth->ppe, entry) < 0) goto free; - } - entry->hash = hash; err = rhashtable_insert_fast(ð->flow_table, &entry->node, mtk_flow_ht_params); if (err < 0) - goto clear_flow; + goto clear; return 0; -clear_flow: - mtk_foe_entry_clear(ð->ppe, hash); + +clear: + mtk_foe_entry_clear(eth->ppe, entry); free: kfree(entry); + if (wed_index >= 0) + mtk_wed_flow_remove(wed_index); return err; } @@ -366,9 +463,11 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f) if (!entry) return -ENOENT; - mtk_foe_entry_clear(ð->ppe, entry->hash); + mtk_foe_entry_clear(eth->ppe, entry); rhashtable_remove_fast(ð->flow_table, &entry->node, mtk_flow_ht_params); + if (entry->wed_index >= 0) + mtk_wed_flow_remove(entry->wed_index); kfree(entry); return 0; @@ -378,7 +477,6 @@ static int mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f) { struct mtk_flow_entry *entry; - int timestamp; u32 idle; entry = rhashtable_lookup(ð->flow_table, &f->cookie, @@ -386,11 +484,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f) if (!entry) return -ENOENT; - timestamp = mtk_foe_entry_timestamp(ð->ppe, entry->hash); - if (timestamp < 0) - return -ETIMEDOUT; - - idle = mtk_eth_timestamp(eth) - timestamp; + idle = mtk_foe_entry_idle_time(eth->ppe, entry); f->stats.lastused = jiffies - idle * HZ; return 0; @@ -442,7 +536,7 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f) struct flow_block_cb *block_cb; flow_setup_cb_t *cb; - if (!eth->ppe.foe_table) + if (!eth->ppe || !eth->ppe->foe_table) return -EOPNOTSUPP; if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) @@ -483,15 +577,18 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f) int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { - if (type == TC_SETUP_FT) + switch (type) { + case TC_SETUP_BLOCK: + case TC_SETUP_FT: return mtk_eth_setup_tc_block(dev, type_data); - - return -EOPNOTSUPP; + default: + return -EOPNOTSUPP; + } } int mtk_eth_offload_init(struct mtk_eth *eth) { - if (!eth->ppe.foe_table) + if (!eth->ppe || !eth->ppe->foe_table) return 0; return rhashtable_init(ð->flow_table, &mtk_flow_ht_params); diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c new file mode 100644 index 000000000000..f0eacf819cd9 --- /dev/null +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */ + +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/bitfield.h> +#include <linux/dma-mapping.h> +#include <linux/skbuff.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/mfd/syscon.h> +#include <linux/debugfs.h> +#include <linux/soc/mediatek/mtk_wed.h> +#include "mtk_eth_soc.h" +#include "mtk_wed_regs.h" +#include "mtk_wed.h" +#include "mtk_ppe.h" + +#define MTK_PCIE_BASE(n) (0x1a143000 + (n) * 0x2000) + +#define MTK_WED_PKT_SIZE 1900 +#define MTK_WED_BUF_SIZE 2048 +#define MTK_WED_BUF_PER_PAGE (PAGE_SIZE / 2048) + +#define MTK_WED_TX_RING_SIZE 2048 +#define MTK_WED_WDMA_RING_SIZE 1024 + +static struct mtk_wed_hw *hw_list[2]; +static DEFINE_MUTEX(hw_lock); + +static void +wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val) +{ + regmap_update_bits(dev->hw->regs, reg, mask | val, val); +} + +static void +wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask) +{ + return wed_m32(dev, reg, 0, mask); +} + +static void +wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask) +{ + return wed_m32(dev, reg, mask, 0); +} + +static void +wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val) +{ + wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val); +} + +static void +wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask) +{ + wdma_m32(dev, reg, 0, mask); +} + +static u32 +mtk_wed_read_reset(struct mtk_wed_device *dev) +{ + return wed_r32(dev, MTK_WED_RESET); +} + +static void +mtk_wed_reset(struct mtk_wed_device *dev, u32 mask) +{ + u32 status; + + wed_w32(dev, MTK_WED_RESET, mask); + if (readx_poll_timeout(mtk_wed_read_reset, dev, status, + !(status & mask), 0, 1000)) + WARN_ON_ONCE(1); +} + +static struct mtk_wed_hw * +mtk_wed_assign(struct mtk_wed_device *dev) +{ + struct mtk_wed_hw *hw; + + hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)]; + if (!hw || hw->wed_dev) + return NULL; + + hw->wed_dev = dev; + return hw; +} + +static int +mtk_wed_buffer_alloc(struct mtk_wed_device *dev) +{ + struct mtk_wdma_desc *desc; + dma_addr_t desc_phys; + void **page_list; + int token = dev->wlan.token_start; + int ring_size; + int n_pages; + int i, page_idx; + + ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1); + n_pages = ring_size / MTK_WED_BUF_PER_PAGE; + + page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL); + if (!page_list) + return -ENOMEM; + + dev->buf_ring.size = ring_size; + dev->buf_ring.pages = page_list; + + desc = dma_alloc_coherent(dev->hw->dev, ring_size * sizeof(*desc), + &desc_phys, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + dev->buf_ring.desc = desc; + dev->buf_ring.desc_phys = desc_phys; + + for (i = 0, page_idx = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) { + dma_addr_t page_phys, buf_phys; + struct page *page; + void *buf; + int s; + + page = __dev_alloc_pages(GFP_KERNEL, 0); + if (!page) + return -ENOMEM; + + page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev->hw->dev, page_phys)) { + __free_page(page); + return -ENOMEM; + } + + page_list[page_idx++] = page; + dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + buf = page_to_virt(page); + buf_phys = page_phys; + + for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) { + u32 txd_size; + + txd_size = dev->wlan.init_buf(buf, buf_phys, token++); + + desc->buf0 = buf_phys; + desc->buf1 = buf_phys + txd_size; + desc->ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, + txd_size) | + FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1, + MTK_WED_BUF_SIZE - txd_size) | + MTK_WDMA_DESC_CTRL_LAST_SEG1; + desc->info = 0; + desc++; + + buf += MTK_WED_BUF_SIZE; + buf_phys += MTK_WED_BUF_SIZE; + } + + dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE, + DMA_BIDIRECTIONAL); + } + + return 0; +} + +static void +mtk_wed_free_buffer(struct mtk_wed_device *dev) +{ + struct mtk_wdma_desc *desc = dev->buf_ring.desc; + void **page_list = dev->buf_ring.pages; + int page_idx; + int i; + + if (!page_list) + return; + + if (!desc) + goto free_pagelist; + + for (i = 0, page_idx = 0; i < dev->buf_ring.size; i += MTK_WED_BUF_PER_PAGE) { + void *page = page_list[page_idx++]; + + if (!page) + break; + + dma_unmap_page(dev->hw->dev, desc[i].buf0, + PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(page); + } + + dma_free_coherent(dev->hw->dev, dev->buf_ring.size * sizeof(*desc), + desc, dev->buf_ring.desc_phys); + +free_pagelist: + kfree(page_list); +} + +static void +mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring) +{ + if (!ring->desc) + return; + + dma_free_coherent(dev->hw->dev, ring->size * sizeof(*ring->desc), + ring->desc, ring->desc_phys); +} + +static void +mtk_wed_free_tx_rings(struct mtk_wed_device *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) + mtk_wed_free_ring(dev, &dev->tx_ring[i]); + for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) + mtk_wed_free_ring(dev, &dev->tx_wdma[i]); +} + +static void +mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en) +{ + u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK; + + if (!dev->hw->num_flows) + mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; + + wed_w32(dev, MTK_WED_EXT_INT_MASK, en ? mask : 0); + wed_r32(dev, MTK_WED_EXT_INT_MASK); +} + +static void +mtk_wed_stop(struct mtk_wed_device *dev) +{ + regmap_write(dev->hw->mirror, dev->hw->index * 4, 0); + mtk_wed_set_ext_int(dev, false); + + wed_clr(dev, MTK_WED_CTRL, + MTK_WED_CTRL_WDMA_INT_AGENT_EN | + MTK_WED_CTRL_WPDMA_INT_AGENT_EN | + MTK_WED_CTRL_WED_TX_BM_EN | + MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); + wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0); + wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0); + wdma_w32(dev, MTK_WDMA_INT_MASK, 0); + wdma_w32(dev, MTK_WDMA_INT_GRP2, 0); + wed_w32(dev, MTK_WED_WPDMA_INT_MASK, 0); + + wed_clr(dev, MTK_WED_GLO_CFG, + MTK_WED_GLO_CFG_TX_DMA_EN | + MTK_WED_GLO_CFG_RX_DMA_EN); + wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | + MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); + wed_clr(dev, MTK_WED_WDMA_GLO_CFG, + MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); +} + +static void +mtk_wed_detach(struct mtk_wed_device *dev) +{ + struct device_node *wlan_node = dev->wlan.pci_dev->dev.of_node; + struct mtk_wed_hw *hw = dev->hw; + + mutex_lock(&hw_lock); + + mtk_wed_stop(dev); + + wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX); + wdma_w32(dev, MTK_WDMA_RESET_IDX, 0); + + mtk_wed_reset(dev, MTK_WED_RESET_WED); + + mtk_wed_free_buffer(dev); + mtk_wed_free_tx_rings(dev); + + if (of_dma_is_coherent(wlan_node)) + regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, + BIT(hw->index), BIT(hw->index)); + + if (!hw_list[!hw->index]->wed_dev && + hw->eth->dma_dev != hw->eth->dev) + mtk_eth_set_dma_device(hw->eth, hw->eth->dev); + + memset(dev, 0, sizeof(*dev)); + module_put(THIS_MODULE); + + hw->wed_dev = NULL; + mutex_unlock(&hw_lock); +} + +static void +mtk_wed_hw_init_early(struct mtk_wed_device *dev) +{ + u32 mask, set; + u32 offset; + + mtk_wed_stop(dev); + mtk_wed_reset(dev, MTK_WED_RESET_WED); + + mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE | + MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE | + MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE; + set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2) | + MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP | + MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY; + wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set); + + wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_INFO_PRERES); + + offset = dev->hw->index ? 0x04000400 : 0; + wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset); + wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset); + + wed_w32(dev, MTK_WED_PCIE_CFG_BASE, MTK_PCIE_BASE(dev->hw->index)); + wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys); +} + +static void +mtk_wed_hw_init(struct mtk_wed_device *dev) +{ + if (dev->init_done) + return; + + dev->init_done = true; + mtk_wed_set_ext_int(dev, false); + wed_w32(dev, MTK_WED_TX_BM_CTRL, + MTK_WED_TX_BM_CTRL_PAUSE | + FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM, + dev->buf_ring.size / 128) | + FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM, + MTK_WED_TX_RING_SIZE / 256)); + + wed_w32(dev, MTK_WED_TX_BM_BASE, dev->buf_ring.desc_phys); + + wed_w32(dev, MTK_WED_TX_BM_TKID, + FIELD_PREP(MTK_WED_TX_BM_TKID_START, + dev->wlan.token_start) | + FIELD_PREP(MTK_WED_TX_BM_TKID_END, + dev->wlan.token_start + dev->wlan.nbuf - 1)); + + wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE); + + wed_w32(dev, MTK_WED_TX_BM_DYN_THR, + FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) | + MTK_WED_TX_BM_DYN_THR_HI); + + mtk_wed_reset(dev, MTK_WED_RESET_TX_BM); + + wed_set(dev, MTK_WED_CTRL, + MTK_WED_CTRL_WED_TX_BM_EN | + MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); + + wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE); +} + +static void +mtk_wed_ring_reset(struct mtk_wdma_desc *desc, int size) +{ + int i; + + for (i = 0; i < size; i++) { + desc[i].buf0 = 0; + desc[i].ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); + desc[i].buf1 = 0; + desc[i].info = 0; + } +} + +static u32 +mtk_wed_check_busy(struct mtk_wed_device *dev) +{ + if (wed_r32(dev, MTK_WED_GLO_CFG) & MTK_WED_GLO_CFG_TX_DMA_BUSY) + return true; + + if (wed_r32(dev, MTK_WED_WPDMA_GLO_CFG) & + MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY) + return true; + + if (wed_r32(dev, MTK_WED_CTRL) & MTK_WED_CTRL_WDMA_INT_AGENT_BUSY) + return true; + + if (wed_r32(dev, MTK_WED_WDMA_GLO_CFG) & + MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY) + return true; + + if (wdma_r32(dev, MTK_WDMA_GLO_CFG) & + MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY) + return true; + + if (wed_r32(dev, MTK_WED_CTRL) & + (MTK_WED_CTRL_WED_TX_BM_BUSY | MTK_WED_CTRL_WED_TX_FREE_AGENT_BUSY)) + return true; + + return false; +} + +static int +mtk_wed_poll_busy(struct mtk_wed_device *dev) +{ + int sleep = 15000; + int timeout = 100 * sleep; + u32 val; + + return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep, + timeout, false, dev); +} + +static void +mtk_wed_reset_dma(struct mtk_wed_device *dev) +{ + bool busy = false; + u32 val; + int i; + + for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) { + struct mtk_wdma_desc *desc = dev->tx_ring[i].desc; + + if (!desc) + continue; + + mtk_wed_ring_reset(desc, MTK_WED_TX_RING_SIZE); + } + + if (mtk_wed_poll_busy(dev)) + busy = mtk_wed_check_busy(dev); + + if (busy) { + mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA); + } else { + wed_w32(dev, MTK_WED_RESET_IDX, + MTK_WED_RESET_IDX_TX | + MTK_WED_RESET_IDX_RX); + wed_w32(dev, MTK_WED_RESET_IDX, 0); + } + + wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX); + wdma_w32(dev, MTK_WDMA_RESET_IDX, 0); + + if (busy) { + mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT); + mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV); + } else { + wed_w32(dev, MTK_WED_WDMA_RESET_IDX, + MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV); + wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0); + + wed_set(dev, MTK_WED_WDMA_GLO_CFG, + MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE); + + wed_clr(dev, MTK_WED_WDMA_GLO_CFG, + MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE); + } + + for (i = 0; i < 100; i++) { + val = wed_r32(dev, MTK_WED_TX_BM_INTF); + if (FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP, val) == 0x40) + break; + } + + mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT); + mtk_wed_reset(dev, MTK_WED_RESET_TX_BM); + + if (busy) { + mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT); + mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV); + mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV); + } else { + wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, + MTK_WED_WPDMA_RESET_IDX_TX | + MTK_WED_WPDMA_RESET_IDX_RX); + wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0); + } + +} + +static int +mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring, + int size) +{ + ring->desc = dma_alloc_coherent(dev->hw->dev, + size * sizeof(*ring->desc), + &ring->desc_phys, GFP_KERNEL); + if (!ring->desc) + return -ENOMEM; + + ring->size = size; + mtk_wed_ring_reset(ring->desc, size); + + return 0; +} + +static int +mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size) +{ + struct mtk_wed_ring *wdma = &dev->tx_wdma[idx]; + + if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE)) + return -ENOMEM; + + wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE, + wdma->desc_phys); + wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, + size); + wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); + + wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE, + wdma->desc_phys); + wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, + size); + + return 0; +} + +static void +mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask) +{ + u32 wdma_mask; + u32 val; + int i; + + for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) + if (!dev->tx_wdma[i].desc) + mtk_wed_wdma_ring_setup(dev, i, 16); + + wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0)); + + mtk_wed_hw_init(dev); + + wed_set(dev, MTK_WED_CTRL, + MTK_WED_CTRL_WDMA_INT_AGENT_EN | + MTK_WED_CTRL_WPDMA_INT_AGENT_EN | + MTK_WED_CTRL_WED_TX_BM_EN | + MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); + + wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, MTK_WED_PCIE_INT_TRIGGER_STATUS); + + wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, + MTK_WED_WPDMA_INT_TRIGGER_RX_DONE | + MTK_WED_WPDMA_INT_TRIGGER_TX_DONE); + + wed_set(dev, MTK_WED_WPDMA_INT_CTRL, + MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV); + + wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask); + wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask); + + wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask); + wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask); + + wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask); + wed_w32(dev, MTK_WED_INT_MASK, irq_mask); + + wed_set(dev, MTK_WED_GLO_CFG, + MTK_WED_GLO_CFG_TX_DMA_EN | + MTK_WED_GLO_CFG_RX_DMA_EN); + wed_set(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | + MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); + wed_set(dev, MTK_WED_WDMA_GLO_CFG, + MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); + + mtk_wed_set_ext_int(dev, true); + val = dev->wlan.wpdma_phys | + MTK_PCIE_MIRROR_MAP_EN | + FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, dev->hw->index); + + if (dev->hw->index) + val |= BIT(1); + val |= BIT(0); + regmap_write(dev->hw->mirror, dev->hw->index * 4, val); + + dev->running = true; +} + +static int +mtk_wed_attach(struct mtk_wed_device *dev) + __releases(RCU) +{ + struct mtk_wed_hw *hw; + int ret = 0; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "mtk_wed_attach without holding the RCU read lock"); + + if (pci_domain_nr(dev->wlan.pci_dev->bus) > 1 || + !try_module_get(THIS_MODULE)) + ret = -ENODEV; + + rcu_read_unlock(); + + if (ret) + return ret; + + mutex_lock(&hw_lock); + + hw = mtk_wed_assign(dev); + if (!hw) { + module_put(THIS_MODULE); + ret = -ENODEV; + goto out; + } + + dev_info(&dev->wlan.pci_dev->dev, "attaching wed device %d\n", hw->index); + + dev->hw = hw; + dev->dev = hw->dev; + dev->irq = hw->irq; + dev->wdma_idx = hw->index; + + if (hw->eth->dma_dev == hw->eth->dev && + of_dma_is_coherent(hw->eth->dev->of_node)) + mtk_eth_set_dma_device(hw->eth, hw->dev); + + ret = mtk_wed_buffer_alloc(dev); + if (ret) { + mtk_wed_detach(dev); + goto out; + } + + mtk_wed_hw_init_early(dev); + regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, BIT(hw->index), 0); + +out: + mutex_unlock(&hw_lock); + + return ret; +} + +static int +mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs) +{ + struct mtk_wed_ring *ring = &dev->tx_ring[idx]; + + /* + * Tx ring redirection: + * Instead of configuring the WLAN PDMA TX ring directly, the WLAN + * driver allocated DMA ring gets configured into WED MTK_WED_RING_TX(n) + * registers. + * + * WED driver posts its own DMA ring as WLAN PDMA TX and configures it + * into MTK_WED_WPDMA_RING_TX(n) registers. + * It gets filled with packets picked up from WED TX ring and from + * WDMA RX. + */ + + BUG_ON(idx > ARRAY_SIZE(dev->tx_ring)); + + if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE)) + return -ENOMEM; + + if (mtk_wed_wdma_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE)) + return -ENOMEM; + + ring->reg_base = MTK_WED_RING_TX(idx); + ring->wpdma = regs; + + /* WED -> WPDMA */ + wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys); + wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE); + wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0); + + wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE, + ring->desc_phys); + wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT, + MTK_WED_TX_RING_SIZE); + wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0); + + return 0; +} + +static int +mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) +{ + struct mtk_wed_ring *ring = &dev->txfree_ring; + int i; + + /* + * For txfree event handling, the same DMA ring is shared between WED + * and WLAN. The WLAN driver accesses the ring index registers through + * WED + */ + ring->reg_base = MTK_WED_RING_RX(1); + ring->wpdma = regs; + + for (i = 0; i < 12; i += 4) { + u32 val = readl(regs + i); + + wed_w32(dev, MTK_WED_RING_RX(1) + i, val); + wed_w32(dev, MTK_WED_WPDMA_RING_RX(1) + i, val); + } + + return 0; +} + +static u32 +mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask) +{ + u32 val; + + val = wed_r32(dev, MTK_WED_EXT_INT_STATUS); + wed_w32(dev, MTK_WED_EXT_INT_STATUS, val); + val &= MTK_WED_EXT_INT_STATUS_ERROR_MASK; + if (!dev->hw->num_flows) + val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; + if (val && net_ratelimit()) + pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val); + + val = wed_r32(dev, MTK_WED_INT_STATUS); + val &= mask; + wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */ + + return val; +} + +static void +mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask) +{ + if (!dev->running) + return; + + mtk_wed_set_ext_int(dev, !!mask); + wed_w32(dev, MTK_WED_INT_MASK, mask); +} + +int mtk_wed_flow_add(int index) +{ + struct mtk_wed_hw *hw = hw_list[index]; + int ret; + + if (!hw || !hw->wed_dev) + return -ENODEV; + + if (hw->num_flows) { + hw->num_flows++; + return 0; + } + + mutex_lock(&hw_lock); + if (!hw->wed_dev) { + ret = -ENODEV; + goto out; + } + + ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev); + if (!ret) + hw->num_flows++; + mtk_wed_set_ext_int(hw->wed_dev, true); + +out: + mutex_unlock(&hw_lock); + + return ret; +} + +void mtk_wed_flow_remove(int index) +{ + struct mtk_wed_hw *hw = hw_list[index]; + + if (!hw) + return; + + if (--hw->num_flows) + return; + + mutex_lock(&hw_lock); + if (!hw->wed_dev) + goto out; + + hw->wed_dev->wlan.offload_disable(hw->wed_dev); + mtk_wed_set_ext_int(hw->wed_dev, true); + +out: + mutex_unlock(&hw_lock); +} + +void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, + void __iomem *wdma, int index) +{ + static const struct mtk_wed_ops wed_ops = { + .attach = mtk_wed_attach, + .tx_ring_setup = mtk_wed_tx_ring_setup, + .txfree_ring_setup = mtk_wed_txfree_ring_setup, + .start = mtk_wed_start, + .stop = mtk_wed_stop, + .reset_dma = mtk_wed_reset_dma, + .reg_read = wed_r32, + .reg_write = wed_w32, + .irq_get = mtk_wed_irq_get, + .irq_set_mask = mtk_wed_irq_set_mask, + .detach = mtk_wed_detach, + }; + struct device_node *eth_np = eth->dev->of_node; + struct platform_device *pdev; + struct mtk_wed_hw *hw; + struct regmap *regs; + int irq; + + if (!np) + return; + + pdev = of_find_device_by_node(np); + if (!pdev) + return; + + get_device(&pdev->dev); + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return; + + regs = syscon_regmap_lookup_by_phandle(np, NULL); + if (!regs) + return; + + rcu_assign_pointer(mtk_soc_wed_ops, &wed_ops); + + mutex_lock(&hw_lock); + + if (WARN_ON(hw_list[index])) + goto unlock; + + hw = kzalloc(sizeof(*hw), GFP_KERNEL); + hw->node = np; + hw->regs = regs; + hw->eth = eth; + hw->dev = &pdev->dev; + hw->wdma = wdma; + hw->index = index; + hw->irq = irq; + hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, + "mediatek,pcie-mirror"); + hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np, + "mediatek,hifsys"); + if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) { + kfree(hw); + goto unlock; + } + + if (!index) { + regmap_write(hw->mirror, 0, 0); + regmap_write(hw->mirror, 4, 0); + } + mtk_wed_hw_add_debugfs(hw); + + hw_list[index] = hw; + +unlock: + mutex_unlock(&hw_lock); +} + +void mtk_wed_exit(void) +{ + int i; + + rcu_assign_pointer(mtk_soc_wed_ops, NULL); + + synchronize_rcu(); + + for (i = 0; i < ARRAY_SIZE(hw_list); i++) { + struct mtk_wed_hw *hw; + + hw = hw_list[i]; + if (!hw) + continue; + + hw_list[i] = NULL; + debugfs_remove(hw->debugfs_dir); + put_device(hw->dev); + kfree(hw); + } +} diff --git a/drivers/net/ethernet/mediatek/mtk_wed.h b/drivers/net/ethernet/mediatek/mtk_wed.h new file mode 100644 index 000000000000..981ec613f4b0 --- /dev/null +++ b/drivers/net/ethernet/mediatek/mtk_wed.h @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */ + +#ifndef __MTK_WED_PRIV_H +#define __MTK_WED_PRIV_H + +#include <linux/soc/mediatek/mtk_wed.h> +#include <linux/debugfs.h> +#include <linux/regmap.h> +#include <linux/netdevice.h> + +struct mtk_eth; + +struct mtk_wed_hw { + struct device_node *node; + struct mtk_eth *eth; + struct regmap *regs; + struct regmap *hifsys; + struct device *dev; + void __iomem *wdma; + struct regmap *mirror; + struct dentry *debugfs_dir; + struct mtk_wed_device *wed_dev; + u32 debugfs_reg; + u32 num_flows; + char dirname[5]; + int irq; + int index; +}; + +struct mtk_wdma_info { + u8 wdma_idx; + u8 queue; + u16 wcid; + u8 bss; +}; + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +static inline void +wed_w32(struct mtk_wed_device *dev, u32 reg, u32 val) +{ + regmap_write(dev->hw->regs, reg, val); +} + +static inline u32 +wed_r32(struct mtk_wed_device *dev, u32 reg) +{ + unsigned int val; + + regmap_read(dev->hw->regs, reg, &val); + + return val; +} + +static inline void +wdma_w32(struct mtk_wed_device *dev, u32 reg, u32 val) +{ + writel(val, dev->hw->wdma + reg); +} + +static inline u32 +wdma_r32(struct mtk_wed_device *dev, u32 reg) +{ + return readl(dev->hw->wdma + reg); +} + +static inline u32 +wpdma_tx_r32(struct mtk_wed_device *dev, int ring, u32 reg) +{ + if (!dev->tx_ring[ring].wpdma) + return 0; + + return readl(dev->tx_ring[ring].wpdma + reg); +} + +static inline void +wpdma_tx_w32(struct mtk_wed_device *dev, int ring, u32 reg, u32 val) +{ + if (!dev->tx_ring[ring].wpdma) + return; + + writel(val, dev->tx_ring[ring].wpdma + reg); +} + +static inline u32 +wpdma_txfree_r32(struct mtk_wed_device *dev, u32 reg) +{ + if (!dev->txfree_ring.wpdma) + return 0; + + return readl(dev->txfree_ring.wpdma + reg); +} + +static inline void +wpdma_txfree_w32(struct mtk_wed_device *dev, u32 reg, u32 val) +{ + if (!dev->txfree_ring.wpdma) + return; + + writel(val, dev->txfree_ring.wpdma + reg); +} + +void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, + void __iomem *wdma, int index); +void mtk_wed_exit(void); +int mtk_wed_flow_add(int index); +void mtk_wed_flow_remove(int index); +#else +static inline void +mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, + void __iomem *wdma, int index) +{ +} +static inline void +mtk_wed_exit(void) +{ +} +static inline int mtk_wed_flow_add(int index) +{ + return -EINVAL; +} +static inline void mtk_wed_flow_remove(int index) +{ +} +#endif + +#ifdef CONFIG_DEBUG_FS +void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw); +#else +static inline void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw) +{ +} +#endif + +#endif diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c new file mode 100644 index 000000000000..a81d3fd1a439 --- /dev/null +++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */ + +#include <linux/seq_file.h> +#include "mtk_wed.h" +#include "mtk_wed_regs.h" + +struct reg_dump { + const char *name; + u16 offset; + u8 type; + u8 base; +}; + +enum { + DUMP_TYPE_STRING, + DUMP_TYPE_WED, + DUMP_TYPE_WDMA, + DUMP_TYPE_WPDMA_TX, + DUMP_TYPE_WPDMA_TXFREE, +}; + +#define DUMP_STR(_str) { _str, 0, DUMP_TYPE_STRING } +#define DUMP_REG(_reg, ...) { #_reg, MTK_##_reg, __VA_ARGS__ } +#define DUMP_RING(_prefix, _base, ...) \ + { _prefix " BASE", _base, __VA_ARGS__ }, \ + { _prefix " CNT", _base + 0x4, __VA_ARGS__ }, \ + { _prefix " CIDX", _base + 0x8, __VA_ARGS__ }, \ + { _prefix " DIDX", _base + 0xc, __VA_ARGS__ } + +#define DUMP_WED(_reg) DUMP_REG(_reg, DUMP_TYPE_WED) +#define DUMP_WED_RING(_base) DUMP_RING(#_base, MTK_##_base, DUMP_TYPE_WED) + +#define DUMP_WDMA(_reg) DUMP_REG(_reg, DUMP_TYPE_WDMA) +#define DUMP_WDMA_RING(_base) DUMP_RING(#_base, MTK_##_base, DUMP_TYPE_WDMA) + +#define DUMP_WPDMA_TX_RING(_n) DUMP_RING("WPDMA_TX" #_n, 0, DUMP_TYPE_WPDMA_TX, _n) +#define DUMP_WPDMA_TXFREE_RING DUMP_RING("WPDMA_RX1", 0, DUMP_TYPE_WPDMA_TXFREE) + +static void +print_reg_val(struct seq_file *s, const char *name, u32 val) +{ + seq_printf(s, "%-32s %08x\n", name, val); +} + +static void +dump_wed_regs(struct seq_file *s, struct mtk_wed_device *dev, + const struct reg_dump *regs, int n_regs) +{ + const struct reg_dump *cur; + u32 val; + + for (cur = regs; cur < ®s[n_regs]; cur++) { + switch (cur->type) { + case DUMP_TYPE_STRING: + seq_printf(s, "%s======== %s:\n", + cur > regs ? "\n" : "", + cur->name); + continue; + case DUMP_TYPE_WED: + val = wed_r32(dev, cur->offset); + break; + case DUMP_TYPE_WDMA: + val = wdma_r32(dev, cur->offset); + break; + case DUMP_TYPE_WPDMA_TX: + val = wpdma_tx_r32(dev, cur->base, cur->offset); + break; + case DUMP_TYPE_WPDMA_TXFREE: + val = wpdma_txfree_r32(dev, cur->offset); + break; + } + print_reg_val(s, cur->name, val); + } +} + + +static int +wed_txinfo_show(struct seq_file *s, void *data) +{ + static const struct reg_dump regs[] = { + DUMP_STR("WED TX"), + DUMP_WED(WED_TX_MIB(0)), + DUMP_WED_RING(WED_RING_TX(0)), + + DUMP_WED(WED_TX_MIB(1)), + DUMP_WED_RING(WED_RING_TX(1)), + + DUMP_STR("WPDMA TX"), + DUMP_WED(WED_WPDMA_TX_MIB(0)), + DUMP_WED_RING(WED_WPDMA_RING_TX(0)), + DUMP_WED(WED_WPDMA_TX_COHERENT_MIB(0)), + + DUMP_WED(WED_WPDMA_TX_MIB(1)), + DUMP_WED_RING(WED_WPDMA_RING_TX(1)), + DUMP_WED(WED_WPDMA_TX_COHERENT_MIB(1)), + + DUMP_STR("WPDMA TX"), + DUMP_WPDMA_TX_RING(0), + DUMP_WPDMA_TX_RING(1), + + DUMP_STR("WED WDMA RX"), + DUMP_WED(WED_WDMA_RX_MIB(0)), + DUMP_WED_RING(WED_WDMA_RING_RX(0)), + DUMP_WED(WED_WDMA_RX_THRES(0)), + DUMP_WED(WED_WDMA_RX_RECYCLE_MIB(0)), + DUMP_WED(WED_WDMA_RX_PROCESSED_MIB(0)), + + DUMP_WED(WED_WDMA_RX_MIB(1)), + DUMP_WED_RING(WED_WDMA_RING_RX(1)), + DUMP_WED(WED_WDMA_RX_THRES(1)), + DUMP_WED(WED_WDMA_RX_RECYCLE_MIB(1)), + DUMP_WED(WED_WDMA_RX_PROCESSED_MIB(1)), + + DUMP_STR("WDMA RX"), + DUMP_WDMA(WDMA_GLO_CFG), + DUMP_WDMA_RING(WDMA_RING_RX(0)), + DUMP_WDMA_RING(WDMA_RING_RX(1)), + }; + struct mtk_wed_hw *hw = s->private; + struct mtk_wed_device *dev = hw->wed_dev; + + if (!dev) + return 0; + + dump_wed_regs(s, dev, regs, ARRAY_SIZE(regs)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(wed_txinfo); + + +static int +mtk_wed_reg_set(void *data, u64 val) +{ + struct mtk_wed_hw *hw = data; + + regmap_write(hw->regs, hw->debugfs_reg, val); + + return 0; +} + +static int +mtk_wed_reg_get(void *data, u64 *val) +{ + struct mtk_wed_hw *hw = data; + unsigned int regval; + int ret; + + ret = regmap_read(hw->regs, hw->debugfs_reg, ®val); + if (ret) + return ret; + + *val = regval; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_regval, mtk_wed_reg_get, mtk_wed_reg_set, + "0x%08llx\n"); + +void mtk_wed_hw_add_debugfs(struct mtk_wed_hw *hw) +{ + struct dentry *dir; + + snprintf(hw->dirname, sizeof(hw->dirname), "wed%d", hw->index); + dir = debugfs_create_dir(hw->dirname, NULL); + if (!dir) + return; + + hw->debugfs_dir = dir; + debugfs_create_u32("regidx", 0600, dir, &hw->debugfs_reg); + debugfs_create_file_unsafe("regval", 0600, dir, hw, &fops_regval); + debugfs_create_file_unsafe("txinfo", 0400, dir, hw, &wed_txinfo_fops); +} diff --git a/drivers/net/ethernet/mediatek/mtk_wed_ops.c b/drivers/net/ethernet/mediatek/mtk_wed_ops.c new file mode 100644 index 000000000000..a5d9d8a5bce2 --- /dev/null +++ b/drivers/net/ethernet/mediatek/mtk_wed_ops.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ + +#include <linux/kernel.h> +#include <linux/soc/mediatek/mtk_wed.h> + +const struct mtk_wed_ops __rcu *mtk_soc_wed_ops; +EXPORT_SYMBOL_GPL(mtk_soc_wed_ops); diff --git a/drivers/net/ethernet/mediatek/mtk_wed_regs.h b/drivers/net/ethernet/mediatek/mtk_wed_regs.h new file mode 100644 index 000000000000..0a0465ea58b4 --- /dev/null +++ b/drivers/net/ethernet/mediatek/mtk_wed_regs.h @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ + +#ifndef __MTK_WED_REGS_H +#define __MTK_WED_REGS_H + +#define MTK_WDMA_DESC_CTRL_LEN1 GENMASK(14, 0) +#define MTK_WDMA_DESC_CTRL_LAST_SEG1 BIT(15) +#define MTK_WDMA_DESC_CTRL_BURST BIT(16) +#define MTK_WDMA_DESC_CTRL_LEN0 GENMASK(29, 16) +#define MTK_WDMA_DESC_CTRL_LAST_SEG0 BIT(30) +#define MTK_WDMA_DESC_CTRL_DMA_DONE BIT(31) + +struct mtk_wdma_desc { + __le32 buf0; + __le32 ctrl; + __le32 buf1; + __le32 info; +} __packed __aligned(4); + +#define MTK_WED_RESET 0x008 +#define MTK_WED_RESET_TX_BM BIT(0) +#define MTK_WED_RESET_TX_FREE_AGENT BIT(4) +#define MTK_WED_RESET_WPDMA_TX_DRV BIT(8) +#define MTK_WED_RESET_WPDMA_RX_DRV BIT(9) +#define MTK_WED_RESET_WPDMA_INT_AGENT BIT(11) +#define MTK_WED_RESET_WED_TX_DMA BIT(12) +#define MTK_WED_RESET_WDMA_RX_DRV BIT(17) +#define MTK_WED_RESET_WDMA_INT_AGENT BIT(19) +#define MTK_WED_RESET_WED BIT(31) + +#define MTK_WED_CTRL 0x00c +#define MTK_WED_CTRL_WPDMA_INT_AGENT_EN BIT(0) +#define MTK_WED_CTRL_WPDMA_INT_AGENT_BUSY BIT(1) +#define MTK_WED_CTRL_WDMA_INT_AGENT_EN BIT(2) +#define MTK_WED_CTRL_WDMA_INT_AGENT_BUSY BIT(3) +#define MTK_WED_CTRL_WED_TX_BM_EN BIT(8) +#define MTK_WED_CTRL_WED_TX_BM_BUSY BIT(9) +#define MTK_WED_CTRL_WED_TX_FREE_AGENT_EN BIT(10) +#define MTK_WED_CTRL_WED_TX_FREE_AGENT_BUSY BIT(11) +#define MTK_WED_CTRL_RESERVE_EN BIT(12) +#define MTK_WED_CTRL_RESERVE_BUSY BIT(13) +#define MTK_WED_CTRL_FINAL_DIDX_READ BIT(24) +#define MTK_WED_CTRL_MIB_READ_CLEAR BIT(28) + +#define MTK_WED_EXT_INT_STATUS 0x020 +#define MTK_WED_EXT_INT_STATUS_TF_LEN_ERR BIT(0) +#define MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD BIT(1) +#define MTK_WED_EXT_INT_STATUS_TKID_TITO_INVALID BIT(4) +#define MTK_WED_EXT_INT_STATUS_TX_FBUF_LO_TH BIT(8) +#define MTK_WED_EXT_INT_STATUS_TX_FBUF_HI_TH BIT(9) +#define MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH BIT(12) +#define MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH BIT(13) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR BIT(16) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR BIT(17) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT BIT(18) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN BIT(19) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_BM_DMAD_COHERENT BIT(20) +#define MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR BIT(21) +#define MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR BIT(22) +#define MTK_WED_EXT_INT_STATUS_RX_DRV_DMA_RECYCLE BIT(24) +#define MTK_WED_EXT_INT_STATUS_ERROR_MASK (MTK_WED_EXT_INT_STATUS_TF_LEN_ERR | \ + MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD | \ + MTK_WED_EXT_INT_STATUS_TKID_TITO_INVALID | \ + MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR | \ + MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR | \ + MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN | \ + MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR | \ + MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR) + +#define MTK_WED_EXT_INT_MASK 0x028 + +#define MTK_WED_STATUS 0x060 +#define MTK_WED_STATUS_TX GENMASK(15, 8) + +#define MTK_WED_TX_BM_CTRL 0x080 +#define MTK_WED_TX_BM_CTRL_VLD_GRP_NUM GENMASK(6, 0) +#define MTK_WED_TX_BM_CTRL_RSV_GRP_NUM GENMASK(22, 16) +#define MTK_WED_TX_BM_CTRL_PAUSE BIT(28) + +#define MTK_WED_TX_BM_BASE 0x084 + +#define MTK_WED_TX_BM_TKID 0x088 +#define MTK_WED_TX_BM_TKID_START GENMASK(15, 0) +#define MTK_WED_TX_BM_TKID_END GENMASK(31, 16) + +#define MTK_WED_TX_BM_BUF_LEN 0x08c + +#define MTK_WED_TX_BM_INTF 0x09c +#define MTK_WED_TX_BM_INTF_TKID GENMASK(15, 0) +#define MTK_WED_TX_BM_INTF_TKFIFO_FDEP GENMASK(23, 16) +#define MTK_WED_TX_BM_INTF_TKID_VALID BIT(28) +#define MTK_WED_TX_BM_INTF_TKID_READ BIT(29) + +#define MTK_WED_TX_BM_DYN_THR 0x0a0 +#define MTK_WED_TX_BM_DYN_THR_LO GENMASK(6, 0) +#define MTK_WED_TX_BM_DYN_THR_HI GENMASK(22, 16) + +#define MTK_WED_INT_STATUS 0x200 +#define MTK_WED_INT_MASK 0x204 + +#define MTK_WED_GLO_CFG 0x208 +#define MTK_WED_GLO_CFG_TX_DMA_EN BIT(0) +#define MTK_WED_GLO_CFG_TX_DMA_BUSY BIT(1) +#define MTK_WED_GLO_CFG_RX_DMA_EN BIT(2) +#define MTK_WED_GLO_CFG_RX_DMA_BUSY BIT(3) +#define MTK_WED_GLO_CFG_RX_BT_SIZE GENMASK(5, 4) +#define MTK_WED_GLO_CFG_TX_WB_DDONE BIT(6) +#define MTK_WED_GLO_CFG_BIG_ENDIAN BIT(7) +#define MTK_WED_GLO_CFG_DIS_BT_SIZE_ALIGN BIT(8) +#define MTK_WED_GLO_CFG_TX_BT_SIZE_LO BIT(9) +#define MTK_WED_GLO_CFG_MULTI_DMA_EN GENMASK(11, 10) +#define MTK_WED_GLO_CFG_FIFO_LITTLE_ENDIAN BIT(12) +#define MTK_WED_GLO_CFG_MI_DEPTH_RD GENMASK(21, 13) +#define MTK_WED_GLO_CFG_TX_BT_SIZE_HI GENMASK(23, 22) +#define MTK_WED_GLO_CFG_SW_RESET BIT(24) +#define MTK_WED_GLO_CFG_FIRST_TOKEN_ONLY BIT(26) +#define MTK_WED_GLO_CFG_OMIT_RX_INFO BIT(27) +#define MTK_WED_GLO_CFG_OMIT_TX_INFO BIT(28) +#define MTK_WED_GLO_CFG_BYTE_SWAP BIT(29) +#define MTK_WED_GLO_CFG_RX_2B_OFFSET BIT(31) + +#define MTK_WED_RESET_IDX 0x20c +#define MTK_WED_RESET_IDX_TX GENMASK(3, 0) +#define MTK_WED_RESET_IDX_RX GENMASK(17, 16) + +#define MTK_WED_TX_MIB(_n) (0x2a0 + (_n) * 4) + +#define MTK_WED_RING_TX(_n) (0x300 + (_n) * 0x10) + +#define MTK_WED_RING_RX(_n) (0x400 + (_n) * 0x10) + +#define MTK_WED_WPDMA_INT_TRIGGER 0x504 +#define MTK_WED_WPDMA_INT_TRIGGER_RX_DONE BIT(1) +#define MTK_WED_WPDMA_INT_TRIGGER_TX_DONE GENMASK(5, 4) + +#define MTK_WED_WPDMA_GLO_CFG 0x508 +#define MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN BIT(0) +#define MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY BIT(1) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN BIT(2) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY BIT(3) +#define MTK_WED_WPDMA_GLO_CFG_RX_BT_SIZE GENMASK(5, 4) +#define MTK_WED_WPDMA_GLO_CFG_TX_WB_DDONE BIT(6) +#define MTK_WED_WPDMA_GLO_CFG_BIG_ENDIAN BIT(7) +#define MTK_WED_WPDMA_GLO_CFG_DIS_BT_SIZE_ALIGN BIT(8) +#define MTK_WED_WPDMA_GLO_CFG_TX_BT_SIZE_LO BIT(9) +#define MTK_WED_WPDMA_GLO_CFG_MULTI_DMA_EN GENMASK(11, 10) +#define MTK_WED_WPDMA_GLO_CFG_FIFO_LITTLE_ENDIAN BIT(12) +#define MTK_WED_WPDMA_GLO_CFG_MI_DEPTH_RD GENMASK(21, 13) +#define MTK_WED_WPDMA_GLO_CFG_TX_BT_SIZE_HI GENMASK(23, 22) +#define MTK_WED_WPDMA_GLO_CFG_SW_RESET BIT(24) +#define MTK_WED_WPDMA_GLO_CFG_FIRST_TOKEN_ONLY BIT(26) +#define MTK_WED_WPDMA_GLO_CFG_OMIT_RX_INFO BIT(27) +#define MTK_WED_WPDMA_GLO_CFG_OMIT_TX_INFO BIT(28) +#define MTK_WED_WPDMA_GLO_CFG_BYTE_SWAP BIT(29) +#define MTK_WED_WPDMA_GLO_CFG_RX_2B_OFFSET BIT(31) + +#define MTK_WED_WPDMA_RESET_IDX 0x50c +#define MTK_WED_WPDMA_RESET_IDX_TX GENMASK(3, 0) +#define MTK_WED_WPDMA_RESET_IDX_RX GENMASK(17, 16) + +#define MTK_WED_WPDMA_INT_CTRL 0x520 +#define MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV BIT(21) + +#define MTK_WED_WPDMA_INT_MASK 0x524 + +#define MTK_WED_PCIE_CFG_BASE 0x560 + +#define MTK_WED_PCIE_INT_TRIGGER 0x570 +#define MTK_WED_PCIE_INT_TRIGGER_STATUS BIT(16) + +#define MTK_WED_WPDMA_CFG_BASE 0x580 + +#define MTK_WED_WPDMA_TX_MIB(_n) (0x5a0 + (_n) * 4) +#define MTK_WED_WPDMA_TX_COHERENT_MIB(_n) (0x5d0 + (_n) * 4) + +#define MTK_WED_WPDMA_RING_TX(_n) (0x600 + (_n) * 0x10) +#define MTK_WED_WPDMA_RING_RX(_n) (0x700 + (_n) * 0x10) +#define MTK_WED_WDMA_RING_RX(_n) (0x900 + (_n) * 0x10) +#define MTK_WED_WDMA_RX_THRES(_n) (0x940 + (_n) * 0x4) + +#define MTK_WED_WDMA_GLO_CFG 0xa04 +#define MTK_WED_WDMA_GLO_CFG_TX_DRV_EN BIT(0) +#define MTK_WED_WDMA_GLO_CFG_RX_DRV_EN BIT(2) +#define MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY BIT(3) +#define MTK_WED_WDMA_GLO_CFG_BT_SIZE GENMASK(5, 4) +#define MTK_WED_WDMA_GLO_CFG_TX_WB_DDONE BIT(6) +#define MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE BIT(13) +#define MTK_WED_WDMA_GLO_CFG_WCOMPLETE_SEL BIT(16) +#define MTK_WED_WDMA_GLO_CFG_INIT_PHASE_RXDMA_BYPASS BIT(17) +#define MTK_WED_WDMA_GLO_CFG_INIT_PHASE_BYPASS BIT(18) +#define MTK_WED_WDMA_GLO_CFG_FSM_RETURN_IDLE BIT(19) +#define MTK_WED_WDMA_GLO_CFG_WAIT_COHERENT BIT(20) +#define MTK_WED_WDMA_GLO_CFG_AXI_W_AFTER_AW BIT(21) +#define MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY_SINGLE_W BIT(22) +#define MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY BIT(23) +#define MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP BIT(24) +#define MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE BIT(25) +#define MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE BIT(26) +#define MTK_WED_WDMA_GLO_CFG_RXDRV_CLKGATE_BYPASS BIT(30) + +#define MTK_WED_WDMA_RESET_IDX 0xa08 +#define MTK_WED_WDMA_RESET_IDX_RX GENMASK(17, 16) +#define MTK_WED_WDMA_RESET_IDX_DRV GENMASK(25, 24) + +#define MTK_WED_WDMA_INT_TRIGGER 0xa28 +#define MTK_WED_WDMA_INT_TRIGGER_RX_DONE GENMASK(17, 16) + +#define MTK_WED_WDMA_INT_CTRL 0xa2c +#define MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL GENMASK(17, 16) + +#define MTK_WED_WDMA_OFFSET0 0xaa4 +#define MTK_WED_WDMA_OFFSET1 0xaa8 + +#define MTK_WED_WDMA_RX_MIB(_n) (0xae0 + (_n) * 4) +#define MTK_WED_WDMA_RX_RECYCLE_MIB(_n) (0xae8 + (_n) * 4) +#define MTK_WED_WDMA_RX_PROCESSED_MIB(_n) (0xaf0 + (_n) * 4) + +#define MTK_WED_RING_OFS_BASE 0x00 +#define MTK_WED_RING_OFS_COUNT 0x04 +#define MTK_WED_RING_OFS_CPU_IDX 0x08 +#define MTK_WED_RING_OFS_DMA_IDX 0x0c + +#define MTK_WDMA_RING_RX(_n) (0x100 + (_n) * 0x10) + +#define MTK_WDMA_GLO_CFG 0x204 +#define MTK_WDMA_GLO_CFG_RX_INFO_PRERES GENMASK(28, 26) + +#define MTK_WDMA_RESET_IDX 0x208 +#define MTK_WDMA_RESET_IDX_TX GENMASK(3, 0) +#define MTK_WDMA_RESET_IDX_RX GENMASK(17, 16) + +#define MTK_WDMA_INT_MASK 0x228 +#define MTK_WDMA_INT_MASK_TX_DONE GENMASK(3, 0) +#define MTK_WDMA_INT_MASK_RX_DONE GENMASK(17, 16) +#define MTK_WDMA_INT_MASK_TX_DELAY BIT(28) +#define MTK_WDMA_INT_MASK_TX_COHERENT BIT(29) +#define MTK_WDMA_INT_MASK_RX_DELAY BIT(30) +#define MTK_WDMA_INT_MASK_RX_COHERENT BIT(31) + +#define MTK_WDMA_INT_GRP1 0x250 +#define MTK_WDMA_INT_GRP2 0x254 + +#define MTK_PCIE_MIRROR_MAP(n) ((n) ? 0x4 : 0x0) +#define MTK_PCIE_MIRROR_MAP_EN BIT(0) +#define MTK_PCIE_MIRROR_MAP_WED_ID BIT(1) + +/* DMA channel mapping */ +#define HIFSYS_DMA_AG_MAP 0x008 + +#endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 79deb19e3a19..79fd486e29e3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -7010,7 +7010,7 @@ mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); if (IS_ERR(mlxsw_sp_rt6)) { err = PTR_ERR(mlxsw_sp_rt6); - goto err_rt6_create; + goto err_rt6_unwind; } list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); @@ -7019,14 +7019,12 @@ mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); if (err) - goto err_nexthop6_group_update; + goto err_rt6_unwind; return 0; -err_nexthop6_group_update: - i = nrt6; -err_rt6_create: - for (i--; i >= 0; i--) { +err_rt6_unwind: + for (; i > 0; i--) { fib6_entry->nrt6--; mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, list); @@ -7154,7 +7152,7 @@ mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); if (IS_ERR(mlxsw_sp_rt6)) { err = PTR_ERR(mlxsw_sp_rt6); - goto err_rt6_create; + goto err_rt6_unwind; } list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); fib6_entry->nrt6++; @@ -7162,7 +7160,7 @@ mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); if (err) - goto err_nexthop6_group_get; + goto err_rt6_unwind; err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, fib_node->fib); @@ -7181,10 +7179,8 @@ err_fib6_entry_type_set: mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); err_nexthop_group_vr_link: mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); -err_nexthop6_group_get: - i = nrt6; -err_rt6_create: - for (i--; i >= 0; i--) { +err_rt6_unwind: + for (; i > 0; i--) { fib6_entry->nrt6--; mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, list); diff --git a/drivers/net/ethernet/micrel/Kconfig b/drivers/net/ethernet/micrel/Kconfig index 93df3049cdc0..830363bafcce 100644 --- a/drivers/net/ethernet/micrel/Kconfig +++ b/drivers/net/ethernet/micrel/Kconfig @@ -28,6 +28,7 @@ config KS8842 config KS8851 tristate "Micrel KS8851 SPI" depends on SPI + depends on PTP_1588_CLOCK_OPTIONAL select MII select CRC32 select EEPROM_93CX6 @@ -39,6 +40,7 @@ config KS8851 config KS8851_MLL tristate "Micrel KS8851 MLL" depends on HAS_IOMEM + depends on PTP_1588_CLOCK_OPTIONAL select MII select CRC32 select EEPROM_93CX6 diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 50ac3ee2577a..21d2645885ce 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -2903,11 +2903,9 @@ static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, status = myri10ge_xmit(curr, dev); if (status != 0) { dev_kfree_skb_any(curr); - if (segs != NULL) { - curr = segs; - segs = next; + skb_list_walk_safe(next, curr, next) { curr->next = NULL; - dev_kfree_skb_any(segs); + dev_kfree_skb_any(curr); } goto drop; } diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c index e3edca187ddf..5250d1d1e49c 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_debug.c +++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c @@ -489,7 +489,7 @@ struct split_type_defs { #define STATIC_DEBUG_LINE_DWORDS 9 -#define NUM_COMMON_GLOBAL_PARAMS 11 +#define NUM_COMMON_GLOBAL_PARAMS 10 #define MAX_RECURSION_DEPTH 10 diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c index 96a2077fd315..7e286cddbedb 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c @@ -161,11 +161,11 @@ EXPORT_SYMBOL(qed_vlan_get_ndev); struct pci_dev *qed_validate_ndev(struct net_device *ndev) { - struct pci_dev *pdev = NULL; struct net_device *upper; + struct pci_dev *pdev; for_each_pci_dev(pdev) { - if (pdev && pdev->driver && + if (pdev->driver && !strcmp(pdev->driver->name, "qede")) { upper = pci_get_drvdata(pdev); if (upper->ifindex == ndev->ifindex) diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index b242000a77fd..b7cc36589f59 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -748,6 +748,9 @@ qede_build_skb(struct qede_rx_queue *rxq, buf = page_address(bd->data) + bd->page_offset; skb = build_skb(buf, rxq->rx_buf_seg_size); + if (unlikely(!skb)) + return NULL; + skb_reserve(skb, pad); skb_put(skb, len); diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index f9064532beb6..377df8b7f015 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -786,6 +786,85 @@ void efx_remove_channels(struct efx_nic *efx) kfree(efx->xdp_tx_queues); } +static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number, + struct efx_tx_queue *tx_queue) +{ + if (xdp_queue_number >= efx->xdp_tx_queue_count) + return -EINVAL; + + netif_dbg(efx, drv, efx->net_dev, + "Channel %u TXQ %u is XDP %u, HW %u\n", + tx_queue->channel->channel, tx_queue->label, + xdp_queue_number, tx_queue->queue); + efx->xdp_tx_queues[xdp_queue_number] = tx_queue; + return 0; +} + +static void efx_set_xdp_channels(struct efx_nic *efx) +{ + struct efx_tx_queue *tx_queue; + struct efx_channel *channel; + unsigned int next_queue = 0; + int xdp_queue_number = 0; + int rc; + + /* We need to mark which channels really have RX and TX + * queues, and adjust the TX queue numbers if we have separate + * RX-only and TX-only channels. + */ + efx_for_each_channel(channel, efx) { + if (channel->channel < efx->tx_channel_offset) + continue; + + if (efx_channel_is_xdp_tx(channel)) { + efx_for_each_channel_tx_queue(tx_queue, channel) { + tx_queue->queue = next_queue++; + rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, + tx_queue); + if (rc == 0) + xdp_queue_number++; + } + } else { + efx_for_each_channel_tx_queue(tx_queue, channel) { + tx_queue->queue = next_queue++; + netif_dbg(efx, drv, efx->net_dev, + "Channel %u TXQ %u is HW %u\n", + channel->channel, tx_queue->label, + tx_queue->queue); + } + + /* If XDP is borrowing queues from net stack, it must + * use the queue with no csum offload, which is the + * first one of the channel + * (note: tx_queue_by_type is not initialized yet) + */ + if (efx->xdp_txq_queues_mode == + EFX_XDP_TX_QUEUES_BORROWED) { + tx_queue = &channel->tx_queue[0]; + rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, + tx_queue); + if (rc == 0) + xdp_queue_number++; + } + } + } + WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED && + xdp_queue_number != efx->xdp_tx_queue_count); + WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED && + xdp_queue_number > efx->xdp_tx_queue_count); + + /* If we have more CPUs than assigned XDP TX queues, assign the already + * existing queues to the exceeding CPUs + */ + next_queue = 0; + while (xdp_queue_number < efx->xdp_tx_queue_count) { + tx_queue = efx->xdp_tx_queues[next_queue++]; + rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); + if (rc == 0) + xdp_queue_number++; + } +} + int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) { struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; @@ -857,6 +936,7 @@ int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) efx_init_napi_channel(efx->channel[i]); } + efx_set_xdp_channels(efx); out: /* Destroy unused channel structures */ for (i = 0; i < efx->n_channels; i++) { @@ -889,26 +969,9 @@ rollback: goto out; } -static inline int -efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number, - struct efx_tx_queue *tx_queue) -{ - if (xdp_queue_number >= efx->xdp_tx_queue_count) - return -EINVAL; - - netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n", - tx_queue->channel->channel, tx_queue->label, - xdp_queue_number, tx_queue->queue); - efx->xdp_tx_queues[xdp_queue_number] = tx_queue; - return 0; -} - int efx_set_channels(struct efx_nic *efx) { - struct efx_tx_queue *tx_queue; struct efx_channel *channel; - unsigned int next_queue = 0; - int xdp_queue_number; int rc; efx->tx_channel_offset = @@ -926,61 +989,14 @@ int efx_set_channels(struct efx_nic *efx) return -ENOMEM; } - /* We need to mark which channels really have RX and TX - * queues, and adjust the TX queue numbers if we have separate - * RX-only and TX-only channels. - */ - xdp_queue_number = 0; efx_for_each_channel(channel, efx) { if (channel->channel < efx->n_rx_channels) channel->rx_queue.core_index = channel->channel; else channel->rx_queue.core_index = -1; - - if (channel->channel >= efx->tx_channel_offset) { - if (efx_channel_is_xdp_tx(channel)) { - efx_for_each_channel_tx_queue(tx_queue, channel) { - tx_queue->queue = next_queue++; - rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); - if (rc == 0) - xdp_queue_number++; - } - } else { - efx_for_each_channel_tx_queue(tx_queue, channel) { - tx_queue->queue = next_queue++; - netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n", - channel->channel, tx_queue->label, - tx_queue->queue); - } - - /* If XDP is borrowing queues from net stack, it must use the queue - * with no csum offload, which is the first one of the channel - * (note: channel->tx_queue_by_type is not initialized yet) - */ - if (efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_BORROWED) { - tx_queue = &channel->tx_queue[0]; - rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); - if (rc == 0) - xdp_queue_number++; - } - } - } } - WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED && - xdp_queue_number != efx->xdp_tx_queue_count); - WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED && - xdp_queue_number > efx->xdp_tx_queue_count); - /* If we have more CPUs than assigned XDP TX queues, assign the already - * existing queues to the exceeding CPUs - */ - next_queue = 0; - while (xdp_queue_number < efx->xdp_tx_queue_count) { - tx_queue = efx->xdp_tx_queues[next_queue++]; - rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); - if (rc == 0) - xdp_queue_number++; - } + efx_set_xdp_channels(efx); rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); if (rc) @@ -1124,7 +1140,7 @@ void efx_start_channels(struct efx_nic *efx) struct efx_rx_queue *rx_queue; struct efx_channel *channel; - efx_for_each_channel(channel, efx) { + efx_for_each_channel_rev(channel, efx) { efx_for_each_channel_tx_queue(tx_queue, channel) { efx_init_tx_queue(tx_queue); atomic_inc(&efx->active_queues); diff --git a/drivers/net/ethernet/sfc/falcon/rx.c b/drivers/net/ethernet/sfc/falcon/rx.c index 0c6cc2191369..6bbdb5d2eebf 100644 --- a/drivers/net/ethernet/sfc/falcon/rx.c +++ b/drivers/net/ethernet/sfc/falcon/rx.c @@ -718,12 +718,14 @@ static void ef4_init_rx_recycle_ring(struct ef4_nic *efx, struct ef4_rx_queue *rx_queue) { unsigned int bufs_in_recycle_ring, page_ring_size; + struct iommu_domain __maybe_unused *domain; /* Set the RX recycle ring size */ #ifdef CONFIG_PPC64 bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_IOMMU; #else - if (iommu_present(&pci_bus_type)) + domain = iommu_get_domain_for_dev(&efx->pci_dev->dev); + if (domain && domain->type != IOMMU_DOMAIN_IDENTITY) bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_IOMMU; else bufs_in_recycle_ring = EF4_RECYCLE_RING_SIZE_NOIOMMU; diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index 1b22c7be0088..fa8b9aacca11 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -150,6 +150,9 @@ static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue) struct efx_nic *efx = rx_queue->efx; int i; + if (unlikely(!rx_queue->page_ring)) + return; + /* Unmap and release the pages in the recycle ring. Remove the ring. */ for (i = 0; i <= rx_queue->page_ptr_mask; i++) { struct page *page = rx_queue->page_ring[i]; diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index d16e031e95f4..138bca611341 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -443,6 +443,9 @@ int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs, if (unlikely(!tx_queue)) return -EINVAL; + if (!tx_queue->initialised) + return -EINVAL; + if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED) HARD_TX_LOCK(efx->net_dev, tx_queue->core_txq, cpu); @@ -524,7 +527,8 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, /* PTP "event" packet */ if (unlikely(efx_xmit_with_hwtstamp(skb)) && - unlikely(efx_ptp_is_ptp_tx(efx, skb))) { + ((efx_ptp_use_mac_tx_timestamps(efx) && efx->ptp_data) || + unlikely(efx_ptp_is_ptp_tx(efx, skb)))) { /* There may be existing transmits on the channel that are * waiting for this packet to trigger the doorbell write. * We need to send the packets at this point. diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c index d530cde2b864..9bc8281b7f5b 100644 --- a/drivers/net/ethernet/sfc/tx_common.c +++ b/drivers/net/ethernet/sfc/tx_common.c @@ -101,6 +101,8 @@ void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, "shutting down TX queue %d\n", tx_queue->queue); + tx_queue->initialised = false; + if (!tx_queue->buffer) return; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index ecf759ee1c9f..017dbbda0c1c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -205,7 +205,7 @@ static const struct pci_device_id loongson_dwmac_id_table[] = { }; MODULE_DEVICE_TABLE(pci, loongson_dwmac_id_table); -struct pci_driver loongson_dwmac_driver = { +static struct pci_driver loongson_dwmac_driver = { .name = "dwmac-loongson-pci", .id_table = loongson_dwmac_id_table, .probe = loongson_dwmac_probe, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 5d29f336315b..11e1055e8260 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -431,8 +431,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) plat->phylink_node = np; /* Get max speed of operation from device tree */ - if (of_property_read_u32(np, "max-speed", &plat->max_speed)) - plat->max_speed = -1; + of_property_read_u32(np, "max-speed", &plat->max_speed); plat->bus_id = of_alias_get_id(np, "ethernet"); if (plat->bus_id < 0) diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index 869e362e09c1..3f6b9dfca095 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -1515,7 +1515,7 @@ static int temac_probe(struct platform_device *pdev) of_node_put(dma_np); return PTR_ERR(lp->sdma_regs); } - if (of_get_property(dma_np, "little-endian", NULL)) { + if (of_property_read_bool(dma_np, "little-endian")) { lp->dma_in = temac_dma_in32_le; lp->dma_out = temac_dma_out32_le; } else { diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h index 0f9c88dd1a4a..d5c1e5c4a508 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet.h +++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h @@ -433,8 +433,6 @@ struct axienet_local { struct net_device *ndev; struct device *dev; - struct device_node *phy_node; - struct phylink *phylink; struct phylink_config phylink_config; diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index c7eb05e4a6bf..d6fc3f7acdf0 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -2064,25 +2064,33 @@ static int axienet_probe(struct platform_device *pdev) if (ret) goto cleanup_clk; - lp->phy_node = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); - if (lp->phy_node) { - ret = axienet_mdio_setup(lp); - if (ret) - dev_warn(&pdev->dev, - "error registering MDIO bus: %d\n", ret); - } + ret = axienet_mdio_setup(lp); + if (ret) + dev_warn(&pdev->dev, + "error registering MDIO bus: %d\n", ret); + if (lp->phy_mode == PHY_INTERFACE_MODE_SGMII || lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX) { - if (!lp->phy_node) { - dev_err(&pdev->dev, "phy-handle required for 1000BaseX/SGMII\n"); + np = of_parse_phandle(pdev->dev.of_node, "pcs-handle", 0); + if (!np) { + /* Deprecated: Always use "pcs-handle" for pcs_phy. + * Falling back to "phy-handle" here is only for + * backward compatibility with old device trees. + */ + np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); + } + if (!np) { + dev_err(&pdev->dev, "pcs-handle (preferred) or phy-handle required for 1000BaseX/SGMII\n"); ret = -EINVAL; goto cleanup_mdio; } - lp->pcs_phy = of_mdio_find_device(lp->phy_node); + lp->pcs_phy = of_mdio_find_device(np); if (!lp->pcs_phy) { ret = -EPROBE_DEFER; + of_node_put(np); goto cleanup_mdio; } + of_node_put(np); lp->pcs.ops = &axienet_pcs_ops; lp->pcs.poll = true; } @@ -2125,8 +2133,6 @@ cleanup_mdio: put_device(&lp->pcs_phy->dev); if (lp->mii_bus) axienet_mdio_teardown(lp); - of_node_put(lp->phy_node); - cleanup_clk: clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks); clk_disable_unprepare(lp->axi_clk); @@ -2155,9 +2161,6 @@ static int axienet_remove(struct platform_device *pdev) clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks); clk_disable_unprepare(lp->axi_clk); - of_node_put(lp->phy_node); - lp->phy_node = NULL; - free_netdev(ndev); return 0; diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 9442f751ad3a..4061af5baaea 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -792,9 +792,9 @@ static void netvsc_send_tx_complete(struct net_device *ndev, int queue_sends; u64 cmd_rqst; - cmd_rqst = channel->request_addr_callback(channel, (u64)desc->trans_id); + cmd_rqst = channel->request_addr_callback(channel, desc->trans_id); if (cmd_rqst == VMBUS_RQST_ERROR) { - netdev_err(ndev, "Incorrect transaction id\n"); + netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id); return; } @@ -854,9 +854,9 @@ static void netvsc_send_completion(struct net_device *ndev, /* First check if this is a VMBUS completion without data payload */ if (!msglen) { cmd_rqst = incoming_channel->request_addr_callback(incoming_channel, - (u64)desc->trans_id); + desc->trans_id); if (cmd_rqst == VMBUS_RQST_ERROR) { - netdev_err(ndev, "Invalid transaction id\n"); + netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id); return; } diff --git a/drivers/net/hyperv/netvsc_bpf.c b/drivers/net/hyperv/netvsc_bpf.c index 7856905414eb..232c4a0efd7b 100644 --- a/drivers/net/hyperv/netvsc_bpf.c +++ b/drivers/net/hyperv/netvsc_bpf.c @@ -137,7 +137,6 @@ int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog, int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) { struct netdev_bpf xdp; - bpf_op_t ndo_bpf; int ret; ASSERT_RTNL(); @@ -145,8 +144,7 @@ int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) if (!vf_netdev) return 0; - ndo_bpf = vf_netdev->netdev_ops->ndo_bpf; - if (!ndo_bpf) + if (!vf_netdev->netdev_ops->ndo_bpf) return 0; memset(&xdp, 0, sizeof(xdp)); @@ -157,7 +155,7 @@ int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) xdp.command = XDP_SETUP_PROG; xdp.prog = prog; - ret = ndo_bpf(vf_netdev, &xdp); + ret = vf_netdev->netdev_ops->ndo_bpf(vf_netdev, &xdp); if (ret && prog) bpf_prog_put(prog); diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index baf7afac7857..53846c6b56ca 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -553,7 +553,7 @@ static int mctp_i2c_header_create(struct sk_buff *skb, struct net_device *dev, hdr->source_slave = ((llsrc << 1) & 0xff) | 0x01; mhdr->ver = 0x01; - return 0; + return sizeof(struct mctp_i2c_hdr); } static int mctp_i2c_tx_thread(void *data) diff --git a/drivers/net/mdio/mdio-aspeed.c b/drivers/net/mdio/mdio-aspeed.c index e2273588c75b..7aa49827196f 100644 --- a/drivers/net/mdio/mdio-aspeed.c +++ b/drivers/net/mdio/mdio-aspeed.c @@ -21,6 +21,10 @@ #define ASPEED_MDIO_CTRL_OP GENMASK(27, 26) #define MDIO_C22_OP_WRITE 0b01 #define MDIO_C22_OP_READ 0b10 +#define MDIO_C45_OP_ADDR 0b00 +#define MDIO_C45_OP_WRITE 0b01 +#define MDIO_C45_OP_PREAD 0b10 +#define MDIO_C45_OP_READ 0b11 #define ASPEED_MDIO_CTRL_PHYAD GENMASK(25, 21) #define ASPEED_MDIO_CTRL_REGAD GENMASK(20, 16) #define ASPEED_MDIO_CTRL_MIIWDATA GENMASK(15, 0) @@ -39,34 +43,35 @@ struct aspeed_mdio { void __iomem *base; }; -static int aspeed_mdio_read(struct mii_bus *bus, int addr, int regnum) +static int aspeed_mdio_op(struct mii_bus *bus, u8 st, u8 op, u8 phyad, u8 regad, + u16 data) { struct aspeed_mdio *ctx = bus->priv; u32 ctrl; - u32 data; - int rc; - - dev_dbg(&bus->dev, "%s: addr: %d, regnum: %d\n", __func__, addr, - regnum); - /* Just clause 22 for the moment */ - if (regnum & MII_ADDR_C45) - return -EOPNOTSUPP; + dev_dbg(&bus->dev, "%s: st: %u op: %u, phyad: %u, regad: %u, data: %u\n", + __func__, st, op, phyad, regad, data); ctrl = ASPEED_MDIO_CTRL_FIRE - | FIELD_PREP(ASPEED_MDIO_CTRL_ST, ASPEED_MDIO_CTRL_ST_C22) - | FIELD_PREP(ASPEED_MDIO_CTRL_OP, MDIO_C22_OP_READ) - | FIELD_PREP(ASPEED_MDIO_CTRL_PHYAD, addr) - | FIELD_PREP(ASPEED_MDIO_CTRL_REGAD, regnum); + | FIELD_PREP(ASPEED_MDIO_CTRL_ST, st) + | FIELD_PREP(ASPEED_MDIO_CTRL_OP, op) + | FIELD_PREP(ASPEED_MDIO_CTRL_PHYAD, phyad) + | FIELD_PREP(ASPEED_MDIO_CTRL_REGAD, regad) + | FIELD_PREP(ASPEED_MDIO_DATA_MIIRDATA, data); iowrite32(ctrl, ctx->base + ASPEED_MDIO_CTRL); - rc = readl_poll_timeout(ctx->base + ASPEED_MDIO_CTRL, ctrl, + return readl_poll_timeout(ctx->base + ASPEED_MDIO_CTRL, ctrl, !(ctrl & ASPEED_MDIO_CTRL_FIRE), ASPEED_MDIO_INTERVAL_US, ASPEED_MDIO_TIMEOUT_US); - if (rc < 0) - return rc; +} + +static int aspeed_mdio_get_data(struct mii_bus *bus) +{ + struct aspeed_mdio *ctx = bus->priv; + u32 data; + int rc; rc = readl_poll_timeout(ctx->base + ASPEED_MDIO_DATA, data, data & ASPEED_MDIO_DATA_IDLE, @@ -78,31 +83,80 @@ static int aspeed_mdio_read(struct mii_bus *bus, int addr, int regnum) return FIELD_GET(ASPEED_MDIO_DATA_MIIRDATA, data); } -static int aspeed_mdio_write(struct mii_bus *bus, int addr, int regnum, u16 val) +static int aspeed_mdio_read_c22(struct mii_bus *bus, int addr, int regnum) { - struct aspeed_mdio *ctx = bus->priv; - u32 ctrl; + int rc; - dev_dbg(&bus->dev, "%s: addr: %d, regnum: %d, val: 0x%x\n", - __func__, addr, regnum, val); + rc = aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C22, MDIO_C22_OP_READ, + addr, regnum, 0); + if (rc < 0) + return rc; + + return aspeed_mdio_get_data(bus); +} + +static int aspeed_mdio_write_c22(struct mii_bus *bus, int addr, int regnum, + u16 val) +{ + return aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C22, MDIO_C22_OP_WRITE, + addr, regnum, val); +} + +static int aspeed_mdio_read_c45(struct mii_bus *bus, int addr, int regnum) +{ + u8 c45_dev = (regnum >> 16) & 0x1F; + u16 c45_addr = regnum & 0xFFFF; + int rc; + + rc = aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C45, MDIO_C45_OP_ADDR, + addr, c45_dev, c45_addr); + if (rc < 0) + return rc; + + rc = aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C45, MDIO_C45_OP_READ, + addr, c45_dev, 0); + if (rc < 0) + return rc; + + return aspeed_mdio_get_data(bus); +} + +static int aspeed_mdio_write_c45(struct mii_bus *bus, int addr, int regnum, + u16 val) +{ + u8 c45_dev = (regnum >> 16) & 0x1F; + u16 c45_addr = regnum & 0xFFFF; + int rc; + + rc = aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C45, MDIO_C45_OP_ADDR, + addr, c45_dev, c45_addr); + if (rc < 0) + return rc; + + return aspeed_mdio_op(bus, ASPEED_MDIO_CTRL_ST_C45, MDIO_C45_OP_WRITE, + addr, c45_dev, val); +} + +static int aspeed_mdio_read(struct mii_bus *bus, int addr, int regnum) +{ + dev_dbg(&bus->dev, "%s: addr: %d, regnum: %d\n", __func__, addr, + regnum); - /* Just clause 22 for the moment */ if (regnum & MII_ADDR_C45) - return -EOPNOTSUPP; + return aspeed_mdio_read_c45(bus, addr, regnum); - ctrl = ASPEED_MDIO_CTRL_FIRE - | FIELD_PREP(ASPEED_MDIO_CTRL_ST, ASPEED_MDIO_CTRL_ST_C22) - | FIELD_PREP(ASPEED_MDIO_CTRL_OP, MDIO_C22_OP_WRITE) - | FIELD_PREP(ASPEED_MDIO_CTRL_PHYAD, addr) - | FIELD_PREP(ASPEED_MDIO_CTRL_REGAD, regnum) - | FIELD_PREP(ASPEED_MDIO_CTRL_MIIWDATA, val); + return aspeed_mdio_read_c22(bus, addr, regnum); +} - iowrite32(ctrl, ctx->base + ASPEED_MDIO_CTRL); +static int aspeed_mdio_write(struct mii_bus *bus, int addr, int regnum, u16 val) +{ + dev_dbg(&bus->dev, "%s: addr: %d, regnum: %d, val: 0x%x\n", + __func__, addr, regnum, val); - return readl_poll_timeout(ctx->base + ASPEED_MDIO_CTRL, ctrl, - !(ctrl & ASPEED_MDIO_CTRL_FIRE), - ASPEED_MDIO_INTERVAL_US, - ASPEED_MDIO_TIMEOUT_US); + if (regnum & MII_ADDR_C45) + return aspeed_mdio_write_c45(bus, addr, regnum, val); + + return aspeed_mdio_write_c22(bus, addr, regnum, val); } static int aspeed_mdio_probe(struct platform_device *pdev) @@ -125,6 +179,7 @@ static int aspeed_mdio_probe(struct platform_device *pdev) bus->parent = &pdev->dev; bus->read = aspeed_mdio_read; bus->write = aspeed_mdio_write; + bus->probe_capabilities = MDIOBUS_C22_C45; rc = of_mdiobus_register(bus, pdev->dev.of_node); if (rc) { diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c index c483ba67c21f..85b4b097da05 100644 --- a/drivers/net/mdio/mdio-mscc-miim.c +++ b/drivers/net/mdio/mdio-mscc-miim.c @@ -7,6 +7,7 @@ */ #include <linux/bitops.h> +#include <linux/clk.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/kernel.h> @@ -30,6 +31,8 @@ #define MSCC_MIIM_CMD_VLD BIT(31) #define MSCC_MIIM_REG_DATA 0xC #define MSCC_MIIM_DATA_ERROR (BIT(16) | BIT(17)) +#define MSCC_MIIM_REG_CFG 0x10 +#define MSCC_MIIM_CFG_PRESCALE_MASK GENMASK(7, 0) #define MSCC_PHY_REG_PHY_CFG 0x0 #define PHY_CFG_PHY_ENA (BIT(0) | BIT(1) | BIT(2) | BIT(3)) @@ -50,6 +53,8 @@ struct mscc_miim_dev { int mii_status_offset; struct regmap *phy_regs; const struct mscc_miim_info *info; + struct clk *clk; + u32 bus_freq; }; /* When high resolution timers aren't built-in: we can't use usleep_range() as @@ -102,6 +107,9 @@ static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum) u32 val; int ret; + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + ret = mscc_miim_wait_pending(bus); if (ret) goto out; @@ -145,6 +153,9 @@ static int mscc_miim_write(struct mii_bus *bus, int mii_id, struct mscc_miim_dev *miim = bus->priv; int ret; + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + ret = mscc_miim_wait_pending(bus); if (ret < 0) goto out; @@ -235,9 +246,32 @@ int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, const char *name, } EXPORT_SYMBOL(mscc_miim_setup); +static int mscc_miim_clk_set(struct mii_bus *bus) +{ + struct mscc_miim_dev *miim = bus->priv; + unsigned long rate; + u32 div; + + /* Keep the current settings */ + if (!miim->bus_freq) + return 0; + + rate = clk_get_rate(miim->clk); + + div = DIV_ROUND_UP(rate, 2 * miim->bus_freq) - 1; + if (div == 0 || div & ~MSCC_MIIM_CFG_PRESCALE_MASK) { + dev_err(&bus->dev, "Incorrect MDIO clock frequency\n"); + return -EINVAL; + } + + return regmap_update_bits(miim->regs, MSCC_MIIM_REG_CFG, + MSCC_MIIM_CFG_PRESCALE_MASK, div); +} + static int mscc_miim_probe(struct platform_device *pdev) { struct regmap *mii_regmap, *phy_regmap = NULL; + struct device_node *np = pdev->dev.of_node; void __iomem *regs, *phy_regs; struct mscc_miim_dev *miim; struct resource *res; @@ -288,21 +322,47 @@ static int mscc_miim_probe(struct platform_device *pdev) if (!miim->info) return -EINVAL; - ret = of_mdiobus_register(bus, pdev->dev.of_node); + miim->clk = devm_clk_get_optional(&pdev->dev, NULL); + if (IS_ERR(miim->clk)) + return PTR_ERR(miim->clk); + + of_property_read_u32(np, "clock-frequency", &miim->bus_freq); + + if (miim->bus_freq && !miim->clk) { + dev_err(&pdev->dev, + "cannot use clock-frequency without a clock\n"); + return -EINVAL; + } + + ret = clk_prepare_enable(miim->clk); + if (ret) + return ret; + + ret = mscc_miim_clk_set(bus); + if (ret) + goto out_disable_clk; + + ret = of_mdiobus_register(bus, np); if (ret < 0) { dev_err(&pdev->dev, "Cannot register MDIO bus (%d)\n", ret); - return ret; + goto out_disable_clk; } platform_set_drvdata(pdev, bus); return 0; + +out_disable_clk: + clk_disable_unprepare(miim->clk); + return ret; } static int mscc_miim_remove(struct platform_device *pdev) { struct mii_bus *bus = platform_get_drvdata(pdev); + struct mscc_miim_dev *miim = bus->priv; + clk_disable_unprepare(miim->clk); mdiobus_unregister(bus); return 0; diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 19b11e896460..96840695debd 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -70,6 +70,27 @@ #define KSZ8081_LMD_SHORT_INDICATOR BIT(12) #define KSZ8081_LMD_DELTA_TIME_MASK GENMASK(8, 0) +#define KSZ9x31_LMD 0x12 +#define KSZ9x31_LMD_VCT_EN BIT(15) +#define KSZ9x31_LMD_VCT_DIS_TX BIT(14) +#define KSZ9x31_LMD_VCT_PAIR(n) (((n) & 0x3) << 12) +#define KSZ9x31_LMD_VCT_SEL_RESULT 0 +#define KSZ9x31_LMD_VCT_SEL_THRES_HI BIT(10) +#define KSZ9x31_LMD_VCT_SEL_THRES_LO BIT(11) +#define KSZ9x31_LMD_VCT_SEL_MASK GENMASK(11, 10) +#define KSZ9x31_LMD_VCT_ST_NORMAL 0 +#define KSZ9x31_LMD_VCT_ST_OPEN 1 +#define KSZ9x31_LMD_VCT_ST_SHORT 2 +#define KSZ9x31_LMD_VCT_ST_FAIL 3 +#define KSZ9x31_LMD_VCT_ST_MASK GENMASK(9, 8) +#define KSZ9x31_LMD_VCT_DATA_REFLECTED_INVALID BIT(7) +#define KSZ9x31_LMD_VCT_DATA_SIG_WAIT_TOO_LONG BIT(6) +#define KSZ9x31_LMD_VCT_DATA_MASK100 BIT(5) +#define KSZ9x31_LMD_VCT_DATA_NLP_FLP BIT(4) +#define KSZ9x31_LMD_VCT_DATA_LO_PULSE_MASK GENMASK(3, 2) +#define KSZ9x31_LMD_VCT_DATA_HI_PULSE_MASK GENMASK(1, 0) +#define KSZ9x31_LMD_VCT_DATA_MASK GENMASK(7, 0) + /* Lan8814 general Interrupt control/status reg in GPHY specific block. */ #define LAN8814_INTC 0x18 #define LAN8814_INTS 0x1B @@ -99,15 +120,6 @@ #define PTP_TIMESTAMP_EN_PDREQ_ BIT(2) #define PTP_TIMESTAMP_EN_PDRES_ BIT(3) -#define PTP_RX_LATENCY_1000 0x0224 -#define PTP_TX_LATENCY_1000 0x0225 - -#define PTP_RX_LATENCY_100 0x0222 -#define PTP_TX_LATENCY_100 0x0223 - -#define PTP_RX_LATENCY_10 0x0220 -#define PTP_TX_LATENCY_10 0x0221 - #define PTP_TX_PARSE_L2_ADDR_EN 0x0284 #define PTP_RX_PARSE_L2_ADDR_EN 0x0244 @@ -268,15 +280,6 @@ struct lan8814_ptp_rx_ts { u16 seq_id; }; -struct kszphy_latencies { - u16 rx_10; - u16 tx_10; - u16 rx_100; - u16 tx_100; - u16 rx_1000; - u16 tx_1000; -}; - struct kszphy_ptp_priv { struct mii_timestamper mii_ts; struct phy_device *phydev; @@ -296,22 +299,14 @@ struct kszphy_ptp_priv { struct kszphy_priv { struct kszphy_ptp_priv ptp_priv; - struct kszphy_latencies latencies; const struct kszphy_type *type; int led_mode; + u16 vct_ctrl1000; bool rmii_ref_clk_sel; bool rmii_ref_clk_sel_val; u64 stats[ARRAY_SIZE(kszphy_hw_stats)]; }; -static struct kszphy_latencies lan8814_latencies = { - .rx_10 = 0x22AA, - .tx_10 = 0x2E4A, - .rx_100 = 0x092A, - .tx_100 = 0x02C1, - .rx_1000 = 0x01AD, - .tx_1000 = 0x00C9, -}; static const struct kszphy_type ksz8021_type = { .led_mode_reg = MII_KSZPHY_CTRL_2, .has_broadcast_disable = true, @@ -1353,6 +1348,199 @@ static int ksz9031_read_status(struct phy_device *phydev) return 0; } +static int ksz9x31_cable_test_start(struct phy_device *phydev) +{ + struct kszphy_priv *priv = phydev->priv; + int ret; + + /* KSZ9131RNX, DS00002841B-page 38, 4.14 LinkMD (R) Cable Diagnostic + * Prior to running the cable diagnostics, Auto-negotiation should + * be disabled, full duplex set and the link speed set to 1000Mbps + * via the Basic Control Register. + */ + ret = phy_modify(phydev, MII_BMCR, + BMCR_SPEED1000 | BMCR_FULLDPLX | + BMCR_ANENABLE | BMCR_SPEED100, + BMCR_SPEED1000 | BMCR_FULLDPLX); + if (ret) + return ret; + + /* KSZ9131RNX, DS00002841B-page 38, 4.14 LinkMD (R) Cable Diagnostic + * The Master-Slave configuration should be set to Slave by writing + * a value of 0x1000 to the Auto-Negotiation Master Slave Control + * Register. + */ + ret = phy_read(phydev, MII_CTRL1000); + if (ret < 0) + return ret; + + /* Cache these bits, they need to be restored once LinkMD finishes. */ + priv->vct_ctrl1000 = ret & (CTL1000_ENABLE_MASTER | CTL1000_AS_MASTER); + ret &= ~(CTL1000_ENABLE_MASTER | CTL1000_AS_MASTER); + ret |= CTL1000_ENABLE_MASTER; + + return phy_write(phydev, MII_CTRL1000, ret); +} + +static int ksz9x31_cable_test_result_trans(u16 status) +{ + switch (FIELD_GET(KSZ9x31_LMD_VCT_ST_MASK, status)) { + case KSZ9x31_LMD_VCT_ST_NORMAL: + return ETHTOOL_A_CABLE_RESULT_CODE_OK; + case KSZ9x31_LMD_VCT_ST_OPEN: + return ETHTOOL_A_CABLE_RESULT_CODE_OPEN; + case KSZ9x31_LMD_VCT_ST_SHORT: + return ETHTOOL_A_CABLE_RESULT_CODE_SAME_SHORT; + case KSZ9x31_LMD_VCT_ST_FAIL: + fallthrough; + default: + return ETHTOOL_A_CABLE_RESULT_CODE_UNSPEC; + } +} + +static bool ksz9x31_cable_test_failed(u16 status) +{ + int stat = FIELD_GET(KSZ9x31_LMD_VCT_ST_MASK, status); + + return stat == KSZ9x31_LMD_VCT_ST_FAIL; +} + +static bool ksz9x31_cable_test_fault_length_valid(u16 status) +{ + switch (FIELD_GET(KSZ9x31_LMD_VCT_ST_MASK, status)) { + case KSZ9x31_LMD_VCT_ST_OPEN: + fallthrough; + case KSZ9x31_LMD_VCT_ST_SHORT: + return true; + } + return false; +} + +static int ksz9x31_cable_test_fault_length(struct phy_device *phydev, u16 stat) +{ + int dt = FIELD_GET(KSZ9x31_LMD_VCT_DATA_MASK, stat); + + /* KSZ9131RNX, DS00002841B-page 38, 4.14 LinkMD (R) Cable Diagnostic + * + * distance to fault = (VCT_DATA - 22) * 4 / cable propagation velocity + */ + if ((phydev->phy_id & MICREL_PHY_ID_MASK) == PHY_ID_KSZ9131) + dt = clamp(dt - 22, 0, 255); + + return (dt * 400) / 10; +} + +static int ksz9x31_cable_test_wait_for_completion(struct phy_device *phydev) +{ + int val, ret; + + ret = phy_read_poll_timeout(phydev, KSZ9x31_LMD, val, + !(val & KSZ9x31_LMD_VCT_EN), + 30000, 100000, true); + + return ret < 0 ? ret : 0; +} + +static int ksz9x31_cable_test_get_pair(int pair) +{ + static const int ethtool_pair[] = { + ETHTOOL_A_CABLE_PAIR_A, + ETHTOOL_A_CABLE_PAIR_B, + ETHTOOL_A_CABLE_PAIR_C, + ETHTOOL_A_CABLE_PAIR_D, + }; + + return ethtool_pair[pair]; +} + +static int ksz9x31_cable_test_one_pair(struct phy_device *phydev, int pair) +{ + int ret, val; + + /* KSZ9131RNX, DS00002841B-page 38, 4.14 LinkMD (R) Cable Diagnostic + * To test each individual cable pair, set the cable pair in the Cable + * Diagnostics Test Pair (VCT_PAIR[1:0]) field of the LinkMD Cable + * Diagnostic Register, along with setting the Cable Diagnostics Test + * Enable (VCT_EN) bit. The Cable Diagnostics Test Enable (VCT_EN) bit + * will self clear when the test is concluded. + */ + ret = phy_write(phydev, KSZ9x31_LMD, + KSZ9x31_LMD_VCT_EN | KSZ9x31_LMD_VCT_PAIR(pair)); + if (ret) + return ret; + + ret = ksz9x31_cable_test_wait_for_completion(phydev); + if (ret) + return ret; + + val = phy_read(phydev, KSZ9x31_LMD); + if (val < 0) + return val; + + if (ksz9x31_cable_test_failed(val)) + return -EAGAIN; + + ret = ethnl_cable_test_result(phydev, + ksz9x31_cable_test_get_pair(pair), + ksz9x31_cable_test_result_trans(val)); + if (ret) + return ret; + + if (!ksz9x31_cable_test_fault_length_valid(val)) + return 0; + + return ethnl_cable_test_fault_length(phydev, + ksz9x31_cable_test_get_pair(pair), + ksz9x31_cable_test_fault_length(phydev, val)); +} + +static int ksz9x31_cable_test_get_status(struct phy_device *phydev, + bool *finished) +{ + struct kszphy_priv *priv = phydev->priv; + unsigned long pair_mask = 0xf; + int retries = 20; + int pair, ret, rv; + + *finished = false; + + /* Try harder if link partner is active */ + while (pair_mask && retries--) { + for_each_set_bit(pair, &pair_mask, 4) { + ret = ksz9x31_cable_test_one_pair(phydev, pair); + if (ret == -EAGAIN) + continue; + if (ret < 0) + return ret; + clear_bit(pair, &pair_mask); + } + /* If link partner is in autonegotiation mode it will send 2ms + * of FLPs with at least 6ms of silence. + * Add 2ms sleep to have better chances to hit this silence. + */ + if (pair_mask) + usleep_range(2000, 3000); + } + + /* Report remaining unfinished pair result as unknown. */ + for_each_set_bit(pair, &pair_mask, 4) { + ret = ethnl_cable_test_result(phydev, + ksz9x31_cable_test_get_pair(pair), + ETHTOOL_A_CABLE_RESULT_CODE_UNSPEC); + } + + *finished = true; + + /* Restore cached bits from before LinkMD got started. */ + rv = phy_modify(phydev, MII_CTRL1000, + CTL1000_ENABLE_MASTER | CTL1000_AS_MASTER, + priv->vct_ctrl1000); + if (rv) + return rv; + + return ret; +} + static int ksz8873mll_config_aneg(struct phy_device *phydev) { return 0; @@ -2618,55 +2806,6 @@ static int lan8814_ptp_probe_once(struct phy_device *phydev) return 0; } -static int lan8814_read_status(struct phy_device *phydev) -{ - struct kszphy_priv *priv = phydev->priv; - struct kszphy_latencies *latencies = &priv->latencies; - int err; - int regval; - - err = genphy_read_status(phydev); - if (err) - return err; - - switch (phydev->speed) { - case SPEED_1000: - lanphy_write_page_reg(phydev, 5, PTP_RX_LATENCY_1000, - latencies->rx_1000); - lanphy_write_page_reg(phydev, 5, PTP_TX_LATENCY_1000, - latencies->tx_1000); - break; - case SPEED_100: - lanphy_write_page_reg(phydev, 5, PTP_RX_LATENCY_100, - latencies->rx_100); - lanphy_write_page_reg(phydev, 5, PTP_TX_LATENCY_100, - latencies->tx_100); - break; - case SPEED_10: - lanphy_write_page_reg(phydev, 5, PTP_RX_LATENCY_10, - latencies->rx_10); - lanphy_write_page_reg(phydev, 5, PTP_TX_LATENCY_10, - latencies->tx_10); - break; - default: - break; - } - - /* Make sure the PHY is not broken. Read idle error count, - * and reset the PHY if it is maxed out. - */ - regval = phy_read(phydev, MII_STAT1000); - if ((regval & 0xFF) == 0xFF) { - phy_init_hw(phydev); - phydev->link = 0; - if (phydev->drv->config_intr && phy_interrupt_is_valid(phydev)) - phydev->drv->config_intr(phydev); - return genphy_config_aneg(phydev); - } - - return 0; -} - static int lan8814_config_init(struct phy_device *phydev) { int val; @@ -2690,30 +2829,8 @@ static int lan8814_config_init(struct phy_device *phydev) return 0; } -static void lan8814_parse_latency(struct phy_device *phydev) -{ - const struct device_node *np = phydev->mdio.dev.of_node; - struct kszphy_priv *priv = phydev->priv; - struct kszphy_latencies *latency = &priv->latencies; - u32 val; - - if (!of_property_read_u32(np, "lan8814,latency_rx_10", &val)) - latency->rx_10 = val; - if (!of_property_read_u32(np, "lan8814,latency_tx_10", &val)) - latency->tx_10 = val; - if (!of_property_read_u32(np, "lan8814,latency_rx_100", &val)) - latency->rx_100 = val; - if (!of_property_read_u32(np, "lan8814,latency_tx_100", &val)) - latency->tx_100 = val; - if (!of_property_read_u32(np, "lan8814,latency_rx_1000", &val)) - latency->rx_1000 = val; - if (!of_property_read_u32(np, "lan8814,latency_tx_1000", &val)) - latency->tx_1000 = val; -} - static int lan8814_probe(struct phy_device *phydev) { - const struct device_node *np = phydev->mdio.dev.of_node; struct kszphy_priv *priv; u16 addr; int err; @@ -2724,13 +2841,10 @@ static int lan8814_probe(struct phy_device *phydev) priv->led_mode = -1; - priv->latencies = lan8814_latencies; - phydev->priv = priv; if (!IS_ENABLED(CONFIG_PTP_1588_CLOCK) || - !IS_ENABLED(CONFIG_NETWORK_PHY_TIMESTAMPING) || - of_property_read_bool(np, "lan8814,ignore-ts")) + !IS_ENABLED(CONFIG_NETWORK_PHY_TIMESTAMPING)) return 0; /* Strap-in value for PHY address, below register read gives starting @@ -2746,7 +2860,6 @@ static int lan8814_probe(struct phy_device *phydev) return err; } - lan8814_parse_latency(phydev); lan8814_ptp_init(phydev); return 0; @@ -2908,6 +3021,7 @@ static struct phy_driver ksphy_driver[] = { .phy_id = PHY_ID_KSZ9031, .phy_id_mask = MICREL_PHY_ID_MASK, .name = "Micrel KSZ9031 Gigabit PHY", + .flags = PHY_POLL_CABLE_TEST, .driver_data = &ksz9021_type, .probe = kszphy_probe, .get_features = ksz9031_get_features, @@ -2921,6 +3035,8 @@ static struct phy_driver ksphy_driver[] = { .get_stats = kszphy_get_stats, .suspend = kszphy_suspend, .resume = kszphy_resume, + .cable_test_start = ksz9x31_cable_test_start, + .cable_test_get_status = ksz9x31_cable_test_get_status, }, { .phy_id = PHY_ID_LAN8814, .phy_id_mask = MICREL_PHY_ID_MASK, @@ -2928,7 +3044,7 @@ static struct phy_driver ksphy_driver[] = { .config_init = lan8814_config_init, .probe = lan8814_probe, .soft_reset = genphy_soft_reset, - .read_status = lan8814_read_status, + .read_status = ksz9031_read_status, .get_sset_count = kszphy_get_sset_count, .get_strings = kszphy_get_strings, .get_stats = kszphy_get_stats, @@ -2955,6 +3071,7 @@ static struct phy_driver ksphy_driver[] = { .phy_id_mask = MICREL_PHY_ID_MASK, .name = "Microchip KSZ9131 Gigabit PHY", /* PHY_GBIT_FEATURES */ + .flags = PHY_POLL_CABLE_TEST, .driver_data = &ksz9021_type, .probe = kszphy_probe, .config_init = ksz9131_config_init, @@ -2965,6 +3082,8 @@ static struct phy_driver ksphy_driver[] = { .get_stats = kszphy_get_stats, .suspend = kszphy_suspend, .resume = kszphy_resume, + .cable_test_start = ksz9x31_cable_test_start, + .cable_test_get_status = ksz9x31_cable_test_get_status, }, { .phy_id = PHY_ID_KSZ8873MLL, .phy_id_mask = MICREL_PHY_ID_MASK, diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c index 3619520340b7..1b41cd9732d7 100644 --- a/drivers/net/ppp/pppoe.c +++ b/drivers/net/ppp/pppoe.c @@ -1011,8 +1011,7 @@ static int pppoe_recvmsg(struct socket *sock, struct msghdr *m, goto end; } - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &error); + skb = skb_recv_datagram(sk, flags, &error); if (error < 0) goto end; diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c index 88396ff99f03..6865d32270e5 100644 --- a/drivers/net/slip/slip.c +++ b/drivers/net/slip/slip.c @@ -469,7 +469,7 @@ static void sl_tx_timeout(struct net_device *dev, unsigned int txqueue) spin_lock(&sl->lock); if (netif_queue_stopped(dev)) { - if (!netif_running(dev)) + if (!netif_running(dev) || !sl->tty) goto out; /* May be we must check transmitter timeout here ? diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c index ea06d10e1c21..ca409d450a29 100644 --- a/drivers/net/usb/aqc111.c +++ b/drivers/net/usb/aqc111.c @@ -1102,10 +1102,15 @@ static int aqc111_rx_fixup(struct usbnet *dev, struct sk_buff *skb) if (start_of_descs != desc_offset) goto err; - /* self check desc_offset from header*/ - if (desc_offset >= skb_len) + /* self check desc_offset from header and make sure that the + * bounds of the metadata array are inside the SKB + */ + if (pkt_count * 2 + desc_offset >= skb_len) goto err; + /* Packets must not overlap the metadata array */ + skb_trim(skb, desc_offset); + if (pkt_count == 0) goto err; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 3353e761016d..17cf0d10fef9 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -190,7 +190,6 @@ static int qmimux_rx_fixup(struct usbnet *dev, struct sk_buff *skb) skbn = netdev_alloc_skb(net, pkt_len + LL_MAX_HEADER); if (!skbn) return 0; - skbn->dev = net; switch (skb->data[offset + qmimux_hdr_sz] & 0xf0) { case 0x40: diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 11f26b00a226..87838cbe38cf 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -169,6 +169,24 @@ struct receive_queue { struct xdp_rxq_info xdp_rxq; }; +/* This structure can contain rss message with maximum settings for indirection table and keysize + * Note, that default structure that describes RSS configuration virtio_net_rss_config + * contains same info but can't handle table values. + * In any case, structure would be passed to virtio hw through sg_buf split by parts + * because table sizes may be differ according to the device configuration. + */ +#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 +#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128 +struct virtio_net_ctrl_rss { + u32 hash_types; + u16 indirection_table_mask; + u16 unclassified_queue; + u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN]; + u16 max_tx_vq; + u8 hash_key_length; + u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE]; +}; + /* Control VQ buffers: protected by the rtnl lock */ struct control_buf { struct virtio_net_ctrl_hdr hdr; @@ -178,6 +196,7 @@ struct control_buf { u8 allmulti; __virtio16 vid; __virtio64 offloads; + struct virtio_net_ctrl_rss rss; }; struct virtnet_info { @@ -206,6 +225,14 @@ struct virtnet_info { /* Host will merge rx buffers for big packets (shake it! shake it!) */ bool mergeable_rx_bufs; + /* Host supports rss and/or hash report */ + bool has_rss; + bool has_rss_hash_report; + u8 rss_key_size; + u16 rss_indir_table_size; + u32 rss_hash_types_supported; + u32 rss_hash_types_saved; + /* Has control virtqueue */ bool has_cvq; @@ -242,13 +269,13 @@ struct virtnet_info { }; struct padded_vnet_hdr { - struct virtio_net_hdr_mrg_rxbuf hdr; + struct virtio_net_hdr_v1_hash hdr; /* * hdr is in a separate sg buffer, and data sg buffer shares same page * with this header sg. This padding makes next sg 16 byte aligned * after the header. */ - char padding[4]; + char padding[12]; }; static bool is_xdp_frame(void *ptr) @@ -396,7 +423,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, hdr_len = vi->hdr_len; if (vi->mergeable_rx_bufs) - hdr_padded_len = sizeof(*hdr); + hdr_padded_len = hdr_len; else hdr_padded_len = sizeof(struct padded_vnet_hdr); @@ -1123,6 +1150,35 @@ xdp_xmit: return NULL; } +static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash, + struct sk_buff *skb) +{ + enum pkt_hash_types rss_hash_type; + + if (!hdr_hash || !skb) + return; + + switch ((int)hdr_hash->hash_report) { + case VIRTIO_NET_HASH_REPORT_TCPv4: + case VIRTIO_NET_HASH_REPORT_UDPv4: + case VIRTIO_NET_HASH_REPORT_TCPv6: + case VIRTIO_NET_HASH_REPORT_UDPv6: + case VIRTIO_NET_HASH_REPORT_TCPv6_EX: + case VIRTIO_NET_HASH_REPORT_UDPv6_EX: + rss_hash_type = PKT_HASH_TYPE_L4; + break; + case VIRTIO_NET_HASH_REPORT_IPv4: + case VIRTIO_NET_HASH_REPORT_IPv6: + case VIRTIO_NET_HASH_REPORT_IPv6_EX: + rss_hash_type = PKT_HASH_TYPE_L3; + break; + case VIRTIO_NET_HASH_REPORT_NONE: + default: + rss_hash_type = PKT_HASH_TYPE_NONE; + } + skb_set_hash(skb, (unsigned int)hdr_hash->hash_value, rss_hash_type); +} + static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, void *buf, unsigned int len, void **ctx, unsigned int *xdp_xmit, @@ -1157,6 +1213,8 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, return; hdr = skb_vnet_hdr(skb); + if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) + virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb); if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) skb->ip_summed = CHECKSUM_UNNECESSARY; @@ -1266,7 +1324,8 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq, struct ewma_pkt_len *avg_pkt_len, unsigned int room) { - const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + struct virtnet_info *vi = rq->vq->vdev->priv; + const size_t hdr_len = vi->hdr_len; unsigned int len; if (room) @@ -2183,6 +2242,174 @@ static void virtnet_get_ringparam(struct net_device *dev, ring->tx_pending = ring->tx_max_pending; } +static bool virtnet_commit_rss_command(struct virtnet_info *vi) +{ + struct net_device *dev = vi->dev; + struct scatterlist sgs[4]; + unsigned int sg_buf_size; + + /* prepare sgs */ + sg_init_table(sgs, 4); + + sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table); + sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size); + + sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1); + sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size); + + sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key) + - offsetof(struct virtio_net_ctrl_rss, max_tx_vq); + sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size); + + sg_buf_size = vi->rss_key_size; + sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, + vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG + : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) { + dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); + return false; + } + return true; +} + +static void virtnet_init_default_rss(struct virtnet_info *vi) +{ + u32 indir_val = 0; + int i = 0; + + vi->ctrl->rss.hash_types = vi->rss_hash_types_supported; + vi->rss_hash_types_saved = vi->rss_hash_types_supported; + vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size + ? vi->rss_indir_table_size - 1 : 0; + vi->ctrl->rss.unclassified_queue = 0; + + for (; i < vi->rss_indir_table_size; ++i) { + indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs); + vi->ctrl->rss.indirection_table[i] = indir_val; + } + + vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs; + vi->ctrl->rss.hash_key_length = vi->rss_key_size; + + netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size); +} + +static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info) +{ + info->data = 0; + switch (info->flow_type) { + case TCP_V4_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { + info->data = RXH_IP_SRC | RXH_IP_DST; + } + break; + case TCP_V6_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { + info->data = RXH_IP_SRC | RXH_IP_DST; + } + break; + case UDP_V4_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { + info->data = RXH_IP_SRC | RXH_IP_DST; + } + break; + case UDP_V6_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { + info->data = RXH_IP_SRC | RXH_IP_DST; + } + break; + case IPV4_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) + info->data = RXH_IP_SRC | RXH_IP_DST; + + break; + case IPV6_FLOW: + if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) + info->data = RXH_IP_SRC | RXH_IP_DST; + + break; + default: + info->data = 0; + break; + } +} + +static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info) +{ + u32 new_hashtypes = vi->rss_hash_types_saved; + bool is_disable = info->data & RXH_DISCARD; + bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3); + + /* supports only 'sd', 'sdfn' and 'r' */ + if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable)) + return false; + + switch (info->flow_type) { + case TCP_V4_FLOW: + new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4); + if (!is_disable) + new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 + | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0); + break; + case UDP_V4_FLOW: + new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4); + if (!is_disable) + new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 + | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0); + break; + case IPV4_FLOW: + new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4; + if (!is_disable) + new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4; + break; + case TCP_V6_FLOW: + new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6); + if (!is_disable) + new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 + | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0); + break; + case UDP_V6_FLOW: + new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6); + if (!is_disable) + new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 + | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0); + break; + case IPV6_FLOW: + new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6; + if (!is_disable) + new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6; + break; + default: + /* unsupported flow */ + return false; + } + + /* if unsupported hashtype was set */ + if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) + return false; + + if (new_hashtypes != vi->rss_hash_types_saved) { + vi->rss_hash_types_saved = new_hashtypes; + vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; + if (vi->dev->features & NETIF_F_RXHASH) + return virtnet_commit_rss_command(vi); + } + + return true; +} static void virtnet_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) @@ -2411,6 +2638,92 @@ static void virtnet_update_settings(struct virtnet_info *vi) vi->duplex = duplex; } +static u32 virtnet_get_rxfh_key_size(struct net_device *dev) +{ + return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; +} + +static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) +{ + return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; +} + +static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) +{ + struct virtnet_info *vi = netdev_priv(dev); + int i; + + if (indir) { + for (i = 0; i < vi->rss_indir_table_size; ++i) + indir[i] = vi->ctrl->rss.indirection_table[i]; + } + + if (key) + memcpy(key, vi->ctrl->rss.key, vi->rss_key_size); + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; + + return 0; +} + +static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc) +{ + struct virtnet_info *vi = netdev_priv(dev); + int i; + + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + if (indir) { + for (i = 0; i < vi->rss_indir_table_size; ++i) + vi->ctrl->rss.indirection_table[i] = indir[i]; + } + if (key) + memcpy(vi->ctrl->rss.key, key, vi->rss_key_size); + + virtnet_commit_rss_command(vi); + + return 0; +} + +static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) +{ + struct virtnet_info *vi = netdev_priv(dev); + int rc = 0; + + switch (info->cmd) { + case ETHTOOL_GRXRINGS: + info->data = vi->curr_queue_pairs; + break; + case ETHTOOL_GRXFH: + virtnet_get_hashflow(vi, info); + break; + default: + rc = -EOPNOTSUPP; + } + + return rc; +} + +static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) +{ + struct virtnet_info *vi = netdev_priv(dev); + int rc = 0; + + switch (info->cmd) { + case ETHTOOL_SRXFH: + if (!virtnet_set_hashflow(vi, info)) + rc = -EINVAL; + + break; + default: + rc = -EOPNOTSUPP; + } + + return rc; +} + static const struct ethtool_ops virtnet_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES, .get_drvinfo = virtnet_get_drvinfo, @@ -2426,6 +2739,12 @@ static const struct ethtool_ops virtnet_ethtool_ops = { .set_link_ksettings = virtnet_set_link_ksettings, .set_coalesce = virtnet_set_coalesce, .get_coalesce = virtnet_get_coalesce, + .get_rxfh_key_size = virtnet_get_rxfh_key_size, + .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, + .get_rxfh = virtnet_get_rxfh, + .set_rxfh = virtnet_set_rxfh, + .get_rxnfc = virtnet_get_rxnfc, + .set_rxnfc = virtnet_set_rxnfc, }; static void virtnet_freeze_down(struct virtio_device *vdev) @@ -2678,6 +2997,16 @@ static int virtnet_set_features(struct net_device *dev, vi->guest_offloads = offloads; } + if ((dev->features ^ features) & NETIF_F_RXHASH) { + if (features & NETIF_F_RXHASH) + vi->ctrl->rss.hash_types = vi->rss_hash_types_saved; + else + vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE; + + if (!virtnet_commit_rss_command(vi)) + return -EINVAL; + } + return 0; } @@ -2851,7 +3180,7 @@ static void virtnet_del_vqs(struct virtnet_info *vi) */ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) { - const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + const unsigned int hdr_len = vi->hdr_len; unsigned int rq_size = virtqueue_get_vring_size(vq); unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; @@ -3072,6 +3401,10 @@ static bool virtnet_validate_features(struct virtio_device *vdev) "VIRTIO_NET_F_CTRL_VQ") || VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, + "VIRTIO_NET_F_CTRL_VQ") || + VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, + "VIRTIO_NET_F_CTRL_VQ") || + VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, "VIRTIO_NET_F_CTRL_VQ"))) { return false; } @@ -3112,13 +3445,14 @@ static int virtnet_probe(struct virtio_device *vdev) u16 max_queue_pairs; int mtu; - /* Find if host supports multiqueue virtio_net device */ - err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ, - struct virtio_net_config, - max_virtqueue_pairs, &max_queue_pairs); + /* Find if host supports multiqueue/rss virtio_net device */ + max_queue_pairs = 1; + if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) + max_queue_pairs = + virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); /* We need at least 2 queue's */ - if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || + if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) max_queue_pairs = 1; @@ -3206,8 +3540,33 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; - if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || - virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) + if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) + vi->has_rss_hash_report = true; + + if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) + vi->has_rss = true; + + if (vi->has_rss || vi->has_rss_hash_report) { + vi->rss_indir_table_size = + virtio_cread16(vdev, offsetof(struct virtio_net_config, + rss_max_indirection_table_length)); + vi->rss_key_size = + virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); + + vi->rss_hash_types_supported = + virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); + vi->rss_hash_types_supported &= + ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | + VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | + VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); + + dev->hw_features |= NETIF_F_RXHASH; + } + + if (vi->has_rss_hash_report) + vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); + else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || + virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); else vi->hdr_len = sizeof(struct virtio_net_hdr); @@ -3274,6 +3633,9 @@ static int virtnet_probe(struct virtio_device *vdev) } } + if (vi->has_rss || vi->has_rss_hash_report) + virtnet_init_default_rss(vi); + err = register_netdev(dev); if (err) { pr_debug("virtio_net: registering device failed\n"); @@ -3405,7 +3767,8 @@ static struct virtio_device_id id_table[] = { VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ VIRTIO_NET_F_CTRL_MAC_ADDR, \ VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ - VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY + VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ + VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT static unsigned int features[] = { VIRTNET_FEATURES, diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 85e362461d71..cfc30ce4c6e1 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1265,6 +1265,7 @@ static int vrf_prepare_mac_header(struct sk_buff *skb, eth = (struct ethhdr *)skb->data; skb_reset_mac_header(skb); + skb_reset_mac_len(skb); /* we set the ethernet destination and the source addresses to the * address of the VRF device. @@ -1294,9 +1295,9 @@ static int vrf_prepare_mac_header(struct sk_buff *skb, */ static int vrf_add_mac_header_if_unset(struct sk_buff *skb, struct net_device *vrf_dev, - u16 proto) + u16 proto, struct net_device *orig_dev) { - if (skb_mac_header_was_set(skb)) + if (skb_mac_header_was_set(skb) && dev_has_header(orig_dev)) return 0; return vrf_prepare_mac_header(skb, vrf_dev, proto); @@ -1402,6 +1403,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev, /* if packet is NDISC then keep the ingress interface */ if (!is_ndisc) { + struct net_device *orig_dev = skb->dev; + vrf_rx_stats(vrf_dev, skb->len); skb->dev = vrf_dev; skb->skb_iif = vrf_dev->ifindex; @@ -1410,7 +1413,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev, int err; err = vrf_add_mac_header_if_unset(skb, vrf_dev, - ETH_P_IPV6); + ETH_P_IPV6, + orig_dev); if (likely(!err)) { skb_push(skb, skb->mac_len); dev_queue_xmit_nit(skb, vrf_dev); @@ -1440,6 +1444,8 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev, static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev, struct sk_buff *skb) { + struct net_device *orig_dev = skb->dev; + skb->dev = vrf_dev; skb->skb_iif = vrf_dev->ifindex; IPCB(skb)->flags |= IPSKB_L3SLAVE; @@ -1460,7 +1466,8 @@ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev, if (!list_empty(&vrf_dev->ptype_all)) { int err; - err = vrf_add_mac_header_if_unset(skb, vrf_dev, ETH_P_IP); + err = vrf_add_mac_header_if_unset(skb, vrf_dev, ETH_P_IP, + orig_dev); if (likely(!err)) { skb_push(skb, skb->mac_len); dev_queue_xmit_nit(skb, vrf_dev); diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig index 140780ac1745..588b2333cdb8 100644 --- a/drivers/net/wan/Kconfig +++ b/drivers/net/wan/Kconfig @@ -57,34 +57,6 @@ config COSA The driver will be compiled as a module: the module will be called cosa. -# -# Lan Media's board. Currently 1000, 1200, 5200, 5245 -# -config LANMEDIA - tristate "LanMedia Corp. SSI/V.35, T1/E1, HSSI, T3 boards" - depends on PCI && VIRT_TO_BUS && HDLC - help - Driver for the following Lan Media family of serial boards: - - - LMC 1000 board allows you to connect synchronous serial devices - (for example base-band modems, or any other device with the X.21, - V.24, V.35 or V.36 interface) to your Linux box. - - - LMC 1200 with on board DSU board allows you to connect your Linux - box directly to a T1 or E1 circuit. - - - LMC 5200 board provides a HSSI interface capable of running up to - 52 Mbits per second. - - - LMC 5245 board connects directly to a T3 circuit saving the - additional external hardware. - - To change setting such as clock source you will need lmcctl. - It is available at <ftp://ftp.lanmedia.com/> (broken link). - - To compile this driver as a module, choose M here: the - module will be called lmc. - # There is no way to detect a Sealevel board. Force it modular config SEALEVEL_4021 tristate "Sealevel Systems 4021 support" diff --git a/drivers/net/wan/Makefile b/drivers/net/wan/Makefile index 480bcd1f6c1c..1cd42147b34f 100644 --- a/drivers/net/wan/Makefile +++ b/drivers/net/wan/Makefile @@ -19,8 +19,6 @@ obj-$(CONFIG_SEALEVEL_4021) += z85230.o sealevel.o obj-$(CONFIG_COSA) += cosa.o obj-$(CONFIG_FARSYNC) += farsync.o -obj-$(CONFIG_LANMEDIA) += lmc/ - obj-$(CONFIG_LAPBETHER) += lapbether.o obj-$(CONFIG_N2) += n2.o obj-$(CONFIG_C101) += c101.o diff --git a/drivers/net/wan/lmc/Makefile b/drivers/net/wan/lmc/Makefile deleted file mode 100644 index f00fe4491d69..000000000000 --- a/drivers/net/wan/lmc/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the Lan Media 21140 based WAN cards -# Specifically the 1000,1200,5200,5245 -# - -obj-$(CONFIG_LANMEDIA) += lmc.o - -lmc-objs := lmc_debug.o lmc_media.o lmc_main.o lmc_proto.o - -# Like above except every packet gets echoed to KERN_DEBUG -# in hex -# -# DBDEF = \ -# -DDEBUG \ -# -DLMC_PACKET_LOG - -ccflags-y := $(DBGDEF) diff --git a/drivers/net/wan/lmc/lmc.h b/drivers/net/wan/lmc/lmc.h deleted file mode 100644 index d7d59b4595f9..000000000000 --- a/drivers/net/wan/lmc/lmc.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LMC_H_ -#define _LMC_H_ - -#include "lmc_var.h" - -/* - * prototypes for everyone - */ -int lmc_probe(struct net_device * dev); -unsigned lmc_mii_readreg(lmc_softc_t * const sc, unsigned - devaddr, unsigned regno); -void lmc_mii_writereg(lmc_softc_t * const sc, unsigned devaddr, - unsigned regno, unsigned data); -void lmc_led_on(lmc_softc_t * const, u32); -void lmc_led_off(lmc_softc_t * const, u32); -unsigned lmc_mii_readreg(lmc_softc_t * const, unsigned, unsigned); -void lmc_mii_writereg(lmc_softc_t * const, unsigned, unsigned, unsigned); -void lmc_gpio_mkinput(lmc_softc_t * const sc, u32 bits); -void lmc_gpio_mkoutput(lmc_softc_t * const sc, u32 bits); - -int lmc_ioctl(struct net_device *dev, struct if_settings *ifs); - -extern lmc_media_t lmc_ds3_media; -extern lmc_media_t lmc_ssi_media; -extern lmc_media_t lmc_t1_media; -extern lmc_media_t lmc_hssi_media; - -#ifdef _DBG_EVENTLOG -static void lmcEventLog(u32 EventNum, u32 arg2, u32 arg3); -#endif - -#endif diff --git a/drivers/net/wan/lmc/lmc_debug.c b/drivers/net/wan/lmc/lmc_debug.c deleted file mode 100644 index 2b6051bda3fb..000000000000 --- a/drivers/net/wan/lmc/lmc_debug.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/types.h> -#include <linux/netdevice.h> -#include <linux/interrupt.h> - -#include "lmc_debug.h" - -/* - * Prints out len, max to 80 octets using printk, 20 per line - */ -#ifdef DEBUG -#ifdef LMC_PACKET_LOG -void lmcConsoleLog(char *type, unsigned char *ucData, int iLen) -{ - int iNewLine = 1; - char str[80], *pstr; - - sprintf(str, KERN_DEBUG "lmc: %s: ", type); - pstr = str+strlen(str); - - if(iLen > 240){ - printk(KERN_DEBUG "lmc: Printing 240 chars... out of: %d\n", iLen); - iLen = 240; - } - else{ - printk(KERN_DEBUG "lmc: Printing %d chars\n", iLen); - } - - while(iLen > 0) - { - sprintf(pstr, "%02x ", *ucData); - pstr+=3; - ucData++; - if( !(iNewLine % 20)) - { - sprintf(pstr, "\n"); - printk(str); - sprintf(str, KERN_DEBUG "lmc: %s: ", type); - pstr=str+strlen(str); - } - iNewLine++; - iLen--; - } - sprintf(pstr, "\n"); - printk(str); -} -#endif -#endif - -#ifdef DEBUG -u32 lmcEventLogIndex; -u32 lmcEventLogBuf[LMC_EVENTLOGSIZE * LMC_EVENTLOGARGS]; - -void lmcEventLog(u32 EventNum, u32 arg2, u32 arg3) -{ - lmcEventLogBuf[lmcEventLogIndex++] = EventNum; - lmcEventLogBuf[lmcEventLogIndex++] = arg2; - lmcEventLogBuf[lmcEventLogIndex++] = arg3; - lmcEventLogBuf[lmcEventLogIndex++] = jiffies; - - lmcEventLogIndex &= (LMC_EVENTLOGSIZE * LMC_EVENTLOGARGS) - 1; -} -#endif /* DEBUG */ - -/* --------------------------- end if_lmc_linux.c ------------------------ */ diff --git a/drivers/net/wan/lmc/lmc_debug.h b/drivers/net/wan/lmc/lmc_debug.h deleted file mode 100644 index cfae9eddf003..000000000000 --- a/drivers/net/wan/lmc/lmc_debug.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LMC_DEBUG_H_ -#define _LMC_DEBUG_H_ - -#ifdef DEBUG -#ifdef LMC_PACKET_LOG -#define LMC_CONSOLE_LOG(x,y,z) lmcConsoleLog((x), (y), (z)) -#else -#define LMC_CONSOLE_LOG(x,y,z) -#endif -#else -#define LMC_CONSOLE_LOG(x,y,z) -#endif - - - -/* Debug --- Event log definitions --- */ -/* EVENTLOGSIZE*EVENTLOGARGS needs to be a power of 2 */ -#define LMC_EVENTLOGSIZE 1024 /* number of events in eventlog */ -#define LMC_EVENTLOGARGS 4 /* number of args for each event */ - -/* event indicators */ -#define LMC_EVENT_XMT 1 -#define LMC_EVENT_XMTEND 2 -#define LMC_EVENT_XMTINT 3 -#define LMC_EVENT_RCVINT 4 -#define LMC_EVENT_RCVEND 5 -#define LMC_EVENT_INT 6 -#define LMC_EVENT_XMTINTTMO 7 -#define LMC_EVENT_XMTPRCTMO 8 -#define LMC_EVENT_INTEND 9 -#define LMC_EVENT_RESET1 10 -#define LMC_EVENT_RESET2 11 -#define LMC_EVENT_FORCEDRESET 12 -#define LMC_EVENT_WATCHDOG 13 -#define LMC_EVENT_BADPKTSURGE 14 -#define LMC_EVENT_TBUSY0 15 -#define LMC_EVENT_TBUSY1 16 - - -#ifdef DEBUG -extern u32 lmcEventLogIndex; -extern u32 lmcEventLogBuf[LMC_EVENTLOGSIZE * LMC_EVENTLOGARGS]; -#define LMC_EVENT_LOG(x, y, z) lmcEventLog((x), (y), (z)) -#else -#define LMC_EVENT_LOG(x,y,z) -#endif /* end ifdef _DBG_EVENTLOG */ - -void lmcConsoleLog(char *type, unsigned char *ucData, int iLen); -void lmcEventLog(u32 EventNum, u32 arg2, u32 arg3); - -#endif diff --git a/drivers/net/wan/lmc/lmc_ioctl.h b/drivers/net/wan/lmc/lmc_ioctl.h deleted file mode 100644 index 8c65e2176e94..000000000000 --- a/drivers/net/wan/lmc/lmc_ioctl.h +++ /dev/null @@ -1,255 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LMC_IOCTL_H_ -#define _LMC_IOCTL_H_ -/* $Id: lmc_ioctl.h,v 1.15 2000/04/06 12:16:43 asj Exp $ */ - - /* - * Copyright (c) 1997-2000 LAN Media Corporation (LMC) - * All rights reserved. www.lanmedia.com - * - * This code is written by: - * Andrew Stanley-Jones (asj@cban.com) - * Rob Braun (bbraun@vix.com), - * Michael Graff (explorer@vix.com) and - * Matt Thomas (matt@3am-software.com). - */ - -#define LMCIOCGINFO SIOCDEVPRIVATE+3 /* get current state */ -#define LMCIOCSINFO SIOCDEVPRIVATE+4 /* set state to user values */ -#define LMCIOCGETLMCSTATS SIOCDEVPRIVATE+5 -#define LMCIOCCLEARLMCSTATS SIOCDEVPRIVATE+6 -#define LMCIOCDUMPEVENTLOG SIOCDEVPRIVATE+7 -#define LMCIOCGETXINFO SIOCDEVPRIVATE+8 -#define LMCIOCSETCIRCUIT SIOCDEVPRIVATE+9 -#define LMCIOCUNUSEDATM SIOCDEVPRIVATE+10 -#define LMCIOCRESET SIOCDEVPRIVATE+11 -#define LMCIOCT1CONTROL SIOCDEVPRIVATE+12 -#define LMCIOCIFTYPE SIOCDEVPRIVATE+13 -#define LMCIOCXILINX SIOCDEVPRIVATE+14 - -#define LMC_CARDTYPE_UNKNOWN -1 -#define LMC_CARDTYPE_HSSI 1 /* probed card is a HSSI card */ -#define LMC_CARDTYPE_DS3 2 /* probed card is a DS3 card */ -#define LMC_CARDTYPE_SSI 3 /* probed card is a SSI card */ -#define LMC_CARDTYPE_T1 4 /* probed card is a T1 card */ - -#define LMC_CTL_CARDTYPE_LMC5200 0 /* HSSI */ -#define LMC_CTL_CARDTYPE_LMC5245 1 /* DS3 */ -#define LMC_CTL_CARDTYPE_LMC1000 2 /* SSI, V.35 */ -#define LMC_CTL_CARDTYPE_LMC1200 3 /* DS1 */ - -#define LMC_CTL_OFF 0 /* generic OFF value */ -#define LMC_CTL_ON 1 /* generic ON value */ - -#define LMC_CTL_CLOCK_SOURCE_EXT 0 /* clock off line */ -#define LMC_CTL_CLOCK_SOURCE_INT 1 /* internal clock */ - -#define LMC_CTL_CRC_LENGTH_16 16 -#define LMC_CTL_CRC_LENGTH_32 32 -#define LMC_CTL_CRC_BYTESIZE_2 2 -#define LMC_CTL_CRC_BYTESIZE_4 4 - - -#define LMC_CTL_CABLE_LENGTH_LT_100FT 0 /* DS3 cable < 100 feet */ -#define LMC_CTL_CABLE_LENGTH_GT_100FT 1 /* DS3 cable >= 100 feet */ - -#define LMC_CTL_CIRCUIT_TYPE_E1 0 -#define LMC_CTL_CIRCUIT_TYPE_T1 1 - -/* - * IFTYPE defines - */ -#define LMC_PPP 1 /* use generic HDLC interface */ -#define LMC_NET 2 /* use direct net interface */ -#define LMC_RAW 3 /* use direct net interface */ - -/* - * These are not in the least IOCTL related, but I want them common. - */ -/* - * assignments for the GPIO register on the DEC chip (common) - */ -#define LMC_GEP_INIT 0x01 /* 0: */ -#define LMC_GEP_RESET 0x02 /* 1: */ -#define LMC_GEP_MODE 0x10 /* 4: */ -#define LMC_GEP_DP 0x20 /* 5: */ -#define LMC_GEP_DATA 0x40 /* 6: serial out */ -#define LMC_GEP_CLK 0x80 /* 7: serial clock */ - -/* - * HSSI GPIO assignments - */ -#define LMC_GEP_HSSI_ST 0x04 /* 2: receive timing sense (deprecated) */ -#define LMC_GEP_HSSI_CLOCK 0x08 /* 3: clock source */ - -/* - * T1 GPIO assignments - */ -#define LMC_GEP_SSI_GENERATOR 0x04 /* 2: enable prog freq gen serial i/f */ -#define LMC_GEP_SSI_TXCLOCK 0x08 /* 3: provide clock on TXCLOCK output */ - -/* - * Common MII16 bits - */ -#define LMC_MII16_LED0 0x0080 -#define LMC_MII16_LED1 0x0100 -#define LMC_MII16_LED2 0x0200 -#define LMC_MII16_LED3 0x0400 /* Error, and the red one */ -#define LMC_MII16_LED_ALL 0x0780 /* LED bit mask */ -#define LMC_MII16_FIFO_RESET 0x0800 - -/* - * definitions for HSSI - */ -#define LMC_MII16_HSSI_TA 0x0001 -#define LMC_MII16_HSSI_CA 0x0002 -#define LMC_MII16_HSSI_LA 0x0004 -#define LMC_MII16_HSSI_LB 0x0008 -#define LMC_MII16_HSSI_LC 0x0010 -#define LMC_MII16_HSSI_TM 0x0020 -#define LMC_MII16_HSSI_CRC 0x0040 - -/* - * assignments for the MII register 16 (DS3) - */ -#define LMC_MII16_DS3_ZERO 0x0001 -#define LMC_MII16_DS3_TRLBK 0x0002 -#define LMC_MII16_DS3_LNLBK 0x0004 -#define LMC_MII16_DS3_RAIS 0x0008 -#define LMC_MII16_DS3_TAIS 0x0010 -#define LMC_MII16_DS3_BIST 0x0020 -#define LMC_MII16_DS3_DLOS 0x0040 -#define LMC_MII16_DS3_CRC 0x1000 -#define LMC_MII16_DS3_SCRAM 0x2000 -#define LMC_MII16_DS3_SCRAM_LARS 0x4000 - -/* Note: 2 pairs of LEDs where swapped by mistake - * in Xilinx code for DS3 & DS1 adapters */ -#define LMC_DS3_LED0 0x0100 /* bit 08 yellow */ -#define LMC_DS3_LED1 0x0080 /* bit 07 blue */ -#define LMC_DS3_LED2 0x0400 /* bit 10 green */ -#define LMC_DS3_LED3 0x0200 /* bit 09 red */ - -/* - * framer register 0 and 7 (7 is latched and reset on read) - */ -#define LMC_FRAMER_REG0_DLOS 0x80 /* digital loss of service */ -#define LMC_FRAMER_REG0_OOFS 0x40 /* out of frame sync */ -#define LMC_FRAMER_REG0_AIS 0x20 /* alarm indication signal */ -#define LMC_FRAMER_REG0_CIS 0x10 /* channel idle */ -#define LMC_FRAMER_REG0_LOC 0x08 /* loss of clock */ - -/* - * Framer register 9 contains the blue alarm signal - */ -#define LMC_FRAMER_REG9_RBLUE 0x02 /* Blue alarm failure */ - -/* - * Framer register 0x10 contains xbit error - */ -#define LMC_FRAMER_REG10_XBIT 0x01 /* X bit error alarm failure */ - -/* - * And SSI, LMC1000 - */ -#define LMC_MII16_SSI_DTR 0x0001 /* DTR output RW */ -#define LMC_MII16_SSI_DSR 0x0002 /* DSR input RO */ -#define LMC_MII16_SSI_RTS 0x0004 /* RTS output RW */ -#define LMC_MII16_SSI_CTS 0x0008 /* CTS input RO */ -#define LMC_MII16_SSI_DCD 0x0010 /* DCD input RO */ -#define LMC_MII16_SSI_RI 0x0020 /* RI input RO */ -#define LMC_MII16_SSI_CRC 0x1000 /* CRC select - RW */ - -/* - * bits 0x0080 through 0x0800 are generic, and described - * above with LMC_MII16_LED[0123] _LED_ALL, and _FIFO_RESET - */ -#define LMC_MII16_SSI_LL 0x1000 /* LL output RW */ -#define LMC_MII16_SSI_RL 0x2000 /* RL output RW */ -#define LMC_MII16_SSI_TM 0x4000 /* TM input RO */ -#define LMC_MII16_SSI_LOOP 0x8000 /* loopback enable RW */ - -/* - * Some of the MII16 bits are mirrored in the MII17 register as well, - * but let's keep thing separate for now, and get only the cable from - * the MII17. - */ -#define LMC_MII17_SSI_CABLE_MASK 0x0038 /* mask to extract the cable type */ -#define LMC_MII17_SSI_CABLE_SHIFT 3 /* shift to extract the cable type */ - -/* - * And T1, LMC1200 - */ -#define LMC_MII16_T1_UNUSED1 0x0003 -#define LMC_MII16_T1_XOE 0x0004 -#define LMC_MII16_T1_RST 0x0008 /* T1 chip reset - RW */ -#define LMC_MII16_T1_Z 0x0010 /* output impedance T1=1, E1=0 output - RW */ -#define LMC_MII16_T1_INTR 0x0020 /* interrupt from 8370 - RO */ -#define LMC_MII16_T1_ONESEC 0x0040 /* one second square wave - ro */ - -#define LMC_MII16_T1_LED0 0x0100 -#define LMC_MII16_T1_LED1 0x0080 -#define LMC_MII16_T1_LED2 0x0400 -#define LMC_MII16_T1_LED3 0x0200 -#define LMC_MII16_T1_FIFO_RESET 0x0800 - -#define LMC_MII16_T1_CRC 0x1000 /* CRC select - RW */ -#define LMC_MII16_T1_UNUSED2 0xe000 - - -/* 8370 framer registers */ - -#define T1FRAMER_ALARM1_STATUS 0x47 -#define T1FRAMER_ALARM2_STATUS 0x48 -#define T1FRAMER_FERR_LSB 0x50 -#define T1FRAMER_FERR_MSB 0x51 /* framing bit error counter */ -#define T1FRAMER_LCV_LSB 0x54 -#define T1FRAMER_LCV_MSB 0x55 /* line code violation counter */ -#define T1FRAMER_AERR 0x5A - -/* mask for the above AERR register */ -#define T1FRAMER_LOF_MASK (0x0f0) /* receive loss of frame */ -#define T1FRAMER_COFA_MASK (0x0c0) /* change of frame alignment */ -#define T1FRAMER_SEF_MASK (0x03) /* severely errored frame */ - -/* 8370 framer register ALM1 (0x47) values - * used to determine link status - */ - -#define T1F_SIGFRZ 0x01 /* signaling freeze */ -#define T1F_RLOF 0x02 /* receive loss of frame alignment */ -#define T1F_RLOS 0x04 /* receive loss of signal */ -#define T1F_RALOS 0x08 /* receive analog loss of signal or RCKI loss of clock */ -#define T1F_RAIS 0x10 /* receive alarm indication signal */ -#define T1F_UNUSED 0x20 -#define T1F_RYEL 0x40 /* receive yellow alarm */ -#define T1F_RMYEL 0x80 /* receive multiframe yellow alarm */ - -#define LMC_T1F_WRITE 0 -#define LMC_T1F_READ 1 - -typedef struct lmc_st1f_control { - int command; - int address; - int value; - char __user *data; -} lmc_t1f_control; - -enum lmc_xilinx_c { - lmc_xilinx_reset = 1, - lmc_xilinx_load_prom = 2, - lmc_xilinx_load = 3 -}; - -struct lmc_xilinx_control { - enum lmc_xilinx_c command; - int len; - char __user *data; -}; - -/* ------------------ end T1 defs ------------------- */ - -#define LMC_MII_LedMask 0x0780 -#define LMC_MII_LedBitPos 7 - -#endif diff --git a/drivers/net/wan/lmc/lmc_main.c b/drivers/net/wan/lmc/lmc_main.c deleted file mode 100644 index 76c6b4f89890..000000000000 --- a/drivers/net/wan/lmc/lmc_main.c +++ /dev/null @@ -1,2009 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - /* - * Copyright (c) 1997-2000 LAN Media Corporation (LMC) - * All rights reserved. www.lanmedia.com - * Generic HDLC port Copyright (C) 2008 Krzysztof Halasa <khc@pm.waw.pl> - * - * This code is written by: - * Andrew Stanley-Jones (asj@cban.com) - * Rob Braun (bbraun@vix.com), - * Michael Graff (explorer@vix.com) and - * Matt Thomas (matt@3am-software.com). - * - * With Help By: - * David Boggs - * Ron Crane - * Alan Cox - * - * Driver for the LanMedia LMC5200, LMC5245, LMC1000, LMC1200 cards. - * - * To control link specific options lmcctl is required. - * It can be obtained from ftp.lanmedia.com. - * - * Linux driver notes: - * Linux uses the device struct lmc_private to pass private information - * around. - * - * The initialization portion of this driver (the lmc_reset() and the - * lmc_dec_reset() functions, as well as the led controls and the - * lmc_initcsrs() functions. - * - * The watchdog function runs every second and checks to see if - * we still have link, and that the timing source is what we expected - * it to be. If link is lost, the interface is marked down, and - * we no longer can transmit. - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/timer.h> -#include <linux/ptrace.h> -#include <linux/errno.h> -#include <linux/ioport.h> -#include <linux/slab.h> -#include <linux/interrupt.h> -#include <linux/pci.h> -#include <linux/delay.h> -#include <linux/hdlc.h> -#include <linux/in.h> -#include <linux/if_arp.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/skbuff.h> -#include <linux/inet.h> -#include <linux/bitops.h> -#include <asm/processor.h> /* Processor type for cache alignment. */ -#include <asm/io.h> -#include <asm/dma.h> -#include <linux/uaccess.h> -#include <linux/jiffies.h> -//#include <asm/spinlock.h> - -#define DRIVER_MAJOR_VERSION 1 -#define DRIVER_MINOR_VERSION 34 -#define DRIVER_SUB_VERSION 0 - -#define DRIVER_VERSION ((DRIVER_MAJOR_VERSION << 8) + DRIVER_MINOR_VERSION) - -#include "lmc.h" -#include "lmc_var.h" -#include "lmc_ioctl.h" -#include "lmc_debug.h" -#include "lmc_proto.h" - -static int LMC_PKT_BUF_SZ = 1542; - -static const struct pci_device_id lmc_pci_tbl[] = { - { PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_TULIP_FAST, - PCI_VENDOR_ID_LMC, PCI_ANY_ID }, - { PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_TULIP_FAST, - PCI_ANY_ID, PCI_VENDOR_ID_LMC }, - { 0 } -}; - -MODULE_DEVICE_TABLE(pci, lmc_pci_tbl); -MODULE_LICENSE("GPL v2"); - - -static netdev_tx_t lmc_start_xmit(struct sk_buff *skb, - struct net_device *dev); -static int lmc_rx (struct net_device *dev); -static int lmc_open(struct net_device *dev); -static int lmc_close(struct net_device *dev); -static struct net_device_stats *lmc_get_stats(struct net_device *dev); -static irqreturn_t lmc_interrupt(int irq, void *dev_instance); -static void lmc_initcsrs(lmc_softc_t * const sc, lmc_csrptr_t csr_base, size_t csr_size); -static void lmc_softreset(lmc_softc_t * const); -static void lmc_running_reset(struct net_device *dev); -static int lmc_ifdown(struct net_device * const); -static void lmc_watchdog(struct timer_list *t); -static void lmc_reset(lmc_softc_t * const sc); -static void lmc_dec_reset(lmc_softc_t * const sc); -static void lmc_driver_timeout(struct net_device *dev, unsigned int txqueue); - -/* - * linux reserves 16 device specific IOCTLs. We call them - * LMCIOC* to control various bits of our world. - */ -static int lmc_siocdevprivate(struct net_device *dev, struct ifreq *ifr, - void __user *data, int cmd) /*fold00*/ -{ - lmc_softc_t *sc = dev_to_sc(dev); - lmc_ctl_t ctl; - int ret = -EOPNOTSUPP; - u16 regVal; - unsigned long flags; - - /* - * Most functions mess with the structure - * Disable interrupts while we do the polling - */ - - switch (cmd) { - /* - * Return current driver state. Since we keep this up - * To date internally, just copy this out to the user. - */ - case LMCIOCGINFO: /*fold01*/ - if (copy_to_user(data, &sc->ictl, sizeof(lmc_ctl_t))) - ret = -EFAULT; - else - ret = 0; - break; - - case LMCIOCSINFO: /*fold01*/ - if (!capable(CAP_NET_ADMIN)) { - ret = -EPERM; - break; - } - - if(dev->flags & IFF_UP){ - ret = -EBUSY; - break; - } - - if (copy_from_user(&ctl, data, sizeof(lmc_ctl_t))) { - ret = -EFAULT; - break; - } - - spin_lock_irqsave(&sc->lmc_lock, flags); - sc->lmc_media->set_status (sc, &ctl); - - if(ctl.crc_length != sc->ictl.crc_length) { - sc->lmc_media->set_crc_length(sc, ctl.crc_length); - if (sc->ictl.crc_length == LMC_CTL_CRC_LENGTH_16) - sc->TxDescriptControlInit |= LMC_TDES_ADD_CRC_DISABLE; - else - sc->TxDescriptControlInit &= ~LMC_TDES_ADD_CRC_DISABLE; - } - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - ret = 0; - break; - - case LMCIOCIFTYPE: /*fold01*/ - { - u16 old_type = sc->if_type; - u16 new_type; - - if (!capable(CAP_NET_ADMIN)) { - ret = -EPERM; - break; - } - - if (copy_from_user(&new_type, data, sizeof(u16))) { - ret = -EFAULT; - break; - } - - - if (new_type == old_type) - { - ret = 0 ; - break; /* no change */ - } - - spin_lock_irqsave(&sc->lmc_lock, flags); - lmc_proto_close(sc); - - sc->if_type = new_type; - lmc_proto_attach(sc); - ret = lmc_proto_open(sc); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - break; - } - - case LMCIOCGETXINFO: /*fold01*/ - spin_lock_irqsave(&sc->lmc_lock, flags); - sc->lmc_xinfo.Magic0 = 0xBEEFCAFE; - - sc->lmc_xinfo.PciCardType = sc->lmc_cardtype; - sc->lmc_xinfo.PciSlotNumber = 0; - sc->lmc_xinfo.DriverMajorVersion = DRIVER_MAJOR_VERSION; - sc->lmc_xinfo.DriverMinorVersion = DRIVER_MINOR_VERSION; - sc->lmc_xinfo.DriverSubVersion = DRIVER_SUB_VERSION; - sc->lmc_xinfo.XilinxRevisionNumber = - lmc_mii_readreg (sc, 0, 3) & 0xf; - sc->lmc_xinfo.MaxFrameSize = LMC_PKT_BUF_SZ; - sc->lmc_xinfo.link_status = sc->lmc_media->get_link_status (sc); - sc->lmc_xinfo.mii_reg16 = lmc_mii_readreg (sc, 0, 16); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - sc->lmc_xinfo.Magic1 = 0xDEADBEEF; - - if (copy_to_user(data, &sc->lmc_xinfo, sizeof(struct lmc_xinfo))) - ret = -EFAULT; - else - ret = 0; - - break; - - case LMCIOCGETLMCSTATS: - spin_lock_irqsave(&sc->lmc_lock, flags); - if (sc->lmc_cardtype == LMC_CARDTYPE_T1) { - lmc_mii_writereg(sc, 0, 17, T1FRAMER_FERR_LSB); - sc->extra_stats.framingBitErrorCount += - lmc_mii_readreg(sc, 0, 18) & 0xff; - lmc_mii_writereg(sc, 0, 17, T1FRAMER_FERR_MSB); - sc->extra_stats.framingBitErrorCount += - (lmc_mii_readreg(sc, 0, 18) & 0xff) << 8; - lmc_mii_writereg(sc, 0, 17, T1FRAMER_LCV_LSB); - sc->extra_stats.lineCodeViolationCount += - lmc_mii_readreg(sc, 0, 18) & 0xff; - lmc_mii_writereg(sc, 0, 17, T1FRAMER_LCV_MSB); - sc->extra_stats.lineCodeViolationCount += - (lmc_mii_readreg(sc, 0, 18) & 0xff) << 8; - lmc_mii_writereg(sc, 0, 17, T1FRAMER_AERR); - regVal = lmc_mii_readreg(sc, 0, 18) & 0xff; - - sc->extra_stats.lossOfFrameCount += - (regVal & T1FRAMER_LOF_MASK) >> 4; - sc->extra_stats.changeOfFrameAlignmentCount += - (regVal & T1FRAMER_COFA_MASK) >> 2; - sc->extra_stats.severelyErroredFrameCount += - regVal & T1FRAMER_SEF_MASK; - } - spin_unlock_irqrestore(&sc->lmc_lock, flags); - if (copy_to_user(data, &sc->lmc_device->stats, - sizeof(sc->lmc_device->stats)) || - copy_to_user(data + sizeof(sc->lmc_device->stats), - &sc->extra_stats, sizeof(sc->extra_stats))) - ret = -EFAULT; - else - ret = 0; - break; - - case LMCIOCCLEARLMCSTATS: - if (!capable(CAP_NET_ADMIN)) { - ret = -EPERM; - break; - } - - spin_lock_irqsave(&sc->lmc_lock, flags); - memset(&sc->lmc_device->stats, 0, sizeof(sc->lmc_device->stats)); - memset(&sc->extra_stats, 0, sizeof(sc->extra_stats)); - sc->extra_stats.check = STATCHECK; - sc->extra_stats.version_size = (DRIVER_VERSION << 16) + - sizeof(sc->lmc_device->stats) + sizeof(sc->extra_stats); - sc->extra_stats.lmc_cardtype = sc->lmc_cardtype; - spin_unlock_irqrestore(&sc->lmc_lock, flags); - ret = 0; - break; - - case LMCIOCSETCIRCUIT: /*fold01*/ - if (!capable(CAP_NET_ADMIN)){ - ret = -EPERM; - break; - } - - if(dev->flags & IFF_UP){ - ret = -EBUSY; - break; - } - - if (copy_from_user(&ctl, data, sizeof(lmc_ctl_t))) { - ret = -EFAULT; - break; - } - spin_lock_irqsave(&sc->lmc_lock, flags); - sc->lmc_media->set_circuit_type(sc, ctl.circuit_type); - sc->ictl.circuit_type = ctl.circuit_type; - spin_unlock_irqrestore(&sc->lmc_lock, flags); - ret = 0; - - break; - - case LMCIOCRESET: /*fold01*/ - if (!capable(CAP_NET_ADMIN)){ - ret = -EPERM; - break; - } - - spin_lock_irqsave(&sc->lmc_lock, flags); - /* Reset driver and bring back to current state */ - printk (" REG16 before reset +%04x\n", lmc_mii_readreg (sc, 0, 16)); - lmc_running_reset (dev); - printk (" REG16 after reset +%04x\n", lmc_mii_readreg (sc, 0, 16)); - - LMC_EVENT_LOG(LMC_EVENT_FORCEDRESET, LMC_CSR_READ (sc, csr_status), lmc_mii_readreg (sc, 0, 16)); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - ret = 0; - break; - -#ifdef DEBUG - case LMCIOCDUMPEVENTLOG: - if (copy_to_user(data, &lmcEventLogIndex, sizeof(u32))) { - ret = -EFAULT; - break; - } - if (copy_to_user(data + sizeof(u32), lmcEventLogBuf, - sizeof(lmcEventLogBuf))) - ret = -EFAULT; - else - ret = 0; - - break; -#endif /* end ifdef _DBG_EVENTLOG */ - case LMCIOCT1CONTROL: /*fold01*/ - if (sc->lmc_cardtype != LMC_CARDTYPE_T1){ - ret = -EOPNOTSUPP; - break; - } - break; - case LMCIOCXILINX: /*fold01*/ - { - struct lmc_xilinx_control xc; /*fold02*/ - - if (!capable(CAP_NET_ADMIN)){ - ret = -EPERM; - break; - } - - /* - * Stop the xwitter whlie we restart the hardware - */ - netif_stop_queue(dev); - - if (copy_from_user(&xc, data, sizeof(struct lmc_xilinx_control))) { - ret = -EFAULT; - break; - } - switch(xc.command){ - case lmc_xilinx_reset: /*fold02*/ - { - spin_lock_irqsave(&sc->lmc_lock, flags); - lmc_mii_readreg (sc, 0, 16); - - /* - * Make all of them 0 and make input - */ - lmc_gpio_mkinput(sc, 0xff); - - /* - * make the reset output - */ - lmc_gpio_mkoutput(sc, LMC_GEP_RESET); - - /* - * RESET low to force configuration. This also forces - * the transmitter clock to be internal, but we expect to reset - * that later anyway. - */ - - sc->lmc_gpio &= ~LMC_GEP_RESET; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - - /* - * hold for more than 10 microseconds - */ - udelay(50); - - sc->lmc_gpio |= LMC_GEP_RESET; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - - /* - * stop driving Xilinx-related signals - */ - lmc_gpio_mkinput(sc, 0xff); - - /* Reset the frammer hardware */ - sc->lmc_media->set_link_status (sc, 1); - sc->lmc_media->set_status (sc, NULL); -// lmc_softreset(sc); - - { - int i; - for(i = 0; i < 5; i++){ - lmc_led_on(sc, LMC_DS3_LED0); - mdelay(100); - lmc_led_off(sc, LMC_DS3_LED0); - lmc_led_on(sc, LMC_DS3_LED1); - mdelay(100); - lmc_led_off(sc, LMC_DS3_LED1); - lmc_led_on(sc, LMC_DS3_LED3); - mdelay(100); - lmc_led_off(sc, LMC_DS3_LED3); - lmc_led_on(sc, LMC_DS3_LED2); - mdelay(100); - lmc_led_off(sc, LMC_DS3_LED2); - } - } - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - - - ret = 0x0; - - } - - break; - case lmc_xilinx_load_prom: /*fold02*/ - { - int timeout = 500000; - spin_lock_irqsave(&sc->lmc_lock, flags); - lmc_mii_readreg (sc, 0, 16); - - /* - * Make all of them 0 and make input - */ - lmc_gpio_mkinput(sc, 0xff); - - /* - * make the reset output - */ - lmc_gpio_mkoutput(sc, LMC_GEP_DP | LMC_GEP_RESET); - - /* - * RESET low to force configuration. This also forces - * the transmitter clock to be internal, but we expect to reset - * that later anyway. - */ - - sc->lmc_gpio &= ~(LMC_GEP_RESET | LMC_GEP_DP); - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - - /* - * hold for more than 10 microseconds - */ - udelay(50); - - sc->lmc_gpio |= LMC_GEP_DP | LMC_GEP_RESET; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - /* - * busy wait for the chip to reset - */ - while( (LMC_CSR_READ(sc, csr_gp) & LMC_GEP_INIT) == 0 && - (timeout-- > 0)) - cpu_relax(); - - - /* - * stop driving Xilinx-related signals - */ - lmc_gpio_mkinput(sc, 0xff); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - ret = 0x0; - - - break; - - } - - case lmc_xilinx_load: /*fold02*/ - { - char *data; - int pos; - int timeout = 500000; - - if (!xc.data) { - ret = -EINVAL; - break; - } - - data = memdup_user(xc.data, xc.len); - if (IS_ERR(data)) { - ret = PTR_ERR(data); - break; - } - - printk("%s: Starting load of data Len: %d at 0x%p == 0x%p\n", dev->name, xc.len, xc.data, data); - - spin_lock_irqsave(&sc->lmc_lock, flags); - lmc_gpio_mkinput(sc, 0xff); - - /* - * Clear the Xilinx and start prgramming from the DEC - */ - - /* - * Set ouput as: - * Reset: 0 (active) - * DP: 0 (active) - * Mode: 1 - * - */ - sc->lmc_gpio = 0x00; - sc->lmc_gpio &= ~LMC_GEP_DP; - sc->lmc_gpio &= ~LMC_GEP_RESET; - sc->lmc_gpio |= LMC_GEP_MODE; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - lmc_gpio_mkoutput(sc, LMC_GEP_MODE | LMC_GEP_DP | LMC_GEP_RESET); - - /* - * Wait at least 10 us 20 to be safe - */ - udelay(50); - - /* - * Clear reset and activate programming lines - * Reset: Input - * DP: Input - * Clock: Output - * Data: Output - * Mode: Output - */ - lmc_gpio_mkinput(sc, LMC_GEP_DP | LMC_GEP_RESET); - - /* - * Set LOAD, DATA, Clock to 1 - */ - sc->lmc_gpio = 0x00; - sc->lmc_gpio |= LMC_GEP_MODE; - sc->lmc_gpio |= LMC_GEP_DATA; - sc->lmc_gpio |= LMC_GEP_CLK; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - lmc_gpio_mkoutput(sc, LMC_GEP_DATA | LMC_GEP_CLK | LMC_GEP_MODE ); - - /* - * busy wait for the chip to reset - */ - while( (LMC_CSR_READ(sc, csr_gp) & LMC_GEP_INIT) == 0 && - (timeout-- > 0)) - cpu_relax(); - - printk(KERN_DEBUG "%s: Waited %d for the Xilinx to clear its memory\n", dev->name, 500000-timeout); - - for(pos = 0; pos < xc.len; pos++){ - switch(data[pos]){ - case 0: - sc->lmc_gpio &= ~LMC_GEP_DATA; /* Data is 0 */ - break; - case 1: - sc->lmc_gpio |= LMC_GEP_DATA; /* Data is 1 */ - break; - default: - printk(KERN_WARNING "%s Bad data in xilinx programming data at %d, got %d wanted 0 or 1\n", dev->name, pos, data[pos]); - sc->lmc_gpio |= LMC_GEP_DATA; /* Assume it's 1 */ - } - sc->lmc_gpio &= ~LMC_GEP_CLK; /* Clock to zero */ - sc->lmc_gpio |= LMC_GEP_MODE; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - udelay(1); - - sc->lmc_gpio |= LMC_GEP_CLK; /* Put the clack back to one */ - sc->lmc_gpio |= LMC_GEP_MODE; - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - udelay(1); - } - if((LMC_CSR_READ(sc, csr_gp) & LMC_GEP_INIT) == 0){ - printk(KERN_WARNING "%s: Reprogramming FAILED. Needs to be reprogrammed. (corrupted data)\n", dev->name); - } - else if((LMC_CSR_READ(sc, csr_gp) & LMC_GEP_DP) == 0){ - printk(KERN_WARNING "%s: Reprogramming FAILED. Needs to be reprogrammed. (done)\n", dev->name); - } - else { - printk(KERN_DEBUG "%s: Done reprogramming Xilinx, %d bits, good luck!\n", dev->name, pos); - } - - lmc_gpio_mkinput(sc, 0xff); - - sc->lmc_miireg16 |= LMC_MII16_FIFO_RESET; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - - sc->lmc_miireg16 &= ~LMC_MII16_FIFO_RESET; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - kfree(data); - - ret = 0; - - break; - } - default: /*fold02*/ - ret = -EBADE; - break; - } - - netif_wake_queue(dev); - sc->lmc_txfull = 0; - - } - break; - default: - break; - } - - return ret; -} - - -/* the watchdog process that cruises around */ -static void lmc_watchdog(struct timer_list *t) /*fold00*/ -{ - lmc_softc_t *sc = from_timer(sc, t, timer); - struct net_device *dev = sc->lmc_device; - int link_status; - u32 ticks; - unsigned long flags; - - spin_lock_irqsave(&sc->lmc_lock, flags); - - if(sc->check != 0xBEAFCAFE){ - printk("LMC: Corrupt net_device struct, breaking out\n"); - spin_unlock_irqrestore(&sc->lmc_lock, flags); - return; - } - - - /* Make sure the tx jabber and rx watchdog are off, - * and the transmit and receive processes are running. - */ - - LMC_CSR_WRITE (sc, csr_15, 0x00000011); - sc->lmc_cmdmode |= TULIP_CMD_TXRUN | TULIP_CMD_RXRUN; - LMC_CSR_WRITE (sc, csr_command, sc->lmc_cmdmode); - - if (sc->lmc_ok == 0) - goto kick_timer; - - LMC_EVENT_LOG(LMC_EVENT_WATCHDOG, LMC_CSR_READ (sc, csr_status), lmc_mii_readreg (sc, 0, 16)); - - /* --- begin time out check ----------------------------------- - * check for a transmit interrupt timeout - * Has the packet xmt vs xmt serviced threshold been exceeded */ - if (sc->lmc_taint_tx == sc->lastlmc_taint_tx && - sc->lmc_device->stats.tx_packets > sc->lasttx_packets && - sc->tx_TimeoutInd == 0) - { - - /* wait for the watchdog to come around again */ - sc->tx_TimeoutInd = 1; - } - else if (sc->lmc_taint_tx == sc->lastlmc_taint_tx && - sc->lmc_device->stats.tx_packets > sc->lasttx_packets && - sc->tx_TimeoutInd) - { - - LMC_EVENT_LOG(LMC_EVENT_XMTINTTMO, LMC_CSR_READ (sc, csr_status), 0); - - sc->tx_TimeoutDisplay = 1; - sc->extra_stats.tx_TimeoutCnt++; - - /* DEC chip is stuck, hit it with a RESET!!!! */ - lmc_running_reset (dev); - - - /* look at receive & transmit process state to make sure they are running */ - LMC_EVENT_LOG(LMC_EVENT_RESET1, LMC_CSR_READ (sc, csr_status), 0); - - /* look at: DSR - 02 for Reg 16 - * CTS - 08 - * DCD - 10 - * RI - 20 - * for Reg 17 - */ - LMC_EVENT_LOG(LMC_EVENT_RESET2, lmc_mii_readreg (sc, 0, 16), lmc_mii_readreg (sc, 0, 17)); - - /* reset the transmit timeout detection flag */ - sc->tx_TimeoutInd = 0; - sc->lastlmc_taint_tx = sc->lmc_taint_tx; - sc->lasttx_packets = sc->lmc_device->stats.tx_packets; - } else { - sc->tx_TimeoutInd = 0; - sc->lastlmc_taint_tx = sc->lmc_taint_tx; - sc->lasttx_packets = sc->lmc_device->stats.tx_packets; - } - - /* --- end time out check ----------------------------------- */ - - - link_status = sc->lmc_media->get_link_status (sc); - - /* - * hardware level link lost, but the interface is marked as up. - * Mark it as down. - */ - if ((link_status == 0) && (sc->last_link_status != 0)) { - printk(KERN_WARNING "%s: hardware/physical link down\n", dev->name); - sc->last_link_status = 0; - /* lmc_reset (sc); Why reset??? The link can go down ok */ - - /* Inform the world that link has been lost */ - netif_carrier_off(dev); - } - - /* - * hardware link is up, but the interface is marked as down. - * Bring it back up again. - */ - if (link_status != 0 && sc->last_link_status == 0) { - printk(KERN_WARNING "%s: hardware/physical link up\n", dev->name); - sc->last_link_status = 1; - /* lmc_reset (sc); Again why reset??? */ - - netif_carrier_on(dev); - } - - /* Call media specific watchdog functions */ - sc->lmc_media->watchdog(sc); - - /* - * Poke the transmitter to make sure it - * never stops, even if we run out of mem - */ - LMC_CSR_WRITE(sc, csr_rxpoll, 0); - - /* - * Check for code that failed - * and try and fix it as appropriate - */ - if(sc->failed_ring == 1){ - /* - * Failed to setup the recv/xmit rin - * Try again - */ - sc->failed_ring = 0; - lmc_softreset(sc); - } - if(sc->failed_recv_alloc == 1){ - /* - * We failed to alloc mem in the - * interrupt handler, go through the rings - * and rebuild them - */ - sc->failed_recv_alloc = 0; - lmc_softreset(sc); - } - - - /* - * remember the timer value - */ -kick_timer: - - ticks = LMC_CSR_READ (sc, csr_gp_timer); - LMC_CSR_WRITE (sc, csr_gp_timer, 0xffffffffUL); - sc->ictl.ticks = 0x0000ffff - (ticks & 0x0000ffff); - - /* - * restart this timer. - */ - sc->timer.expires = jiffies + (HZ); - add_timer (&sc->timer); - - spin_unlock_irqrestore(&sc->lmc_lock, flags); -} - -static int lmc_attach(struct net_device *dev, unsigned short encoding, - unsigned short parity) -{ - if (encoding == ENCODING_NRZ && parity == PARITY_CRC16_PR1_CCITT) - return 0; - return -EINVAL; -} - -static const struct net_device_ops lmc_ops = { - .ndo_open = lmc_open, - .ndo_stop = lmc_close, - .ndo_start_xmit = hdlc_start_xmit, - .ndo_siocwandev = hdlc_ioctl, - .ndo_siocdevprivate = lmc_siocdevprivate, - .ndo_tx_timeout = lmc_driver_timeout, - .ndo_get_stats = lmc_get_stats, -}; - -static int lmc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - lmc_softc_t *sc; - struct net_device *dev; - u16 subdevice; - u16 AdapModelNum; - int err; - static int cards_found; - - err = pcim_enable_device(pdev); - if (err) { - printk(KERN_ERR "lmc: pci enable failed: %d\n", err); - return err; - } - - err = pci_request_regions(pdev, "lmc"); - if (err) { - printk(KERN_ERR "lmc: pci_request_region failed\n"); - return err; - } - - /* - * Allocate our own device structure - */ - sc = devm_kzalloc(&pdev->dev, sizeof(lmc_softc_t), GFP_KERNEL); - if (!sc) - return -ENOMEM; - - dev = alloc_hdlcdev(sc); - if (!dev) { - printk(KERN_ERR "lmc:alloc_netdev for device failed\n"); - return -ENOMEM; - } - - - dev->type = ARPHRD_HDLC; - dev_to_hdlc(dev)->xmit = lmc_start_xmit; - dev_to_hdlc(dev)->attach = lmc_attach; - dev->netdev_ops = &lmc_ops; - dev->watchdog_timeo = HZ; /* 1 second */ - dev->tx_queue_len = 100; - sc->lmc_device = dev; - sc->name = dev->name; - sc->if_type = LMC_PPP; - sc->check = 0xBEAFCAFE; - dev->base_addr = pci_resource_start(pdev, 0); - dev->irq = pdev->irq; - pci_set_drvdata(pdev, dev); - SET_NETDEV_DEV(dev, &pdev->dev); - - /* - * This will get the protocol layer ready and do any 1 time init's - * Must have a valid sc and dev structure - */ - lmc_proto_attach(sc); - - /* Init the spin lock so can call it latter */ - - spin_lock_init(&sc->lmc_lock); - pci_set_master(pdev); - - printk(KERN_INFO "hdlc: detected at %lx, irq %d\n", - dev->base_addr, dev->irq); - - err = register_hdlc_device(dev); - if (err) { - printk(KERN_ERR "%s: register_netdev failed.\n", dev->name); - free_netdev(dev); - return err; - } - - sc->lmc_cardtype = LMC_CARDTYPE_UNKNOWN; - sc->lmc_timing = LMC_CTL_CLOCK_SOURCE_EXT; - - /* - * - * Check either the subvendor or the subdevice, some systems reverse - * the setting in the bois, seems to be version and arch dependent? - * Fix the error, exchange the two values - */ - if ((subdevice = pdev->subsystem_device) == PCI_VENDOR_ID_LMC) - subdevice = pdev->subsystem_vendor; - - switch (subdevice) { - case PCI_DEVICE_ID_LMC_HSSI: - printk(KERN_INFO "%s: LMC HSSI\n", dev->name); - sc->lmc_cardtype = LMC_CARDTYPE_HSSI; - sc->lmc_media = &lmc_hssi_media; - break; - case PCI_DEVICE_ID_LMC_DS3: - printk(KERN_INFO "%s: LMC DS3\n", dev->name); - sc->lmc_cardtype = LMC_CARDTYPE_DS3; - sc->lmc_media = &lmc_ds3_media; - break; - case PCI_DEVICE_ID_LMC_SSI: - printk(KERN_INFO "%s: LMC SSI\n", dev->name); - sc->lmc_cardtype = LMC_CARDTYPE_SSI; - sc->lmc_media = &lmc_ssi_media; - break; - case PCI_DEVICE_ID_LMC_T1: - printk(KERN_INFO "%s: LMC T1\n", dev->name); - sc->lmc_cardtype = LMC_CARDTYPE_T1; - sc->lmc_media = &lmc_t1_media; - break; - default: - printk(KERN_WARNING "%s: LMC UNKNOWN CARD!\n", dev->name); - unregister_hdlc_device(dev); - return -EIO; - break; - } - - lmc_initcsrs (sc, dev->base_addr, 8); - - lmc_gpio_mkinput (sc, 0xff); - sc->lmc_gpio = 0; /* drive no signals yet */ - - sc->lmc_media->defaults (sc); - - sc->lmc_media->set_link_status (sc, LMC_LINK_UP); - - /* verify that the PCI Sub System ID matches the Adapter Model number - * from the MII register - */ - AdapModelNum = (lmc_mii_readreg (sc, 0, 3) & 0x3f0) >> 4; - - if ((AdapModelNum != LMC_ADAP_T1 || /* detect LMC1200 */ - subdevice != PCI_DEVICE_ID_LMC_T1) && - (AdapModelNum != LMC_ADAP_SSI || /* detect LMC1000 */ - subdevice != PCI_DEVICE_ID_LMC_SSI) && - (AdapModelNum != LMC_ADAP_DS3 || /* detect LMC5245 */ - subdevice != PCI_DEVICE_ID_LMC_DS3) && - (AdapModelNum != LMC_ADAP_HSSI || /* detect LMC5200 */ - subdevice != PCI_DEVICE_ID_LMC_HSSI)) - printk(KERN_WARNING "%s: Model number (%d) miscompare for PCI" - " Subsystem ID = 0x%04x\n", - dev->name, AdapModelNum, subdevice); - - /* - * reset clock - */ - LMC_CSR_WRITE (sc, csr_gp_timer, 0xFFFFFFFFUL); - - sc->board_idx = cards_found++; - sc->extra_stats.check = STATCHECK; - sc->extra_stats.version_size = (DRIVER_VERSION << 16) + - sizeof(sc->lmc_device->stats) + sizeof(sc->extra_stats); - sc->extra_stats.lmc_cardtype = sc->lmc_cardtype; - - sc->lmc_ok = 0; - sc->last_link_status = 0; - - return 0; -} - -/* - * Called from pci when removing module. - */ -static void lmc_remove_one(struct pci_dev *pdev) -{ - struct net_device *dev = pci_get_drvdata(pdev); - - if (dev) { - printk(KERN_DEBUG "%s: removing...\n", dev->name); - unregister_hdlc_device(dev); - free_netdev(dev); - } -} - -/* After this is called, packets can be sent. - * Does not initialize the addresses - */ -static int lmc_open(struct net_device *dev) -{ - lmc_softc_t *sc = dev_to_sc(dev); - int err; - - lmc_led_on(sc, LMC_DS3_LED0); - - lmc_dec_reset(sc); - lmc_reset(sc); - - LMC_EVENT_LOG(LMC_EVENT_RESET1, LMC_CSR_READ(sc, csr_status), 0); - LMC_EVENT_LOG(LMC_EVENT_RESET2, lmc_mii_readreg(sc, 0, 16), - lmc_mii_readreg(sc, 0, 17)); - - if (sc->lmc_ok) - return 0; - - lmc_softreset (sc); - - /* Since we have to use PCI bus, this should work on x86,alpha,ppc */ - if (request_irq (dev->irq, lmc_interrupt, IRQF_SHARED, dev->name, dev)){ - printk(KERN_WARNING "%s: could not get irq: %d\n", dev->name, dev->irq); - return -EAGAIN; - } - sc->got_irq = 1; - - /* Assert Terminal Active */ - sc->lmc_miireg16 |= LMC_MII16_LED_ALL; - sc->lmc_media->set_link_status (sc, LMC_LINK_UP); - - /* - * reset to last state. - */ - sc->lmc_media->set_status (sc, NULL); - - /* setup default bits to be used in tulip_desc_t transmit descriptor - * -baz */ - sc->TxDescriptControlInit = ( - LMC_TDES_INTERRUPT_ON_COMPLETION - | LMC_TDES_FIRST_SEGMENT - | LMC_TDES_LAST_SEGMENT - | LMC_TDES_SECOND_ADDR_CHAINED - | LMC_TDES_DISABLE_PADDING - ); - - if (sc->ictl.crc_length == LMC_CTL_CRC_LENGTH_16) { - /* disable 32 bit CRC generated by ASIC */ - sc->TxDescriptControlInit |= LMC_TDES_ADD_CRC_DISABLE; - } - sc->lmc_media->set_crc_length(sc, sc->ictl.crc_length); - /* Acknoledge the Terminal Active and light LEDs */ - - /* dev->flags |= IFF_UP; */ - - if ((err = lmc_proto_open(sc)) != 0) - return err; - - netif_start_queue(dev); - sc->extra_stats.tx_tbusy0++; - - /* - * select what interrupts we want to get - */ - sc->lmc_intrmask = 0; - /* Should be using the default interrupt mask defined in the .h file. */ - sc->lmc_intrmask |= (TULIP_STS_NORMALINTR - | TULIP_STS_RXINTR - | TULIP_STS_TXINTR - | TULIP_STS_ABNRMLINTR - | TULIP_STS_SYSERROR - | TULIP_STS_TXSTOPPED - | TULIP_STS_TXUNDERFLOW - | TULIP_STS_RXSTOPPED - | TULIP_STS_RXNOBUF - ); - LMC_CSR_WRITE (sc, csr_intr, sc->lmc_intrmask); - - sc->lmc_cmdmode |= TULIP_CMD_TXRUN; - sc->lmc_cmdmode |= TULIP_CMD_RXRUN; - LMC_CSR_WRITE (sc, csr_command, sc->lmc_cmdmode); - - sc->lmc_ok = 1; /* Run watchdog */ - - /* - * Set the if up now - pfb - */ - - sc->last_link_status = 1; - - /* - * Setup a timer for the watchdog on probe, and start it running. - * Since lmc_ok == 0, it will be a NOP for now. - */ - timer_setup(&sc->timer, lmc_watchdog, 0); - sc->timer.expires = jiffies + HZ; - add_timer (&sc->timer); - - return 0; -} - -/* Total reset to compensate for the AdTran DSU doing bad things - * under heavy load - */ - -static void lmc_running_reset (struct net_device *dev) /*fold00*/ -{ - lmc_softc_t *sc = dev_to_sc(dev); - - /* stop interrupts */ - /* Clear the interrupt mask */ - LMC_CSR_WRITE (sc, csr_intr, 0x00000000); - - lmc_dec_reset (sc); - lmc_reset (sc); - lmc_softreset (sc); - /* sc->lmc_miireg16 |= LMC_MII16_LED_ALL; */ - sc->lmc_media->set_link_status (sc, 1); - sc->lmc_media->set_status (sc, NULL); - - netif_wake_queue(dev); - - sc->lmc_txfull = 0; - sc->extra_stats.tx_tbusy0++; - - sc->lmc_intrmask = TULIP_DEFAULT_INTR_MASK; - LMC_CSR_WRITE (sc, csr_intr, sc->lmc_intrmask); - - sc->lmc_cmdmode |= (TULIP_CMD_TXRUN | TULIP_CMD_RXRUN); - LMC_CSR_WRITE (sc, csr_command, sc->lmc_cmdmode); -} - - -/* This is what is called when you ifconfig down a device. - * This disables the timer for the watchdog and keepalives, - * and disables the irq for dev. - */ -static int lmc_close(struct net_device *dev) -{ - /* not calling release_region() as we should */ - lmc_softc_t *sc = dev_to_sc(dev); - - sc->lmc_ok = 0; - sc->lmc_media->set_link_status (sc, 0); - del_timer (&sc->timer); - lmc_proto_close(sc); - lmc_ifdown (dev); - - return 0; -} - -/* Ends the transfer of packets */ -/* When the interface goes down, this is called */ -static int lmc_ifdown (struct net_device *dev) /*fold00*/ -{ - lmc_softc_t *sc = dev_to_sc(dev); - u32 csr6; - int i; - - /* Don't let anything else go on right now */ - // dev->start = 0; - netif_stop_queue(dev); - sc->extra_stats.tx_tbusy1++; - - /* stop interrupts */ - /* Clear the interrupt mask */ - LMC_CSR_WRITE (sc, csr_intr, 0x00000000); - - /* Stop Tx and Rx on the chip */ - csr6 = LMC_CSR_READ (sc, csr_command); - csr6 &= ~LMC_DEC_ST; /* Turn off the Transmission bit */ - csr6 &= ~LMC_DEC_SR; /* Turn off the Receive bit */ - LMC_CSR_WRITE (sc, csr_command, csr6); - - sc->lmc_device->stats.rx_missed_errors += - LMC_CSR_READ(sc, csr_missed_frames) & 0xffff; - - /* release the interrupt */ - if(sc->got_irq == 1){ - free_irq (dev->irq, dev); - sc->got_irq = 0; - } - - /* free skbuffs in the Rx queue */ - for (i = 0; i < LMC_RXDESCS; i++) - { - struct sk_buff *skb = sc->lmc_rxq[i]; - sc->lmc_rxq[i] = NULL; - sc->lmc_rxring[i].status = 0; - sc->lmc_rxring[i].length = 0; - sc->lmc_rxring[i].buffer1 = 0xDEADBEEF; - if (skb != NULL) - dev_kfree_skb(skb); - sc->lmc_rxq[i] = NULL; - } - - for (i = 0; i < LMC_TXDESCS; i++) - { - if (sc->lmc_txq[i] != NULL) - dev_kfree_skb(sc->lmc_txq[i]); - sc->lmc_txq[i] = NULL; - } - - lmc_led_off (sc, LMC_MII16_LED_ALL); - - netif_wake_queue(dev); - sc->extra_stats.tx_tbusy0++; - - return 0; -} - -/* Interrupt handling routine. This will take an incoming packet, or clean - * up after a trasmit. - */ -static irqreturn_t lmc_interrupt (int irq, void *dev_instance) /*fold00*/ -{ - struct net_device *dev = (struct net_device *) dev_instance; - lmc_softc_t *sc = dev_to_sc(dev); - u32 csr; - int i; - s32 stat; - unsigned int badtx; - int max_work = LMC_RXDESCS; - int handled = 0; - - spin_lock(&sc->lmc_lock); - - /* - * Read the csr to find what interrupts we have (if any) - */ - csr = LMC_CSR_READ (sc, csr_status); - - /* - * Make sure this is our interrupt - */ - if ( ! (csr & sc->lmc_intrmask)) { - goto lmc_int_fail_out; - } - - /* always go through this loop at least once */ - while (csr & sc->lmc_intrmask) { - handled = 1; - - /* - * Clear interrupt bits, we handle all case below - */ - LMC_CSR_WRITE (sc, csr_status, csr); - - /* - * One of - * - Transmit process timed out CSR5<1> - * - Transmit jabber timeout CSR5<3> - * - Transmit underflow CSR5<5> - * - Transmit Receiver buffer unavailable CSR5<7> - * - Receive process stopped CSR5<8> - * - Receive watchdog timeout CSR5<9> - * - Early transmit interrupt CSR5<10> - * - * Is this really right? Should we do a running reset for jabber? - * (being a WAN card and all) - */ - if (csr & TULIP_STS_ABNRMLINTR){ - lmc_running_reset (dev); - break; - } - - if (csr & TULIP_STS_RXINTR) - lmc_rx (dev); - - if (csr & (TULIP_STS_TXINTR | TULIP_STS_TXNOBUF | TULIP_STS_TXSTOPPED)) { - - int n_compl = 0 ; - /* reset the transmit timeout detection flag -baz */ - sc->extra_stats.tx_NoCompleteCnt = 0; - - badtx = sc->lmc_taint_tx; - i = badtx % LMC_TXDESCS; - - while ((badtx < sc->lmc_next_tx)) { - stat = sc->lmc_txring[i].status; - - LMC_EVENT_LOG (LMC_EVENT_XMTINT, stat, - sc->lmc_txring[i].length); - /* - * If bit 31 is 1 the tulip owns it break out of the loop - */ - if (stat & 0x80000000) - break; - - n_compl++ ; /* i.e., have an empty slot in ring */ - /* - * If we have no skbuff or have cleared it - * Already continue to the next buffer - */ - if (sc->lmc_txq[i] == NULL) - continue; - - /* - * Check the total error summary to look for any errors - */ - if (stat & 0x8000) { - sc->lmc_device->stats.tx_errors++; - if (stat & 0x4104) - sc->lmc_device->stats.tx_aborted_errors++; - if (stat & 0x0C00) - sc->lmc_device->stats.tx_carrier_errors++; - if (stat & 0x0200) - sc->lmc_device->stats.tx_window_errors++; - if (stat & 0x0002) - sc->lmc_device->stats.tx_fifo_errors++; - } else { - sc->lmc_device->stats.tx_bytes += sc->lmc_txring[i].length & 0x7ff; - - sc->lmc_device->stats.tx_packets++; - } - - dev_consume_skb_irq(sc->lmc_txq[i]); - sc->lmc_txq[i] = NULL; - - badtx++; - i = badtx % LMC_TXDESCS; - } - - if (sc->lmc_next_tx - badtx > LMC_TXDESCS) - { - printk ("%s: out of sync pointer\n", dev->name); - badtx += LMC_TXDESCS; - } - LMC_EVENT_LOG(LMC_EVENT_TBUSY0, n_compl, 0); - sc->lmc_txfull = 0; - netif_wake_queue(dev); - sc->extra_stats.tx_tbusy0++; - - -#ifdef DEBUG - sc->extra_stats.dirtyTx = badtx; - sc->extra_stats.lmc_next_tx = sc->lmc_next_tx; - sc->extra_stats.lmc_txfull = sc->lmc_txfull; -#endif - sc->lmc_taint_tx = badtx; - - /* - * Why was there a break here??? - */ - } /* end handle transmit interrupt */ - - if (csr & TULIP_STS_SYSERROR) { - u32 error; - printk (KERN_WARNING "%s: system bus error csr: %#8.8x\n", dev->name, csr); - error = csr>>23 & 0x7; - switch(error){ - case 0x000: - printk(KERN_WARNING "%s: Parity Fault (bad)\n", dev->name); - break; - case 0x001: - printk(KERN_WARNING "%s: Master Abort (naughty)\n", dev->name); - break; - case 0x002: - printk(KERN_WARNING "%s: Target Abort (not so naughty)\n", dev->name); - break; - default: - printk(KERN_WARNING "%s: This bus error code was supposed to be reserved!\n", dev->name); - } - lmc_dec_reset (sc); - lmc_reset (sc); - LMC_EVENT_LOG(LMC_EVENT_RESET1, LMC_CSR_READ (sc, csr_status), 0); - LMC_EVENT_LOG(LMC_EVENT_RESET2, - lmc_mii_readreg (sc, 0, 16), - lmc_mii_readreg (sc, 0, 17)); - - } - - - if(max_work-- <= 0) - break; - - /* - * Get current csr status to make sure - * we've cleared all interrupts - */ - csr = LMC_CSR_READ (sc, csr_status); - } /* end interrupt loop */ - LMC_EVENT_LOG(LMC_EVENT_INT, firstcsr, csr); - -lmc_int_fail_out: - - spin_unlock(&sc->lmc_lock); - - return IRQ_RETVAL(handled); -} - -static netdev_tx_t lmc_start_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - lmc_softc_t *sc = dev_to_sc(dev); - u32 flag; - int entry; - unsigned long flags; - - spin_lock_irqsave(&sc->lmc_lock, flags); - - /* normal path, tbusy known to be zero */ - - entry = sc->lmc_next_tx % LMC_TXDESCS; - - sc->lmc_txq[entry] = skb; - sc->lmc_txring[entry].buffer1 = virt_to_bus (skb->data); - - LMC_CONSOLE_LOG("xmit", skb->data, skb->len); - -#ifndef GCOM - /* If the queue is less than half full, don't interrupt */ - if (sc->lmc_next_tx - sc->lmc_taint_tx < LMC_TXDESCS / 2) - { - /* Do not interrupt on completion of this packet */ - flag = 0x60000000; - netif_wake_queue(dev); - } - else if (sc->lmc_next_tx - sc->lmc_taint_tx == LMC_TXDESCS / 2) - { - /* This generates an interrupt on completion of this packet */ - flag = 0xe0000000; - netif_wake_queue(dev); - } - else if (sc->lmc_next_tx - sc->lmc_taint_tx < LMC_TXDESCS - 1) - { - /* Do not interrupt on completion of this packet */ - flag = 0x60000000; - netif_wake_queue(dev); - } - else - { - /* This generates an interrupt on completion of this packet */ - flag = 0xe0000000; - sc->lmc_txfull = 1; - netif_stop_queue(dev); - } -#else - flag = LMC_TDES_INTERRUPT_ON_COMPLETION; - - if (sc->lmc_next_tx - sc->lmc_taint_tx >= LMC_TXDESCS - 1) - { /* ring full, go busy */ - sc->lmc_txfull = 1; - netif_stop_queue(dev); - sc->extra_stats.tx_tbusy1++; - LMC_EVENT_LOG(LMC_EVENT_TBUSY1, entry, 0); - } -#endif - - - if (entry == LMC_TXDESCS - 1) /* last descriptor in ring */ - flag |= LMC_TDES_END_OF_RING; /* flag as such for Tulip */ - - /* don't pad small packets either */ - flag = sc->lmc_txring[entry].length = (skb->len) | flag | - sc->TxDescriptControlInit; - - /* set the transmit timeout flag to be checked in - * the watchdog timer handler. -baz - */ - - sc->extra_stats.tx_NoCompleteCnt++; - sc->lmc_next_tx++; - - /* give ownership to the chip */ - LMC_EVENT_LOG(LMC_EVENT_XMT, flag, entry); - sc->lmc_txring[entry].status = 0x80000000; - - /* send now! */ - LMC_CSR_WRITE (sc, csr_txpoll, 0); - - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - return NETDEV_TX_OK; -} - - -static int lmc_rx(struct net_device *dev) -{ - lmc_softc_t *sc = dev_to_sc(dev); - int i; - int rx_work_limit = LMC_RXDESCS; - int rxIntLoopCnt; /* debug -baz */ - int localLengthErrCnt = 0; - long stat; - struct sk_buff *skb, *nsb; - u16 len; - - lmc_led_on(sc, LMC_DS3_LED3); - - rxIntLoopCnt = 0; /* debug -baz */ - - i = sc->lmc_next_rx % LMC_RXDESCS; - - while (((stat = sc->lmc_rxring[i].status) & LMC_RDES_OWN_BIT) != DESC_OWNED_BY_DC21X4) - { - rxIntLoopCnt++; /* debug -baz */ - len = ((stat & LMC_RDES_FRAME_LENGTH) >> RDES_FRAME_LENGTH_BIT_NUMBER); - if ((stat & 0x0300) != 0x0300) { /* Check first segment and last segment */ - if ((stat & 0x0000ffff) != 0x7fff) { - /* Oversized frame */ - sc->lmc_device->stats.rx_length_errors++; - goto skip_packet; - } - } - - if (stat & 0x00000008) { /* Catch a dribbling bit error */ - sc->lmc_device->stats.rx_errors++; - sc->lmc_device->stats.rx_frame_errors++; - goto skip_packet; - } - - - if (stat & 0x00000004) { /* Catch a CRC error by the Xilinx */ - sc->lmc_device->stats.rx_errors++; - sc->lmc_device->stats.rx_crc_errors++; - goto skip_packet; - } - - if (len > LMC_PKT_BUF_SZ) { - sc->lmc_device->stats.rx_length_errors++; - localLengthErrCnt++; - goto skip_packet; - } - - if (len < sc->lmc_crcSize + 2) { - sc->lmc_device->stats.rx_length_errors++; - sc->extra_stats.rx_SmallPktCnt++; - localLengthErrCnt++; - goto skip_packet; - } - - if(stat & 0x00004000){ - printk(KERN_WARNING "%s: Receiver descriptor error, receiver out of sync?\n", dev->name); - } - - len -= sc->lmc_crcSize; - - skb = sc->lmc_rxq[i]; - - /* - * We ran out of memory at some point - * just allocate an skb buff and continue. - */ - - if (!skb) { - nsb = dev_alloc_skb (LMC_PKT_BUF_SZ + 2); - if (nsb) { - sc->lmc_rxq[i] = nsb; - nsb->dev = dev; - sc->lmc_rxring[i].buffer1 = virt_to_bus(skb_tail_pointer(nsb)); - } - sc->failed_recv_alloc = 1; - goto skip_packet; - } - - sc->lmc_device->stats.rx_packets++; - sc->lmc_device->stats.rx_bytes += len; - - LMC_CONSOLE_LOG("recv", skb->data, len); - - /* - * I'm not sure of the sanity of this - * Packets could be arriving at a constant - * 44.210mbits/sec and we're going to copy - * them into a new buffer?? - */ - - if(len > (LMC_MTU - (LMC_MTU>>2))){ /* len > LMC_MTU * 0.75 */ - /* - * If it's a large packet don't copy it just hand it up - */ - give_it_anyways: - - sc->lmc_rxq[i] = NULL; - sc->lmc_rxring[i].buffer1 = 0x0; - - skb_put (skb, len); - skb->protocol = lmc_proto_type(sc, skb); - skb_reset_mac_header(skb); - /* skb_reset_network_header(skb); */ - skb->dev = dev; - lmc_proto_netif(sc, skb); - - /* - * This skb will be destroyed by the upper layers, make a new one - */ - nsb = dev_alloc_skb (LMC_PKT_BUF_SZ + 2); - if (nsb) { - sc->lmc_rxq[i] = nsb; - nsb->dev = dev; - sc->lmc_rxring[i].buffer1 = virt_to_bus(skb_tail_pointer(nsb)); - /* Transferred to 21140 below */ - } - else { - /* - * We've run out of memory, stop trying to allocate - * memory and exit the interrupt handler - * - * The chip may run out of receivers and stop - * in which care we'll try to allocate the buffer - * again. (once a second) - */ - sc->extra_stats.rx_BuffAllocErr++; - LMC_EVENT_LOG(LMC_EVENT_RCVINT, stat, len); - sc->failed_recv_alloc = 1; - goto skip_out_of_mem; - } - } - else { - nsb = dev_alloc_skb(len); - if(!nsb) { - goto give_it_anyways; - } - skb_copy_from_linear_data(skb, skb_put(nsb, len), len); - - nsb->protocol = lmc_proto_type(sc, nsb); - skb_reset_mac_header(nsb); - /* skb_reset_network_header(nsb); */ - nsb->dev = dev; - lmc_proto_netif(sc, nsb); - } - - skip_packet: - LMC_EVENT_LOG(LMC_EVENT_RCVINT, stat, len); - sc->lmc_rxring[i].status = DESC_OWNED_BY_DC21X4; - - sc->lmc_next_rx++; - i = sc->lmc_next_rx % LMC_RXDESCS; - rx_work_limit--; - if (rx_work_limit < 0) - break; - } - - /* detect condition for LMC1000 where DSU cable attaches and fills - * descriptors with bogus packets - * - if (localLengthErrCnt > LMC_RXDESCS - 3) { - sc->extra_stats.rx_BadPktSurgeCnt++; - LMC_EVENT_LOG(LMC_EVENT_BADPKTSURGE, localLengthErrCnt, - sc->extra_stats.rx_BadPktSurgeCnt); - } */ - - /* save max count of receive descriptors serviced */ - if (rxIntLoopCnt > sc->extra_stats.rxIntLoopCnt) - sc->extra_stats.rxIntLoopCnt = rxIntLoopCnt; /* debug -baz */ - -#ifdef DEBUG - if (rxIntLoopCnt == 0) - { - for (i = 0; i < LMC_RXDESCS; i++) - { - if ((sc->lmc_rxring[i].status & LMC_RDES_OWN_BIT) - != DESC_OWNED_BY_DC21X4) - { - rxIntLoopCnt++; - } - } - LMC_EVENT_LOG(LMC_EVENT_RCVEND, rxIntLoopCnt, 0); - } -#endif - - - lmc_led_off(sc, LMC_DS3_LED3); - -skip_out_of_mem: - return 0; -} - -static struct net_device_stats *lmc_get_stats(struct net_device *dev) -{ - lmc_softc_t *sc = dev_to_sc(dev); - unsigned long flags; - - spin_lock_irqsave(&sc->lmc_lock, flags); - - sc->lmc_device->stats.rx_missed_errors += LMC_CSR_READ(sc, csr_missed_frames) & 0xffff; - - spin_unlock_irqrestore(&sc->lmc_lock, flags); - - return &sc->lmc_device->stats; -} - -static struct pci_driver lmc_driver = { - .name = "lmc", - .id_table = lmc_pci_tbl, - .probe = lmc_init_one, - .remove = lmc_remove_one, -}; - -module_pci_driver(lmc_driver); - -unsigned lmc_mii_readreg (lmc_softc_t * const sc, unsigned devaddr, unsigned regno) /*fold00*/ -{ - int i; - int command = (0xf6 << 10) | (devaddr << 5) | regno; - int retval = 0; - - LMC_MII_SYNC (sc); - - for (i = 15; i >= 0; i--) - { - int dataval = (command & (1 << i)) ? 0x20000 : 0; - - LMC_CSR_WRITE (sc, csr_9, dataval); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - LMC_CSR_WRITE (sc, csr_9, dataval | 0x10000); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - } - - for (i = 19; i > 0; i--) - { - LMC_CSR_WRITE (sc, csr_9, 0x40000); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - retval = (retval << 1) | ((LMC_CSR_READ (sc, csr_9) & 0x80000) ? 1 : 0); - LMC_CSR_WRITE (sc, csr_9, 0x40000 | 0x10000); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - } - - return (retval >> 1) & 0xffff; -} - -void lmc_mii_writereg (lmc_softc_t * const sc, unsigned devaddr, unsigned regno, unsigned data) /*fold00*/ -{ - int i = 32; - int command = (0x5002 << 16) | (devaddr << 23) | (regno << 18) | data; - - LMC_MII_SYNC (sc); - - i = 31; - while (i >= 0) - { - int datav; - - if (command & (1 << i)) - datav = 0x20000; - else - datav = 0x00000; - - LMC_CSR_WRITE (sc, csr_9, datav); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - LMC_CSR_WRITE (sc, csr_9, (datav | 0x10000)); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - i--; - } - - i = 2; - while (i > 0) - { - LMC_CSR_WRITE (sc, csr_9, 0x40000); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - LMC_CSR_WRITE (sc, csr_9, 0x50000); - lmc_delay (); - /* __SLOW_DOWN_IO; */ - i--; - } -} - -static void lmc_softreset (lmc_softc_t * const sc) /*fold00*/ -{ - int i; - - /* Initialize the receive rings and buffers. */ - sc->lmc_txfull = 0; - sc->lmc_next_rx = 0; - sc->lmc_next_tx = 0; - sc->lmc_taint_rx = 0; - sc->lmc_taint_tx = 0; - - /* - * Setup each one of the receiver buffers - * allocate an skbuff for each one, setup the descriptor table - * and point each buffer at the next one - */ - - for (i = 0; i < LMC_RXDESCS; i++) - { - struct sk_buff *skb; - - if (sc->lmc_rxq[i] == NULL) - { - skb = dev_alloc_skb (LMC_PKT_BUF_SZ + 2); - if(skb == NULL){ - printk(KERN_WARNING "%s: Failed to allocate receiver ring, will try again\n", sc->name); - sc->failed_ring = 1; - break; - } - else{ - sc->lmc_rxq[i] = skb; - } - } - else - { - skb = sc->lmc_rxq[i]; - } - - skb->dev = sc->lmc_device; - - /* owned by 21140 */ - sc->lmc_rxring[i].status = 0x80000000; - - /* used to be PKT_BUF_SZ now uses skb since we lose some to head room */ - sc->lmc_rxring[i].length = skb_tailroom(skb); - - /* use to be tail which is dumb since you're thinking why write - * to the end of the packj,et but since there's nothing there tail == data - */ - sc->lmc_rxring[i].buffer1 = virt_to_bus (skb->data); - - /* This is fair since the structure is static and we have the next address */ - sc->lmc_rxring[i].buffer2 = virt_to_bus (&sc->lmc_rxring[i + 1]); - - } - - /* - * Sets end of ring - */ - if (i != 0) { - sc->lmc_rxring[i - 1].length |= 0x02000000; /* Set end of buffers flag */ - sc->lmc_rxring[i - 1].buffer2 = virt_to_bus(&sc->lmc_rxring[0]); /* Point back to the start */ - } - LMC_CSR_WRITE (sc, csr_rxlist, virt_to_bus (sc->lmc_rxring)); /* write base address */ - - /* Initialize the transmit rings and buffers */ - for (i = 0; i < LMC_TXDESCS; i++) - { - if (sc->lmc_txq[i] != NULL){ /* have buffer */ - dev_kfree_skb(sc->lmc_txq[i]); /* free it */ - sc->lmc_device->stats.tx_dropped++; /* We just dropped a packet */ - } - sc->lmc_txq[i] = NULL; - sc->lmc_txring[i].status = 0x00000000; - sc->lmc_txring[i].buffer2 = virt_to_bus (&sc->lmc_txring[i + 1]); - } - sc->lmc_txring[i - 1].buffer2 = virt_to_bus (&sc->lmc_txring[0]); - LMC_CSR_WRITE (sc, csr_txlist, virt_to_bus (sc->lmc_txring)); -} - -void lmc_gpio_mkinput(lmc_softc_t * const sc, u32 bits) /*fold00*/ -{ - sc->lmc_gpio_io &= ~bits; - LMC_CSR_WRITE(sc, csr_gp, TULIP_GP_PINSET | (sc->lmc_gpio_io)); -} - -void lmc_gpio_mkoutput(lmc_softc_t * const sc, u32 bits) /*fold00*/ -{ - sc->lmc_gpio_io |= bits; - LMC_CSR_WRITE(sc, csr_gp, TULIP_GP_PINSET | (sc->lmc_gpio_io)); -} - -void lmc_led_on(lmc_softc_t * const sc, u32 led) /*fold00*/ -{ - if ((~sc->lmc_miireg16) & led) /* Already on! */ - return; - - sc->lmc_miireg16 &= ~led; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); -} - -void lmc_led_off(lmc_softc_t * const sc, u32 led) /*fold00*/ -{ - if (sc->lmc_miireg16 & led) /* Already set don't do anything */ - return; - - sc->lmc_miireg16 |= led; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); -} - -static void lmc_reset(lmc_softc_t * const sc) /*fold00*/ -{ - sc->lmc_miireg16 |= LMC_MII16_FIFO_RESET; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - - sc->lmc_miireg16 &= ~LMC_MII16_FIFO_RESET; - lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - - /* - * make some of the GPIO pins be outputs - */ - lmc_gpio_mkoutput(sc, LMC_GEP_RESET); - - /* - * RESET low to force state reset. This also forces - * the transmitter clock to be internal, but we expect to reset - * that later anyway. - */ - sc->lmc_gpio &= ~(LMC_GEP_RESET); - LMC_CSR_WRITE(sc, csr_gp, sc->lmc_gpio); - - /* - * hold for more than 10 microseconds - */ - udelay(50); - - /* - * stop driving Xilinx-related signals - */ - lmc_gpio_mkinput(sc, LMC_GEP_RESET); - - /* - * Call media specific init routine - */ - sc->lmc_media->init(sc); - - sc->extra_stats.resetCount++; -} - -static void lmc_dec_reset(lmc_softc_t * const sc) /*fold00*/ -{ - u32 val; - - /* - * disable all interrupts - */ - sc->lmc_intrmask = 0; - LMC_CSR_WRITE(sc, csr_intr, sc->lmc_intrmask); - - /* - * Reset the chip with a software reset command. - * Wait 10 microseconds (actually 50 PCI cycles but at - * 33MHz that comes to two microseconds but wait a - * bit longer anyways) - */ - LMC_CSR_WRITE(sc, csr_busmode, TULIP_BUSMODE_SWRESET); - udelay(25); -#ifdef __sparc__ - sc->lmc_busmode = LMC_CSR_READ(sc, csr_busmode); - sc->lmc_busmode = 0x00100000; - sc->lmc_busmode &= ~TULIP_BUSMODE_SWRESET; - LMC_CSR_WRITE(sc, csr_busmode, sc->lmc_busmode); -#endif - sc->lmc_cmdmode = LMC_CSR_READ(sc, csr_command); - - /* - * We want: - * no ethernet address in frames we write - * disable padding (txdesc, padding disable) - * ignore runt frames (rdes0 bit 15) - * no receiver watchdog or transmitter jabber timer - * (csr15 bit 0,14 == 1) - * if using 16-bit CRC, turn off CRC (trans desc, crc disable) - */ - - sc->lmc_cmdmode |= ( TULIP_CMD_PROMISCUOUS - | TULIP_CMD_FULLDUPLEX - | TULIP_CMD_PASSBADPKT - | TULIP_CMD_NOHEARTBEAT - | TULIP_CMD_PORTSELECT - | TULIP_CMD_RECEIVEALL - | TULIP_CMD_MUSTBEONE - ); - sc->lmc_cmdmode &= ~( TULIP_CMD_OPERMODE - | TULIP_CMD_THRESHOLDCTL - | TULIP_CMD_STOREFWD - | TULIP_CMD_TXTHRSHLDCTL - ); - - LMC_CSR_WRITE(sc, csr_command, sc->lmc_cmdmode); - - /* - * disable receiver watchdog and transmit jabber - */ - val = LMC_CSR_READ(sc, csr_sia_general); - val |= (TULIP_WATCHDOG_TXDISABLE | TULIP_WATCHDOG_RXDISABLE); - LMC_CSR_WRITE(sc, csr_sia_general, val); -} - -static void lmc_initcsrs(lmc_softc_t * const sc, lmc_csrptr_t csr_base, /*fold00*/ - size_t csr_size) -{ - sc->lmc_csrs.csr_busmode = csr_base + 0 * csr_size; - sc->lmc_csrs.csr_txpoll = csr_base + 1 * csr_size; - sc->lmc_csrs.csr_rxpoll = csr_base + 2 * csr_size; - sc->lmc_csrs.csr_rxlist = csr_base + 3 * csr_size; - sc->lmc_csrs.csr_txlist = csr_base + 4 * csr_size; - sc->lmc_csrs.csr_status = csr_base + 5 * csr_size; - sc->lmc_csrs.csr_command = csr_base + 6 * csr_size; - sc->lmc_csrs.csr_intr = csr_base + 7 * csr_size; - sc->lmc_csrs.csr_missed_frames = csr_base + 8 * csr_size; - sc->lmc_csrs.csr_9 = csr_base + 9 * csr_size; - sc->lmc_csrs.csr_10 = csr_base + 10 * csr_size; - sc->lmc_csrs.csr_11 = csr_base + 11 * csr_size; - sc->lmc_csrs.csr_12 = csr_base + 12 * csr_size; - sc->lmc_csrs.csr_13 = csr_base + 13 * csr_size; - sc->lmc_csrs.csr_14 = csr_base + 14 * csr_size; - sc->lmc_csrs.csr_15 = csr_base + 15 * csr_size; -} - -static void lmc_driver_timeout(struct net_device *dev, unsigned int txqueue) -{ - lmc_softc_t *sc = dev_to_sc(dev); - u32 csr6; - unsigned long flags; - - spin_lock_irqsave(&sc->lmc_lock, flags); - - printk("%s: Xmitter busy|\n", dev->name); - - sc->extra_stats.tx_tbusy_calls++; - if (time_is_before_jiffies(dev_trans_start(dev) + TX_TIMEOUT)) - goto bug_out; - - /* - * Chip seems to have locked up - * Reset it - * This whips out all our descriptor - * table and starts from scartch - */ - - LMC_EVENT_LOG(LMC_EVENT_XMTPRCTMO, - LMC_CSR_READ (sc, csr_status), - sc->extra_stats.tx_ProcTimeout); - - lmc_running_reset (dev); - - LMC_EVENT_LOG(LMC_EVENT_RESET1, LMC_CSR_READ (sc, csr_status), 0); - LMC_EVENT_LOG(LMC_EVENT_RESET2, - lmc_mii_readreg (sc, 0, 16), - lmc_mii_readreg (sc, 0, 17)); - - /* restart the tx processes */ - csr6 = LMC_CSR_READ (sc, csr_command); - LMC_CSR_WRITE (sc, csr_command, csr6 | 0x0002); - LMC_CSR_WRITE (sc, csr_command, csr6 | 0x2002); - - /* immediate transmit */ - LMC_CSR_WRITE (sc, csr_txpoll, 0); - - sc->lmc_device->stats.tx_errors++; - sc->extra_stats.tx_ProcTimeout++; /* -baz */ - - netif_trans_update(dev); /* prevent tx timeout */ - -bug_out: - - spin_unlock_irqrestore(&sc->lmc_lock, flags); -} diff --git a/drivers/net/wan/lmc/lmc_media.c b/drivers/net/wan/lmc/lmc_media.c deleted file mode 100644 index ec1ac7b1f3fd..000000000000 --- a/drivers/net/wan/lmc/lmc_media.c +++ /dev/null @@ -1,1206 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* $Id: lmc_media.c,v 1.13 2000/04/11 05:25:26 asj Exp $ */ - -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/timer.h> -#include <linux/ptrace.h> -#include <linux/errno.h> -#include <linux/ioport.h> -#include <linux/interrupt.h> -#include <linux/in.h> -#include <linux/if_arp.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/skbuff.h> -#include <linux/inet.h> -#include <linux/bitops.h> - -#include <asm/processor.h> /* Processor type for cache alignment. */ -#include <asm/io.h> -#include <asm/dma.h> - -#include <linux/uaccess.h> - -#include "lmc.h" -#include "lmc_var.h" -#include "lmc_ioctl.h" -#include "lmc_debug.h" - -#define CONFIG_LMC_IGNORE_HARDWARE_HANDSHAKE 1 - - /* - * Copyright (c) 1997-2000 LAN Media Corporation (LMC) - * All rights reserved. www.lanmedia.com - * - * This code is written by: - * Andrew Stanley-Jones (asj@cban.com) - * Rob Braun (bbraun@vix.com), - * Michael Graff (explorer@vix.com) and - * Matt Thomas (matt@3am-software.com). - */ - -/* - * protocol independent method. - */ -static void lmc_set_protocol (lmc_softc_t * const, lmc_ctl_t *); - -/* - * media independent methods to check on media status, link, light LEDs, - * etc. - */ -static void lmc_ds3_init (lmc_softc_t * const); -static void lmc_ds3_default (lmc_softc_t * const); -static void lmc_ds3_set_status (lmc_softc_t * const, lmc_ctl_t *); -static void lmc_ds3_set_100ft (lmc_softc_t * const, int); -static int lmc_ds3_get_link_status (lmc_softc_t * const); -static void lmc_ds3_set_crc_length (lmc_softc_t * const, int); -static void lmc_ds3_set_scram (lmc_softc_t * const, int); -static void lmc_ds3_watchdog (lmc_softc_t * const); - -static void lmc_hssi_init (lmc_softc_t * const); -static void lmc_hssi_default (lmc_softc_t * const); -static void lmc_hssi_set_status (lmc_softc_t * const, lmc_ctl_t *); -static void lmc_hssi_set_clock (lmc_softc_t * const, int); -static int lmc_hssi_get_link_status (lmc_softc_t * const); -static void lmc_hssi_set_link_status (lmc_softc_t * const, int); -static void lmc_hssi_set_crc_length (lmc_softc_t * const, int); -static void lmc_hssi_watchdog (lmc_softc_t * const); - -static void lmc_ssi_init (lmc_softc_t * const); -static void lmc_ssi_default (lmc_softc_t * const); -static void lmc_ssi_set_status (lmc_softc_t * const, lmc_ctl_t *); -static void lmc_ssi_set_clock (lmc_softc_t * const, int); -static void lmc_ssi_set_speed (lmc_softc_t * const, lmc_ctl_t *); -static int lmc_ssi_get_link_status (lmc_softc_t * const); -static void lmc_ssi_set_link_status (lmc_softc_t * const, int); -static void lmc_ssi_set_crc_length (lmc_softc_t * const, int); -static void lmc_ssi_watchdog (lmc_softc_t * const); - -static void lmc_t1_init (lmc_softc_t * const); -static void lmc_t1_default (lmc_softc_t * const); -static void lmc_t1_set_status (lmc_softc_t * const, lmc_ctl_t *); -static int lmc_t1_get_link_status (lmc_softc_t * const); -static void lmc_t1_set_circuit_type (lmc_softc_t * const, int); -static void lmc_t1_set_crc_length (lmc_softc_t * const, int); -static void lmc_t1_set_clock (lmc_softc_t * const, int); -static void lmc_t1_watchdog (lmc_softc_t * const); - -static void lmc_dummy_set_1 (lmc_softc_t * const, int); -static void lmc_dummy_set2_1 (lmc_softc_t * const, lmc_ctl_t *); - -static inline void write_av9110_bit (lmc_softc_t *, int); -static void write_av9110(lmc_softc_t *, u32, u32, u32, u32, u32); - -lmc_media_t lmc_ds3_media = { - .init = lmc_ds3_init, /* special media init stuff */ - .defaults = lmc_ds3_default, /* reset to default state */ - .set_status = lmc_ds3_set_status, /* reset status to state provided */ - .set_clock_source = lmc_dummy_set_1, /* set clock source */ - .set_speed = lmc_dummy_set2_1, /* set line speed */ - .set_cable_length = lmc_ds3_set_100ft, /* set cable length */ - .set_scrambler = lmc_ds3_set_scram, /* set scrambler */ - .get_link_status = lmc_ds3_get_link_status, /* get link status */ - .set_link_status = lmc_dummy_set_1, /* set link status */ - .set_crc_length = lmc_ds3_set_crc_length, /* set CRC length */ - .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */ - .watchdog = lmc_ds3_watchdog -}; - -lmc_media_t lmc_hssi_media = { - .init = lmc_hssi_init, /* special media init stuff */ - .defaults = lmc_hssi_default, /* reset to default state */ - .set_status = lmc_hssi_set_status, /* reset status to state provided */ - .set_clock_source = lmc_hssi_set_clock, /* set clock source */ - .set_speed = lmc_dummy_set2_1, /* set line speed */ - .set_cable_length = lmc_dummy_set_1, /* set cable length */ - .set_scrambler = lmc_dummy_set_1, /* set scrambler */ - .get_link_status = lmc_hssi_get_link_status, /* get link status */ - .set_link_status = lmc_hssi_set_link_status, /* set link status */ - .set_crc_length = lmc_hssi_set_crc_length, /* set CRC length */ - .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */ - .watchdog = lmc_hssi_watchdog -}; - -lmc_media_t lmc_ssi_media = { - .init = lmc_ssi_init, /* special media init stuff */ - .defaults = lmc_ssi_default, /* reset to default state */ - .set_status = lmc_ssi_set_status, /* reset status to state provided */ - .set_clock_source = lmc_ssi_set_clock, /* set clock source */ - .set_speed = lmc_ssi_set_speed, /* set line speed */ - .set_cable_length = lmc_dummy_set_1, /* set cable length */ - .set_scrambler = lmc_dummy_set_1, /* set scrambler */ - .get_link_status = lmc_ssi_get_link_status, /* get link status */ - .set_link_status = lmc_ssi_set_link_status, /* set link status */ - .set_crc_length = lmc_ssi_set_crc_length, /* set CRC length */ - .set_circuit_type = lmc_dummy_set_1, /* set T1 or E1 circuit type */ - .watchdog = lmc_ssi_watchdog -}; - -lmc_media_t lmc_t1_media = { - .init = lmc_t1_init, /* special media init stuff */ - .defaults = lmc_t1_default, /* reset to default state */ - .set_status = lmc_t1_set_status, /* reset status to state provided */ - .set_clock_source = lmc_t1_set_clock, /* set clock source */ - .set_speed = lmc_dummy_set2_1, /* set line speed */ - .set_cable_length = lmc_dummy_set_1, /* set cable length */ - .set_scrambler = lmc_dummy_set_1, /* set scrambler */ - .get_link_status = lmc_t1_get_link_status, /* get link status */ - .set_link_status = lmc_dummy_set_1, /* set link status */ - .set_crc_length = lmc_t1_set_crc_length, /* set CRC length */ - .set_circuit_type = lmc_t1_set_circuit_type, /* set T1 or E1 circuit type */ - .watchdog = lmc_t1_watchdog -}; - -static void -lmc_dummy_set_1 (lmc_softc_t * const sc, int a) -{ -} - -static void -lmc_dummy_set2_1 (lmc_softc_t * const sc, lmc_ctl_t * a) -{ -} - -/* - * HSSI methods - */ - -static void -lmc_hssi_init (lmc_softc_t * const sc) -{ - sc->ictl.cardtype = LMC_CTL_CARDTYPE_LMC5200; - - lmc_gpio_mkoutput (sc, LMC_GEP_HSSI_CLOCK); -} - -static void -lmc_hssi_default (lmc_softc_t * const sc) -{ - sc->lmc_miireg16 = LMC_MII16_LED_ALL; - - sc->lmc_media->set_link_status (sc, LMC_LINK_DOWN); - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_EXT); - sc->lmc_media->set_crc_length (sc, LMC_CTL_CRC_LENGTH_16); -} - -/* - * Given a user provided state, set ourselves up to match it. This will - * always reset the card if needed. - */ -static void -lmc_hssi_set_status (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - if (ctl == NULL) - { - sc->lmc_media->set_clock_source (sc, sc->ictl.clock_source); - lmc_set_protocol (sc, NULL); - - return; - } - - /* - * check for change in clock source - */ - if (ctl->clock_source && !sc->ictl.clock_source) - { - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_INT); - sc->lmc_timing = LMC_CTL_CLOCK_SOURCE_INT; - } - else if (!ctl->clock_source && sc->ictl.clock_source) - { - sc->lmc_timing = LMC_CTL_CLOCK_SOURCE_EXT; - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_EXT); - } - - lmc_set_protocol (sc, ctl); -} - -/* - * 1 == internal, 0 == external - */ -static void -lmc_hssi_set_clock (lmc_softc_t * const sc, int ie) -{ - int old; - old = sc->ictl.clock_source; - if (ie == LMC_CTL_CLOCK_SOURCE_EXT) - { - sc->lmc_gpio |= LMC_GEP_HSSI_CLOCK; - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_EXT; - if(old != ie) - printk (LMC_PRINTF_FMT ": clock external\n", LMC_PRINTF_ARGS); - } - else - { - sc->lmc_gpio &= ~(LMC_GEP_HSSI_CLOCK); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_INT; - if(old != ie) - printk (LMC_PRINTF_FMT ": clock internal\n", LMC_PRINTF_ARGS); - } -} - -/* - * return hardware link status. - * 0 == link is down, 1 == link is up. - */ -static int -lmc_hssi_get_link_status (lmc_softc_t * const sc) -{ - /* - * We're using the same code as SSI since - * they're practically the same - */ - return lmc_ssi_get_link_status(sc); -} - -static void -lmc_hssi_set_link_status (lmc_softc_t * const sc, int state) -{ - if (state == LMC_LINK_UP) - sc->lmc_miireg16 |= LMC_MII16_HSSI_TA; - else - sc->lmc_miireg16 &= ~LMC_MII16_HSSI_TA; - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -/* - * 0 == 16bit, 1 == 32bit - */ -static void -lmc_hssi_set_crc_length (lmc_softc_t * const sc, int state) -{ - if (state == LMC_CTL_CRC_LENGTH_32) - { - /* 32 bit */ - sc->lmc_miireg16 |= LMC_MII16_HSSI_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_32; - } - else - { - /* 16 bit */ - sc->lmc_miireg16 &= ~LMC_MII16_HSSI_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_16; - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -static void -lmc_hssi_watchdog (lmc_softc_t * const sc) -{ - /* HSSI is blank */ -} - -/* - * DS3 methods - */ - -/* - * Set cable length - */ -static void -lmc_ds3_set_100ft (lmc_softc_t * const sc, int ie) -{ - if (ie == LMC_CTL_CABLE_LENGTH_GT_100FT) - { - sc->lmc_miireg16 &= ~LMC_MII16_DS3_ZERO; - sc->ictl.cable_length = LMC_CTL_CABLE_LENGTH_GT_100FT; - } - else if (ie == LMC_CTL_CABLE_LENGTH_LT_100FT) - { - sc->lmc_miireg16 |= LMC_MII16_DS3_ZERO; - sc->ictl.cable_length = LMC_CTL_CABLE_LENGTH_LT_100FT; - } - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -static void -lmc_ds3_default (lmc_softc_t * const sc) -{ - sc->lmc_miireg16 = LMC_MII16_LED_ALL; - - sc->lmc_media->set_link_status (sc, LMC_LINK_DOWN); - sc->lmc_media->set_cable_length (sc, LMC_CTL_CABLE_LENGTH_LT_100FT); - sc->lmc_media->set_scrambler (sc, LMC_CTL_OFF); - sc->lmc_media->set_crc_length (sc, LMC_CTL_CRC_LENGTH_16); -} - -/* - * Given a user provided state, set ourselves up to match it. This will - * always reset the card if needed. - */ -static void -lmc_ds3_set_status (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - if (ctl == NULL) - { - sc->lmc_media->set_cable_length (sc, sc->ictl.cable_length); - sc->lmc_media->set_scrambler (sc, sc->ictl.scrambler_onoff); - lmc_set_protocol (sc, NULL); - - return; - } - - /* - * check for change in cable length setting - */ - if (ctl->cable_length && !sc->ictl.cable_length) - lmc_ds3_set_100ft (sc, LMC_CTL_CABLE_LENGTH_GT_100FT); - else if (!ctl->cable_length && sc->ictl.cable_length) - lmc_ds3_set_100ft (sc, LMC_CTL_CABLE_LENGTH_LT_100FT); - - /* - * Check for change in scrambler setting (requires reset) - */ - if (ctl->scrambler_onoff && !sc->ictl.scrambler_onoff) - lmc_ds3_set_scram (sc, LMC_CTL_ON); - else if (!ctl->scrambler_onoff && sc->ictl.scrambler_onoff) - lmc_ds3_set_scram (sc, LMC_CTL_OFF); - - lmc_set_protocol (sc, ctl); -} - -static void -lmc_ds3_init (lmc_softc_t * const sc) -{ - int i; - - sc->ictl.cardtype = LMC_CTL_CARDTYPE_LMC5245; - - /* writes zeros everywhere */ - for (i = 0; i < 21; i++) - { - lmc_mii_writereg (sc, 0, 17, i); - lmc_mii_writereg (sc, 0, 18, 0); - } - - /* set some essential bits */ - lmc_mii_writereg (sc, 0, 17, 1); - lmc_mii_writereg (sc, 0, 18, 0x25); /* ser, xtx */ - - lmc_mii_writereg (sc, 0, 17, 5); - lmc_mii_writereg (sc, 0, 18, 0x80); /* emode */ - - lmc_mii_writereg (sc, 0, 17, 14); - lmc_mii_writereg (sc, 0, 18, 0x30); /* rcgen, tcgen */ - - /* clear counters and latched bits */ - for (i = 0; i < 21; i++) - { - lmc_mii_writereg (sc, 0, 17, i); - lmc_mii_readreg (sc, 0, 18); - } -} - -/* - * 1 == DS3 payload scrambled, 0 == not scrambled - */ -static void -lmc_ds3_set_scram (lmc_softc_t * const sc, int ie) -{ - if (ie == LMC_CTL_ON) - { - sc->lmc_miireg16 |= LMC_MII16_DS3_SCRAM; - sc->ictl.scrambler_onoff = LMC_CTL_ON; - } - else - { - sc->lmc_miireg16 &= ~LMC_MII16_DS3_SCRAM; - sc->ictl.scrambler_onoff = LMC_CTL_OFF; - } - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -/* - * return hardware link status. - * 0 == link is down, 1 == link is up. - */ -static int -lmc_ds3_get_link_status (lmc_softc_t * const sc) -{ - u16 link_status, link_status_11; - int ret = 1; - - lmc_mii_writereg (sc, 0, 17, 7); - link_status = lmc_mii_readreg (sc, 0, 18); - - /* LMC5245 (DS3) & LMC1200 (DS1) LED definitions - * led0 yellow = far-end adapter is in Red alarm condition - * led1 blue = received an Alarm Indication signal - * (upstream failure) - * led2 Green = power to adapter, Gate Array loaded & driver - * attached - * led3 red = Loss of Signal (LOS) or out of frame (OOF) - * conditions detected on T3 receive signal - */ - - lmc_led_on(sc, LMC_DS3_LED2); - - if ((link_status & LMC_FRAMER_REG0_DLOS) || - (link_status & LMC_FRAMER_REG0_OOFS)){ - ret = 0; - if(sc->last_led_err[3] != 1){ - u16 r1; - lmc_mii_writereg (sc, 0, 17, 01); /* Turn on Xbit error as our cisco does */ - r1 = lmc_mii_readreg (sc, 0, 18); - r1 &= 0xfe; - lmc_mii_writereg(sc, 0, 18, r1); - printk(KERN_WARNING "%s: Red Alarm - Loss of Signal or Loss of Framing\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED3); /* turn on red LED */ - sc->last_led_err[3] = 1; - } - else { - lmc_led_off(sc, LMC_DS3_LED3); /* turn on red LED */ - if(sc->last_led_err[3] == 1){ - u16 r1; - lmc_mii_writereg (sc, 0, 17, 01); /* Turn off Xbit error */ - r1 = lmc_mii_readreg (sc, 0, 18); - r1 |= 0x01; - lmc_mii_writereg(sc, 0, 18, r1); - } - sc->last_led_err[3] = 0; - } - - lmc_mii_writereg(sc, 0, 17, 0x10); - link_status_11 = lmc_mii_readreg(sc, 0, 18); - if((link_status & LMC_FRAMER_REG0_AIS) || - (link_status_11 & LMC_FRAMER_REG10_XBIT)) { - ret = 0; - if(sc->last_led_err[0] != 1){ - printk(KERN_WARNING "%s: AIS Alarm or XBit Error\n", sc->name); - printk(KERN_WARNING "%s: Remote end has loss of signal or framing\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED0); - sc->last_led_err[0] = 1; - } - else { - lmc_led_off(sc, LMC_DS3_LED0); - sc->last_led_err[0] = 0; - } - - lmc_mii_writereg (sc, 0, 17, 9); - link_status = lmc_mii_readreg (sc, 0, 18); - - if(link_status & LMC_FRAMER_REG9_RBLUE){ - ret = 0; - if(sc->last_led_err[1] != 1){ - printk(KERN_WARNING "%s: Blue Alarm - Receiving all 1's\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED1); - sc->last_led_err[1] = 1; - } - else { - lmc_led_off(sc, LMC_DS3_LED1); - sc->last_led_err[1] = 0; - } - - return ret; -} - -/* - * 0 == 16bit, 1 == 32bit - */ -static void -lmc_ds3_set_crc_length (lmc_softc_t * const sc, int state) -{ - if (state == LMC_CTL_CRC_LENGTH_32) - { - /* 32 bit */ - sc->lmc_miireg16 |= LMC_MII16_DS3_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_32; - } - else - { - /* 16 bit */ - sc->lmc_miireg16 &= ~LMC_MII16_DS3_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_16; - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -static void -lmc_ds3_watchdog (lmc_softc_t * const sc) -{ - -} - - -/* - * SSI methods - */ - -static void lmc_ssi_init(lmc_softc_t * const sc) -{ - u16 mii17; - int cable; - - sc->ictl.cardtype = LMC_CTL_CARDTYPE_LMC1000; - - mii17 = lmc_mii_readreg(sc, 0, 17); - - cable = (mii17 & LMC_MII17_SSI_CABLE_MASK) >> LMC_MII17_SSI_CABLE_SHIFT; - sc->ictl.cable_type = cable; - - lmc_gpio_mkoutput(sc, LMC_GEP_SSI_TXCLOCK); -} - -static void -lmc_ssi_default (lmc_softc_t * const sc) -{ - sc->lmc_miireg16 = LMC_MII16_LED_ALL; - - /* - * make TXCLOCK always be an output - */ - lmc_gpio_mkoutput (sc, LMC_GEP_SSI_TXCLOCK); - - sc->lmc_media->set_link_status (sc, LMC_LINK_DOWN); - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_EXT); - sc->lmc_media->set_speed (sc, NULL); - sc->lmc_media->set_crc_length (sc, LMC_CTL_CRC_LENGTH_16); -} - -/* - * Given a user provided state, set ourselves up to match it. This will - * always reset the card if needed. - */ -static void -lmc_ssi_set_status (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - if (ctl == NULL) - { - sc->lmc_media->set_clock_source (sc, sc->ictl.clock_source); - sc->lmc_media->set_speed (sc, &sc->ictl); - lmc_set_protocol (sc, NULL); - - return; - } - - /* - * check for change in clock source - */ - if (ctl->clock_source == LMC_CTL_CLOCK_SOURCE_INT - && sc->ictl.clock_source == LMC_CTL_CLOCK_SOURCE_EXT) - { - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_INT); - sc->lmc_timing = LMC_CTL_CLOCK_SOURCE_INT; - } - else if (ctl->clock_source == LMC_CTL_CLOCK_SOURCE_EXT - && sc->ictl.clock_source == LMC_CTL_CLOCK_SOURCE_INT) - { - sc->lmc_media->set_clock_source (sc, LMC_CTL_CLOCK_SOURCE_EXT); - sc->lmc_timing = LMC_CTL_CLOCK_SOURCE_EXT; - } - - if (ctl->clock_rate != sc->ictl.clock_rate) - sc->lmc_media->set_speed (sc, ctl); - - lmc_set_protocol (sc, ctl); -} - -/* - * 1 == internal, 0 == external - */ -static void -lmc_ssi_set_clock (lmc_softc_t * const sc, int ie) -{ - int old; - old = ie; - if (ie == LMC_CTL_CLOCK_SOURCE_EXT) - { - sc->lmc_gpio &= ~(LMC_GEP_SSI_TXCLOCK); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_EXT; - if(ie != old) - printk (LMC_PRINTF_FMT ": clock external\n", LMC_PRINTF_ARGS); - } - else - { - sc->lmc_gpio |= LMC_GEP_SSI_TXCLOCK; - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_INT; - if(ie != old) - printk (LMC_PRINTF_FMT ": clock internal\n", LMC_PRINTF_ARGS); - } -} - -static void -lmc_ssi_set_speed (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - lmc_ctl_t *ictl = &sc->ictl; - lmc_av9110_t *av; - - /* original settings for clock rate of: - * 100 Khz (8,25,0,0,2) were incorrect - * they should have been 80,125,1,3,3 - * There are 17 param combinations to produce this freq. - * For 1.5 Mhz use 120,100,1,1,2 (226 param. combinations) - */ - if (ctl == NULL) - { - av = &ictl->cardspec.ssi; - ictl->clock_rate = 1500000; - av->f = ictl->clock_rate; - av->n = 120; - av->m = 100; - av->v = 1; - av->x = 1; - av->r = 2; - - write_av9110 (sc, av->n, av->m, av->v, av->x, av->r); - return; - } - - av = &ctl->cardspec.ssi; - - if (av->f == 0) - return; - - ictl->clock_rate = av->f; /* really, this is the rate we are */ - ictl->cardspec.ssi = *av; - - write_av9110 (sc, av->n, av->m, av->v, av->x, av->r); -} - -/* - * return hardware link status. - * 0 == link is down, 1 == link is up. - */ -static int -lmc_ssi_get_link_status (lmc_softc_t * const sc) -{ - u16 link_status; - u32 ticks; - int ret = 1; - int hw_hdsk = 1; - - /* - * missing CTS? Hmm. If we require CTS on, we may never get the - * link to come up, so omit it in this test. - * - * Also, it seems that with a loopback cable, DCD isn't asserted, - * so just check for things like this: - * DSR _must_ be asserted. - * One of DCD or CTS must be asserted. - */ - - /* LMC 1000 (SSI) LED definitions - * led0 Green = power to adapter, Gate Array loaded & - * driver attached - * led1 Green = DSR and DTR and RTS and CTS are set - * led2 Green = Cable detected - * led3 red = No timing is available from the - * cable or the on-board frequency - * generator. - */ - - link_status = lmc_mii_readreg (sc, 0, 16); - - /* Is the transmit clock still available */ - ticks = LMC_CSR_READ (sc, csr_gp_timer); - ticks = 0x0000ffff - (ticks & 0x0000ffff); - - lmc_led_on (sc, LMC_MII16_LED0); - - /* ====== transmit clock determination ===== */ - if (sc->lmc_timing == LMC_CTL_CLOCK_SOURCE_INT) { - lmc_led_off(sc, LMC_MII16_LED3); - } - else if (ticks == 0 ) { /* no clock found ? */ - ret = 0; - if (sc->last_led_err[3] != 1) { - sc->extra_stats.tx_lossOfClockCnt++; - printk(KERN_WARNING "%s: Lost Clock, Link Down\n", sc->name); - } - sc->last_led_err[3] = 1; - lmc_led_on (sc, LMC_MII16_LED3); /* turn ON red LED */ - } - else { - if(sc->last_led_err[3] == 1) - printk(KERN_WARNING "%s: Clock Returned\n", sc->name); - sc->last_led_err[3] = 0; - lmc_led_off (sc, LMC_MII16_LED3); /* turn OFF red LED */ - } - - if ((link_status & LMC_MII16_SSI_DSR) == 0) { /* Also HSSI CA */ - ret = 0; - hw_hdsk = 0; - } - -#ifdef CONFIG_LMC_IGNORE_HARDWARE_HANDSHAKE - if ((link_status & (LMC_MII16_SSI_CTS | LMC_MII16_SSI_DCD)) == 0){ - ret = 0; - hw_hdsk = 0; - } -#endif - - if(hw_hdsk == 0){ - if(sc->last_led_err[1] != 1) - printk(KERN_WARNING "%s: DSR not asserted\n", sc->name); - sc->last_led_err[1] = 1; - lmc_led_off(sc, LMC_MII16_LED1); - } - else { - if(sc->last_led_err[1] != 0) - printk(KERN_WARNING "%s: DSR now asserted\n", sc->name); - sc->last_led_err[1] = 0; - lmc_led_on(sc, LMC_MII16_LED1); - } - - if(ret == 1) { - lmc_led_on(sc, LMC_MII16_LED2); /* Over all good status? */ - } - - return ret; -} - -static void -lmc_ssi_set_link_status (lmc_softc_t * const sc, int state) -{ - if (state == LMC_LINK_UP) - { - sc->lmc_miireg16 |= (LMC_MII16_SSI_DTR | LMC_MII16_SSI_RTS); - printk (LMC_PRINTF_FMT ": asserting DTR and RTS\n", LMC_PRINTF_ARGS); - } - else - { - sc->lmc_miireg16 &= ~(LMC_MII16_SSI_DTR | LMC_MII16_SSI_RTS); - printk (LMC_PRINTF_FMT ": deasserting DTR and RTS\n", LMC_PRINTF_ARGS); - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); - -} - -/* - * 0 == 16bit, 1 == 32bit - */ -static void -lmc_ssi_set_crc_length (lmc_softc_t * const sc, int state) -{ - if (state == LMC_CTL_CRC_LENGTH_32) - { - /* 32 bit */ - sc->lmc_miireg16 |= LMC_MII16_SSI_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_32; - sc->lmc_crcSize = LMC_CTL_CRC_BYTESIZE_4; - - } - else - { - /* 16 bit */ - sc->lmc_miireg16 &= ~LMC_MII16_SSI_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_16; - sc->lmc_crcSize = LMC_CTL_CRC_BYTESIZE_2; - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -/* - * These are bits to program the ssi frequency generator - */ -static inline void -write_av9110_bit (lmc_softc_t * sc, int c) -{ - /* - * set the data bit as we need it. - */ - sc->lmc_gpio &= ~(LMC_GEP_CLK); - if (c & 0x01) - sc->lmc_gpio |= LMC_GEP_DATA; - else - sc->lmc_gpio &= ~(LMC_GEP_DATA); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - - /* - * set the clock to high - */ - sc->lmc_gpio |= LMC_GEP_CLK; - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - - /* - * set the clock to low again. - */ - sc->lmc_gpio &= ~(LMC_GEP_CLK); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); -} - -static void write_av9110(lmc_softc_t *sc, u32 n, u32 m, u32 v, u32 x, u32 r) -{ - int i; - -#if 0 - printk (LMC_PRINTF_FMT ": speed %u, %d %d %d %d %d\n", - LMC_PRINTF_ARGS, sc->ictl.clock_rate, n, m, v, x, r); -#endif - - sc->lmc_gpio |= LMC_GEP_SSI_GENERATOR; - sc->lmc_gpio &= ~(LMC_GEP_DATA | LMC_GEP_CLK); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - - /* - * Set the TXCLOCK, GENERATOR, SERIAL, and SERIALCLK - * as outputs. - */ - lmc_gpio_mkoutput (sc, (LMC_GEP_DATA | LMC_GEP_CLK - | LMC_GEP_SSI_GENERATOR)); - - sc->lmc_gpio &= ~(LMC_GEP_SSI_GENERATOR); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - - /* - * a shifting we will go... - */ - for (i = 0; i < 7; i++) - write_av9110_bit (sc, n >> i); - for (i = 0; i < 7; i++) - write_av9110_bit (sc, m >> i); - for (i = 0; i < 1; i++) - write_av9110_bit (sc, v >> i); - for (i = 0; i < 2; i++) - write_av9110_bit (sc, x >> i); - for (i = 0; i < 2; i++) - write_av9110_bit (sc, r >> i); - for (i = 0; i < 5; i++) - write_av9110_bit (sc, 0x17 >> i); - - /* - * stop driving serial-related signals - */ - lmc_gpio_mkinput (sc, - (LMC_GEP_DATA | LMC_GEP_CLK - | LMC_GEP_SSI_GENERATOR)); -} - -static void lmc_ssi_watchdog(lmc_softc_t * const sc) -{ - u16 mii17 = lmc_mii_readreg(sc, 0, 17); - if (((mii17 >> 3) & 7) == 7) - lmc_led_off(sc, LMC_MII16_LED2); - else - lmc_led_on(sc, LMC_MII16_LED2); -} - -/* - * T1 methods - */ - -/* - * The framer regs are multiplexed through MII regs 17 & 18 - * write the register address to MII reg 17 and the * data to MII reg 18. */ -static void -lmc_t1_write (lmc_softc_t * const sc, int a, int d) -{ - lmc_mii_writereg (sc, 0, 17, a); - lmc_mii_writereg (sc, 0, 18, d); -} - -/* Save a warning -static int -lmc_t1_read (lmc_softc_t * const sc, int a) -{ - lmc_mii_writereg (sc, 0, 17, a); - return lmc_mii_readreg (sc, 0, 18); -} -*/ - - -static void -lmc_t1_init (lmc_softc_t * const sc) -{ - u16 mii16; - int i; - - sc->ictl.cardtype = LMC_CTL_CARDTYPE_LMC1200; - mii16 = lmc_mii_readreg (sc, 0, 16); - - /* reset 8370 */ - mii16 &= ~LMC_MII16_T1_RST; - lmc_mii_writereg (sc, 0, 16, mii16 | LMC_MII16_T1_RST); - lmc_mii_writereg (sc, 0, 16, mii16); - - /* set T1 or E1 line. Uses sc->lmcmii16 reg in function so update it */ - sc->lmc_miireg16 = mii16; - lmc_t1_set_circuit_type(sc, LMC_CTL_CIRCUIT_TYPE_T1); - mii16 = sc->lmc_miireg16; - - lmc_t1_write (sc, 0x01, 0x1B); /* CR0 - primary control */ - lmc_t1_write (sc, 0x02, 0x42); /* JAT_CR - jitter atten config */ - lmc_t1_write (sc, 0x14, 0x00); /* LOOP - loopback config */ - lmc_t1_write (sc, 0x15, 0x00); /* DL3_TS - external data link timeslot */ - lmc_t1_write (sc, 0x18, 0xFF); /* PIO - programmable I/O */ - lmc_t1_write (sc, 0x19, 0x30); /* POE - programmable OE */ - lmc_t1_write (sc, 0x1A, 0x0F); /* CMUX - clock input mux */ - lmc_t1_write (sc, 0x20, 0x41); /* LIU_CR - RX LIU config */ - lmc_t1_write (sc, 0x22, 0x76); /* RLIU_CR - RX LIU config */ - lmc_t1_write (sc, 0x40, 0x03); /* RCR0 - RX config */ - lmc_t1_write (sc, 0x45, 0x00); /* RALM - RX alarm config */ - lmc_t1_write (sc, 0x46, 0x05); /* LATCH - RX alarm/err/cntr latch */ - lmc_t1_write (sc, 0x68, 0x40); /* TLIU_CR - TX LIU config */ - lmc_t1_write (sc, 0x70, 0x0D); /* TCR0 - TX framer config */ - lmc_t1_write (sc, 0x71, 0x05); /* TCR1 - TX config */ - lmc_t1_write (sc, 0x72, 0x0B); /* TFRM - TX frame format */ - lmc_t1_write (sc, 0x73, 0x00); /* TERROR - TX error insert */ - lmc_t1_write (sc, 0x74, 0x00); /* TMAN - TX manual Sa/FEBE config */ - lmc_t1_write (sc, 0x75, 0x00); /* TALM - TX alarm signal config */ - lmc_t1_write (sc, 0x76, 0x00); /* TPATT - TX test pattern config */ - lmc_t1_write (sc, 0x77, 0x00); /* TLB - TX inband loopback config */ - lmc_t1_write (sc, 0x90, 0x05); /* CLAD_CR - clock rate adapter config */ - lmc_t1_write (sc, 0x91, 0x05); /* CSEL - clad freq sel */ - lmc_t1_write (sc, 0xA6, 0x00); /* DL1_CTL - DL1 control */ - lmc_t1_write (sc, 0xB1, 0x00); /* DL2_CTL - DL2 control */ - lmc_t1_write (sc, 0xD0, 0x47); /* SBI_CR - sys bus iface config */ - lmc_t1_write (sc, 0xD1, 0x70); /* RSB_CR - RX sys bus config */ - lmc_t1_write (sc, 0xD4, 0x30); /* TSB_CR - TX sys bus config */ - for (i = 0; i < 32; i++) - { - lmc_t1_write (sc, 0x0E0 + i, 0x00); /* SBCn - sys bus per-channel ctl */ - lmc_t1_write (sc, 0x100 + i, 0x00); /* TPCn - TX per-channel ctl */ - lmc_t1_write (sc, 0x180 + i, 0x00); /* RPCn - RX per-channel ctl */ - } - for (i = 1; i < 25; i++) - { - lmc_t1_write (sc, 0x0E0 + i, 0x0D); /* SBCn - sys bus per-channel ctl */ - } - - mii16 |= LMC_MII16_T1_XOE; - lmc_mii_writereg (sc, 0, 16, mii16); - sc->lmc_miireg16 = mii16; -} - -static void -lmc_t1_default (lmc_softc_t * const sc) -{ - sc->lmc_miireg16 = LMC_MII16_LED_ALL; - sc->lmc_media->set_link_status (sc, LMC_LINK_DOWN); - sc->lmc_media->set_circuit_type (sc, LMC_CTL_CIRCUIT_TYPE_T1); - sc->lmc_media->set_crc_length (sc, LMC_CTL_CRC_LENGTH_16); - /* Right now we can only clock from out internal source */ - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_INT; -} -/* * Given a user provided state, set ourselves up to match it. This will * always reset the card if needed. - */ -static void -lmc_t1_set_status (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - if (ctl == NULL) - { - sc->lmc_media->set_circuit_type (sc, sc->ictl.circuit_type); - lmc_set_protocol (sc, NULL); - - return; - } - /* - * check for change in circuit type */ - if (ctl->circuit_type == LMC_CTL_CIRCUIT_TYPE_T1 - && sc->ictl.circuit_type == - LMC_CTL_CIRCUIT_TYPE_E1) sc->lmc_media->set_circuit_type (sc, - LMC_CTL_CIRCUIT_TYPE_E1); - else if (ctl->circuit_type == LMC_CTL_CIRCUIT_TYPE_E1 - && sc->ictl.circuit_type == LMC_CTL_CIRCUIT_TYPE_T1) - sc->lmc_media->set_circuit_type (sc, LMC_CTL_CIRCUIT_TYPE_T1); - lmc_set_protocol (sc, ctl); -} -/* - * return hardware link status. - * 0 == link is down, 1 == link is up. - */ static int -lmc_t1_get_link_status (lmc_softc_t * const sc) -{ - u16 link_status; - int ret = 1; - - /* LMC5245 (DS3) & LMC1200 (DS1) LED definitions - * led0 yellow = far-end adapter is in Red alarm condition - * led1 blue = received an Alarm Indication signal - * (upstream failure) - * led2 Green = power to adapter, Gate Array loaded & driver - * attached - * led3 red = Loss of Signal (LOS) or out of frame (OOF) - * conditions detected on T3 receive signal - */ - lmc_led_on(sc, LMC_DS3_LED2); - - lmc_mii_writereg (sc, 0, 17, T1FRAMER_ALARM1_STATUS); - link_status = lmc_mii_readreg (sc, 0, 18); - - - if (link_status & T1F_RAIS) { /* turn on blue LED */ - ret = 0; - if(sc->last_led_err[1] != 1){ - printk(KERN_WARNING "%s: Receive AIS/Blue Alarm. Far end in RED alarm\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED1); - sc->last_led_err[1] = 1; - } - else { - if(sc->last_led_err[1] != 0){ - printk(KERN_WARNING "%s: End AIS/Blue Alarm\n", sc->name); - } - lmc_led_off (sc, LMC_DS3_LED1); - sc->last_led_err[1] = 0; - } - - /* - * Yellow Alarm is nasty evil stuff, looks at data patterns - * inside the channel and confuses it with HDLC framing - * ignore all yellow alarms. - * - * Do listen to MultiFrame Yellow alarm which while implemented - * different ways isn't in the channel and hence somewhat - * more reliable - */ - - if (link_status & T1F_RMYEL) { - ret = 0; - if(sc->last_led_err[0] != 1){ - printk(KERN_WARNING "%s: Receive Yellow AIS Alarm\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED0); - sc->last_led_err[0] = 1; - } - else { - if(sc->last_led_err[0] != 0){ - printk(KERN_WARNING "%s: End of Yellow AIS Alarm\n", sc->name); - } - lmc_led_off(sc, LMC_DS3_LED0); - sc->last_led_err[0] = 0; - } - - /* - * Loss of signal and los of frame - * Use the green bit to identify which one lit the led - */ - if(link_status & T1F_RLOF){ - ret = 0; - if(sc->last_led_err[3] != 1){ - printk(KERN_WARNING "%s: Local Red Alarm: Loss of Framing\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED3); - sc->last_led_err[3] = 1; - - } - else { - if(sc->last_led_err[3] != 0){ - printk(KERN_WARNING "%s: End Red Alarm (LOF)\n", sc->name); - } - if( ! (link_status & T1F_RLOS)) - lmc_led_off(sc, LMC_DS3_LED3); - sc->last_led_err[3] = 0; - } - - if(link_status & T1F_RLOS){ - ret = 0; - if(sc->last_led_err[2] != 1){ - printk(KERN_WARNING "%s: Local Red Alarm: Loss of Signal\n", sc->name); - } - lmc_led_on(sc, LMC_DS3_LED3); - sc->last_led_err[2] = 1; - - } - else { - if(sc->last_led_err[2] != 0){ - printk(KERN_WARNING "%s: End Red Alarm (LOS)\n", sc->name); - } - if( ! (link_status & T1F_RLOF)) - lmc_led_off(sc, LMC_DS3_LED3); - sc->last_led_err[2] = 0; - } - - sc->lmc_xinfo.t1_alarm1_status = link_status; - - lmc_mii_writereg (sc, 0, 17, T1FRAMER_ALARM2_STATUS); - sc->lmc_xinfo.t1_alarm2_status = lmc_mii_readreg (sc, 0, 18); - - return ret; -} - -/* - * 1 == T1 Circuit Type , 0 == E1 Circuit Type - */ -static void -lmc_t1_set_circuit_type (lmc_softc_t * const sc, int ie) -{ - if (ie == LMC_CTL_CIRCUIT_TYPE_T1) { - sc->lmc_miireg16 |= LMC_MII16_T1_Z; - sc->ictl.circuit_type = LMC_CTL_CIRCUIT_TYPE_T1; - printk(KERN_INFO "%s: In T1 Mode\n", sc->name); - } - else { - sc->lmc_miireg16 &= ~LMC_MII16_T1_Z; - sc->ictl.circuit_type = LMC_CTL_CIRCUIT_TYPE_E1; - printk(KERN_INFO "%s: In E1 Mode\n", sc->name); - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); - -} - -/* - * 0 == 16bit, 1 == 32bit */ -static void -lmc_t1_set_crc_length (lmc_softc_t * const sc, int state) -{ - if (state == LMC_CTL_CRC_LENGTH_32) - { - /* 32 bit */ - sc->lmc_miireg16 |= LMC_MII16_T1_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_32; - sc->lmc_crcSize = LMC_CTL_CRC_BYTESIZE_4; - - } - else - { - /* 16 bit */ sc->lmc_miireg16 &= ~LMC_MII16_T1_CRC; - sc->ictl.crc_length = LMC_CTL_CRC_LENGTH_16; - sc->lmc_crcSize = LMC_CTL_CRC_BYTESIZE_2; - - } - - lmc_mii_writereg (sc, 0, 16, sc->lmc_miireg16); -} - -/* - * 1 == internal, 0 == external - */ -static void -lmc_t1_set_clock (lmc_softc_t * const sc, int ie) -{ - int old; - old = ie; - if (ie == LMC_CTL_CLOCK_SOURCE_EXT) - { - sc->lmc_gpio &= ~(LMC_GEP_SSI_TXCLOCK); - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_EXT; - if(old != ie) - printk (LMC_PRINTF_FMT ": clock external\n", LMC_PRINTF_ARGS); - } - else - { - sc->lmc_gpio |= LMC_GEP_SSI_TXCLOCK; - LMC_CSR_WRITE (sc, csr_gp, sc->lmc_gpio); - sc->ictl.clock_source = LMC_CTL_CLOCK_SOURCE_INT; - if(old != ie) - printk (LMC_PRINTF_FMT ": clock internal\n", LMC_PRINTF_ARGS); - } -} - -static void -lmc_t1_watchdog (lmc_softc_t * const sc) -{ -} - -static void -lmc_set_protocol (lmc_softc_t * const sc, lmc_ctl_t * ctl) -{ - if (!ctl) - sc->ictl.keepalive_onoff = LMC_CTL_ON; -} diff --git a/drivers/net/wan/lmc/lmc_proto.c b/drivers/net/wan/lmc/lmc_proto.c deleted file mode 100644 index e5487616a816..000000000000 --- a/drivers/net/wan/lmc/lmc_proto.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - /* - * Copyright (c) 1997-2000 LAN Media Corporation (LMC) - * All rights reserved. www.lanmedia.com - * - * This code is written by: - * Andrew Stanley-Jones (asj@cban.com) - * Rob Braun (bbraun@vix.com), - * Michael Graff (explorer@vix.com) and - * Matt Thomas (matt@3am-software.com). - * - * With Help By: - * David Boggs - * Ron Crane - * Allan Cox - * - * Driver for the LanMedia LMC5200, LMC5245, LMC1000, LMC1200 cards. - */ - -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/timer.h> -#include <linux/ptrace.h> -#include <linux/errno.h> -#include <linux/ioport.h> -#include <linux/interrupt.h> -#include <linux/in.h> -#include <linux/if_arp.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/skbuff.h> -#include <linux/inet.h> -#include <linux/workqueue.h> -#include <linux/proc_fs.h> -#include <linux/bitops.h> -#include <asm/processor.h> /* Processor type for cache alignment. */ -#include <asm/io.h> -#include <asm/dma.h> -#include <linux/smp.h> - -#include "lmc.h" -#include "lmc_var.h" -#include "lmc_debug.h" -#include "lmc_ioctl.h" -#include "lmc_proto.h" - -// attach -void lmc_proto_attach(lmc_softc_t *sc) /*FOLD00*/ -{ - if (sc->if_type == LMC_NET) { - struct net_device *dev = sc->lmc_device; - /* - * They set a few basics because they don't use HDLC - */ - dev->flags |= IFF_POINTOPOINT; - dev->hard_header_len = 0; - dev->addr_len = 0; - } -} - -int lmc_proto_open(lmc_softc_t *sc) -{ - int ret = 0; - - if (sc->if_type == LMC_PPP) { - ret = hdlc_open(sc->lmc_device); - if (ret < 0) - printk(KERN_WARNING "%s: HDLC open failed: %d\n", - sc->name, ret); - } - return ret; -} - -void lmc_proto_close(lmc_softc_t *sc) -{ - if (sc->if_type == LMC_PPP) - hdlc_close(sc->lmc_device); -} - -__be16 lmc_proto_type(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ -{ - switch(sc->if_type){ - case LMC_PPP: - return hdlc_type_trans(skb, sc->lmc_device); - case LMC_NET: - return htons(ETH_P_802_2); - case LMC_RAW: /* Packet type for skbuff kind of useless */ - return htons(ETH_P_802_2); - default: - printk(KERN_WARNING "%s: No protocol set for this interface, assuming 802.2 (which is wrong!!)\n", sc->name); - return htons(ETH_P_802_2); - } -} - -void lmc_proto_netif(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ -{ - switch(sc->if_type){ - case LMC_PPP: - case LMC_NET: - default: - netif_rx(skb); - break; - case LMC_RAW: - break; - } -} diff --git a/drivers/net/wan/lmc/lmc_proto.h b/drivers/net/wan/lmc/lmc_proto.h deleted file mode 100644 index e56e7072de44..000000000000 --- a/drivers/net/wan/lmc/lmc_proto.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LMC_PROTO_H_ -#define _LMC_PROTO_H_ - -#include <linux/hdlc.h> - -void lmc_proto_attach(lmc_softc_t *sc); -int lmc_proto_open(lmc_softc_t *sc); -void lmc_proto_close(lmc_softc_t *sc); -__be16 lmc_proto_type(lmc_softc_t *sc, struct sk_buff *skb); -void lmc_proto_netif(lmc_softc_t *sc, struct sk_buff *skb); - -static inline lmc_softc_t* dev_to_sc(struct net_device *dev) -{ - return (lmc_softc_t *)dev_to_hdlc(dev)->priv; -} - -#endif diff --git a/drivers/net/wan/lmc/lmc_var.h b/drivers/net/wan/lmc/lmc_var.h deleted file mode 100644 index 99f0aa787a35..000000000000 --- a/drivers/net/wan/lmc/lmc_var.h +++ /dev/null @@ -1,468 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _LMC_VAR_H_ -#define _LMC_VAR_H_ - - /* - * Copyright (c) 1997-2000 LAN Media Corporation (LMC) - * All rights reserved. www.lanmedia.com - * - * This code is written by: - * Andrew Stanley-Jones (asj@cban.com) - * Rob Braun (bbraun@vix.com), - * Michael Graff (explorer@vix.com) and - * Matt Thomas (matt@3am-software.com). - */ - -#include <linux/timer.h> - -/* - * basic definitions used in lmc include files - */ - -typedef struct lmc___softc lmc_softc_t; -typedef struct lmc___media lmc_media_t; -typedef struct lmc___ctl lmc_ctl_t; - -#define lmc_csrptr_t unsigned long - -#define LMC_REG_RANGE 0x80 - -#define LMC_PRINTF_FMT "%s" -#define LMC_PRINTF_ARGS (sc->lmc_device->name) - -#define TX_TIMEOUT (2*HZ) - -#define LMC_TXDESCS 32 -#define LMC_RXDESCS 32 - -#define LMC_LINK_UP 1 -#define LMC_LINK_DOWN 0 - -/* These macros for generic read and write to and from the dec chip */ -#define LMC_CSR_READ(sc, csr) \ - inl((sc)->lmc_csrs.csr) -#define LMC_CSR_WRITE(sc, reg, val) \ - outl((val), (sc)->lmc_csrs.reg) - -//#ifdef _LINUX_DELAY_H -// #define SLOW_DOWN_IO udelay(2); -// #undef __SLOW_DOWN_IO -// #define __SLOW_DOWN_IO udelay(2); -//#endif - -#define DELAY(n) SLOW_DOWN_IO - -#define lmc_delay() inl(sc->lmc_csrs.csr_9) - -/* This macro sync's up with the mii so that reads and writes can take place */ -#define LMC_MII_SYNC(sc) do {int n=32; while( n >= 0 ) { \ - LMC_CSR_WRITE((sc), csr_9, 0x20000); \ - lmc_delay(); \ - LMC_CSR_WRITE((sc), csr_9, 0x30000); \ - lmc_delay(); \ - n--; }} while(0) - -struct lmc_regfile_t { - lmc_csrptr_t csr_busmode; /* CSR0 */ - lmc_csrptr_t csr_txpoll; /* CSR1 */ - lmc_csrptr_t csr_rxpoll; /* CSR2 */ - lmc_csrptr_t csr_rxlist; /* CSR3 */ - lmc_csrptr_t csr_txlist; /* CSR4 */ - lmc_csrptr_t csr_status; /* CSR5 */ - lmc_csrptr_t csr_command; /* CSR6 */ - lmc_csrptr_t csr_intr; /* CSR7 */ - lmc_csrptr_t csr_missed_frames; /* CSR8 */ - lmc_csrptr_t csr_9; /* CSR9 */ - lmc_csrptr_t csr_10; /* CSR10 */ - lmc_csrptr_t csr_11; /* CSR11 */ - lmc_csrptr_t csr_12; /* CSR12 */ - lmc_csrptr_t csr_13; /* CSR13 */ - lmc_csrptr_t csr_14; /* CSR14 */ - lmc_csrptr_t csr_15; /* CSR15 */ -}; - -#define csr_enetrom csr_9 /* 21040 */ -#define csr_reserved csr_10 /* 21040 */ -#define csr_full_duplex csr_11 /* 21040 */ -#define csr_bootrom csr_10 /* 21041/21140A/?? */ -#define csr_gp csr_12 /* 21140* */ -#define csr_watchdog csr_15 /* 21140* */ -#define csr_gp_timer csr_11 /* 21041/21140* */ -#define csr_srom_mii csr_9 /* 21041/21140* */ -#define csr_sia_status csr_12 /* 2104x */ -#define csr_sia_connectivity csr_13 /* 2104x */ -#define csr_sia_tx_rx csr_14 /* 2104x */ -#define csr_sia_general csr_15 /* 2104x */ - -/* tulip length/control transmit descriptor definitions - * used to define bits in the second tulip_desc_t field (length) - * for the transmit descriptor -baz */ - -#define LMC_TDES_FIRST_BUFFER_SIZE ((u32)(0x000007FF)) -#define LMC_TDES_SECOND_BUFFER_SIZE ((u32)(0x003FF800)) -#define LMC_TDES_HASH_FILTERING ((u32)(0x00400000)) -#define LMC_TDES_DISABLE_PADDING ((u32)(0x00800000)) -#define LMC_TDES_SECOND_ADDR_CHAINED ((u32)(0x01000000)) -#define LMC_TDES_END_OF_RING ((u32)(0x02000000)) -#define LMC_TDES_ADD_CRC_DISABLE ((u32)(0x04000000)) -#define LMC_TDES_SETUP_PACKET ((u32)(0x08000000)) -#define LMC_TDES_INVERSE_FILTERING ((u32)(0x10000000)) -#define LMC_TDES_FIRST_SEGMENT ((u32)(0x20000000)) -#define LMC_TDES_LAST_SEGMENT ((u32)(0x40000000)) -#define LMC_TDES_INTERRUPT_ON_COMPLETION ((u32)(0x80000000)) - -#define TDES_SECOND_BUFFER_SIZE_BIT_NUMBER 11 -#define TDES_COLLISION_COUNT_BIT_NUMBER 3 - -/* Constants for the RCV descriptor RDES */ - -#define LMC_RDES_OVERFLOW ((u32)(0x00000001)) -#define LMC_RDES_CRC_ERROR ((u32)(0x00000002)) -#define LMC_RDES_DRIBBLING_BIT ((u32)(0x00000004)) -#define LMC_RDES_REPORT_ON_MII_ERR ((u32)(0x00000008)) -#define LMC_RDES_RCV_WATCHDOG_TIMEOUT ((u32)(0x00000010)) -#define LMC_RDES_FRAME_TYPE ((u32)(0x00000020)) -#define LMC_RDES_COLLISION_SEEN ((u32)(0x00000040)) -#define LMC_RDES_FRAME_TOO_LONG ((u32)(0x00000080)) -#define LMC_RDES_LAST_DESCRIPTOR ((u32)(0x00000100)) -#define LMC_RDES_FIRST_DESCRIPTOR ((u32)(0x00000200)) -#define LMC_RDES_MULTICAST_FRAME ((u32)(0x00000400)) -#define LMC_RDES_RUNT_FRAME ((u32)(0x00000800)) -#define LMC_RDES_DATA_TYPE ((u32)(0x00003000)) -#define LMC_RDES_LENGTH_ERROR ((u32)(0x00004000)) -#define LMC_RDES_ERROR_SUMMARY ((u32)(0x00008000)) -#define LMC_RDES_FRAME_LENGTH ((u32)(0x3FFF0000)) -#define LMC_RDES_OWN_BIT ((u32)(0x80000000)) - -#define RDES_FRAME_LENGTH_BIT_NUMBER 16 - -#define LMC_RDES_ERROR_MASK ( (u32)( \ - LMC_RDES_OVERFLOW \ - | LMC_RDES_DRIBBLING_BIT \ - | LMC_RDES_REPORT_ON_MII_ERR \ - | LMC_RDES_COLLISION_SEEN ) ) - - -/* - * Ioctl info - */ - -typedef struct { - u32 n; - u32 m; - u32 v; - u32 x; - u32 r; - u32 f; - u32 exact; -} lmc_av9110_t; - -/* - * Common structure passed to the ioctl code. - */ -struct lmc___ctl { - u32 cardtype; - u32 clock_source; /* HSSI, T1 */ - u32 clock_rate; /* T1 */ - u32 crc_length; - u32 cable_length; /* DS3 */ - u32 scrambler_onoff; /* DS3 */ - u32 cable_type; /* T1 */ - u32 keepalive_onoff; /* protocol */ - u32 ticks; /* ticks/sec */ - union { - lmc_av9110_t ssi; - } cardspec; - u32 circuit_type; /* T1 or E1 */ -}; - - -/* - * Careful, look at the data sheet, there's more to this - * structure than meets the eye. It should probably be: - * - * struct tulip_desc_t { - * u8 own:1; - * u32 status:31; - * u32 control:10; - * u32 buffer1; - * u32 buffer2; - * }; - * You could also expand status control to provide more bit information - */ - -struct tulip_desc_t { - s32 status; - s32 length; - u32 buffer1; - u32 buffer2; -}; - -/* - * media independent methods to check on media status, link, light LEDs, - * etc. - */ -struct lmc___media { - void (* init)(lmc_softc_t * const); - void (* defaults)(lmc_softc_t * const); - void (* set_status)(lmc_softc_t * const, lmc_ctl_t *); - void (* set_clock_source)(lmc_softc_t * const, int); - void (* set_speed)(lmc_softc_t * const, lmc_ctl_t *); - void (* set_cable_length)(lmc_softc_t * const, int); - void (* set_scrambler)(lmc_softc_t * const, int); - int (* get_link_status)(lmc_softc_t * const); - void (* set_link_status)(lmc_softc_t * const, int); - void (* set_crc_length)(lmc_softc_t * const, int); - void (* set_circuit_type)(lmc_softc_t * const, int); - void (* watchdog)(lmc_softc_t * const); -}; - - -#define STATCHECK 0xBEEFCAFE - -struct lmc_extra_statistics -{ - u32 version_size; - u32 lmc_cardtype; - - u32 tx_ProcTimeout; - u32 tx_IntTimeout; - u32 tx_NoCompleteCnt; - u32 tx_MaxXmtsB4Int; - u32 tx_TimeoutCnt; - u32 tx_OutOfSyncPtr; - u32 tx_tbusy0; - u32 tx_tbusy1; - u32 tx_tbusy_calls; - u32 resetCount; - u32 lmc_txfull; - u32 tbusy; - u32 dirtyTx; - u32 lmc_next_tx; - u32 otherTypeCnt; - u32 lastType; - u32 lastTypeOK; - u32 txLoopCnt; - u32 usedXmtDescripCnt; - u32 txIndexCnt; - u32 rxIntLoopCnt; - - u32 rx_SmallPktCnt; - u32 rx_BadPktSurgeCnt; - u32 rx_BuffAllocErr; - u32 tx_lossOfClockCnt; - - /* T1 error counters */ - u32 framingBitErrorCount; - u32 lineCodeViolationCount; - - u32 lossOfFrameCount; - u32 changeOfFrameAlignmentCount; - u32 severelyErroredFrameCount; - - u32 check; -}; - -typedef struct lmc_xinfo { - u32 Magic0; /* BEEFCAFE */ - - u32 PciCardType; - u32 PciSlotNumber; /* PCI slot number */ - - u16 DriverMajorVersion; - u16 DriverMinorVersion; - u16 DriverSubVersion; - - u16 XilinxRevisionNumber; - u16 MaxFrameSize; - - u16 t1_alarm1_status; - u16 t1_alarm2_status; - - int link_status; - u32 mii_reg16; - - u32 Magic1; /* DEADBEEF */ -} LMC_XINFO; - - -/* - * forward decl - */ -struct lmc___softc { - char *name; - u8 board_idx; - struct lmc_extra_statistics extra_stats; - struct net_device *lmc_device; - - int hang, rxdesc, bad_packet, some_counter; - u32 txgo; - struct lmc_regfile_t lmc_csrs; - volatile u32 lmc_txtick; - volatile u32 lmc_rxtick; - u32 lmc_flags; - u32 lmc_intrmask; /* our copy of csr_intr */ - u32 lmc_cmdmode; /* our copy of csr_cmdmode */ - u32 lmc_busmode; /* our copy of csr_busmode */ - u32 lmc_gpio_io; /* state of in/out settings */ - u32 lmc_gpio; /* state of outputs */ - struct sk_buff* lmc_txq[LMC_TXDESCS]; - struct sk_buff* lmc_rxq[LMC_RXDESCS]; - volatile - struct tulip_desc_t lmc_rxring[LMC_RXDESCS]; - volatile - struct tulip_desc_t lmc_txring[LMC_TXDESCS]; - unsigned int lmc_next_rx, lmc_next_tx; - volatile - unsigned int lmc_taint_tx, lmc_taint_rx; - int lmc_tx_start, lmc_txfull; - int lmc_txbusy; - u16 lmc_miireg16; - int lmc_ok; - int last_link_status; - int lmc_cardtype; - u32 last_frameerr; - lmc_media_t *lmc_media; - struct timer_list timer; - lmc_ctl_t ictl; - u32 TxDescriptControlInit; - - int tx_TimeoutInd; /* additional driver state */ - int tx_TimeoutDisplay; - unsigned int lastlmc_taint_tx; - int lasttx_packets; - u32 tx_clockState; - u32 lmc_crcSize; - LMC_XINFO lmc_xinfo; - char lmc_yel, lmc_blue, lmc_red; /* for T1 and DS3 */ - char lmc_timing; /* for HSSI and SSI */ - int got_irq; - - char last_led_err[4]; - - u32 last_int; - u32 num_int; - - spinlock_t lmc_lock; - u16 if_type; /* HDLC/PPP or NET */ - - /* Failure cases */ - u8 failed_ring; - u8 failed_recv_alloc; - - /* Structure check */ - u32 check; -}; - -#define LMC_PCI_TIME 1 -#define LMC_EXT_TIME 0 - -#define PKT_BUF_SZ 1542 /* was 1536 */ - -/* CSR5 settings */ -#define TIMER_INT 0x00000800 -#define TP_LINK_FAIL 0x00001000 -#define TP_LINK_PASS 0x00000010 -#define NORMAL_INT 0x00010000 -#define ABNORMAL_INT 0x00008000 -#define RX_JABBER_INT 0x00000200 -#define RX_DIED 0x00000100 -#define RX_NOBUFF 0x00000080 -#define RX_INT 0x00000040 -#define TX_FIFO_UNDER 0x00000020 -#define TX_JABBER 0x00000008 -#define TX_NOBUFF 0x00000004 -#define TX_DIED 0x00000002 -#define TX_INT 0x00000001 - -/* CSR6 settings */ -#define OPERATION_MODE 0x00000200 /* Full Duplex */ -#define PROMISC_MODE 0x00000040 /* Promiscuous Mode */ -#define RECEIVE_ALL 0x40000000 /* Receive All */ -#define PASS_BAD_FRAMES 0x00000008 /* Pass Bad Frames */ - -/* Dec control registers CSR6 as well */ -#define LMC_DEC_ST 0x00002000 -#define LMC_DEC_SR 0x00000002 - -/* CSR15 settings */ -#define RECV_WATCHDOG_DISABLE 0x00000010 -#define JABBER_DISABLE 0x00000001 - -/* More settings */ -/* - * aSR6 -- Command (Operation Mode) Register - */ -#define TULIP_CMD_RECEIVEALL 0x40000000L /* (RW) Receivel all frames? */ -#define TULIP_CMD_MUSTBEONE 0x02000000L /* (RW) Must Be One (21140) */ -#define TULIP_CMD_TXTHRSHLDCTL 0x00400000L /* (RW) Transmit Threshold Mode (21140) */ -#define TULIP_CMD_STOREFWD 0x00200000L /* (RW) Store and Forward (21140) */ -#define TULIP_CMD_NOHEARTBEAT 0x00080000L /* (RW) No Heartbeat (21140) */ -#define TULIP_CMD_PORTSELECT 0x00040000L /* (RW) Post Select (100Mb) (21140) */ -#define TULIP_CMD_FULLDUPLEX 0x00000200L /* (RW) Full Duplex Mode */ -#define TULIP_CMD_OPERMODE 0x00000C00L /* (RW) Operating Mode */ -#define TULIP_CMD_PROMISCUOUS 0x00000041L /* (RW) Promiscuous Mode */ -#define TULIP_CMD_PASSBADPKT 0x00000008L /* (RW) Pass Bad Frames */ -#define TULIP_CMD_THRESHOLDCTL 0x0000C000L /* (RW) Threshold Control */ - -#define TULIP_GP_PINSET 0x00000100L -#define TULIP_BUSMODE_SWRESET 0x00000001L -#define TULIP_WATCHDOG_TXDISABLE 0x00000001L -#define TULIP_WATCHDOG_RXDISABLE 0x00000010L - -#define TULIP_STS_NORMALINTR 0x00010000L /* (RW) Normal Interrupt */ -#define TULIP_STS_ABNRMLINTR 0x00008000L /* (RW) Abnormal Interrupt */ -#define TULIP_STS_ERI 0x00004000L /* (RW) Early Receive Interrupt */ -#define TULIP_STS_SYSERROR 0x00002000L /* (RW) System Error */ -#define TULIP_STS_GTE 0x00000800L /* (RW) General Pupose Timer Exp */ -#define TULIP_STS_ETI 0x00000400L /* (RW) Early Transmit Interrupt */ -#define TULIP_STS_RXWT 0x00000200L /* (RW) Receiver Watchdog Timeout */ -#define TULIP_STS_RXSTOPPED 0x00000100L /* (RW) Receiver Process Stopped */ -#define TULIP_STS_RXNOBUF 0x00000080L /* (RW) Receive Buf Unavail */ -#define TULIP_STS_RXINTR 0x00000040L /* (RW) Receive Interrupt */ -#define TULIP_STS_TXUNDERFLOW 0x00000020L /* (RW) Transmit Underflow */ -#define TULIP_STS_TXJABER 0x00000008L /* (RW) Jabber timeout */ -#define TULIP_STS_TXNOBUF 0x00000004L -#define TULIP_STS_TXSTOPPED 0x00000002L /* (RW) Transmit Process Stopped */ -#define TULIP_STS_TXINTR 0x00000001L /* (RW) Transmit Interrupt */ - -#define TULIP_STS_RXS_STOPPED 0x00000000L /* 000 - Stopped */ - -#define TULIP_STS_RXSTOPPED 0x00000100L /* (RW) Receive Process Stopped */ -#define TULIP_STS_RXNOBUF 0x00000080L - -#define TULIP_CMD_TXRUN 0x00002000L /* (RW) Start/Stop Transmitter */ -#define TULIP_CMD_RXRUN 0x00000002L /* (RW) Start/Stop Receive Filtering */ -#define TULIP_DSTS_TxDEFERRED 0x00000001 /* Initially Deferred */ -#define TULIP_DSTS_OWNER 0x80000000 /* Owner (1 = 21040) */ -#define TULIP_DSTS_RxMIIERR 0x00000008 -#define LMC_DSTS_ERRSUM (TULIP_DSTS_RxMIIERR) - -#define TULIP_DEFAULT_INTR_MASK (TULIP_STS_NORMALINTR \ - | TULIP_STS_RXINTR \ - | TULIP_STS_TXINTR \ - | TULIP_STS_ABNRMLINTR \ - | TULIP_STS_SYSERROR \ - | TULIP_STS_TXSTOPPED \ - | TULIP_STS_TXUNDERFLOW\ - | TULIP_STS_RXSTOPPED ) - -#define DESC_OWNED_BY_SYSTEM ((u32)(0x00000000)) -#define DESC_OWNED_BY_DC21X4 ((u32)(0x80000000)) - -#ifndef TULIP_CMD_RECEIVEALL -#define TULIP_CMD_RECEIVEALL 0x40000000L -#endif - -/* Adapter module number */ -#define LMC_ADAP_HSSI 2 -#define LMC_ADAP_DS3 3 -#define LMC_ADAP_SSI 4 -#define LMC_ADAP_T1 5 - -#define LMC_MTU 1500 - -#define LMC_CRC_LEN_16 2 /* 16-bit CRC */ -#define LMC_CRC_LEN_32 4 - -#endif /* _LMC_VAR_H_ */ diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 677fa4bf76d3..efb85c6d8e2d 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1830,9 +1830,6 @@ static void nvme_update_disk_info(struct gendisk *disk, nvme_config_discard(disk, ns); blk_queue_max_write_zeroes_sectors(disk->queue, ns->ctrl->max_zeroes_sectors); - - set_disk_ro(disk, (id->nsattr & NVME_NS_ATTR_RO) || - test_bit(NVME_NS_FORCE_RO, &ns->flags)); } static inline bool nvme_first_scan(struct gendisk *disk) @@ -1891,6 +1888,8 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) goto out_unfreeze; } + set_disk_ro(ns->disk, (id->nsattr & NVME_NS_ATTR_RO) || + test_bit(NVME_NS_FORCE_RO, &ns->flags)); set_bit(NVME_NS_READY, &ns->flags); blk_mq_unfreeze_queue(ns->disk->queue); @@ -1903,6 +1902,9 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) if (nvme_ns_head_multipath(ns->head)) { blk_mq_freeze_queue(ns->head->disk->queue); nvme_update_disk_info(ns->head->disk, ns, id); + set_disk_ro(ns->head->disk, + (id->nsattr & NVME_NS_ATTR_RO) || + test_bit(NVME_NS_FORCE_RO, &ns->flags)); nvme_mpath_revalidate_paths(ns); blk_stack_limits(&ns->head->disk->queue->limits, &ns->queue->limits, 0); @@ -3589,15 +3591,20 @@ static const struct attribute_group *nvme_dev_attr_groups[] = { NULL, }; -static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys, +static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, unsigned nsid) { struct nvme_ns_head *h; - lockdep_assert_held(&subsys->lock); + lockdep_assert_held(&ctrl->subsys->lock); - list_for_each_entry(h, &subsys->nsheads, entry) { - if (h->ns_id != nsid) + list_for_each_entry(h, &ctrl->subsys->nsheads, entry) { + /* + * Private namespaces can share NSIDs under some conditions. + * In that case we can't use the same ns_head for namespaces + * with the same NSID. + */ + if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h)) continue; if (!list_empty(&h->list) && nvme_tryget_ns_head(h)) return h; @@ -3791,7 +3798,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, } mutex_lock(&ctrl->subsys->lock); - head = nvme_find_ns_head(ctrl->subsys, nsid); + head = nvme_find_ns_head(ctrl, nsid); if (!head) { ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, ids); if (ret) { @@ -3988,6 +3995,16 @@ static void nvme_ns_remove(struct nvme_ns *ns) set_capacity(ns->disk, 0); nvme_fault_inject_fini(&ns->fault_inject); + /* + * Ensure that !NVME_NS_READY is seen by other threads to prevent + * this ns going back into current_path. + */ + synchronize_srcu(&ns->head->srcu); + + /* wait for concurrent submissions */ + if (nvme_mpath_clear_current_path(ns)) + synchronize_srcu(&ns->head->srcu); + mutex_lock(&ns->ctrl->subsys->lock); list_del_rcu(&ns->siblings); if (list_empty(&ns->head->list)) { @@ -3999,10 +4016,6 @@ static void nvme_ns_remove(struct nvme_ns *ns) /* guarantee not available in head->list */ synchronize_rcu(); - /* wait for concurrent submissions */ - if (nvme_mpath_clear_current_path(ns)) - synchronize_srcu(&ns->head->srcu); - if (!nvme_ns_head_multipath(ns->head)) nvme_cdev_del(&ns->cdev, &ns->cdev_device); del_gendisk(ns->disk); @@ -4480,6 +4493,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl) if (ctrl->queue_count > 1) { nvme_queue_scan(ctrl); nvme_start_queues(ctrl); + nvme_mpath_update(ctrl); } nvme_change_uevent(ctrl, "NVME_EVENT=connected"); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 1b31f19e1053..d464fdf978fb 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -482,10 +482,11 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) /* * Add a multipath node if the subsystems supports multiple controllers. - * We also do this for private namespaces as the namespace sharing data could - * change after a rescan. + * We also do this for private namespaces as the namespace sharing flag + * could change after a rescan. */ - if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || !multipath) + if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || + !nvme_is_unique_nsid(ctrl, head) || !multipath) return 0; head->disk = blk_alloc_disk(ctrl->numa_node); @@ -612,8 +613,17 @@ static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, ns->ana_grpid = le32_to_cpu(desc->grpid); ns->ana_state = desc->state; clear_bit(NVME_NS_ANA_PENDING, &ns->flags); - - if (nvme_state_is_live(ns->ana_state)) + /* + * nvme_mpath_set_live() will trigger I/O to the multipath path device + * and in turn to this path device. However we cannot accept this I/O + * if the controller is not live. This may deadlock if called from + * nvme_mpath_init_identify() and the ctrl will never complete + * initialization, preventing I/O from completing. For this case we + * will reprocess the ANA log page in nvme_mpath_update() once the + * controller is ready. + */ + if (nvme_state_is_live(ns->ana_state) && + ns->ctrl->state == NVME_CTRL_LIVE) nvme_mpath_set_live(ns); } @@ -700,6 +710,18 @@ static void nvme_ana_work(struct work_struct *work) nvme_read_ana_log(ctrl); } +void nvme_mpath_update(struct nvme_ctrl *ctrl) +{ + u32 nr_change_groups = 0; + + if (!ctrl->ana_log_buf) + return; + + mutex_lock(&ctrl->ana_lock); + nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); + mutex_unlock(&ctrl->ana_lock); +} + static void nvme_anatt_timeout(struct timer_list *t) { struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index f4b674a8ce20..1393bbf82d71 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -723,6 +723,25 @@ static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, return queue_live; return __nvme_check_ready(ctrl, rq, queue_live); } + +/* + * NSID shall be unique for all shared namespaces, or if at least one of the + * following conditions is met: + * 1. Namespace Management is supported by the controller + * 2. ANA is supported by the controller + * 3. NVM Set are supported by the controller + * + * In other case, private namespace are not required to report a unique NSID. + */ +static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl, + struct nvme_ns_head *head) +{ + return head->shared || + (ctrl->oacs & NVME_CTRL_OACS_NS_MNGT_SUPP) || + (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) || + (ctrl->ctratt & NVME_CTRL_CTRATT_NVM_SETS); +} + int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, void *buf, unsigned bufflen); int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, @@ -782,6 +801,7 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id); void nvme_mpath_remove_disk(struct nvme_ns_head *head); int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id); void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl); +void nvme_mpath_update(struct nvme_ctrl *ctrl); void nvme_mpath_uninit(struct nvme_ctrl *ctrl); void nvme_mpath_stop(struct nvme_ctrl *ctrl); bool nvme_mpath_clear_current_path(struct nvme_ns *ns); @@ -853,6 +873,9 @@ static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n"); return 0; } +static inline void nvme_mpath_update(struct nvme_ctrl *ctrl) +{ +} static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl) { } diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 2e98ac3f3ad6..d817ca17463e 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -45,7 +45,7 @@ #define NVME_MAX_SEGS 127 static int use_threaded_interrupts; -module_param(use_threaded_interrupts, int, 0); +module_param(use_threaded_interrupts, int, 0444); static bool use_cmb_sqes = true; module_param(use_cmb_sqes, bool, 0444); @@ -3467,7 +3467,10 @@ static const struct pci_device_id nvme_id_table[] = { NVME_QUIRK_128_BYTES_SQES | NVME_QUIRK_SHARED_TAGS | NVME_QUIRK_SKIP_CID_GEN }, - + { PCI_DEVICE(0x144d, 0xa808), /* Samsung X5 */ + .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY| + NVME_QUIRK_NO_DEEPEST_PS | + NVME_QUIRK_IGNORE_DEV_SUBNQN, }, { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, { 0, } }; diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 46d0dab686dd..397daaf51f1b 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -988,7 +988,7 @@ void nvmet_execute_async_event(struct nvmet_req *req) ctrl->async_event_cmds[ctrl->nr_async_event_cmds++] = req; mutex_unlock(&ctrl->lock); - schedule_work(&ctrl->async_event_work); + queue_work(nvmet_wq, &ctrl->async_event_work); } void nvmet_execute_keep_alive(struct nvmet_req *req) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 8fedd1e052fe..e44b2988759e 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1555,7 +1555,7 @@ static void nvmet_port_release(struct config_item *item) struct nvmet_port *port = to_nvmet_port(item); /* Let inflight controllers teardown complete */ - flush_scheduled_work(); + flush_workqueue(nvmet_wq); list_del(&port->global_entry); kfree(port->ana_state); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 64c2d2f3e25c..90e75324dae0 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -20,6 +20,9 @@ struct workqueue_struct *zbd_wq; static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX]; static DEFINE_IDA(cntlid_ida); +struct workqueue_struct *nvmet_wq; +EXPORT_SYMBOL_GPL(nvmet_wq); + /* * This read/write semaphore is used to synchronize access to configuration * information on a target system that will result in discovery log page @@ -205,7 +208,7 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, list_add_tail(&aen->entry, &ctrl->async_events); mutex_unlock(&ctrl->lock); - schedule_work(&ctrl->async_event_work); + queue_work(nvmet_wq, &ctrl->async_event_work); } static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) @@ -385,7 +388,7 @@ static void nvmet_keep_alive_timer(struct work_struct *work) if (reset_tbkas) { pr_debug("ctrl %d reschedule traffic based keep-alive timer\n", ctrl->cntlid); - schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ); + queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); return; } @@ -403,7 +406,7 @@ void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl) pr_debug("ctrl %d start keep-alive timer for %d secs\n", ctrl->cntlid, ctrl->kato); - schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ); + queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); } void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl) @@ -1120,7 +1123,7 @@ static inline u8 nvmet_cc_iocqes(u32 cc) static inline bool nvmet_css_supported(u8 cc_css) { - switch (cc_css <<= NVME_CC_CSS_SHIFT) { + switch (cc_css << NVME_CC_CSS_SHIFT) { case NVME_CC_CSS_NVM: case NVME_CC_CSS_CSI: return true; @@ -1478,7 +1481,7 @@ void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl) mutex_lock(&ctrl->lock); if (!(ctrl->csts & NVME_CSTS_CFS)) { ctrl->csts |= NVME_CSTS_CFS; - schedule_work(&ctrl->fatal_err_work); + queue_work(nvmet_wq, &ctrl->fatal_err_work); } mutex_unlock(&ctrl->lock); } @@ -1619,9 +1622,15 @@ static int __init nvmet_init(void) goto out_free_zbd_work_queue; } + nvmet_wq = alloc_workqueue("nvmet-wq", WQ_MEM_RECLAIM, 0); + if (!nvmet_wq) { + error = -ENOMEM; + goto out_free_buffered_work_queue; + } + error = nvmet_init_discovery(); if (error) - goto out_free_work_queue; + goto out_free_nvmet_work_queue; error = nvmet_init_configfs(); if (error) @@ -1630,7 +1639,9 @@ static int __init nvmet_init(void) out_exit_discovery: nvmet_exit_discovery(); -out_free_work_queue: +out_free_nvmet_work_queue: + destroy_workqueue(nvmet_wq); +out_free_buffered_work_queue: destroy_workqueue(buffered_io_wq); out_free_zbd_work_queue: destroy_workqueue(zbd_wq); @@ -1642,6 +1653,7 @@ static void __exit nvmet_exit(void) nvmet_exit_configfs(); nvmet_exit_discovery(); ida_destroy(&cntlid_ida); + destroy_workqueue(nvmet_wq); destroy_workqueue(buffered_io_wq); destroy_workqueue(zbd_wq); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index de90001fc5c4..ab2627e17bb9 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1491,7 +1491,7 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) { if (!nvmet_fc_tgt_a_get(assoc)) continue; - if (!schedule_work(&assoc->del_work)) + if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); } @@ -1546,7 +1546,7 @@ nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port, continue; assoc->hostport->invalid = 1; noassoc = false; - if (!schedule_work(&assoc->del_work)) + if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); } @@ -1592,7 +1592,7 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) nvmet_fc_tgtport_put(tgtport); if (found_ctrl) { - if (!schedule_work(&assoc->del_work)) + if (!queue_work(nvmet_wq, &assoc->del_work)) /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); return; @@ -2060,7 +2060,7 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, iod->rqstdatalen = lsreqbuf_len; iod->hosthandle = hosthandle; - schedule_work(&iod->work); + queue_work(nvmet_wq, &iod->work); return 0; } diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 54606f1872b4..5c16372f3b53 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -360,7 +360,7 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport, spin_lock(&rport->lock); list_add_tail(&rport->ls_list, &tls_req->ls_list); spin_unlock(&rport->lock); - schedule_work(&rport->ls_work); + queue_work(nvmet_wq, &rport->ls_work); return ret; } @@ -393,7 +393,7 @@ fcloop_h2t_xmt_ls_rsp(struct nvmet_fc_target_port *targetport, spin_lock(&rport->lock); list_add_tail(&rport->ls_list, &tls_req->ls_list); spin_unlock(&rport->lock); - schedule_work(&rport->ls_work); + queue_work(nvmet_wq, &rport->ls_work); } return 0; @@ -448,7 +448,7 @@ fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle, spin_lock(&tport->lock); list_add_tail(&tport->ls_list, &tls_req->ls_list); spin_unlock(&tport->lock); - schedule_work(&tport->ls_work); + queue_work(nvmet_wq, &tport->ls_work); return ret; } @@ -480,7 +480,7 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, spin_lock(&tport->lock); list_add_tail(&tport->ls_list, &tls_req->ls_list); spin_unlock(&tport->lock); - schedule_work(&tport->ls_work); + queue_work(nvmet_wq, &tport->ls_work); } return 0; @@ -520,7 +520,7 @@ fcloop_tgt_discovery_evt(struct nvmet_fc_target_port *tgtport) tgt_rscn->tport = tgtport->private; INIT_WORK(&tgt_rscn->work, fcloop_tgt_rscn_work); - schedule_work(&tgt_rscn->work); + queue_work(nvmet_wq, &tgt_rscn->work); } static void @@ -739,7 +739,7 @@ fcloop_fcp_req(struct nvme_fc_local_port *localport, INIT_WORK(&tfcp_req->tio_done_work, fcloop_tgt_fcprqst_done_work); kref_init(&tfcp_req->ref); - schedule_work(&tfcp_req->fcp_rcv_work); + queue_work(nvmet_wq, &tfcp_req->fcp_rcv_work); return 0; } @@ -921,7 +921,7 @@ fcloop_fcp_req_release(struct nvmet_fc_target_port *tgtport, { struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq); - schedule_work(&tfcp_req->tio_done_work); + queue_work(nvmet_wq, &tfcp_req->tio_done_work); } static void @@ -976,7 +976,7 @@ fcloop_fcp_abort(struct nvme_fc_local_port *localport, if (abortio) /* leave the reference while the work item is scheduled */ - WARN_ON(!schedule_work(&tfcp_req->abort_rcv_work)); + WARN_ON(!queue_work(nvmet_wq, &tfcp_req->abort_rcv_work)); else { /* * as the io has already had the done callback made, diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index 6485dc8eb974..f3d58abf11e0 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -283,7 +283,7 @@ static void nvmet_file_execute_flush(struct nvmet_req *req) if (!nvmet_check_transfer_len(req, 0)) return; INIT_WORK(&req->f.work, nvmet_file_flush_work); - schedule_work(&req->f.work); + queue_work(nvmet_wq, &req->f.work); } static void nvmet_file_execute_discard(struct nvmet_req *req) @@ -343,7 +343,7 @@ static void nvmet_file_execute_dsm(struct nvmet_req *req) if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req))) return; INIT_WORK(&req->f.work, nvmet_file_dsm_work); - schedule_work(&req->f.work); + queue_work(nvmet_wq, &req->f.work); } static void nvmet_file_write_zeroes_work(struct work_struct *w) @@ -373,7 +373,7 @@ static void nvmet_file_execute_write_zeroes(struct nvmet_req *req) if (!nvmet_check_transfer_len(req, 0)) return; INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work); - schedule_work(&req->f.work); + queue_work(nvmet_wq, &req->f.work); } u16 nvmet_file_parse_io_cmd(struct nvmet_req *req) diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index 23f9d6f88804..59024af2da2e 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -166,7 +166,7 @@ static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, iod->req.transfer_len = blk_rq_payload_bytes(req); } - schedule_work(&iod->work); + queue_work(nvmet_wq, &iod->work); return BLK_STS_OK; } @@ -187,7 +187,7 @@ static void nvme_loop_submit_async_event(struct nvme_ctrl *arg) return; } - schedule_work(&iod->work); + queue_work(nvmet_wq, &iod->work); } static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index d910c6aad4b6..69818752a33a 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -366,6 +366,7 @@ struct nvmet_req { extern struct workqueue_struct *buffered_io_wq; extern struct workqueue_struct *zbd_wq; +extern struct workqueue_struct *nvmet_wq; static inline void nvmet_set_result(struct nvmet_req *req, u32 result) { diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index a4de1e0d518b..5247c24538eb 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -283,7 +283,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req) if (req->p.use_workqueue || effects) { INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work); req->p.rq = rq; - schedule_work(&req->p.work); + queue_work(nvmet_wq, &req->p.work); } else { rq->end_io_data = req; blk_execute_rq_nowait(rq, false, nvmet_passthru_req_done); diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 2446d0918a41..2fab0b219b25 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -1584,7 +1584,7 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, if (queue->host_qid == 0) { /* Let inflight controller teardown complete */ - flush_scheduled_work(); + flush_workqueue(nvmet_wq); } ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); @@ -1669,7 +1669,7 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) if (disconnect) { rdma_disconnect(queue->cm_id); - schedule_work(&queue->release_work); + queue_work(nvmet_wq, &queue->release_work); } } @@ -1699,7 +1699,7 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, mutex_unlock(&nvmet_rdma_queue_mutex); pr_err("failed to connect queue %d\n", queue->idx); - schedule_work(&queue->release_work); + queue_work(nvmet_wq, &queue->release_work); } /** @@ -1773,7 +1773,7 @@ static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id, if (!queue) { struct nvmet_rdma_port *port = cm_id->context; - schedule_delayed_work(&port->repair_work, 0); + queue_delayed_work(nvmet_wq, &port->repair_work, 0); break; } fallthrough; @@ -1903,7 +1903,7 @@ static void nvmet_rdma_repair_port_work(struct work_struct *w) nvmet_rdma_disable_port(port); ret = nvmet_rdma_enable_port(port); if (ret) - schedule_delayed_work(&port->repair_work, 5 * HZ); + queue_delayed_work(nvmet_wq, &port->repair_work, 5 * HZ); } static int nvmet_rdma_add_port(struct nvmet_port *nport) @@ -2053,7 +2053,7 @@ static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data } mutex_unlock(&nvmet_rdma_queue_mutex); - flush_scheduled_work(); + flush_workqueue(nvmet_wq); } static struct ib_client nvmet_rdma_ib_client = { diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 83ca577f72be..2793554e622e 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1269,7 +1269,7 @@ static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue) spin_lock(&queue->state_lock); if (queue->state != NVMET_TCP_Q_DISCONNECTING) { queue->state = NVMET_TCP_Q_DISCONNECTING; - schedule_work(&queue->release_work); + queue_work(nvmet_wq, &queue->release_work); } spin_unlock(&queue->state_lock); } @@ -1684,7 +1684,7 @@ static void nvmet_tcp_listen_data_ready(struct sock *sk) goto out; if (sk->sk_state == TCP_LISTEN) - schedule_work(&port->accept_work); + queue_work(nvmet_wq, &port->accept_work); out: read_unlock_bh(&sk->sk_callback_lock); } @@ -1815,7 +1815,7 @@ static u16 nvmet_tcp_install_queue(struct nvmet_sq *sq) if (sq->qid == 0) { /* Let inflight controller teardown complete */ - flush_scheduled_work(); + flush_workqueue(nvmet_wq); } queue->nr_cmds = sq->size * 2; @@ -1876,12 +1876,12 @@ static void __exit nvmet_tcp_exit(void) nvmet_unregister_transport(&nvmet_tcp_ops); - flush_scheduled_work(); + flush_workqueue(nvmet_wq); mutex_lock(&nvmet_tcp_queue_mutex); list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) kernel_sock_shutdown(queue->sock, SHUT_RDWR); mutex_unlock(&nvmet_tcp_queue_mutex); - flush_scheduled_work(); + flush_workqueue(nvmet_wq); destroy_workqueue(nvmet_tcp_wq); } diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index df84d221e3de..d270a204324e 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -766,14 +766,6 @@ static unsigned int hv_msi_get_int_vector(struct irq_data *irqd) return irqd->parent_data->hwirq; } -static void hv_set_msi_entry_from_desc(union hv_msi_entry *msi_entry, - struct msi_desc *msi_desc) -{ - msi_entry->address = ((u64)msi_desc->msg.address_hi << 32) | - msi_desc->msg.address_lo; - msi_entry->data = msi_desc->msg.data; -} - /* * @nr_bm_irqs: Indicates the number of IRQs that were allocated from * the bitmap. @@ -3415,6 +3407,15 @@ static int hv_pci_probe(struct hv_device *hdev, hbus->bridge->domain_nr = dom; #ifdef CONFIG_X86 hbus->sysdata.domain = dom; +#elif defined(CONFIG_ARM64) + /* + * Set the PCI bus parent to be the corresponding VMbus + * device. Then the VMbus device will be assigned as the + * ACPI companion in pcibios_root_bridge_prepare() and + * pci_dma_configure() will propagate device coherence + * information to devices created on the bus. + */ + hbus->sysdata.parent = hdev->device.parent; #endif hbus->hdev = hdev; diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index 5d4be9735d9d..6420ca129548 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -2,6 +2,7 @@ # tell define_trace.h where to find the cros ec trace header CFLAGS_cros_ec_trace.o:= -I$(src) +CFLAGS_cros_ec_sensorhub_ring.o:= -I$(src) obj-$(CONFIG_CHROMEOS_LAPTOP) += chromeos_laptop.o obj-$(CONFIG_CHROMEOS_PRIVACY_SCREEN) += chromeos_privacy_screen.o @@ -21,7 +22,7 @@ obj-$(CONFIG_CROS_EC_CHARDEV) += cros_ec_chardev.o obj-$(CONFIG_CROS_EC_LIGHTBAR) += cros_ec_lightbar.o obj-$(CONFIG_CROS_EC_VBC) += cros_ec_vbc.o obj-$(CONFIG_CROS_EC_DEBUGFS) += cros_ec_debugfs.o -cros-ec-sensorhub-objs := cros_ec_sensorhub.o cros_ec_sensorhub_ring.o cros_ec_trace.o +cros-ec-sensorhub-objs := cros_ec_sensorhub.o cros_ec_sensorhub_ring.o obj-$(CONFIG_CROS_EC_SENSORHUB) += cros-ec-sensorhub.o obj-$(CONFIG_CROS_EC_SYSFS) += cros_ec_sysfs.o obj-$(CONFIG_CROS_USBPD_LOGGER) += cros_usbpd_logger.o diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 272c89837d74..0dbceee87a4b 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -25,6 +25,9 @@ #define CIRC_ADD(idx, size, value) (((idx) + (value)) & ((size) - 1)) +/* waitqueue for log readers */ +static DECLARE_WAIT_QUEUE_HEAD(cros_ec_debugfs_log_wq); + /** * struct cros_ec_debugfs - EC debugging information. * @@ -33,7 +36,6 @@ * @log_buffer: circular buffer for console log information * @read_msg: preallocated EC command and buffer to read console log * @log_mutex: mutex to protect circular buffer - * @log_wq: waitqueue for log readers * @log_poll_work: recurring task to poll EC for new console log data * @panicinfo_blob: panicinfo debugfs blob */ @@ -44,7 +46,6 @@ struct cros_ec_debugfs { struct circ_buf log_buffer; struct cros_ec_command *read_msg; struct mutex log_mutex; - wait_queue_head_t log_wq; struct delayed_work log_poll_work; /* EC panicinfo */ struct debugfs_blob_wrapper panicinfo_blob; @@ -107,7 +108,7 @@ static void cros_ec_console_log_work(struct work_struct *__work) buf_space--; } - wake_up(&debug_info->log_wq); + wake_up(&cros_ec_debugfs_log_wq); } mutex_unlock(&debug_info->log_mutex); @@ -141,7 +142,7 @@ static ssize_t cros_ec_console_log_read(struct file *file, char __user *buf, mutex_unlock(&debug_info->log_mutex); - ret = wait_event_interruptible(debug_info->log_wq, + ret = wait_event_interruptible(cros_ec_debugfs_log_wq, CIRC_CNT(cb->head, cb->tail, LOG_SIZE)); if (ret < 0) return ret; @@ -173,7 +174,7 @@ static __poll_t cros_ec_console_log_poll(struct file *file, struct cros_ec_debugfs *debug_info = file->private_data; __poll_t mask = 0; - poll_wait(file, &debug_info->log_wq, wait); + poll_wait(file, &cros_ec_debugfs_log_wq, wait); mutex_lock(&debug_info->log_mutex); if (CIRC_CNT(debug_info->log_buffer.head, @@ -377,7 +378,6 @@ static int cros_ec_create_console_log(struct cros_ec_debugfs *debug_info) debug_info->log_buffer.tail = 0; mutex_init(&debug_info->log_mutex); - init_waitqueue_head(&debug_info->log_wq); debugfs_create_file("console_log", S_IFREG | 0444, debug_info->dir, debug_info, &cros_ec_console_log_fops); diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index 98e37080f760..71948dade0e2 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -17,7 +17,8 @@ #include <linux/sort.h> #include <linux/slab.h> -#include "cros_ec_trace.h" +#define CREATE_TRACE_POINTS +#include "cros_ec_sensorhub_trace.h" /* Precision of fixed point for the m values from the filter */ #define M_PRECISION BIT(23) diff --git a/drivers/platform/chrome/cros_ec_sensorhub_trace.h b/drivers/platform/chrome/cros_ec_sensorhub_trace.h new file mode 100644 index 000000000000..57d9b4785969 --- /dev/null +++ b/drivers/platform/chrome/cros_ec_sensorhub_trace.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Trace events for the ChromeOS Sensorhub kernel module + * + * Copyright 2021 Google LLC. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cros_ec + +#if !defined(_CROS_EC_SENSORHUB_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _CROS_EC_SENSORHUB_TRACE_H_ + +#include <linux/types.h> +#include <linux/platform_data/cros_ec_sensorhub.h> + +#include <linux/tracepoint.h> + +TRACE_EVENT(cros_ec_sensorhub_timestamp, + TP_PROTO(u32 ec_sample_timestamp, u32 ec_fifo_timestamp, s64 fifo_timestamp, + s64 current_timestamp, s64 current_time), + TP_ARGS(ec_sample_timestamp, ec_fifo_timestamp, fifo_timestamp, current_timestamp, + current_time), + TP_STRUCT__entry( + __field(u32, ec_sample_timestamp) + __field(u32, ec_fifo_timestamp) + __field(s64, fifo_timestamp) + __field(s64, current_timestamp) + __field(s64, current_time) + __field(s64, delta) + ), + TP_fast_assign( + __entry->ec_sample_timestamp = ec_sample_timestamp; + __entry->ec_fifo_timestamp = ec_fifo_timestamp; + __entry->fifo_timestamp = fifo_timestamp; + __entry->current_timestamp = current_timestamp; + __entry->current_time = current_time; + __entry->delta = current_timestamp - current_time; + ), + TP_printk("ec_ts: %9u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld", + __entry->ec_sample_timestamp, + __entry->ec_fifo_timestamp, + __entry->fifo_timestamp, + __entry->current_timestamp, + __entry->current_time, + __entry->delta + ) +); + +TRACE_EVENT(cros_ec_sensorhub_data, + TP_PROTO(u32 ec_sensor_num, u32 ec_fifo_timestamp, s64 fifo_timestamp, + s64 current_timestamp, s64 current_time), + TP_ARGS(ec_sensor_num, ec_fifo_timestamp, fifo_timestamp, current_timestamp, current_time), + TP_STRUCT__entry( + __field(u32, ec_sensor_num) + __field(u32, ec_fifo_timestamp) + __field(s64, fifo_timestamp) + __field(s64, current_timestamp) + __field(s64, current_time) + __field(s64, delta) + ), + TP_fast_assign( + __entry->ec_sensor_num = ec_sensor_num; + __entry->ec_fifo_timestamp = ec_fifo_timestamp; + __entry->fifo_timestamp = fifo_timestamp; + __entry->current_timestamp = current_timestamp; + __entry->current_time = current_time; + __entry->delta = current_timestamp - current_time; + ), + TP_printk("ec_num: %4u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld", + __entry->ec_sensor_num, + __entry->ec_fifo_timestamp, + __entry->fifo_timestamp, + __entry->current_timestamp, + __entry->current_time, + __entry->delta + ) +); + +TRACE_EVENT(cros_ec_sensorhub_filter, + TP_PROTO(struct cros_ec_sensors_ts_filter_state *state, s64 dx, s64 dy), + TP_ARGS(state, dx, dy), + TP_STRUCT__entry( + __field(s64, dx) + __field(s64, dy) + __field(s64, median_m) + __field(s64, median_error) + __field(s64, history_len) + __field(s64, x) + __field(s64, y) + ), + TP_fast_assign( + __entry->dx = dx; + __entry->dy = dy; + __entry->median_m = state->median_m; + __entry->median_error = state->median_error; + __entry->history_len = state->history_len; + __entry->x = state->x_offset; + __entry->y = state->y_offset; + ), + TP_printk("dx: %12lld. dy: %12lld median_m: %12lld median_error: %12lld len: %lld x: %12lld y: %12lld", + __entry->dx, + __entry->dy, + __entry->median_m, + __entry->median_error, + __entry->history_len, + __entry->x, + __entry->y + ) +); + + +#endif /* _CROS_EC_SENSORHUB_TRACE_H_ */ + +/* this part must be outside header guard */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE cros_ec_sensorhub_trace + +#include <trace/define_trace.h> diff --git a/drivers/platform/chrome/cros_ec_trace.h b/drivers/platform/chrome/cros_ec_trace.h index 7e7cfc98657a..9bb5cd2c98b8 100644 --- a/drivers/platform/chrome/cros_ec_trace.h +++ b/drivers/platform/chrome/cros_ec_trace.h @@ -15,7 +15,6 @@ #include <linux/types.h> #include <linux/platform_data/cros_ec_commands.h> #include <linux/platform_data/cros_ec_proto.h> -#include <linux/platform_data/cros_ec_sensorhub.h> #include <linux/tracepoint.h> @@ -71,100 +70,6 @@ TRACE_EVENT(cros_ec_request_done, __entry->retval) ); -TRACE_EVENT(cros_ec_sensorhub_timestamp, - TP_PROTO(u32 ec_sample_timestamp, u32 ec_fifo_timestamp, s64 fifo_timestamp, - s64 current_timestamp, s64 current_time), - TP_ARGS(ec_sample_timestamp, ec_fifo_timestamp, fifo_timestamp, current_timestamp, - current_time), - TP_STRUCT__entry( - __field(u32, ec_sample_timestamp) - __field(u32, ec_fifo_timestamp) - __field(s64, fifo_timestamp) - __field(s64, current_timestamp) - __field(s64, current_time) - __field(s64, delta) - ), - TP_fast_assign( - __entry->ec_sample_timestamp = ec_sample_timestamp; - __entry->ec_fifo_timestamp = ec_fifo_timestamp; - __entry->fifo_timestamp = fifo_timestamp; - __entry->current_timestamp = current_timestamp; - __entry->current_time = current_time; - __entry->delta = current_timestamp - current_time; - ), - TP_printk("ec_ts: %9u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld", - __entry->ec_sample_timestamp, - __entry->ec_fifo_timestamp, - __entry->fifo_timestamp, - __entry->current_timestamp, - __entry->current_time, - __entry->delta - ) -); - -TRACE_EVENT(cros_ec_sensorhub_data, - TP_PROTO(u32 ec_sensor_num, u32 ec_fifo_timestamp, s64 fifo_timestamp, - s64 current_timestamp, s64 current_time), - TP_ARGS(ec_sensor_num, ec_fifo_timestamp, fifo_timestamp, current_timestamp, current_time), - TP_STRUCT__entry( - __field(u32, ec_sensor_num) - __field(u32, ec_fifo_timestamp) - __field(s64, fifo_timestamp) - __field(s64, current_timestamp) - __field(s64, current_time) - __field(s64, delta) - ), - TP_fast_assign( - __entry->ec_sensor_num = ec_sensor_num; - __entry->ec_fifo_timestamp = ec_fifo_timestamp; - __entry->fifo_timestamp = fifo_timestamp; - __entry->current_timestamp = current_timestamp; - __entry->current_time = current_time; - __entry->delta = current_timestamp - current_time; - ), - TP_printk("ec_num: %4u, ec_fifo_ts: %9u, fifo_ts: %12lld, curr_ts: %12lld, curr_time: %12lld, delta %12lld", - __entry->ec_sensor_num, - __entry->ec_fifo_timestamp, - __entry->fifo_timestamp, - __entry->current_timestamp, - __entry->current_time, - __entry->delta - ) -); - -TRACE_EVENT(cros_ec_sensorhub_filter, - TP_PROTO(struct cros_ec_sensors_ts_filter_state *state, s64 dx, s64 dy), - TP_ARGS(state, dx, dy), - TP_STRUCT__entry( - __field(s64, dx) - __field(s64, dy) - __field(s64, median_m) - __field(s64, median_error) - __field(s64, history_len) - __field(s64, x) - __field(s64, y) - ), - TP_fast_assign( - __entry->dx = dx; - __entry->dy = dy; - __entry->median_m = state->median_m; - __entry->median_error = state->median_error; - __entry->history_len = state->history_len; - __entry->x = state->x_offset; - __entry->y = state->y_offset; - ), - TP_printk("dx: %12lld. dy: %12lld median_m: %12lld median_error: %12lld len: %lld x: %12lld y: %12lld", - __entry->dx, - __entry->dy, - __entry->median_m, - __entry->median_error, - __entry->history_len, - __entry->x, - __entry->y - ) -); - - #endif /* _CROS_EC_TRACE_H_ */ /* this part must be outside header guard */ diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 5de0bfb0bc4d..4bd2752c0823 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -115,17 +115,18 @@ static int cros_typec_parse_port_props(struct typec_capability *cap, return ret; cap->data = ret; + /* Try-power-role is optional. */ ret = fwnode_property_read_string(fwnode, "try-power-role", &buf); if (ret) { - dev_err(dev, "try-power-role not found: %d\n", ret); - return ret; + dev_warn(dev, "try-power-role not found: %d\n", ret); + cap->prefer_role = TYPEC_NO_PREFERRED_ROLE; + } else { + ret = typec_find_power_role(buf); + if (ret < 0) + return ret; + cap->prefer_role = ret; } - ret = typec_find_power_role(buf); - if (ret < 0) - return ret; - cap->prefer_role = ret; - cap->fwnode = fwnode; return 0; @@ -227,6 +228,7 @@ static void cros_typec_remove_partner(struct cros_typec_data *typec, cros_typec_unregister_altmodes(typec, port_num, true); cros_typec_usb_disconnect_state(port); + port->mux_flags = USB_PD_MUX_NONE; typec_unregister_partner(port->partner); port->partner = NULL; @@ -512,20 +514,38 @@ static int cros_typec_enable_usb4(struct cros_typec_data *typec, } static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, - uint8_t mux_flags, struct ec_response_usb_pd_control_v2 *pd_ctrl) { struct cros_typec_port *port = typec->ports[port_num]; + struct ec_response_usb_pd_mux_info resp; + struct ec_params_usb_pd_mux_info req = { + .port = port_num, + }; struct ec_params_usb_pd_mux_ack mux_ack; enum typec_orientation orientation; int ret; - if (mux_flags == USB_PD_MUX_NONE) { + ret = cros_ec_command(typec->ec, 0, EC_CMD_USB_PD_MUX_INFO, + &req, sizeof(req), &resp, sizeof(resp)); + if (ret < 0) { + dev_warn(typec->dev, "Failed to get mux info for port: %d, err = %d\n", + port_num, ret); + return ret; + } + + /* No change needs to be made, let's exit early. */ + if (port->mux_flags == resp.flags && port->role == pd_ctrl->role) + return 0; + + port->mux_flags = resp.flags; + port->role = pd_ctrl->role; + + if (port->mux_flags == USB_PD_MUX_NONE) { ret = cros_typec_usb_disconnect_state(port); goto mux_ack; } - if (mux_flags & USB_PD_MUX_POLARITY_INVERTED) + if (port->mux_flags & USB_PD_MUX_POLARITY_INVERTED) orientation = TYPEC_ORIENTATION_REVERSE; else orientation = TYPEC_ORIENTATION_NORMAL; @@ -540,22 +560,22 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, if (ret) return ret; - if (mux_flags & USB_PD_MUX_USB4_ENABLED) { + if (port->mux_flags & USB_PD_MUX_USB4_ENABLED) { ret = cros_typec_enable_usb4(typec, port_num, pd_ctrl); - } else if (mux_flags & USB_PD_MUX_TBT_COMPAT_ENABLED) { + } else if (port->mux_flags & USB_PD_MUX_TBT_COMPAT_ENABLED) { ret = cros_typec_enable_tbt(typec, port_num, pd_ctrl); - } else if (mux_flags & USB_PD_MUX_DP_ENABLED) { + } else if (port->mux_flags & USB_PD_MUX_DP_ENABLED) { ret = cros_typec_enable_dp(typec, port_num, pd_ctrl); - } else if (mux_flags & USB_PD_MUX_SAFE_MODE) { + } else if (port->mux_flags & USB_PD_MUX_SAFE_MODE) { ret = cros_typec_usb_safe_state(port); - } else if (mux_flags & USB_PD_MUX_USB_ENABLED) { + } else if (port->mux_flags & USB_PD_MUX_USB_ENABLED) { port->state.alt = NULL; port->state.mode = TYPEC_STATE_USB; ret = typec_mux_set(port->mux, &port->state); } else { dev_dbg(typec->dev, "Unrecognized mode requested, mux flags: %x\n", - mux_flags); + port->mux_flags); } mux_ack: @@ -630,17 +650,6 @@ static void cros_typec_set_port_params_v1(struct cros_typec_data *typec, } } -static int cros_typec_get_mux_info(struct cros_typec_data *typec, int port_num, - struct ec_response_usb_pd_mux_info *resp) -{ - struct ec_params_usb_pd_mux_info req = { - .port = port_num, - }; - - return cros_ec_command(typec->ec, 0, EC_CMD_USB_PD_MUX_INFO, &req, - sizeof(req), resp, sizeof(*resp)); -} - /* * Helper function to register partner/plug altmodes. */ @@ -938,7 +947,6 @@ static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) { struct ec_params_usb_pd_control req; struct ec_response_usb_pd_control_v2 resp; - struct ec_response_usb_pd_mux_info mux_resp; int ret; if (port_num < 0 || port_num >= typec->num_ports) { @@ -958,6 +966,11 @@ static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) if (ret < 0) return ret; + /* Update the switches if they exist, according to requested state */ + ret = cros_typec_configure_mux(typec, port_num, &resp); + if (ret) + dev_warn(typec->dev, "Configure muxes failed, err = %d\n", ret); + dev_dbg(typec->dev, "Enabled %d: 0x%hhx\n", port_num, resp.enabled); dev_dbg(typec->dev, "Role %d: 0x%hhx\n", port_num, resp.role); dev_dbg(typec->dev, "Polarity %d: 0x%hhx\n", port_num, resp.polarity); @@ -973,27 +986,7 @@ static int cros_typec_port_update(struct cros_typec_data *typec, int port_num) if (typec->typec_cmd_supported) cros_typec_handle_status(typec, port_num); - /* Update the switches if they exist, according to requested state */ - ret = cros_typec_get_mux_info(typec, port_num, &mux_resp); - if (ret < 0) { - dev_warn(typec->dev, - "Failed to get mux info for port: %d, err = %d\n", - port_num, ret); - return 0; - } - - /* No change needs to be made, let's exit early. */ - if (typec->ports[port_num]->mux_flags == mux_resp.flags && - typec->ports[port_num]->role == resp.role) - return 0; - - typec->ports[port_num]->mux_flags = mux_resp.flags; - typec->ports[port_num]->role = resp.role; - ret = cros_typec_configure_mux(typec, port_num, mux_resp.flags, &resp); - if (ret) - dev_warn(typec->dev, "Configure muxes failed, err = %d\n", ret); - - return ret; + return 0; } static int cros_typec_get_cmd_version(struct cros_typec_data *typec) @@ -1075,7 +1068,13 @@ static int cros_typec_probe(struct platform_device *pdev) return -ENOMEM; typec->dev = dev; + typec->ec = dev_get_drvdata(pdev->dev.parent); + if (!typec->ec) { + dev_err(dev, "couldn't find parent EC device\n"); + return -ENODEV; + } + platform_set_drvdata(pdev, typec); ret = cros_typec_get_cmd_version(typec); diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index f6d6d4c26361..41c65b4d2baf 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -1293,6 +1293,16 @@ config RTC_DRV_OPAL This driver can also be built as a module. If so, the module will be called rtc-opal. +config RTC_DRV_OPTEE + tristate "OP-TEE based RTC driver" + depends on OPTEE + help + Select this to get support for OP-TEE based RTC control on SoCs where + RTC are not accessible to the normal world (Linux). + + This driver can also be built as a module. If so, the module + will be called rtc-optee. + config RTC_DRV_ZYNQMP tristate "Xilinx Zynq Ultrascale+ MPSoC RTC" depends on OF && HAS_IOMEM diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index e92f3e943245..2d827d8261d5 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -115,6 +115,7 @@ obj-$(CONFIG_RTC_DRV_GAMECUBE) += rtc-gamecube.o obj-$(CONFIG_RTC_DRV_NTXEC) += rtc-ntxec.o obj-$(CONFIG_RTC_DRV_OMAP) += rtc-omap.o obj-$(CONFIG_RTC_DRV_OPAL) += rtc-opal.o +obj-$(CONFIG_RTC_DRV_OPTEE) += rtc-optee.o obj-$(CONFIG_RTC_DRV_PALMAS) += rtc-palmas.o obj-$(CONFIG_RTC_DRV_PCAP) += rtc-pcap.o obj-$(CONFIG_RTC_DRV_PCF2123) += rtc-pcf2123.o diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c index 4b460c61f1d8..3c8eec2218df 100644 --- a/drivers/rtc/class.c +++ b/drivers/rtc/class.c @@ -26,6 +26,15 @@ struct class *rtc_class; static void rtc_device_release(struct device *dev) { struct rtc_device *rtc = to_rtc_device(dev); + struct timerqueue_head *head = &rtc->timerqueue; + struct timerqueue_node *node; + + mutex_lock(&rtc->ops_lock); + while ((node = timerqueue_getnext(head))) + timerqueue_del(head, node); + mutex_unlock(&rtc->ops_lock); + + cancel_work_sync(&rtc->irqwork); ida_simple_remove(&rtc_ida, rtc->id); mutex_destroy(&rtc->ops_lock); @@ -390,9 +399,6 @@ int __devm_rtc_register_device(struct module *owner, struct rtc_device *rtc) if (!rtc->ops->set_alarm) clear_bit(RTC_FEATURE_ALARM, rtc->features); - if (rtc->uie_unsupported) - clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); - if (rtc->ops->set_offset) set_bit(RTC_FEATURE_CORRECTION, rtc->features); diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index d8e835798153..9edd662c69ac 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -804,9 +804,13 @@ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer) struct timerqueue_node *next = timerqueue_getnext(&rtc->timerqueue); struct rtc_time tm; ktime_t now; + int err; + + err = __rtc_read_time(rtc, &tm); + if (err) + return err; timer->enabled = 1; - __rtc_read_time(rtc, &tm); now = rtc_tm_to_ktime(tm); /* Skip over expired timers */ @@ -820,7 +824,6 @@ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer) trace_rtc_timer_enqueue(timer); if (!next || ktime_before(timer->node.expires, next->expires)) { struct rtc_wkalrm alarm; - int err; alarm.time = rtc_ktime_to_tm(timer->node.expires); alarm.enabled = 1; diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index 336cb9aa5e33..d51565bcc189 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -1955,7 +1955,7 @@ static int ds1307_probe(struct i2c_client *client, dev_info(ds1307->dev, "'wakeup-source' is set, request for an IRQ is disabled!\n"); /* We cannot support UIE mode if we do not have an IRQ line */ - ds1307->rtc->uie_unsupported = 1; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, ds1307->rtc->features); } if (want_irq) { diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c index 75db7ab654a5..a24331ba8a5f 100644 --- a/drivers/rtc/rtc-ds1685.c +++ b/drivers/rtc/rtc-ds1685.c @@ -1273,7 +1273,7 @@ ds1685_rtc_probe(struct platform_device *pdev) /* See if the platform doesn't support UIE. */ if (pdata->uie_unsupported) - rtc_dev->uie_unsupported = 1; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc_dev->features); rtc->dev = rtc_dev; @@ -1285,13 +1285,10 @@ ds1685_rtc_probe(struct platform_device *pdev) * there won't be an automatic way of notifying the kernel about it, * unless ctrlc is explicitly polled. */ - if (!pdata->no_irq) { - ret = platform_get_irq(pdev, 0); - if (ret <= 0) - return ret; - - rtc->irq_num = ret; - + rtc->irq_num = platform_get_irq(pdev, 0); + if (rtc->irq_num <= 0) { + clear_bit(RTC_FEATURE_ALARM, rtc_dev->features); + } else { /* Request an IRQ. */ ret = devm_request_threaded_irq(&pdev->dev, rtc->irq_num, NULL, ds1685_rtc_irq_handler, @@ -1305,7 +1302,6 @@ ds1685_rtc_probe(struct platform_device *pdev) rtc->irq_num = 0; } } - rtc->no_irq = pdata->no_irq; /* Setup complete. */ ds1685_rtc_switch_to_bank0(rtc); @@ -1394,7 +1390,7 @@ ds1685_rtc_poweroff(struct platform_device *pdev) * have been taken care of by the shutdown scripts and this * is the final function call. */ - if (!rtc->no_irq) + if (rtc->irq_num) disable_irq_nosync(rtc->irq_num); /* Oscillator must be on and the countdown chain enabled. */ diff --git a/drivers/rtc/rtc-efi.c b/drivers/rtc/rtc-efi.c index 138c5e0046c8..11850c2880ad 100644 --- a/drivers/rtc/rtc-efi.c +++ b/drivers/rtc/rtc-efi.c @@ -261,15 +261,17 @@ static int __init efi_rtc_probe(struct platform_device *dev) if (efi.get_time(&eft, &cap) != EFI_SUCCESS) return -ENODEV; - rtc = devm_rtc_device_register(&dev->dev, "rtc-efi", &efi_rtc_ops, - THIS_MODULE); + rtc = devm_rtc_allocate_device(&dev->dev); if (IS_ERR(rtc)) return PTR_ERR(rtc); - rtc->uie_unsupported = 1; platform_set_drvdata(dev, rtc); - return 0; + rtc->ops = &efi_rtc_ops; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); + set_bit(RTC_FEATURE_ALARM_WAKEUP_ONLY, rtc->features); + + return devm_rtc_register_device(rtc); } static struct platform_driver efi_rtc_driver = { diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index f717b36f4738..18ca3b38b2d0 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -235,6 +235,7 @@ static int gamecube_rtc_read_offset_from_sram(struct priv *d) } ret = of_address_to_resource(np, 0, &res); + of_node_put(np); if (ret) { pr_err("no io memory range found\n"); return -1; diff --git a/drivers/rtc/rtc-hym8563.c b/drivers/rtc/rtc-hym8563.c index 0751cae27285..90e602e99d03 100644 --- a/drivers/rtc/rtc-hym8563.c +++ b/drivers/rtc/rtc-hym8563.c @@ -220,24 +220,6 @@ static int hym8563_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm) u8 buf[4]; int ret; - /* - * The alarm has no seconds so deal with it - */ - if (alm_tm->tm_sec) { - alm_tm->tm_sec = 0; - alm_tm->tm_min++; - if (alm_tm->tm_min >= 60) { - alm_tm->tm_min = 0; - alm_tm->tm_hour++; - if (alm_tm->tm_hour >= 24) { - alm_tm->tm_hour = 0; - alm_tm->tm_mday++; - if (alm_tm->tm_mday > 31) - alm_tm->tm_mday = 0; - } - } - } - ret = i2c_smbus_read_byte_data(client, HYM8563_CTL2); if (ret < 0) return ret; @@ -523,6 +505,10 @@ static int hym8563_probe(struct i2c_client *client, if (!hym8563) return -ENOMEM; + hym8563->rtc = devm_rtc_allocate_device(&client->dev); + if (IS_ERR(hym8563->rtc)) + return PTR_ERR(hym8563->rtc); + hym8563->client = client; i2c_set_clientdata(client, hym8563); @@ -557,19 +543,15 @@ static int hym8563_probe(struct i2c_client *client, dev_dbg(&client->dev, "rtc information is %s\n", (ret & HYM8563_SEC_VL) ? "invalid" : "valid"); - hym8563->rtc = devm_rtc_device_register(&client->dev, client->name, - &hym8563_rtc_ops, THIS_MODULE); - if (IS_ERR(hym8563->rtc)) - return PTR_ERR(hym8563->rtc); - - /* the hym8563 alarm only supports a minute accuracy */ - hym8563->rtc->uie_unsupported = 1; + hym8563->rtc->ops = &hym8563_rtc_ops; + set_bit(RTC_FEATURE_ALARM_RES_MINUTE, hym8563->rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, hym8563->rtc->features); #ifdef CONFIG_COMMON_CLK hym8563_clkout_register_clk(hym8563); #endif - return 0; + return devm_rtc_register_device(hym8563->rtc); } static const struct i2c_device_id hym8563_id[] = { diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c index 6d383b629d20..d868458cd40e 100644 --- a/drivers/rtc/rtc-m41t80.c +++ b/drivers/rtc/rtc-m41t80.c @@ -932,10 +932,8 @@ static int m41t80_probe(struct i2c_client *client, m41t80_data->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; m41t80_data->rtc->range_max = RTC_TIMESTAMP_END_2099; - if (client->irq <= 0) { - /* We cannot support UIE mode if we do not have an IRQ line */ - m41t80_data->rtc->uie_unsupported = 1; - } + if (client->irq <= 0) + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, m41t80_data->rtc->features); /* Make sure HT (Halt Update) bit is cleared */ rc = i2c_smbus_read_byte_data(client, M41T80_REG_ALARM_HOUR); diff --git a/drivers/rtc/rtc-mc146818-lib.c b/drivers/rtc/rtc-mc146818-lib.c index ae9f131b43c0..522449b25921 100644 --- a/drivers/rtc/rtc-mc146818-lib.c +++ b/drivers/rtc/rtc-mc146818-lib.c @@ -176,6 +176,17 @@ int mc146818_get_time(struct rtc_time *time) } EXPORT_SYMBOL_GPL(mc146818_get_time); +/* AMD systems don't allow access to AltCentury with DV1 */ +static bool apply_amd_register_a_behavior(void) +{ +#ifdef CONFIG_X86 + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || + boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + return true; +#endif + return false; +} + /* Set the current date and time in the real time clock. */ int mc146818_set_time(struct rtc_time *time) { @@ -232,8 +243,10 @@ int mc146818_set_time(struct rtc_time *time) if (yrs >= 100) yrs -= 100; - if (!(CMOS_READ(RTC_CONTROL) & RTC_DM_BINARY) - || RTC_ALWAYS_BCD) { + spin_lock_irqsave(&rtc_lock, flags); + save_control = CMOS_READ(RTC_CONTROL); + spin_unlock_irqrestore(&rtc_lock, flags); + if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { sec = bin2bcd(sec); min = bin2bcd(min); hrs = bin2bcd(hrs); @@ -247,7 +260,10 @@ int mc146818_set_time(struct rtc_time *time) save_control = CMOS_READ(RTC_CONTROL); CMOS_WRITE((save_control|RTC_SET), RTC_CONTROL); save_freq_select = CMOS_READ(RTC_FREQ_SELECT); - CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT); + if (apply_amd_register_a_behavior()) + CMOS_WRITE((save_freq_select & ~RTC_AMD_BANK_SELECT), RTC_FREQ_SELECT); + else + CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT); #ifdef CONFIG_MACH_DECSTATION CMOS_WRITE(real_yrs, RTC_DEC_YEAR); diff --git a/drivers/rtc/rtc-mpc5121.c b/drivers/rtc/rtc-mpc5121.c index bb2ea9bc56f2..6d7656a75cae 100644 --- a/drivers/rtc/rtc-mpc5121.c +++ b/drivers/rtc/rtc-mpc5121.c @@ -210,20 +210,6 @@ static int mpc5121_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) struct mpc5121_rtc_data *rtc = dev_get_drvdata(dev); struct mpc5121_rtc_regs __iomem *regs = rtc->regs; - /* - * the alarm has no seconds so deal with it - */ - if (alarm->time.tm_sec) { - alarm->time.tm_sec = 0; - alarm->time.tm_min++; - if (alarm->time.tm_min >= 60) { - alarm->time.tm_min = 0; - alarm->time.tm_hour++; - if (alarm->time.tm_hour >= 24) - alarm->time.tm_hour = 0; - } - } - alarm->time.tm_mday = -1; alarm->time.tm_mon = -1; alarm->time.tm_year = -1; @@ -349,7 +335,8 @@ static int mpc5121_rtc_probe(struct platform_device *op) } rtc->rtc->ops = &mpc5200_rtc_ops; - rtc->rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_MINUTE, rtc->rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->rtc->features); rtc->rtc->range_min = RTC_TIMESTAMP_BEGIN_0000; rtc->rtc->range_max = 65733206399ULL; /* 4052-12-31 23:59:59 */ diff --git a/drivers/rtc/rtc-opal.c b/drivers/rtc/rtc-opal.c index f8f49a969c23..ad41aaf8a17f 100644 --- a/drivers/rtc/rtc-opal.c +++ b/drivers/rtc/rtc-opal.c @@ -250,7 +250,7 @@ static int opal_rtc_probe(struct platform_device *pdev) rtc->ops = &opal_rtc_ops; rtc->range_min = RTC_TIMESTAMP_BEGIN_0000; rtc->range_max = RTC_TIMESTAMP_END_9999; - rtc->uie_unsupported = 1; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); return devm_rtc_register_device(rtc); } diff --git a/drivers/rtc/rtc-optee.c b/drivers/rtc/rtc-optee.c new file mode 100644 index 000000000000..9f8b5d4a8f6b --- /dev/null +++ b/drivers/rtc/rtc-optee.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Microchip. + */ + +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/rtc.h> +#include <linux/tee_drv.h> + +#define RTC_INFO_VERSION 0x1 + +#define TA_CMD_RTC_GET_INFO 0x0 +#define TA_CMD_RTC_GET_TIME 0x1 +#define TA_CMD_RTC_SET_TIME 0x2 +#define TA_CMD_RTC_GET_OFFSET 0x3 +#define TA_CMD_RTC_SET_OFFSET 0x4 + +#define TA_RTC_FEATURE_CORRECTION BIT(0) + +struct optee_rtc_time { + u32 tm_sec; + u32 tm_min; + u32 tm_hour; + u32 tm_mday; + u32 tm_mon; + u32 tm_year; + u32 tm_wday; +}; + +struct optee_rtc_info { + u64 version; + u64 features; + struct optee_rtc_time range_min; + struct optee_rtc_time range_max; +}; + +/** + * struct optee_rtc - OP-TEE RTC private data + * @dev: OP-TEE based RTC device. + * @ctx: OP-TEE context handler. + * @session_id: RTC TA session identifier. + * @shm: Memory pool shared with RTC device. + * @features: Bitfield of RTC features + */ +struct optee_rtc { + struct device *dev; + struct tee_context *ctx; + u32 session_id; + struct tee_shm *shm; + u64 features; +}; + +static int optee_rtc_readtime(struct device *dev, struct rtc_time *tm) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + struct tee_ioctl_invoke_arg inv_arg = {0}; + struct optee_rtc_time *optee_tm; + struct tee_param param[4] = {0}; + int ret; + + inv_arg.func = TA_CMD_RTC_GET_TIME; + inv_arg.session = priv->session_id; + inv_arg.num_params = 4; + + /* Fill invoke cmd params */ + param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT; + param[0].u.memref.shm = priv->shm; + param[0].u.memref.size = sizeof(struct optee_rtc_time); + + ret = tee_client_invoke_func(priv->ctx, &inv_arg, param); + if (ret < 0 || inv_arg.ret != 0) + return ret ? ret : -EPROTO; + + optee_tm = tee_shm_get_va(priv->shm, 0); + if (IS_ERR(optee_tm)) + return PTR_ERR(optee_tm); + + if (param[0].u.memref.size != sizeof(*optee_tm)) + return -EPROTO; + + tm->tm_sec = optee_tm->tm_sec; + tm->tm_min = optee_tm->tm_min; + tm->tm_hour = optee_tm->tm_hour; + tm->tm_mday = optee_tm->tm_mday; + tm->tm_mon = optee_tm->tm_mon; + tm->tm_year = optee_tm->tm_year - 1900; + tm->tm_wday = optee_tm->tm_wday; + tm->tm_yday = rtc_year_days(tm->tm_mday, tm->tm_mon, tm->tm_year); + + return 0; +} + +static int optee_rtc_settime(struct device *dev, struct rtc_time *tm) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + struct tee_ioctl_invoke_arg inv_arg = {0}; + struct tee_param param[4] = {0}; + struct optee_rtc_time optee_tm; + void *rtc_data; + int ret; + + optee_tm.tm_sec = tm->tm_sec; + optee_tm.tm_min = tm->tm_min; + optee_tm.tm_hour = tm->tm_hour; + optee_tm.tm_mday = tm->tm_mday; + optee_tm.tm_mon = tm->tm_mon; + optee_tm.tm_year = tm->tm_year + 1900; + optee_tm.tm_wday = tm->tm_wday; + + inv_arg.func = TA_CMD_RTC_SET_TIME; + inv_arg.session = priv->session_id; + inv_arg.num_params = 4; + + param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; + param[0].u.memref.shm = priv->shm; + param[0].u.memref.size = sizeof(struct optee_rtc_time); + + rtc_data = tee_shm_get_va(priv->shm, 0); + if (IS_ERR(rtc_data)) + return PTR_ERR(rtc_data); + + memcpy(rtc_data, &optee_tm, sizeof(struct optee_rtc_time)); + + ret = tee_client_invoke_func(priv->ctx, &inv_arg, param); + if (ret < 0 || inv_arg.ret != 0) + return ret ? ret : -EPROTO; + + return 0; +} + +static int optee_rtc_readoffset(struct device *dev, long *offset) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + struct tee_ioctl_invoke_arg inv_arg = {0}; + struct tee_param param[4] = {0}; + int ret; + + if (!(priv->features & TA_RTC_FEATURE_CORRECTION)) + return -EOPNOTSUPP; + + inv_arg.func = TA_CMD_RTC_GET_OFFSET; + inv_arg.session = priv->session_id; + inv_arg.num_params = 4; + + param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT; + + ret = tee_client_invoke_func(priv->ctx, &inv_arg, param); + if (ret < 0 || inv_arg.ret != 0) + return ret ? ret : -EPROTO; + + *offset = param[0].u.value.a; + + return 0; +} + +static int optee_rtc_setoffset(struct device *dev, long offset) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + struct tee_ioctl_invoke_arg inv_arg = {0}; + struct tee_param param[4] = {0}; + int ret; + + if (!(priv->features & TA_RTC_FEATURE_CORRECTION)) + return -EOPNOTSUPP; + + inv_arg.func = TA_CMD_RTC_SET_OFFSET; + inv_arg.session = priv->session_id; + inv_arg.num_params = 4; + + param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT; + param[0].u.value.a = offset; + + ret = tee_client_invoke_func(priv->ctx, &inv_arg, param); + if (ret < 0 || inv_arg.ret != 0) + return ret ? ret : -EPROTO; + + return 0; +} + +static const struct rtc_class_ops optee_rtc_ops = { + .read_time = optee_rtc_readtime, + .set_time = optee_rtc_settime, + .set_offset = optee_rtc_setoffset, + .read_offset = optee_rtc_readoffset, +}; + +static int optee_rtc_read_info(struct device *dev, struct rtc_device *rtc, + u64 *features) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + struct tee_ioctl_invoke_arg inv_arg = {0}; + struct tee_param param[4] = {0}; + struct optee_rtc_info *info; + struct optee_rtc_time *tm; + int ret; + + inv_arg.func = TA_CMD_RTC_GET_INFO; + inv_arg.session = priv->session_id; + inv_arg.num_params = 4; + + param[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT; + param[0].u.memref.shm = priv->shm; + param[0].u.memref.size = sizeof(*info); + + ret = tee_client_invoke_func(priv->ctx, &inv_arg, param); + if (ret < 0 || inv_arg.ret != 0) + return ret ? ret : -EPROTO; + + info = tee_shm_get_va(priv->shm, 0); + if (IS_ERR(info)) + return PTR_ERR(info); + + if (param[0].u.memref.size != sizeof(*info)) + return -EPROTO; + + if (info->version != RTC_INFO_VERSION) + return -EPROTO; + + *features = info->features; + + tm = &info->range_min; + rtc->range_min = mktime64(tm->tm_year, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, + tm->tm_sec); + tm = &info->range_max; + rtc->range_max = mktime64(tm->tm_year, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, + tm->tm_sec); + + return 0; +} + +static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data) +{ + if (ver->impl_id == TEE_IMPL_ID_OPTEE) + return 1; + else + return 0; +} + +static int optee_rtc_probe(struct device *dev) +{ + struct tee_client_device *rtc_device = to_tee_client_device(dev); + struct tee_ioctl_open_session_arg sess_arg; + struct optee_rtc *priv; + struct rtc_device *rtc; + struct tee_shm *shm; + int ret, err; + + memset(&sess_arg, 0, sizeof(sess_arg)); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + rtc = devm_rtc_allocate_device(dev); + if (IS_ERR(rtc)) + return PTR_ERR(rtc); + + /* Open context with TEE driver */ + priv->ctx = tee_client_open_context(NULL, optee_ctx_match, NULL, NULL); + if (IS_ERR(priv->ctx)) + return -ENODEV; + + /* Open session with rtc Trusted App */ + export_uuid(sess_arg.uuid, &rtc_device->id.uuid); + sess_arg.clnt_login = TEE_IOCTL_LOGIN_REE_KERNEL; + + ret = tee_client_open_session(priv->ctx, &sess_arg, NULL); + if (ret < 0 || sess_arg.ret != 0) { + dev_err(dev, "tee_client_open_session failed, err: %x\n", sess_arg.ret); + err = -EINVAL; + goto out_ctx; + } + priv->session_id = sess_arg.session; + + shm = tee_shm_alloc_kernel_buf(priv->ctx, sizeof(struct optee_rtc_info)); + if (IS_ERR(shm)) { + dev_err(priv->dev, "tee_shm_alloc_kernel_buf failed\n"); + err = PTR_ERR(shm); + goto out_sess; + } + + priv->shm = shm; + priv->dev = dev; + dev_set_drvdata(dev, priv); + + rtc->ops = &optee_rtc_ops; + + err = optee_rtc_read_info(dev, rtc, &priv->features); + if (err) { + dev_err(dev, "Failed to get RTC features from OP-TEE\n"); + goto out_shm; + } + + err = devm_rtc_register_device(rtc); + if (err) + goto out_shm; + + /* + * We must clear this bit after registering because rtc_register_device + * will set it if it sees that .set_offset is provided. + */ + if (!(priv->features & TA_RTC_FEATURE_CORRECTION)) + clear_bit(RTC_FEATURE_CORRECTION, rtc->features); + + return 0; + +out_shm: + tee_shm_free(priv->shm); +out_sess: + tee_client_close_session(priv->ctx, priv->session_id); +out_ctx: + tee_client_close_context(priv->ctx); + + return err; +} + +static int optee_rtc_remove(struct device *dev) +{ + struct optee_rtc *priv = dev_get_drvdata(dev); + + tee_client_close_session(priv->ctx, priv->session_id); + tee_client_close_context(priv->ctx); + + return 0; +} + +static const struct tee_client_device_id optee_rtc_id_table[] = { + {UUID_INIT(0xf389f8c8, 0x845f, 0x496c, + 0x8b, 0xbe, 0xd6, 0x4b, 0xd2, 0x4c, 0x92, 0xfd)}, + {} +}; + +MODULE_DEVICE_TABLE(tee, optee_rtc_id_table); + +static struct tee_client_driver optee_rtc_driver = { + .id_table = optee_rtc_id_table, + .driver = { + .name = "optee_rtc", + .bus = &tee_bus_type, + .probe = optee_rtc_probe, + .remove = optee_rtc_remove, + }, +}; + +static int __init optee_rtc_mod_init(void) +{ + return driver_register(&optee_rtc_driver.driver); +} + +static void __exit optee_rtc_mod_exit(void) +{ + driver_unregister(&optee_rtc_driver.driver); +} + +module_init(optee_rtc_mod_init); +module_exit(optee_rtc_mod_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Clément Léger <clement.leger@bootlin.com>"); +MODULE_DESCRIPTION("OP-TEE based RTC driver"); diff --git a/drivers/rtc/rtc-pcf2123.c b/drivers/rtc/rtc-pcf2123.c index 7473e6c8a183..e13b5e695d06 100644 --- a/drivers/rtc/rtc-pcf2123.c +++ b/drivers/rtc/rtc-pcf2123.c @@ -427,7 +427,8 @@ static int pcf2123_probe(struct spi_device *spi) * support to this driver to generate interrupts more than once * per minute. */ - rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_MINUTE, rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); rtc->ops = &pcf2123_rtc_ops; rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; rtc->range_max = RTC_TIMESTAMP_END_2099; diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c index 81a5b1f2e68c..63b275b014bd 100644 --- a/drivers/rtc/rtc-pcf2127.c +++ b/drivers/rtc/rtc-pcf2127.c @@ -374,7 +374,8 @@ static int pcf2127_watchdog_init(struct device *dev, struct pcf2127 *pcf2127) static int pcf2127_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) { struct pcf2127 *pcf2127 = dev_get_drvdata(dev); - unsigned int buf[5], ctrl2; + u8 buf[5]; + unsigned int ctrl2; int ret; ret = regmap_read(pcf2127->regmap, PCF2127_REG_CTRL2, &ctrl2); @@ -655,13 +656,25 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap, pcf2127->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; pcf2127->rtc->range_max = RTC_TIMESTAMP_END_2099; pcf2127->rtc->set_start_time = true; /* Sets actual start to 1970 */ - pcf2127->rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_2S, pcf2127->rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, pcf2127->rtc->features); clear_bit(RTC_FEATURE_ALARM, pcf2127->rtc->features); if (alarm_irq > 0) { + unsigned long flags; + + /* + * If flags = 0, devm_request_threaded_irq() will use IRQ flags + * obtained from device tree. + */ + if (dev_fwnode(dev)) + flags = 0; + else + flags = IRQF_TRIGGER_LOW; + ret = devm_request_threaded_irq(dev, alarm_irq, NULL, pcf2127_rtc_irq, - IRQF_TRIGGER_LOW | IRQF_ONESHOT, + flags | IRQF_ONESHOT, dev_name(dev), dev); if (ret) { dev_err(dev, "failed to request alarm irq\n"); diff --git a/drivers/rtc/rtc-pcf85063.c b/drivers/rtc/rtc-pcf85063.c index df2b072c394d..9760824ec199 100644 --- a/drivers/rtc/rtc-pcf85063.c +++ b/drivers/rtc/rtc-pcf85063.c @@ -616,7 +616,8 @@ static int pcf85063_probe(struct i2c_client *client) pcf85063->rtc->ops = &pcf85063_rtc_ops; pcf85063->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; pcf85063->rtc->range_max = RTC_TIMESTAMP_END_2099; - pcf85063->rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_2S, pcf85063->rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, pcf85063->rtc->features); clear_bit(RTC_FEATURE_ALARM, pcf85063->rtc->features); if (config->has_alarms && client->irq > 0) { diff --git a/drivers/rtc/rtc-pcf8523.c b/drivers/rtc/rtc-pcf8523.c index c93acade7205..b1b1943de844 100644 --- a/drivers/rtc/rtc-pcf8523.c +++ b/drivers/rtc/rtc-pcf8523.c @@ -212,14 +212,6 @@ static int pcf8523_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *tm) if (err < 0) return err; - /* The alarm has no seconds, round up to nearest minute */ - if (tm->time.tm_sec) { - time64_t alarm_time = rtc_tm_to_time64(&tm->time); - - alarm_time += 60 - tm->time.tm_sec; - rtc_time64_to_tm(alarm_time, &tm->time); - } - regs[0] = bin2bcd(tm->time.tm_min); regs[1] = bin2bcd(tm->time.tm_hour); regs[2] = bin2bcd(tm->time.tm_mday); @@ -240,9 +232,9 @@ static int pcf8523_param_get(struct device *dev, struct rtc_param *param) { struct pcf8523 *pcf8523 = dev_get_drvdata(dev); int ret; + u32 value; switch(param->param) { - u32 value; case RTC_PARAM_BACKUP_SWITCH_MODE: ret = regmap_read(pcf8523->regmap, PCF8523_REG_CONTROL3, &value); @@ -279,9 +271,9 @@ static int pcf8523_param_get(struct device *dev, struct rtc_param *param) static int pcf8523_param_set(struct device *dev, struct rtc_param *param) { struct pcf8523 *pcf8523 = dev_get_drvdata(dev); + u8 mode; switch(param->param) { - u8 mode; case RTC_PARAM_BACKUP_SWITCH_MODE: switch (param->uvalue) { case RTC_BSM_DISABLED: @@ -450,7 +442,8 @@ static int pcf8523_probe(struct i2c_client *client, rtc->ops = &pcf8523_rtc_ops; rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; rtc->range_max = RTC_TIMESTAMP_END_2099; - rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_MINUTE, rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); if (client->irq > 0) { err = regmap_write(pcf8523->regmap, PCF8523_TMR_CLKOUT_CTRL, 0x38); diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index c8bddfb94129..9d06813e2e6d 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -330,19 +330,6 @@ static int pcf8563_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *tm) unsigned char buf[4]; int err; - /* The alarm has no seconds, round up to nearest minute */ - if (tm->time.tm_sec) { - time64_t alarm_time = rtc_tm_to_time64(&tm->time); - - alarm_time += 60 - tm->time.tm_sec; - rtc_time64_to_tm(alarm_time, &tm->time); - } - - dev_dbg(dev, "%s, min=%d hour=%d wday=%d mday=%d " - "enabled=%d pending=%d\n", __func__, - tm->time.tm_min, tm->time.tm_hour, tm->time.tm_wday, - tm->time.tm_mday, tm->enabled, tm->pending); - buf[0] = bin2bcd(tm->time.tm_min); buf[1] = bin2bcd(tm->time.tm_hour); buf[2] = bin2bcd(tm->time.tm_mday); @@ -565,7 +552,8 @@ static int pcf8563_probe(struct i2c_client *client, pcf8563->rtc->ops = &pcf8563_rtc_ops; /* the pcf8563 alarm only supports a minute accuracy */ - pcf8563->rtc->uie_unsupported = 1; + set_bit(RTC_FEATURE_ALARM_RES_MINUTE, pcf8563->rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, pcf8563->rtc->features); pcf8563->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; pcf8563->rtc->range_max = RTC_TIMESTAMP_END_2099; pcf8563->rtc->set_start_time = true; diff --git a/drivers/rtc/rtc-pl031.c b/drivers/rtc/rtc-pl031.c index e38ee8848385..bad6a5d9c683 100644 --- a/drivers/rtc/rtc-pl031.c +++ b/drivers/rtc/rtc-pl031.c @@ -350,9 +350,6 @@ static int pl031_probe(struct amba_device *adev, const struct amba_id *id) } } - if (!adev->irq[0]) - clear_bit(RTC_FEATURE_ALARM, ldata->rtc->features); - device_init_wakeup(&adev->dev, true); ldata->rtc = devm_rtc_allocate_device(&adev->dev); if (IS_ERR(ldata->rtc)) { @@ -360,6 +357,9 @@ static int pl031_probe(struct amba_device *adev, const struct amba_id *id) goto out; } + if (!adev->irq[0]) + clear_bit(RTC_FEATURE_ALARM, ldata->rtc->features); + ldata->rtc->ops = ops; ldata->rtc->range_min = vendor->range_min; ldata->rtc->range_max = vendor->range_max; diff --git a/drivers/rtc/rtc-pm8xxx.c b/drivers/rtc/rtc-pm8xxx.c index 29a1c65661e9..dc6d1476baa5 100644 --- a/drivers/rtc/rtc-pm8xxx.c +++ b/drivers/rtc/rtc-pm8xxx.c @@ -7,6 +7,7 @@ #include <linux/rtc.h> #include <linux/platform_device.h> #include <linux/pm.h> +#include <linux/pm_wakeirq.h> #include <linux/regmap.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -83,7 +84,7 @@ static int pm8xxx_rtc_set_time(struct device *dev, struct rtc_time *tm) const struct pm8xxx_rtc_regs *regs = rtc_dd->regs; if (!rtc_dd->allow_set_time) - return -EACCES; + return -ENODEV; secs = rtc_tm_to_time64(tm); @@ -527,40 +528,28 @@ static int pm8xxx_rtc_probe(struct platform_device *pdev) return rc; } - return devm_rtc_register_device(rtc_dd->rtc); -} - -#ifdef CONFIG_PM_SLEEP -static int pm8xxx_rtc_resume(struct device *dev) -{ - struct pm8xxx_rtc *rtc_dd = dev_get_drvdata(dev); + rc = devm_rtc_register_device(rtc_dd->rtc); + if (rc) + return rc; - if (device_may_wakeup(dev)) - disable_irq_wake(rtc_dd->rtc_alarm_irq); + rc = dev_pm_set_wake_irq(&pdev->dev, rtc_dd->rtc_alarm_irq); + if (rc) + return rc; return 0; } -static int pm8xxx_rtc_suspend(struct device *dev) +static int pm8xxx_remove(struct platform_device *pdev) { - struct pm8xxx_rtc *rtc_dd = dev_get_drvdata(dev); - - if (device_may_wakeup(dev)) - enable_irq_wake(rtc_dd->rtc_alarm_irq); - + dev_pm_clear_wake_irq(&pdev->dev); return 0; } -#endif - -static SIMPLE_DEV_PM_OPS(pm8xxx_rtc_pm_ops, - pm8xxx_rtc_suspend, - pm8xxx_rtc_resume); static struct platform_driver pm8xxx_rtc_driver = { .probe = pm8xxx_rtc_probe, + .remove = pm8xxx_remove, .driver = { .name = "rtc-pm8xxx", - .pm = &pm8xxx_rtc_pm_ops, .of_match_table = pm8xxx_id_table, }, }; diff --git a/drivers/rtc/rtc-spear.c b/drivers/rtc/rtc-spear.c index b4a520056b1a..d4777b01ab22 100644 --- a/drivers/rtc/rtc-spear.c +++ b/drivers/rtc/rtc-spear.c @@ -204,8 +204,10 @@ static int spear_rtc_read_time(struct device *dev, struct rtc_time *tm) /* we don't report wday/yday/isdst ... */ rtc_wait_not_busy(config); - time = readl(config->ioaddr + TIME_REG); - date = readl(config->ioaddr + DATE_REG); + do { + time = readl(config->ioaddr + TIME_REG); + date = readl(config->ioaddr + DATE_REG); + } while (time == readl(config->ioaddr + TIME_REG)); tm->tm_sec = (time >> SECOND_SHIFT) & SECOND_MASK; tm->tm_min = (time >> MINUTE_SHIFT) & MIN_MASK; tm->tm_hour = (time >> HOUR_SHIFT) & HOUR_MASK; @@ -352,6 +354,10 @@ static int spear_rtc_probe(struct platform_device *pdev) if (!config) return -ENOMEM; + config->rtc = devm_rtc_allocate_device(&pdev->dev); + if (IS_ERR(config->rtc)) + return PTR_ERR(config->rtc); + /* alarm irqs */ irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -380,16 +386,13 @@ static int spear_rtc_probe(struct platform_device *pdev) spin_lock_init(&config->lock); platform_set_drvdata(pdev, config); - config->rtc = devm_rtc_device_register(&pdev->dev, pdev->name, - &spear_rtc_ops, THIS_MODULE); - if (IS_ERR(config->rtc)) { - dev_err(&pdev->dev, "can't register RTC device, err %ld\n", - PTR_ERR(config->rtc)); - status = PTR_ERR(config->rtc); - goto err_disable_clock; - } + config->rtc->ops = &spear_rtc_ops; + config->rtc->range_min = RTC_TIMESTAMP_BEGIN_0000; + config->rtc->range_min = RTC_TIMESTAMP_END_9999; - config->rtc->uie_unsupported = 1; + status = devm_rtc_register_device(config->rtc); + if (status) + goto err_disable_clock; if (!device_can_wakeup(&pdev->dev)) device_init_wakeup(&pdev->dev, 1); diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c index 711832c758ae..5b3e4da63406 100644 --- a/drivers/rtc/rtc-sun6i.c +++ b/drivers/rtc/rtc-sun6i.c @@ -13,6 +13,7 @@ #include <linux/clk.h> #include <linux/clk-provider.h> +#include <linux/clk/sunxi-ng.h> #include <linux/delay.h> #include <linux/err.h> #include <linux/fs.h> @@ -48,7 +49,8 @@ /* Alarm 0 (counter) */ #define SUN6I_ALRM_COUNTER 0x0020 -#define SUN6I_ALRM_CUR_VAL 0x0024 +/* This holds the remaining alarm seconds on older SoCs (current value) */ +#define SUN6I_ALRM_COUNTER_HMS 0x0024 #define SUN6I_ALRM_EN 0x0028 #define SUN6I_ALRM_EN_CNT_EN BIT(0) #define SUN6I_ALRM_IRQ_EN 0x002c @@ -110,6 +112,8 @@ #define SUN6I_YEAR_MIN 1970 #define SUN6I_YEAR_OFF (SUN6I_YEAR_MIN - 1900) +#define SECS_PER_DAY (24 * 3600ULL) + /* * There are other differences between models, including: * @@ -133,12 +137,15 @@ struct sun6i_rtc_clk_data { unsigned int has_auto_swt : 1; }; +#define RTC_LINEAR_DAY BIT(0) + struct sun6i_rtc_dev { struct rtc_device *rtc; const struct sun6i_rtc_clk_data *data; void __iomem *base; int irq; - unsigned long alarm; + time64_t alarm; + unsigned long flags; struct clk_hw hw; struct clk_hw *int_osc; @@ -363,23 +370,6 @@ CLK_OF_DECLARE_DRIVER(sun8i_h3_rtc_clk, "allwinner,sun8i-h3-rtc", CLK_OF_DECLARE_DRIVER(sun50i_h5_rtc_clk, "allwinner,sun50i-h5-rtc", sun8i_h3_rtc_clk_init); -static const struct sun6i_rtc_clk_data sun50i_h6_rtc_data = { - .rc_osc_rate = 16000000, - .fixed_prescaler = 32, - .has_prescaler = 1, - .has_out_clk = 1, - .export_iosc = 1, - .has_losc_en = 1, - .has_auto_swt = 1, -}; - -static void __init sun50i_h6_rtc_clk_init(struct device_node *node) -{ - sun6i_rtc_clk_init(node, &sun50i_h6_rtc_data); -} -CLK_OF_DECLARE_DRIVER(sun50i_h6_rtc_clk, "allwinner,sun50i-h6-rtc", - sun50i_h6_rtc_clk_init); - /* * The R40 user manual is self-conflicting on whether the prescaler is * fixed or configurable. The clock diagram shows it as fixed, but there @@ -467,22 +457,30 @@ static int sun6i_rtc_gettime(struct device *dev, struct rtc_time *rtc_tm) } while ((date != readl(chip->base + SUN6I_RTC_YMD)) || (time != readl(chip->base + SUN6I_RTC_HMS))); + if (chip->flags & RTC_LINEAR_DAY) { + /* + * Newer chips store a linear day number, the manual + * does not mandate any epoch base. The BSP driver uses + * the UNIX epoch, let's just copy that, as it's the + * easiest anyway. + */ + rtc_time64_to_tm((date & 0xffff) * SECS_PER_DAY, rtc_tm); + } else { + rtc_tm->tm_mday = SUN6I_DATE_GET_DAY_VALUE(date); + rtc_tm->tm_mon = SUN6I_DATE_GET_MON_VALUE(date) - 1; + rtc_tm->tm_year = SUN6I_DATE_GET_YEAR_VALUE(date); + + /* + * switch from (data_year->min)-relative offset to + * a (1900)-relative one + */ + rtc_tm->tm_year += SUN6I_YEAR_OFF; + } + rtc_tm->tm_sec = SUN6I_TIME_GET_SEC_VALUE(time); rtc_tm->tm_min = SUN6I_TIME_GET_MIN_VALUE(time); rtc_tm->tm_hour = SUN6I_TIME_GET_HOUR_VALUE(time); - rtc_tm->tm_mday = SUN6I_DATE_GET_DAY_VALUE(date); - rtc_tm->tm_mon = SUN6I_DATE_GET_MON_VALUE(date); - rtc_tm->tm_year = SUN6I_DATE_GET_YEAR_VALUE(date); - - rtc_tm->tm_mon -= 1; - - /* - * switch from (data_year->min)-relative offset to - * a (1900)-relative one - */ - rtc_tm->tm_year += SUN6I_YEAR_OFF; - return 0; } @@ -510,36 +508,54 @@ static int sun6i_rtc_setalarm(struct device *dev, struct rtc_wkalrm *wkalrm) struct sun6i_rtc_dev *chip = dev_get_drvdata(dev); struct rtc_time *alrm_tm = &wkalrm->time; struct rtc_time tm_now; - unsigned long time_now = 0; - unsigned long time_set = 0; - unsigned long time_gap = 0; - int ret = 0; - - ret = sun6i_rtc_gettime(dev, &tm_now); - if (ret < 0) { - dev_err(dev, "Error in getting time\n"); - return -EINVAL; - } + time64_t time_set; + u32 counter_val, counter_val_hms; + int ret; time_set = rtc_tm_to_time64(alrm_tm); - time_now = rtc_tm_to_time64(&tm_now); - if (time_set <= time_now) { - dev_err(dev, "Date to set in the past\n"); - return -EINVAL; - } - - time_gap = time_set - time_now; - if (time_gap > U32_MAX) { - dev_err(dev, "Date too far in the future\n"); - return -EINVAL; + if (chip->flags & RTC_LINEAR_DAY) { + /* + * The alarm registers hold the actual alarm time, encoded + * in the same way (linear day + HMS) as the current time. + */ + counter_val_hms = SUN6I_TIME_SET_SEC_VALUE(alrm_tm->tm_sec) | + SUN6I_TIME_SET_MIN_VALUE(alrm_tm->tm_min) | + SUN6I_TIME_SET_HOUR_VALUE(alrm_tm->tm_hour); + /* The division will cut off the H:M:S part of alrm_tm. */ + counter_val = div_u64(rtc_tm_to_time64(alrm_tm), SECS_PER_DAY); + } else { + /* The alarm register holds the number of seconds left. */ + time64_t time_now; + + ret = sun6i_rtc_gettime(dev, &tm_now); + if (ret < 0) { + dev_err(dev, "Error in getting time\n"); + return -EINVAL; + } + + time_now = rtc_tm_to_time64(&tm_now); + if (time_set <= time_now) { + dev_err(dev, "Date to set in the past\n"); + return -EINVAL; + } + if ((time_set - time_now) > U32_MAX) { + dev_err(dev, "Date too far in the future\n"); + return -EINVAL; + } + + counter_val = time_set - time_now; } sun6i_rtc_setaie(0, chip); writel(0, chip->base + SUN6I_ALRM_COUNTER); + if (chip->flags & RTC_LINEAR_DAY) + writel(0, chip->base + SUN6I_ALRM_COUNTER_HMS); usleep_range(100, 300); - writel(time_gap, chip->base + SUN6I_ALRM_COUNTER); + writel(counter_val, chip->base + SUN6I_ALRM_COUNTER); + if (chip->flags & RTC_LINEAR_DAY) + writel(counter_val_hms, chip->base + SUN6I_ALRM_COUNTER_HMS); chip->alarm = time_set; sun6i_rtc_setaie(wkalrm->enabled, chip); @@ -571,20 +587,25 @@ static int sun6i_rtc_settime(struct device *dev, struct rtc_time *rtc_tm) u32 date = 0; u32 time = 0; - rtc_tm->tm_year -= SUN6I_YEAR_OFF; - rtc_tm->tm_mon += 1; - - date = SUN6I_DATE_SET_DAY_VALUE(rtc_tm->tm_mday) | - SUN6I_DATE_SET_MON_VALUE(rtc_tm->tm_mon) | - SUN6I_DATE_SET_YEAR_VALUE(rtc_tm->tm_year); - - if (is_leap_year(rtc_tm->tm_year + SUN6I_YEAR_MIN)) - date |= SUN6I_LEAP_SET_VALUE(1); - time = SUN6I_TIME_SET_SEC_VALUE(rtc_tm->tm_sec) | SUN6I_TIME_SET_MIN_VALUE(rtc_tm->tm_min) | SUN6I_TIME_SET_HOUR_VALUE(rtc_tm->tm_hour); + if (chip->flags & RTC_LINEAR_DAY) { + /* The division will cut off the H:M:S part of rtc_tm. */ + date = div_u64(rtc_tm_to_time64(rtc_tm), SECS_PER_DAY); + } else { + rtc_tm->tm_year -= SUN6I_YEAR_OFF; + rtc_tm->tm_mon += 1; + + date = SUN6I_DATE_SET_DAY_VALUE(rtc_tm->tm_mday) | + SUN6I_DATE_SET_MON_VALUE(rtc_tm->tm_mon) | + SUN6I_DATE_SET_YEAR_VALUE(rtc_tm->tm_year); + + if (is_leap_year(rtc_tm->tm_year + SUN6I_YEAR_MIN)) + date |= SUN6I_LEAP_SET_VALUE(1); + } + /* Check whether registers are writable */ if (sun6i_rtc_wait(chip, SUN6I_LOSC_CTRL, SUN6I_LOSC_CTRL_ACC_MASK, 50)) { @@ -668,11 +689,35 @@ static int sun6i_rtc_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(sun6i_rtc_pm_ops, sun6i_rtc_suspend, sun6i_rtc_resume); +static void sun6i_rtc_bus_clk_cleanup(void *data) +{ + struct clk *bus_clk = data; + + clk_disable_unprepare(bus_clk); +} + static int sun6i_rtc_probe(struct platform_device *pdev) { struct sun6i_rtc_dev *chip = sun6i_rtc; + struct device *dev = &pdev->dev; + struct clk *bus_clk; int ret; + bus_clk = devm_clk_get_optional(dev, "bus"); + if (IS_ERR(bus_clk)) + return PTR_ERR(bus_clk); + + if (bus_clk) { + ret = clk_prepare_enable(bus_clk); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, sun6i_rtc_bus_clk_cleanup, + bus_clk); + if (ret) + return ret; + } + if (!chip) { chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); if (!chip) @@ -683,10 +728,18 @@ static int sun6i_rtc_probe(struct platform_device *pdev) chip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->base)) return PTR_ERR(chip->base); + + if (IS_REACHABLE(CONFIG_SUN6I_RTC_CCU)) { + ret = sun6i_rtc_ccu_probe(dev, chip->base); + if (ret) + return ret; + } } platform_set_drvdata(pdev, chip); + chip->flags = (unsigned long)of_device_get_match_data(&pdev->dev); + chip->irq = platform_get_irq(pdev, 0); if (chip->irq < 0) return chip->irq; @@ -733,7 +786,10 @@ static int sun6i_rtc_probe(struct platform_device *pdev) return PTR_ERR(chip->rtc); chip->rtc->ops = &sun6i_rtc_ops; - chip->rtc->range_max = 2019686399LL; /* 2033-12-31 23:59:59 */ + if (chip->flags & RTC_LINEAR_DAY) + chip->rtc->range_max = (65536 * SECS_PER_DAY) - 1; + else + chip->rtc->range_max = 2019686399LL; /* 2033-12-31 23:59:59 */ ret = devm_rtc_register_device(chip->rtc); if (ret) @@ -758,6 +814,8 @@ static const struct of_device_id sun6i_rtc_dt_ids[] = { { .compatible = "allwinner,sun8i-v3-rtc" }, { .compatible = "allwinner,sun50i-h5-rtc" }, { .compatible = "allwinner,sun50i-h6-rtc" }, + { .compatible = "allwinner,sun50i-h616-rtc", + .data = (void *)RTC_LINEAR_DAY }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun6i_rtc_dt_ids); diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c index 2018614f258f..6eaa9321c074 100644 --- a/drivers/rtc/rtc-wm8350.c +++ b/drivers/rtc/rtc-wm8350.c @@ -432,14 +432,21 @@ static int wm8350_rtc_probe(struct platform_device *pdev) return ret; } - wm8350_register_irq(wm8350, WM8350_IRQ_RTC_SEC, + ret = wm8350_register_irq(wm8350, WM8350_IRQ_RTC_SEC, wm8350_rtc_update_handler, 0, "RTC Seconds", wm8350); + if (ret) + return ret; + wm8350_mask_irq(wm8350, WM8350_IRQ_RTC_SEC); - wm8350_register_irq(wm8350, WM8350_IRQ_RTC_ALM, + ret = wm8350_register_irq(wm8350, WM8350_IRQ_RTC_ALM, wm8350_rtc_alarm_handler, 0, "RTC Alarm", wm8350); + if (ret) { + wm8350_free_irq(wm8350, WM8350_IRQ_RTC_SEC, wm8350); + return ret; + } return 0; } diff --git a/drivers/rtc/rtc-xgene.c b/drivers/rtc/rtc-xgene.c index cf68a9b1c9eb..d3d0054e21fd 100644 --- a/drivers/rtc/rtc-xgene.c +++ b/drivers/rtc/rtc-xgene.c @@ -180,8 +180,6 @@ static int xgene_rtc_probe(struct platform_device *pdev) return ret; } - /* HW does not support update faster than 1 seconds */ - pdata->rtc->uie_unsupported = 1; pdata->rtc->ops = &xgene_rtc_ops; pdata->rtc->range_max = U32_MAX; diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index f0763e36b861..cb2491761958 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -745,9 +745,7 @@ sclp_sync_wait(void) /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { /* Check for expired request timer */ - if (timer_pending(&sclp_request_timer) && - get_tod_clock_fast() > timeout && - del_timer(&sclp_request_timer)) + if (get_tod_clock_fast() > timeout && del_timer(&sclp_request_timer)) sclp_request_timer.function(&sclp_request_timer); cpu_relax(); } diff --git a/drivers/s390/char/sclp_con.c b/drivers/s390/char/sclp_con.c index de028868c6f4..fe5ee2646fcf 100644 --- a/drivers/s390/char/sclp_con.c +++ b/drivers/s390/char/sclp_con.c @@ -109,8 +109,7 @@ static void sclp_console_sync_queue(void) unsigned long flags; spin_lock_irqsave(&sclp_con_lock, flags); - if (timer_pending(&sclp_con_timer)) - del_timer(&sclp_con_timer); + del_timer(&sclp_con_timer); while (sclp_con_queue_running) { spin_unlock_irqrestore(&sclp_con_lock, flags); sclp_sync_wait(); diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 7bc4e4a10937..3b4e7e5d9b71 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -231,8 +231,7 @@ sclp_vt220_emit_current(void) list_add_tail(&sclp_vt220_current_request->list, &sclp_vt220_outqueue); sclp_vt220_current_request = NULL; - if (timer_pending(&sclp_vt220_timer)) - del_timer(&sclp_vt220_timer); + del_timer(&sclp_vt220_timer); } sclp_vt220_flush_later = 0; } @@ -776,8 +775,7 @@ static void __sclp_vt220_flush_buffer(void) sclp_vt220_emit_current(); spin_lock_irqsave(&sclp_vt220_lock, flags); - if (timer_pending(&sclp_vt220_timer)) - del_timer(&sclp_vt220_timer); + del_timer(&sclp_vt220_timer); while (sclp_vt220_queue_running) { spin_unlock_irqrestore(&sclp_vt220_lock, flags); sclp_sync_wait(); diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 7ada994d4592..38cc1565d6ae 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -354,10 +354,10 @@ tape_34xx_unit_check(struct tape_device *device, struct tape_request *request, if (( sense[0] == SENSE_DATA_CHECK || sense[0] == SENSE_EQUIPMENT_CHECK || - sense[0] == SENSE_EQUIPMENT_CHECK + SENSE_DEFERRED_UNIT_CHECK + sense[0] == (SENSE_EQUIPMENT_CHECK | SENSE_DEFERRED_UNIT_CHECK) ) && ( sense[1] == SENSE_DRIVE_ONLINE || - sense[1] == SENSE_BEGINNING_OF_TAPE + SENSE_WRITE_MODE + sense[1] == (SENSE_BEGINNING_OF_TAPE | SENSE_WRITE_MODE) )) { switch (request->op) { /* diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index 05e136cfb8be..6d63b968309a 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -113,16 +113,10 @@ ccw_device_timeout(struct timer_list *t) void ccw_device_set_timeout(struct ccw_device *cdev, int expires) { - if (expires == 0) { + if (expires == 0) del_timer(&cdev->private->timer); - return; - } - if (timer_pending(&cdev->private->timer)) { - if (mod_timer(&cdev->private->timer, jiffies + expires)) - return; - } - cdev->private->timer.expires = jiffies + expires; - add_timer(&cdev->private->timer); + else + mod_timer(&cdev->private->timer, jiffies + expires); } int diff --git a/drivers/s390/cio/eadm_sch.c b/drivers/s390/cio/eadm_sch.c index 8b463681a149..ab6a7495180a 100644 --- a/drivers/s390/cio/eadm_sch.c +++ b/drivers/s390/cio/eadm_sch.c @@ -112,16 +112,10 @@ static void eadm_subchannel_set_timeout(struct subchannel *sch, int expires) { struct eadm_private *private = get_eadm_private(sch); - if (expires == 0) { + if (expires == 0) del_timer(&private->timer); - return; - } - if (timer_pending(&private->timer)) { - if (mod_timer(&private->timer, jiffies + expires)) - return; - } - private->timer.expires = jiffies + expires; - add_timer(&private->timer); + else + mod_timer(&private->timer, jiffies + expires); } static void eadm_subchannel_irq(struct subchannel *sch) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 8fd5a17bdf99..6a65885f5f43 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -315,6 +315,7 @@ struct ap_perms { unsigned long ioctlm[BITS_TO_LONGS(AP_IOCTLS)]; unsigned long apm[BITS_TO_LONGS(AP_DEVICES)]; unsigned long aqm[BITS_TO_LONGS(AP_DOMAINS)]; + unsigned long adm[BITS_TO_LONGS(AP_DOMAINS)]; }; extern struct ap_perms ap_perms; extern struct mutex ap_perms_mutex; diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index cf23ce1b1146..7f69ca695fc2 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -155,7 +155,7 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) /* * The cca_xxx2protkey call may fail when a card has been * addressed where the master key was changed after last fetch - * of the mkvp into the cache. Try 3 times: First witout verify + * of the mkvp into the cache. Try 3 times: First without verify * then with verify and last round with verify and old master * key verification pattern match not ignored. */ diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 7dc26365e29a..6e08d04b605d 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1189,13 +1189,6 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { * @matrix_mdev: a mediated matrix device * @kvm: reference to KVM instance * - * Note: The matrix_dev->lock must be taken prior to calling - * this function; however, the lock will be temporarily released while the - * guest's AP configuration is set to avoid a potential lockdep splat. - * The kvm->lock is taken to set the guest's AP configuration which, under - * certain circumstances, will result in a circular lock dependency if this is - * done under the @matrix_mdev->lock. - * * Return: 0 if no other mediated matrix device has a reference to @kvm; * otherwise, returns an -EPERM. */ @@ -1269,18 +1262,11 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, * by @matrix_mdev. * * @matrix_mdev: a matrix mediated device - * @kvm: the pointer to the kvm structure being unset. - * - * Note: The matrix_dev->lock must be taken prior to calling - * this function; however, the lock will be temporarily released while the - * guest's AP configuration is cleared to avoid a potential lockdep splat. - * The kvm->lock is taken to clear the guest's AP configuration which, under - * certain circumstances, will result in a circular lock dependency if this is - * done under the @matrix_mdev->lock. */ -static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev, - struct kvm *kvm) +static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) { + struct kvm *kvm = matrix_mdev->kvm; + if (kvm && kvm->arch.crypto.crycbd) { down_write(&kvm->arch.crypto.pqap_hook_rwsem); kvm->arch.crypto.pqap_hook = NULL; @@ -1311,7 +1297,7 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb, matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier); if (!data) - vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm); + vfio_ap_mdev_unset_kvm(matrix_mdev); else if (vfio_ap_mdev_set_kvm(matrix_mdev, data)) notify_rc = NOTIFY_DONE; @@ -1448,7 +1434,7 @@ static void vfio_ap_mdev_close_device(struct vfio_device *vdev) &matrix_mdev->iommu_notifier); vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY, &matrix_mdev->group_notifier); - vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm); + vfio_ap_mdev_unset_kvm(matrix_mdev); } static int vfio_ap_mdev_get_device_info(unsigned long arg) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 80e2a306709a..aa6dc3c0c353 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -285,10 +285,53 @@ static ssize_t aqmask_store(struct device *dev, static DEVICE_ATTR_RW(aqmask); +static ssize_t admask_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i, rc; + struct zcdn_device *zcdndev = to_zcdn_dev(dev); + + if (mutex_lock_interruptible(&ap_perms_mutex)) + return -ERESTARTSYS; + + buf[0] = '0'; + buf[1] = 'x'; + for (i = 0; i < sizeof(zcdndev->perms.adm) / sizeof(long); i++) + snprintf(buf + 2 + 2 * i * sizeof(long), + PAGE_SIZE - 2 - 2 * i * sizeof(long), + "%016lx", zcdndev->perms.adm[i]); + buf[2 + 2 * i * sizeof(long)] = '\n'; + buf[2 + 2 * i * sizeof(long) + 1] = '\0'; + rc = 2 + 2 * i * sizeof(long) + 1; + + mutex_unlock(&ap_perms_mutex); + + return rc; +} + +static ssize_t admask_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc; + struct zcdn_device *zcdndev = to_zcdn_dev(dev); + + rc = ap_parse_mask_str(buf, zcdndev->perms.adm, + AP_DOMAINS, &ap_perms_mutex); + if (rc) + return rc; + + return count; +} + +static DEVICE_ATTR_RW(admask); + static struct attribute *zcdn_dev_attrs[] = { &dev_attr_ioctlmask.attr, &dev_attr_apmask.attr, &dev_attr_aqmask.attr, + &dev_attr_admask.attr, NULL }; @@ -880,11 +923,22 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, if (rc) goto out; + tdom = *domain; + if (perms != &ap_perms && tdom < AP_DOMAINS) { + if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { + if (!test_bit_inv(tdom, perms->adm)) { + rc = -ENODEV; + goto out; + } + } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) { + rc = -EOPNOTSUPP; + goto out; + } + } /* * If a valid target domain is set and this domain is NOT a usage * domain but a control only domain, autoselect target domain. */ - tdom = *domain; if (tdom < AP_DOMAINS && !ap_test_config_usage_domain(tdom) && ap_test_config_ctrl_domain(tdom)) @@ -1062,6 +1116,18 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (rc) goto out_free; + if (perms != &ap_perms && domain < AUTOSEL_DOM) { + if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { + if (!test_bit_inv(domain, perms->adm)) { + rc = -ENODEV; + goto out_free; + } + } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) { + rc = -EOPNOTSUPP; + goto out_free; + } + } + pref_zc = NULL; pref_zq = NULL; spin_lock(&zcrypt_list_lock); diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index 3e259befd30a..fcbd537530e8 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -90,7 +90,7 @@ static ssize_t online_store(struct device *dev, list_for_each_entry(zq, &zc->zqueues, list) maxzqs++; if (maxzqs > 0) - zq_uelist = kcalloc(maxzqs + 1, sizeof(zq), GFP_ATOMIC); + zq_uelist = kcalloc(maxzqs + 1, sizeof(*zq_uelist), GFP_ATOMIC); list_for_each_entry(zq, &zc->zqueues, list) if (zcrypt_queue_force_online(zq, online)) if (zq_uelist) { diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 9ce5a71da69b..98d33f932b0b 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -1109,7 +1109,7 @@ static int ep11_wrapkey(u16 card, u16 domain, if (kb->head.type == TOKTYPE_NON_CCA && kb->head.version == TOKVER_EP11_AES) { has_header = true; - keysize = kb->head.len < keysize ? kb->head.len : keysize; + keysize = min_t(size_t, kb->head.len, keysize); } /* request cprb and payload */ diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 7d41dfe48ade..48c4dadb0c7c 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -10,45 +10,29 @@ #include "ifcvf_base.h" -static inline u8 ifc_ioread8(u8 __iomem *addr) -{ - return ioread8(addr); -} -static inline u16 ifc_ioread16 (__le16 __iomem *addr) +struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw) { - return ioread16(addr); + return container_of(hw, struct ifcvf_adapter, vf); } -static inline u32 ifc_ioread32(__le32 __iomem *addr) +u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector) { - return ioread32(addr); -} + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; -static inline void ifc_iowrite8(u8 value, u8 __iomem *addr) -{ - iowrite8(value, addr); -} + vp_iowrite16(qid, &cfg->queue_select); + vp_iowrite16(vector, &cfg->queue_msix_vector); -static inline void ifc_iowrite16(u16 value, __le16 __iomem *addr) -{ - iowrite16(value, addr); + return vp_ioread16(&cfg->queue_msix_vector); } -static inline void ifc_iowrite32(u32 value, __le32 __iomem *addr) +u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector) { - iowrite32(value, addr); -} + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; -static void ifc_iowrite64_twopart(u64 val, - __le32 __iomem *lo, __le32 __iomem *hi) -{ - ifc_iowrite32((u32)val, lo); - ifc_iowrite32(val >> 32, hi); -} + cfg = hw->common_cfg; + vp_iowrite16(vector, &cfg->msix_config); -struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw) -{ - return container_of(hw, struct ifcvf_adapter, vf); + return vp_ioread16(&cfg->msix_config); } static void __iomem *get_cap_addr(struct ifcvf_hw *hw, @@ -158,15 +142,16 @@ next: return -EIO; } - hw->nr_vring = ifc_ioread16(&hw->common_cfg->num_queues); + hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues); for (i = 0; i < hw->nr_vring; i++) { - ifc_iowrite16(i, &hw->common_cfg->queue_select); - notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off); + vp_iowrite16(i, &hw->common_cfg->queue_select); + notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off); hw->vring[i].notify_addr = hw->notify_base + notify_off * hw->notify_off_multiplier; hw->vring[i].notify_pa = hw->notify_base_pa + notify_off * hw->notify_off_multiplier; + hw->vring[i].irq = -EINVAL; } hw->lm_cfg = hw->base[IFCVF_LM_BAR]; @@ -176,17 +161,20 @@ next: hw->common_cfg, hw->notify_base, hw->isr, hw->dev_cfg, hw->notify_off_multiplier); + hw->vqs_reused_irq = -EINVAL; + hw->config_irq = -EINVAL; + return 0; } u8 ifcvf_get_status(struct ifcvf_hw *hw) { - return ifc_ioread8(&hw->common_cfg->device_status); + return vp_ioread8(&hw->common_cfg->device_status); } void ifcvf_set_status(struct ifcvf_hw *hw, u8 status) { - ifc_iowrite8(status, &hw->common_cfg->device_status); + vp_iowrite8(status, &hw->common_cfg->device_status); } void ifcvf_reset(struct ifcvf_hw *hw) @@ -214,11 +202,11 @@ u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) u32 features_lo, features_hi; u64 features; - ifc_iowrite32(0, &cfg->device_feature_select); - features_lo = ifc_ioread32(&cfg->device_feature); + vp_iowrite32(0, &cfg->device_feature_select); + features_lo = vp_ioread32(&cfg->device_feature); - ifc_iowrite32(1, &cfg->device_feature_select); - features_hi = ifc_ioread32(&cfg->device_feature); + vp_iowrite32(1, &cfg->device_feature_select); + features_hi = vp_ioread32(&cfg->device_feature); features = ((u64)features_hi << 32) | features_lo; @@ -271,12 +259,12 @@ void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset, WARN_ON(offset + length > hw->config_size); do { - old_gen = ifc_ioread8(&hw->common_cfg->config_generation); + old_gen = vp_ioread8(&hw->common_cfg->config_generation); p = dst; for (i = 0; i < length; i++) - *p++ = ifc_ioread8(hw->dev_cfg + offset + i); + *p++ = vp_ioread8(hw->dev_cfg + offset + i); - new_gen = ifc_ioread8(&hw->common_cfg->config_generation); + new_gen = vp_ioread8(&hw->common_cfg->config_generation); } while (old_gen != new_gen); } @@ -289,18 +277,18 @@ void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset, p = src; WARN_ON(offset + length > hw->config_size); for (i = 0; i < length; i++) - ifc_iowrite8(*p++, hw->dev_cfg + offset + i); + vp_iowrite8(*p++, hw->dev_cfg + offset + i); } static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) { struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; - ifc_iowrite32(0, &cfg->guest_feature_select); - ifc_iowrite32((u32)features, &cfg->guest_feature); + vp_iowrite32(0, &cfg->guest_feature_select); + vp_iowrite32((u32)features, &cfg->guest_feature); - ifc_iowrite32(1, &cfg->guest_feature_select); - ifc_iowrite32(features >> 32, &cfg->guest_feature); + vp_iowrite32(1, &cfg->guest_feature_select); + vp_iowrite32(features >> 32, &cfg->guest_feature); } static int ifcvf_config_features(struct ifcvf_hw *hw) @@ -329,7 +317,7 @@ u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; q_pair_id = qid / hw->nr_vring; avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; - last_avail_idx = ifc_ioread16(avail_idx_addr); + last_avail_idx = vp_ioread16(avail_idx_addr); return last_avail_idx; } @@ -344,7 +332,7 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) q_pair_id = qid / hw->nr_vring; avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; hw->vring[qid].last_avail_idx = num; - ifc_iowrite16(num, avail_idx_addr); + vp_iowrite16(num, avail_idx_addr); return 0; } @@ -352,41 +340,23 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) static int ifcvf_hw_enable(struct ifcvf_hw *hw) { struct virtio_pci_common_cfg __iomem *cfg; - struct ifcvf_adapter *ifcvf; u32 i; - ifcvf = vf_to_adapter(hw); cfg = hw->common_cfg; - ifc_iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config); - - if (ifc_ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) { - IFCVF_ERR(ifcvf->pdev, "No msix vector for device config\n"); - return -EINVAL; - } - for (i = 0; i < hw->nr_vring; i++) { if (!hw->vring[i].ready) break; - ifc_iowrite16(i, &cfg->queue_select); - ifc_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, + vp_iowrite16(i, &cfg->queue_select); + vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi); - ifc_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, + vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, &cfg->queue_avail_hi); - ifc_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, + vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, &cfg->queue_used_hi); - ifc_iowrite16(hw->vring[i].size, &cfg->queue_size); - ifc_iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector); - - if (ifc_ioread16(&cfg->queue_msix_vector) == - VIRTIO_MSI_NO_VECTOR) { - IFCVF_ERR(ifcvf->pdev, - "No msix vector for queue %u\n", i); - return -EINVAL; - } - + vp_iowrite16(hw->vring[i].size, &cfg->queue_size); ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx); - ifc_iowrite16(1, &cfg->queue_enable); + vp_iowrite16(1, &cfg->queue_enable); } return 0; @@ -394,18 +364,12 @@ static int ifcvf_hw_enable(struct ifcvf_hw *hw) static void ifcvf_hw_disable(struct ifcvf_hw *hw) { - struct virtio_pci_common_cfg __iomem *cfg; u32 i; - cfg = hw->common_cfg; - ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config); - + ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR); for (i = 0; i < hw->nr_vring; i++) { - ifc_iowrite16(i, &cfg->queue_select); - ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector); + ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR); } - - ifc_ioread16(&cfg->queue_msix_vector); } int ifcvf_start_hw(struct ifcvf_hw *hw) @@ -433,5 +397,5 @@ void ifcvf_stop_hw(struct ifcvf_hw *hw) void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid) { - ifc_iowrite16(qid, hw->vring[qid].notify_addr); + vp_iowrite16(qid, hw->vring[qid].notify_addr); } diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index c486873f370a..115b61f4924b 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -14,6 +14,7 @@ #include <linux/pci.h> #include <linux/pci_regs.h> #include <linux/vdpa.h> +#include <linux/virtio_pci_modern.h> #include <uapi/linux/virtio_net.h> #include <uapi/linux/virtio_blk.h> #include <uapi/linux/virtio_config.h> @@ -27,8 +28,6 @@ #define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE #define IFCVF_QUEUE_MAX 32768 -#define IFCVF_MSI_CONFIG_OFF 0 -#define IFCVF_MSI_QUEUE_OFF 1 #define IFCVF_PCI_MAX_RESOURCE 6 #define IFCVF_LM_CFG_SIZE 0x40 @@ -42,6 +41,13 @@ #define ifcvf_private_to_vf(adapter) \ (&((struct ifcvf_adapter *)adapter)->vf) +/* all vqs and config interrupt has its own vector */ +#define MSIX_VECTOR_PER_VQ_AND_CONFIG 1 +/* all vqs share a vector, and config interrupt has a separate vector */ +#define MSIX_VECTOR_SHARED_VQ_AND_CONFIG 2 +/* all vqs and config interrupt share a vector */ +#define MSIX_VECTOR_DEV_SHARED 3 + struct vring_info { u64 desc; u64 avail; @@ -60,25 +66,27 @@ struct ifcvf_hw { u8 __iomem *isr; /* Live migration */ u8 __iomem *lm_cfg; - u16 nr_vring; /* Notification bar number */ u8 notify_bar; + u8 msix_vector_status; + /* virtio-net or virtio-blk device config size */ + u32 config_size; /* Notificaiton bar address */ void __iomem *notify_base; phys_addr_t notify_base_pa; u32 notify_off_multiplier; + u32 dev_type; u64 req_features; u64 hw_features; - u32 dev_type; struct virtio_pci_common_cfg __iomem *common_cfg; void __iomem *dev_cfg; struct vring_info vring[IFCVF_MAX_QUEUES]; void __iomem * const *base; char config_msix_name[256]; struct vdpa_callback config_cb; - unsigned int config_irq; - /* virtio-net or virtio-blk device config size */ - u32 config_size; + int config_irq; + int vqs_reused_irq; + u16 nr_vring; }; struct ifcvf_adapter { @@ -123,4 +131,6 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num); struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw); int ifcvf_probed_virtio_net(struct ifcvf_hw *hw); u32 ifcvf_get_config_size(struct ifcvf_hw *hw); +u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector); +u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector); #endif /* _IFCVF_H_ */ diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index d1a6b5ab543c..4366320fb68d 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -27,7 +27,7 @@ static irqreturn_t ifcvf_config_changed(int irq, void *arg) return IRQ_HANDLED; } -static irqreturn_t ifcvf_intr_handler(int irq, void *arg) +static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg) { struct vring_info *vring = arg; @@ -37,76 +37,324 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg) return IRQ_HANDLED; } +static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg) +{ + struct ifcvf_hw *vf = arg; + struct vring_info *vring; + int i; + + for (i = 0; i < vf->nr_vring; i++) { + vring = &vf->vring[i]; + if (vring->cb.callback) + vring->cb.callback(vring->cb.private); + } + + return IRQ_HANDLED; +} + +static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg) +{ + struct ifcvf_hw *vf = arg; + u8 isr; + + isr = vp_ioread8(vf->isr); + if (isr & VIRTIO_PCI_ISR_CONFIG) + ifcvf_config_changed(irq, arg); + + return ifcvf_vqs_reused_intr_handler(irq, arg); +} + static void ifcvf_free_irq_vectors(void *data) { pci_free_irq_vectors(data); } -static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues) +static void ifcvf_free_per_vq_irq(struct ifcvf_adapter *adapter) { struct pci_dev *pdev = adapter->pdev; struct ifcvf_hw *vf = &adapter->vf; int i; + for (i = 0; i < vf->nr_vring; i++) { + if (vf->vring[i].irq != -EINVAL) { + devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]); + vf->vring[i].irq = -EINVAL; + } + } +} - for (i = 0; i < queues; i++) { - devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]); - vf->vring[i].irq = -EINVAL; +static void ifcvf_free_vqs_reused_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + + if (vf->vqs_reused_irq != -EINVAL) { + devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf); + vf->vqs_reused_irq = -EINVAL; } - devm_free_irq(&pdev->dev, vf->config_irq, vf); +} + +static void ifcvf_free_vq_irq(struct ifcvf_adapter *adapter) +{ + struct ifcvf_hw *vf = &adapter->vf; + + if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG) + ifcvf_free_per_vq_irq(adapter); + else + ifcvf_free_vqs_reused_irq(adapter); +} + +static void ifcvf_free_config_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + + if (vf->config_irq == -EINVAL) + return; + + /* If the irq is shared by all vqs and the config interrupt, + * it is already freed in ifcvf_free_vq_irq, so here only + * need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED + */ + if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) { + devm_free_irq(&pdev->dev, vf->config_irq, vf); + vf->config_irq = -EINVAL; + } +} + +static void ifcvf_free_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + + ifcvf_free_vq_irq(adapter); + ifcvf_free_config_irq(adapter); ifcvf_free_irq_vectors(pdev); } -static int ifcvf_request_irq(struct ifcvf_adapter *adapter) +/* ifcvf MSIX vectors allocator, this helper tries to allocate + * vectors for all virtqueues and the config interrupt. + * It returns the number of allocated vectors, negative + * return value when fails. + */ +static int ifcvf_alloc_vectors(struct ifcvf_adapter *adapter) { struct pci_dev *pdev = adapter->pdev; struct ifcvf_hw *vf = &adapter->vf; - int vector, i, ret, irq; - u16 max_intr; + int max_intr, ret; /* all queues and config interrupt */ max_intr = vf->nr_vring + 1; + ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); - ret = pci_alloc_irq_vectors(pdev, max_intr, - max_intr, PCI_IRQ_MSIX); if (ret < 0) { IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n"); return ret; } + if (ret < max_intr) + IFCVF_INFO(pdev, + "Requested %u vectors, however only %u allocated, lower performance\n", + max_intr, ret); + + return ret; +} + +static int ifcvf_request_per_vq_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int i, vector, ret, irq; + + vf->vqs_reused_irq = -EINVAL; + for (i = 0; i < vf->nr_vring; i++) { + snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i); + vector = i; + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_vq_intr_handler, 0, + vf->vring[i].msix_name, + &vf->vring[i]); + if (ret) { + IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i); + goto err; + } + + vf->vring[i].irq = irq; + ret = ifcvf_set_vq_vector(vf, i, vector); + if (ret == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(pdev, "No msix vector for vq %u\n", i); + goto err; + } + } + + return 0; +err: + ifcvf_free_irq(adapter); + + return -EFAULT; +} + +static int ifcvf_request_vqs_reused_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int i, vector, ret, irq; + + vector = 0; + snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev)); + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_vqs_reused_intr_handler, 0, + vf->vring[0].msix_name, vf); + if (ret) { + IFCVF_ERR(pdev, "Failed to request reused irq for the device\n"); + goto err; + } + + vf->vqs_reused_irq = irq; + for (i = 0; i < vf->nr_vring; i++) { + vf->vring[i].irq = -EINVAL; + ret = ifcvf_set_vq_vector(vf, i, vector); + if (ret == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(pdev, "No msix vector for vq %u\n", i); + goto err; + } + } + + return 0; +err: + ifcvf_free_irq(adapter); + + return -EFAULT; +} + +static int ifcvf_request_dev_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int i, vector, ret, irq; + + vector = 0; + snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev)); + irq = pci_irq_vector(pdev, vector); + ret = devm_request_irq(&pdev->dev, irq, + ifcvf_dev_intr_handler, 0, + vf->vring[0].msix_name, vf); + if (ret) { + IFCVF_ERR(pdev, "Failed to request irq for the device\n"); + goto err; + } + + vf->vqs_reused_irq = irq; + for (i = 0; i < vf->nr_vring; i++) { + vf->vring[i].irq = -EINVAL; + ret = ifcvf_set_vq_vector(vf, i, vector); + if (ret == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(pdev, "No msix vector for vq %u\n", i); + goto err; + } + } + + vf->config_irq = irq; + ret = ifcvf_set_config_vector(vf, vector); + if (ret == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(pdev, "No msix vector for device config\n"); + goto err; + } + + return 0; +err: + ifcvf_free_irq(adapter); + + return -EFAULT; + +} + +static int ifcvf_request_vq_irq(struct ifcvf_adapter *adapter) +{ + struct ifcvf_hw *vf = &adapter->vf; + int ret; + + if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG) + ret = ifcvf_request_per_vq_irq(adapter); + else + ret = ifcvf_request_vqs_reused_irq(adapter); + + return ret; +} + +static int ifcvf_request_config_irq(struct ifcvf_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + struct ifcvf_hw *vf = &adapter->vf; + int config_vector, ret; + + if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED) + return 0; + + if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG) + /* vector 0 ~ vf->nr_vring for vqs, num vf->nr_vring vector for config interrupt */ + config_vector = vf->nr_vring; + + if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG) + /* vector 0 for vqs and 1 for config interrupt */ + config_vector = 1; + snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n", pci_name(pdev)); - vector = 0; - vf->config_irq = pci_irq_vector(pdev, vector); + vf->config_irq = pci_irq_vector(pdev, config_vector); ret = devm_request_irq(&pdev->dev, vf->config_irq, ifcvf_config_changed, 0, vf->config_msix_name, vf); if (ret) { IFCVF_ERR(pdev, "Failed to request config irq\n"); - return ret; + goto err; } - for (i = 0; i < vf->nr_vring; i++) { - snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", - pci_name(pdev), i); - vector = i + IFCVF_MSI_QUEUE_OFF; - irq = pci_irq_vector(pdev, vector); - ret = devm_request_irq(&pdev->dev, irq, - ifcvf_intr_handler, 0, - vf->vring[i].msix_name, - &vf->vring[i]); - if (ret) { - IFCVF_ERR(pdev, - "Failed to request irq for vq %d\n", i); - ifcvf_free_irq(adapter, i); + ret = ifcvf_set_config_vector(vf, config_vector); + if (ret == VIRTIO_MSI_NO_VECTOR) { + IFCVF_ERR(pdev, "No msix vector for device config\n"); + goto err; + } - return ret; - } + return 0; +err: + ifcvf_free_irq(adapter); - vf->vring[i].irq = irq; + return -EFAULT; +} + +static int ifcvf_request_irq(struct ifcvf_adapter *adapter) +{ + struct ifcvf_hw *vf = &adapter->vf; + int nvectors, ret, max_intr; + + nvectors = ifcvf_alloc_vectors(adapter); + if (nvectors <= 0) + return -EFAULT; + + vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG; + max_intr = vf->nr_vring + 1; + if (nvectors < max_intr) + vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG; + + if (nvectors == 1) { + vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED; + ret = ifcvf_request_dev_irq(adapter); + + return ret; } + ret = ifcvf_request_vq_irq(adapter); + if (ret) + return ret; + + ret = ifcvf_request_config_irq(adapter); + + if (ret) + return ret; + return 0; } @@ -263,7 +511,7 @@ static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) { ifcvf_stop_datapath(adapter); - ifcvf_free_irq(adapter, vf->nr_vring); + ifcvf_free_irq(adapter); } ifcvf_reset_vring(adapter); @@ -348,7 +596,7 @@ static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev) { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - return ioread8(&vf->common_cfg->config_generation); + return vp_ioread8(&vf->common_cfg->config_generation); } static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev) @@ -410,7 +658,10 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); - return vf->vring[qid].irq; + if (vf->vqs_reused_irq < 0) + return vf->vring[qid].irq; + else + return -EINVAL; } static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev, diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index d0f91078600e..79001301b383 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -163,6 +163,7 @@ struct mlx5_vdpa_net { u32 cur_num_vqs; struct notifier_block nb; struct vdpa_callback config_cb; + struct mlx5_vdpa_wq_ent cvq_ent; }; static void free_resources(struct mlx5_vdpa_net *ndev); @@ -1475,7 +1476,7 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) virtio_net_ctrl_ack status = VIRTIO_NET_ERR; struct mlx5_core_dev *pfmdev; size_t read; - u8 mac[ETH_ALEN]; + u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); switch (cmd) { @@ -1489,6 +1490,9 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) break; } + if (is_zero_ether_addr(mac)) + break; + if (!is_zero_ether_addr(ndev->config.mac)) { if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", @@ -1503,7 +1507,47 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) break; } + /* backup the original mac address so that if failed to add the forward rules + * we could restore it + */ + memcpy(mac_back, ndev->config.mac, ETH_ALEN); + memcpy(ndev->config.mac, mac, ETH_ALEN); + + /* Need recreate the flow table entry, so that the packet could forward back + */ + remove_fwd_to_tir(ndev); + + if (add_fwd_to_tir(ndev)) { + mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); + + /* Although it hardly run here, we still need double check */ + if (is_zero_ether_addr(mac_back)) { + mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); + break; + } + + /* Try to restore original mac address to MFPS table, and try to restore + * the forward rule entry. + */ + if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { + mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", + ndev->config.mac); + } + + if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { + mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", + mac_back); + } + + memcpy(ndev->config.mac, mac_back, ETH_ALEN); + + if (add_fwd_to_tir(ndev)) + mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); + + break; + } + status = VIRTIO_NET_OK; break; @@ -1615,6 +1659,12 @@ static void mlx5_cvq_kick_handler(struct work_struct *work) mvdev = wqent->mvdev; ndev = to_mlx5_vdpa_ndev(mvdev); cvq = &mvdev->cvq; + + mutex_lock(&ndev->reslock); + + if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) + goto out; + if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) goto out; @@ -1653,9 +1703,13 @@ static void mlx5_cvq_kick_handler(struct work_struct *work) if (vringh_need_notify_iotlb(&cvq->vring)) vringh_notify(&cvq->vring); + + queue_work(mvdev->wq, &wqent->work); + break; } + out: - kfree(wqent); + mutex_unlock(&ndev->reslock); } static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) @@ -1663,22 +1717,15 @@ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct mlx5_vdpa_virtqueue *mvq; - struct mlx5_vdpa_wq_ent *wqent; if (!is_index_valid(mvdev, idx)) return; if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { - if (!mvdev->cvq.ready) - return; - - wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); - if (!wqent) + if (!mvdev->wq || !mvdev->cvq.ready) return; - wqent->mvdev = mvdev; - INIT_WORK(&wqent->work, mlx5_cvq_kick_handler); - queue_work(mvdev->wq, &wqent->work); + queue_work(mvdev->wq, &ndev->cvq_ent.work); return; } @@ -2137,7 +2184,7 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb goto err_mr; if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) - return 0; + goto err_mr; restore_channels_info(ndev); err = setup_driver(mvdev); @@ -2152,12 +2199,14 @@ err_mr: return err; } +/* reslock must be held for this function */ static int setup_driver(struct mlx5_vdpa_dev *mvdev) { struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); int err; - mutex_lock(&ndev->reslock); + WARN_ON(!mutex_is_locked(&ndev->reslock)); + if (ndev->setup) { mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); err = 0; @@ -2187,7 +2236,6 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev) goto err_fwd; } ndev->setup = true; - mutex_unlock(&ndev->reslock); return 0; @@ -2198,23 +2246,23 @@ err_tir: err_rqt: teardown_virtqueues(ndev); out: - mutex_unlock(&ndev->reslock); return err; } +/* reslock must be held for this function */ static void teardown_driver(struct mlx5_vdpa_net *ndev) { - mutex_lock(&ndev->reslock); + + WARN_ON(!mutex_is_locked(&ndev->reslock)); + if (!ndev->setup) - goto out; + return; remove_fwd_to_tir(ndev); destroy_tir(ndev); destroy_rqt(ndev); teardown_virtqueues(ndev); ndev->setup = false; -out: - mutex_unlock(&ndev->reslock); } static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) @@ -2235,6 +2283,8 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) print_status(mvdev, status, true); + mutex_lock(&ndev->reslock); + if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { if (status & VIRTIO_CONFIG_S_DRIVER_OK) { err = setup_driver(mvdev); @@ -2244,16 +2294,19 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) } } else { mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); - return; + goto err_clear; } } ndev->mvdev.status = status; + mutex_unlock(&ndev->reslock); return; err_setup: mlx5_vdpa_destroy_mr(&ndev->mvdev); ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; +err_clear: + mutex_unlock(&ndev->reslock); } static int mlx5_vdpa_reset(struct vdpa_device *vdev) @@ -2263,6 +2316,8 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) print_status(mvdev, 0, true); mlx5_vdpa_info(mvdev, "performing device reset\n"); + + mutex_lock(&ndev->reslock); teardown_driver(ndev); clear_vqs_ready(ndev); mlx5_vdpa_destroy_mr(&ndev->mvdev); @@ -2275,6 +2330,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) if (mlx5_vdpa_create_mr(mvdev, NULL)) mlx5_vdpa_warn(mvdev, "create MR failed\n"); } + mutex_unlock(&ndev->reslock); return 0; } @@ -2310,19 +2366,24 @@ static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); bool change_map; int err; + mutex_lock(&ndev->reslock); + err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); if (err) { mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); - return err; + goto err; } if (change_map) - return mlx5_vdpa_change_map(mvdev, iotlb); + err = mlx5_vdpa_change_map(mvdev, iotlb); - return 0; +err: + mutex_unlock(&ndev->reslock); + return err; } static void mlx5_vdpa_free(struct vdpa_device *vdev) @@ -2565,6 +2626,28 @@ static int event_handler(struct notifier_block *nb, unsigned long event, void *p return ret; } +static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) +{ + int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); + void *in; + int err; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); + MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, + mtu + MLX5V_ETH_HARD_MTU); + MLX5_SET(modify_nic_vport_context_in, in, opcode, + MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); + + err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); + + kvfree(in); + return err; +} + static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, const struct vdpa_dev_set_config *add_config) { @@ -2624,6 +2707,13 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, init_mvqs(ndev); mutex_init(&ndev->reslock); config = &ndev->config; + + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { + err = config_func_mtu(mdev, add_config->net.mtu); + if (err) + goto err_mtu; + } + err = query_mtu(mdev, &mtu); if (err) goto err_mtu; @@ -2668,6 +2758,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, if (err) goto err_mr; + ndev->cvq_ent.mvdev = mvdev; + INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); if (!mvdev->wq) { err = -ENOMEM; @@ -2707,9 +2799,12 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device * struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct workqueue_struct *wq; mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); - destroy_workqueue(mvdev->wq); + wq = mvdev->wq; + mvdev->wq = NULL; + destroy_workqueue(wq); _vdpa_unregister_device(dev); mgtdev->ndev = NULL; } @@ -2741,7 +2836,8 @@ static int mlx5v_probe(struct auxiliary_device *adev, mgtdev->mgtdev.device = mdev->device; mgtdev->mgtdev.id_table = id_table; mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | - BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); + BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | + BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); mgtdev->mgtdev.max_supported_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; mgtdev->mgtdev.supported_features = get_supported_features(mdev); diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 1ea525433a5c..2b75c00b1005 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -232,7 +232,7 @@ static int vdpa_name_match(struct device *dev, const void *data) return (strcmp(dev_name(&vdev->dev), data) == 0); } -static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs) +static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) { struct device *dev; @@ -257,7 +257,7 @@ static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs) * * Return: Returns an error when fail to add device to vDPA bus */ -int _vdpa_register_device(struct vdpa_device *vdev, int nvqs) +int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) { if (!vdev->mdev) return -EINVAL; @@ -274,7 +274,7 @@ EXPORT_SYMBOL_GPL(_vdpa_register_device); * * Return: Returns an error when fail to add to vDPA bus */ -int vdpa_register_device(struct vdpa_device *vdev, int nvqs) +int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) { int err; diff --git a/drivers/vhost/iotlb.c b/drivers/vhost/iotlb.c index 40b098320b2a..5829cf2d0552 100644 --- a/drivers/vhost/iotlb.c +++ b/drivers/vhost/iotlb.c @@ -62,8 +62,12 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, */ if (start == 0 && last == ULONG_MAX) { u64 mid = last / 2; + int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, + perm, opaque); + + if (err) + return err; - vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, perm, opaque); addr += mid + 1; start = mid + 1; } diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index ec5249e8c32d..4c2f0bd06285 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -42,7 +42,7 @@ struct vhost_vdpa { struct device dev; struct cdev cdev; atomic_t opened; - int nvqs; + u32 nvqs; int virtio_id; int minor; struct eventfd_ctx *config_ctx; @@ -97,8 +97,11 @@ static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid) return; irq = ops->get_vq_irq(vdpa, qid); + if (irq < 0) + return; + irq_bypass_unregister_producer(&vq->call_ctx.producer); - if (!vq->call_ctx.ctx || irq < 0) + if (!vq->call_ctx.ctx) return; vq->call_ctx.producer.token = vq->call_ctx.ctx; @@ -158,7 +161,8 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u8 status, status_old; - int ret, nvqs = v->nvqs; + u32 nvqs = v->nvqs; + int ret; u16 i; if (copy_from_user(&status, statusp, sizeof(status))) @@ -355,6 +359,30 @@ static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp) return 0; } +static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + u32 size; + + size = ops->get_config_size(vdpa); + + if (copy_to_user(argp, &size, sizeof(size))) + return -EFAULT; + + return 0; +} + +static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp) +{ + struct vdpa_device *vdpa = v->vdpa; + + if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs))) + return -EFAULT; + + return 0; +} + static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { @@ -492,6 +520,12 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_VDPA_GET_IOVA_RANGE: r = vhost_vdpa_get_iova_range(v, argp); break; + case VHOST_VDPA_GET_CONFIG_SIZE: + r = vhost_vdpa_get_config_size(v, argp); + break; + case VHOST_VDPA_GET_VQS_COUNT: + r = vhost_vdpa_get_vqs_count(v, argp); + break; default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) @@ -948,7 +982,8 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) struct vhost_vdpa *v; struct vhost_dev *dev; struct vhost_virtqueue **vqs; - int nvqs, i, r, opened; + int r, opened; + u32 i, nvqs; v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); @@ -1001,7 +1036,7 @@ err: static void vhost_vdpa_clean_irq(struct vhost_vdpa *v) { - int i; + u32 i; for (i = 0; i < v->nvqs; i++) vhost_vdpa_unsetup_vq_irq(v, i); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 1768362115c6..d02173fb290c 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -2550,8 +2550,9 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) &vq->avail->idx, r); return false; } + vq->avail_idx = vhost16_to_cpu(vq, avail_idx); - return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx; + return vq->avail_idx != vq->last_avail_idx; } EXPORT_SYMBOL_GPL(vhost_enable_notify); diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c index 0ae1a39f2e28..a1c467a0e9f7 100644 --- a/drivers/virt/vmgenid.c +++ b/drivers/virt/vmgenid.c @@ -78,6 +78,7 @@ static void vmgenid_notify(struct acpi_device *device, u32 event) } static const struct acpi_device_id vmgenid_ids[] = { + { "VMGENCTR", 0 }, { "VM_GEN_COUNTER", 0 }, { } }; diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 492fc26f0b65..b5adf6abd241 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -105,7 +105,7 @@ config VIRTIO_BALLOON config VIRTIO_MEM tristate "Virtio mem driver" - depends on X86_64 + depends on X86_64 || ARM64 depends on VIRTIO depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE @@ -115,8 +115,9 @@ config VIRTIO_MEM This driver provides access to virtio-mem paravirtualized memory devices, allowing to hotplug and hotunplug memory. - This driver was only tested under x86-64, but should theoretically - work on all architectures that support memory hotplug and hotremove. + This driver was only tested under x86-64 and arm64, but should + theoretically work on all architectures that support memory hotplug + and hotremove. If unsure, say M. diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index fdbde1db5ec5..d724f676608b 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -24,46 +24,17 @@ MODULE_PARM_DESC(force_legacy, "Force legacy mode for transitional virtio 1 devices"); #endif -/* disable irq handlers */ -void vp_disable_cbs(struct virtio_device *vdev) +/* wait for pending irq handlers */ +void vp_synchronize_vectors(struct virtio_device *vdev) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); int i; - if (vp_dev->intx_enabled) { - /* - * The below synchronize() guarantees that any - * interrupt for this line arriving after - * synchronize_irq() has completed is guaranteed to see - * intx_soft_enabled == false. - */ - WRITE_ONCE(vp_dev->intx_soft_enabled, false); + if (vp_dev->intx_enabled) synchronize_irq(vp_dev->pci_dev->irq); - } - - for (i = 0; i < vp_dev->msix_vectors; ++i) - disable_irq(pci_irq_vector(vp_dev->pci_dev, i)); -} - -/* enable irq handlers */ -void vp_enable_cbs(struct virtio_device *vdev) -{ - struct virtio_pci_device *vp_dev = to_vp_device(vdev); - int i; - - if (vp_dev->intx_enabled) { - disable_irq(vp_dev->pci_dev->irq); - /* - * The above disable_irq() provides TSO ordering and - * as such promotes the below store to store-release. - */ - WRITE_ONCE(vp_dev->intx_soft_enabled, true); - enable_irq(vp_dev->pci_dev->irq); - return; - } for (i = 0; i < vp_dev->msix_vectors; ++i) - enable_irq(pci_irq_vector(vp_dev->pci_dev, i)); + synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i)); } /* the notify function used when creating a virt queue */ @@ -113,9 +84,6 @@ static irqreturn_t vp_interrupt(int irq, void *opaque) struct virtio_pci_device *vp_dev = opaque; u8 isr; - if (!READ_ONCE(vp_dev->intx_soft_enabled)) - return IRQ_NONE; - /* reading the ISR has the effect of also clearing it so it's very * important to save off the value. */ isr = ioread8(vp_dev->isr); @@ -173,8 +141,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, "%s-config", name); err = request_irq(pci_irq_vector(vp_dev->pci_dev, v), - vp_config_changed, IRQF_NO_AUTOEN, - vp_dev->msix_names[v], + vp_config_changed, 0, vp_dev->msix_names[v], vp_dev); if (err) goto error; @@ -193,8 +160,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, "%s-virtqueues", name); err = request_irq(pci_irq_vector(vp_dev->pci_dev, v), - vp_vring_interrupt, IRQF_NO_AUTOEN, - vp_dev->msix_names[v], + vp_vring_interrupt, 0, vp_dev->msix_names[v], vp_dev); if (err) goto error; @@ -371,7 +337,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, "%s-%s", dev_name(&vp_dev->vdev.dev), names[i]); err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec), - vring_interrupt, IRQF_NO_AUTOEN, + vring_interrupt, 0, vp_dev->msix_names[msix_vec], vqs[i]); if (err) diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 23f6c5c678d5..eb17a29fc7ef 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -63,7 +63,6 @@ struct virtio_pci_device { /* MSI-X support */ int msix_enabled; int intx_enabled; - bool intx_soft_enabled; cpumask_var_t *msix_affinity_masks; /* Name strings for interrupts. This size should be enough, * and I'm too lazy to allocate each name separately. */ @@ -102,10 +101,8 @@ static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev) return container_of(vdev, struct virtio_pci_device, vdev); } -/* disable irq handlers */ -void vp_disable_cbs(struct virtio_device *vdev); -/* enable irq handlers */ -void vp_enable_cbs(struct virtio_device *vdev); +/* wait for pending irq handlers */ +void vp_synchronize_vectors(struct virtio_device *vdev); /* the notify function used when creating a virt queue */ bool vp_notify(struct virtqueue *vq); /* the config->del_vqs() implementation */ diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 34141b9abe27..6f4e34ce96b8 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -98,8 +98,8 @@ static void vp_reset(struct virtio_device *vdev) /* Flush out the status write, and flush in device writes, * including MSi-X interrupts, if any. */ vp_legacy_get_status(&vp_dev->ldev); - /* Disable VQ/configuration callbacks. */ - vp_disable_cbs(vdev); + /* Flush pending VQ/configuration callbacks. */ + vp_synchronize_vectors(vdev); } static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) @@ -185,7 +185,6 @@ static void del_vq(struct virtio_pci_vq_info *info) } static const struct virtio_config_ops virtio_pci_config_ops = { - .enable_cbs = vp_enable_cbs, .get = vp_get, .set = vp_set, .get_status = vp_get_status, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 5455bc041fb6..a2671a20ef77 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -172,8 +172,8 @@ static void vp_reset(struct virtio_device *vdev) */ while (vp_modern_get_status(mdev)) msleep(1); - /* Disable VQ/configuration callbacks. */ - vp_disable_cbs(vdev); + /* Flush pending VQ/configuration callbacks. */ + vp_synchronize_vectors(vdev); } static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) @@ -293,7 +293,7 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0; pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { - u8 type, cap_len, id; + u8 type, cap_len, id, res_bar; u32 tmp32; u64 res_offset, res_length; @@ -315,9 +315,14 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, if (id != required_id) continue; - /* Type, and ID match, looks good */ pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, - bar), bar); + bar), &res_bar); + if (res_bar >= PCI_STD_NUM_BARS) + continue; + + /* Type and ID match, and the BAR value isn't reserved. + * Looks good. + */ /* Read the lower 32bit of length and offset */ pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap, @@ -337,6 +342,7 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, length_hi), &tmp32); res_length |= ((u64)tmp32) << 32; + *bar = res_bar; *offset = res_offset; *len = res_length; @@ -380,7 +386,6 @@ static bool vp_get_shm_region(struct virtio_device *vdev, } static const struct virtio_config_ops virtio_pci_config_nodev_ops = { - .enable_cbs = vp_enable_cbs, .get = NULL, .set = NULL, .generation = vp_generation, @@ -398,7 +403,6 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { }; static const struct virtio_config_ops virtio_pci_config_ops = { - .enable_cbs = vp_enable_cbs, .get = vp_get, .set = vp_set, .generation = vp_generation, diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index e8b3ff2b9fbc..591738ad3d56 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -35,6 +35,13 @@ vp_modern_map_capability(struct virtio_pci_modern_device *mdev, int off, pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), &length); + /* Check if the BAR may have changed since we requested the region. */ + if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) { + dev_err(&dev->dev, + "virtio_pci: bar unexpectedly changed to %u\n", bar); + return NULL; + } + if (length <= start) { dev_err(&dev->dev, "virtio_pci: bad capability len %u (>%u expected)\n", @@ -120,7 +127,7 @@ static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, &bar); /* Ignore structures with reserved BAR values */ - if (bar > 0x5) + if (bar >= PCI_STD_NUM_BARS) continue; if (type == cfg_type) { diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 962f1477b1fa..cfb028ca238e 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -379,19 +379,11 @@ static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); - if (flags & VRING_DESC_F_INDIRECT) { - dma_unmap_single(vring_dma_dev(vq), - virtio64_to_cpu(vq->vq.vdev, desc->addr), - virtio32_to_cpu(vq->vq.vdev, desc->len), - (flags & VRING_DESC_F_WRITE) ? - DMA_FROM_DEVICE : DMA_TO_DEVICE); - } else { - dma_unmap_page(vring_dma_dev(vq), - virtio64_to_cpu(vq->vq.vdev, desc->addr), - virtio32_to_cpu(vq->vq.vdev, desc->len), - (flags & VRING_DESC_F_WRITE) ? - DMA_FROM_DEVICE : DMA_TO_DEVICE); - } + dma_unmap_page(vring_dma_dev(vq), + virtio64_to_cpu(vq->vq.vdev, desc->addr), + virtio32_to_cpu(vq->vq.vdev, desc->len), + (flags & VRING_DESC_F_WRITE) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); } static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, @@ -984,24 +976,24 @@ static struct virtqueue *vring_create_virtqueue_split( * Packed ring specific functions - *_packed(). */ -static void vring_unmap_state_packed(const struct vring_virtqueue *vq, - struct vring_desc_extra *state) +static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, + struct vring_desc_extra *extra) { u16 flags; if (!vq->use_dma_api) return; - flags = state->flags; + flags = extra->flags; if (flags & VRING_DESC_F_INDIRECT) { dma_unmap_single(vring_dma_dev(vq), - state->addr, state->len, + extra->addr, extra->len, (flags & VRING_DESC_F_WRITE) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); } else { dma_unmap_page(vring_dma_dev(vq), - state->addr, state->len, + extra->addr, extra->len, (flags & VRING_DESC_F_WRITE) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); } @@ -1017,19 +1009,11 @@ static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, flags = le16_to_cpu(desc->flags); - if (flags & VRING_DESC_F_INDIRECT) { - dma_unmap_single(vring_dma_dev(vq), - le64_to_cpu(desc->addr), - le32_to_cpu(desc->len), - (flags & VRING_DESC_F_WRITE) ? - DMA_FROM_DEVICE : DMA_TO_DEVICE); - } else { - dma_unmap_page(vring_dma_dev(vq), - le64_to_cpu(desc->addr), - le32_to_cpu(desc->len), - (flags & VRING_DESC_F_WRITE) ? - DMA_FROM_DEVICE : DMA_TO_DEVICE); - } + dma_unmap_page(vring_dma_dev(vq), + le64_to_cpu(desc->addr), + le32_to_cpu(desc->len), + (flags & VRING_DESC_F_WRITE) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); } static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg, @@ -1303,8 +1287,7 @@ unmap_release: for (n = 0; n < total_sg; n++) { if (i == err_idx) break; - vring_unmap_state_packed(vq, - &vq->packed.desc_extra[curr]); + vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); curr = vq->packed.desc_extra[curr].next; i++; if (i >= vq->packed.vring.num) @@ -1383,8 +1366,8 @@ static void detach_buf_packed(struct vring_virtqueue *vq, if (unlikely(vq->use_dma_api)) { curr = id; for (i = 0; i < state->num; i++) { - vring_unmap_state_packed(vq, - &vq->packed.desc_extra[curr]); + vring_unmap_extra_packed(vq, + &vq->packed.desc_extra[curr]); curr = vq->packed.desc_extra[curr].next; } } diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 085f5a4ad06c..c4e82a8d863f 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -1779,7 +1779,7 @@ config BCM7038_WDT tristate "BCM63xx/BCM7038 Watchdog" select WATCHDOG_CORE depends on HAS_IOMEM - depends on ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST + depends on ARCH_BCM4908 || ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST help Watchdog driver for the built-in hardware in Broadcom 7038 and later SoCs used in set-top boxes. BCM7038 was made public diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c index 436571b6fc79..bd06622813eb 100644 --- a/drivers/watchdog/aspeed_wdt.c +++ b/drivers/watchdog/aspeed_wdt.c @@ -13,6 +13,11 @@ #include <linux/platform_device.h> #include <linux/watchdog.h> +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); + struct aspeed_wdt { struct watchdog_device wdd; void __iomem *base; @@ -266,6 +271,8 @@ static int aspeed_wdt_probe(struct platform_device *pdev) wdt->wdd.timeout = WDT_DEFAULT_TIMEOUT; watchdog_init_timeout(&wdt->wdd, 0, dev); + watchdog_set_nowayout(&wdt->wdd, nowayout); + np = dev->of_node; ofdid = of_match_node(aspeed_wdt_of_table, np); diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c index 51bfb796898b..d0c5d47ddede 100644 --- a/drivers/watchdog/imx2_wdt.c +++ b/drivers/watchdog/imx2_wdt.c @@ -66,6 +66,7 @@ struct imx2_wdt_device { struct watchdog_device wdog; bool ext_reset; bool clk_is_on; + bool no_ping; }; static bool nowayout = WATCHDOG_NOWAYOUT; @@ -312,12 +313,18 @@ static int __init imx2_wdt_probe(struct platform_device *pdev) wdev->ext_reset = of_property_read_bool(dev->of_node, "fsl,ext-reset-output"); + /* + * The i.MX7D doesn't support low power mode, so we need to ping the watchdog + * during suspend. + */ + wdev->no_ping = !of_device_is_compatible(dev->of_node, "fsl,imx7d-wdt"); platform_set_drvdata(pdev, wdog); watchdog_set_drvdata(wdog, wdev); watchdog_set_nowayout(wdog, nowayout); watchdog_set_restart_priority(wdog, 128); watchdog_init_timeout(wdog, timeout, dev); - watchdog_stop_ping_on_suspend(wdog); + if (wdev->no_ping) + watchdog_stop_ping_on_suspend(wdog); if (imx2_wdt_is_running(wdev)) { imx2_wdt_set_timeout(wdog, wdog->timeout); @@ -366,9 +373,11 @@ static int __maybe_unused imx2_wdt_suspend(struct device *dev) imx2_wdt_ping(wdog); } - clk_disable_unprepare(wdev->clk); + if (wdev->no_ping) { + clk_disable_unprepare(wdev->clk); - wdev->clk_is_on = false; + wdev->clk_is_on = false; + } return 0; } @@ -380,11 +389,14 @@ static int __maybe_unused imx2_wdt_resume(struct device *dev) struct imx2_wdt_device *wdev = watchdog_get_drvdata(wdog); int ret; - ret = clk_prepare_enable(wdev->clk); - if (ret) - return ret; + if (wdev->no_ping) { + ret = clk_prepare_enable(wdev->clk); - wdev->clk_is_on = true; + if (ret) + return ret; + + wdev->clk_is_on = true; + } if (watchdog_active(wdog) && !imx2_wdt_is_running(wdev)) { /* @@ -407,6 +419,7 @@ static SIMPLE_DEV_PM_OPS(imx2_wdt_pm_ops, imx2_wdt_suspend, static const struct of_device_id imx2_wdt_dt_ids[] = { { .compatible = "fsl,imx21-wdt", }, + { .compatible = "fsl,imx7d-wdt", }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, imx2_wdt_dt_ids); diff --git a/drivers/watchdog/ixp4xx_wdt.c b/drivers/watchdog/ixp4xx_wdt.c index 31b03fa71341..281a48d9889f 100644 --- a/drivers/watchdog/ixp4xx_wdt.c +++ b/drivers/watchdog/ixp4xx_wdt.c @@ -84,10 +84,24 @@ static int ixp4xx_wdt_set_timeout(struct watchdog_device *wdd, return 0; } +static int ixp4xx_wdt_restart(struct watchdog_device *wdd, + unsigned long action, void *data) +{ + struct ixp4xx_wdt *iwdt = to_ixp4xx_wdt(wdd); + + __raw_writel(IXP4XX_WDT_KEY, iwdt->base + IXP4XX_OSWK_OFFSET); + __raw_writel(0, iwdt->base + IXP4XX_OSWT_OFFSET); + __raw_writel(IXP4XX_WDT_COUNT_ENABLE | IXP4XX_WDT_RESET_ENABLE, + iwdt->base + IXP4XX_OSWE_OFFSET); + + return 0; +} + static const struct watchdog_ops ixp4xx_wdt_ops = { .start = ixp4xx_wdt_start, .stop = ixp4xx_wdt_stop, .set_timeout = ixp4xx_wdt_set_timeout, + .restart = ixp4xx_wdt_restart, .owner = THIS_MODULE, }; diff --git a/drivers/watchdog/orion_wdt.c b/drivers/watchdog/orion_wdt.c index 127eefc9161d..e25e6bf4647f 100644 --- a/drivers/watchdog/orion_wdt.c +++ b/drivers/watchdog/orion_wdt.c @@ -238,8 +238,10 @@ static int armada370_start(struct watchdog_device *wdt_dev) atomic_io_modify(dev->reg + TIMER_A370_STATUS, WDT_A370_EXPIRED, 0); /* Enable watchdog timer */ - atomic_io_modify(dev->reg + TIMER_CTRL, dev->data->wdt_enable_bit, - dev->data->wdt_enable_bit); + reg = dev->data->wdt_enable_bit; + if (dev->wdt.info->options & WDIOF_PRETIMEOUT) + reg |= TIMER1_ENABLE_BIT; + atomic_io_modify(dev->reg + TIMER_CTRL, reg, reg); /* Enable reset on watchdog */ reg = readl(dev->rstout); @@ -312,7 +314,7 @@ static int armada375_stop(struct watchdog_device *wdt_dev) static int armada370_stop(struct watchdog_device *wdt_dev) { struct orion_watchdog *dev = watchdog_get_drvdata(wdt_dev); - u32 reg; + u32 reg, mask; /* Disable reset on watchdog */ reg = readl(dev->rstout); @@ -320,7 +322,10 @@ static int armada370_stop(struct watchdog_device *wdt_dev) writel(reg, dev->rstout); /* Disable watchdog timer */ - atomic_io_modify(dev->reg + TIMER_CTRL, dev->data->wdt_enable_bit, 0); + mask = dev->data->wdt_enable_bit; + if (wdt_dev->info->options & WDIOF_PRETIMEOUT) + mask |= TIMER1_ENABLE_BIT; + atomic_io_modify(dev->reg + TIMER_CTRL, mask, 0); return 0; } diff --git a/drivers/watchdog/renesas_wdt.c b/drivers/watchdog/renesas_wdt.c index 5791198960e6..41d58ea5eb2f 100644 --- a/drivers/watchdog/renesas_wdt.c +++ b/drivers/watchdog/renesas_wdt.c @@ -327,6 +327,7 @@ static SIMPLE_DEV_PM_OPS(rwdt_pm_ops, rwdt_suspend, rwdt_resume); static const struct of_device_id rwdt_ids[] = { { .compatible = "renesas,rcar-gen2-wdt", }, { .compatible = "renesas,rcar-gen3-wdt", }, + { .compatible = "renesas,rcar-gen4-wdt", }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, rwdt_ids); diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 117bc2a8eb0a..db843f825860 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -228,6 +228,7 @@ static int rti_wdt_probe(struct platform_device *pdev) ret = pm_runtime_get_sync(dev); if (ret) { pm_runtime_put_noidle(dev); + pm_runtime_disable(&pdev->dev); return dev_err_probe(dev, ret, "runtime pm failed\n"); } diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index dd9a744f82f8..86ffb58fbc85 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -49,7 +49,7 @@ /* internal variables */ enum tco_reg_layout { - sp5100, sb800, efch + sp5100, sb800, efch, efch_mmio }; struct sp5100_tco { @@ -87,6 +87,10 @@ static enum tco_reg_layout tco_reg_layout(struct pci_dev *dev) dev->revision < 0x40) { return sp5100; } else if (dev->vendor == PCI_VENDOR_ID_AMD && + sp5100_tco_pci->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS && + sp5100_tco_pci->revision >= AMD_ZEN_SMBUS_PCI_REV) { + return efch_mmio; + } else if (dev->vendor == PCI_VENDOR_ID_AMD && ((dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS && dev->revision >= 0x41) || (dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS && @@ -209,6 +213,8 @@ static void tco_timer_enable(struct sp5100_tco *tco) ~EFCH_PM_WATCHDOG_DISABLE, EFCH_PM_DECODEEN_SECOND_RES); break; + default: + break; } } @@ -223,14 +229,195 @@ static u32 sp5100_tco_read_pm_reg32(u8 index) return val; } +static u32 sp5100_tco_request_region(struct device *dev, + u32 mmio_addr, + const char *dev_name) +{ + if (!devm_request_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE, + dev_name)) { + dev_dbg(dev, "MMIO address 0x%08x already in use\n", mmio_addr); + return 0; + } + + return mmio_addr; +} + +static u32 sp5100_tco_prepare_base(struct sp5100_tco *tco, + u32 mmio_addr, + u32 alt_mmio_addr, + const char *dev_name) +{ + struct device *dev = tco->wdd.parent; + + dev_dbg(dev, "Got 0x%08x from SBResource_MMIO register\n", mmio_addr); + + if (!mmio_addr && !alt_mmio_addr) + return -ENODEV; + + /* Check for MMIO address and alternate MMIO address conflicts */ + if (mmio_addr) + mmio_addr = sp5100_tco_request_region(dev, mmio_addr, dev_name); + + if (!mmio_addr && alt_mmio_addr) + mmio_addr = sp5100_tco_request_region(dev, alt_mmio_addr, dev_name); + + if (!mmio_addr) { + dev_err(dev, "Failed to reserve MMIO or alternate MMIO region\n"); + return -EBUSY; + } + + tco->tcobase = devm_ioremap(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE); + if (!tco->tcobase) { + dev_err(dev, "MMIO address 0x%08x failed mapping\n", mmio_addr); + devm_release_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE); + return -ENOMEM; + } + + dev_info(dev, "Using 0x%08x for watchdog MMIO address\n", mmio_addr); + + return 0; +} + +static int sp5100_tco_timer_init(struct sp5100_tco *tco) +{ + struct watchdog_device *wdd = &tco->wdd; + struct device *dev = wdd->parent; + u32 val; + + val = readl(SP5100_WDT_CONTROL(tco->tcobase)); + if (val & SP5100_WDT_DISABLED) { + dev_err(dev, "Watchdog hardware is disabled\n"); + return -ENODEV; + } + + /* + * Save WatchDogFired status, because WatchDogFired flag is + * cleared here. + */ + if (val & SP5100_WDT_FIRED) + wdd->bootstatus = WDIOF_CARDRESET; + + /* Set watchdog action to reset the system */ + val &= ~SP5100_WDT_ACTION_RESET; + writel(val, SP5100_WDT_CONTROL(tco->tcobase)); + + /* Set a reasonable heartbeat before we stop the timer */ + tco_timer_set_timeout(wdd, wdd->timeout); + + /* + * Stop the TCO before we change anything so we don't race with + * a zeroed timer. + */ + tco_timer_stop(wdd); + + return 0; +} + +static u8 efch_read_pm_reg8(void __iomem *addr, u8 index) +{ + return readb(addr + index); +} + +static void efch_update_pm_reg8(void __iomem *addr, u8 index, u8 reset, u8 set) +{ + u8 val; + + val = readb(addr + index); + val &= reset; + val |= set; + writeb(val, addr + index); +} + +static void tco_timer_enable_mmio(void __iomem *addr) +{ + efch_update_pm_reg8(addr, EFCH_PM_DECODEEN3, + ~EFCH_PM_WATCHDOG_DISABLE, + EFCH_PM_DECODEEN_SECOND_RES); +} + +static int sp5100_tco_setupdevice_mmio(struct device *dev, + struct watchdog_device *wdd) +{ + struct sp5100_tco *tco = watchdog_get_drvdata(wdd); + const char *dev_name = SB800_DEVNAME; + u32 mmio_addr = 0, alt_mmio_addr = 0; + struct resource *res; + void __iomem *addr; + int ret; + u32 val; + + res = request_mem_region_muxed(EFCH_PM_ACPI_MMIO_PM_ADDR, + EFCH_PM_ACPI_MMIO_PM_SIZE, + "sp5100_tco"); + + if (!res) { + dev_err(dev, + "Memory region 0x%08x already in use\n", + EFCH_PM_ACPI_MMIO_PM_ADDR); + return -EBUSY; + } + + addr = ioremap(EFCH_PM_ACPI_MMIO_PM_ADDR, EFCH_PM_ACPI_MMIO_PM_SIZE); + if (!addr) { + dev_err(dev, "Address mapping failed\n"); + ret = -ENOMEM; + goto out; + } + + /* + * EFCH_PM_DECODEEN_WDT_TMREN is dual purpose. This bitfield + * enables sp5100_tco register MMIO space decoding. The bitfield + * also starts the timer operation. Enable if not already enabled. + */ + val = efch_read_pm_reg8(addr, EFCH_PM_DECODEEN); + if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) { + efch_update_pm_reg8(addr, EFCH_PM_DECODEEN, 0xff, + EFCH_PM_DECODEEN_WDT_TMREN); + } + + /* Error if the timer could not be enabled */ + val = efch_read_pm_reg8(addr, EFCH_PM_DECODEEN); + if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) { + dev_err(dev, "Failed to enable the timer\n"); + ret = -EFAULT; + goto out; + } + + mmio_addr = EFCH_PM_WDT_ADDR; + + /* Determine alternate MMIO base address */ + val = efch_read_pm_reg8(addr, EFCH_PM_ISACONTROL); + if (val & EFCH_PM_ISACONTROL_MMIOEN) + alt_mmio_addr = EFCH_PM_ACPI_MMIO_ADDR + + EFCH_PM_ACPI_MMIO_WDT_OFFSET; + + ret = sp5100_tco_prepare_base(tco, mmio_addr, alt_mmio_addr, dev_name); + if (!ret) { + tco_timer_enable_mmio(addr); + ret = sp5100_tco_timer_init(tco); + } + +out: + if (addr) + iounmap(addr); + + release_resource(res); + + return ret; +} + static int sp5100_tco_setupdevice(struct device *dev, struct watchdog_device *wdd) { struct sp5100_tco *tco = watchdog_get_drvdata(wdd); const char *dev_name; u32 mmio_addr = 0, val; + u32 alt_mmio_addr = 0; int ret; + if (tco->tco_reg_layout == efch_mmio) + return sp5100_tco_setupdevice_mmio(dev, wdd); + /* Request the IO ports used by this driver */ if (!request_muxed_region(SP5100_IO_PM_INDEX_REG, SP5100_PM_IOPORTS_SIZE, "sp5100_tco")) { @@ -247,138 +434,55 @@ static int sp5100_tco_setupdevice(struct device *dev, dev_name = SP5100_DEVNAME; mmio_addr = sp5100_tco_read_pm_reg32(SP5100_PM_WATCHDOG_BASE) & 0xfffffff8; + + /* + * Secondly, find the watchdog timer MMIO address + * from SBResource_MMIO register. + */ + + /* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */ + pci_read_config_dword(sp5100_tco_pci, + SP5100_SB_RESOURCE_MMIO_BASE, + &val); + + /* Verify MMIO is enabled and using bar0 */ + if ((val & SB800_ACPI_MMIO_MASK) == SB800_ACPI_MMIO_DECODE_EN) + alt_mmio_addr = (val & ~0xfff) + SB800_PM_WDT_MMIO_OFFSET; break; case sb800: dev_name = SB800_DEVNAME; mmio_addr = sp5100_tco_read_pm_reg32(SB800_PM_WATCHDOG_BASE) & 0xfffffff8; + + /* Read SBResource_MMIO from AcpiMmioEn(PM_Reg: 24h) */ + val = sp5100_tco_read_pm_reg32(SB800_PM_ACPI_MMIO_EN); + + /* Verify MMIO is enabled and using bar0 */ + if ((val & SB800_ACPI_MMIO_MASK) == SB800_ACPI_MMIO_DECODE_EN) + alt_mmio_addr = (val & ~0xfff) + SB800_PM_WDT_MMIO_OFFSET; break; case efch: dev_name = SB800_DEVNAME; - /* - * On Family 17h devices, the EFCH_PM_DECODEEN_WDT_TMREN bit of - * EFCH_PM_DECODEEN not only enables the EFCH_PM_WDT_ADDR memory - * region, it also enables the watchdog itself. - */ - if (boot_cpu_data.x86 == 0x17) { - val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN); - if (!(val & EFCH_PM_DECODEEN_WDT_TMREN)) { - sp5100_tco_update_pm_reg8(EFCH_PM_DECODEEN, 0xff, - EFCH_PM_DECODEEN_WDT_TMREN); - } - } val = sp5100_tco_read_pm_reg8(EFCH_PM_DECODEEN); if (val & EFCH_PM_DECODEEN_WDT_TMREN) mmio_addr = EFCH_PM_WDT_ADDR; + + val = sp5100_tco_read_pm_reg8(EFCH_PM_ISACONTROL); + if (val & EFCH_PM_ISACONTROL_MMIOEN) + alt_mmio_addr = EFCH_PM_ACPI_MMIO_ADDR + + EFCH_PM_ACPI_MMIO_WDT_OFFSET; break; default: return -ENODEV; } - /* Check MMIO address conflict */ - if (!mmio_addr || - !devm_request_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE, - dev_name)) { - if (mmio_addr) - dev_dbg(dev, "MMIO address 0x%08x already in use\n", - mmio_addr); - switch (tco->tco_reg_layout) { - case sp5100: - /* - * Secondly, Find the watchdog timer MMIO address - * from SBResource_MMIO register. - */ - /* Read SBResource_MMIO from PCI config(PCI_Reg: 9Ch) */ - pci_read_config_dword(sp5100_tco_pci, - SP5100_SB_RESOURCE_MMIO_BASE, - &mmio_addr); - if ((mmio_addr & (SB800_ACPI_MMIO_DECODE_EN | - SB800_ACPI_MMIO_SEL)) != - SB800_ACPI_MMIO_DECODE_EN) { - ret = -ENODEV; - goto unreg_region; - } - mmio_addr &= ~0xFFF; - mmio_addr += SB800_PM_WDT_MMIO_OFFSET; - break; - case sb800: - /* Read SBResource_MMIO from AcpiMmioEn(PM_Reg: 24h) */ - mmio_addr = - sp5100_tco_read_pm_reg32(SB800_PM_ACPI_MMIO_EN); - if ((mmio_addr & (SB800_ACPI_MMIO_DECODE_EN | - SB800_ACPI_MMIO_SEL)) != - SB800_ACPI_MMIO_DECODE_EN) { - ret = -ENODEV; - goto unreg_region; - } - mmio_addr &= ~0xFFF; - mmio_addr += SB800_PM_WDT_MMIO_OFFSET; - break; - case efch: - val = sp5100_tco_read_pm_reg8(EFCH_PM_ISACONTROL); - if (!(val & EFCH_PM_ISACONTROL_MMIOEN)) { - ret = -ENODEV; - goto unreg_region; - } - mmio_addr = EFCH_PM_ACPI_MMIO_ADDR + - EFCH_PM_ACPI_MMIO_WDT_OFFSET; - break; - } - dev_dbg(dev, "Got 0x%08x from SBResource_MMIO register\n", - mmio_addr); - if (!devm_request_mem_region(dev, mmio_addr, - SP5100_WDT_MEM_MAP_SIZE, - dev_name)) { - dev_dbg(dev, "MMIO address 0x%08x already in use\n", - mmio_addr); - ret = -EBUSY; - goto unreg_region; - } - } - - tco->tcobase = devm_ioremap(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE); - if (!tco->tcobase) { - dev_err(dev, "failed to get tcobase address\n"); - ret = -ENOMEM; - goto unreg_region; - } - - dev_info(dev, "Using 0x%08x for watchdog MMIO address\n", mmio_addr); - - /* Setup the watchdog timer */ - tco_timer_enable(tco); - - val = readl(SP5100_WDT_CONTROL(tco->tcobase)); - if (val & SP5100_WDT_DISABLED) { - dev_err(dev, "Watchdog hardware is disabled\n"); - ret = -ENODEV; - goto unreg_region; + ret = sp5100_tco_prepare_base(tco, mmio_addr, alt_mmio_addr, dev_name); + if (!ret) { + /* Setup the watchdog timer */ + tco_timer_enable(tco); + ret = sp5100_tco_timer_init(tco); } - /* - * Save WatchDogFired status, because WatchDogFired flag is - * cleared here. - */ - if (val & SP5100_WDT_FIRED) - wdd->bootstatus = WDIOF_CARDRESET; - /* Set watchdog action to reset the system */ - val &= ~SP5100_WDT_ACTION_RESET; - writel(val, SP5100_WDT_CONTROL(tco->tcobase)); - - /* Set a reasonable heartbeat before we stop the timer */ - tco_timer_set_timeout(wdd, wdd->timeout); - - /* - * Stop the TCO before we change anything so we don't race with - * a zeroed timer. - */ - tco_timer_stop(wdd); - - release_region(SP5100_IO_PM_INDEX_REG, SP5100_PM_IOPORTS_SIZE); - - return 0; - -unreg_region: release_region(SP5100_IO_PM_INDEX_REG, SP5100_PM_IOPORTS_SIZE); return ret; } diff --git a/drivers/watchdog/sp5100_tco.h b/drivers/watchdog/sp5100_tco.h index adf015aa4126..6a0986d2c94b 100644 --- a/drivers/watchdog/sp5100_tco.h +++ b/drivers/watchdog/sp5100_tco.h @@ -58,6 +58,7 @@ #define SB800_PM_WATCHDOG_SECOND_RES GENMASK(1, 0) #define SB800_ACPI_MMIO_DECODE_EN BIT(0) #define SB800_ACPI_MMIO_SEL BIT(1) +#define SB800_ACPI_MMIO_MASK GENMASK(1, 0) #define SB800_PM_WDT_MMIO_OFFSET 0xB00 @@ -82,4 +83,10 @@ #define EFCH_PM_ISACONTROL_MMIOEN BIT(1) #define EFCH_PM_ACPI_MMIO_ADDR 0xfed80000 +#define EFCH_PM_ACPI_MMIO_PM_OFFSET 0x00000300 #define EFCH_PM_ACPI_MMIO_WDT_OFFSET 0x00000b00 + +#define EFCH_PM_ACPI_MMIO_PM_ADDR (EFCH_PM_ACPI_MMIO_ADDR + \ + EFCH_PM_ACPI_MMIO_PM_OFFSET) +#define EFCH_PM_ACPI_MMIO_PM_SIZE 8 +#define AMD_ZEN_SMBUS_PCI_REV 0x51 diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 3a3d8b5c7ad5..54903f3c851e 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -171,17 +171,17 @@ static int __watchdog_ping(struct watchdog_device *wdd) } /* - * watchdog_ping: ping the watchdog. - * @wdd: the watchdog device to ping + * watchdog_ping - ping the watchdog + * @wdd: The watchdog device to ping * - * The caller must hold wd_data->lock. + * If the watchdog has no own ping operation then it needs to be + * restarted via the start operation. This wrapper function does + * exactly that. + * We only ping when the watchdog device is running. + * The caller must hold wd_data->lock. * - * If the watchdog has no own ping operation then it needs to be - * restarted via the start operation. This wrapper function does - * exactly that. - * We only ping when the watchdog device is running. + * Return: 0 on success, error otherwise. */ - static int watchdog_ping(struct watchdog_device *wdd) { struct watchdog_core_data *wd_data = wdd->wd_data; @@ -231,16 +231,14 @@ static enum hrtimer_restart watchdog_timer_expired(struct hrtimer *timer) } /* - * watchdog_start: wrapper to start the watchdog. - * @wdd: the watchdog device to start + * watchdog_start - wrapper to start the watchdog + * @wdd: The watchdog device to start * - * The caller must hold wd_data->lock. + * Start the watchdog if it is not active and mark it active. + * The caller must hold wd_data->lock. * - * Start the watchdog if it is not active and mark it active. - * This function returns zero on success or a negative errno code for - * failure. + * Return: 0 on success or a negative errno code for failure. */ - static int watchdog_start(struct watchdog_device *wdd) { struct watchdog_core_data *wd_data = wdd->wd_data; @@ -274,17 +272,15 @@ static int watchdog_start(struct watchdog_device *wdd) } /* - * watchdog_stop: wrapper to stop the watchdog. - * @wdd: the watchdog device to stop + * watchdog_stop - wrapper to stop the watchdog + * @wdd: The watchdog device to stop * - * The caller must hold wd_data->lock. + * Stop the watchdog if it is still active and unmark it active. + * If the 'nowayout' feature was set, the watchdog cannot be stopped. + * The caller must hold wd_data->lock. * - * Stop the watchdog if it is still active and unmark it active. - * This function returns zero on success or a negative errno code for - * failure. - * If the 'nowayout' feature was set, the watchdog cannot be stopped. + * Return: 0 on success or a negative errno code for failure. */ - static int watchdog_stop(struct watchdog_device *wdd) { int err = 0; @@ -315,14 +311,14 @@ static int watchdog_stop(struct watchdog_device *wdd) } /* - * watchdog_get_status: wrapper to get the watchdog status - * @wdd: the watchdog device to get the status from + * watchdog_get_status - wrapper to get the watchdog status + * @wdd: The watchdog device to get the status from * - * The caller must hold wd_data->lock. + * Get the watchdog's status flags. + * The caller must hold wd_data->lock. * - * Get the watchdog's status flags. + * Return: watchdog's status flags. */ - static unsigned int watchdog_get_status(struct watchdog_device *wdd) { struct watchdog_core_data *wd_data = wdd->wd_data; @@ -352,13 +348,14 @@ static unsigned int watchdog_get_status(struct watchdog_device *wdd) } /* - * watchdog_set_timeout: set the watchdog timer timeout - * @wdd: the watchdog device to set the timeout for - * @timeout: timeout to set in seconds + * watchdog_set_timeout - set the watchdog timer timeout + * @wdd: The watchdog device to set the timeout for + * @timeout: Timeout to set in seconds + * + * The caller must hold wd_data->lock. * - * The caller must hold wd_data->lock. + * Return: 0 if successful, error otherwise. */ - static int watchdog_set_timeout(struct watchdog_device *wdd, unsigned int timeout) { @@ -385,11 +382,12 @@ static int watchdog_set_timeout(struct watchdog_device *wdd, } /* - * watchdog_set_pretimeout: set the watchdog timer pretimeout - * @wdd: the watchdog device to set the timeout for - * @timeout: pretimeout to set in seconds + * watchdog_set_pretimeout - set the watchdog timer pretimeout + * @wdd: The watchdog device to set the timeout for + * @timeout: pretimeout to set in seconds + * + * Return: 0 if successful, error otherwise. */ - static int watchdog_set_pretimeout(struct watchdog_device *wdd, unsigned int timeout) { @@ -410,15 +408,15 @@ static int watchdog_set_pretimeout(struct watchdog_device *wdd, } /* - * watchdog_get_timeleft: wrapper to get the time left before a reboot - * @wdd: the watchdog device to get the remaining time from - * @timeleft: the time that's left + * watchdog_get_timeleft - wrapper to get the time left before a reboot + * @wdd: The watchdog device to get the remaining time from + * @timeleft: The time that's left * - * The caller must hold wd_data->lock. + * Get the time before a watchdog will reboot (if not pinged). + * The caller must hold wd_data->lock. * - * Get the time before a watchdog will reboot (if not pinged). + * Return: 0 if successful, error otherwise. */ - static int watchdog_get_timeleft(struct watchdog_device *wdd, unsigned int *timeleft) { @@ -635,14 +633,15 @@ __ATTRIBUTE_GROUPS(wdt); #endif /* - * watchdog_ioctl_op: call the watchdog drivers ioctl op if defined - * @wdd: the watchdog device to do the ioctl on - * @cmd: watchdog command - * @arg: argument pointer + * watchdog_ioctl_op - call the watchdog drivers ioctl op if defined + * @wdd: The watchdog device to do the ioctl on + * @cmd: Watchdog command + * @arg: Argument pointer * - * The caller must hold wd_data->lock. + * The caller must hold wd_data->lock. + * + * Return: 0 if successful, error otherwise. */ - static int watchdog_ioctl_op(struct watchdog_device *wdd, unsigned int cmd, unsigned long arg) { @@ -653,17 +652,18 @@ static int watchdog_ioctl_op(struct watchdog_device *wdd, unsigned int cmd, } /* - * watchdog_write: writes to the watchdog. - * @file: file from VFS - * @data: user address of data - * @len: length of data - * @ppos: pointer to the file offset + * watchdog_write - writes to the watchdog + * @file: File from VFS + * @data: User address of data + * @len: Length of data + * @ppos: Pointer to the file offset * - * A write to a watchdog device is defined as a keepalive ping. - * Writing the magic 'V' sequence allows the next close to turn - * off the watchdog (if 'nowayout' is not set). + * A write to a watchdog device is defined as a keepalive ping. + * Writing the magic 'V' sequence allows the next close to turn + * off the watchdog (if 'nowayout' is not set). + * + * Return: @len if successful, error otherwise. */ - static ssize_t watchdog_write(struct file *file, const char __user *data, size_t len, loff_t *ppos) { @@ -706,13 +706,15 @@ static ssize_t watchdog_write(struct file *file, const char __user *data, } /* - * watchdog_ioctl: handle the different ioctl's for the watchdog device. - * @file: file handle to the device - * @cmd: watchdog command - * @arg: argument pointer + * watchdog_ioctl - handle the different ioctl's for the watchdog device + * @file: File handle to the device + * @cmd: Watchdog command + * @arg: Argument pointer * - * The watchdog API defines a common set of functions for all watchdogs - * according to their available features. + * The watchdog API defines a common set of functions for all watchdogs + * according to their available features. + * + * Return: 0 if successful, error otherwise. */ static long watchdog_ioctl(struct file *file, unsigned int cmd, @@ -819,15 +821,16 @@ out_ioctl: } /* - * watchdog_open: open the /dev/watchdog* devices. - * @inode: inode of device - * @file: file handle to device + * watchdog_open - open the /dev/watchdog* devices + * @inode: Inode of device + * @file: File handle to device + * + * When the /dev/watchdog* device gets opened, we start the watchdog. + * Watch out: the /dev/watchdog device is single open, so we make sure + * it can only be opened once. * - * When the /dev/watchdog* device gets opened, we start the watchdog. - * Watch out: the /dev/watchdog device is single open, so we make sure - * it can only be opened once. + * Return: 0 if successful, error otherwise. */ - static int watchdog_open(struct inode *inode, struct file *file) { struct watchdog_core_data *wd_data; @@ -896,15 +899,16 @@ static void watchdog_core_data_release(struct device *dev) } /* - * watchdog_release: release the watchdog device. - * @inode: inode of device - * @file: file handle to device + * watchdog_release - release the watchdog device + * @inode: Inode of device + * @file: File handle to device + * + * This is the code for when /dev/watchdog gets closed. We will only + * stop the watchdog when we have received the magic char (and nowayout + * was not set), else the watchdog will keep running. * - * This is the code for when /dev/watchdog gets closed. We will only - * stop the watchdog when we have received the magic char (and nowayout - * was not set), else the watchdog will keep running. + * Always returns 0. */ - static int watchdog_release(struct inode *inode, struct file *file) { struct watchdog_core_data *wd_data = file->private_data; @@ -977,14 +981,15 @@ static struct class watchdog_class = { }; /* - * watchdog_cdev_register: register watchdog character device - * @wdd: watchdog device + * watchdog_cdev_register - register watchdog character device + * @wdd: Watchdog device + * + * Register a watchdog character device including handling the legacy + * /dev/watchdog node. /dev/watchdog is actually a miscdevice and + * thus we set it up like that. * - * Register a watchdog character device including handling the legacy - * /dev/watchdog node. /dev/watchdog is actually a miscdevice and - * thus we set it up like that. + * Return: 0 if successful, error otherwise. */ - static int watchdog_cdev_register(struct watchdog_device *wdd) { struct watchdog_core_data *wd_data; @@ -1074,13 +1079,12 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) } /* - * watchdog_cdev_unregister: unregister watchdog character device - * @watchdog: watchdog device + * watchdog_cdev_unregister - unregister watchdog character device + * @wdd: Watchdog device * - * Unregister watchdog character device and if needed the legacy - * /dev/watchdog device. + * Unregister watchdog character device and if needed the legacy + * /dev/watchdog device. */ - static void watchdog_cdev_unregister(struct watchdog_device *wdd) { struct watchdog_core_data *wd_data = wdd->wd_data; @@ -1109,15 +1113,16 @@ static void watchdog_cdev_unregister(struct watchdog_device *wdd) put_device(&wd_data->dev); } -/* - * watchdog_dev_register: register a watchdog device - * @wdd: watchdog device +/** + * watchdog_dev_register - register a watchdog device + * @wdd: Watchdog device + * + * Register a watchdog device including handling the legacy + * /dev/watchdog node. /dev/watchdog is actually a miscdevice and + * thus we set it up like that. * - * Register a watchdog device including handling the legacy - * /dev/watchdog node. /dev/watchdog is actually a miscdevice and - * thus we set it up like that. + * Return: 0 if successful, error otherwise. */ - int watchdog_dev_register(struct watchdog_device *wdd) { int ret; @@ -1133,30 +1138,31 @@ int watchdog_dev_register(struct watchdog_device *wdd) return ret; } -/* - * watchdog_dev_unregister: unregister a watchdog device - * @watchdog: watchdog device +/** + * watchdog_dev_unregister - unregister a watchdog device + * @wdd: watchdog device * - * Unregister watchdog device and if needed the legacy - * /dev/watchdog device. + * Unregister watchdog device and if needed the legacy + * /dev/watchdog device. */ - void watchdog_dev_unregister(struct watchdog_device *wdd) { watchdog_unregister_pretimeout(wdd); watchdog_cdev_unregister(wdd); } -/* - * watchdog_set_last_hw_keepalive: set last HW keepalive time for watchdog - * @wdd: watchdog device - * @last_ping_ms: time since last HW heartbeat +/** + * watchdog_set_last_hw_keepalive - set last HW keepalive time for watchdog + * @wdd: Watchdog device + * @last_ping_ms: Time since last HW heartbeat * - * Adjusts the last known HW keepalive time for a watchdog timer. - * This is needed if the watchdog is already running when the probe - * function is called, and it can't be pinged immediately. This - * function must be called immediately after watchdog registration, - * and min_hw_heartbeat_ms must be set for this to be useful. + * Adjusts the last known HW keepalive time for a watchdog timer. + * This is needed if the watchdog is already running when the probe + * function is called, and it can't be pinged immediately. This + * function must be called immediately after watchdog registration, + * and min_hw_heartbeat_ms must be set for this to be useful. + * + * Return: 0 if successful, error otherwise. */ int watchdog_set_last_hw_keepalive(struct watchdog_device *wdd, unsigned int last_ping_ms) @@ -1180,12 +1186,13 @@ int watchdog_set_last_hw_keepalive(struct watchdog_device *wdd, } EXPORT_SYMBOL_GPL(watchdog_set_last_hw_keepalive); -/* - * watchdog_dev_init: init dev part of watchdog core +/** + * watchdog_dev_init - init dev part of watchdog core * - * Allocate a range of chardev nodes to use for watchdog devices + * Allocate a range of chardev nodes to use for watchdog devices. + * + * Return: 0 if successful, error otherwise. */ - int __init watchdog_dev_init(void) { int err; @@ -1218,12 +1225,11 @@ err_register: return err; } -/* - * watchdog_dev_exit: exit dev part of watchdog core +/** + * watchdog_dev_exit - exit dev part of watchdog core * - * Release the range of chardev nodes used for watchdog devices + * Release the range of chardev nodes used for watchdog devices. */ - void __exit watchdog_dev_exit(void) { unregister_chrdev_region(watchdog_devt, MAX_DOGS); diff --git a/fs/9p/cache.c b/fs/9p/cache.c index 55e108e5e133..1c8dc696d516 100644 --- a/fs/9p/cache.c +++ b/fs/9p/cache.c @@ -49,22 +49,20 @@ int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, void v9fs_cache_inode_get_cookie(struct inode *inode) { - struct v9fs_inode *v9inode; + struct v9fs_inode *v9inode = V9FS_I(inode); struct v9fs_session_info *v9ses; __le32 version; __le64 path; if (!S_ISREG(inode->i_mode)) return; - - v9inode = V9FS_I(inode); - if (WARN_ON(v9inode->fscache)) + if (WARN_ON(v9fs_inode_cookie(v9inode))) return; version = cpu_to_le32(v9inode->qid.version); path = cpu_to_le64(v9inode->qid.path); v9ses = v9fs_inode2v9ses(inode); - v9inode->fscache = + v9inode->netfs_ctx.cache = fscache_acquire_cookie(v9fs_session_cache(v9ses), 0, &path, sizeof(path), @@ -72,5 +70,5 @@ void v9fs_cache_inode_get_cookie(struct inode *inode) i_size_read(&v9inode->vfs_inode)); p9_debug(P9_DEBUG_FSC, "inode %p get cookie %p\n", - inode, v9inode->fscache); + inode, v9fs_inode_cookie(v9inode)); } diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 08f65c40af4f..e28ddf763b3b 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -623,9 +623,7 @@ static void v9fs_sysfs_cleanup(void) static void v9fs_inode_init_once(void *foo) { struct v9fs_inode *v9inode = (struct v9fs_inode *)foo; -#ifdef CONFIG_9P_FSCACHE - v9inode->fscache = NULL; -#endif + memset(&v9inode->qid, 0, sizeof(v9inode->qid)); inode_init_once(&v9inode->vfs_inode); } diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h index bc8b30205d36..ec0e8df3b2eb 100644 --- a/fs/9p/v9fs.h +++ b/fs/9p/v9fs.h @@ -9,6 +9,7 @@ #define FS_9P_V9FS_H #include <linux/backing-dev.h> +#include <linux/netfs.h> /** * enum p9_session_flags - option flags for each 9P session @@ -108,14 +109,15 @@ struct v9fs_session_info { #define V9FS_INO_INVALID_ATTR 0x01 struct v9fs_inode { -#ifdef CONFIG_9P_FSCACHE - struct fscache_cookie *fscache; -#endif + struct { + /* These must be contiguous */ + struct inode vfs_inode; /* the VFS's inode record */ + struct netfs_i_context netfs_ctx; /* Netfslib context */ + }; struct p9_qid qid; unsigned int cache_validity; struct p9_fid *writeback_fid; struct mutex v_mutex; - struct inode vfs_inode; }; static inline struct v9fs_inode *V9FS_I(const struct inode *inode) @@ -126,7 +128,7 @@ static inline struct v9fs_inode *V9FS_I(const struct inode *inode) static inline struct fscache_cookie *v9fs_inode_cookie(struct v9fs_inode *v9inode) { #ifdef CONFIG_9P_FSCACHE - return v9inode->fscache; + return netfs_i_cookie(&v9inode->vfs_inode); #else return NULL; #endif @@ -163,6 +165,7 @@ extern struct inode *v9fs_inode_from_fid(struct v9fs_session_info *v9ses, extern const struct inode_operations v9fs_dir_inode_operations_dotl; extern const struct inode_operations v9fs_file_inode_operations_dotl; extern const struct inode_operations v9fs_symlink_inode_operations_dotl; +extern const struct netfs_request_ops v9fs_req_ops; extern struct inode *v9fs_inode_from_fid_dotl(struct v9fs_session_info *v9ses, struct p9_fid *fid, struct super_block *sb, int new); diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index 76956c9d2af9..501128188343 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -28,12 +28,12 @@ #include "fid.h" /** - * v9fs_req_issue_op - Issue a read from 9P + * v9fs_issue_read - Issue a read from 9P * @subreq: The read to make */ -static void v9fs_req_issue_op(struct netfs_read_subrequest *subreq) +static void v9fs_issue_read(struct netfs_io_subrequest *subreq) { - struct netfs_read_request *rreq = subreq->rreq; + struct netfs_io_request *rreq = subreq->rreq; struct p9_fid *fid = rreq->netfs_priv; struct iov_iter to; loff_t pos = subreq->start + subreq->transferred; @@ -52,20 +52,21 @@ static void v9fs_req_issue_op(struct netfs_read_subrequest *subreq) } /** - * v9fs_init_rreq - Initialise a read request + * v9fs_init_request - Initialise a read request * @rreq: The read request * @file: The file being read from */ -static void v9fs_init_rreq(struct netfs_read_request *rreq, struct file *file) +static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file) { struct p9_fid *fid = file->private_data; refcount_inc(&fid->count); rreq->netfs_priv = fid; + return 0; } /** - * v9fs_req_cleanup - Cleanup request initialized by v9fs_init_rreq + * v9fs_req_cleanup - Cleanup request initialized by v9fs_init_request * @mapping: unused mapping of request to cleanup * @priv: private data to cleanup, a fid, guaranted non-null. */ @@ -77,21 +78,10 @@ static void v9fs_req_cleanup(struct address_space *mapping, void *priv) } /** - * v9fs_is_cache_enabled - Determine if caching is enabled for an inode - * @inode: The inode to check - */ -static bool v9fs_is_cache_enabled(struct inode *inode) -{ - struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(inode)); - - return fscache_cookie_enabled(cookie) && cookie->cache_priv; -} - -/** * v9fs_begin_cache_operation - Begin a cache operation for a read * @rreq: The read request */ -static int v9fs_begin_cache_operation(struct netfs_read_request *rreq) +static int v9fs_begin_cache_operation(struct netfs_io_request *rreq) { #ifdef CONFIG_9P_FSCACHE struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(rreq->inode)); @@ -102,37 +92,14 @@ static int v9fs_begin_cache_operation(struct netfs_read_request *rreq) #endif } -static const struct netfs_read_request_ops v9fs_req_ops = { - .init_rreq = v9fs_init_rreq, - .is_cache_enabled = v9fs_is_cache_enabled, +const struct netfs_request_ops v9fs_req_ops = { + .init_request = v9fs_init_request, .begin_cache_operation = v9fs_begin_cache_operation, - .issue_op = v9fs_req_issue_op, + .issue_read = v9fs_issue_read, .cleanup = v9fs_req_cleanup, }; /** - * v9fs_vfs_readpage - read an entire page in from 9P - * @file: file being read - * @page: structure to page - * - */ -static int v9fs_vfs_readpage(struct file *file, struct page *page) -{ - struct folio *folio = page_folio(page); - - return netfs_readpage(file, folio, &v9fs_req_ops, NULL); -} - -/** - * v9fs_vfs_readahead - read a set of pages from 9P - * @ractl: The readahead parameters - */ -static void v9fs_vfs_readahead(struct readahead_control *ractl) -{ - netfs_readahead(ractl, &v9fs_req_ops, NULL); -} - -/** * v9fs_release_page - release the private state associated with a page * @page: The page to be released * @gfp: The caller's allocation restrictions @@ -308,8 +275,7 @@ static int v9fs_write_begin(struct file *filp, struct address_space *mapping, * file. We need to do this before we get a lock on the page in case * there's more than one writer competing for the same cache block. */ - retval = netfs_write_begin(filp, mapping, pos, len, flags, &folio, fsdata, - &v9fs_req_ops, NULL); + retval = netfs_write_begin(filp, mapping, pos, len, flags, &folio, fsdata); if (retval < 0) return retval; @@ -370,8 +336,8 @@ static bool v9fs_dirty_folio(struct address_space *mapping, struct folio *folio) #endif const struct address_space_operations v9fs_addr_operations = { - .readpage = v9fs_vfs_readpage, - .readahead = v9fs_vfs_readahead, + .readpage = netfs_readpage, + .readahead = netfs_readahead, .dirty_folio = v9fs_dirty_folio, .writepage = v9fs_vfs_writepage, .write_begin = v9fs_write_begin, diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index 84c3cf7dffa5..55367ecb9442 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -231,9 +231,6 @@ struct inode *v9fs_alloc_inode(struct super_block *sb) v9inode = alloc_inode_sb(sb, v9fs_inode_cache, GFP_KERNEL); if (!v9inode) return NULL; -#ifdef CONFIG_9P_FSCACHE - v9inode->fscache = NULL; -#endif v9inode->writeback_fid = NULL; v9inode->cache_validity = 0; mutex_init(&v9inode->v_mutex); @@ -250,6 +247,14 @@ void v9fs_free_inode(struct inode *inode) kmem_cache_free(v9fs_inode_cache, V9FS_I(inode)); } +/* + * Set parameters for the netfs library + */ +static void v9fs_set_netfs_context(struct inode *inode) +{ + netfs_i_context_init(inode, &v9fs_req_ops); +} + int v9fs_init_inode(struct v9fs_session_info *v9ses, struct inode *inode, umode_t mode, dev_t rdev) { @@ -338,6 +343,8 @@ int v9fs_init_inode(struct v9fs_session_info *v9ses, err = -EINVAL; goto error; } + + v9fs_set_netfs_context(inode); error: return err; diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c index db832cc931c8..f120bcb8bf73 100644 --- a/fs/afs/dynroot.c +++ b/fs/afs/dynroot.c @@ -76,6 +76,7 @@ struct inode *afs_iget_pseudo_dir(struct super_block *sb, bool root) /* there shouldn't be an existing inode */ BUG_ON(!(inode->i_state & I_NEW)); + netfs_i_context_init(inode, NULL); inode->i_size = 0; inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO; if (root) { diff --git a/fs/afs/file.c b/fs/afs/file.c index 0f9fdb284a20..26292a110a8f 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -19,13 +19,11 @@ #include "internal.h" static int afs_file_mmap(struct file *file, struct vm_area_struct *vma); -static int afs_readpage(struct file *file, struct page *page); static int afs_symlink_readpage(struct file *file, struct page *page); static void afs_invalidate_folio(struct folio *folio, size_t offset, size_t length); static int afs_releasepage(struct page *page, gfp_t gfp_flags); -static void afs_readahead(struct readahead_control *ractl); static ssize_t afs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter); static void afs_vm_open(struct vm_area_struct *area); static void afs_vm_close(struct vm_area_struct *area); @@ -52,8 +50,8 @@ const struct inode_operations afs_file_inode_operations = { }; const struct address_space_operations afs_file_aops = { - .readpage = afs_readpage, - .readahead = afs_readahead, + .readpage = netfs_readpage, + .readahead = netfs_readahead, .dirty_folio = afs_dirty_folio, .launder_folio = afs_launder_folio, .releasepage = afs_releasepage, @@ -240,7 +238,7 @@ void afs_put_read(struct afs_read *req) static void afs_fetch_data_notify(struct afs_operation *op) { struct afs_read *req = op->fetch.req; - struct netfs_read_subrequest *subreq = req->subreq; + struct netfs_io_subrequest *subreq = req->subreq; int error = op->error; if (error == -ECONNABORTED) @@ -310,7 +308,7 @@ int afs_fetch_data(struct afs_vnode *vnode, struct afs_read *req) return afs_do_sync_operation(op); } -static void afs_req_issue_op(struct netfs_read_subrequest *subreq) +static void afs_issue_read(struct netfs_io_subrequest *subreq) { struct afs_vnode *vnode = AFS_FS_I(subreq->rreq->inode); struct afs_read *fsreq; @@ -359,19 +357,13 @@ static int afs_symlink_readpage(struct file *file, struct page *page) return ret; } -static void afs_init_rreq(struct netfs_read_request *rreq, struct file *file) +static int afs_init_request(struct netfs_io_request *rreq, struct file *file) { rreq->netfs_priv = key_get(afs_file_key(file)); + return 0; } -static bool afs_is_cache_enabled(struct inode *inode) -{ - struct fscache_cookie *cookie = afs_vnode_cache(AFS_FS_I(inode)); - - return fscache_cookie_enabled(cookie) && cookie->cache_priv; -} - -static int afs_begin_cache_operation(struct netfs_read_request *rreq) +static int afs_begin_cache_operation(struct netfs_io_request *rreq) { #ifdef CONFIG_AFS_FSCACHE struct afs_vnode *vnode = AFS_FS_I(rreq->inode); @@ -396,27 +388,14 @@ static void afs_priv_cleanup(struct address_space *mapping, void *netfs_priv) key_put(netfs_priv); } -const struct netfs_read_request_ops afs_req_ops = { - .init_rreq = afs_init_rreq, - .is_cache_enabled = afs_is_cache_enabled, +const struct netfs_request_ops afs_req_ops = { + .init_request = afs_init_request, .begin_cache_operation = afs_begin_cache_operation, .check_write_begin = afs_check_write_begin, - .issue_op = afs_req_issue_op, + .issue_read = afs_issue_read, .cleanup = afs_priv_cleanup, }; -static int afs_readpage(struct file *file, struct page *page) -{ - struct folio *folio = page_folio(page); - - return netfs_readpage(file, folio, &afs_req_ops, NULL); -} - -static void afs_readahead(struct readahead_control *ractl) -{ - netfs_readahead(ractl, &afs_req_ops, NULL); -} - int afs_write_inode(struct inode *inode, struct writeback_control *wbc) { fscache_unpin_writeback(wbc, afs_vnode_cache(AFS_FS_I(inode))); diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 5964f8aee090..2fe402483ad5 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -54,6 +54,14 @@ static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *paren } /* + * Set parameters for the netfs library + */ +static void afs_set_netfs_context(struct afs_vnode *vnode) +{ + netfs_i_context_init(&vnode->vfs_inode, &afs_req_ops); +} + +/* * Initialise an inode from the vnode status. */ static int afs_inode_init_from_status(struct afs_operation *op, @@ -128,6 +136,7 @@ static int afs_inode_init_from_status(struct afs_operation *op, } afs_set_i_size(vnode, status->size); + afs_set_netfs_context(vnode); vnode->invalid_before = status->data_version; inode_set_iversion_raw(&vnode->vfs_inode, status->data_version); @@ -237,6 +246,7 @@ static void afs_apply_status(struct afs_operation *op, * idea of what the size should be that's not the same as * what's on the server. */ + vnode->netfs_ctx.remote_i_size = status->size; if (change_size) { afs_set_i_size(vnode, status->size); inode->i_ctime = t; @@ -420,7 +430,7 @@ static void afs_get_inode_cache(struct afs_vnode *vnode) struct afs_vnode_cache_aux aux; if (vnode->status.type != AFS_FTYPE_FILE) { - vnode->cache = NULL; + vnode->netfs_ctx.cache = NULL; return; } @@ -430,12 +440,14 @@ static void afs_get_inode_cache(struct afs_vnode *vnode) key.vnode_id_ext[1] = htonl(vnode->fid.vnode_hi); afs_set_cache_aux(vnode, &aux); - vnode->cache = fscache_acquire_cookie( - vnode->volume->cache, - vnode->status.type == AFS_FTYPE_FILE ? 0 : FSCACHE_ADV_SINGLE_CHUNK, - &key, sizeof(key), - &aux, sizeof(aux), - vnode->status.size); + afs_vnode_set_cache(vnode, + fscache_acquire_cookie( + vnode->volume->cache, + vnode->status.type == AFS_FTYPE_FILE ? + 0 : FSCACHE_ADV_SINGLE_CHUNK, + &key, sizeof(key), + &aux, sizeof(aux), + vnode->status.size)); #endif } @@ -528,6 +540,7 @@ struct inode *afs_root_iget(struct super_block *sb, struct key *key) vnode = AFS_FS_I(inode); vnode->cb_v_break = as->volume->cb_v_break, + afs_set_netfs_context(vnode); op = afs_alloc_operation(key, as->volume); if (IS_ERR(op)) { @@ -786,11 +799,8 @@ void afs_evict_inode(struct inode *inode) afs_put_wb_key(wbk); } -#ifdef CONFIG_AFS_FSCACHE - fscache_relinquish_cookie(vnode->cache, + fscache_relinquish_cookie(afs_vnode_cache(vnode), test_bit(AFS_VNODE_DELETED, &vnode->flags)); - vnode->cache = NULL; -#endif afs_prune_wb_keys(vnode); afs_put_permits(rcu_access_pointer(vnode->permit_cache)); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index dc5032e10244..7b7ef945dc78 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -207,7 +207,7 @@ struct afs_read { loff_t file_size; /* File size returned by server */ struct key *key; /* The key to use to reissue the read */ struct afs_vnode *vnode; /* The file being read into. */ - struct netfs_read_subrequest *subreq; /* Fscache helper read request this belongs to */ + struct netfs_io_subrequest *subreq; /* Fscache helper read request this belongs to */ afs_dataversion_t data_version; /* Version number returned by server */ refcount_t usage; unsigned int call_debug_id; @@ -619,15 +619,16 @@ enum afs_lock_state { * leak from one inode to another. */ struct afs_vnode { - struct inode vfs_inode; /* the VFS's inode record */ + struct { + /* These must be contiguous */ + struct inode vfs_inode; /* the VFS's inode record */ + struct netfs_i_context netfs_ctx; /* Netfslib context */ + }; struct afs_volume *volume; /* volume on which vnode resides */ struct afs_fid fid; /* the file identifier for this inode */ struct afs_file_status status; /* AFS status info for this file */ afs_dataversion_t invalid_before; /* Child dentries are invalid before this */ -#ifdef CONFIG_AFS_FSCACHE - struct fscache_cookie *cache; /* caching cookie */ -#endif struct afs_permits __rcu *permit_cache; /* cache of permits so far obtained */ struct mutex io_lock; /* Lock for serialising I/O on this mutex */ struct rw_semaphore validate_lock; /* lock for validating this vnode */ @@ -674,12 +675,20 @@ struct afs_vnode { static inline struct fscache_cookie *afs_vnode_cache(struct afs_vnode *vnode) { #ifdef CONFIG_AFS_FSCACHE - return vnode->cache; + return netfs_i_cookie(&vnode->vfs_inode); #else return NULL; #endif } +static inline void afs_vnode_set_cache(struct afs_vnode *vnode, + struct fscache_cookie *cookie) +{ +#ifdef CONFIG_AFS_FSCACHE + vnode->netfs_ctx.cache = cookie; +#endif +} + /* * cached security record for one user's attempt to access a vnode */ @@ -1063,7 +1072,7 @@ extern const struct address_space_operations afs_file_aops; extern const struct address_space_operations afs_symlink_aops; extern const struct inode_operations afs_file_inode_operations; extern const struct file_operations afs_file_operations; -extern const struct netfs_read_request_ops afs_req_ops; +extern const struct netfs_request_ops afs_req_ops; extern int afs_cache_wb_key(struct afs_vnode *, struct afs_file *); extern void afs_put_wb_key(struct afs_wb_key *); diff --git a/fs/afs/super.c b/fs/afs/super.c index 7592c0f469f1..1fea195b0b27 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -688,13 +688,11 @@ static struct inode *afs_alloc_inode(struct super_block *sb) /* Reset anything that shouldn't leak from one inode to the next. */ memset(&vnode->fid, 0, sizeof(vnode->fid)); memset(&vnode->status, 0, sizeof(vnode->status)); + afs_vnode_set_cache(vnode, NULL); vnode->volume = NULL; vnode->lock_key = NULL; vnode->permit_cache = NULL; -#ifdef CONFIG_AFS_FSCACHE - vnode->cache = NULL; -#endif vnode->flags = 1 << AFS_VNODE_UNSET; vnode->lock_state = AFS_VNODE_LOCK_NONE; diff --git a/fs/afs/write.c b/fs/afs/write.c index e1c17081d18e..6bcf1475511b 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -60,8 +60,7 @@ int afs_write_begin(struct file *file, struct address_space *mapping, * file. We need to do this before we get a lock on the page in case * there's more than one writer competing for the same cache block. */ - ret = netfs_write_begin(file, mapping, pos, len, flags, &folio, fsdata, - &afs_req_ops, NULL); + ret = netfs_write_begin(file, mapping, pos, len, flags, &folio, fsdata); if (ret < 0) return ret; @@ -355,9 +354,10 @@ static const struct afs_operation_ops afs_store_data_operation = { static int afs_store_data(struct afs_vnode *vnode, struct iov_iter *iter, loff_t pos, bool laundering) { + struct netfs_i_context *ictx = &vnode->netfs_ctx; struct afs_operation *op; struct afs_wb_key *wbk = NULL; - loff_t size = iov_iter_count(iter), i_size; + loff_t size = iov_iter_count(iter); int ret = -ENOKEY; _enter("%s{%llx:%llu.%u},%llx,%llx", @@ -379,15 +379,13 @@ static int afs_store_data(struct afs_vnode *vnode, struct iov_iter *iter, loff_t return -ENOMEM; } - i_size = i_size_read(&vnode->vfs_inode); - afs_op_set_vnode(op, 0, vnode); op->file[0].dv_delta = 1; op->file[0].modification = true; op->store.write_iter = iter; op->store.pos = pos; op->store.size = size; - op->store.i_size = max(pos + size, i_size); + op->store.i_size = max(pos + size, ictx->remote_i_size); op->store.laundering = laundering; op->mtime = vnode->vfs_inode.i_mtime; op->flags |= AFS_OPERATION_UNINTR; @@ -1552,7 +1552,6 @@ static int aio_read(struct kiocb *req, const struct iocb *iocb, file = req->ki_filp; if (unlikely(!(file->f_mode & FMODE_READ))) return -EBADF; - ret = -EINVAL; if (unlikely(!file->f_op->read_iter)) return -EINVAL; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 0399cf8e3c32..151e9da5da2d 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -118,7 +118,7 @@ struct btrfs_bio_ctrl { */ struct extent_changeset { /* How many bytes are set/cleared in this operation */ - unsigned int bytes_changed; + u64 bytes_changed; /* Changed ranges */ struct ulist range_changed; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 9f455c96c974..380054c94e4b 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -2957,8 +2957,9 @@ out: return ret; } -static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) +static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len) { + struct inode *inode = file_inode(file); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_root *root = BTRFS_I(inode)->root; struct extent_state *cached_state = NULL; @@ -2990,6 +2991,10 @@ static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) goto out_only_mutex; } + ret = file_modified(file); + if (ret) + goto out_only_mutex; + lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode))); lockend = round_down(offset + len, btrfs_inode_sectorsize(BTRFS_I(inode))) - 1; @@ -3430,7 +3435,7 @@ static long btrfs_fallocate(struct file *file, int mode, return -EOPNOTSUPP; if (mode & FALLOC_FL_PUNCH_HOLE) - return btrfs_punch_hole(inode, offset, len); + return btrfs_punch_hole(file, offset, len); /* * Only trigger disk allocation, don't trigger qgroup reserve @@ -3452,6 +3457,10 @@ static long btrfs_fallocate(struct file *file, int mode, goto out; } + ret = file_modified(file); + if (ret) + goto out; + /* * TODO: Move these two operations after we have checked * accurate reserved space, or fallocate can still fail but diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index aa0a60ee26cb..17d5557f98ec 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1128,7 +1128,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode, int ret = 0; if (btrfs_is_free_space_inode(inode)) { - WARN_ON_ONCE(1); ret = -EINVAL; goto out_unlock; } @@ -4488,6 +4487,13 @@ int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry) dest->root_key.objectid); return -EPERM; } + if (atomic_read(&dest->nr_swapfiles)) { + spin_unlock(&dest->root_item_lock); + btrfs_warn(fs_info, + "attempt to delete subvolume %llu with active swapfile", + root->root_key.objectid); + return -EPERM; + } root_flags = btrfs_root_flags(&dest->root_item); btrfs_set_root_flags(&dest->root_item, root_flags | BTRFS_ROOT_SUBVOL_DEAD); @@ -8296,7 +8302,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * cover the full folio, like invalidating the last folio, we're * still safe to wait for ordered extent to finish. */ - if (!(offset == 0 && length == PAGE_SIZE)) { + if (!(offset == 0 && length == folio_size(folio))) { btrfs_releasepage(&folio->page, GFP_NOFS); return; } @@ -11107,8 +11113,23 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, * set. We use this counter to prevent snapshots. We must increment it * before walking the extents because we don't want a concurrent * snapshot to run after we've already checked the extents. + * + * It is possible that subvolume is marked for deletion but still not + * removed yet. To prevent this race, we check the root status before + * activating the swapfile. */ + spin_lock(&root->root_item_lock); + if (btrfs_root_dead(root)) { + spin_unlock(&root->root_item_lock); + + btrfs_exclop_finish(fs_info); + btrfs_warn(fs_info, + "cannot activate swapfile because subvolume %llu is being deleted", + root->root_key.objectid); + return -EPERM; + } atomic_inc(&root->nr_swapfiles); + spin_unlock(&root->root_item_lock); isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 238cee5b5254..f46e71061942 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -1239,7 +1239,7 @@ static u32 get_extent_max_capacity(const struct extent_map *em) } static bool defrag_check_next_extent(struct inode *inode, struct extent_map *em, - bool locked) + u32 extent_thresh, u64 newer_than, bool locked) { struct extent_map *next; bool ret = false; @@ -1249,11 +1249,12 @@ static bool defrag_check_next_extent(struct inode *inode, struct extent_map *em, return false; /* - * We want to check if the next extent can be merged with the current - * one, which can be an extent created in a past generation, so we pass - * a minimum generation of 0 to defrag_lookup_extent(). + * Here we need to pass @newer_then when checking the next extent, or + * we will hit a case we mark current extent for defrag, but the next + * one will not be a target. + * This will just cause extra IO without really reducing the fragments. */ - next = defrag_lookup_extent(inode, em->start + em->len, 0, locked); + next = defrag_lookup_extent(inode, em->start + em->len, newer_than, locked); /* No more em or hole */ if (!next || next->block_start >= EXTENT_MAP_LAST_BYTE) goto out; @@ -1265,6 +1266,13 @@ static bool defrag_check_next_extent(struct inode *inode, struct extent_map *em, */ if (next->len >= get_extent_max_capacity(em)) goto out; + /* Skip older extent */ + if (next->generation < newer_than) + goto out; + /* Also check extent size */ + if (next->len >= extent_thresh) + goto out; + ret = true; out: free_extent_map(next); @@ -1470,7 +1478,7 @@ static int defrag_collect_targets(struct btrfs_inode *inode, goto next; next_mergeable = defrag_check_next_extent(&inode->vfs_inode, em, - locked); + extent_thresh, newer_than, locked); if (!next_mergeable) { struct defrag_target_range *last; diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 04a88bfe4fcf..998e3f180d90 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -645,7 +645,7 @@ static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len, int ret; /* - * Lock destination range to serialize with concurrent readpages() and + * Lock destination range to serialize with concurrent readahead() and * source range to serialize with relocation. */ btrfs_double_extent_lock(src, loff, dst, dst_loff, len); @@ -739,7 +739,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, } /* - * Lock destination range to serialize with concurrent readpages() and + * Lock destination range to serialize with concurrent readahead() and * source range to serialize with relocation. */ btrfs_double_extent_lock(src, off, inode, destoff, len); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 1be7cb2f955f..2cfbc74a3b4e 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1896,23 +1896,18 @@ static void update_dev_time(const char *device_path) path_put(&path); } -static int btrfs_rm_dev_item(struct btrfs_device *device) +static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans, + struct btrfs_device *device) { struct btrfs_root *root = device->fs_info->chunk_root; int ret; struct btrfs_path *path; struct btrfs_key key; - struct btrfs_trans_handle *trans; path = btrfs_alloc_path(); if (!path) return -ENOMEM; - trans = btrfs_start_transaction(root, 0); - if (IS_ERR(trans)) { - btrfs_free_path(path); - return PTR_ERR(trans); - } key.objectid = BTRFS_DEV_ITEMS_OBJECTID; key.type = BTRFS_DEV_ITEM_KEY; key.offset = device->devid; @@ -1923,21 +1918,12 @@ static int btrfs_rm_dev_item(struct btrfs_device *device) if (ret) { if (ret > 0) ret = -ENOENT; - btrfs_abort_transaction(trans, ret); - btrfs_end_transaction(trans); goto out; } ret = btrfs_del_item(trans, root, path); - if (ret) { - btrfs_abort_transaction(trans, ret); - btrfs_end_transaction(trans); - } - out: btrfs_free_path(path); - if (!ret) - ret = btrfs_commit_transaction(trans); return ret; } @@ -2078,6 +2064,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, struct btrfs_dev_lookup_args *args, struct block_device **bdev, fmode_t *mode) { + struct btrfs_trans_handle *trans; struct btrfs_device *device; struct btrfs_fs_devices *cur_devices; struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; @@ -2098,7 +2085,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1); if (ret) - goto out; + return ret; device = btrfs_find_device(fs_info->fs_devices, args); if (!device) { @@ -2106,27 +2093,22 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND; else ret = -ENOENT; - goto out; + return ret; } if (btrfs_pinned_by_swapfile(fs_info, device)) { btrfs_warn_in_rcu(fs_info, "cannot remove device %s (devid %llu) due to active swapfile", rcu_str_deref(device->name), device->devid); - ret = -ETXTBSY; - goto out; + return -ETXTBSY; } - if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { - ret = BTRFS_ERROR_DEV_TGT_REPLACE; - goto out; - } + if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + return BTRFS_ERROR_DEV_TGT_REPLACE; if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && - fs_info->fs_devices->rw_devices == 1) { - ret = BTRFS_ERROR_DEV_ONLY_WRITABLE; - goto out; - } + fs_info->fs_devices->rw_devices == 1) + return BTRFS_ERROR_DEV_ONLY_WRITABLE; if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { mutex_lock(&fs_info->chunk_mutex); @@ -2139,14 +2121,22 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, if (ret) goto error_undo; - /* - * TODO: the superblock still includes this device in its num_devices - * counter although write_all_supers() is not locked out. This - * could give a filesystem state which requires a degraded mount. - */ - ret = btrfs_rm_dev_item(device); - if (ret) + trans = btrfs_start_transaction(fs_info->chunk_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); goto error_undo; + } + + ret = btrfs_rm_dev_item(trans, device); + if (ret) { + /* Any error in dev item removal is critical */ + btrfs_crit(fs_info, + "failed to remove device item for devid %llu: %d", + device->devid, ret); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); btrfs_scrub_cancel_dev(device); @@ -2229,7 +2219,8 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, free_fs_devices(cur_devices); } -out: + ret = btrfs_commit_transaction(trans); + return ret; error_undo: @@ -2240,7 +2231,7 @@ error_undo: device->fs_devices->rw_devices++; mutex_unlock(&fs_info->chunk_mutex); } - goto out; + return ret; } void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index b7b5fac1c779..1b1b310c3c51 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1801,7 +1801,6 @@ struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info, map = em->map_lookup; /* We only support single profile for now */ - ASSERT(map->num_stripes == 1); device = map->stripes[0].dev; free_extent_map(em); @@ -1976,18 +1975,16 @@ int btrfs_zone_finish(struct btrfs_block_group *block_group) bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) { + struct btrfs_fs_info *fs_info = fs_devices->fs_info; struct btrfs_device *device; bool ret = false; - if (!btrfs_is_zoned(fs_devices->fs_info)) + if (!btrfs_is_zoned(fs_info)) return true; - /* Non-single profiles are not supported yet */ - ASSERT((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0); - /* Check if there is a device with active zones left */ - mutex_lock(&fs_devices->device_list_mutex); - list_for_each_entry(device, &fs_devices->devices, dev_list) { + mutex_lock(&fs_info->chunk_mutex); + list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { struct btrfs_zoned_device_info *zinfo = device->zone_info; if (!device->bdev) @@ -1999,7 +1996,7 @@ bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) break; } } - mutex_unlock(&fs_devices->device_list_mutex); + mutex_unlock(&fs_info->chunk_mutex); return ret; } diff --git a/fs/buffer.c b/fs/buffer.c index d67fbe063a3a..2b5561ae5d0b 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2352,8 +2352,7 @@ int generic_cont_expand_simple(struct inode *inode, loff_t size) if (err) goto out; - err = pagecache_write_begin(NULL, mapping, size, 0, - AOP_FLAG_CONT_EXPAND, &page, &fsdata); + err = pagecache_write_begin(NULL, mapping, size, 0, 0, &page, &fsdata); if (err) goto out; diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index bc7c7a7d9260..9dc81e781f2b 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -380,18 +380,18 @@ presubmission_error: * Prepare a read operation, shortening it to a cached/uncached * boundary as appropriate. */ -static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subrequest *subreq, +static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq, loff_t i_size) { enum cachefiles_prepare_read_trace why; - struct netfs_read_request *rreq = subreq->rreq; + struct netfs_io_request *rreq = subreq->rreq; struct netfs_cache_resources *cres = &rreq->cache_resources; struct cachefiles_object *object; struct cachefiles_cache *cache; struct fscache_cookie *cookie = fscache_cres_cookie(cres); const struct cred *saved_cred; struct file *file = cachefiles_cres_file(cres); - enum netfs_read_source ret = NETFS_DOWNLOAD_FROM_SERVER; + enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER; loff_t off, to; ino_t ino = file ? file_inode(file)->i_ino : 0; @@ -404,7 +404,7 @@ static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subreque } if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) { - __set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); + __set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); why = cachefiles_trace_read_no_data; goto out_no_object; } @@ -473,7 +473,7 @@ static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subreque goto out; download_and_store: - __set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); + __set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); out: cachefiles_end_secure(cache, saved_cred); out_no_object: diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index c7a0ab0d298b..aa25bffd4823 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -182,7 +182,7 @@ static int ceph_releasepage(struct page *page, gfp_t gfp) return 1; } -static void ceph_netfs_expand_readahead(struct netfs_read_request *rreq) +static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) { struct inode *inode = rreq->inode; struct ceph_inode_info *ci = ceph_inode(inode); @@ -199,7 +199,7 @@ static void ceph_netfs_expand_readahead(struct netfs_read_request *rreq) rreq->len = roundup(rreq->len, lo->stripe_unit); } -static bool ceph_netfs_clamp_length(struct netfs_read_subrequest *subreq) +static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq) { struct inode *inode = subreq->rreq->inode; struct ceph_fs_client *fsc = ceph_inode_to_client(inode); @@ -218,7 +218,7 @@ static void finish_netfs_read(struct ceph_osd_request *req) { struct ceph_fs_client *fsc = ceph_inode_to_client(req->r_inode); struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0); - struct netfs_read_subrequest *subreq = req->r_priv; + struct netfs_io_subrequest *subreq = req->r_priv; int num_pages; int err = req->r_result; @@ -244,9 +244,9 @@ static void finish_netfs_read(struct ceph_osd_request *req) iput(req->r_inode); } -static bool ceph_netfs_issue_op_inline(struct netfs_read_subrequest *subreq) +static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) { - struct netfs_read_request *rreq = subreq->rreq; + struct netfs_io_request *rreq = subreq->rreq; struct inode *inode = rreq->inode; struct ceph_mds_reply_info_parsed *rinfo; struct ceph_mds_reply_info_in *iinfo; @@ -258,7 +258,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_read_subrequest *subreq) size_t len; __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); - __clear_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); + __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); if (subreq->start >= inode->i_size) goto out; @@ -297,9 +297,9 @@ out: return true; } -static void ceph_netfs_issue_op(struct netfs_read_subrequest *subreq) +static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) { - struct netfs_read_request *rreq = subreq->rreq; + struct netfs_io_request *rreq = subreq->rreq; struct inode *inode = rreq->inode; struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_client(inode); @@ -353,6 +353,45 @@ out: dout("%s: result %d\n", __func__, err); } +static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) +{ + struct inode *inode = rreq->inode; + int got = 0, want = CEPH_CAP_FILE_CACHE; + int ret = 0; + + if (rreq->origin != NETFS_READAHEAD) + return 0; + + if (file) { + struct ceph_rw_context *rw_ctx; + struct ceph_file_info *fi = file->private_data; + + rw_ctx = ceph_find_rw_context(fi); + if (rw_ctx) + return 0; + } + + /* + * readahead callers do not necessarily hold Fcb caps + * (e.g. fadvise, madvise). + */ + ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got); + if (ret < 0) { + dout("start_read %p, error getting cap\n", inode); + return ret; + } + + if (!(got & want)) { + dout("start_read %p, no cache cap\n", inode); + return -EACCES; + } + if (ret == 0) + return -EACCES; + + rreq->netfs_priv = (void *)(uintptr_t)got; + return 0; +} + static void ceph_readahead_cleanup(struct address_space *mapping, void *priv) { struct inode *inode = mapping->host; @@ -363,64 +402,16 @@ static void ceph_readahead_cleanup(struct address_space *mapping, void *priv) ceph_put_cap_refs(ci, got); } -static const struct netfs_read_request_ops ceph_netfs_read_ops = { - .is_cache_enabled = ceph_is_cache_enabled, +const struct netfs_request_ops ceph_netfs_ops = { + .init_request = ceph_init_request, .begin_cache_operation = ceph_begin_cache_operation, - .issue_op = ceph_netfs_issue_op, + .issue_read = ceph_netfs_issue_read, .expand_readahead = ceph_netfs_expand_readahead, .clamp_length = ceph_netfs_clamp_length, .check_write_begin = ceph_netfs_check_write_begin, .cleanup = ceph_readahead_cleanup, }; -/* read a single page, without unlocking it. */ -static int ceph_readpage(struct file *file, struct page *subpage) -{ - struct folio *folio = page_folio(subpage); - struct inode *inode = file_inode(file); - struct ceph_inode_info *ci = ceph_inode(inode); - struct ceph_vino vino = ceph_vino(inode); - size_t len = folio_size(folio); - u64 off = folio_file_pos(folio); - - dout("readpage ino %llx.%llx file %p off %llu len %zu folio %p index %lu\n inline %d", - vino.ino, vino.snap, file, off, len, folio, folio_index(folio), - ci->i_inline_version != CEPH_INLINE_NONE); - - return netfs_readpage(file, folio, &ceph_netfs_read_ops, NULL); -} - -static void ceph_readahead(struct readahead_control *ractl) -{ - struct inode *inode = file_inode(ractl->file); - struct ceph_file_info *fi = ractl->file->private_data; - struct ceph_rw_context *rw_ctx; - int got = 0; - int ret = 0; - - if (ceph_inode(inode)->i_inline_version != CEPH_INLINE_NONE) - return; - - rw_ctx = ceph_find_rw_context(fi); - if (!rw_ctx) { - /* - * readahead callers do not necessarily hold Fcb caps - * (e.g. fadvise, madvise). - */ - int want = CEPH_CAP_FILE_CACHE; - - ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got); - if (ret < 0) - dout("start_read %p, error getting cap\n", inode); - else if (!(got & want)) - dout("start_read %p, no cache cap\n", inode); - - if (ret <= 0) - return; - } - netfs_readahead(ractl, &ceph_netfs_read_ops, (void *)(uintptr_t)got); -} - #ifdef CONFIG_CEPH_FSCACHE static void ceph_set_page_fscache(struct page *page) { @@ -1327,8 +1318,7 @@ static int ceph_write_begin(struct file *file, struct address_space *mapping, struct folio *folio = NULL; int r; - r = netfs_write_begin(file, inode->i_mapping, pos, len, 0, &folio, NULL, - &ceph_netfs_read_ops, NULL); + r = netfs_write_begin(file, inode->i_mapping, pos, len, 0, &folio, NULL); if (r == 0) folio_wait_fscache(folio); if (r < 0) { @@ -1382,8 +1372,8 @@ out: } const struct address_space_operations ceph_aops = { - .readpage = ceph_readpage, - .readahead = ceph_readahead, + .readpage = netfs_readpage, + .readahead = netfs_readahead, .writepage = ceph_writepage, .writepages = ceph_writepages_start, .write_begin = ceph_write_begin, diff --git a/fs/ceph/cache.c b/fs/ceph/cache.c index 7d22850623ef..ddea99922073 100644 --- a/fs/ceph/cache.c +++ b/fs/ceph/cache.c @@ -29,26 +29,25 @@ void ceph_fscache_register_inode_cookie(struct inode *inode) if (!(inode->i_state & I_NEW)) return; - WARN_ON_ONCE(ci->fscache); + WARN_ON_ONCE(ci->netfs_ctx.cache); - ci->fscache = fscache_acquire_cookie(fsc->fscache, 0, - &ci->i_vino, sizeof(ci->i_vino), - &ci->i_version, sizeof(ci->i_version), - i_size_read(inode)); + ci->netfs_ctx.cache = + fscache_acquire_cookie(fsc->fscache, 0, + &ci->i_vino, sizeof(ci->i_vino), + &ci->i_version, sizeof(ci->i_version), + i_size_read(inode)); } -void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) +void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info *ci) { - struct fscache_cookie *cookie = ci->fscache; - - fscache_relinquish_cookie(cookie, false); + fscache_relinquish_cookie(ceph_fscache_cookie(ci), false); } void ceph_fscache_use_cookie(struct inode *inode, bool will_modify) { struct ceph_inode_info *ci = ceph_inode(inode); - fscache_use_cookie(ci->fscache, will_modify); + fscache_use_cookie(ceph_fscache_cookie(ci), will_modify); } void ceph_fscache_unuse_cookie(struct inode *inode, bool update) @@ -58,9 +57,10 @@ void ceph_fscache_unuse_cookie(struct inode *inode, bool update) if (update) { loff_t i_size = i_size_read(inode); - fscache_unuse_cookie(ci->fscache, &ci->i_version, &i_size); + fscache_unuse_cookie(ceph_fscache_cookie(ci), + &ci->i_version, &i_size); } else { - fscache_unuse_cookie(ci->fscache, NULL, NULL); + fscache_unuse_cookie(ceph_fscache_cookie(ci), NULL, NULL); } } @@ -69,14 +69,14 @@ void ceph_fscache_update(struct inode *inode) struct ceph_inode_info *ci = ceph_inode(inode); loff_t i_size = i_size_read(inode); - fscache_update_cookie(ci->fscache, &ci->i_version, &i_size); + fscache_update_cookie(ceph_fscache_cookie(ci), &ci->i_version, &i_size); } void ceph_fscache_invalidate(struct inode *inode, bool dio_write) { struct ceph_inode_info *ci = ceph_inode(inode); - fscache_invalidate(ceph_inode(inode)->fscache, + fscache_invalidate(ceph_fscache_cookie(ci), &ci->i_version, i_size_read(inode), dio_write ? FSCACHE_INVAL_DIO_WRITE : 0); } diff --git a/fs/ceph/cache.h b/fs/ceph/cache.h index b90f3016994d..7255b790a4c1 100644 --- a/fs/ceph/cache.h +++ b/fs/ceph/cache.h @@ -26,14 +26,9 @@ void ceph_fscache_unuse_cookie(struct inode *inode, bool update); void ceph_fscache_update(struct inode *inode); void ceph_fscache_invalidate(struct inode *inode, bool dio_write); -static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci) -{ - ci->fscache = NULL; -} - static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info *ci) { - return ci->fscache; + return netfs_i_cookie(&ci->vfs_inode); } static inline void ceph_fscache_resize(struct inode *inode, loff_t to) @@ -62,7 +57,7 @@ static inline int ceph_fscache_dirty_folio(struct address_space *mapping, return fscache_dirty_folio(mapping, folio, ceph_fscache_cookie(ci)); } -static inline int ceph_begin_cache_operation(struct netfs_read_request *rreq) +static inline int ceph_begin_cache_operation(struct netfs_io_request *rreq) { struct fscache_cookie *cookie = ceph_fscache_cookie(ceph_inode(rreq->inode)); @@ -91,10 +86,6 @@ static inline void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) { } -static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci) -{ -} - static inline void ceph_fscache_register_inode_cookie(struct inode *inode) { } @@ -144,7 +135,7 @@ static inline bool ceph_is_cache_enabled(struct inode *inode) return false; } -static inline int ceph_begin_cache_operation(struct netfs_read_request *rreq) +static inline int ceph_begin_cache_operation(struct netfs_io_request *rreq) { return -ENOBUFS; } diff --git a/fs/ceph/file.c b/fs/ceph/file.c index feb75eb1cd82..6c9e837aa1d3 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -1869,7 +1869,7 @@ retry_snap: * are pending vmtruncate. So write and vmtruncate * can not run at the same time */ - written = generic_perform_write(file, from, pos); + written = generic_perform_write(iocb, from); if (likely(written >= 0)) iocb->ki_pos = pos + written; ceph_end_io_write(inode); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index d80911dc91c2..63113e2a4890 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -459,6 +459,9 @@ struct inode *ceph_alloc_inode(struct super_block *sb) dout("alloc_inode %p\n", &ci->vfs_inode); + /* Set parameters for the netfs library */ + netfs_i_context_init(&ci->vfs_inode, &ceph_netfs_ops); + spin_lock_init(&ci->i_ceph_lock); ci->i_version = 0; @@ -544,9 +547,6 @@ struct inode *ceph_alloc_inode(struct super_block *sb) INIT_WORK(&ci->i_work, ceph_inode_work); ci->i_work_mask = 0; memset(&ci->i_btime, '\0', sizeof(ci->i_btime)); - - ceph_fscache_inode_init(ci); - return &ci->vfs_inode; } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index a1ecc410a495..20ceab74e871 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -17,13 +17,11 @@ #include <linux/posix_acl.h> #include <linux/refcount.h> #include <linux/security.h> +#include <linux/netfs.h> +#include <linux/fscache.h> #include <linux/ceph/libceph.h> -#ifdef CONFIG_CEPH_FSCACHE -#include <linux/fscache.h> -#endif - /* large granularity for statfs utilization stats to facilitate * large volume sizes on 32-bit machines. */ #define CEPH_BLOCK_SHIFT 22 /* 4 MB */ @@ -318,6 +316,11 @@ struct ceph_inode_xattrs_info { * Ceph inode. */ struct ceph_inode_info { + struct { + /* These must be contiguous */ + struct inode vfs_inode; + struct netfs_i_context netfs_ctx; /* Netfslib context */ + }; struct ceph_vino i_vino; /* ceph ino + snap */ spinlock_t i_ceph_lock; @@ -428,11 +431,6 @@ struct ceph_inode_info { struct work_struct i_work; unsigned long i_work_mask; - -#ifdef CONFIG_CEPH_FSCACHE - struct fscache_cookie *fscache; -#endif - struct inode vfs_inode; /* at end */ }; static inline struct ceph_inode_info * @@ -1216,6 +1214,7 @@ extern void __ceph_touch_fmode(struct ceph_inode_info *ci, /* addr.c */ extern const struct address_space_operations ceph_aops; +extern const struct netfs_request_ops ceph_netfs_ops; extern int ceph_mmap(struct file *file, struct vm_area_struct *vma); extern int ceph_uninline_data(struct file *file); extern int ceph_pool_perm_check(struct inode *inode, int need); diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index ea00e1a91250..9d334816eac0 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -94,7 +94,7 @@ static void cifs_debug_tcon(struct seq_file *m, struct cifs_tcon *tcon) le32_to_cpu(tcon->fsDevInfo.DeviceCharacteristics), le32_to_cpu(tcon->fsAttrInfo.Attributes), le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength), - tcon->tidStatus); + tcon->status); if (dev_type == FILE_DEVICE_DISK) seq_puts(m, " type: DISK "); else if (dev_type == FILE_DEVICE_CD_ROM) diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index d1211ad4e85b..a47fa44b6d52 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -699,14 +699,14 @@ static void cifs_umount_begin(struct super_block *sb) tcon = cifs_sb_master_tcon(cifs_sb); spin_lock(&cifs_tcp_ses_lock); - if ((tcon->tc_count > 1) || (tcon->tidStatus == CifsExiting)) { + if ((tcon->tc_count > 1) || (tcon->status == TID_EXITING)) { /* we have other mounts to same share or we have already tried to force umount this and woken up all waiting network requests, nothing to do */ spin_unlock(&cifs_tcp_ses_lock); return; } else if (tcon->tc_count == 1) - tcon->tidStatus = CifsExiting; + tcon->status = TID_EXITING; spin_unlock(&cifs_tcp_ses_lock); /* cancel_brl_requests(tcon); */ /* BB mark all brl mids as exiting */ diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 48b343d03430..8de977c359b1 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -16,6 +16,7 @@ #include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/utsname.h> +#include <linux/netfs.h> #include "cifs_fs_sb.h" #include "cifsacl.h" #include <crypto/internal/hash.h> @@ -115,10 +116,18 @@ enum statusEnum { CifsInNegotiate, CifsNeedSessSetup, CifsInSessSetup, - CifsNeedTcon, - CifsInTcon, - CifsNeedFilesInvalidate, - CifsInFilesInvalidate +}; + +/* associated with each tree connection to the server */ +enum tid_status_enum { + TID_NEW = 0, + TID_GOOD, + TID_EXITING, + TID_NEED_RECON, + TID_NEED_TCON, + TID_IN_TCON, + TID_NEED_FILES_INVALIDATE, /* currently unused */ + TID_IN_FILES_INVALIDATE }; enum securityEnum { @@ -852,13 +861,7 @@ compare_mid(__u16 mid, const struct smb_hdr *smb) #define CIFS_MAX_RFC1002_WSIZE ((1<<17) - 1 - sizeof(WRITE_REQ) + 4) #define CIFS_MAX_RFC1002_RSIZE ((1<<17) - 1 - sizeof(READ_RSP) + 4) -/* - * The default wsize is 1M. find_get_pages seems to return a maximum of 256 - * pages in a single call. With PAGE_SIZE == 4k, this means we can fill - * a single wsize request with a single call. - */ #define CIFS_DEFAULT_IOSIZE (1024 * 1024) -#define SMB3_DEFAULT_IOSIZE (4 * 1024 * 1024) /* * Windows only supports a max of 60kb reads and 65535 byte writes. Default to @@ -1038,7 +1041,7 @@ struct cifs_tcon { char *password; /* for share-level security */ __u32 tid; /* The 4 byte tree id */ __u16 Flags; /* optional support bits */ - enum statusEnum tidStatus; + enum tid_status_enum status; atomic_t num_smbs_sent; union { struct { @@ -1402,6 +1405,11 @@ void cifsFileInfo_put(struct cifsFileInfo *cifs_file); */ struct cifsInodeInfo { + struct { + /* These must be contiguous */ + struct inode vfs_inode; /* the VFS's inode record */ + struct netfs_i_context netfs_ctx; /* Netfslib context */ + }; bool can_cache_brlcks; struct list_head llist; /* locks helb by this inode */ /* @@ -1432,10 +1440,6 @@ struct cifsInodeInfo { u64 uniqueid; /* server inode number */ u64 createtime; /* creation time on server */ __u8 lease_key[SMB2_LEASE_KEY_SIZE]; /* lease key for this inode */ -#ifdef CONFIG_CIFS_FSCACHE - struct fscache_cookie *fscache; -#endif - struct inode vfs_inode; struct list_head deferred_closes; /* list of deferred closes */ spinlock_t deferred_lock; /* protection on deferred list */ bool lease_granted; /* Flag to indicate whether lease or oplock is granted. */ diff --git a/fs/cifs/cifspdu.h b/fs/cifs/cifspdu.h index 68b9a436af4b..aeba371c4c70 100644 --- a/fs/cifs/cifspdu.h +++ b/fs/cifs/cifspdu.h @@ -123,18 +123,6 @@ */ #define CIFS_SESS_KEY_SIZE (16) -/* - * Size of the smb3 signing key - */ -#define SMB3_SIGN_KEY_SIZE (16) - -/* - * Size of the smb3 encryption/decryption key storage. - * This size is big enough to store any cipher key types. - */ -#define SMB3_ENC_DEC_KEY_SIZE (32) - -#define CIFS_CLIENT_CHALLENGE_SIZE (8) #define CIFS_SERVER_CHALLENGE_SIZE (8) #define CIFS_HMAC_MD5_HASH_SIZE (16) #define CIFS_CPHTXT_SIZE (16) @@ -1658,7 +1646,7 @@ struct smb_t2_rsp { #define SMB_FIND_FILE_ID_FULL_DIR_INFO 0x105 #define SMB_FIND_FILE_ID_BOTH_DIR_INFO 0x106 #define SMB_FIND_FILE_UNIX 0x202 -#define SMB_FIND_FILE_POSIX_INFO 0x064 +/* #define SMB_FIND_FILE_POSIX_INFO 0x064 */ typedef struct smb_com_transaction2_qpi_req { struct smb_hdr hdr; /* wct = 14+ */ diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c index 071e2f21a7db..47e927c4ff8d 100644 --- a/fs/cifs/cifssmb.c +++ b/fs/cifs/cifssmb.c @@ -75,12 +75,11 @@ cifs_mark_open_files_invalid(struct cifs_tcon *tcon) /* only send once per connect */ spin_lock(&cifs_tcp_ses_lock); - if (tcon->ses->status != CifsGood || - tcon->tidStatus != CifsNeedReconnect) { + if ((tcon->ses->status != CifsGood) || (tcon->status != TID_NEED_RECON)) { spin_unlock(&cifs_tcp_ses_lock); return; } - tcon->tidStatus = CifsInFilesInvalidate; + tcon->status = TID_IN_FILES_INVALIDATE; spin_unlock(&cifs_tcp_ses_lock); /* list all files open on tree connection and mark them invalid */ @@ -100,8 +99,8 @@ cifs_mark_open_files_invalid(struct cifs_tcon *tcon) mutex_unlock(&tcon->crfid.fid_mutex); spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsInFilesInvalidate) - tcon->tidStatus = CifsNeedTcon; + if (tcon->status == TID_IN_FILES_INVALIDATE) + tcon->status = TID_NEED_TCON; spin_unlock(&cifs_tcp_ses_lock); /* @@ -136,7 +135,7 @@ cifs_reconnect_tcon(struct cifs_tcon *tcon, int smb_command) * have tcon) are allowed as we start force umount */ spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsExiting) { + if (tcon->status == TID_EXITING) { if (smb_command != SMB_COM_WRITE_ANDX && smb_command != SMB_COM_OPEN_ANDX && smb_command != SMB_COM_TREE_DISCONNECT) { @@ -597,7 +596,7 @@ CIFSSMBNegotiate(const unsigned int xid, set_credits(server, server->maxReq); /* probably no need to store and check maxvcs */ server->maxBuf = le32_to_cpu(pSMBr->MaxBufferSize); - /* set up max_read for readpages check */ + /* set up max_read for readahead check */ server->max_read = server->maxBuf; server->max_rw = le32_to_cpu(pSMBr->MaxRawSize); cifs_dbg(NOISY, "Max buf = %d\n", ses->server->maxBuf); diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 9964c3634322..ee3b7c15e884 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -245,7 +245,7 @@ cifs_mark_tcp_ses_conns_for_reconnect(struct TCP_Server_Info *server, list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { tcon->need_reconnect = true; - tcon->tidStatus = CifsNeedReconnect; + tcon->status = TID_NEED_RECON; } if (ses->tcon_ipc) ses->tcon_ipc->need_reconnect = true; @@ -2207,7 +2207,7 @@ get_ses_fail: static int match_tcon(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) { - if (tcon->tidStatus == CifsExiting) + if (tcon->status == TID_EXITING) return 0; if (strncmp(tcon->treeName, ctx->UNC, MAX_TREE_SIZE)) return 0; @@ -3513,6 +3513,9 @@ static int connect_dfs_target(struct mount_ctx *mnt_ctx, const char *full_path, struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; char *oldmnt = cifs_sb->ctx->mount_options; + cifs_dbg(FYI, "%s: full_path=%s ref_path=%s target=%s\n", __func__, full_path, ref_path, + dfs_cache_get_tgt_name(tit)); + rc = dfs_cache_get_tgt_referral(ref_path, tit, &ref); if (rc) goto out; @@ -3611,13 +3614,18 @@ static int __follow_dfs_link(struct mount_ctx *mnt_ctx) if (rc) goto out; - /* Try all dfs link targets */ + /* Try all dfs link targets. If an I/O fails from currently connected DFS target with an + * error other than STATUS_PATH_NOT_COVERED (-EREMOTE), then retry it from other targets as + * specified in MS-DFSC "3.1.5.2 I/O Operation to Target Fails with an Error Other Than + * STATUS_PATH_NOT_COVERED." + */ for (rc = -ENOENT, tit = dfs_cache_get_tgt_iterator(&tl); tit; tit = dfs_cache_get_next_tgt(&tl, tit)) { rc = connect_dfs_target(mnt_ctx, full_path, mnt_ctx->leaf_fullpath + 1, tit); if (!rc) { rc = is_path_remote(mnt_ctx); - break; + if (!rc || rc == -EREMOTE) + break; } } @@ -3691,7 +3699,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) goto error; rc = is_path_remote(&mnt_ctx); - if (rc == -EREMOTE) + if (rc) rc = follow_dfs_link(&mnt_ctx); if (rc) goto error; @@ -4478,12 +4486,12 @@ int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const stru /* only send once per connect */ spin_lock(&cifs_tcp_ses_lock); if (tcon->ses->status != CifsGood || - (tcon->tidStatus != CifsNew && - tcon->tidStatus != CifsNeedTcon)) { + (tcon->status != TID_NEW && + tcon->status != TID_NEED_TCON)) { spin_unlock(&cifs_tcp_ses_lock); return 0; } - tcon->tidStatus = CifsInTcon; + tcon->status = TID_IN_TCON; spin_unlock(&cifs_tcp_ses_lock); tree = kzalloc(MAX_TREE_SIZE, GFP_KERNEL); @@ -4524,13 +4532,13 @@ out: if (rc) { spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsInTcon) - tcon->tidStatus = CifsNeedTcon; + if (tcon->status == TID_IN_TCON) + tcon->status = TID_NEED_TCON; spin_unlock(&cifs_tcp_ses_lock); } else { spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsInTcon) - tcon->tidStatus = CifsGood; + if (tcon->status == TID_IN_TCON) + tcon->status = TID_GOOD; spin_unlock(&cifs_tcp_ses_lock); tcon->need_reconnect = false; } @@ -4546,24 +4554,24 @@ int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const stru /* only send once per connect */ spin_lock(&cifs_tcp_ses_lock); if (tcon->ses->status != CifsGood || - (tcon->tidStatus != CifsNew && - tcon->tidStatus != CifsNeedTcon)) { + (tcon->status != TID_NEW && + tcon->status != TID_NEED_TCON)) { spin_unlock(&cifs_tcp_ses_lock); return 0; } - tcon->tidStatus = CifsInTcon; + tcon->status = TID_IN_TCON; spin_unlock(&cifs_tcp_ses_lock); rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, nlsc); if (rc) { spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsInTcon) - tcon->tidStatus = CifsNeedTcon; + if (tcon->status == TID_IN_TCON) + tcon->status = TID_NEED_TCON; spin_unlock(&cifs_tcp_ses_lock); } else { spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsInTcon) - tcon->tidStatus = CifsGood; + if (tcon->status == TID_IN_TCON) + tcon->status = TID_GOOD; spin_unlock(&cifs_tcp_ses_lock); tcon->need_reconnect = false; } diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 60f43bff7ccb..d511a78383c3 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -4210,13 +4210,19 @@ cifs_page_mkwrite(struct vm_fault *vmf) { struct page *page = vmf->page; + /* Wait for the page to be written to the cache before we allow it to + * be modified. We then assume the entire page will need writing back. + */ #ifdef CONFIG_CIFS_FSCACHE if (PageFsCache(page) && wait_on_page_fscache_killable(page) < 0) return VM_FAULT_RETRY; #endif - lock_page(page); + wait_on_page_writeback(page); + + if (lock_page_killable(page) < 0) + return VM_FAULT_RETRY; return VM_FAULT_LOCKED; } diff --git a/fs/cifs/fscache.c b/fs/cifs/fscache.c index 33af72e0ac0c..a638b29e9062 100644 --- a/fs/cifs/fscache.c +++ b/fs/cifs/fscache.c @@ -103,7 +103,7 @@ void cifs_fscache_get_inode_cookie(struct inode *inode) cifs_fscache_fill_coherency(&cifsi->vfs_inode, &cd); - cifsi->fscache = + cifsi->netfs_ctx.cache = fscache_acquire_cookie(tcon->fscache, 0, &cifsi->uniqueid, sizeof(cifsi->uniqueid), &cd, sizeof(cd), @@ -126,22 +126,15 @@ void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update) void cifs_fscache_release_inode_cookie(struct inode *inode) { struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct fscache_cookie *cookie = cifs_inode_cookie(inode); - if (cifsi->fscache) { - cifs_dbg(FYI, "%s: (0x%p)\n", __func__, cifsi->fscache); - fscache_relinquish_cookie(cifsi->fscache, false); - cifsi->fscache = NULL; + if (cookie) { + cifs_dbg(FYI, "%s: (0x%p)\n", __func__, cookie); + fscache_relinquish_cookie(cookie, false); + cifsi->netfs_ctx.cache = NULL; } } -static inline void fscache_end_operation(struct netfs_cache_resources *cres) -{ - const struct netfs_cache_ops *ops = fscache_operation_valid(cres); - - if (ops) - ops->end_operation(cres); -} - /* * Fallback page reading interface. */ diff --git a/fs/cifs/fscache.h b/fs/cifs/fscache.h index 55129908e2c1..52355c0912ae 100644 --- a/fs/cifs/fscache.h +++ b/fs/cifs/fscache.h @@ -61,7 +61,7 @@ void cifs_fscache_fill_coherency(struct inode *inode, static inline struct fscache_cookie *cifs_inode_cookie(struct inode *inode) { - return CIFS_I(inode)->fscache; + return netfs_i_cookie(inode); } static inline void cifs_invalidate_cache(struct inode *inode, unsigned int flags) diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 60d853c92f6a..2f9e7d2f81b6 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -49,7 +49,7 @@ static void cifs_set_ops(struct inode *inode) inode->i_fop = &cifs_file_ops; } - /* check if server can support readpages */ + /* check if server can support readahead */ if (cifs_sb_master_tcon(cifs_sb)->ses->server->max_read < PAGE_SIZE + MAX_CIFS_HDR_SIZE) inode->i_data.a_ops = &cifs_addr_ops_smallbuf; diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c index 56598f7dbe00..afaf59c22193 100644 --- a/fs/cifs/misc.c +++ b/fs/cifs/misc.c @@ -116,7 +116,7 @@ tconInfoAlloc(void) } atomic_inc(&tconInfoAllocCount); - ret_buf->tidStatus = CifsNew; + ret_buf->status = TID_NEW; ++ret_buf->tc_count; INIT_LIST_HEAD(&ret_buf->openFileList); INIT_LIST_HEAD(&ret_buf->tcon_list); diff --git a/fs/cifs/smb2glob.h b/fs/cifs/smb2glob.h index 4125fd113cfb..82e916ad167c 100644 --- a/fs/cifs/smb2glob.h +++ b/fs/cifs/smb2glob.h @@ -41,15 +41,4 @@ #define END_OF_CHAIN 4 #define RELATED_REQUEST 8 -#define SMB2_SIGNATURE_SIZE (16) -#define SMB2_NTLMV2_SESSKEY_SIZE (16) -#define SMB2_HMACSHA256_SIZE (32) -#define SMB2_CMACAES_SIZE (16) -#define SMB3_SIGNKEY_SIZE (16) -#define SMB3_GCM128_CRYPTKEY_SIZE (16) -#define SMB3_GCM256_CRYPTKEY_SIZE (32) - -/* Maximum buffer size value we can send with 1 credit */ -#define SMB2_MAX_BUFFER_SIZE 65536 - #endif /* _SMB2_GLOB_H */ diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index b25623e3fe3d..c653beb735b8 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -203,7 +203,7 @@ smb2_check_message(char *buf, unsigned int len, struct TCP_Server_Info *srvr) if (smb2_rsp_struct_sizes[command] != pdu->StructureSize2) { if (command != SMB2_OPLOCK_BREAK_HE && (shdr->Status == 0 || - pdu->StructureSize2 != SMB2_ERROR_STRUCTURE_SIZE2)) { + pdu->StructureSize2 != SMB2_ERROR_STRUCTURE_SIZE2_LE)) { /* error packets have 9 byte structure size */ cifs_dbg(VFS, "Invalid response size %u for command %d\n", le16_to_cpu(pdu->StructureSize2), command); @@ -303,7 +303,7 @@ smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr) /* error responses do not have data area */ if (shdr->Status && shdr->Status != STATUS_MORE_PROCESSING_REQUIRED && (((struct smb2_err_rsp *)shdr)->StructureSize) == - SMB2_ERROR_STRUCTURE_SIZE2) + SMB2_ERROR_STRUCTURE_SIZE2_LE) return NULL; /* @@ -478,11 +478,11 @@ smb2_get_lease_state(struct cifsInodeInfo *cinode) __le32 lease = 0; if (CIFS_CACHE_WRITE(cinode)) - lease |= SMB2_LEASE_WRITE_CACHING; + lease |= SMB2_LEASE_WRITE_CACHING_LE; if (CIFS_CACHE_HANDLE(cinode)) - lease |= SMB2_LEASE_HANDLE_CACHING; + lease |= SMB2_LEASE_HANDLE_CACHING_LE; if (CIFS_CACHE_READ(cinode)) - lease |= SMB2_LEASE_READ_CACHING; + lease |= SMB2_LEASE_READ_CACHING_LE; return lease; } @@ -832,8 +832,8 @@ smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *serve rc = __smb2_handle_cancelled_cmd(tcon, le16_to_cpu(hdr->Command), le64_to_cpu(hdr->MessageId), - le64_to_cpu(rsp->PersistentFileId), - le64_to_cpu(rsp->VolatileFileId)); + rsp->PersistentFileId, + rsp->VolatileFileId); if (rc) cifs_put_tcon(tcon); diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 891b11576e55..db23f5b404ba 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -897,8 +897,8 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, atomic_inc(&tcon->num_remote_opens); o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; - oparms.fid->persistent_fid = le64_to_cpu(o_rsp->PersistentFileId); - oparms.fid->volatile_fid = le64_to_cpu(o_rsp->VolatileFileId); + oparms.fid->persistent_fid = o_rsp->PersistentFileId; + oparms.fid->volatile_fid = o_rsp->VolatileFileId; #ifdef CONFIG_CIFS_DEBUG2 oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId); #endif /* CIFS_DEBUG2 */ @@ -1192,17 +1192,12 @@ smb2_query_eas(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb) { int rc; - __le16 *utf16_path; struct kvec rsp_iov = {NULL, 0}; int buftype = CIFS_NO_BUFFER; struct smb2_query_info_rsp *rsp; struct smb2_file_full_ea_info *info = NULL; - utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); - if (!utf16_path) - return -ENOMEM; - - rc = smb2_query_info_compound(xid, tcon, utf16_path, + rc = smb2_query_info_compound(xid, tcon, path, FILE_READ_EA, FILE_FULL_EA_INFORMATION, SMB2_O_INFO_FILE, @@ -1235,7 +1230,6 @@ smb2_query_eas(const unsigned int xid, struct cifs_tcon *tcon, le32_to_cpu(rsp->OutputBufferLength), ea_name); qeas_exit: - kfree(utf16_path); free_rsp_buf(buftype, rsp_iov.iov_base); return rc; } @@ -1295,7 +1289,7 @@ smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, * the new EA. If not we should not add it since we * would not be able to even read the EAs back. */ - rc = smb2_query_info_compound(xid, tcon, utf16_path, + rc = smb2_query_info_compound(xid, tcon, path, FILE_READ_EA, FILE_FULL_EA_INFORMATION, SMB2_O_INFO_FILE, @@ -1643,6 +1637,7 @@ smb2_ioctl_query_info(const unsigned int xid, unsigned int size[2]; void *data[2]; int create_options = is_dir ? CREATE_NOT_FILE : CREATE_NOT_DIR; + void (*free_req1_func)(struct smb_rqst *r); vars = kzalloc(sizeof(*vars), GFP_ATOMIC); if (vars == NULL) @@ -1652,27 +1647,29 @@ smb2_ioctl_query_info(const unsigned int xid, resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER; - if (copy_from_user(&qi, arg, sizeof(struct smb_query_info))) - goto e_fault; - + if (copy_from_user(&qi, arg, sizeof(struct smb_query_info))) { + rc = -EFAULT; + goto free_vars; + } if (qi.output_buffer_length > 1024) { - kfree(vars); - return -EINVAL; + rc = -EINVAL; + goto free_vars; } if (!ses || !server) { - kfree(vars); - return -EIO; + rc = -EIO; + goto free_vars; } if (smb3_encryption_required(tcon)) flags |= CIFS_TRANSFORM_REQ; - buffer = memdup_user(arg + sizeof(struct smb_query_info), - qi.output_buffer_length); - if (IS_ERR(buffer)) { - kfree(vars); - return PTR_ERR(buffer); + if (qi.output_buffer_length) { + buffer = memdup_user(arg + sizeof(struct smb_query_info), qi.output_buffer_length); + if (IS_ERR(buffer)) { + rc = PTR_ERR(buffer); + goto free_vars; + } } /* Open */ @@ -1710,45 +1707,45 @@ smb2_ioctl_query_info(const unsigned int xid, rc = SMB2_open_init(tcon, server, &rqst[0], &oplock, &oparms, path); if (rc) - goto iqinf_exit; + goto free_output_buffer; smb2_set_next_command(tcon, &rqst[0]); /* Query */ if (qi.flags & PASSTHRU_FSCTL) { /* Can eventually relax perm check since server enforces too */ - if (!capable(CAP_SYS_ADMIN)) + if (!capable(CAP_SYS_ADMIN)) { rc = -EPERM; - else { - rqst[1].rq_iov = &vars->io_iov[0]; - rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE; - - rc = SMB2_ioctl_init(tcon, server, - &rqst[1], - COMPOUND_FID, COMPOUND_FID, - qi.info_type, true, buffer, - qi.output_buffer_length, - CIFSMaxBufSize - - MAX_SMB2_CREATE_RESPONSE_SIZE - - MAX_SMB2_CLOSE_RESPONSE_SIZE); + goto free_open_req; } + rqst[1].rq_iov = &vars->io_iov[0]; + rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE; + + rc = SMB2_ioctl_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, + qi.info_type, true, buffer, qi.output_buffer_length, + CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE); + free_req1_func = SMB2_ioctl_free; } else if (qi.flags == PASSTHRU_SET_INFO) { /* Can eventually relax perm check since server enforces too */ - if (!capable(CAP_SYS_ADMIN)) + if (!capable(CAP_SYS_ADMIN)) { rc = -EPERM; - else { - rqst[1].rq_iov = &vars->si_iov[0]; - rqst[1].rq_nvec = 1; - - size[0] = 8; - data[0] = buffer; - - rc = SMB2_set_info_init(tcon, server, - &rqst[1], - COMPOUND_FID, COMPOUND_FID, - current->tgid, - FILE_END_OF_FILE_INFORMATION, - SMB2_O_INFO_FILE, 0, data, size); + goto free_open_req; + } + if (qi.output_buffer_length < 8) { + rc = -EINVAL; + goto free_open_req; } + rqst[1].rq_iov = &vars->si_iov[0]; + rqst[1].rq_nvec = 1; + + /* MS-FSCC 2.4.13 FileEndOfFileInformation */ + size[0] = 8; + data[0] = buffer; + + rc = SMB2_set_info_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, + current->tgid, FILE_END_OF_FILE_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + free_req1_func = SMB2_set_info_free; } else if (qi.flags == PASSTHRU_QUERY_INFO) { rqst[1].rq_iov = &vars->qi_iov[0]; rqst[1].rq_nvec = 1; @@ -1759,6 +1756,7 @@ smb2_ioctl_query_info(const unsigned int xid, qi.info_type, qi.additional_information, qi.input_buffer_length, qi.output_buffer_length, buffer); + free_req1_func = SMB2_query_info_free; } else { /* unknown flags */ cifs_tcon_dbg(VFS, "Invalid passthru query flags: 0x%x\n", qi.flags); @@ -1766,7 +1764,7 @@ smb2_ioctl_query_info(const unsigned int xid, } if (rc) - goto iqinf_exit; + goto free_open_req; smb2_set_next_command(tcon, &rqst[1]); smb2_set_related(&rqst[1]); @@ -1777,14 +1775,14 @@ smb2_ioctl_query_info(const unsigned int xid, rc = SMB2_close_init(tcon, server, &rqst[2], COMPOUND_FID, COMPOUND_FID, false); if (rc) - goto iqinf_exit; + goto free_req_1; smb2_set_related(&rqst[2]); rc = compound_send_recv(xid, ses, server, flags, 3, rqst, resp_buftype, rsp_iov); if (rc) - goto iqinf_exit; + goto out; /* No need to bump num_remote_opens since handle immediately closed */ if (qi.flags & PASSTHRU_FSCTL) { @@ -1794,18 +1792,22 @@ smb2_ioctl_query_info(const unsigned int xid, qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount); if (qi.input_buffer_length > 0 && le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length - > rsp_iov[1].iov_len) - goto e_fault; + > rsp_iov[1].iov_len) { + rc = -EFAULT; + goto out; + } if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, - sizeof(qi.input_buffer_length))) - goto e_fault; + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto out; + } if (copy_to_user((void __user *)pqi + sizeof(struct smb_query_info), (const void *)io_rsp + le32_to_cpu(io_rsp->OutputOffset), qi.input_buffer_length)) - goto e_fault; + rc = -EFAULT; } else { pqi = (struct smb_query_info __user *)arg; qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; @@ -1813,28 +1815,30 @@ smb2_ioctl_query_info(const unsigned int xid, qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength); if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, - sizeof(qi.input_buffer_length))) - goto e_fault; + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto out; + } if (copy_to_user(pqi + 1, qi_rsp->Buffer, qi.input_buffer_length)) - goto e_fault; + rc = -EFAULT; } - iqinf_exit: - cifs_small_buf_release(rqst[0].rq_iov[0].iov_base); - cifs_small_buf_release(rqst[1].rq_iov[0].iov_base); - cifs_small_buf_release(rqst[2].rq_iov[0].iov_base); +out: free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); - kfree(vars); + SMB2_close_free(&rqst[2]); +free_req_1: + free_req1_func(&rqst[1]); +free_open_req: + SMB2_open_free(&rqst[0]); +free_output_buffer: kfree(buffer); +free_vars: + kfree(vars); return rc; - -e_fault: - rc = -EFAULT; - goto iqinf_exit; } static ssize_t @@ -2407,8 +2411,8 @@ again: cifs_dbg(FYI, "query_dir_first: open failed rc=%d\n", rc); goto qdf_free; } - fid->persistent_fid = le64_to_cpu(op_rsp->PersistentFileId); - fid->volatile_fid = le64_to_cpu(op_rsp->VolatileFileId); + fid->persistent_fid = op_rsp->PersistentFileId; + fid->volatile_fid = op_rsp->VolatileFileId; /* Anything else than ENODATA means a genuine error */ if (rc && rc != -ENODATA) { @@ -2646,7 +2650,7 @@ smb2_set_next_command(struct cifs_tcon *tcon, struct smb_rqst *rqst) */ int smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, - __le16 *utf16_path, u32 desired_access, + const char *path, u32 desired_access, u32 class, u32 type, u32 output_len, struct kvec *rsp, int *buftype, struct cifs_sb_info *cifs_sb) @@ -2664,6 +2668,14 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_fid fid; int rc; + __le16 *utf16_path; + struct cached_fid *cfid = NULL; + + if (!path) + path = ""; + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; if (smb3_encryption_required(tcon)) flags |= CIFS_TRANSFORM_REQ; @@ -2672,6 +2684,8 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER; memset(rsp_iov, 0, sizeof(rsp_iov)); + rc = open_cached_dir(xid, tcon, path, cifs_sb, &cfid); + memset(&open_iov, 0, sizeof(open_iov)); rqst[0].rq_iov = open_iov; rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; @@ -2693,15 +2707,29 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, rqst[1].rq_iov = qi_iov; rqst[1].rq_nvec = 1; - rc = SMB2_query_info_init(tcon, server, - &rqst[1], COMPOUND_FID, COMPOUND_FID, - class, type, 0, - output_len, 0, - NULL); + if (cfid) { + rc = SMB2_query_info_init(tcon, server, + &rqst[1], + cfid->fid->persistent_fid, + cfid->fid->volatile_fid, + class, type, 0, + output_len, 0, + NULL); + } else { + rc = SMB2_query_info_init(tcon, server, + &rqst[1], + COMPOUND_FID, + COMPOUND_FID, + class, type, 0, + output_len, 0, + NULL); + } if (rc) goto qic_exit; - smb2_set_next_command(tcon, &rqst[1]); - smb2_set_related(&rqst[1]); + if (!cfid) { + smb2_set_next_command(tcon, &rqst[1]); + smb2_set_related(&rqst[1]); + } memset(&close_iov, 0, sizeof(close_iov)); rqst[2].rq_iov = close_iov; @@ -2713,9 +2741,15 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, goto qic_exit; smb2_set_related(&rqst[2]); - rc = compound_send_recv(xid, ses, server, - flags, 3, rqst, - resp_buftype, rsp_iov); + if (cfid) { + rc = compound_send_recv(xid, ses, server, + flags, 1, &rqst[1], + &resp_buftype[1], &rsp_iov[1]); + } else { + rc = compound_send_recv(xid, ses, server, + flags, 3, rqst, + resp_buftype, rsp_iov); + } if (rc) { free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); if (rc == -EREMCHG) { @@ -2729,11 +2763,14 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, *buftype = resp_buftype[1]; qic_exit: + kfree(utf16_path); SMB2_open_free(&rqst[0]); SMB2_query_info_free(&rqst[1]); SMB2_close_free(&rqst[2]); free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + if (cfid) + close_cached_dir(cfid); return rc; } @@ -2743,13 +2780,12 @@ smb2_queryfs(const unsigned int xid, struct cifs_tcon *tcon, { struct smb2_query_info_rsp *rsp; struct smb2_fs_full_size_info *info = NULL; - __le16 utf16_path = 0; /* Null - open root of share */ struct kvec rsp_iov = {NULL, 0}; int buftype = CIFS_NO_BUFFER; int rc; - rc = smb2_query_info_compound(xid, tcon, &utf16_path, + rc = smb2_query_info_compound(xid, tcon, "", FILE_READ_ATTRIBUTES, FS_FULL_SIZE_INFORMATION, SMB2_O_INFO_FILESYSTEM, @@ -4293,12 +4329,12 @@ static __le32 map_oplock_to_lease(u8 oplock) { if (oplock == SMB2_OPLOCK_LEVEL_EXCLUSIVE) - return SMB2_LEASE_WRITE_CACHING | SMB2_LEASE_READ_CACHING; + return SMB2_LEASE_WRITE_CACHING_LE | SMB2_LEASE_READ_CACHING_LE; else if (oplock == SMB2_OPLOCK_LEVEL_II) - return SMB2_LEASE_READ_CACHING; + return SMB2_LEASE_READ_CACHING_LE; else if (oplock == SMB2_OPLOCK_LEVEL_BATCH) - return SMB2_LEASE_HANDLE_CACHING | SMB2_LEASE_READ_CACHING | - SMB2_LEASE_WRITE_CACHING; + return SMB2_LEASE_HANDLE_CACHING_LE | SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE; return 0; } @@ -4360,7 +4396,7 @@ smb2_parse_lease_buf(void *buf, unsigned int *epoch, char *lease_key) struct create_lease *lc = (struct create_lease *)buf; *epoch = 0; /* not used */ - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS) + if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; return le32_to_cpu(lc->lcontext.LeaseState); } @@ -4371,7 +4407,7 @@ smb3_parse_lease_buf(void *buf, unsigned int *epoch, char *lease_key) struct create_lease_v2 *lc = (struct create_lease_v2 *)buf; *epoch = le16_to_cpu(lc->lcontext.Epoch); - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS) + if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; if (lease_key) memcpy(lease_key, &lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); @@ -5814,8 +5850,8 @@ struct smb_version_values smb20_values = { .protocol_id = SMB20_PROT_ID, .req_capabilities = 0, /* MBZ */ .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5835,8 +5871,8 @@ struct smb_version_values smb21_values = { .protocol_id = SMB21_PROT_ID, .req_capabilities = 0, /* MBZ on negotiate req until SMB3 dialect */ .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5856,8 +5892,8 @@ struct smb_version_values smb3any_values = { .protocol_id = SMB302_PROT_ID, /* doesn't matter, send protocol array */ .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5877,8 +5913,8 @@ struct smb_version_values smbdefault_values = { .protocol_id = SMB302_PROT_ID, /* doesn't matter, send protocol array */ .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5898,8 +5934,8 @@ struct smb_version_values smb30_values = { .protocol_id = SMB30_PROT_ID, .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5919,8 +5955,8 @@ struct smb_version_values smb302_values = { .protocol_id = SMB302_PROT_ID, .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, @@ -5940,8 +5976,8 @@ struct smb_version_values smb311_values = { .protocol_id = SMB311_PROT_ID, .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, .large_lock_type = 0, - .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE_LOCK, - .shared_lock_type = SMB2_LOCKFLAG_SHARED_LOCK, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, .header_size = sizeof(struct smb2_hdr), .header_preamble_size = 0, diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 7e7909b1ae11..1b7ad0c09566 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -163,7 +163,7 @@ smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon, return 0; spin_lock(&cifs_tcp_ses_lock); - if (tcon->tidStatus == CifsExiting) { + if (tcon->status == TID_EXITING) { /* * only tree disconnect, open, and write, * (and ulogoff which does not have tcon) @@ -2734,13 +2734,10 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, goto err_free_req; } - trace_smb3_posix_mkdir_done(xid, le64_to_cpu(rsp->PersistentFileId), - tcon->tid, - ses->Suid, CREATE_NOT_FILE, - FILE_WRITE_ATTRIBUTES); + trace_smb3_posix_mkdir_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, + CREATE_NOT_FILE, FILE_WRITE_ATTRIBUTES); - SMB2_close(xid, tcon, le64_to_cpu(rsp->PersistentFileId), - le64_to_cpu(rsp->VolatileFileId)); + SMB2_close(xid, tcon, rsp->PersistentFileId, rsp->VolatileFileId); /* Eventually save off posix specific response info and timestaps */ @@ -3009,14 +3006,12 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, } else if (rsp == NULL) /* unlikely to happen, but safer to check */ goto creat_exit; else - trace_smb3_open_done(xid, le64_to_cpu(rsp->PersistentFileId), - tcon->tid, - ses->Suid, oparms->create_options, - oparms->desired_access); + trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, + oparms->create_options, oparms->desired_access); atomic_inc(&tcon->num_remote_opens); - oparms->fid->persistent_fid = le64_to_cpu(rsp->PersistentFileId); - oparms->fid->volatile_fid = le64_to_cpu(rsp->VolatileFileId); + oparms->fid->persistent_fid = rsp->PersistentFileId; + oparms->fid->volatile_fid = rsp->VolatileFileId; oparms->fid->access = oparms->desired_access; #ifdef CONFIG_CIFS_DEBUG2 oparms->fid->mid = le64_to_cpu(rsp->hdr.MessageId); @@ -3313,8 +3308,8 @@ SMB2_close_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, if (rc) return rc; - req->PersistentFileId = cpu_to_le64(persistent_fid); - req->VolatileFileId = cpu_to_le64(volatile_fid); + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; if (query_attrs) req->Flags = SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB; else @@ -3677,8 +3672,8 @@ SMB2_notify_init(const unsigned int xid, struct smb_rqst *rqst, if (rc) return rc; - req->PersistentFileId = cpu_to_le64(persistent_fid); - req->VolatileFileId = cpu_to_le64(volatile_fid); + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; /* See note 354 of MS-SMB2, 64K max */ req->OutputBufferLength = cpu_to_le32(SMB2_MAX_BUFFER_SIZE - MAX_SMB2_HDR_SIZE); @@ -3858,12 +3853,14 @@ void smb2_reconnect_server(struct work_struct *work) tcon = kzalloc(sizeof(struct cifs_tcon), GFP_KERNEL); if (!tcon) { resched = true; - list_del_init(&ses->rlist); - cifs_put_smb_ses(ses); + list_for_each_entry_safe(ses, ses2, &tmp_ses_list, rlist) { + list_del_init(&ses->rlist); + cifs_put_smb_ses(ses); + } goto done; } - tcon->tidStatus = CifsGood; + tcon->status = TID_GOOD; tcon->retry = false; tcon->need_reconnect = false; @@ -3951,8 +3948,8 @@ SMB2_flush_init(const unsigned int xid, struct smb_rqst *rqst, if (rc) return rc; - req->PersistentFileId = cpu_to_le64(persistent_fid); - req->VolatileFileId = cpu_to_le64(volatile_fid); + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; iov[0].iov_base = (char *)req; iov[0].iov_len = total_len; @@ -4033,8 +4030,8 @@ smb2_new_read_req(void **buf, unsigned int *total_len, shdr = &req->hdr; shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); - req->PersistentFileId = cpu_to_le64(io_parms->persistent_fid); - req->VolatileFileId = cpu_to_le64(io_parms->volatile_fid); + req->PersistentFileId = io_parms->persistent_fid; + req->VolatileFileId = io_parms->volatile_fid; req->ReadChannelInfoOffset = 0; /* reserved */ req->ReadChannelInfoLength = 0; /* reserved */ req->Channel = 0; /* reserved */ @@ -4094,8 +4091,8 @@ smb2_new_read_req(void **buf, unsigned int *total_len, */ shdr->SessionId = cpu_to_le64(0xFFFFFFFFFFFFFFFF); shdr->Id.SyncId.TreeId = cpu_to_le32(0xFFFFFFFF); - req->PersistentFileId = cpu_to_le64(0xFFFFFFFFFFFFFFFF); - req->VolatileFileId = cpu_to_le64(0xFFFFFFFFFFFFFFFF); + req->PersistentFileId = (u64)-1; + req->VolatileFileId = (u64)-1; } } if (remaining_bytes > io_parms->length) @@ -4307,21 +4304,19 @@ SMB2_read(const unsigned int xid, struct cifs_io_parms *io_parms, cifs_stats_fail_inc(io_parms->tcon, SMB2_READ_HE); cifs_dbg(VFS, "Send error in read = %d\n", rc); trace_smb3_read_err(xid, - le64_to_cpu(req->PersistentFileId), + req->PersistentFileId, io_parms->tcon->tid, ses->Suid, io_parms->offset, io_parms->length, rc); } else - trace_smb3_read_done(xid, - le64_to_cpu(req->PersistentFileId), - io_parms->tcon->tid, ses->Suid, - io_parms->offset, 0); + trace_smb3_read_done(xid, req->PersistentFileId, io_parms->tcon->tid, + ses->Suid, io_parms->offset, 0); free_rsp_buf(resp_buftype, rsp_iov.iov_base); cifs_small_buf_release(req); return rc == -ENODATA ? 0 : rc; } else trace_smb3_read_done(xid, - le64_to_cpu(req->PersistentFileId), + req->PersistentFileId, io_parms->tcon->tid, ses->Suid, io_parms->offset, io_parms->length); @@ -4463,8 +4458,8 @@ smb2_async_writev(struct cifs_writedata *wdata, shdr = (struct smb2_hdr *)req; shdr->Id.SyncId.ProcessId = cpu_to_le32(wdata->cfile->pid); - req->PersistentFileId = cpu_to_le64(wdata->cfile->fid.persistent_fid); - req->VolatileFileId = cpu_to_le64(wdata->cfile->fid.volatile_fid); + req->PersistentFileId = wdata->cfile->fid.persistent_fid; + req->VolatileFileId = wdata->cfile->fid.volatile_fid; req->WriteChannelInfoOffset = 0; req->WriteChannelInfoLength = 0; req->Channel = 0; @@ -4562,7 +4557,7 @@ smb2_async_writev(struct cifs_writedata *wdata, if (rc) { trace_smb3_write_err(0 /* no xid */, - le64_to_cpu(req->PersistentFileId), + req->PersistentFileId, tcon->tid, tcon->ses->Suid, wdata->offset, wdata->bytes, rc); kref_put(&wdata->refcount, release); @@ -4615,8 +4610,8 @@ SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, req->hdr.Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); - req->PersistentFileId = cpu_to_le64(io_parms->persistent_fid); - req->VolatileFileId = cpu_to_le64(io_parms->volatile_fid); + req->PersistentFileId = io_parms->persistent_fid; + req->VolatileFileId = io_parms->volatile_fid; req->WriteChannelInfoOffset = 0; req->WriteChannelInfoLength = 0; req->Channel = 0; @@ -4645,7 +4640,7 @@ SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, if (rc) { trace_smb3_write_err(xid, - le64_to_cpu(req->PersistentFileId), + req->PersistentFileId, io_parms->tcon->tid, io_parms->tcon->ses->Suid, io_parms->offset, io_parms->length, rc); @@ -4654,7 +4649,7 @@ SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, } else { *nbytes = le32_to_cpu(rsp->DataLength); trace_smb3_write_done(xid, - le64_to_cpu(req->PersistentFileId), + req->PersistentFileId, io_parms->tcon->tid, io_parms->tcon->ses->Suid, io_parms->offset, *nbytes); diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index 33cfd0a1adf1..d8c4388b190d 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -56,16 +56,6 @@ struct smb2_rdma_crypto_transform { #define COMPOUND_FID 0xFFFFFFFFFFFFFFFFULL -#define SMB2_ERROR_STRUCTURE_SIZE2 cpu_to_le16(9) - -struct smb2_err_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; - __le16 Reserved; /* MBZ */ - __le32 ByteCount; /* even if zero, at least one byte follows */ - __u8 ErrorData[1]; /* variable length */ -} __packed; - #define SYMLINK_ERROR_TAG 0x4c4d5953 struct smb2_symlink_err_rsp { @@ -139,47 +129,6 @@ struct share_redirect_error_context_rsp { #define SMB2_LEASE_HANDLE_CACHING_HE 0x02 #define SMB2_LEASE_WRITE_CACHING_HE 0x04 -#define SMB2_LEASE_NONE cpu_to_le32(0x00) -#define SMB2_LEASE_READ_CACHING cpu_to_le32(0x01) -#define SMB2_LEASE_HANDLE_CACHING cpu_to_le32(0x02) -#define SMB2_LEASE_WRITE_CACHING cpu_to_le32(0x04) - -#define SMB2_LEASE_FLAG_BREAK_IN_PROGRESS cpu_to_le32(0x00000002) -#define SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET cpu_to_le32(0x00000004) - -#define SMB2_LEASE_KEY_SIZE 16 - -struct lease_context { - u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; - __le32 LeaseState; - __le32 LeaseFlags; - __le64 LeaseDuration; -} __packed; - -struct lease_context_v2 { - u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; - __le32 LeaseState; - __le32 LeaseFlags; - __le64 LeaseDuration; - __le64 ParentLeaseKeyLow; - __le64 ParentLeaseKeyHigh; - __le16 Epoch; - __le16 Reserved; -} __packed; - -struct create_lease { - struct create_context ccontext; - __u8 Name[8]; - struct lease_context lcontext; -} __packed; - -struct create_lease_v2 { - struct create_context ccontext; - __u8 Name[8]; - struct lease_context_v2 lcontext; - __u8 Pad[4]; -} __packed; - struct create_durable { struct create_context ccontext; __u8 Name[8]; @@ -192,13 +141,6 @@ struct create_durable { } Data; } __packed; -struct create_posix { - struct create_context ccontext; - __u8 Name[16]; - __le32 Mode; - __u32 Reserved; -} __packed; - /* See MS-SMB2 2.2.13.2.11 */ /* Flags */ #define SMB2_DHANDLE_FLAG_PERSISTENT 0x00000002 @@ -287,12 +229,6 @@ struct copychunk_ioctl { __u32 Reserved2; } __packed; -/* this goes in the ioctl buffer when doing FSCTL_SET_ZERO_DATA */ -struct file_zero_data_information { - __le64 FileOffset; - __le64 BeyondFinalZero; -} __packed; - struct copychunk_ioctl_rsp { __le32 ChunksWritten; __le32 ChunkBytesWritten; @@ -338,11 +274,6 @@ struct fsctl_get_integrity_information_rsp { __le32 ClusterSizeInBytes; } __packed; -struct file_allocated_range_buffer { - __le64 file_offset; - __le64 length; -} __packed; - /* Integrity ChecksumAlgorithm choices for above */ #define CHECKSUM_TYPE_NONE 0x0000 #define CHECKSUM_TYPE_CRC64 0x0002 @@ -351,53 +282,6 @@ struct file_allocated_range_buffer { /* Integrity flags for above */ #define FSCTL_INTEGRITY_FLAG_CHECKSUM_ENFORCEMENT_OFF 0x00000001 -/* Reparse structures - see MS-FSCC 2.1.2 */ - -/* struct fsctl_reparse_info_req is empty, only response structs (see below) */ - -struct reparse_data_buffer { - __le32 ReparseTag; - __le16 ReparseDataLength; - __u16 Reserved; - __u8 DataBuffer[]; /* Variable Length */ -} __packed; - -struct reparse_guid_data_buffer { - __le32 ReparseTag; - __le16 ReparseDataLength; - __u16 Reserved; - __u8 ReparseGuid[16]; - __u8 DataBuffer[]; /* Variable Length */ -} __packed; - -struct reparse_mount_point_data_buffer { - __le32 ReparseTag; - __le16 ReparseDataLength; - __u16 Reserved; - __le16 SubstituteNameOffset; - __le16 SubstituteNameLength; - __le16 PrintNameOffset; - __le16 PrintNameLength; - __u8 PathBuffer[]; /* Variable Length */ -} __packed; - -#define SYMLINK_FLAG_RELATIVE 0x00000001 - -struct reparse_symlink_data_buffer { - __le32 ReparseTag; - __le16 ReparseDataLength; - __u16 Reserved; - __le16 SubstituteNameOffset; - __le16 SubstituteNameLength; - __le16 PrintNameOffset; - __le16 PrintNameLength; - __le32 Flags; - __u8 PathBuffer[]; /* Variable Length */ -} __packed; - -/* See MS-FSCC 2.1.2.6 and cifspdu.h for struct reparse_posix_data */ - - /* See MS-DFSC 2.2.2 */ struct fsctl_get_dfs_referral_req { __le16 MaxReferralLevel; @@ -413,22 +297,6 @@ struct network_resiliency_req { } __packed; /* There is no buffer for the response ie no struct network_resiliency_rsp */ - -struct validate_negotiate_info_req { - __le32 Capabilities; - __u8 Guid[SMB2_CLIENT_GUID_SIZE]; - __le16 SecurityMode; - __le16 DialectCount; - __le16 Dialects[4]; /* BB expand this if autonegotiate > 4 dialects */ -} __packed; - -struct validate_negotiate_info_rsp { - __le32 Capabilities; - __u8 Guid[SMB2_CLIENT_GUID_SIZE]; - __le16 SecurityMode; - __le16 Dialect; /* Dialect in use for the connection */ -} __packed; - #define RSS_CAPABLE cpu_to_le32(0x00000001) #define RDMA_CAPABLE cpu_to_le32(0x00000002) @@ -464,14 +332,6 @@ struct compress_ioctl { __le16 CompressionState; /* See cifspdu.h for possible flag values */ } __packed; -struct duplicate_extents_to_file { - __u64 PersistentFileHandle; /* source file handle, opaque endianness */ - __u64 VolatileFileHandle; - __le64 SourceFileOffset; - __le64 TargetFileOffset; - __le64 ByteCount; /* Bytes to be copied */ -} __packed; - /* * Maximum number of iovs we need for an ioctl request. * [0] : struct smb2_ioctl_req @@ -479,370 +339,11 @@ struct duplicate_extents_to_file { */ #define SMB2_IOCTL_IOV_SIZE 2 -struct smb2_ioctl_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 57 */ - __u16 Reserved; - __le32 CtlCode; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - __le32 InputOffset; - __le32 InputCount; - __le32 MaxInputResponse; - __le32 OutputOffset; - __le32 OutputCount; - __le32 MaxOutputResponse; - __le32 Flags; - __u32 Reserved2; - __u8 Buffer[]; -} __packed; - -struct smb2_ioctl_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 57 */ - __u16 Reserved; - __le32 CtlCode; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - __le32 InputOffset; - __le32 InputCount; - __le32 OutputOffset; - __le32 OutputCount; - __le32 Flags; - __u32 Reserved2; - /* char * buffer[] */ -} __packed; - -#define SMB2_LOCKFLAG_SHARED_LOCK 0x0001 -#define SMB2_LOCKFLAG_EXCLUSIVE_LOCK 0x0002 -#define SMB2_LOCKFLAG_UNLOCK 0x0004 -#define SMB2_LOCKFLAG_FAIL_IMMEDIATELY 0x0010 - -struct smb2_lock_element { - __le64 Offset; - __le64 Length; - __le32 Flags; - __le32 Reserved; -} __packed; - -struct smb2_lock_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 48 */ - __le16 LockCount; - /* - * The least significant four bits are the index, the other 28 bits are - * the lock sequence number (0 to 64). See MS-SMB2 2.2.26 - */ - __le32 LockSequenceNumber; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - /* Followed by at least one */ - struct smb2_lock_element locks[1]; -} __packed; - -struct smb2_lock_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __le16 Reserved; -} __packed; - -struct smb2_echo_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __u16 Reserved; -} __packed; - -struct smb2_echo_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __u16 Reserved; -} __packed; - -/* search (query_directory) Flags field */ -#define SMB2_RESTART_SCANS 0x01 -#define SMB2_RETURN_SINGLE_ENTRY 0x02 -#define SMB2_INDEX_SPECIFIED 0x04 -#define SMB2_REOPEN 0x10 - -#define SMB2_QUERY_DIRECTORY_IOV_SIZE 2 - -/* - * Valid FileInformation classes. - * - * Note that these are a subset of the (file) QUERY_INFO levels defined - * later in this file (but since QUERY_DIRECTORY uses equivalent numbers - * we do not redefine them here) - * - * FileDirectoryInfomation 0x01 - * FileFullDirectoryInformation 0x02 - * FileIdFullDirectoryInformation 0x26 - * FileBothDirectoryInformation 0x03 - * FileIdBothDirectoryInformation 0x25 - * FileNamesInformation 0x0C - * FileIdExtdDirectoryInformation 0x3C - */ - -struct smb2_query_directory_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 33 */ - __u8 FileInformationClass; - __u8 Flags; - __le32 FileIndex; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - __le16 FileNameOffset; - __le16 FileNameLength; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -struct smb2_query_directory_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 9 */ - __le16 OutputBufferOffset; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -/* Possible InfoType values */ -#define SMB2_O_INFO_FILE 0x01 -#define SMB2_O_INFO_FILESYSTEM 0x02 -#define SMB2_O_INFO_SECURITY 0x03 -#define SMB2_O_INFO_QUOTA 0x04 - -/* Security info type additionalinfo flags. See MS-SMB2 (2.2.37) or MS-DTYP */ -#define OWNER_SECINFO 0x00000001 -#define GROUP_SECINFO 0x00000002 -#define DACL_SECINFO 0x00000004 -#define SACL_SECINFO 0x00000008 -#define LABEL_SECINFO 0x00000010 -#define ATTRIBUTE_SECINFO 0x00000020 -#define SCOPE_SECINFO 0x00000040 -#define BACKUP_SECINFO 0x00010000 -#define UNPROTECTED_SACL_SECINFO 0x10000000 -#define UNPROTECTED_DACL_SECINFO 0x20000000 -#define PROTECTED_SACL_SECINFO 0x40000000 -#define PROTECTED_DACL_SECINFO 0x80000000 - -/* Flags used for FileFullEAinfo */ -#define SL_RESTART_SCAN 0x00000001 -#define SL_RETURN_SINGLE_ENTRY 0x00000002 -#define SL_INDEX_SPECIFIED 0x00000004 - -struct smb2_query_info_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 41 */ - __u8 InfoType; - __u8 FileInfoClass; - __le32 OutputBufferLength; - __le16 InputBufferOffset; - __u16 Reserved; - __le32 InputBufferLength; - __le32 AdditionalInformation; - __le32 Flags; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - __u8 Buffer[1]; -} __packed; - -struct smb2_query_info_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 9 */ - __le16 OutputBufferOffset; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -/* - * Maximum number of iovs we need for a set-info request. - * The largest one is rename/hardlink - * [0] : struct smb2_set_info_req + smb2_file_[rename|link]_info - * [1] : path - * [2] : compound padding - */ -#define SMB2_SET_INFO_IOV_SIZE 3 - -struct smb2_set_info_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 33 */ - __u8 InfoType; - __u8 FileInfoClass; - __le32 BufferLength; - __le16 BufferOffset; - __u16 Reserved; - __le32 AdditionalInformation; - __u64 PersistentFileId; /* opaque endianness */ - __u64 VolatileFileId; /* opaque endianness */ - __u8 Buffer[1]; -} __packed; - -struct smb2_set_info_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 2 */ -} __packed; - -struct smb2_oplock_break { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 24 */ - __u8 OplockLevel; - __u8 Reserved; - __le32 Reserved2; - __u64 PersistentFid; - __u64 VolatileFid; -} __packed; - -#define SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED cpu_to_le32(0x01) - -struct smb2_lease_break { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 44 */ - __le16 Epoch; - __le32 Flags; - __u8 LeaseKey[16]; - __le32 CurrentLeaseState; - __le32 NewLeaseState; - __le32 BreakReason; - __le32 AccessMaskHint; - __le32 ShareMaskHint; -} __packed; - -struct smb2_lease_ack { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 36 */ - __le16 Reserved; - __le32 Flags; - __u8 LeaseKey[16]; - __le32 LeaseState; - __le64 LeaseDuration; -} __packed; - /* - * PDU infolevel structure definitions + * PDU query infolevel structure definitions * BB consider moving to a different header */ -/* File System Information Classes */ -#define FS_VOLUME_INFORMATION 1 /* Query */ -#define FS_LABEL_INFORMATION 2 /* Local only */ -#define FS_SIZE_INFORMATION 3 /* Query */ -#define FS_DEVICE_INFORMATION 4 /* Query */ -#define FS_ATTRIBUTE_INFORMATION 5 /* Query */ -#define FS_CONTROL_INFORMATION 6 /* Query, Set */ -#define FS_FULL_SIZE_INFORMATION 7 /* Query */ -#define FS_OBJECT_ID_INFORMATION 8 /* Query, Set */ -#define FS_DRIVER_PATH_INFORMATION 9 /* Local only */ -#define FS_VOLUME_FLAGS_INFORMATION 10 /* Local only */ -#define FS_SECTOR_SIZE_INFORMATION 11 /* SMB3 or later. Query */ -#define FS_POSIX_INFORMATION 100 /* SMB3.1.1 POSIX. Query */ - -struct smb2_fs_full_size_info { - __le64 TotalAllocationUnits; - __le64 CallerAvailableAllocationUnits; - __le64 ActualAvailableAllocationUnits; - __le32 SectorsPerAllocationUnit; - __le32 BytesPerSector; -} __packed; - -#define SSINFO_FLAGS_ALIGNED_DEVICE 0x00000001 -#define SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE 0x00000002 -#define SSINFO_FLAGS_NO_SEEK_PENALTY 0x00000004 -#define SSINFO_FLAGS_TRIM_ENABLED 0x00000008 - -/* sector size info struct */ -struct smb3_fs_ss_info { - __le32 LogicalBytesPerSector; - __le32 PhysicalBytesPerSectorForAtomicity; - __le32 PhysicalBytesPerSectorForPerf; - __le32 FileSystemEffectivePhysicalBytesPerSectorForAtomicity; - __le32 Flags; - __le32 ByteOffsetForSectorAlignment; - __le32 ByteOffsetForPartitionAlignment; -} __packed; - -/* volume info struct - see MS-FSCC 2.5.9 */ -#define MAX_VOL_LABEL_LEN 32 -struct smb3_fs_vol_info { - __le64 VolumeCreationTime; - __u32 VolumeSerialNumber; - __le32 VolumeLabelLength; /* includes trailing null */ - __u8 SupportsObjects; /* True if eg like NTFS, supports objects */ - __u8 Reserved; - __u8 VolumeLabel[]; /* variable len */ -} __packed; - -/* partial list of QUERY INFO levels */ -#define FILE_DIRECTORY_INFORMATION 1 -#define FILE_FULL_DIRECTORY_INFORMATION 2 -#define FILE_BOTH_DIRECTORY_INFORMATION 3 -#define FILE_BASIC_INFORMATION 4 -#define FILE_STANDARD_INFORMATION 5 -#define FILE_INTERNAL_INFORMATION 6 -#define FILE_EA_INFORMATION 7 -#define FILE_ACCESS_INFORMATION 8 -#define FILE_NAME_INFORMATION 9 -#define FILE_RENAME_INFORMATION 10 -#define FILE_LINK_INFORMATION 11 -#define FILE_NAMES_INFORMATION 12 -#define FILE_DISPOSITION_INFORMATION 13 -#define FILE_POSITION_INFORMATION 14 -#define FILE_FULL_EA_INFORMATION 15 -#define FILE_MODE_INFORMATION 16 -#define FILE_ALIGNMENT_INFORMATION 17 -#define FILE_ALL_INFORMATION 18 -#define FILE_ALLOCATION_INFORMATION 19 -#define FILE_END_OF_FILE_INFORMATION 20 -#define FILE_ALTERNATE_NAME_INFORMATION 21 -#define FILE_STREAM_INFORMATION 22 -#define FILE_PIPE_INFORMATION 23 -#define FILE_PIPE_LOCAL_INFORMATION 24 -#define FILE_PIPE_REMOTE_INFORMATION 25 -#define FILE_MAILSLOT_QUERY_INFORMATION 26 -#define FILE_MAILSLOT_SET_INFORMATION 27 -#define FILE_COMPRESSION_INFORMATION 28 -#define FILE_OBJECT_ID_INFORMATION 29 -/* Number 30 not defined in documents */ -#define FILE_MOVE_CLUSTER_INFORMATION 31 -#define FILE_QUOTA_INFORMATION 32 -#define FILE_REPARSE_POINT_INFORMATION 33 -#define FILE_NETWORK_OPEN_INFORMATION 34 -#define FILE_ATTRIBUTE_TAG_INFORMATION 35 -#define FILE_TRACKING_INFORMATION 36 -#define FILEID_BOTH_DIRECTORY_INFORMATION 37 -#define FILEID_FULL_DIRECTORY_INFORMATION 38 -#define FILE_VALID_DATA_LENGTH_INFORMATION 39 -#define FILE_SHORT_NAME_INFORMATION 40 -#define FILE_SFIO_RESERVE_INFORMATION 44 -#define FILE_SFIO_VOLUME_INFORMATION 45 -#define FILE_HARD_LINK_INFORMATION 46 -#define FILE_NORMALIZED_NAME_INFORMATION 48 -#define FILEID_GLOBAL_TX_DIRECTORY_INFORMATION 50 -#define FILE_STANDARD_LINK_INFORMATION 54 -#define FILE_ID_INFORMATION 59 -#define FILE_ID_EXTD_DIRECTORY_INFORMATION 60 - -struct smb2_file_internal_info { - __le64 IndexNumber; -} __packed; /* level 6 Query */ - -struct smb2_file_rename_info { /* encoding of request for level 10 */ - __u8 ReplaceIfExists; /* 1 = replace existing target with new */ - /* 0 = fail if target already exists */ - __u8 Reserved[7]; - __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ - __le32 FileNameLength; - char FileName[]; /* New name to be assigned */ - /* padding - overall struct size must be >= 24 so filename + pad >= 6 */ -} __packed; /* level 10 Set */ - -struct smb2_file_link_info { /* encoding of request for level 11 */ - __u8 ReplaceIfExists; /* 1 = replace existing link with new */ - /* 0 = fail if link already exists */ - __u8 Reserved[7]; - __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ - __le32 FileNameLength; - char FileName[]; /* Name to be assigned to new link */ -} __packed; /* level 11 Set */ - struct smb2_file_full_ea_info { /* encoding of response for level 15 */ __le32 next_entry_offset; __u8 flags; @@ -851,38 +352,6 @@ struct smb2_file_full_ea_info { /* encoding of response for level 15 */ char ea_data[]; /* \0 terminated name plus value */ } __packed; /* level 15 Set */ -/* - * This level 18, although with struct with same name is different from cifs - * level 0x107. Level 0x107 has an extra u64 between AccessFlags and - * CurrentByteOffset. - */ -struct smb2_file_all_info { /* data block encoding of response to level 18 */ - __le64 CreationTime; /* Beginning of FILE_BASIC_INFO equivalent */ - __le64 LastAccessTime; - __le64 LastWriteTime; - __le64 ChangeTime; - __le32 Attributes; - __u32 Pad1; /* End of FILE_BASIC_INFO_INFO equivalent */ - __le64 AllocationSize; /* Beginning of FILE_STANDARD_INFO equivalent */ - __le64 EndOfFile; /* size ie offset to first free byte in file */ - __le32 NumberOfLinks; /* hard links */ - __u8 DeletePending; - __u8 Directory; - __u16 Pad2; /* End of FILE_STANDARD_INFO equivalent */ - __le64 IndexNumber; - __le32 EASize; - __le32 AccessFlags; - __le64 CurrentByteOffset; - __le32 Mode; - __le32 AlignmentRequirement; - __le32 FileNameLength; - char FileName[1]; -} __packed; /* level 18 Query */ - -struct smb2_file_eof_info { /* encoding of request for level 10 */ - __le64 EndOfFile; /* new end of file value */ -} __packed; /* level 20 Set */ - struct smb2_file_reparse_point_info { __le64 IndexNumber; __le32 Tag; @@ -935,6 +404,8 @@ struct create_posix_rsp { struct cifs_sid group; /* var-sized on the wire */ } __packed; +#define SMB2_QUERY_DIRECTORY_IOV_SIZE 2 + /* * SMB2-only POSIX info level for query dir * @@ -966,31 +437,6 @@ struct smb2_posix_info { */ } __packed; -/* Level 100 query info */ -struct smb311_posix_qinfo { - __le64 CreationTime; - __le64 LastAccessTime; - __le64 LastWriteTime; - __le64 ChangeTime; - __le64 EndOfFile; - __le64 AllocationSize; - __le32 DosAttributes; - __le64 Inode; - __le32 DeviceId; - __le32 Zero; - /* beginning of POSIX Create Context Response */ - __le32 HardLinks; - __le32 ReparseTag; - __le32 Mode; - u8 Sids[]; - /* - * var sized owner SID - * var sized group SID - * le32 filenamelength - * u8 filename[] - */ -} __packed; - /* * Parsed version of the above struct. Allows direct access to the * variable length fields diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h index 4a7062fd1c26..a69f1eed1cfe 100644 --- a/fs/cifs/smb2proto.h +++ b/fs/cifs/smb2proto.h @@ -283,7 +283,7 @@ extern int smb311_update_preauth_hash(struct cifs_ses *ses, struct kvec *iov, int nvec); extern int smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, - __le16 *utf16_path, u32 desired_access, + const char *path, u32 desired_access, u32 class, u32 type, u32 output_len, struct kvec *rsp, int *buftype, struct cifs_sb_info *cifs_sb); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 4fcca79f39ae..526a4c1bed99 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -248,7 +248,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); * which must still be locked and not uptodate. Normally, blocksize == * PAGE_SIZE and the whole page is decrypted at once. * - * This is for use by the filesystem's ->readpages() method. + * This is for use by the filesystem's ->readahead() method. * * Return: 0 on success; -errno on failure */ diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h index 619e5b4bed10..c6800b880920 100644 --- a/fs/exfat/exfat_fs.h +++ b/fs/exfat/exfat_fs.h @@ -203,7 +203,8 @@ struct exfat_mount_options { /* on error: continue, panic, remount-ro */ enum exfat_error_mode errors; unsigned utf8:1, /* Use of UTF-8 character set */ - discard:1; /* Issue discard requests on deletions */ + discard:1, /* Issue discard requests on deletions */ + keep_last_dots:1; /* Keep trailing periods in paths */ int time_offset; /* Offset of timestamps from UTC (in minutes) */ }; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index d890fd34bb2d..2f5130059236 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -218,8 +218,6 @@ int __exfat_truncate(struct inode *inode, loff_t new_size) if (exfat_free_cluster(inode, &clu)) return -EIO; - exfat_clear_volume_dirty(sb); - return 0; } diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c index af4eb39cc0c3..a02a04a993bf 100644 --- a/fs/exfat/namei.c +++ b/fs/exfat/namei.c @@ -65,11 +65,14 @@ static int exfat_d_revalidate(struct dentry *dentry, unsigned int flags) return ret; } -/* returns the length of a struct qstr, ignoring trailing dots */ -static unsigned int exfat_striptail_len(unsigned int len, const char *name) +/* returns the length of a struct qstr, ignoring trailing dots if necessary */ +static unsigned int exfat_striptail_len(unsigned int len, const char *name, + bool keep_last_dots) { - while (len && name[len - 1] == '.') - len--; + if (!keep_last_dots) { + while (len && name[len - 1] == '.') + len--; + } return len; } @@ -83,7 +86,8 @@ static int exfat_d_hash(const struct dentry *dentry, struct qstr *qstr) struct super_block *sb = dentry->d_sb; struct nls_table *t = EXFAT_SB(sb)->nls_io; const unsigned char *name = qstr->name; - unsigned int len = exfat_striptail_len(qstr->len, qstr->name); + unsigned int len = exfat_striptail_len(qstr->len, qstr->name, + EXFAT_SB(sb)->options.keep_last_dots); unsigned long hash = init_name_hash(dentry); int i, charlen; wchar_t c; @@ -104,8 +108,10 @@ static int exfat_d_cmp(const struct dentry *dentry, unsigned int len, { struct super_block *sb = dentry->d_sb; struct nls_table *t = EXFAT_SB(sb)->nls_io; - unsigned int alen = exfat_striptail_len(name->len, name->name); - unsigned int blen = exfat_striptail_len(len, str); + unsigned int alen = exfat_striptail_len(name->len, name->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned int blen = exfat_striptail_len(len, str, + EXFAT_SB(sb)->options.keep_last_dots); wchar_t c1, c2; int charlen, i; @@ -136,7 +142,8 @@ static int exfat_utf8_d_hash(const struct dentry *dentry, struct qstr *qstr) { struct super_block *sb = dentry->d_sb; const unsigned char *name = qstr->name; - unsigned int len = exfat_striptail_len(qstr->len, qstr->name); + unsigned int len = exfat_striptail_len(qstr->len, qstr->name, + EXFAT_SB(sb)->options.keep_last_dots); unsigned long hash = init_name_hash(dentry); int i, charlen; unicode_t u; @@ -161,8 +168,11 @@ static int exfat_utf8_d_cmp(const struct dentry *dentry, unsigned int len, const char *str, const struct qstr *name) { struct super_block *sb = dentry->d_sb; - unsigned int alen = exfat_striptail_len(name->len, name->name); - unsigned int blen = exfat_striptail_len(len, str); + unsigned int alen = exfat_striptail_len(name->len, name->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned int blen = exfat_striptail_len(len, str, + EXFAT_SB(sb)->options.keep_last_dots); + unicode_t u_a, u_b; int charlen, i; @@ -416,13 +426,25 @@ static int __exfat_resolve_path(struct inode *inode, const unsigned char *path, struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct exfat_inode_info *ei = EXFAT_I(inode); + int pathlen = strlen(path); - /* strip all trailing periods */ - namelen = exfat_striptail_len(strlen(path), path); + /* + * get the length of the pathname excluding + * trailing periods, if any. + */ + namelen = exfat_striptail_len(pathlen, path, false); + if (EXFAT_SB(sb)->options.keep_last_dots) { + /* + * Do not allow the creation of files with names + * ending with period(s). + */ + if (!lookup && (namelen < pathlen)) + return -EINVAL; + namelen = pathlen; + } if (!namelen) return -ENOENT; - - if (strlen(path) > (MAX_NAME_LENGTH * MAX_CHARSET_SIZE)) + if (pathlen > (MAX_NAME_LENGTH * MAX_CHARSET_SIZE)) return -ENAMETOOLONG; /* @@ -554,7 +576,6 @@ static int exfat_create(struct user_namespace *mnt_userns, struct inode *dir, exfat_set_volume_dirty(sb); err = exfat_add_entry(dir, dentry->d_name.name, &cdir, TYPE_FILE, &info); - exfat_clear_volume_dirty(sb); if (err) goto unlock; @@ -812,7 +833,6 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry) /* This doesn't modify ei */ ei->dir.dir = DIR_DELETED; - exfat_clear_volume_dirty(sb); inode_inc_iversion(dir); dir->i_mtime = dir->i_atime = current_time(dir); @@ -846,7 +866,6 @@ static int exfat_mkdir(struct user_namespace *mnt_userns, struct inode *dir, exfat_set_volume_dirty(sb); err = exfat_add_entry(dir, dentry->d_name.name, &cdir, TYPE_DIR, &info); - exfat_clear_volume_dirty(sb); if (err) goto unlock; @@ -976,7 +995,6 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) goto unlock; } ei->dir.dir = DIR_DELETED; - exfat_clear_volume_dirty(sb); inode_inc_iversion(dir); dir->i_mtime = dir->i_atime = current_time(dir); @@ -1311,7 +1329,6 @@ del_out: */ new_ei->dir.dir = DIR_DELETED; } - exfat_clear_volume_dirty(sb); out: return ret; } diff --git a/fs/exfat/super.c b/fs/exfat/super.c index 9f892903419a..8ca21e7917d1 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -100,7 +100,6 @@ static int exfat_set_vol_flags(struct super_block *sb, unsigned short new_flags) { struct exfat_sb_info *sbi = EXFAT_SB(sb); struct boot_sector *p_boot = (struct boot_sector *)sbi->boot_bh->b_data; - bool sync; /* retain persistent-flags */ new_flags |= sbi->vol_flags_persistent; @@ -119,16 +118,11 @@ static int exfat_set_vol_flags(struct super_block *sb, unsigned short new_flags) p_boot->vol_flags = cpu_to_le16(new_flags); - if ((new_flags & VOLUME_DIRTY) && !buffer_dirty(sbi->boot_bh)) - sync = true; - else - sync = false; - set_buffer_uptodate(sbi->boot_bh); mark_buffer_dirty(sbi->boot_bh); - if (sync) - sync_dirty_buffer(sbi->boot_bh); + __sync_dirty_buffer(sbi->boot_bh, REQ_SYNC | REQ_FUA | REQ_PREFLUSH); + return 0; } @@ -174,6 +168,8 @@ static int exfat_show_options(struct seq_file *m, struct dentry *root) seq_puts(m, ",errors=remount-ro"); if (opts->discard) seq_puts(m, ",discard"); + if (opts->keep_last_dots) + seq_puts(m, ",keep_last_dots"); if (opts->time_offset) seq_printf(m, ",time_offset=%d", opts->time_offset); return 0; @@ -217,6 +213,7 @@ enum { Opt_charset, Opt_errors, Opt_discard, + Opt_keep_last_dots, Opt_time_offset, /* Deprecated options */ @@ -243,6 +240,7 @@ static const struct fs_parameter_spec exfat_parameters[] = { fsparam_string("iocharset", Opt_charset), fsparam_enum("errors", Opt_errors, exfat_param_enums), fsparam_flag("discard", Opt_discard), + fsparam_flag("keep_last_dots", Opt_keep_last_dots), fsparam_s32("time_offset", Opt_time_offset), __fsparam(NULL, "utf8", Opt_utf8, fs_param_deprecated, NULL), @@ -297,6 +295,9 @@ static int exfat_parse_param(struct fs_context *fc, struct fs_parameter *param) case Opt_discard: opts->discard = 1; break; + case Opt_keep_last_dots: + opts->keep_last_dots = 1; + break; case Opt_time_offset: /* * Make the limit 24 just in case someone invents something diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 8bd66cdc41be..6feb07e3e1eb 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -267,7 +267,7 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, goto out; current->backing_dev_info = inode_to_bdi(inode); - ret = generic_perform_write(iocb->ki_filp, from, iocb->ki_pos); + ret = generic_perform_write(iocb, from); current->backing_dev_info = NULL; out: diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 1ce13f69fbec..13740f2d0e61 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3589,7 +3589,7 @@ const struct iomap_ops ext4_iomap_report_ops = { static bool ext4_journalled_dirty_folio(struct address_space *mapping, struct folio *folio) { - WARN_ON_ONCE(!page_has_buffers(&folio->page)); + WARN_ON_ONCE(!folio_buffers(folio)); folio_set_checked(folio); return filemap_dirty_folio(mapping, folio); } diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 1aa26d6634fc..af491e170c4a 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -109,7 +109,7 @@ static void verity_work(struct work_struct *work) struct bio *bio = ctx->bio; /* - * fsverity_verify_bio() may call readpages() again, and although verity + * fsverity_verify_bio() may call readahead() again, and although verity * will be disabled for that, decryption may still be needed, causing * another bio_post_read_ctx to be allocated. So to guarantee that * mempool_alloc() never deadlocks we must free the current ctx first. diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index a8fc4fa511a8..f5366feea82d 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -456,7 +456,7 @@ static bool f2fs_dirty_meta_folio(struct address_space *mapping, folio_mark_uptodate(folio); if (!folio_test_dirty(folio)) { filemap_dirty_folio(mapping, folio); - inc_page_count(F2FS_P_SB(&folio->page), F2FS_DIRTY_META); + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META); set_page_private_reference(&folio->page); return true; } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index f8fcbe91059b..8e0c2e773c8d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -164,7 +164,7 @@ static void f2fs_verify_bio(struct work_struct *work) bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); /* - * fsverity_verify_bio() may call readpages() again, and while verity + * fsverity_verify_bio() may call readahead() again, and while verity * will be disabled for this, decryption and/or decompression may still * be needed, resulting in another bio_post_read_ctx being allocated. * So to prevent deadlocks we need to release the current ctx to the @@ -2392,7 +2392,7 @@ static void f2fs_readahead(struct readahead_control *rac) if (!f2fs_is_compress_backend_ready(inode)) return; - /* If the file has inline data, skip readpages */ + /* If the file has inline data, skip readahead */ if (f2fs_has_inline_data(inode)) return; @@ -3571,7 +3571,7 @@ static bool f2fs_dirty_data_folio(struct address_space *mapping, f2fs_update_dirty_folio(inode, folio); return true; } - return true; + return false; } diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d3f39a704b8b..5b89af0f27f0 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4448,7 +4448,7 @@ static ssize_t f2fs_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; current->backing_dev_info = inode_to_bdi(inode); - ret = generic_perform_write(file, from, iocb->ki_pos); + ret = generic_perform_write(iocb, from); current->backing_dev_info = NULL; if (ret > 0) { diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 0b6e741e94a0..c45d341dcf6e 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -2146,11 +2146,11 @@ static bool f2fs_dirty_node_folio(struct address_space *mapping, folio_mark_uptodate(folio); #ifdef CONFIG_F2FS_CHECK_FS if (IS_INODE(&folio->page)) - f2fs_inode_chksum_set(F2FS_P_SB(&folio->page), &folio->page); + f2fs_inode_chksum_set(F2FS_M_SB(mapping), &folio->page); #endif if (!folio_test_dirty(folio)) { filemap_dirty_folio(mapping, folio); - inc_page_count(F2FS_P_SB(&folio->page), F2FS_DIRTY_NODES); + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); set_page_private_reference(&folio->page); return true; } diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index f121c21590dc..ed1c9ed737f2 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -71,17 +71,6 @@ static inline void fscache_see_cookie(struct fscache_cookie *cookie, } /* - * io.c - */ -static inline void fscache_end_operation(struct netfs_cache_resources *cres) -{ - const struct netfs_cache_ops *ops = fscache_operation_valid(cres); - - if (ops) - ops->end_operation(cres); -} - -/* * main.c */ extern unsigned fscache_debug; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index eac4984cc753..488b460e046f 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -627,7 +627,7 @@ struct fuse_conn { /** Connection successful. Only set in INIT */ unsigned conn_init:1; - /** Do readpages asynchronously? Only set in INIT */ + /** Do readahead asynchronously? Only set in INIT */ unsigned async_read:1; /** Return an unique read error after abort. Only set in INIT */ diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index d67108489148..39080b2d6cf8 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -606,9 +606,9 @@ out: return ret; } -static inline __be64 *gfs2_indirect_init(struct metapath *mp, - struct gfs2_glock *gl, unsigned int i, - unsigned offset, u64 bn) +static inline void gfs2_indirect_init(struct metapath *mp, + struct gfs2_glock *gl, unsigned int i, + unsigned offset, u64 bn) { __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data + ((i > 1) ? sizeof(struct gfs2_meta_header) : @@ -621,7 +621,6 @@ static inline __be64 *gfs2_indirect_init(struct metapath *mp, gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header)); ptr += offset; *ptr = cpu_to_be64(bn); - return ptr; } enum alloc_state { @@ -2146,7 +2145,7 @@ int gfs2_setattr_size(struct inode *inode, u64 newsize) ret = do_shrink(inode, newsize); out: - gfs2_rs_delete(ip, NULL); + gfs2_rs_delete(ip); gfs2_qa_put(ip); return ret; } diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 8c39a8571b1f..22b41acfbbc3 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -706,7 +706,7 @@ static int gfs2_release(struct inode *inode, struct file *file) if (file->f_mode & FMODE_WRITE) { if (gfs2_rs_active(&ip->i_res)) - gfs2_rs_delete(ip, &inode->i_writecount); + gfs2_rs_delete(ip); gfs2_qa_put(ip); } return 0; @@ -775,8 +775,7 @@ static inline bool should_fault_in_pages(ssize_t ret, struct iov_iter *i, size_t *window_size) { size_t count = iov_iter_count(i); - char __user *p; - int pages = 1; + size_t size, offs; if (likely(!count)) return false; @@ -785,18 +784,20 @@ static inline bool should_fault_in_pages(ssize_t ret, struct iov_iter *i, if (!iter_is_iovec(i)) return false; + size = PAGE_SIZE; + offs = offset_in_page(i->iov[0].iov_base + i->iov_offset); if (*prev_count != count || !*window_size) { - int pages, nr_dirtied; + size_t nr_dirtied; - pages = min_t(int, BIO_MAX_VECS, DIV_ROUND_UP(count, PAGE_SIZE)); + size = ALIGN(offs + count, PAGE_SIZE); + size = min_t(size_t, size, SZ_1M); nr_dirtied = max(current->nr_dirtied_pause - - current->nr_dirtied, 1); - pages = min(pages, nr_dirtied); + current->nr_dirtied, 8); + size = min(size, nr_dirtied << PAGE_SHIFT); } *prev_count = count; - p = i->iov[0].iov_base + i->iov_offset; - *window_size = (size_t)PAGE_SIZE * pages - offset_in_page(p); + *window_size = size - offs; return true; } @@ -851,9 +852,9 @@ retry_under_glock: leftover = fault_in_iov_iter_writeable(to, window_size); gfs2_holder_disallow_demote(gh); if (leftover != window_size) { - if (!gfs2_holder_queued(gh)) - goto retry; - goto retry_under_glock; + if (gfs2_holder_queued(gh)) + goto retry_under_glock; + goto retry; } } if (gfs2_holder_queued(gh)) @@ -920,9 +921,9 @@ retry_under_glock: leftover = fault_in_iov_iter_readable(from, window_size); gfs2_holder_disallow_demote(gh); if (leftover != window_size) { - if (!gfs2_holder_queued(gh)) - goto retry; - goto retry_under_glock; + if (gfs2_holder_queued(gh)) + goto retry_under_glock; + goto retry; } } out: @@ -950,20 +951,19 @@ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) * and retry. */ - if (iocb->ki_flags & IOCB_DIRECT) { - ret = gfs2_file_direct_read(iocb, to, &gh); - if (likely(ret != -ENOTBLK)) - return ret; - iocb->ki_flags &= ~IOCB_DIRECT; - } + if (iocb->ki_flags & IOCB_DIRECT) + return gfs2_file_direct_read(iocb, to, &gh); + + pagefault_disable(); iocb->ki_flags |= IOCB_NOIO; ret = generic_file_read_iter(iocb, to); iocb->ki_flags &= ~IOCB_NOIO; + pagefault_enable(); if (ret >= 0) { if (!iov_iter_count(to)) return ret; written = ret; - } else { + } else if (ret != -EFAULT) { if (ret != -EAGAIN) return ret; if (iocb->ki_flags & IOCB_NOWAIT) @@ -989,12 +989,11 @@ retry_under_glock: leftover = fault_in_iov_iter_writeable(to, window_size); gfs2_holder_disallow_demote(&gh); if (leftover != window_size) { - if (!gfs2_holder_queued(&gh)) { - if (written) - goto out_uninit; - goto retry; - } - goto retry_under_glock; + if (gfs2_holder_queued(&gh)) + goto retry_under_glock; + if (written) + goto out_uninit; + goto retry; } } if (gfs2_holder_queued(&gh)) @@ -1068,12 +1067,11 @@ retry_under_glock: gfs2_holder_disallow_demote(gh); if (leftover != window_size) { from->count = min(from->count, window_size - leftover); - if (!gfs2_holder_queued(gh)) { - if (read) - goto out_uninit; - goto retry; - } - goto retry_under_glock; + if (gfs2_holder_queued(gh)) + goto retry_under_glock; + if (read && !(iocb->ki_flags & IOCB_DIRECT)) + goto out_uninit; + goto retry; } } out_unlock: @@ -1083,6 +1081,7 @@ out_uninit: gfs2_holder_uninit(gh); if (statfs_gh) kfree(statfs_gh); + from->count = orig_count - read; return read ? read : ret; } @@ -1497,7 +1496,6 @@ static int do_flock(struct file *file, int cmd, struct file_lock *fl) if (error != GLR_TRYFAILED) break; fl_gh->gh_flags = LM_FLAG_TRY | GL_EXACT; - fl_gh->gh_error = 0; msleep(sleeptime); } if (error) { diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 6b23399eaee0..630c6550eacf 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -542,7 +542,7 @@ restart: * some reason. If this holder is the head of the list, it * means we have a blocked holder at the head, so return 1. */ - if (gh->gh_list.prev == &gl->gl_holders) + if (list_is_first(&gh->gh_list, &gl->gl_holders)) return 1; do_error(gl, 0); break; @@ -669,6 +669,8 @@ static void finish_xmote(struct gfs2_glock *gl, unsigned int ret) /* Check for state != intended state */ if (unlikely(state != gl->gl_target)) { + if (gh && (ret & LM_OUT_CANCELED)) + gfs2_holder_wake(gh); if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) { /* move to back of queue and try next entry */ if (ret & LM_OUT_CANCELED) { @@ -1259,7 +1261,6 @@ void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags, gh->gh_owner_pid = get_pid(task_pid(current)); gh->gh_state = state; gh->gh_flags = flags; - gh->gh_error = 0; gh->gh_iflags = 0; gfs2_glock_hold(gl); } @@ -1565,6 +1566,7 @@ int gfs2_glock_nq(struct gfs2_holder *gh) if (test_bit(GLF_LRU, &gl->gl_flags)) gfs2_glock_remove_from_lru(gl); + gh->gh_error = 0; spin_lock(&gl->gl_lockref.lock); add_to_queue(gh); if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) && @@ -1691,6 +1693,14 @@ void gfs2_glock_dq(struct gfs2_holder *gh) struct gfs2_glock *gl = gh->gh_gl; spin_lock(&gl->gl_lockref.lock); + if (list_is_first(&gh->gh_list, &gl->gl_holders) && + !test_bit(HIF_HOLDER, &gh->gh_iflags)) { + spin_unlock(&gl->gl_lockref.lock); + gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl); + wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE); + spin_lock(&gl->gl_lockref.lock); + } + __gfs2_glock_dq(gh); spin_unlock(&gl->gl_lockref.lock); } diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 89905f4f29bb..c8ec876f33ea 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -131,7 +131,21 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, struct gfs2_sbd *sdp = GFS2_SB(inode); struct gfs2_glock *io_gl; - error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE, &ip->i_gl); + error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE, + &ip->i_gl); + if (unlikely(error)) + goto fail; + + error = gfs2_glock_get(sdp, no_addr, &gfs2_iopen_glops, CREATE, + &io_gl); + if (unlikely(error)) + goto fail; + + if (blktype != GFS2_BLKST_UNLINKED) + gfs2_cancel_delete_work(io_gl); + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, + &ip->i_iopen_gh); + gfs2_glock_put(io_gl); if (unlikely(error)) goto fail; @@ -161,16 +175,6 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, set_bit(GLF_INSTANTIATE_NEEDED, &ip->i_gl->gl_flags); - error = gfs2_glock_get(sdp, no_addr, &gfs2_iopen_glops, CREATE, &io_gl); - if (unlikely(error)) - goto fail; - if (blktype != GFS2_BLKST_UNLINKED) - gfs2_cancel_delete_work(io_gl); - error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, &ip->i_iopen_gh); - gfs2_glock_put(io_gl); - if (unlikely(error)) - goto fail; - /* Lowest possible timestamp; will be overwritten in gfs2_dinode_in. */ inode->i_atime.tv_sec = 1LL << (8 * sizeof(inode->i_atime.tv_sec) - 1); inode->i_atime.tv_nsec = 0; @@ -716,13 +720,17 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, error = insert_inode_locked4(inode, ip->i_no_addr, iget_test, &ip->i_no_addr); BUG_ON(error); - error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, ghs + 1); + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, &ip->i_iopen_gh); if (error) goto fail_gunlock2; + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, ghs + 1); + if (error) + goto fail_gunlock3; + error = gfs2_trans_begin(sdp, blocks, 0); if (error) - goto fail_gunlock2; + goto fail_gunlock3; if (blocks > 1) { ip->i_eattr = ip->i_no_addr + 1; @@ -731,10 +739,6 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, init_dinode(dip, ip, symname); gfs2_trans_end(sdp); - error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, &ip->i_iopen_gh); - if (error) - goto fail_gunlock2; - glock_set_object(ip->i_gl, ip); glock_set_object(io_gl, ip); gfs2_set_iop(inode); @@ -745,14 +749,14 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, if (default_acl) { error = __gfs2_set_acl(inode, default_acl, ACL_TYPE_DEFAULT); if (error) - goto fail_gunlock3; + goto fail_gunlock4; posix_acl_release(default_acl); default_acl = NULL; } if (acl) { error = __gfs2_set_acl(inode, acl, ACL_TYPE_ACCESS); if (error) - goto fail_gunlock3; + goto fail_gunlock4; posix_acl_release(acl); acl = NULL; } @@ -760,11 +764,11 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, error = security_inode_init_security(&ip->i_inode, &dip->i_inode, name, &gfs2_initxattrs, NULL); if (error) - goto fail_gunlock3; + goto fail_gunlock4; error = link_dinode(dip, name, ip, &da); if (error) - goto fail_gunlock3; + goto fail_gunlock4; mark_inode_dirty(inode); d_instantiate(dentry, inode); @@ -782,9 +786,10 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, unlock_new_inode(inode); return error; -fail_gunlock3: +fail_gunlock4: glock_clear_object(ip->i_gl, ip); glock_clear_object(io_gl, ip); +fail_gunlock3: gfs2_glock_dq_uninit(&ip->i_iopen_gh); fail_gunlock2: gfs2_glock_put(io_gl); @@ -793,7 +798,7 @@ fail_free_inode: if (free_vfs_inode) /* else evict will do the put for us */ gfs2_glock_put(ip->i_gl); } - gfs2_rs_delete(ip, NULL); + gfs2_rs_deltree(&ip->i_res); gfs2_qa_put(ip); fail_free_acls: posix_acl_release(default_acl); diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c index 50578f881e6d..2559a79cf14b 100644 --- a/fs/gfs2/lock_dlm.c +++ b/fs/gfs2/lock_dlm.c @@ -261,6 +261,7 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state, int req; u32 lkf; char strname[GDLM_STRNAME_BYTES] = ""; + int error; req = make_mode(gl->gl_name.ln_sbd, req_state); lkf = make_flags(gl, flags, req); @@ -279,8 +280,14 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state, * Submit the actual lock request. */ - return dlm_lock(ls->ls_dlm, req, &gl->gl_lksb, lkf, strname, +again: + error = dlm_lock(ls->ls_dlm, req, &gl->gl_lksb, lkf, strname, GDLM_STRNAME_BYTES - 1, 0, gdlm_ast, gl, gdlm_bast); + if (error == -EBUSY) { + msleep(20); + goto again; + } + return error; } static void gdlm_put_lock(struct gfs2_glock *gl) @@ -312,8 +319,14 @@ static void gdlm_put_lock(struct gfs2_glock *gl) return; } +again: error = dlm_unlock(ls->ls_dlm, gl->gl_lksb.sb_lkid, DLM_LKF_VALBLK, NULL, gl); + if (error == -EBUSY) { + msleep(20); + goto again; + } + if (error) { fs_err(sdp, "gdlm_unlock %x,%llx err=%d\n", gl->gl_name.ln_type, diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c index 0fb3c01bc557..801ad9f4f2be 100644 --- a/fs/gfs2/rgrp.c +++ b/fs/gfs2/rgrp.c @@ -680,13 +680,14 @@ void gfs2_rs_deltree(struct gfs2_blkreserv *rs) /** * gfs2_rs_delete - delete a multi-block reservation * @ip: The inode for this reservation - * @wcount: The inode's write count, or NULL * */ -void gfs2_rs_delete(struct gfs2_inode *ip, atomic_t *wcount) +void gfs2_rs_delete(struct gfs2_inode *ip) { + struct inode *inode = &ip->i_inode; + down_write(&ip->i_rw_mutex); - if ((wcount == NULL) || (atomic_read(wcount) <= 1)) + if (atomic_read(&inode->i_writecount) <= 1) gfs2_rs_deltree(&ip->i_res); up_write(&ip->i_rw_mutex); } @@ -922,15 +923,15 @@ static int read_rindex_entry(struct gfs2_inode *ip) spin_lock_init(&rgd->rd_rsspin); mutex_init(&rgd->rd_mutex); - error = compute_bitstructs(rgd); - if (error) - goto fail; - error = gfs2_glock_get(sdp, rgd->rd_addr, &gfs2_rgrp_glops, CREATE, &rgd->rd_gl); if (error) goto fail; + error = compute_bitstructs(rgd); + if (error) + goto fail_glock; + rgd->rd_rgl = (struct gfs2_rgrp_lvb *)rgd->rd_gl->gl_lksb.sb_lvbptr; rgd->rd_flags &= ~GFS2_RDF_PREFERRED; if (rgd->rd_data > sdp->sd_max_rg_data) @@ -944,6 +945,7 @@ static int read_rindex_entry(struct gfs2_inode *ip) } error = 0; /* someone else read in the rgrp; free it and ignore it */ +fail_glock: gfs2_glock_put(rgd->rd_gl); fail: @@ -1415,7 +1417,8 @@ int gfs2_fitrim(struct file *filp, void __user *argp) start = r.start >> bs_shift; end = start + (r.len >> bs_shift); - minlen = max_t(u64, r.minlen, + minlen = max_t(u64, r.minlen, sdp->sd_sb.sb_bsize); + minlen = max_t(u64, minlen, q->limits.discard_granularity) >> bs_shift; if (end <= start || minlen > sdp->sd_max_rg_data) diff --git a/fs/gfs2/rgrp.h b/fs/gfs2/rgrp.h index 3e2ca1fb4305..46dd94e9e085 100644 --- a/fs/gfs2/rgrp.h +++ b/fs/gfs2/rgrp.h @@ -45,7 +45,7 @@ extern int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *n, bool dinode, u64 *generation); extern void gfs2_rs_deltree(struct gfs2_blkreserv *rs); -extern void gfs2_rs_delete(struct gfs2_inode *ip, atomic_t *wcount); +extern void gfs2_rs_delete(struct gfs2_inode *ip); extern void __gfs2_free_blocks(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, u64 bstart, u32 blen, int meta); extern void gfs2_free_meta(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index cf9cf66522b3..bdb773e5c88f 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -1396,7 +1396,7 @@ out: truncate_inode_pages_final(&inode->i_data); if (ip->i_qadata) gfs2_assert_warn(sdp, ip->i_qadata->qa_ref == 0); - gfs2_rs_delete(ip, NULL); + gfs2_rs_deltree(&ip->i_res); gfs2_ordered_del_inode(ip); clear_inode(inode); gfs2_dir_hash_inval(ip); diff --git a/fs/internal.h b/fs/internal.h index fb2c2ea807d7..08503dc68d2b 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -74,7 +74,7 @@ int do_linkat(int olddfd, struct filename *old, int newdfd, * namespace.c */ extern struct vfsmount *lookup_mnt(const struct path *); -extern int finish_automount(struct vfsmount *, struct path *); +extern int finish_automount(struct vfsmount *, const struct path *); extern int sb_prepare_remount_readonly(struct super_block *); diff --git a/fs/io_uring.c b/fs/io_uring.c index b94d57c1b0e5..a8413f006417 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -611,6 +611,7 @@ struct io_sr_msg { int msg_flags; int bgid; size_t len; + size_t done_io; }; struct io_open { @@ -781,6 +782,7 @@ enum { REQ_F_SKIP_LINK_CQES_BIT, REQ_F_SINGLE_POLL_BIT, REQ_F_DOUBLE_POLL_BIT, + REQ_F_PARTIAL_IO_BIT, /* keep async read/write and isreg together and in order */ REQ_F_SUPPORT_NOWAIT_BIT, REQ_F_ISREG_BIT, @@ -843,6 +845,8 @@ enum { REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT), /* double poll may active */ REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT), + /* request has already done partial IO */ + REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT), }; struct async_poll { @@ -923,7 +927,6 @@ struct io_kiocb { struct io_wq_work_node comp_list; atomic_t refs; atomic_t poll_refs; - struct io_kiocb *link; struct io_task_work io_task_work; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; @@ -931,9 +934,11 @@ struct io_kiocb { struct async_poll *apoll; /* opcode allocated if it needs to store data for async defer */ void *async_data; - /* custom credentials, valid IFF REQ_F_CREDS is set */ /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ struct io_buffer *kbuf; + /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */ + struct io_kiocb *link; + /* custom credentials, valid IFF REQ_F_CREDS is set */ const struct cred *creds; struct io_wq_work work; }; @@ -962,6 +967,7 @@ struct io_op_def { /* set if opcode supports polled "wait" */ unsigned pollin : 1; unsigned pollout : 1; + unsigned poll_exclusive : 1; /* op supports buffer selection */ unsigned buffer_select : 1; /* do prep async if is going to be punted */ @@ -1056,6 +1062,7 @@ static const struct io_op_def io_op_defs[] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, + .poll_exclusive = 1, }, [IORING_OP_ASYNC_CANCEL] = { .audit_skip = 1, @@ -1330,6 +1337,8 @@ static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list) static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req) { + lockdep_assert_held(&req->ctx->completion_lock); + if (likely(!(req->flags & REQ_F_BUFFER_SELECTED))) return 0; return __io_put_kbuf(req, &req->ctx->io_buffers_comp); @@ -1362,6 +1371,8 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, cflags = __io_put_kbuf(req, &ctx->io_buffers_comp); spin_unlock(&ctx->completion_lock); } else { + lockdep_assert_held(&req->ctx->uring_lock); + cflags = __io_put_kbuf(req, &req->ctx->io_buffers_cache); } @@ -1382,7 +1393,7 @@ static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, return NULL; } -static void io_kbuf_recycle(struct io_kiocb *req) +static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; @@ -1390,6 +1401,12 @@ static void io_kbuf_recycle(struct io_kiocb *req) if (likely(!(req->flags & REQ_F_BUFFER_SELECTED))) return; + /* don't recycle if we already did IO to this buffer */ + if (req->flags & REQ_F_PARTIAL_IO) + return; + + if (issue_flags & IO_URING_F_UNLOCKED) + mutex_lock(&ctx->uring_lock); lockdep_assert_held(&ctx->uring_lock); @@ -1398,6 +1415,9 @@ static void io_kbuf_recycle(struct io_kiocb *req) list_add(&buf->list, &bl->buf_list); req->flags &= ~REQ_F_BUFFER_SELECTED; req->kbuf = NULL; + + if (issue_flags & IO_URING_F_UNLOCKED) + mutex_unlock(&ctx->uring_lock); } static bool io_match_task(struct io_kiocb *head, struct task_struct *task, @@ -2104,6 +2124,12 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, } } io_req_put_rsrc(req, ctx); + /* + * Selected buffer deallocation in io_clean_op() assumes that + * we don't hold ->completion_lock. Clean them here to avoid + * deadlocks. + */ + io_put_kbuf_comp(req); io_dismantle_req(req); io_put_task(req->task, 1); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); @@ -2148,7 +2174,7 @@ static inline void io_req_complete(struct io_kiocb *req, s32 res) static void io_req_complete_failed(struct io_kiocb *req, s32 res) { req_set_fail(req); - io_req_complete_post(req, res, io_put_kbuf(req, 0)); + io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); } static void io_req_complete_fail_submit(struct io_kiocb *req) @@ -2437,6 +2463,8 @@ static void handle_prev_tw_list(struct io_wq_work_node *node, struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); + prefetch(container_of(next, struct io_kiocb, io_task_work.node)); + if (req->ctx != *ctx) { if (unlikely(!*uring_locked && *ctx)) ctx_commit_and_unlock(*ctx); @@ -2469,6 +2497,8 @@ static void handle_tw_list(struct io_wq_work_node *node, struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); + prefetch(container_of(next, struct io_kiocb, io_task_work.node)); + if (req->ctx != *ctx) { ctx_flush_and_put(*ctx, locked); *ctx = req->ctx; @@ -2974,8 +3004,12 @@ static bool io_rw_should_reissue(struct io_kiocb *req) static bool __io_complete_rw_common(struct io_kiocb *req, long res) { - if (req->rw.kiocb.ki_flags & IOCB_WRITE) + if (req->rw.kiocb.ki_flags & IOCB_WRITE) { kiocb_end_write(req); + fsnotify_modify(req->file); + } else { + fsnotify_access(req->file); + } if (unlikely(res != req->result)) { if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) { @@ -4439,9 +4473,6 @@ static int io_msg_ring_prep(struct io_kiocb *req, sqe->splice_fd_in || sqe->buf_index || sqe->personality)) return -EINVAL; - if (req->file->f_op != &io_uring_fops) - return -EBADFD; - req->msg.user_data = READ_ONCE(sqe->off); req->msg.len = READ_ONCE(sqe->len); return 0; @@ -4451,14 +4482,18 @@ static int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *target_ctx; struct io_msg *msg = &req->msg; - int ret = -EOVERFLOW; bool filled; + int ret; + + ret = -EBADFD; + if (req->file->f_op != &io_uring_fops) + goto done; + ret = -EOVERFLOW; target_ctx = req->file->private_data; spin_lock(&target_ctx->completion_lock); - filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len, - IORING_CQE_F_MSG); + filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len, 0); io_commit_cqring(target_ctx); spin_unlock(&target_ctx->completion_lock); @@ -4467,6 +4502,9 @@ static int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) ret = 0; } +done: + if (ret < 0) + req_set_fail(req); __io_req_complete(req, issue_flags, ret, 0); return 0; } @@ -4537,6 +4575,8 @@ static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) req->sync.len); if (ret < 0) req_set_fail(req); + else + fsnotify_modify(req->file); io_req_complete(req, ret); return 0; } @@ -5419,12 +5459,21 @@ static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->ctx->compat) sr->msg_flags |= MSG_CMSG_COMPAT; #endif + sr->done_io = 0; return 0; } +static bool io_net_retry(struct socket *sock, int flags) +{ + if (!(flags & MSG_WAITALL)) + return false; + return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; +} + static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_msghdr iomsg, *kmsg; + struct io_sr_msg *sr = &req->sr_msg; struct socket *sock; struct io_buffer *kbuf; unsigned flags; @@ -5467,6 +5516,11 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) return io_setup_async_msg(req, kmsg); if (ret == -ERESTARTSYS) ret = -EINTR; + if (ret > 0 && io_net_retry(sock, flags)) { + sr->done_io += ret; + req->flags |= REQ_F_PARTIAL_IO; + return io_setup_async_msg(req, kmsg); + } req_set_fail(req); } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { req_set_fail(req); @@ -5476,6 +5530,10 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (kmsg->free_iov) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; + if (ret >= 0) + ret += sr->done_io; + else if (sr->done_io) + ret = sr->done_io; __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags)); return 0; } @@ -5526,12 +5584,23 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) return -EAGAIN; if (ret == -ERESTARTSYS) ret = -EINTR; + if (ret > 0 && io_net_retry(sock, flags)) { + sr->len -= ret; + sr->buf += ret; + sr->done_io += ret; + req->flags |= REQ_F_PARTIAL_IO; + return -EAGAIN; + } req_set_fail(req); } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { out_free: req_set_fail(req); } + if (ret >= 0) + ret += sr->done_io; + else if (sr->done_io) + ret = sr->done_io; __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags)); return 0; } @@ -5569,9 +5638,6 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) struct file *file; int ret, fd; - if (req->file->f_flags & O_NONBLOCK) - req->flags |= REQ_F_NOWAIT; - if (!fixed) { fd = __get_unused_fd_flags(accept->flags, accept->nofile); if (unlikely(fd < 0)) @@ -5801,7 +5867,7 @@ struct io_poll_table { }; #define IO_POLL_CANCEL_FLAG BIT(31) -#define IO_POLL_REF_MASK ((1u << 20)-1) +#define IO_POLL_REF_MASK GENMASK(30, 0) /* * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can @@ -6035,10 +6101,13 @@ static void io_poll_cancel_req(struct io_kiocb *req) io_poll_execute(req, 0, 0); } +#define wqe_to_req(wait) ((void *)((unsigned long) (wait)->private & ~1)) +#define wqe_is_double(wait) ((unsigned long) (wait)->private & 1) + static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, void *key) { - struct io_kiocb *req = wait->private; + struct io_kiocb *req = wqe_to_req(wait); struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, wait); __poll_t mask = key_to_poll(key); @@ -6076,7 +6145,10 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, if (mask && poll->events & EPOLLONESHOT) { list_del_init(&poll->wait.entry); poll->head = NULL; - req->flags &= ~REQ_F_SINGLE_POLL; + if (wqe_is_double(wait)) + req->flags &= ~REQ_F_DOUBLE_POLL; + else + req->flags &= ~REQ_F_SINGLE_POLL; } __io_poll_execute(req, mask, poll->events); } @@ -6088,6 +6160,7 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, struct io_poll_iocb **poll_ptr) { struct io_kiocb *req = pt->req; + unsigned long wqe_private = (unsigned long) req; /* * The file being polled uses multiple waitqueues for poll handling @@ -6113,6 +6186,8 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, pt->error = -ENOMEM; return; } + /* mark as double wq entry */ + wqe_private |= 1; req->flags |= REQ_F_DOUBLE_POLL; io_init_poll_iocb(poll, first->events, first->wait.func); *poll_ptr = poll; @@ -6123,7 +6198,7 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, req->flags |= REQ_F_SINGLE_POLL; pt->nr_entries++; poll->head = head; - poll->wait.private = req; + poll->wait.private = (void *) wqe_private; if (poll->events & EPOLLEXCLUSIVE) add_wait_queue_exclusive(head, &poll->wait); @@ -6150,7 +6225,6 @@ static int __io_arm_poll_handler(struct io_kiocb *req, INIT_HLIST_NODE(&req->hash_node); io_init_poll_iocb(poll, mask, io_poll_wake); poll->file = req->file; - poll->wait.private = req; ipt->pt._key = mask; ipt->req = req; @@ -6238,7 +6312,8 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) } else { mask |= POLLOUT | POLLWRNORM; } - + if (def->poll_exclusive) + mask |= EPOLLEXCLUSIVE; if (!(issue_flags & IO_URING_F_UNLOCKED) && !list_empty(&ctx->apoll_cache)) { apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, @@ -6254,6 +6329,8 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) req->flags |= REQ_F_POLLED; ipt.pt._qproc = io_async_queue_proc; + io_kbuf_recycle(req, issue_flags); + ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask); if (ret || ipt.error) return ret ? IO_APOLL_READY : IO_APOLL_ABORTED; @@ -6281,6 +6358,7 @@ static __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, list = &ctx->cancel_hash[i]; hlist_for_each_entry_safe(req, tmp, list, hash_node) { if (io_match_task_safe(req, tsk, cancel_all)) { + hlist_del_init(&req->hash_node); io_poll_cancel_req(req); found = true; } @@ -7075,8 +7153,11 @@ fail: static void io_clean_op(struct io_kiocb *req) { - if (req->flags & REQ_F_BUFFER_SELECTED) + if (req->flags & REQ_F_BUFFER_SELECTED) { + spin_lock(&req->ctx->completion_lock); io_put_kbuf_comp(req); + spin_unlock(&req->ctx->completion_lock); + } if (req->flags & REQ_F_NEED_CLEANUP) { switch (req->opcode) { @@ -7505,11 +7586,9 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) * Queued up for async execution, worker will release * submit reference when the iocb is actually submitted. */ - io_kbuf_recycle(req); io_queue_async_work(req, NULL); break; case IO_APOLL_OK: - io_kbuf_recycle(req); break; } @@ -8053,6 +8132,13 @@ static int io_sq_thread(void *data) needs_sched = false; break; } + + /* + * Ensure the store of the wakeup flag is not + * reordered with the load of the SQ tail + */ + smp_mb(); + if (io_sqring_entries(ctx)) { needs_sched = false; break; @@ -8782,6 +8868,7 @@ static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset) fput(fpl->fp[i]); } else { kfree_skb(skb); + free_uid(fpl->user); kfree(fpl); } diff --git a/fs/ioctl.c b/fs/ioctl.c index 090bf47606ab..80ac36aea913 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -173,7 +173,7 @@ int fiemap_prep(struct inode *inode, struct fiemap_extent_info *fieinfo, if (*len == 0) return -EINVAL; - if (start > maxbytes) + if (start >= maxbytes) return -EFBIG; /* diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 49dccd9050f1..8ce8720093b9 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -435,18 +435,17 @@ bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count) { struct iomap_page *iop = to_iomap_page(folio); struct inode *inode = folio->mapping->host; - size_t len; unsigned first, last, i; if (!iop) return false; - /* Limit range to this folio */ - len = min(folio_size(folio) - from, count); + /* Caller's range may extend past the end of this folio */ + count = min(folio_size(folio) - from, count); - /* First and last blocks in range within page */ + /* First and last blocks in range within folio */ first = from >> inode->i_blkbits; - last = (from + len - 1) >> inode->i_blkbits; + last = (from + count - 1) >> inode->i_blkbits; for (i = first; i <= last; i++) if (!test_bit(i, iop->uptodate)) diff --git a/fs/jffs2/build.c b/fs/jffs2/build.c index b288c8ae1236..837cd55fd4c5 100644 --- a/fs/jffs2/build.c +++ b/fs/jffs2/build.c @@ -415,13 +415,15 @@ int jffs2_do_mount_fs(struct jffs2_sb_info *c) jffs2_free_ino_caches(c); jffs2_free_raw_node_refs(c); ret = -EIO; - goto out_free; + goto out_sum_exit; } jffs2_calc_trigger_levels(c); return 0; + out_sum_exit: + jffs2_sum_exit(c); out_free: kvfree(c->blocks); diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c index 2ac410477c4f..71f03a5d36ed 100644 --- a/fs/jffs2/fs.c +++ b/fs/jffs2/fs.c @@ -603,8 +603,8 @@ out_root: jffs2_free_ino_caches(c); jffs2_free_raw_node_refs(c); kvfree(c->blocks); - out_inohash: jffs2_clear_xattr_subsystem(c); + out_inohash: kfree(c->inocache_list); out_wbuf: jffs2_flash_cleanup(c); diff --git a/fs/jffs2/jffs2_fs_i.h b/fs/jffs2/jffs2_fs_i.h index 2e4a86763c07..93a2951538ce 100644 --- a/fs/jffs2/jffs2_fs_i.h +++ b/fs/jffs2/jffs2_fs_i.h @@ -18,11 +18,11 @@ #include <linux/mutex.h> struct jffs2_inode_info { - /* We need an internal mutex similar to inode->i_mutex. + /* We need an internal mutex similar to inode->i_rwsem. Unfortunately, we can't used the existing one, because either the GC would deadlock, or we'd have to release it before letting GC proceed. Or we'd have to put ugliness - into the GC code so it didn't attempt to obtain the i_mutex + into the GC code so it didn't attempt to obtain the i_rwsem for the inode(s) which are already locked */ struct mutex sem; diff --git a/fs/jffs2/scan.c b/fs/jffs2/scan.c index b676056826be..29671e33a171 100644 --- a/fs/jffs2/scan.c +++ b/fs/jffs2/scan.c @@ -136,7 +136,7 @@ int jffs2_scan_medium(struct jffs2_sb_info *c) if (!s) { JFFS2_WARNING("Can't allocate memory for summary\n"); ret = -ENOMEM; - goto out; + goto out_buf; } } @@ -275,13 +275,15 @@ int jffs2_scan_medium(struct jffs2_sb_info *c) } ret = 0; out: + jffs2_sum_reset_collected(s); + kfree(s); + out_buf: if (buf_size) kfree(flashbuf); #ifndef __ECOS else mtd_unpoint(c->mtd, 0, c->mtd->size); #endif - kfree(s); return ret; } diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 74067a73ff78..88423069407c 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -120,13 +120,8 @@ static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos) if (next == ERR_PTR(-ENODEV)) kernfs_seq_stop_active(sf, next); return next; - } else { - /* - * The same behavior and code as single_open(). Returns - * !NULL if pos is at the beginning; otherwise, NULL. - */ - return NULL + !*ppos; } + return single_start(sf, ppos); } static void *kernfs_seq_next(struct seq_file *sf, void *v, loff_t *ppos) diff --git a/fs/ksmbd/oplock.c b/fs/ksmbd/oplock.c index 077b8761d099..23871b18a429 100644 --- a/fs/ksmbd/oplock.c +++ b/fs/ksmbd/oplock.c @@ -656,8 +656,8 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; rsp->Reserved = 0; rsp->Reserved2 = 0; - rsp->PersistentFid = cpu_to_le64(fp->persistent_id); - rsp->VolatileFid = cpu_to_le64(fp->volatile_id); + rsp->PersistentFid = fp->persistent_id; + rsp->VolatileFid = fp->volatile_id; inc_rfc1001_len(work->response_buf, 24); diff --git a/fs/ksmbd/server.c b/fs/ksmbd/server.c index 2e12f6d8483b..4cd03d661df0 100644 --- a/fs/ksmbd/server.c +++ b/fs/ksmbd/server.c @@ -585,7 +585,7 @@ static int __init ksmbd_server_init(void) if (ret) goto err_crypto_destroy; - pr_warn_once("The ksmbd server is experimental, use at your own risk.\n"); + pr_warn_once("The ksmbd server is experimental\n"); return 0; diff --git a/fs/ksmbd/smb2pdu.c b/fs/ksmbd/smb2pdu.c index 67e8e28e3fc3..3bf6c56c654c 100644 --- a/fs/ksmbd/smb2pdu.c +++ b/fs/ksmbd/smb2pdu.c @@ -377,12 +377,8 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) * command in the compound request */ if (req->Command == SMB2_CREATE && rsp->Status == STATUS_SUCCESS) { - work->compound_fid = - le64_to_cpu(((struct smb2_create_rsp *)rsp)-> - VolatileFileId); - work->compound_pfid = - le64_to_cpu(((struct smb2_create_rsp *)rsp)-> - PersistentFileId); + work->compound_fid = ((struct smb2_create_rsp *)rsp)->VolatileFileId; + work->compound_pfid = ((struct smb2_create_rsp *)rsp)->PersistentFileId; work->compound_sid = le64_to_cpu(rsp->SessionId); } @@ -2129,7 +2125,7 @@ static noinline int create_smb2_pipe(struct ksmbd_work *work) rsp->EndofFile = cpu_to_le64(0); rsp->FileAttributes = FILE_ATTRIBUTE_NORMAL_LE; rsp->Reserved2 = 0; - rsp->VolatileFileId = cpu_to_le64(id); + rsp->VolatileFileId = id; rsp->PersistentFileId = 0; rsp->CreateContextsOffset = 0; rsp->CreateContextsLength = 0; @@ -3157,8 +3153,8 @@ int smb2_open(struct ksmbd_work *work) rsp->Reserved2 = 0; - rsp->PersistentFileId = cpu_to_le64(fp->persistent_id); - rsp->VolatileFileId = cpu_to_le64(fp->volatile_id); + rsp->PersistentFileId = fp->persistent_id; + rsp->VolatileFileId = fp->volatile_id; rsp->CreateContextsOffset = 0; rsp->CreateContextsLength = 0; @@ -3865,9 +3861,7 @@ int smb2_query_dir(struct ksmbd_work *work) goto err_out2; } - dir_fp = ksmbd_lookup_fd_slow(work, - le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + dir_fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); if (!dir_fp) { rc = -EBADF; goto err_out2; @@ -4088,12 +4082,12 @@ static int smb2_get_info_file_pipe(struct ksmbd_session *sess, * Windows can sometime send query file info request on * pipe without opening it, checking error condition here */ - id = le64_to_cpu(req->VolatileFileId); + id = req->VolatileFileId; if (!ksmbd_session_rpc_method(sess, id)) return -ENOENT; ksmbd_debug(SMB, "FileInfoClass %u, FileId 0x%llx\n", - req->FileInfoClass, le64_to_cpu(req->VolatileFileId)); + req->FileInfoClass, req->VolatileFileId); switch (req->FileInfoClass) { case FILE_STANDARD_INFORMATION: @@ -4738,7 +4732,7 @@ static int smb2_get_info_file(struct ksmbd_work *work, } if (work->next_smb2_rcv_hdr_off) { - if (!has_file_id(le64_to_cpu(req->VolatileFileId))) { + if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; @@ -4747,8 +4741,8 @@ static int smb2_get_info_file(struct ksmbd_work *work, } if (!has_file_id(id)) { - id = le64_to_cpu(req->VolatileFileId); - pid = le64_to_cpu(req->PersistentFileId); + id = req->VolatileFileId; + pid = req->PersistentFileId; } fp = ksmbd_lookup_fd_slow(work, id, pid); @@ -5113,7 +5107,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work, } if (work->next_smb2_rcv_hdr_off) { - if (!has_file_id(le64_to_cpu(req->VolatileFileId))) { + if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; @@ -5122,8 +5116,8 @@ static int smb2_get_info_sec(struct ksmbd_work *work, } if (!has_file_id(id)) { - id = le64_to_cpu(req->VolatileFileId); - pid = le64_to_cpu(req->PersistentFileId); + id = req->VolatileFileId; + pid = req->PersistentFileId; } fp = ksmbd_lookup_fd_slow(work, id, pid); @@ -5221,7 +5215,7 @@ static noinline int smb2_close_pipe(struct ksmbd_work *work) struct smb2_close_req *req = smb2_get_msg(work->request_buf); struct smb2_close_rsp *rsp = smb2_get_msg(work->response_buf); - id = le64_to_cpu(req->VolatileFileId); + id = req->VolatileFileId; ksmbd_session_rpc_close(work->sess, id); rsp->StructureSize = cpu_to_le16(60); @@ -5280,7 +5274,7 @@ int smb2_close(struct ksmbd_work *work) } if (work->next_smb2_rcv_hdr_off && - !has_file_id(le64_to_cpu(req->VolatileFileId))) { + !has_file_id(req->VolatileFileId)) { if (!has_file_id(work->compound_fid)) { /* file already closed, return FILE_CLOSED */ ksmbd_debug(SMB, "file already closed\n"); @@ -5299,7 +5293,7 @@ int smb2_close(struct ksmbd_work *work) work->compound_pfid = KSMBD_NO_FID; } } else { - volatile_id = le64_to_cpu(req->VolatileFileId); + volatile_id = req->VolatileFileId; } ksmbd_debug(SMB, "volatile_id = %llu\n", volatile_id); @@ -5988,7 +5982,7 @@ int smb2_set_info(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); - if (!has_file_id(le64_to_cpu(req->VolatileFileId))) { + if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; @@ -6000,8 +5994,8 @@ int smb2_set_info(struct ksmbd_work *work) } if (!has_file_id(id)) { - id = le64_to_cpu(req->VolatileFileId); - pid = le64_to_cpu(req->PersistentFileId); + id = req->VolatileFileId; + pid = req->PersistentFileId; } fp = ksmbd_lookup_fd_slow(work, id, pid); @@ -6079,7 +6073,7 @@ static noinline int smb2_read_pipe(struct ksmbd_work *work) struct smb2_read_req *req = smb2_get_msg(work->request_buf); struct smb2_read_rsp *rsp = smb2_get_msg(work->response_buf); - id = le64_to_cpu(req->VolatileFileId); + id = req->VolatileFileId; inc_rfc1001_len(work->response_buf, 16); rpc_resp = ksmbd_rpc_read(work->sess, id); @@ -6215,8 +6209,7 @@ int smb2_read(struct ksmbd_work *work) goto out; } - fp = ksmbd_lookup_fd_slow(work, le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); if (!fp) { err = -ENOENT; goto out; @@ -6335,7 +6328,7 @@ static noinline int smb2_write_pipe(struct ksmbd_work *work) size_t length; length = le32_to_cpu(req->Length); - id = le64_to_cpu(req->VolatileFileId); + id = req->VolatileFileId; if (le16_to_cpu(req->DataOffset) == offsetof(struct smb2_write_req, Buffer)) { @@ -6471,8 +6464,7 @@ int smb2_write(struct ksmbd_work *work) goto out; } - fp = ksmbd_lookup_fd_slow(work, le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); if (!fp) { err = -ENOENT; goto out; @@ -6584,12 +6576,9 @@ int smb2_flush(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); - ksmbd_debug(SMB, "SMB2_FLUSH called for fid %llu\n", - le64_to_cpu(req->VolatileFileId)); + ksmbd_debug(SMB, "SMB2_FLUSH called for fid %llu\n", req->VolatileFileId); - err = ksmbd_vfs_fsync(work, - le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + err = ksmbd_vfs_fsync(work, req->VolatileFileId, req->PersistentFileId); if (err) goto out; @@ -6618,8 +6607,7 @@ int smb2_cancel(struct ksmbd_work *work) struct ksmbd_conn *conn = work->conn; struct smb2_hdr *hdr = smb2_get_msg(work->request_buf); struct smb2_hdr *chdr; - struct ksmbd_work *cancel_work = NULL; - int canceled = 0; + struct ksmbd_work *cancel_work = NULL, *iter; struct list_head *command_list; ksmbd_debug(SMB, "smb2 cancel called on mid %llu, async flags 0x%x\n", @@ -6629,11 +6617,11 @@ int smb2_cancel(struct ksmbd_work *work) command_list = &conn->async_requests; spin_lock(&conn->request_lock); - list_for_each_entry(cancel_work, command_list, + list_for_each_entry(iter, command_list, async_request_entry) { - chdr = smb2_get_msg(cancel_work->request_buf); + chdr = smb2_get_msg(iter->request_buf); - if (cancel_work->async_id != + if (iter->async_id != le64_to_cpu(hdr->Id.AsyncId)) continue; @@ -6641,7 +6629,7 @@ int smb2_cancel(struct ksmbd_work *work) "smb2 with AsyncId %llu cancelled command = 0x%x\n", le64_to_cpu(hdr->Id.AsyncId), le16_to_cpu(chdr->Command)); - canceled = 1; + cancel_work = iter; break; } spin_unlock(&conn->request_lock); @@ -6649,24 +6637,24 @@ int smb2_cancel(struct ksmbd_work *work) command_list = &conn->requests; spin_lock(&conn->request_lock); - list_for_each_entry(cancel_work, command_list, request_entry) { - chdr = smb2_get_msg(cancel_work->request_buf); + list_for_each_entry(iter, command_list, request_entry) { + chdr = smb2_get_msg(iter->request_buf); if (chdr->MessageId != hdr->MessageId || - cancel_work == work) + iter == work) continue; ksmbd_debug(SMB, "smb2 with mid %llu cancelled command = 0x%x\n", le64_to_cpu(hdr->MessageId), le16_to_cpu(chdr->Command)); - canceled = 1; + cancel_work = iter; break; } spin_unlock(&conn->request_lock); } - if (canceled) { + if (cancel_work) { cancel_work->state = KSMBD_WORK_CANCELLED; if (cancel_work->cancel_fn) cancel_work->cancel_fn(cancel_work->cancel_argv); @@ -6804,12 +6792,9 @@ int smb2_lock(struct ksmbd_work *work) int prior_lock = 0; ksmbd_debug(SMB, "Received lock request\n"); - fp = ksmbd_lookup_fd_slow(work, - le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); if (!fp) { - ksmbd_debug(SMB, "Invalid file id for lock : %llu\n", - le64_to_cpu(req->VolatileFileId)); + ksmbd_debug(SMB, "Invalid file id for lock : %llu\n", req->VolatileFileId); err = -ENOENT; goto out2; } @@ -7164,8 +7149,8 @@ static int fsctl_copychunk(struct ksmbd_work *work, ci_rsp = (struct copychunk_ioctl_rsp *)&rsp->Buffer[0]; - rsp->VolatileFileId = cpu_to_le64(volatile_id); - rsp->PersistentFileId = cpu_to_le64(persistent_id); + rsp->VolatileFileId = volatile_id; + rsp->PersistentFileId = persistent_id; ci_rsp->ChunksWritten = cpu_to_le32(ksmbd_server_side_copy_max_chunk_count()); ci_rsp->ChunkBytesWritten = @@ -7379,8 +7364,8 @@ ipv6_retry: if (nii_rsp) nii_rsp->Next = 0; - rsp->PersistentFileId = cpu_to_le64(SMB2_NO_FID); - rsp->VolatileFileId = cpu_to_le64(SMB2_NO_FID); + rsp->PersistentFileId = SMB2_NO_FID; + rsp->VolatileFileId = SMB2_NO_FID; return nbytes; } @@ -7547,9 +7532,7 @@ static int fsctl_request_resume_key(struct ksmbd_work *work, { struct ksmbd_file *fp; - fp = ksmbd_lookup_fd_slow(work, - le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId)); + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); if (!fp) return -ENOENT; @@ -7579,7 +7562,7 @@ int smb2_ioctl(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); - if (!has_file_id(le64_to_cpu(req->VolatileFileId))) { + if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); id = work->compound_fid; @@ -7590,14 +7573,14 @@ int smb2_ioctl(struct ksmbd_work *work) } if (!has_file_id(id)) - id = le64_to_cpu(req->VolatileFileId); + id = req->VolatileFileId; if (req->Flags != cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL)) { rsp->hdr.Status = STATUS_NOT_SUPPORTED; goto out; } - cnt_code = le32_to_cpu(req->CntCode); + cnt_code = le32_to_cpu(req->CtlCode); ret = smb2_calc_max_out_buf_len(work, 48, le32_to_cpu(req->MaxOutputResponse)); if (ret < 0) { @@ -7656,8 +7639,8 @@ int smb2_ioctl(struct ksmbd_work *work) goto out; nbytes = sizeof(struct validate_negotiate_info_rsp); - rsp->PersistentFileId = cpu_to_le64(SMB2_NO_FID); - rsp->VolatileFileId = cpu_to_le64(SMB2_NO_FID); + rsp->PersistentFileId = SMB2_NO_FID; + rsp->VolatileFileId = SMB2_NO_FID; break; case FSCTL_QUERY_NETWORK_INTERFACE_INFO: ret = fsctl_query_iface_info_ioctl(conn, rsp, out_buf_len); @@ -7703,10 +7686,10 @@ int smb2_ioctl(struct ksmbd_work *work) rsp->PersistentFileId = req->PersistentFileId; fsctl_copychunk(work, (struct copychunk_ioctl_req *)&req->Buffer[0], - le32_to_cpu(req->CntCode), + le32_to_cpu(req->CtlCode), le32_to_cpu(req->InputCount), - le64_to_cpu(req->VolatileFileId), - le64_to_cpu(req->PersistentFileId), + req->VolatileFileId, + req->PersistentFileId, rsp); break; case FSCTL_SET_SPARSE: @@ -7857,7 +7840,7 @@ dup_ext_out: goto out; } - rsp->CntCode = cpu_to_le32(cnt_code); + rsp->CtlCode = cpu_to_le32(cnt_code); rsp->InputCount = cpu_to_le32(0); rsp->InputOffset = cpu_to_le32(112); rsp->OutputOffset = cpu_to_le32(112); @@ -7903,8 +7886,8 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) char req_oplevel = 0, rsp_oplevel = 0; unsigned int oplock_change_type; - volatile_id = le64_to_cpu(req->VolatileFid); - persistent_id = le64_to_cpu(req->PersistentFid); + volatile_id = req->VolatileFid; + persistent_id = req->PersistentFid; req_oplevel = req->OplockLevel; ksmbd_debug(OPLOCK, "v_id %llu, p_id %llu request oplock level %d\n", volatile_id, persistent_id, req_oplevel); @@ -7999,8 +7982,8 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) rsp->OplockLevel = rsp_oplevel; rsp->Reserved = 0; rsp->Reserved2 = 0; - rsp->VolatileFid = cpu_to_le64(volatile_id); - rsp->PersistentFid = cpu_to_le64(persistent_id); + rsp->VolatileFid = volatile_id; + rsp->PersistentFid = persistent_id; inc_rfc1001_len(work->response_buf, 24); return; @@ -8500,7 +8483,7 @@ static void fill_transform_hdr(void *tr_buf, char *old_buf, __le16 cipher_type) struct smb2_hdr *hdr = smb2_get_msg(old_buf); unsigned int orig_len = get_rfc1002_len(old_buf); - memset(tr_buf, 0, sizeof(struct smb2_transform_hdr) + 4); + /* tr_buf must be cleared by the caller */ tr_hdr->ProtocolId = SMB2_TRANSFORM_PROTO_NUM; tr_hdr->OriginalMessageSize = cpu_to_le32(orig_len); tr_hdr->Flags = cpu_to_le16(TRANSFORM_FLAG_ENCRYPTED); diff --git a/fs/ksmbd/smb2pdu.h b/fs/ksmbd/smb2pdu.h index d49468426576..af455278d005 100644 --- a/fs/ksmbd/smb2pdu.h +++ b/fs/ksmbd/smb2pdu.h @@ -16,42 +16,13 @@ #define FILE_CREATED 0x00000002 #define FILE_OVERWRITTEN 0x00000003 -/* - * Size of the session key (crypto key encrypted with the password - */ -#define SMB2_NTLMV2_SESSKEY_SIZE 16 -#define SMB2_SIGNATURE_SIZE 16 -#define SMB2_HMACSHA256_SIZE 32 -#define SMB2_CMACAES_SIZE 16 -#define SMB3_GCM128_CRYPTKEY_SIZE 16 -#define SMB3_GCM256_CRYPTKEY_SIZE 32 - -/* - * Size of the smb3 encryption/decryption keys - */ -#define SMB3_ENC_DEC_KEY_SIZE 32 - -/* - * Size of the smb3 signing key - */ -#define SMB3_SIGN_KEY_SIZE 16 - -#define CIFS_CLIENT_CHALLENGE_SIZE 8 -#define SMB_SERVER_CHALLENGE_SIZE 8 - /* SMB2 Max Credits */ #define SMB2_MAX_CREDITS 8192 -/* Maximum buffer size value we can send with 1 credit */ -#define SMB2_MAX_BUFFER_SIZE 65536 - -#define NUMBER_OF_SMB2_COMMANDS 0x0013 - /* BB FIXME - analyze following length BB */ #define MAX_SMB2_HDR_SIZE 0x78 /* 4 len + 64 hdr + (2*24 wct) + 2 bct + 2 pad */ #define SMB21_DEFAULT_IOSIZE (1024 * 1024) -#define SMB3_DEFAULT_IOSIZE (4 * 1024 * 1024) #define SMB3_DEFAULT_TRANS_SIZE (1024 * 1024) #define SMB3_MIN_IOSIZE (64 * 1024) #define SMB3_MAX_IOSIZE (8 * 1024 * 1024) @@ -65,18 +36,6 @@ * */ -#define SMB2_ERROR_STRUCTURE_SIZE2 9 -#define SMB2_ERROR_STRUCTURE_SIZE2_LE cpu_to_le16(SMB2_ERROR_STRUCTURE_SIZE2) - -struct smb2_err_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; - __u8 ErrorContextCount; - __u8 Reserved; - __le32 ByteCount; /* even if zero, at least one byte follows */ - __u8 ErrorData[1]; /* variable length */ -} __packed; - struct preauth_integrity_info { /* PreAuth integrity Hash ID */ __le16 Preauth_HashId; @@ -116,8 +75,8 @@ struct create_durable_reconn_req { union { __u8 Reserved[16]; struct { - __le64 PersistentFileId; - __le64 VolatileFileId; + __u64 PersistentFileId; + __u64 VolatileFileId; } Fid; } Data; } __packed; @@ -126,8 +85,8 @@ struct create_durable_reconn_v2_req { struct create_context ccontext; __u8 Name[8]; struct { - __le64 PersistentFileId; - __le64 VolatileFileId; + __u64 PersistentFileId; + __u64 VolatileFileId; } Fid; __u8 CreateGuid[16]; __le32 Flags; @@ -161,13 +120,6 @@ struct create_alloc_size_req { __le64 AllocationSize; } __packed; -struct create_posix { - struct create_context ccontext; - __u8 Name[16]; - __le32 Mode; - __u32 Reserved; -} __packed; - struct create_durable_rsp { struct create_context ccontext; __u8 Name[8]; @@ -209,45 +161,6 @@ struct create_posix_rsp { u8 SidBuffer[40]; } __packed; -#define SMB2_LEASE_NONE_LE cpu_to_le32(0x00) -#define SMB2_LEASE_READ_CACHING_LE cpu_to_le32(0x01) -#define SMB2_LEASE_HANDLE_CACHING_LE cpu_to_le32(0x02) -#define SMB2_LEASE_WRITE_CACHING_LE cpu_to_le32(0x04) - -#define SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE cpu_to_le32(0x02) - -#define SMB2_LEASE_KEY_SIZE 16 - -struct lease_context { - __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; - __le32 LeaseState; - __le32 LeaseFlags; - __le64 LeaseDuration; -} __packed; - -struct lease_context_v2 { - __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; - __le32 LeaseState; - __le32 LeaseFlags; - __le64 LeaseDuration; - __u8 ParentLeaseKey[SMB2_LEASE_KEY_SIZE]; - __le16 Epoch; - __le16 Reserved; -} __packed; - -struct create_lease { - struct create_context ccontext; - __u8 Name[8]; - struct lease_context lcontext; -} __packed; - -struct create_lease_v2 { - struct create_context ccontext; - __u8 Name[8]; - struct lease_context_v2 lcontext; - __u8 Pad[4]; -} __packed; - struct smb2_buffer_desc_v1 { __le64 offset; __le32 token; @@ -256,63 +169,6 @@ struct smb2_buffer_desc_v1 { #define SMB2_0_IOCTL_IS_FSCTL 0x00000001 -struct duplicate_extents_to_file { - __u64 PersistentFileHandle; /* source file handle, opaque endianness */ - __u64 VolatileFileHandle; - __le64 SourceFileOffset; - __le64 TargetFileOffset; - __le64 ByteCount; /* Bytes to be copied */ -} __packed; - -struct smb2_ioctl_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 57 */ - __le16 Reserved; /* offset from start of SMB2 header to write data */ - __le32 CntCode; - __le64 PersistentFileId; - __le64 VolatileFileId; - __le32 InputOffset; /* Reserved MBZ */ - __le32 InputCount; - __le32 MaxInputResponse; - __le32 OutputOffset; - __le32 OutputCount; - __le32 MaxOutputResponse; - __le32 Flags; - __le32 Reserved2; - __u8 Buffer[1]; -} __packed; - -struct smb2_ioctl_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 49 */ - __le16 Reserved; /* offset from start of SMB2 header to write data */ - __le32 CntCode; - __le64 PersistentFileId; - __le64 VolatileFileId; - __le32 InputOffset; /* Reserved MBZ */ - __le32 InputCount; - __le32 OutputOffset; - __le32 OutputCount; - __le32 Flags; - __le32 Reserved2; - __u8 Buffer[1]; -} __packed; - -struct validate_negotiate_info_req { - __le32 Capabilities; - __u8 Guid[SMB2_CLIENT_GUID_SIZE]; - __le16 SecurityMode; - __le16 DialectCount; - __le16 Dialects[1]; /* dialect (someday maybe list) client asked for */ -} __packed; - -struct validate_negotiate_info_rsp { - __le32 Capabilities; - __u8 Guid[SMB2_CLIENT_GUID_SIZE]; - __le16 SecurityMode; - __le16 Dialect; /* Dialect in use for the connection */ -} __packed; - struct smb_sockaddr_in { __be16 Port; __be32 IPv4address; @@ -357,7 +213,7 @@ struct file_object_buf_type1_ioctl_rsp { } __packed; struct resume_key_ioctl_rsp { - __le64 ResumeKey[3]; + __u64 ResumeKey[3]; __le32 ContextLength; __u8 Context[4]; /* ignored, Windows sets to 4 bytes of zero */ } __packed; @@ -386,167 +242,6 @@ struct file_sparse { __u8 SetSparse; } __packed; -struct file_zero_data_information { - __le64 FileOffset; - __le64 BeyondFinalZero; -} __packed; - -struct file_allocated_range_buffer { - __le64 file_offset; - __le64 length; -} __packed; - -struct reparse_data_buffer { - __le32 ReparseTag; - __le16 ReparseDataLength; - __u16 Reserved; - __u8 DataBuffer[]; /* Variable Length */ -} __packed; - -/* SMB2 Notify Action Flags */ -#define FILE_ACTION_ADDED 0x00000001 -#define FILE_ACTION_REMOVED 0x00000002 -#define FILE_ACTION_MODIFIED 0x00000003 -#define FILE_ACTION_RENAMED_OLD_NAME 0x00000004 -#define FILE_ACTION_RENAMED_NEW_NAME 0x00000005 -#define FILE_ACTION_ADDED_STREAM 0x00000006 -#define FILE_ACTION_REMOVED_STREAM 0x00000007 -#define FILE_ACTION_MODIFIED_STREAM 0x00000008 -#define FILE_ACTION_REMOVED_BY_DELETE 0x00000009 - -#define SMB2_LOCKFLAG_SHARED 0x0001 -#define SMB2_LOCKFLAG_EXCLUSIVE 0x0002 -#define SMB2_LOCKFLAG_UNLOCK 0x0004 -#define SMB2_LOCKFLAG_FAIL_IMMEDIATELY 0x0010 -#define SMB2_LOCKFLAG_MASK 0x0007 - -struct smb2_lock_element { - __le64 Offset; - __le64 Length; - __le32 Flags; - __le32 Reserved; -} __packed; - -struct smb2_lock_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 48 */ - __le16 LockCount; - __le32 Reserved; - __le64 PersistentFileId; - __le64 VolatileFileId; - /* Followed by at least one */ - struct smb2_lock_element locks[1]; -} __packed; - -struct smb2_lock_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __le16 Reserved; -} __packed; - -struct smb2_echo_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __u16 Reserved; -} __packed; - -struct smb2_echo_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 4 */ - __u16 Reserved; -} __packed; - -/* search (query_directory) Flags field */ -#define SMB2_RESTART_SCANS 0x01 -#define SMB2_RETURN_SINGLE_ENTRY 0x02 -#define SMB2_INDEX_SPECIFIED 0x04 -#define SMB2_REOPEN 0x10 - -struct smb2_query_directory_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 33 */ - __u8 FileInformationClass; - __u8 Flags; - __le32 FileIndex; - __le64 PersistentFileId; - __le64 VolatileFileId; - __le16 FileNameOffset; - __le16 FileNameLength; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -struct smb2_query_directory_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 9 */ - __le16 OutputBufferOffset; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -/* Possible InfoType values */ -#define SMB2_O_INFO_FILE 0x01 -#define SMB2_O_INFO_FILESYSTEM 0x02 -#define SMB2_O_INFO_SECURITY 0x03 -#define SMB2_O_INFO_QUOTA 0x04 - -/* Security info type additionalinfo flags. See MS-SMB2 (2.2.37) or MS-DTYP */ -#define OWNER_SECINFO 0x00000001 -#define GROUP_SECINFO 0x00000002 -#define DACL_SECINFO 0x00000004 -#define SACL_SECINFO 0x00000008 -#define LABEL_SECINFO 0x00000010 -#define ATTRIBUTE_SECINFO 0x00000020 -#define SCOPE_SECINFO 0x00000040 -#define BACKUP_SECINFO 0x00010000 -#define UNPROTECTED_SACL_SECINFO 0x10000000 -#define UNPROTECTED_DACL_SECINFO 0x20000000 -#define PROTECTED_SACL_SECINFO 0x40000000 -#define PROTECTED_DACL_SECINFO 0x80000000 - -struct smb2_query_info_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 41 */ - __u8 InfoType; - __u8 FileInfoClass; - __le32 OutputBufferLength; - __le16 InputBufferOffset; - __u16 Reserved; - __le32 InputBufferLength; - __le32 AdditionalInformation; - __le32 Flags; - __le64 PersistentFileId; - __le64 VolatileFileId; - __u8 Buffer[1]; -} __packed; - -struct smb2_query_info_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 9 */ - __le16 OutputBufferOffset; - __le32 OutputBufferLength; - __u8 Buffer[1]; -} __packed; - -struct smb2_set_info_req { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 33 */ - __u8 InfoType; - __u8 FileInfoClass; - __le32 BufferLength; - __le16 BufferOffset; - __u16 Reserved; - __le32 AdditionalInformation; - __le64 PersistentFileId; - __le64 VolatileFileId; - __u8 Buffer[1]; -} __packed; - -struct smb2_set_info_rsp { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 2 */ -} __packed; - /* FILE Info response size */ #define FILE_DIRECTORY_INFORMATION_SIZE 1 #define FILE_FULL_DIRECTORY_INFORMATION_SIZE 2 @@ -602,145 +297,11 @@ struct fs_type_info { long magic_number; } __packed; -struct smb2_oplock_break { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 24 */ - __u8 OplockLevel; - __u8 Reserved; - __le32 Reserved2; - __le64 PersistentFid; - __le64 VolatileFid; -} __packed; - -#define SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED cpu_to_le32(0x01) - -struct smb2_lease_break { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 44 */ - __le16 Epoch; - __le32 Flags; - __u8 LeaseKey[16]; - __le32 CurrentLeaseState; - __le32 NewLeaseState; - __le32 BreakReason; - __le32 AccessMaskHint; - __le32 ShareMaskHint; -} __packed; - -struct smb2_lease_ack { - struct smb2_hdr hdr; - __le16 StructureSize; /* Must be 36 */ - __le16 Reserved; - __le32 Flags; - __u8 LeaseKey[16]; - __le32 LeaseState; - __le64 LeaseDuration; -} __packed; - /* - * PDU infolevel structure definitions + * PDU query infolevel structure definitions * BB consider moving to a different header */ -/* File System Information Classes */ -#define FS_VOLUME_INFORMATION 1 /* Query */ -#define FS_LABEL_INFORMATION 2 /* Set */ -#define FS_SIZE_INFORMATION 3 /* Query */ -#define FS_DEVICE_INFORMATION 4 /* Query */ -#define FS_ATTRIBUTE_INFORMATION 5 /* Query */ -#define FS_CONTROL_INFORMATION 6 /* Query, Set */ -#define FS_FULL_SIZE_INFORMATION 7 /* Query */ -#define FS_OBJECT_ID_INFORMATION 8 /* Query, Set */ -#define FS_DRIVER_PATH_INFORMATION 9 /* Query */ -#define FS_SECTOR_SIZE_INFORMATION 11 /* SMB3 or later. Query */ -#define FS_POSIX_INFORMATION 100 /* SMB3.1.1 POSIX. Query */ - -struct smb2_fs_full_size_info { - __le64 TotalAllocationUnits; - __le64 CallerAvailableAllocationUnits; - __le64 ActualAvailableAllocationUnits; - __le32 SectorsPerAllocationUnit; - __le32 BytesPerSector; -} __packed; - -#define SSINFO_FLAGS_ALIGNED_DEVICE 0x00000001 -#define SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE 0x00000002 -#define SSINFO_FLAGS_NO_SEEK_PENALTY 0x00000004 -#define SSINFO_FLAGS_TRIM_ENABLED 0x00000008 - -/* sector size info struct */ -struct smb3_fs_ss_info { - __le32 LogicalBytesPerSector; - __le32 PhysicalBytesPerSectorForAtomicity; - __le32 PhysicalBytesPerSectorForPerf; - __le32 FSEffPhysicalBytesPerSectorForAtomicity; - __le32 Flags; - __le32 ByteOffsetForSectorAlignment; - __le32 ByteOffsetForPartitionAlignment; -} __packed; - -/* File System Control Information */ -struct smb2_fs_control_info { - __le64 FreeSpaceStartFiltering; - __le64 FreeSpaceThreshold; - __le64 FreeSpaceStopFiltering; - __le64 DefaultQuotaThreshold; - __le64 DefaultQuotaLimit; - __le32 FileSystemControlFlags; - __le32 Padding; -} __packed; - -/* partial list of QUERY INFO levels */ -#define FILE_DIRECTORY_INFORMATION 1 -#define FILE_FULL_DIRECTORY_INFORMATION 2 -#define FILE_BOTH_DIRECTORY_INFORMATION 3 -#define FILE_BASIC_INFORMATION 4 -#define FILE_STANDARD_INFORMATION 5 -#define FILE_INTERNAL_INFORMATION 6 -#define FILE_EA_INFORMATION 7 -#define FILE_ACCESS_INFORMATION 8 -#define FILE_NAME_INFORMATION 9 -#define FILE_RENAME_INFORMATION 10 -#define FILE_LINK_INFORMATION 11 -#define FILE_NAMES_INFORMATION 12 -#define FILE_DISPOSITION_INFORMATION 13 -#define FILE_POSITION_INFORMATION 14 -#define FILE_FULL_EA_INFORMATION 15 -#define FILE_MODE_INFORMATION 16 -#define FILE_ALIGNMENT_INFORMATION 17 -#define FILE_ALL_INFORMATION 18 -#define FILE_ALLOCATION_INFORMATION 19 -#define FILE_END_OF_FILE_INFORMATION 20 -#define FILE_ALTERNATE_NAME_INFORMATION 21 -#define FILE_STREAM_INFORMATION 22 -#define FILE_PIPE_INFORMATION 23 -#define FILE_PIPE_LOCAL_INFORMATION 24 -#define FILE_PIPE_REMOTE_INFORMATION 25 -#define FILE_MAILSLOT_QUERY_INFORMATION 26 -#define FILE_MAILSLOT_SET_INFORMATION 27 -#define FILE_COMPRESSION_INFORMATION 28 -#define FILE_OBJECT_ID_INFORMATION 29 -/* Number 30 not defined in documents */ -#define FILE_MOVE_CLUSTER_INFORMATION 31 -#define FILE_QUOTA_INFORMATION 32 -#define FILE_REPARSE_POINT_INFORMATION 33 -#define FILE_NETWORK_OPEN_INFORMATION 34 -#define FILE_ATTRIBUTE_TAG_INFORMATION 35 -#define FILE_TRACKING_INFORMATION 36 -#define FILEID_BOTH_DIRECTORY_INFORMATION 37 -#define FILEID_FULL_DIRECTORY_INFORMATION 38 -#define FILE_VALID_DATA_LENGTH_INFORMATION 39 -#define FILE_SHORT_NAME_INFORMATION 40 -#define FILE_SFIO_RESERVE_INFORMATION 44 -#define FILE_SFIO_VOLUME_INFORMATION 45 -#define FILE_HARD_LINK_INFORMATION 46 -#define FILE_NORMALIZED_NAME_INFORMATION 48 -#define FILEID_GLOBAL_TX_DIRECTORY_INFORMATION 50 -#define FILE_STANDARD_LINK_INFORMATION 54 - -#define OP_BREAK_STRUCT_SIZE_20 24 -#define OP_BREAK_STRUCT_SIZE_21 36 - struct smb2_file_access_info { __le32 AccessFlags; } __packed; @@ -749,56 +310,6 @@ struct smb2_file_alignment_info { __le32 AlignmentRequirement; } __packed; -struct smb2_file_internal_info { - __le64 IndexNumber; -} __packed; /* level 6 Query */ - -struct smb2_file_rename_info { /* encoding of request for level 10 */ - __u8 ReplaceIfExists; /* 1 = replace existing target with new */ - /* 0 = fail if target already exists */ - __u8 Reserved[7]; - __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ - __le32 FileNameLength; - char FileName[]; /* New name to be assigned */ -} __packed; /* level 10 Set */ - -struct smb2_file_link_info { /* encoding of request for level 11 */ - __u8 ReplaceIfExists; /* 1 = replace existing link with new */ - /* 0 = fail if link already exists */ - __u8 Reserved[7]; - __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ - __le32 FileNameLength; - char FileName[]; /* Name to be assigned to new link */ -} __packed; /* level 11 Set */ - -/* - * This level 18, although with struct with same name is different from cifs - * level 0x107. Level 0x107 has an extra u64 between AccessFlags and - * CurrentByteOffset. - */ -struct smb2_file_all_info { /* data block encoding of response to level 18 */ - __le64 CreationTime; /* Beginning of FILE_BASIC_INFO equivalent */ - __le64 LastAccessTime; - __le64 LastWriteTime; - __le64 ChangeTime; - __le32 Attributes; - __u32 Pad1; /* End of FILE_BASIC_INFO_INFO equivalent */ - __le64 AllocationSize; /* Beginning of FILE_STANDARD_INFO equivalent */ - __le64 EndOfFile; /* size ie offset to first free byte in file */ - __le32 NumberOfLinks; /* hard links */ - __u8 DeletePending; - __u8 Directory; - __u16 Pad2; /* End of FILE_STANDARD_INFO equivalent */ - __le64 IndexNumber; - __le32 EASize; - __le32 AccessFlags; - __le64 CurrentByteOffset; - __le32 Mode; - __le32 AlignmentRequirement; - __le32 FileNameLength; - char FileName[1]; -} __packed; /* level 18 Query */ - struct smb2_file_basic_info { /* data block encoding of response to level 18 */ __le64 CreationTime; /* Beginning of FILE_BASIC_INFO equivalent */ __le64 LastAccessTime; @@ -821,10 +332,6 @@ struct smb2_file_stream_info { char StreamName[]; } __packed; -struct smb2_file_eof_info { /* encoding of request for level 10 */ - __le64 EndOfFile; /* new end of file value */ -} __packed; /* level 20 Set */ - struct smb2_file_ntwrk_info { __le64 CreationTime; __le64 LastAccessTime; @@ -915,34 +422,6 @@ struct create_sd_buf_req { struct smb_ntsd ntsd; } __packed; -/* Find File infolevels */ -#define SMB_FIND_FILE_POSIX_INFO 0x064 - -/* Level 100 query info */ -struct smb311_posix_qinfo { - __le64 CreationTime; - __le64 LastAccessTime; - __le64 LastWriteTime; - __le64 ChangeTime; - __le64 EndOfFile; - __le64 AllocationSize; - __le32 DosAttributes; - __le64 Inode; - __le32 DeviceId; - __le32 Zero; - /* beginning of POSIX Create Context Response */ - __le32 HardLinks; - __le32 ReparseTag; - __le32 Mode; - u8 Sids[]; - /* - * var sized owner SID - * var sized group SID - * le32 filenamelength - * u8 filename[] - */ -} __packed; - struct smb2_posix_info { __le32 NextEntryOffset; __u32 Ignored; diff --git a/fs/ksmbd/transport_tcp.c b/fs/ksmbd/transport_tcp.c index 82a1429bbe12..8fef9de787d3 100644 --- a/fs/ksmbd/transport_tcp.c +++ b/fs/ksmbd/transport_tcp.c @@ -476,7 +476,7 @@ static int ksmbd_netdev_event(struct notifier_block *nb, unsigned long event, switch (event) { case NETDEV_UP: - if (netdev->priv_flags & IFF_BRIDGE_PORT) + if (netif_is_bridge_port(netdev)) return NOTIFY_OK; list_for_each_entry(iface, &iface_list, entry) { @@ -585,7 +585,7 @@ int ksmbd_tcp_set_interfaces(char *ifc_list, int ifc_list_sz) rtnl_lock(); for_each_netdev(&init_net, netdev) { - if (netdev->priv_flags & IFF_BRIDGE_PORT) + if (netif_is_bridge_port(netdev)) continue; if (!alloc_iface(kstrdup(netdev->name, GFP_KERNEL))) return -ENOMEM; diff --git a/fs/namespace.c b/fs/namespace.c index 6e9844b8c6fb..a0a36bfa3aa0 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -2112,22 +2112,23 @@ static int invent_group_ids(struct mount *mnt, bool recurse) int count_mounts(struct mnt_namespace *ns, struct mount *mnt) { unsigned int max = READ_ONCE(sysctl_mount_max); - unsigned int mounts = 0, old, pending, sum; + unsigned int mounts = 0; struct mount *p; + if (ns->mounts >= max) + return -ENOSPC; + max -= ns->mounts; + if (ns->pending_mounts >= max) + return -ENOSPC; + max -= ns->pending_mounts; + for (p = mnt; p; p = next_mnt(p, mnt)) mounts++; - old = ns->mounts; - pending = ns->pending_mounts; - sum = old + pending; - if ((old > sum) || - (pending > sum) || - (max < sum) || - (mounts > (max - sum))) + if (mounts > max) return -ENOSPC; - ns->pending_mounts = pending + mounts; + ns->pending_mounts += mounts; return 0; } @@ -2921,7 +2922,7 @@ static int do_move_mount_old(struct path *path, const char *old_name) * add a mount into a namespace's mount tree */ static int do_add_mount(struct mount *newmnt, struct mountpoint *mp, - struct path *path, int mnt_flags) + const struct path *path, int mnt_flags) { struct mount *parent = real_mount(path->mnt); @@ -3044,7 +3045,7 @@ static int do_new_mount(struct path *path, const char *fstype, int sb_flags, return err; } -int finish_automount(struct vfsmount *m, struct path *path) +int finish_automount(struct vfsmount *m, const struct path *path) { struct dentry *dentry = path->dentry; struct mountpoint *mp; diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile index c15bfc966d96..f684c0cd1ec5 100644 --- a/fs/netfs/Makefile +++ b/fs/netfs/Makefile @@ -1,5 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 -netfs-y := read_helper.o stats.o +netfs-y := \ + buffered_read.o \ + io.o \ + main.o \ + objects.o + +netfs-$(CONFIG_NETFS_STATS) += stats.o obj-$(CONFIG_NETFS_SUPPORT) := netfs.o diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c new file mode 100644 index 000000000000..281a88a5b8dc --- /dev/null +++ b/fs/netfs/buffered_read.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Network filesystem high-level buffered read support. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/export.h> +#include <linux/task_io_accounting_ops.h> +#include "internal.h" + +/* + * Unlock the folios in a read operation. We need to set PG_fscache on any + * folios we're going to write back before we unlock them. + */ +void netfs_rreq_unlock_folios(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + struct folio *folio; + unsigned int iopos, account = 0; + pgoff_t start_page = rreq->start / PAGE_SIZE; + pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; + bool subreq_failed = false; + + XA_STATE(xas, &rreq->mapping->i_pages, start_page); + + if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) { + __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags); + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { + __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); + } + } + + /* Walk through the pagecache and the I/O request lists simultaneously. + * We may have a mixture of cached and uncached sections and we only + * really want to write out the uncached sections. This is slightly + * complicated by the possibility that we might have huge pages with a + * mixture inside. + */ + subreq = list_first_entry(&rreq->subrequests, + struct netfs_io_subrequest, rreq_link); + iopos = 0; + subreq_failed = (subreq->error < 0); + + trace_netfs_rreq(rreq, netfs_rreq_trace_unlock); + + rcu_read_lock(); + xas_for_each(&xas, folio, last_page) { + unsigned int pgpos = (folio_index(folio) - start_page) * PAGE_SIZE; + unsigned int pgend = pgpos + folio_size(folio); + bool pg_failed = false; + + for (;;) { + if (!subreq) { + pg_failed = true; + break; + } + if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) + folio_start_fscache(folio); + pg_failed |= subreq_failed; + if (pgend < iopos + subreq->len) + break; + + account += subreq->transferred; + iopos += subreq->len; + if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { + subreq = list_next_entry(subreq, rreq_link); + subreq_failed = (subreq->error < 0); + } else { + subreq = NULL; + subreq_failed = false; + } + if (pgend == iopos) + break; + } + + if (!pg_failed) { + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + } + + if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { + if (folio_index(folio) == rreq->no_unlock_folio && + test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) + _debug("no unlock"); + else + folio_unlock(folio); + } + } + rcu_read_unlock(); + + task_io_account_read(account); + if (rreq->netfs_ops->done) + rreq->netfs_ops->done(rreq); +} + +static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, + loff_t *_start, size_t *_len, loff_t i_size) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + if (cres->ops && cres->ops->expand_readahead) + cres->ops->expand_readahead(cres, _start, _len, i_size); +} + +static void netfs_rreq_expand(struct netfs_io_request *rreq, + struct readahead_control *ractl) +{ + /* Give the cache a chance to change the request parameters. The + * resultant request must contain the original region. + */ + netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); + + /* Give the netfs a chance to change the request parameters. The + * resultant request must contain the original region. + */ + if (rreq->netfs_ops->expand_readahead) + rreq->netfs_ops->expand_readahead(rreq); + + /* Expand the request if the cache wants it to start earlier. Note + * that the expansion may get further extended if the VM wishes to + * insert THPs and the preferred start and/or end wind up in the middle + * of THPs. + * + * If this is the case, however, the THP size should be an integer + * multiple of the cache granule size, so we get a whole number of + * granules to deal with. + */ + if (rreq->start != readahead_pos(ractl) || + rreq->len != readahead_length(ractl)) { + readahead_expand(ractl, rreq->start, rreq->len); + rreq->start = readahead_pos(ractl); + rreq->len = readahead_length(ractl); + + trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), + netfs_read_trace_expanded); + } +} + +/** + * netfs_readahead - Helper to manage a read request + * @ractl: The description of the readahead request + * + * Fulfil a readahead request by drawing data from the cache if possible, or + * the netfs if not. Space beyond the EOF is zero-filled. Multiple I/O + * requests from different sources will get munged together. If necessary, the + * readahead window can be expanded in either direction to a more convenient + * alighment for RPC efficiency or to make storage in the cache feasible. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + */ +void netfs_readahead(struct readahead_control *ractl) +{ + struct netfs_io_request *rreq; + struct netfs_i_context *ctx = netfs_i_context(ractl->mapping->host); + int ret; + + _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl)); + + if (readahead_count(ractl) == 0) + return; + + rreq = netfs_alloc_request(ractl->mapping, ractl->file, + readahead_pos(ractl), + readahead_length(ractl), + NETFS_READAHEAD); + if (IS_ERR(rreq)) + return; + + if (ctx->ops->begin_cache_operation) { + ret = ctx->ops->begin_cache_operation(rreq); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto cleanup_free; + } + + netfs_stat(&netfs_n_rh_readahead); + trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), + netfs_read_trace_readahead); + + netfs_rreq_expand(rreq, ractl); + + /* Drop the refs on the folios here rather than in the cache or + * filesystem. The locks will be dropped in netfs_rreq_unlock(). + */ + while (readahead_folio(ractl)) + ; + + netfs_begin_read(rreq, false); + return; + +cleanup_free: + netfs_put_request(rreq, false, netfs_rreq_trace_put_failed); + return; +} +EXPORT_SYMBOL(netfs_readahead); + +/** + * netfs_readpage - Helper to manage a readpage request + * @file: The file to read from + * @subpage: A subpage of the folio to read + * + * Fulfil a readpage request by drawing data from the cache if possible, or the + * netfs if not. Space beyond the EOF is zero-filled. Multiple I/O requests + * from different sources will get munged together. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + */ +int netfs_readpage(struct file *file, struct page *subpage) +{ + struct folio *folio = page_folio(subpage); + struct address_space *mapping = folio_file_mapping(folio); + struct netfs_io_request *rreq; + struct netfs_i_context *ctx = netfs_i_context(mapping->host); + int ret; + + _enter("%lx", folio_index(folio)); + + rreq = netfs_alloc_request(mapping, file, + folio_file_pos(folio), folio_size(folio), + NETFS_READPAGE); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto alloc_error; + } + + if (ctx->ops->begin_cache_operation) { + ret = ctx->ops->begin_cache_operation(rreq); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto discard; + } + + netfs_stat(&netfs_n_rh_readpage); + trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); + return netfs_begin_read(rreq, true); + +discard: + netfs_put_request(rreq, false, netfs_rreq_trace_put_discard); +alloc_error: + folio_unlock(folio); + return ret; +} +EXPORT_SYMBOL(netfs_readpage); + +/* + * Prepare a folio for writing without reading first + * @folio: The folio being prepared + * @pos: starting position for the write + * @len: length of write + * @always_fill: T if the folio should always be completely filled/cleared + * + * In some cases, write_begin doesn't need to read at all: + * - full folio write + * - write that lies in a folio that is completely beyond EOF + * - write that covers the folio from start to EOF or beyond it + * + * If any of these criteria are met, then zero out the unwritten parts + * of the folio and return true. Otherwise, return false. + */ +static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len, + bool always_fill) +{ + struct inode *inode = folio_inode(folio); + loff_t i_size = i_size_read(inode); + size_t offset = offset_in_folio(folio, pos); + size_t plen = folio_size(folio); + + if (unlikely(always_fill)) { + if (pos - offset + len <= i_size) + return false; /* Page entirely before EOF */ + zero_user_segment(&folio->page, 0, plen); + folio_mark_uptodate(folio); + return true; + } + + /* Full folio write */ + if (offset == 0 && len >= plen) + return true; + + /* Page entirely beyond the end of the file */ + if (pos - offset >= i_size) + goto zero_out; + + /* Write that covers from the start of the folio to EOF or beyond */ + if (offset == 0 && (pos + len) >= i_size) + goto zero_out; + + return false; +zero_out: + zero_user_segments(&folio->page, 0, offset, offset + len, plen); + return true; +} + +/** + * netfs_write_begin - Helper to prepare for writing + * @file: The file to read from + * @mapping: The mapping to read from + * @pos: File position at which the write will begin + * @len: The length of the write (may extend beyond the end of the folio chosen) + * @aop_flags: AOP_* flags + * @_folio: Where to put the resultant folio + * @_fsdata: Place for the netfs to store a cookie + * + * Pre-read data for a write-begin request by drawing data from the cache if + * possible, or the netfs if not. Space beyond the EOF is zero-filled. + * Multiple I/O requests from different sources will get munged together. If + * necessary, the readahead window can be expanded in either direction to a + * more convenient alighment for RPC efficiency or to make storage in the cache + * feasible. + * + * The calling netfs must provide a table of operations, only one of which, + * issue_op, is mandatory. + * + * The check_write_begin() operation can be provided to check for and flush + * conflicting writes once the folio is grabbed and locked. It is passed a + * pointer to the fsdata cookie that gets returned to the VM to be passed to + * write_end. It is permitted to sleep. It should return 0 if the request + * should go ahead; unlock the folio and return -EAGAIN to cause the folio to + * be regot; or return an error. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + */ +int netfs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned int len, unsigned int aop_flags, + struct folio **_folio, void **_fsdata) +{ + struct netfs_io_request *rreq; + struct netfs_i_context *ctx = netfs_i_context(file_inode(file )); + struct folio *folio; + unsigned int fgp_flags; + pgoff_t index = pos >> PAGE_SHIFT; + int ret; + + DEFINE_READAHEAD(ractl, file, NULL, mapping, index); + +retry: + fgp_flags = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE; + if (aop_flags & AOP_FLAG_NOFS) + fgp_flags |= FGP_NOFS; + folio = __filemap_get_folio(mapping, index, fgp_flags, + mapping_gfp_mask(mapping)); + if (!folio) + return -ENOMEM; + + if (ctx->ops->check_write_begin) { + /* Allow the netfs (eg. ceph) to flush conflicts. */ + ret = ctx->ops->check_write_begin(file, pos, len, folio, _fsdata); + if (ret < 0) { + trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin); + if (ret == -EAGAIN) + goto retry; + goto error; + } + } + + if (folio_test_uptodate(folio)) + goto have_folio; + + /* If the page is beyond the EOF, we want to clear it - unless it's + * within the cache granule containing the EOF, in which case we need + * to preload the granule. + */ + if (!netfs_is_cache_enabled(ctx) && + netfs_skip_folio_read(folio, pos, len, false)) { + netfs_stat(&netfs_n_rh_write_zskip); + goto have_folio_no_wait; + } + + rreq = netfs_alloc_request(mapping, file, + folio_file_pos(folio), folio_size(folio), + NETFS_READ_FOR_WRITE); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto error; + } + rreq->no_unlock_folio = folio_index(folio); + __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); + + if (ctx->ops->begin_cache_operation) { + ret = ctx->ops->begin_cache_operation(rreq); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto error_put; + } + + netfs_stat(&netfs_n_rh_write_begin); + trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); + + /* Expand the request to meet caching requirements and download + * preferences. + */ + ractl._nr_pages = folio_nr_pages(folio); + netfs_rreq_expand(rreq, &ractl); + + /* We hold the folio locks, so we can drop the references */ + folio_get(folio); + while (readahead_folio(&ractl)) + ; + + ret = netfs_begin_read(rreq, true); + if (ret < 0) + goto error; + +have_folio: + ret = folio_wait_fscache_killable(folio); + if (ret < 0) + goto error; +have_folio_no_wait: + *_folio = folio; + _leave(" = 0"); + return 0; + +error_put: + netfs_put_request(rreq, false, netfs_rreq_trace_put_failed); +error: + folio_unlock(folio); + folio_put(folio); + _leave(" = %d", ret); + return ret; +} +EXPORT_SYMBOL(netfs_write_begin); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index b7f2c4459f33..b7b0e3d18d9e 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -5,6 +5,10 @@ * Written by David Howells (dhowells@redhat.com) */ +#include <linux/netfs.h> +#include <linux/fscache.h> +#include <trace/events/netfs.h> + #ifdef pr_fmt #undef pr_fmt #endif @@ -12,11 +16,40 @@ #define pr_fmt(fmt) "netfs: " fmt /* - * read_helper.c + * buffered_read.c + */ +void netfs_rreq_unlock_folios(struct netfs_io_request *rreq); + +/* + * io.c + */ +int netfs_begin_read(struct netfs_io_request *rreq, bool sync); + +/* + * main.c */ extern unsigned int netfs_debug; /* + * objects.c + */ +struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, + struct file *file, + loff_t start, size_t len, + enum netfs_io_origin origin); +void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); +void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async); +void netfs_put_request(struct netfs_io_request *rreq, bool was_async, + enum netfs_rreq_ref_trace what); +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq); + +static inline void netfs_see_request(struct netfs_io_request *rreq, + enum netfs_rreq_ref_trace what) +{ + trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); +} + +/* * stats.c */ #ifdef CONFIG_NETFS_STATS @@ -55,6 +88,21 @@ static inline void netfs_stat_d(atomic_t *stat) #define netfs_stat_d(x) do {} while(0) #endif +/* + * Miscellaneous functions. + */ +static inline bool netfs_is_cache_enabled(struct netfs_i_context *ctx) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cookie = ctx->cache; + + return fscache_cookie_valid(cookie) && cookie->cache_priv && + fscache_cookie_enabled(cookie); +#else + return false; +#endif +} + /*****************************************************************************/ /* * debug tracing diff --git a/fs/netfs/io.c b/fs/netfs/io.c new file mode 100644 index 000000000000..428925899282 --- /dev/null +++ b/fs/netfs/io.c @@ -0,0 +1,657 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Network filesystem high-level read support. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/module.h> +#include <linux/export.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/slab.h> +#include <linux/uio.h> +#include <linux/sched/mm.h> +#include <linux/task_io_accounting_ops.h> +#include "internal.h" + +/* + * Clear the unread part of an I/O request. + */ +static void netfs_clear_unread(struct netfs_io_subrequest *subreq) +{ + struct iov_iter iter; + + iov_iter_xarray(&iter, READ, &subreq->rreq->mapping->i_pages, + subreq->start + subreq->transferred, + subreq->len - subreq->transferred); + iov_iter_zero(iov_iter_count(&iter), &iter); +} + +static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct netfs_io_subrequest *subreq = priv; + + netfs_subreq_terminated(subreq, transferred_or_error, was_async); +} + +/* + * Issue a read against the cache. + * - Eats the caller's ref on subreq. + */ +static void netfs_read_from_cache(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq, + enum netfs_read_from_hole read_hole) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + struct iov_iter iter; + + netfs_stat(&netfs_n_rh_read); + iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages, + subreq->start + subreq->transferred, + subreq->len - subreq->transferred); + + cres->ops->read(cres, subreq->start, &iter, read_hole, + netfs_cache_read_terminated, subreq); +} + +/* + * Fill a subrequest region with zeroes. + */ +static void netfs_fill_with_zeroes(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + netfs_stat(&netfs_n_rh_zero); + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + netfs_subreq_terminated(subreq, 0, false); +} + +/* + * Ask the netfs to issue a read request to the server for us. + * + * The netfs is expected to read from subreq->pos + subreq->transferred to + * subreq->pos + subreq->len - 1. It may not backtrack and write data into the + * buffer prior to the transferred point as it might clobber dirty data + * obtained from the cache. + * + * Alternatively, the netfs is allowed to indicate one of two things: + * + * - NETFS_SREQ_SHORT_READ: A short read - it will get called again to try and + * make progress. + * + * - NETFS_SREQ_CLEAR_TAIL: A short read - the rest of the buffer will be + * cleared. + */ +static void netfs_read_from_server(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + netfs_stat(&netfs_n_rh_download); + rreq->netfs_ops->issue_read(subreq); +} + +/* + * Release those waiting. + */ +static void netfs_rreq_completed(struct netfs_io_request *rreq, bool was_async) +{ + trace_netfs_rreq(rreq, netfs_rreq_trace_done); + netfs_clear_subrequests(rreq, was_async); + netfs_put_request(rreq, was_async, netfs_rreq_trace_put_complete); +} + +/* + * Deal with the completion of writing the data to the cache. We have to clear + * the PG_fscache bits on the folios involved and release the caller's ref. + * + * May be called in softirq mode and we inherit a ref from the caller. + */ +static void netfs_rreq_unmark_after_write(struct netfs_io_request *rreq, + bool was_async) +{ + struct netfs_io_subrequest *subreq; + struct folio *folio; + pgoff_t unlocked = 0; + bool have_unlocked = false; + + rcu_read_lock(); + + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { + XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE); + + xas_for_each(&xas, folio, (subreq->start + subreq->len - 1) / PAGE_SIZE) { + /* We might have multiple writes from the same huge + * folio, but we mustn't unlock a folio more than once. + */ + if (have_unlocked && folio_index(folio) <= unlocked) + continue; + unlocked = folio_index(folio); + folio_end_fscache(folio); + have_unlocked = true; + } + } + + rcu_read_unlock(); + netfs_rreq_completed(rreq, was_async); +} + +static void netfs_rreq_copy_terminated(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct netfs_io_subrequest *subreq = priv; + struct netfs_io_request *rreq = subreq->rreq; + + if (IS_ERR_VALUE(transferred_or_error)) { + netfs_stat(&netfs_n_rh_write_failed); + trace_netfs_failure(rreq, subreq, transferred_or_error, + netfs_fail_copy_to_cache); + } else { + netfs_stat(&netfs_n_rh_write_done); + } + + trace_netfs_sreq(subreq, netfs_sreq_trace_write_term); + + /* If we decrement nr_copy_ops to 0, the ref belongs to us. */ + if (atomic_dec_and_test(&rreq->nr_copy_ops)) + netfs_rreq_unmark_after_write(rreq, was_async); + + netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated); +} + +/* + * Perform any outstanding writes to the cache. We inherit a ref from the + * caller. + */ +static void netfs_rreq_do_write_to_cache(struct netfs_io_request *rreq) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + struct netfs_io_subrequest *subreq, *next, *p; + struct iov_iter iter; + int ret; + + trace_netfs_rreq(rreq, netfs_rreq_trace_copy); + + /* We don't want terminating writes trying to wake us up whilst we're + * still going through the list. + */ + atomic_inc(&rreq->nr_copy_ops); + + list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) { + if (!test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) { + list_del_init(&subreq->rreq_link); + netfs_put_subrequest(subreq, false, + netfs_sreq_trace_put_no_copy); + } + } + + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { + /* Amalgamate adjacent writes */ + while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { + next = list_next_entry(subreq, rreq_link); + if (next->start != subreq->start + subreq->len) + break; + subreq->len += next->len; + list_del_init(&next->rreq_link); + netfs_put_subrequest(next, false, + netfs_sreq_trace_put_merged); + } + + ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len, + rreq->i_size, true); + if (ret < 0) { + trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write); + trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip); + continue; + } + + iov_iter_xarray(&iter, WRITE, &rreq->mapping->i_pages, + subreq->start, subreq->len); + + atomic_inc(&rreq->nr_copy_ops); + netfs_stat(&netfs_n_rh_write); + netfs_get_subrequest(subreq, netfs_sreq_trace_get_copy_to_cache); + trace_netfs_sreq(subreq, netfs_sreq_trace_write); + cres->ops->write(cres, subreq->start, &iter, + netfs_rreq_copy_terminated, subreq); + } + + /* If we decrement nr_copy_ops to 0, the usage ref belongs to us. */ + if (atomic_dec_and_test(&rreq->nr_copy_ops)) + netfs_rreq_unmark_after_write(rreq, false); +} + +static void netfs_rreq_write_to_cache_work(struct work_struct *work) +{ + struct netfs_io_request *rreq = + container_of(work, struct netfs_io_request, work); + + netfs_rreq_do_write_to_cache(rreq); +} + +static void netfs_rreq_write_to_cache(struct netfs_io_request *rreq) +{ + rreq->work.func = netfs_rreq_write_to_cache_work; + if (!queue_work(system_unbound_wq, &rreq->work)) + BUG(); +} + +/* + * Handle a short read. + */ +static void netfs_rreq_short_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + __clear_bit(NETFS_SREQ_SHORT_IO, &subreq->flags); + __set_bit(NETFS_SREQ_SEEK_DATA_READ, &subreq->flags); + + netfs_stat(&netfs_n_rh_short_read); + trace_netfs_sreq(subreq, netfs_sreq_trace_resubmit_short); + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_short_read); + atomic_inc(&rreq->nr_outstanding); + if (subreq->source == NETFS_READ_FROM_CACHE) + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR); + else + netfs_read_from_server(rreq, subreq); +} + +/* + * Resubmit any short or failed operations. Returns true if we got the rreq + * ref back. + */ +static bool netfs_rreq_perform_resubmissions(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + + WARN_ON(in_interrupt()); + + trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); + + /* We don't want terminating submissions trying to wake us up whilst + * we're still going through the list. + */ + atomic_inc(&rreq->nr_outstanding); + + __clear_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { + if (subreq->error) { + if (subreq->source != NETFS_READ_FROM_CACHE) + break; + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; + subreq->error = 0; + netfs_stat(&netfs_n_rh_download_instead); + trace_netfs_sreq(subreq, netfs_sreq_trace_download_instead); + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + atomic_inc(&rreq->nr_outstanding); + netfs_read_from_server(rreq, subreq); + } else if (test_bit(NETFS_SREQ_SHORT_IO, &subreq->flags)) { + netfs_rreq_short_read(rreq, subreq); + } + } + + /* If we decrement nr_outstanding to 0, the usage ref belongs to us. */ + if (atomic_dec_and_test(&rreq->nr_outstanding)) + return true; + + wake_up_var(&rreq->nr_outstanding); + return false; +} + +/* + * Check to see if the data read is still valid. + */ +static void netfs_rreq_is_still_valid(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + + if (!rreq->netfs_ops->is_still_valid || + rreq->netfs_ops->is_still_valid(rreq)) + return; + + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { + if (subreq->source == NETFS_READ_FROM_CACHE) { + subreq->error = -ESTALE; + __set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); + } + } +} + +/* + * Assess the state of a read request and decide what to do next. + * + * Note that we could be in an ordinary kernel thread, on a workqueue or in + * softirq context at this point. We inherit a ref from the caller. + */ +static void netfs_rreq_assess(struct netfs_io_request *rreq, bool was_async) +{ + trace_netfs_rreq(rreq, netfs_rreq_trace_assess); + +again: + netfs_rreq_is_still_valid(rreq); + + if (!test_bit(NETFS_RREQ_FAILED, &rreq->flags) && + test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags)) { + if (netfs_rreq_perform_resubmissions(rreq)) + goto again; + return; + } + + netfs_rreq_unlock_folios(rreq); + + clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags); + wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS); + + if (test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) + return netfs_rreq_write_to_cache(rreq); + + netfs_rreq_completed(rreq, was_async); +} + +static void netfs_rreq_work(struct work_struct *work) +{ + struct netfs_io_request *rreq = + container_of(work, struct netfs_io_request, work); + netfs_rreq_assess(rreq, false); +} + +/* + * Handle the completion of all outstanding I/O operations on a read request. + * We inherit a ref from the caller. + */ +static void netfs_rreq_terminated(struct netfs_io_request *rreq, + bool was_async) +{ + if (test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags) && + was_async) { + if (!queue_work(system_unbound_wq, &rreq->work)) + BUG(); + } else { + netfs_rreq_assess(rreq, was_async); + } +} + +/** + * netfs_subreq_terminated - Note the termination of an I/O operation. + * @subreq: The I/O request that has terminated. + * @transferred_or_error: The amount of data transferred or an error code. + * @was_async: The termination was asynchronous + * + * This tells the read helper that a contributory I/O operation has terminated, + * one way or another, and that it should integrate the results. + * + * The caller indicates in @transferred_or_error the outcome of the operation, + * supplying a positive value to indicate the number of bytes transferred, 0 to + * indicate a failure to transfer anything that should be retried or a negative + * error code. The helper will look after reissuing I/O operations as + * appropriate and writing downloaded data to the cache. + * + * If @was_async is true, the caller might be running in softirq or interrupt + * context and we can't sleep. + */ +void netfs_subreq_terminated(struct netfs_io_subrequest *subreq, + ssize_t transferred_or_error, + bool was_async) +{ + struct netfs_io_request *rreq = subreq->rreq; + int u; + + _enter("[%u]{%llx,%lx},%zd", + subreq->debug_index, subreq->start, subreq->flags, + transferred_or_error); + + switch (subreq->source) { + case NETFS_READ_FROM_CACHE: + netfs_stat(&netfs_n_rh_read_done); + break; + case NETFS_DOWNLOAD_FROM_SERVER: + netfs_stat(&netfs_n_rh_download_done); + break; + default: + break; + } + + if (IS_ERR_VALUE(transferred_or_error)) { + subreq->error = transferred_or_error; + trace_netfs_failure(rreq, subreq, transferred_or_error, + netfs_fail_read); + goto failed; + } + + if (WARN(transferred_or_error > subreq->len - subreq->transferred, + "Subreq overread: R%x[%x] %zd > %zu - %zu", + rreq->debug_id, subreq->debug_index, + transferred_or_error, subreq->len, subreq->transferred)) + transferred_or_error = subreq->len - subreq->transferred; + + subreq->error = 0; + subreq->transferred += transferred_or_error; + if (subreq->transferred < subreq->len) + goto incomplete; + +complete: + __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); + if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) + set_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags); + +out: + trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); + + /* If we decrement nr_outstanding to 0, the ref belongs to us. */ + u = atomic_dec_return(&rreq->nr_outstanding); + if (u == 0) + netfs_rreq_terminated(rreq, was_async); + else if (u == 1) + wake_up_var(&rreq->nr_outstanding); + + netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated); + return; + +incomplete: + if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) { + netfs_clear_unread(subreq); + subreq->transferred = subreq->len; + goto complete; + } + + if (transferred_or_error == 0) { + if (__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) { + subreq->error = -ENODATA; + goto failed; + } + } else { + __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); + } + + __set_bit(NETFS_SREQ_SHORT_IO, &subreq->flags); + set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); + goto out; + +failed: + if (subreq->source == NETFS_READ_FROM_CACHE) { + netfs_stat(&netfs_n_rh_read_failed); + set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); + } else { + netfs_stat(&netfs_n_rh_download_failed); + set_bit(NETFS_RREQ_FAILED, &rreq->flags); + rreq->error = subreq->error; + } + goto out; +} +EXPORT_SYMBOL(netfs_subreq_terminated); + +static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_subrequest *subreq, + loff_t i_size) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct netfs_cache_resources *cres = &rreq->cache_resources; + + if (cres->ops) + return cres->ops->prepare_read(subreq, i_size); + if (subreq->start >= rreq->i_size) + return NETFS_FILL_WITH_ZEROES; + return NETFS_DOWNLOAD_FROM_SERVER; +} + +/* + * Work out what sort of subrequest the next one will be. + */ +static enum netfs_io_source +netfs_rreq_prepare_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + enum netfs_io_source source; + + _enter("%llx-%llx,%llx", subreq->start, subreq->start + subreq->len, rreq->i_size); + + source = netfs_cache_prepare_read(subreq, rreq->i_size); + if (source == NETFS_INVALID_READ) + goto out; + + if (source == NETFS_DOWNLOAD_FROM_SERVER) { + /* Call out to the netfs to let it shrink the request to fit + * its own I/O sizes and boundaries. If it shinks it here, it + * will be called again to make simultaneous calls; if it wants + * to make serial calls, it can indicate a short read and then + * we will call it again. + */ + if (subreq->len > rreq->i_size - subreq->start) + subreq->len = rreq->i_size - subreq->start; + + if (rreq->netfs_ops->clamp_length && + !rreq->netfs_ops->clamp_length(subreq)) { + source = NETFS_INVALID_READ; + goto out; + } + } + + if (WARN_ON(subreq->len == 0)) + source = NETFS_INVALID_READ; + +out: + subreq->source = source; + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + return source; +} + +/* + * Slice off a piece of a read request and submit an I/O request for it. + */ +static bool netfs_rreq_submit_slice(struct netfs_io_request *rreq, + unsigned int *_debug_index) +{ + struct netfs_io_subrequest *subreq; + enum netfs_io_source source; + + subreq = netfs_alloc_subrequest(rreq); + if (!subreq) + return false; + + subreq->debug_index = (*_debug_index)++; + subreq->start = rreq->start + rreq->submitted; + subreq->len = rreq->len - rreq->submitted; + + _debug("slice %llx,%zx,%zx", subreq->start, subreq->len, rreq->submitted); + list_add_tail(&subreq->rreq_link, &rreq->subrequests); + + /* Call out to the cache to find out what it can do with the remaining + * subset. It tells us in subreq->flags what it decided should be done + * and adjusts subreq->len down if the subset crosses a cache boundary. + * + * Then when we hand the subset, it can choose to take a subset of that + * (the starts must coincide), in which case, we go around the loop + * again and ask it to download the next piece. + */ + source = netfs_rreq_prepare_read(rreq, subreq); + if (source == NETFS_INVALID_READ) + goto subreq_failed; + + atomic_inc(&rreq->nr_outstanding); + + rreq->submitted += subreq->len; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + switch (source) { + case NETFS_FILL_WITH_ZEROES: + netfs_fill_with_zeroes(rreq, subreq); + break; + case NETFS_DOWNLOAD_FROM_SERVER: + netfs_read_from_server(rreq, subreq); + break; + case NETFS_READ_FROM_CACHE: + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE); + break; + default: + BUG(); + } + + return true; + +subreq_failed: + rreq->error = subreq->error; + netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_failed); + return false; +} + +/* + * Begin the process of reading in a chunk of data, where that data may be + * stitched together from multiple sources, including multiple servers and the + * local cache. + */ +int netfs_begin_read(struct netfs_io_request *rreq, bool sync) +{ + unsigned int debug_index = 0; + int ret; + + _enter("R=%x %llx-%llx", + rreq->debug_id, rreq->start, rreq->start + rreq->len - 1); + + if (rreq->len == 0) { + pr_err("Zero-sized read [R=%x]\n", rreq->debug_id); + netfs_put_request(rreq, false, netfs_rreq_trace_put_zero_len); + return -EIO; + } + + INIT_WORK(&rreq->work, netfs_rreq_work); + + if (sync) + netfs_get_request(rreq, netfs_rreq_trace_get_hold); + + /* Chop the read into slices according to what the cache and the netfs + * want and submit each one. + */ + atomic_set(&rreq->nr_outstanding, 1); + do { + if (!netfs_rreq_submit_slice(rreq, &debug_index)) + break; + + } while (rreq->submitted < rreq->len); + + if (sync) { + /* Keep nr_outstanding incremented so that the ref always belongs to + * us, and the service code isn't punted off to a random thread pool to + * process. + */ + for (;;) { + wait_var_event(&rreq->nr_outstanding, + atomic_read(&rreq->nr_outstanding) == 1); + netfs_rreq_assess(rreq, false); + if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags)) + break; + cond_resched(); + } + + ret = rreq->error; + if (ret == 0 && rreq->submitted < rreq->len) { + trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); + ret = -EIO; + } + netfs_put_request(rreq, false, netfs_rreq_trace_put_hold); + } else { + /* If we decrement nr_outstanding to 0, the ref belongs to us. */ + if (atomic_dec_and_test(&rreq->nr_outstanding)) + netfs_rreq_assess(rreq, false); + ret = 0; + } + return ret; +} diff --git a/fs/netfs/main.c b/fs/netfs/main.c new file mode 100644 index 000000000000..068568702957 --- /dev/null +++ b/fs/netfs/main.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Miscellaneous bits for the netfs support library. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/module.h> +#include <linux/export.h> +#include "internal.h" +#define CREATE_TRACE_POINTS +#include <trace/events/netfs.h> + +MODULE_DESCRIPTION("Network fs support"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +unsigned netfs_debug; +module_param_named(debug, netfs_debug, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(netfs_debug, "Netfs support debugging mask"); diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c new file mode 100644 index 000000000000..e86107b30ba4 --- /dev/null +++ b/fs/netfs/objects.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Object lifetime handling and tracing. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include <linux/slab.h> +#include "internal.h" + +/* + * Allocate an I/O request and initialise it. + */ +struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, + struct file *file, + loff_t start, size_t len, + enum netfs_io_origin origin) +{ + static atomic_t debug_ids; + struct inode *inode = file ? file_inode(file) : mapping->host; + struct netfs_i_context *ctx = netfs_i_context(inode); + struct netfs_io_request *rreq; + int ret; + + rreq = kzalloc(sizeof(struct netfs_io_request), GFP_KERNEL); + if (!rreq) + return ERR_PTR(-ENOMEM); + + rreq->start = start; + rreq->len = len; + rreq->origin = origin; + rreq->netfs_ops = ctx->ops; + rreq->mapping = mapping; + rreq->inode = inode; + rreq->i_size = i_size_read(inode); + rreq->debug_id = atomic_inc_return(&debug_ids); + INIT_LIST_HEAD(&rreq->subrequests); + refcount_set(&rreq->ref, 1); + __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); + if (rreq->netfs_ops->init_request) { + ret = rreq->netfs_ops->init_request(rreq, file); + if (ret < 0) { + kfree(rreq); + return ERR_PTR(ret); + } + } + + netfs_stat(&netfs_n_rh_rreq); + return rreq; +} + +void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) +{ + int r; + + __refcount_inc(&rreq->ref, &r); + trace_netfs_rreq_ref(rreq->debug_id, r + 1, what); +} + +void netfs_clear_subrequests(struct netfs_io_request *rreq, bool was_async) +{ + struct netfs_io_subrequest *subreq; + + while (!list_empty(&rreq->subrequests)) { + subreq = list_first_entry(&rreq->subrequests, + struct netfs_io_subrequest, rreq_link); + list_del(&subreq->rreq_link); + netfs_put_subrequest(subreq, was_async, + netfs_sreq_trace_put_clear); + } +} + +static void netfs_free_request(struct work_struct *work) +{ + struct netfs_io_request *rreq = + container_of(work, struct netfs_io_request, work); + + netfs_clear_subrequests(rreq, false); + if (rreq->netfs_priv) + rreq->netfs_ops->cleanup(rreq->mapping, rreq->netfs_priv); + trace_netfs_rreq(rreq, netfs_rreq_trace_free); + if (rreq->cache_resources.ops) + rreq->cache_resources.ops->end_operation(&rreq->cache_resources); + kfree(rreq); + netfs_stat_d(&netfs_n_rh_rreq); +} + +void netfs_put_request(struct netfs_io_request *rreq, bool was_async, + enum netfs_rreq_ref_trace what) +{ + unsigned int debug_id = rreq->debug_id; + bool dead; + int r; + + dead = __refcount_dec_and_test(&rreq->ref, &r); + trace_netfs_rreq_ref(debug_id, r - 1, what); + if (dead) { + if (was_async) { + rreq->work.func = netfs_free_request; + if (!queue_work(system_unbound_wq, &rreq->work)) + BUG(); + } else { + netfs_free_request(&rreq->work); + } + } +} + +/* + * Allocate and partially initialise an I/O request structure. + */ +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + + subreq = kzalloc(sizeof(struct netfs_io_subrequest), GFP_KERNEL); + if (subreq) { + INIT_LIST_HEAD(&subreq->rreq_link); + refcount_set(&subreq->ref, 2); + subreq->rreq = rreq; + netfs_get_request(rreq, netfs_rreq_trace_get_subreq); + netfs_stat(&netfs_n_rh_sreq); + } + + return subreq; +} + +void netfs_get_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what) +{ + int r; + + __refcount_inc(&subreq->ref, &r); + trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1, + what); +} + +static void netfs_free_subrequest(struct netfs_io_subrequest *subreq, + bool was_async) +{ + struct netfs_io_request *rreq = subreq->rreq; + + trace_netfs_sreq(subreq, netfs_sreq_trace_free); + kfree(subreq); + netfs_stat_d(&netfs_n_rh_sreq); + netfs_put_request(rreq, was_async, netfs_rreq_trace_put_subreq); +} + +void netfs_put_subrequest(struct netfs_io_subrequest *subreq, bool was_async, + enum netfs_sreq_ref_trace what) +{ + unsigned int debug_index = subreq->debug_index; + unsigned int debug_id = subreq->rreq->debug_id; + bool dead; + int r; + + dead = __refcount_dec_and_test(&subreq->ref, &r); + trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what); + if (dead) + netfs_free_subrequest(subreq, was_async); +} diff --git a/fs/netfs/read_helper.c b/fs/netfs/read_helper.c deleted file mode 100644 index 501da990c259..000000000000 --- a/fs/netfs/read_helper.c +++ /dev/null @@ -1,1205 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Network filesystem high-level read support. - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include <linux/module.h> -#include <linux/export.h> -#include <linux/fs.h> -#include <linux/mm.h> -#include <linux/pagemap.h> -#include <linux/slab.h> -#include <linux/uio.h> -#include <linux/sched/mm.h> -#include <linux/task_io_accounting_ops.h> -#include <linux/netfs.h> -#include "internal.h" -#define CREATE_TRACE_POINTS -#include <trace/events/netfs.h> - -MODULE_DESCRIPTION("Network fs support"); -MODULE_AUTHOR("Red Hat, Inc."); -MODULE_LICENSE("GPL"); - -unsigned netfs_debug; -module_param_named(debug, netfs_debug, uint, S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(netfs_debug, "Netfs support debugging mask"); - -static void netfs_rreq_work(struct work_struct *); -static void __netfs_put_subrequest(struct netfs_read_subrequest *, bool); - -static void netfs_put_subrequest(struct netfs_read_subrequest *subreq, - bool was_async) -{ - if (refcount_dec_and_test(&subreq->usage)) - __netfs_put_subrequest(subreq, was_async); -} - -static struct netfs_read_request *netfs_alloc_read_request( - const struct netfs_read_request_ops *ops, void *netfs_priv, - struct file *file) -{ - static atomic_t debug_ids; - struct netfs_read_request *rreq; - - rreq = kzalloc(sizeof(struct netfs_read_request), GFP_KERNEL); - if (rreq) { - rreq->netfs_ops = ops; - rreq->netfs_priv = netfs_priv; - rreq->inode = file_inode(file); - rreq->i_size = i_size_read(rreq->inode); - rreq->debug_id = atomic_inc_return(&debug_ids); - INIT_LIST_HEAD(&rreq->subrequests); - INIT_WORK(&rreq->work, netfs_rreq_work); - refcount_set(&rreq->usage, 1); - __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); - if (ops->init_rreq) - ops->init_rreq(rreq, file); - netfs_stat(&netfs_n_rh_rreq); - } - - return rreq; -} - -static void netfs_get_read_request(struct netfs_read_request *rreq) -{ - refcount_inc(&rreq->usage); -} - -static void netfs_rreq_clear_subreqs(struct netfs_read_request *rreq, - bool was_async) -{ - struct netfs_read_subrequest *subreq; - - while (!list_empty(&rreq->subrequests)) { - subreq = list_first_entry(&rreq->subrequests, - struct netfs_read_subrequest, rreq_link); - list_del(&subreq->rreq_link); - netfs_put_subrequest(subreq, was_async); - } -} - -static void netfs_free_read_request(struct work_struct *work) -{ - struct netfs_read_request *rreq = - container_of(work, struct netfs_read_request, work); - netfs_rreq_clear_subreqs(rreq, false); - if (rreq->netfs_priv) - rreq->netfs_ops->cleanup(rreq->mapping, rreq->netfs_priv); - trace_netfs_rreq(rreq, netfs_rreq_trace_free); - if (rreq->cache_resources.ops) - rreq->cache_resources.ops->end_operation(&rreq->cache_resources); - kfree(rreq); - netfs_stat_d(&netfs_n_rh_rreq); -} - -static void netfs_put_read_request(struct netfs_read_request *rreq, bool was_async) -{ - if (refcount_dec_and_test(&rreq->usage)) { - if (was_async) { - rreq->work.func = netfs_free_read_request; - if (!queue_work(system_unbound_wq, &rreq->work)) - BUG(); - } else { - netfs_free_read_request(&rreq->work); - } - } -} - -/* - * Allocate and partially initialise an I/O request structure. - */ -static struct netfs_read_subrequest *netfs_alloc_subrequest( - struct netfs_read_request *rreq) -{ - struct netfs_read_subrequest *subreq; - - subreq = kzalloc(sizeof(struct netfs_read_subrequest), GFP_KERNEL); - if (subreq) { - INIT_LIST_HEAD(&subreq->rreq_link); - refcount_set(&subreq->usage, 2); - subreq->rreq = rreq; - netfs_get_read_request(rreq); - netfs_stat(&netfs_n_rh_sreq); - } - - return subreq; -} - -static void netfs_get_read_subrequest(struct netfs_read_subrequest *subreq) -{ - refcount_inc(&subreq->usage); -} - -static void __netfs_put_subrequest(struct netfs_read_subrequest *subreq, - bool was_async) -{ - struct netfs_read_request *rreq = subreq->rreq; - - trace_netfs_sreq(subreq, netfs_sreq_trace_free); - kfree(subreq); - netfs_stat_d(&netfs_n_rh_sreq); - netfs_put_read_request(rreq, was_async); -} - -/* - * Clear the unread part of an I/O request. - */ -static void netfs_clear_unread(struct netfs_read_subrequest *subreq) -{ - struct iov_iter iter; - - iov_iter_xarray(&iter, READ, &subreq->rreq->mapping->i_pages, - subreq->start + subreq->transferred, - subreq->len - subreq->transferred); - iov_iter_zero(iov_iter_count(&iter), &iter); -} - -static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error, - bool was_async) -{ - struct netfs_read_subrequest *subreq = priv; - - netfs_subreq_terminated(subreq, transferred_or_error, was_async); -} - -/* - * Issue a read against the cache. - * - Eats the caller's ref on subreq. - */ -static void netfs_read_from_cache(struct netfs_read_request *rreq, - struct netfs_read_subrequest *subreq, - enum netfs_read_from_hole read_hole) -{ - struct netfs_cache_resources *cres = &rreq->cache_resources; - struct iov_iter iter; - - netfs_stat(&netfs_n_rh_read); - iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages, - subreq->start + subreq->transferred, - subreq->len - subreq->transferred); - - cres->ops->read(cres, subreq->start, &iter, read_hole, - netfs_cache_read_terminated, subreq); -} - -/* - * Fill a subrequest region with zeroes. - */ -static void netfs_fill_with_zeroes(struct netfs_read_request *rreq, - struct netfs_read_subrequest *subreq) -{ - netfs_stat(&netfs_n_rh_zero); - __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); - netfs_subreq_terminated(subreq, 0, false); -} - -/* - * Ask the netfs to issue a read request to the server for us. - * - * The netfs is expected to read from subreq->pos + subreq->transferred to - * subreq->pos + subreq->len - 1. It may not backtrack and write data into the - * buffer prior to the transferred point as it might clobber dirty data - * obtained from the cache. - * - * Alternatively, the netfs is allowed to indicate one of two things: - * - * - NETFS_SREQ_SHORT_READ: A short read - it will get called again to try and - * make progress. - * - * - NETFS_SREQ_CLEAR_TAIL: A short read - the rest of the buffer will be - * cleared. - */ -static void netfs_read_from_server(struct netfs_read_request *rreq, - struct netfs_read_subrequest *subreq) -{ - netfs_stat(&netfs_n_rh_download); - rreq->netfs_ops->issue_op(subreq); -} - -/* - * Release those waiting. - */ -static void netfs_rreq_completed(struct netfs_read_request *rreq, bool was_async) -{ - trace_netfs_rreq(rreq, netfs_rreq_trace_done); - netfs_rreq_clear_subreqs(rreq, was_async); - netfs_put_read_request(rreq, was_async); -} - -/* - * Deal with the completion of writing the data to the cache. We have to clear - * the PG_fscache bits on the folios involved and release the caller's ref. - * - * May be called in softirq mode and we inherit a ref from the caller. - */ -static void netfs_rreq_unmark_after_write(struct netfs_read_request *rreq, - bool was_async) -{ - struct netfs_read_subrequest *subreq; - struct folio *folio; - pgoff_t unlocked = 0; - bool have_unlocked = false; - - rcu_read_lock(); - - list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { - XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE); - - xas_for_each(&xas, folio, (subreq->start + subreq->len - 1) / PAGE_SIZE) { - /* We might have multiple writes from the same huge - * folio, but we mustn't unlock a folio more than once. - */ - if (have_unlocked && folio_index(folio) <= unlocked) - continue; - unlocked = folio_index(folio); - folio_end_fscache(folio); - have_unlocked = true; - } - } - - rcu_read_unlock(); - netfs_rreq_completed(rreq, was_async); -} - -static void netfs_rreq_copy_terminated(void *priv, ssize_t transferred_or_error, - bool was_async) -{ - struct netfs_read_subrequest *subreq = priv; - struct netfs_read_request *rreq = subreq->rreq; - - if (IS_ERR_VALUE(transferred_or_error)) { - netfs_stat(&netfs_n_rh_write_failed); - trace_netfs_failure(rreq, subreq, transferred_or_error, - netfs_fail_copy_to_cache); - } else { - netfs_stat(&netfs_n_rh_write_done); - } - - trace_netfs_sreq(subreq, netfs_sreq_trace_write_term); - - /* If we decrement nr_wr_ops to 0, the ref belongs to us. */ - if (atomic_dec_and_test(&rreq->nr_wr_ops)) - netfs_rreq_unmark_after_write(rreq, was_async); - - netfs_put_subrequest(subreq, was_async); -} - -/* - * Perform any outstanding writes to the cache. We inherit a ref from the - * caller. - */ -static void netfs_rreq_do_write_to_cache(struct netfs_read_request *rreq) -{ - struct netfs_cache_resources *cres = &rreq->cache_resources; - struct netfs_read_subrequest *subreq, *next, *p; - struct iov_iter iter; - int ret; - - trace_netfs_rreq(rreq, netfs_rreq_trace_write); - - /* We don't want terminating writes trying to wake us up whilst we're - * still going through the list. - */ - atomic_inc(&rreq->nr_wr_ops); - - list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) { - if (!test_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags)) { - list_del_init(&subreq->rreq_link); - netfs_put_subrequest(subreq, false); - } - } - - list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { - /* Amalgamate adjacent writes */ - while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { - next = list_next_entry(subreq, rreq_link); - if (next->start != subreq->start + subreq->len) - break; - subreq->len += next->len; - list_del_init(&next->rreq_link); - netfs_put_subrequest(next, false); - } - - ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len, - rreq->i_size, true); - if (ret < 0) { - trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write); - trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip); - continue; - } - - iov_iter_xarray(&iter, WRITE, &rreq->mapping->i_pages, - subreq->start, subreq->len); - - atomic_inc(&rreq->nr_wr_ops); - netfs_stat(&netfs_n_rh_write); - netfs_get_read_subrequest(subreq); - trace_netfs_sreq(subreq, netfs_sreq_trace_write); - cres->ops->write(cres, subreq->start, &iter, - netfs_rreq_copy_terminated, subreq); - } - - /* If we decrement nr_wr_ops to 0, the usage ref belongs to us. */ - if (atomic_dec_and_test(&rreq->nr_wr_ops)) - netfs_rreq_unmark_after_write(rreq, false); -} - -static void netfs_rreq_write_to_cache_work(struct work_struct *work) -{ - struct netfs_read_request *rreq = - container_of(work, struct netfs_read_request, work); - - netfs_rreq_do_write_to_cache(rreq); -} - -static void netfs_rreq_write_to_cache(struct netfs_read_request *rreq) -{ - rreq->work.func = netfs_rreq_write_to_cache_work; - if (!queue_work(system_unbound_wq, &rreq->work)) - BUG(); -} - -/* - * Unlock the folios in a read operation. We need to set PG_fscache on any - * folios we're going to write back before we unlock them. - */ -static void netfs_rreq_unlock(struct netfs_read_request *rreq) -{ - struct netfs_read_subrequest *subreq; - struct folio *folio; - unsigned int iopos, account = 0; - pgoff_t start_page = rreq->start / PAGE_SIZE; - pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; - bool subreq_failed = false; - - XA_STATE(xas, &rreq->mapping->i_pages, start_page); - - if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) { - __clear_bit(NETFS_RREQ_WRITE_TO_CACHE, &rreq->flags); - list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { - __clear_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags); - } - } - - /* Walk through the pagecache and the I/O request lists simultaneously. - * We may have a mixture of cached and uncached sections and we only - * really want to write out the uncached sections. This is slightly - * complicated by the possibility that we might have huge pages with a - * mixture inside. - */ - subreq = list_first_entry(&rreq->subrequests, - struct netfs_read_subrequest, rreq_link); - iopos = 0; - subreq_failed = (subreq->error < 0); - - trace_netfs_rreq(rreq, netfs_rreq_trace_unlock); - - rcu_read_lock(); - xas_for_each(&xas, folio, last_page) { - unsigned int pgpos = (folio_index(folio) - start_page) * PAGE_SIZE; - unsigned int pgend = pgpos + folio_size(folio); - bool pg_failed = false; - - for (;;) { - if (!subreq) { - pg_failed = true; - break; - } - if (test_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags)) - folio_start_fscache(folio); - pg_failed |= subreq_failed; - if (pgend < iopos + subreq->len) - break; - - account += subreq->transferred; - iopos += subreq->len; - if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { - subreq = list_next_entry(subreq, rreq_link); - subreq_failed = (subreq->error < 0); - } else { - subreq = NULL; - subreq_failed = false; - } - if (pgend == iopos) - break; - } - - if (!pg_failed) { - flush_dcache_folio(folio); - folio_mark_uptodate(folio); - } - - if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { - if (folio_index(folio) == rreq->no_unlock_folio && - test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) - _debug("no unlock"); - else - folio_unlock(folio); - } - } - rcu_read_unlock(); - - task_io_account_read(account); - if (rreq->netfs_ops->done) - rreq->netfs_ops->done(rreq); -} - -/* - * Handle a short read. - */ -static void netfs_rreq_short_read(struct netfs_read_request *rreq, - struct netfs_read_subrequest *subreq) -{ - __clear_bit(NETFS_SREQ_SHORT_READ, &subreq->flags); - __set_bit(NETFS_SREQ_SEEK_DATA_READ, &subreq->flags); - - netfs_stat(&netfs_n_rh_short_read); - trace_netfs_sreq(subreq, netfs_sreq_trace_resubmit_short); - - netfs_get_read_subrequest(subreq); - atomic_inc(&rreq->nr_rd_ops); - if (subreq->source == NETFS_READ_FROM_CACHE) - netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR); - else - netfs_read_from_server(rreq, subreq); -} - -/* - * Resubmit any short or failed operations. Returns true if we got the rreq - * ref back. - */ -static bool netfs_rreq_perform_resubmissions(struct netfs_read_request *rreq) -{ - struct netfs_read_subrequest *subreq; - - WARN_ON(in_interrupt()); - - trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); - - /* We don't want terminating submissions trying to wake us up whilst - * we're still going through the list. - */ - atomic_inc(&rreq->nr_rd_ops); - - __clear_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); - list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { - if (subreq->error) { - if (subreq->source != NETFS_READ_FROM_CACHE) - break; - subreq->source = NETFS_DOWNLOAD_FROM_SERVER; - subreq->error = 0; - netfs_stat(&netfs_n_rh_download_instead); - trace_netfs_sreq(subreq, netfs_sreq_trace_download_instead); - netfs_get_read_subrequest(subreq); - atomic_inc(&rreq->nr_rd_ops); - netfs_read_from_server(rreq, subreq); - } else if (test_bit(NETFS_SREQ_SHORT_READ, &subreq->flags)) { - netfs_rreq_short_read(rreq, subreq); - } - } - - /* If we decrement nr_rd_ops to 0, the usage ref belongs to us. */ - if (atomic_dec_and_test(&rreq->nr_rd_ops)) - return true; - - wake_up_var(&rreq->nr_rd_ops); - return false; -} - -/* - * Check to see if the data read is still valid. - */ -static void netfs_rreq_is_still_valid(struct netfs_read_request *rreq) -{ - struct netfs_read_subrequest *subreq; - - if (!rreq->netfs_ops->is_still_valid || - rreq->netfs_ops->is_still_valid(rreq)) - return; - - list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { - if (subreq->source == NETFS_READ_FROM_CACHE) { - subreq->error = -ESTALE; - __set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); - } - } -} - -/* - * Assess the state of a read request and decide what to do next. - * - * Note that we could be in an ordinary kernel thread, on a workqueue or in - * softirq context at this point. We inherit a ref from the caller. - */ -static void netfs_rreq_assess(struct netfs_read_request *rreq, bool was_async) -{ - trace_netfs_rreq(rreq, netfs_rreq_trace_assess); - -again: - netfs_rreq_is_still_valid(rreq); - - if (!test_bit(NETFS_RREQ_FAILED, &rreq->flags) && - test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags)) { - if (netfs_rreq_perform_resubmissions(rreq)) - goto again; - return; - } - - netfs_rreq_unlock(rreq); - - clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags); - wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS); - - if (test_bit(NETFS_RREQ_WRITE_TO_CACHE, &rreq->flags)) - return netfs_rreq_write_to_cache(rreq); - - netfs_rreq_completed(rreq, was_async); -} - -static void netfs_rreq_work(struct work_struct *work) -{ - struct netfs_read_request *rreq = - container_of(work, struct netfs_read_request, work); - netfs_rreq_assess(rreq, false); -} - -/* - * Handle the completion of all outstanding I/O operations on a read request. - * We inherit a ref from the caller. - */ -static void netfs_rreq_terminated(struct netfs_read_request *rreq, - bool was_async) -{ - if (test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags) && - was_async) { - if (!queue_work(system_unbound_wq, &rreq->work)) - BUG(); - } else { - netfs_rreq_assess(rreq, was_async); - } -} - -/** - * netfs_subreq_terminated - Note the termination of an I/O operation. - * @subreq: The I/O request that has terminated. - * @transferred_or_error: The amount of data transferred or an error code. - * @was_async: The termination was asynchronous - * - * This tells the read helper that a contributory I/O operation has terminated, - * one way or another, and that it should integrate the results. - * - * The caller indicates in @transferred_or_error the outcome of the operation, - * supplying a positive value to indicate the number of bytes transferred, 0 to - * indicate a failure to transfer anything that should be retried or a negative - * error code. The helper will look after reissuing I/O operations as - * appropriate and writing downloaded data to the cache. - * - * If @was_async is true, the caller might be running in softirq or interrupt - * context and we can't sleep. - */ -void netfs_subreq_terminated(struct netfs_read_subrequest *subreq, - ssize_t transferred_or_error, - bool was_async) -{ - struct netfs_read_request *rreq = subreq->rreq; - int u; - - _enter("[%u]{%llx,%lx},%zd", - subreq->debug_index, subreq->start, subreq->flags, - transferred_or_error); - - switch (subreq->source) { - case NETFS_READ_FROM_CACHE: - netfs_stat(&netfs_n_rh_read_done); - break; - case NETFS_DOWNLOAD_FROM_SERVER: - netfs_stat(&netfs_n_rh_download_done); - break; - default: - break; - } - - if (IS_ERR_VALUE(transferred_or_error)) { - subreq->error = transferred_or_error; - trace_netfs_failure(rreq, subreq, transferred_or_error, - netfs_fail_read); - goto failed; - } - - if (WARN(transferred_or_error > subreq->len - subreq->transferred, - "Subreq overread: R%x[%x] %zd > %zu - %zu", - rreq->debug_id, subreq->debug_index, - transferred_or_error, subreq->len, subreq->transferred)) - transferred_or_error = subreq->len - subreq->transferred; - - subreq->error = 0; - subreq->transferred += transferred_or_error; - if (subreq->transferred < subreq->len) - goto incomplete; - -complete: - __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); - if (test_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags)) - set_bit(NETFS_RREQ_WRITE_TO_CACHE, &rreq->flags); - -out: - trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); - - /* If we decrement nr_rd_ops to 0, the ref belongs to us. */ - u = atomic_dec_return(&rreq->nr_rd_ops); - if (u == 0) - netfs_rreq_terminated(rreq, was_async); - else if (u == 1) - wake_up_var(&rreq->nr_rd_ops); - - netfs_put_subrequest(subreq, was_async); - return; - -incomplete: - if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) { - netfs_clear_unread(subreq); - subreq->transferred = subreq->len; - goto complete; - } - - if (transferred_or_error == 0) { - if (__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) { - subreq->error = -ENODATA; - goto failed; - } - } else { - __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); - } - - __set_bit(NETFS_SREQ_SHORT_READ, &subreq->flags); - set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); - goto out; - -failed: - if (subreq->source == NETFS_READ_FROM_CACHE) { - netfs_stat(&netfs_n_rh_read_failed); - set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags); - } else { - netfs_stat(&netfs_n_rh_download_failed); - set_bit(NETFS_RREQ_FAILED, &rreq->flags); - rreq->error = subreq->error; - } - goto out; -} -EXPORT_SYMBOL(netfs_subreq_terminated); - -static enum netfs_read_source netfs_cache_prepare_read(struct netfs_read_subrequest *subreq, - loff_t i_size) -{ - struct netfs_read_request *rreq = subreq->rreq; - struct netfs_cache_resources *cres = &rreq->cache_resources; - - if (cres->ops) - return cres->ops->prepare_read(subreq, i_size); - if (subreq->start >= rreq->i_size) - return NETFS_FILL_WITH_ZEROES; - return NETFS_DOWNLOAD_FROM_SERVER; -} - -/* - * Work out what sort of subrequest the next one will be. - */ -static enum netfs_read_source -netfs_rreq_prepare_read(struct netfs_read_request *rreq, - struct netfs_read_subrequest *subreq) -{ - enum netfs_read_source source; - - _enter("%llx-%llx,%llx", subreq->start, subreq->start + subreq->len, rreq->i_size); - - source = netfs_cache_prepare_read(subreq, rreq->i_size); - if (source == NETFS_INVALID_READ) - goto out; - - if (source == NETFS_DOWNLOAD_FROM_SERVER) { - /* Call out to the netfs to let it shrink the request to fit - * its own I/O sizes and boundaries. If it shinks it here, it - * will be called again to make simultaneous calls; if it wants - * to make serial calls, it can indicate a short read and then - * we will call it again. - */ - if (subreq->len > rreq->i_size - subreq->start) - subreq->len = rreq->i_size - subreq->start; - - if (rreq->netfs_ops->clamp_length && - !rreq->netfs_ops->clamp_length(subreq)) { - source = NETFS_INVALID_READ; - goto out; - } - } - - if (WARN_ON(subreq->len == 0)) - source = NETFS_INVALID_READ; - -out: - subreq->source = source; - trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); - return source; -} - -/* - * Slice off a piece of a read request and submit an I/O request for it. - */ -static bool netfs_rreq_submit_slice(struct netfs_read_request *rreq, - unsigned int *_debug_index) -{ - struct netfs_read_subrequest *subreq; - enum netfs_read_source source; - - subreq = netfs_alloc_subrequest(rreq); - if (!subreq) - return false; - - subreq->debug_index = (*_debug_index)++; - subreq->start = rreq->start + rreq->submitted; - subreq->len = rreq->len - rreq->submitted; - - _debug("slice %llx,%zx,%zx", subreq->start, subreq->len, rreq->submitted); - list_add_tail(&subreq->rreq_link, &rreq->subrequests); - - /* Call out to the cache to find out what it can do with the remaining - * subset. It tells us in subreq->flags what it decided should be done - * and adjusts subreq->len down if the subset crosses a cache boundary. - * - * Then when we hand the subset, it can choose to take a subset of that - * (the starts must coincide), in which case, we go around the loop - * again and ask it to download the next piece. - */ - source = netfs_rreq_prepare_read(rreq, subreq); - if (source == NETFS_INVALID_READ) - goto subreq_failed; - - atomic_inc(&rreq->nr_rd_ops); - - rreq->submitted += subreq->len; - - trace_netfs_sreq(subreq, netfs_sreq_trace_submit); - switch (source) { - case NETFS_FILL_WITH_ZEROES: - netfs_fill_with_zeroes(rreq, subreq); - break; - case NETFS_DOWNLOAD_FROM_SERVER: - netfs_read_from_server(rreq, subreq); - break; - case NETFS_READ_FROM_CACHE: - netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE); - break; - default: - BUG(); - } - - return true; - -subreq_failed: - rreq->error = subreq->error; - netfs_put_subrequest(subreq, false); - return false; -} - -static void netfs_cache_expand_readahead(struct netfs_read_request *rreq, - loff_t *_start, size_t *_len, loff_t i_size) -{ - struct netfs_cache_resources *cres = &rreq->cache_resources; - - if (cres->ops && cres->ops->expand_readahead) - cres->ops->expand_readahead(cres, _start, _len, i_size); -} - -static void netfs_rreq_expand(struct netfs_read_request *rreq, - struct readahead_control *ractl) -{ - /* Give the cache a chance to change the request parameters. The - * resultant request must contain the original region. - */ - netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); - - /* Give the netfs a chance to change the request parameters. The - * resultant request must contain the original region. - */ - if (rreq->netfs_ops->expand_readahead) - rreq->netfs_ops->expand_readahead(rreq); - - /* Expand the request if the cache wants it to start earlier. Note - * that the expansion may get further extended if the VM wishes to - * insert THPs and the preferred start and/or end wind up in the middle - * of THPs. - * - * If this is the case, however, the THP size should be an integer - * multiple of the cache granule size, so we get a whole number of - * granules to deal with. - */ - if (rreq->start != readahead_pos(ractl) || - rreq->len != readahead_length(ractl)) { - readahead_expand(ractl, rreq->start, rreq->len); - rreq->start = readahead_pos(ractl); - rreq->len = readahead_length(ractl); - - trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), - netfs_read_trace_expanded); - } -} - -/** - * netfs_readahead - Helper to manage a read request - * @ractl: The description of the readahead request - * @ops: The network filesystem's operations for the helper to use - * @netfs_priv: Private netfs data to be retained in the request - * - * Fulfil a readahead request by drawing data from the cache if possible, or - * the netfs if not. Space beyond the EOF is zero-filled. Multiple I/O - * requests from different sources will get munged together. If necessary, the - * readahead window can be expanded in either direction to a more convenient - * alighment for RPC efficiency or to make storage in the cache feasible. - * - * The calling netfs must provide a table of operations, only one of which, - * issue_op, is mandatory. It may also be passed a private token, which will - * be retained in rreq->netfs_priv and will be cleaned up by ops->cleanup(). - * - * This is usable whether or not caching is enabled. - */ -void netfs_readahead(struct readahead_control *ractl, - const struct netfs_read_request_ops *ops, - void *netfs_priv) -{ - struct netfs_read_request *rreq; - unsigned int debug_index = 0; - int ret; - - _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl)); - - if (readahead_count(ractl) == 0) - goto cleanup; - - rreq = netfs_alloc_read_request(ops, netfs_priv, ractl->file); - if (!rreq) - goto cleanup; - rreq->mapping = ractl->mapping; - rreq->start = readahead_pos(ractl); - rreq->len = readahead_length(ractl); - - if (ops->begin_cache_operation) { - ret = ops->begin_cache_operation(rreq); - if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) - goto cleanup_free; - } - - netfs_stat(&netfs_n_rh_readahead); - trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), - netfs_read_trace_readahead); - - netfs_rreq_expand(rreq, ractl); - - atomic_set(&rreq->nr_rd_ops, 1); - do { - if (!netfs_rreq_submit_slice(rreq, &debug_index)) - break; - - } while (rreq->submitted < rreq->len); - - /* Drop the refs on the folios here rather than in the cache or - * filesystem. The locks will be dropped in netfs_rreq_unlock(). - */ - while (readahead_folio(ractl)) - ; - - /* If we decrement nr_rd_ops to 0, the ref belongs to us. */ - if (atomic_dec_and_test(&rreq->nr_rd_ops)) - netfs_rreq_assess(rreq, false); - return; - -cleanup_free: - netfs_put_read_request(rreq, false); - return; -cleanup: - if (netfs_priv) - ops->cleanup(ractl->mapping, netfs_priv); - return; -} -EXPORT_SYMBOL(netfs_readahead); - -/** - * netfs_readpage - Helper to manage a readpage request - * @file: The file to read from - * @folio: The folio to read - * @ops: The network filesystem's operations for the helper to use - * @netfs_priv: Private netfs data to be retained in the request - * - * Fulfil a readpage request by drawing data from the cache if possible, or the - * netfs if not. Space beyond the EOF is zero-filled. Multiple I/O requests - * from different sources will get munged together. - * - * The calling netfs must provide a table of operations, only one of which, - * issue_op, is mandatory. It may also be passed a private token, which will - * be retained in rreq->netfs_priv and will be cleaned up by ops->cleanup(). - * - * This is usable whether or not caching is enabled. - */ -int netfs_readpage(struct file *file, - struct folio *folio, - const struct netfs_read_request_ops *ops, - void *netfs_priv) -{ - struct netfs_read_request *rreq; - unsigned int debug_index = 0; - int ret; - - _enter("%lx", folio_index(folio)); - - rreq = netfs_alloc_read_request(ops, netfs_priv, file); - if (!rreq) { - if (netfs_priv) - ops->cleanup(folio_file_mapping(folio), netfs_priv); - folio_unlock(folio); - return -ENOMEM; - } - rreq->mapping = folio_file_mapping(folio); - rreq->start = folio_file_pos(folio); - rreq->len = folio_size(folio); - - if (ops->begin_cache_operation) { - ret = ops->begin_cache_operation(rreq); - if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) { - folio_unlock(folio); - goto out; - } - } - - netfs_stat(&netfs_n_rh_readpage); - trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); - - netfs_get_read_request(rreq); - - atomic_set(&rreq->nr_rd_ops, 1); - do { - if (!netfs_rreq_submit_slice(rreq, &debug_index)) - break; - - } while (rreq->submitted < rreq->len); - - /* Keep nr_rd_ops incremented so that the ref always belongs to us, and - * the service code isn't punted off to a random thread pool to - * process. - */ - do { - wait_var_event(&rreq->nr_rd_ops, atomic_read(&rreq->nr_rd_ops) == 1); - netfs_rreq_assess(rreq, false); - } while (test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags)); - - ret = rreq->error; - if (ret == 0 && rreq->submitted < rreq->len) { - trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_readpage); - ret = -EIO; - } -out: - netfs_put_read_request(rreq, false); - return ret; -} -EXPORT_SYMBOL(netfs_readpage); - -/* - * Prepare a folio for writing without reading first - * @folio: The folio being prepared - * @pos: starting position for the write - * @len: length of write - * - * In some cases, write_begin doesn't need to read at all: - * - full folio write - * - write that lies in a folio that is completely beyond EOF - * - write that covers the folio from start to EOF or beyond it - * - * If any of these criteria are met, then zero out the unwritten parts - * of the folio and return true. Otherwise, return false. - */ -static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len) -{ - struct inode *inode = folio_inode(folio); - loff_t i_size = i_size_read(inode); - size_t offset = offset_in_folio(folio, pos); - - /* Full folio write */ - if (offset == 0 && len >= folio_size(folio)) - return true; - - /* pos beyond last folio in the file */ - if (pos - offset >= i_size) - goto zero_out; - - /* Write that covers from the start of the folio to EOF or beyond */ - if (offset == 0 && (pos + len) >= i_size) - goto zero_out; - - return false; -zero_out: - zero_user_segments(&folio->page, 0, offset, offset + len, folio_size(folio)); - return true; -} - -/** - * netfs_write_begin - Helper to prepare for writing - * @file: The file to read from - * @mapping: The mapping to read from - * @pos: File position at which the write will begin - * @len: The length of the write (may extend beyond the end of the folio chosen) - * @aop_flags: AOP_* flags - * @_folio: Where to put the resultant folio - * @_fsdata: Place for the netfs to store a cookie - * @ops: The network filesystem's operations for the helper to use - * @netfs_priv: Private netfs data to be retained in the request - * - * Pre-read data for a write-begin request by drawing data from the cache if - * possible, or the netfs if not. Space beyond the EOF is zero-filled. - * Multiple I/O requests from different sources will get munged together. If - * necessary, the readahead window can be expanded in either direction to a - * more convenient alighment for RPC efficiency or to make storage in the cache - * feasible. - * - * The calling netfs must provide a table of operations, only one of which, - * issue_op, is mandatory. - * - * The check_write_begin() operation can be provided to check for and flush - * conflicting writes once the folio is grabbed and locked. It is passed a - * pointer to the fsdata cookie that gets returned to the VM to be passed to - * write_end. It is permitted to sleep. It should return 0 if the request - * should go ahead; unlock the folio and return -EAGAIN to cause the folio to - * be regot; or return an error. - * - * This is usable whether or not caching is enabled. - */ -int netfs_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned int len, unsigned int aop_flags, - struct folio **_folio, void **_fsdata, - const struct netfs_read_request_ops *ops, - void *netfs_priv) -{ - struct netfs_read_request *rreq; - struct folio *folio; - struct inode *inode = file_inode(file); - unsigned int debug_index = 0, fgp_flags; - pgoff_t index = pos >> PAGE_SHIFT; - int ret; - - DEFINE_READAHEAD(ractl, file, NULL, mapping, index); - -retry: - fgp_flags = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE; - if (aop_flags & AOP_FLAG_NOFS) - fgp_flags |= FGP_NOFS; - folio = __filemap_get_folio(mapping, index, fgp_flags, - mapping_gfp_mask(mapping)); - if (!folio) - return -ENOMEM; - - if (ops->check_write_begin) { - /* Allow the netfs (eg. ceph) to flush conflicts. */ - ret = ops->check_write_begin(file, pos, len, folio, _fsdata); - if (ret < 0) { - trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin); - if (ret == -EAGAIN) - goto retry; - goto error; - } - } - - if (folio_test_uptodate(folio)) - goto have_folio; - - /* If the page is beyond the EOF, we want to clear it - unless it's - * within the cache granule containing the EOF, in which case we need - * to preload the granule. - */ - if (!ops->is_cache_enabled(inode) && - netfs_skip_folio_read(folio, pos, len)) { - netfs_stat(&netfs_n_rh_write_zskip); - goto have_folio_no_wait; - } - - ret = -ENOMEM; - rreq = netfs_alloc_read_request(ops, netfs_priv, file); - if (!rreq) - goto error; - rreq->mapping = folio_file_mapping(folio); - rreq->start = folio_file_pos(folio); - rreq->len = folio_size(folio); - rreq->no_unlock_folio = folio_index(folio); - __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); - netfs_priv = NULL; - - if (ops->begin_cache_operation) { - ret = ops->begin_cache_operation(rreq); - if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) - goto error_put; - } - - netfs_stat(&netfs_n_rh_write_begin); - trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); - - /* Expand the request to meet caching requirements and download - * preferences. - */ - ractl._nr_pages = folio_nr_pages(folio); - netfs_rreq_expand(rreq, &ractl); - netfs_get_read_request(rreq); - - /* We hold the folio locks, so we can drop the references */ - folio_get(folio); - while (readahead_folio(&ractl)) - ; - - atomic_set(&rreq->nr_rd_ops, 1); - do { - if (!netfs_rreq_submit_slice(rreq, &debug_index)) - break; - - } while (rreq->submitted < rreq->len); - - /* Keep nr_rd_ops incremented so that the ref always belongs to us, and - * the service code isn't punted off to a random thread pool to - * process. - */ - for (;;) { - wait_var_event(&rreq->nr_rd_ops, atomic_read(&rreq->nr_rd_ops) == 1); - netfs_rreq_assess(rreq, false); - if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags)) - break; - cond_resched(); - } - - ret = rreq->error; - if (ret == 0 && rreq->submitted < rreq->len) { - trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_write_begin); - ret = -EIO; - } - netfs_put_read_request(rreq, false); - if (ret < 0) - goto error; - -have_folio: - ret = folio_wait_fscache_killable(folio); - if (ret < 0) - goto error; -have_folio_no_wait: - if (netfs_priv) - ops->cleanup(mapping, netfs_priv); - *_folio = folio; - _leave(" = 0"); - return 0; - -error_put: - netfs_put_read_request(rreq, false); -error: - folio_unlock(folio); - folio_put(folio); - if (netfs_priv) - ops->cleanup(mapping, netfs_priv); - _leave(" = %d", ret); - return ret; -} -EXPORT_SYMBOL(netfs_write_begin); diff --git a/fs/netfs/stats.c b/fs/netfs/stats.c index 9ae538c85378..5510a7a14a40 100644 --- a/fs/netfs/stats.c +++ b/fs/netfs/stats.c @@ -7,7 +7,6 @@ #include <linux/export.h> #include <linux/seq_file.h> -#include <linux/netfs.h> #include "internal.h" atomic_t netfs_n_rh_readahead; diff --git a/fs/nfs/file.c b/fs/nfs/file.c index b0ca244c50d0..150b7fa8f0a7 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -646,7 +646,7 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) result = generic_write_checks(iocb, from); if (result > 0) { current->backing_dev_info = inode_to_bdi(inode); - result = generic_perform_write(file, from, iocb->ki_pos); + result = generic_perform_write(iocb, from); current->backing_dev_info = NULL; } nfs_end_io_write(inode); diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index 4dee53ceb941..f73c09a9cf0a 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -238,14 +238,6 @@ void nfs_fscache_release_file(struct inode *inode, struct file *filp) } } -static inline void fscache_end_operation(struct netfs_cache_resources *cres) -{ - const struct netfs_cache_ops *ops = fscache_operation_valid(cres); - - if (ops) - ops->end_operation(cres); -} - /* * Fallback page reading interface. */ diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index 66bdaa2cf496..ca611ac09f7c 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c @@ -20,6 +20,23 @@ #include "page.h" #include "btnode.h" + +/** + * nilfs_init_btnc_inode - initialize B-tree node cache inode + * @btnc_inode: inode to be initialized + * + * nilfs_init_btnc_inode() sets up an inode for B-tree node cache. + */ +void nilfs_init_btnc_inode(struct inode *btnc_inode) +{ + struct nilfs_inode_info *ii = NILFS_I(btnc_inode); + + btnc_inode->i_mode = S_IFREG; + ii->i_flags = 0; + memset(&ii->i_bmap_data, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(btnc_inode->i_mapping, GFP_NOFS); +} + void nilfs_btnode_cache_clear(struct address_space *btnc) { invalidate_mapping_pages(btnc, 0, -1); @@ -29,7 +46,7 @@ void nilfs_btnode_cache_clear(struct address_space *btnc) struct buffer_head * nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr) { - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; struct buffer_head *bh; bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node)); @@ -57,7 +74,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, struct buffer_head **pbh, sector_t *submit_ptr) { struct buffer_head *bh; - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; struct page *page; int err; @@ -157,7 +174,7 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc, struct nilfs_btnode_chkey_ctxt *ctxt) { struct buffer_head *obh, *nbh; - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey; int err; diff --git a/fs/nilfs2/btnode.h b/fs/nilfs2/btnode.h index 11663650add7..bd5544e63a01 100644 --- a/fs/nilfs2/btnode.h +++ b/fs/nilfs2/btnode.h @@ -30,6 +30,7 @@ struct nilfs_btnode_chkey_ctxt { struct buffer_head *newbh; }; +void nilfs_init_btnc_inode(struct inode *btnc_inode); void nilfs_btnode_cache_clear(struct address_space *); struct buffer_head *nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr); diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index 3594eabe1419..f544c22fff78 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -58,7 +58,8 @@ static void nilfs_btree_free_path(struct nilfs_btree_path *path) static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree, __u64 ptr, struct buffer_head **bhp) { - struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; struct buffer_head *bh; bh = nilfs_btnode_create_block(btnc, ptr); @@ -470,7 +471,8 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, struct buffer_head **bhp, const struct nilfs_btree_readahead_info *ra) { - struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; struct buffer_head *bh, *ra_bh; sector_t submit_ptr = 0; int ret; @@ -1741,6 +1743,10 @@ nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key, dat = nilfs_bmap_get_dat(btree); } + ret = nilfs_attach_btree_node_cache(&NILFS_BMAP_I(btree)->vfs_inode); + if (ret < 0) + return ret; + ret = nilfs_bmap_prepare_alloc_ptr(btree, dreq, dat); if (ret < 0) return ret; @@ -1913,7 +1919,7 @@ static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree, path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr; path[level].bp_ctxt.bh = path[level].bp_bh; ret = nilfs_btnode_prepare_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); if (ret < 0) { nilfs_dat_abort_update(dat, @@ -1939,7 +1945,7 @@ static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree, if (buffer_nilfs_node(path[level].bp_bh)) { nilfs_btnode_commit_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); path[level].bp_bh = path[level].bp_ctxt.bh; } @@ -1958,7 +1964,7 @@ static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree, &path[level].bp_newreq.bpr_req); if (buffer_nilfs_node(path[level].bp_bh)) nilfs_btnode_abort_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); } @@ -2134,7 +2140,8 @@ static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree, static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, struct list_head *listp) { - struct address_space *btcache = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btcache = btnc_inode->i_mapping; struct list_head lists[NILFS_BTREE_LEVEL_MAX]; struct pagevec pvec; struct buffer_head *bh, *head; @@ -2188,12 +2195,12 @@ static int nilfs_btree_assign_p(struct nilfs_bmap *btree, path[level].bp_ctxt.newkey = blocknr; path[level].bp_ctxt.bh = *bh; ret = nilfs_btnode_prepare_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); if (ret < 0) return ret; nilfs_btnode_commit_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); *bh = path[level].bp_ctxt.bh; } @@ -2398,6 +2405,10 @@ int nilfs_btree_init(struct nilfs_bmap *bmap) if (nilfs_btree_root_broken(nilfs_btree_get_root(bmap), bmap->b_inode)) ret = -EIO; + else + ret = nilfs_attach_btree_node_cache( + &NILFS_BMAP_I(bmap)->vfs_inode); + return ret; } diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index dc51d3b7a7bf..3b55e239705f 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -497,7 +497,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size, di = NILFS_DAT_I(dat); lockdep_set_class(&di->mi.mi_sem, &dat_lock_key); nilfs_palloc_setup_cache(dat, &di->palloc_cache); - nilfs_mdt_setup_shadow_map(dat, &di->shadow); + err = nilfs_mdt_setup_shadow_map(dat, &di->shadow); + if (err) + goto failed; err = nilfs_read_inode_common(dat, raw_inode); if (err) diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index a8f5315f01e3..04fdd420eae7 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -126,9 +126,10 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, int nilfs_gccache_submit_read_node(struct inode *inode, sector_t pbn, __u64 vbn, struct buffer_head **out_bh) { + struct inode *btnc_inode = NILFS_I(inode)->i_assoc_inode; int ret; - ret = nilfs_btnode_submit_block(&NILFS_I(inode)->i_btnode_cache, + ret = nilfs_btnode_submit_block(btnc_inode->i_mapping, vbn ? : pbn, pbn, REQ_OP_READ, 0, out_bh, &pbn); if (ret == -EEXIST) /* internal code (cache hit) */ @@ -170,7 +171,7 @@ int nilfs_init_gcinode(struct inode *inode) ii->i_flags = 0; nilfs_bmap_init_gc(ii->i_bmap); - return 0; + return nilfs_attach_btree_node_cache(inode); } /** @@ -185,7 +186,7 @@ void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs) ii = list_first_entry(head, struct nilfs_inode_info, i_dirty); list_del_init(&ii->i_dirty); truncate_inode_pages(&ii->vfs_inode.i_data, 0); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); iput(&ii->vfs_inode); } } diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 476a4a649f38..6045cea21f52 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -29,12 +29,16 @@ * @cno: checkpoint number * @root: pointer on NILFS root object (mounted checkpoint) * @for_gc: inode for GC flag + * @for_btnc: inode for B-tree node cache flag + * @for_shadow: inode for shadowed page cache flag */ struct nilfs_iget_args { u64 ino; __u64 cno; struct nilfs_root *root; - int for_gc; + bool for_gc; + bool for_btnc; + bool for_shadow; }; static int nilfs_iget_test(struct inode *inode, void *opaque); @@ -312,7 +316,8 @@ static int nilfs_insert_inode_locked(struct inode *inode, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false, .for_shadow = false }; return insert_inode_locked4(inode, ino, nilfs_iget_test, &args); @@ -525,6 +530,19 @@ static int nilfs_iget_test(struct inode *inode, void *opaque) return 0; ii = NILFS_I(inode); + if (test_bit(NILFS_I_BTNC, &ii->i_state)) { + if (!args->for_btnc) + return 0; + } else if (args->for_btnc) { + return 0; + } + if (test_bit(NILFS_I_SHADOW, &ii->i_state)) { + if (!args->for_shadow) + return 0; + } else if (args->for_shadow) { + return 0; + } + if (!test_bit(NILFS_I_GCINODE, &ii->i_state)) return !args->for_gc; @@ -536,15 +554,17 @@ static int nilfs_iget_set(struct inode *inode, void *opaque) struct nilfs_iget_args *args = opaque; inode->i_ino = args->ino; - if (args->for_gc) { + NILFS_I(inode)->i_cno = args->cno; + NILFS_I(inode)->i_root = args->root; + if (args->root && args->ino == NILFS_ROOT_INO) + nilfs_get_root(args->root); + + if (args->for_gc) NILFS_I(inode)->i_state = BIT(NILFS_I_GCINODE); - NILFS_I(inode)->i_cno = args->cno; - NILFS_I(inode)->i_root = NULL; - } else { - if (args->root && args->ino == NILFS_ROOT_INO) - nilfs_get_root(args->root); - NILFS_I(inode)->i_root = args->root; - } + if (args->for_btnc) + NILFS_I(inode)->i_state |= BIT(NILFS_I_BTNC); + if (args->for_shadow) + NILFS_I(inode)->i_state |= BIT(NILFS_I_SHADOW); return 0; } @@ -552,7 +572,8 @@ struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false, .for_shadow = false }; return ilookup5(sb, ino, nilfs_iget_test, &args); @@ -562,7 +583,8 @@ struct inode *nilfs_iget_locked(struct super_block *sb, struct nilfs_root *root, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false, .for_shadow = false }; return iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args); @@ -593,7 +615,8 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, __u64 cno) { struct nilfs_iget_args args = { - .ino = ino, .root = NULL, .cno = cno, .for_gc = 1 + .ino = ino, .root = NULL, .cno = cno, .for_gc = true, + .for_btnc = false, .for_shadow = false }; struct inode *inode; int err; @@ -613,6 +636,113 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, return inode; } +/** + * nilfs_attach_btree_node_cache - attach a B-tree node cache to the inode + * @inode: inode object + * + * nilfs_attach_btree_node_cache() attaches a B-tree node cache to @inode, + * or does nothing if the inode already has it. This function allocates + * an additional inode to maintain page cache of B-tree nodes one-on-one. + * + * Return Value: On success, 0 is returned. On errors, one of the following + * negative error code is returned. + * + * %-ENOMEM - Insufficient memory available. + */ +int nilfs_attach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode; + struct nilfs_iget_args args; + + if (ii->i_assoc_inode) + return 0; + + args.ino = inode->i_ino; + args.root = ii->i_root; + args.cno = ii->i_cno; + args.for_gc = test_bit(NILFS_I_GCINODE, &ii->i_state) != 0; + args.for_btnc = true; + args.for_shadow = test_bit(NILFS_I_SHADOW, &ii->i_state) != 0; + + btnc_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!btnc_inode)) + return -ENOMEM; + if (btnc_inode->i_state & I_NEW) { + nilfs_init_btnc_inode(btnc_inode); + unlock_new_inode(btnc_inode); + } + NILFS_I(btnc_inode)->i_assoc_inode = inode; + NILFS_I(btnc_inode)->i_bmap = ii->i_bmap; + ii->i_assoc_inode = btnc_inode; + + return 0; +} + +/** + * nilfs_detach_btree_node_cache - detach the B-tree node cache from the inode + * @inode: inode object + * + * nilfs_detach_btree_node_cache() detaches the B-tree node cache and its + * holder inode bound to @inode, or does nothing if @inode doesn't have it. + */ +void nilfs_detach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; + + if (btnc_inode) { + NILFS_I(btnc_inode)->i_assoc_inode = NULL; + ii->i_assoc_inode = NULL; + iput(btnc_inode); + } +} + +/** + * nilfs_iget_for_shadow - obtain inode for shadow mapping + * @inode: inode object that uses shadow mapping + * + * nilfs_iget_for_shadow() allocates a pair of inodes that holds page + * caches for shadow mapping. The page cache for data pages is set up + * in one inode and the one for b-tree node pages is set up in the + * other inode, which is attached to the former inode. + * + * Return Value: On success, a pointer to the inode for data pages is + * returned. On errors, one of the following negative error code is returned + * in a pointer type. + * + * %-ENOMEM - Insufficient memory available. + */ +struct inode *nilfs_iget_for_shadow(struct inode *inode) +{ + struct nilfs_iget_args args = { + .ino = inode->i_ino, .root = NULL, .cno = 0, .for_gc = false, + .for_btnc = false, .for_shadow = true + }; + struct inode *s_inode; + int err; + + s_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!s_inode)) + return ERR_PTR(-ENOMEM); + if (!(s_inode->i_state & I_NEW)) + return inode; + + NILFS_I(s_inode)->i_flags = 0; + memset(NILFS_I(s_inode)->i_bmap, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(s_inode->i_mapping, GFP_NOFS); + + err = nilfs_attach_btree_node_cache(s_inode); + if (unlikely(err)) { + iget_failed(s_inode); + return ERR_PTR(err); + } + unlock_new_inode(s_inode); + return s_inode; +} + void nilfs_write_inode_common(struct inode *inode, struct nilfs_inode *raw_inode, int has_bmap) { @@ -760,7 +890,8 @@ static void nilfs_clear_inode(struct inode *inode) if (test_bit(NILFS_I_BMAP, &ii->i_state)) nilfs_bmap_clear(ii->i_bmap); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + if (!test_bit(NILFS_I_BTNC, &ii->i_state)) + nilfs_detach_btree_node_cache(inode); if (ii->i_root && inode->i_ino == NILFS_ROOT_INO) nilfs_put_root(ii->i_root); diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index 78db33decd72..d29a0f2b9c16 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -471,9 +471,18 @@ int nilfs_mdt_init(struct inode *inode, gfp_t gfp_mask, size_t objsz) void nilfs_mdt_clear(struct inode *inode) { struct nilfs_mdt_info *mdi = NILFS_MDT(inode); + struct nilfs_shadow_map *shadow = mdi->mi_shadow; if (mdi->mi_palloc_cache) nilfs_palloc_destroy_cache(inode); + + if (shadow) { + struct inode *s_inode = shadow->inode; + + shadow->inode = NULL; + iput(s_inode); + mdi->mi_shadow = NULL; + } } /** @@ -507,12 +516,15 @@ int nilfs_mdt_setup_shadow_map(struct inode *inode, struct nilfs_shadow_map *shadow) { struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct inode *s_inode; INIT_LIST_HEAD(&shadow->frozen_buffers); - address_space_init_once(&shadow->frozen_data); - nilfs_mapping_init(&shadow->frozen_data, inode); - address_space_init_once(&shadow->frozen_btnodes); - nilfs_mapping_init(&shadow->frozen_btnodes, inode); + + s_inode = nilfs_iget_for_shadow(inode); + if (IS_ERR(s_inode)) + return PTR_ERR(s_inode); + + shadow->inode = s_inode; mi->mi_shadow = shadow; return 0; } @@ -526,14 +538,15 @@ int nilfs_mdt_save_to_shadow_map(struct inode *inode) struct nilfs_mdt_info *mi = NILFS_MDT(inode); struct nilfs_inode_info *ii = NILFS_I(inode); struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *s_inode = shadow->inode; int ret; - ret = nilfs_copy_dirty_pages(&shadow->frozen_data, inode->i_mapping); + ret = nilfs_copy_dirty_pages(s_inode->i_mapping, inode->i_mapping); if (ret) goto out; - ret = nilfs_copy_dirty_pages(&shadow->frozen_btnodes, - &ii->i_btnode_cache); + ret = nilfs_copy_dirty_pages(NILFS_I(s_inode)->i_assoc_inode->i_mapping, + ii->i_assoc_inode->i_mapping); if (ret) goto out; @@ -549,7 +562,7 @@ int nilfs_mdt_freeze_buffer(struct inode *inode, struct buffer_head *bh) struct page *page; int blkbits = inode->i_blkbits; - page = grab_cache_page(&shadow->frozen_data, bh->b_page->index); + page = grab_cache_page(shadow->inode->i_mapping, bh->b_page->index); if (!page) return -ENOMEM; @@ -581,7 +594,7 @@ nilfs_mdt_get_frozen_buffer(struct inode *inode, struct buffer_head *bh) struct page *page; int n; - page = find_lock_page(&shadow->frozen_data, bh->b_page->index); + page = find_lock_page(shadow->inode->i_mapping, bh->b_page->index); if (page) { if (page_has_buffers(page)) { n = bh_offset(bh) >> inode->i_blkbits; @@ -622,10 +635,11 @@ void nilfs_mdt_restore_from_shadow_map(struct inode *inode) nilfs_palloc_clear_cache(inode); nilfs_clear_dirty_pages(inode->i_mapping, true); - nilfs_copy_back_pages(inode->i_mapping, &shadow->frozen_data); + nilfs_copy_back_pages(inode->i_mapping, shadow->inode->i_mapping); - nilfs_clear_dirty_pages(&ii->i_btnode_cache, true); - nilfs_copy_back_pages(&ii->i_btnode_cache, &shadow->frozen_btnodes); + nilfs_clear_dirty_pages(ii->i_assoc_inode->i_mapping, true); + nilfs_copy_back_pages(ii->i_assoc_inode->i_mapping, + NILFS_I(shadow->inode)->i_assoc_inode->i_mapping); nilfs_bmap_restore(ii->i_bmap, &shadow->bmap_store); @@ -640,10 +654,11 @@ void nilfs_mdt_clear_shadow_map(struct inode *inode) { struct nilfs_mdt_info *mi = NILFS_MDT(inode); struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *shadow_btnc_inode = NILFS_I(shadow->inode)->i_assoc_inode; down_write(&mi->mi_sem); nilfs_release_frozen_buffers(shadow); - truncate_inode_pages(&shadow->frozen_data, 0); - truncate_inode_pages(&shadow->frozen_btnodes, 0); + truncate_inode_pages(shadow->inode->i_mapping, 0); + truncate_inode_pages(shadow_btnc_inode->i_mapping, 0); up_write(&mi->mi_sem); } diff --git a/fs/nilfs2/mdt.h b/fs/nilfs2/mdt.h index 8f86080a436d..9e23bab3ff12 100644 --- a/fs/nilfs2/mdt.h +++ b/fs/nilfs2/mdt.h @@ -18,14 +18,12 @@ /** * struct nilfs_shadow_map - shadow mapping of meta data file * @bmap_store: shadow copy of bmap state - * @frozen_data: shadowed dirty data pages - * @frozen_btnodes: shadowed dirty b-tree nodes' pages + * @inode: holder of page caches used in shadow mapping * @frozen_buffers: list of frozen buffers */ struct nilfs_shadow_map { struct nilfs_bmap_store bmap_store; - struct address_space frozen_data; - struct address_space frozen_btnodes; + struct inode *inode; struct list_head frozen_buffers; }; diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h index a7b81755c350..1344f7d475d3 100644 --- a/fs/nilfs2/nilfs.h +++ b/fs/nilfs2/nilfs.h @@ -28,7 +28,7 @@ * @i_xattr: <TODO> * @i_dir_start_lookup: page index of last successful search * @i_cno: checkpoint number for GC inode - * @i_btnode_cache: cached pages of b-tree nodes + * @i_assoc_inode: associated inode (B-tree node cache holder or back pointer) * @i_dirty: list for connecting dirty files * @xattr_sem: semaphore for extended attributes processing * @i_bh: buffer contains disk inode @@ -43,7 +43,7 @@ struct nilfs_inode_info { __u64 i_xattr; /* sector_t ??? */ __u32 i_dir_start_lookup; __u64 i_cno; /* check point number for GC inode */ - struct address_space i_btnode_cache; + struct inode *i_assoc_inode; struct list_head i_dirty; /* List for connecting dirty files */ #ifdef CONFIG_NILFS_XATTR @@ -75,13 +75,6 @@ NILFS_BMAP_I(const struct nilfs_bmap *bmap) return container_of(bmap, struct nilfs_inode_info, i_bmap_data); } -static inline struct inode *NILFS_BTNC_I(struct address_space *btnc) -{ - struct nilfs_inode_info *ii = - container_of(btnc, struct nilfs_inode_info, i_btnode_cache); - return &ii->vfs_inode; -} - /* * Dynamic state flags of NILFS on-memory inode (i_state) */ @@ -98,6 +91,8 @@ enum { NILFS_I_INODE_SYNC, /* dsync is not allowed for inode */ NILFS_I_BMAP, /* has bmap and btnode_cache */ NILFS_I_GCINODE, /* inode for GC, on memory only */ + NILFS_I_BTNC, /* inode for btree node cache */ + NILFS_I_SHADOW, /* inode for shadowed page cache */ }; /* @@ -267,6 +262,9 @@ struct inode *nilfs_iget(struct super_block *sb, struct nilfs_root *root, unsigned long ino); extern struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, __u64 cno); +int nilfs_attach_btree_node_cache(struct inode *inode); +void nilfs_detach_btree_node_cache(struct inode *inode); +struct inode *nilfs_iget_for_shadow(struct inode *inode); extern void nilfs_update_inode(struct inode *, struct buffer_head *, int); extern void nilfs_truncate(struct inode *); extern void nilfs_evict_inode(struct inode *); diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 063dd16d75b5..a8e88cc38e16 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -436,22 +436,12 @@ unsigned int nilfs_page_count_clean_buffers(struct page *page, return nc; } -void nilfs_mapping_init(struct address_space *mapping, struct inode *inode) -{ - mapping->host = inode; - mapping->flags = 0; - mapping_set_gfp_mask(mapping, GFP_NOFS); - mapping->private_data = NULL; - mapping->a_ops = &empty_aops; -} - /* * NILFS2 needs clear_page_dirty() in the following two cases: * - * 1) For B-tree node pages and data pages of the dat/gcdat, NILFS2 clears - * page dirty flags when it copies back pages from the shadow cache - * (gcdat->{i_mapping,i_btnode_cache}) to its original cache - * (dat->{i_mapping,i_btnode_cache}). + * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty + * flag of pages when it copies back pages from shadow cache to the + * original cache. * * 2) Some B-tree operations like insertion or deletion may dispose buffers * in dirty state, and this needs to cancel the dirty state of their pages. diff --git a/fs/nilfs2/page.h b/fs/nilfs2/page.h index 569263b23c0c..21ddcdd4d63e 100644 --- a/fs/nilfs2/page.h +++ b/fs/nilfs2/page.h @@ -43,7 +43,6 @@ int nilfs_copy_dirty_pages(struct address_space *, struct address_space *); void nilfs_copy_back_pages(struct address_space *, struct address_space *); void nilfs_clear_dirty_page(struct page *, bool); void nilfs_clear_dirty_pages(struct address_space *, bool); -void nilfs_mapping_init(struct address_space *mapping, struct inode *inode); unsigned int nilfs_page_count_clean_buffers(struct page *, unsigned int, unsigned int); unsigned long nilfs_find_uncommitted_extent(struct inode *inode, diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 85a853334771..0afe0832c754 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -733,15 +733,18 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, struct list_head *listp) { struct nilfs_inode_info *ii = NILFS_I(inode); - struct address_space *mapping = &ii->i_btnode_cache; + struct inode *btnc_inode = ii->i_assoc_inode; struct pagevec pvec; struct buffer_head *bh, *head; unsigned int i; pgoff_t index = 0; + if (!btnc_inode) + return; + pagevec_init(&pvec); - while (pagevec_lookup_tag(&pvec, mapping, &index, + while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, PAGECACHE_TAG_DIRTY)) { for (i = 0; i < pagevec_count(&pvec); i++) { bh = head = page_buffers(pvec.pages[i]); @@ -2410,7 +2413,7 @@ nilfs_remove_written_gcinodes(struct the_nilfs *nilfs, struct list_head *head) continue; list_del_init(&ii->i_dirty); truncate_inode_pages(&ii->vfs_inode.i_data, 0); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); iput(&ii->vfs_inode); } } diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index 3e05c98631ec..ba108f915391 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -157,7 +157,8 @@ struct inode *nilfs_alloc_inode(struct super_block *sb) ii->i_bh = NULL; ii->i_state = 0; ii->i_cno = 0; - nilfs_mapping_init(&ii->i_btnode_cache, &ii->vfs_inode); + ii->i_assoc_inode = NULL; + ii->i_bmap = &ii->i_bmap_data; return &ii->vfs_inode; } @@ -1377,8 +1378,6 @@ static void nilfs_inode_init_once(void *obj) #ifdef CONFIG_NILFS_XATTR init_rwsem(&ii->xattr_sem); #endif - address_space_init_once(&ii->i_btnode_cache); - ii->i_bmap = &ii->i_bmap_data; inode_init_once(&ii->vfs_inode); } diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index d154dcfe06af..90e3dad8ee45 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -1746,7 +1746,7 @@ void mark_ntfs_record_dirty(struct page *page, const unsigned int ofs) { set_buffer_dirty(bh); } while ((bh = bh->b_this_page) != head); spin_unlock(&mapping->private_lock); - block_dirty_folio(mapping, page_folio(page)); + filemap_dirty_folio(mapping, page_folio(page)); if (unlikely(buffers_to_free)) { do { bh = buffers_to_free->b_this_page; diff --git a/fs/ocfs2/quota_global.c b/fs/ocfs2/quota_global.c index 273f65e0aaba..0b6f551a342a 100644 --- a/fs/ocfs2/quota_global.c +++ b/fs/ocfs2/quota_global.c @@ -337,7 +337,6 @@ void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex) /* Read information header from global quota file */ int ocfs2_global_read_info(struct super_block *sb, int type) { - struct inode *gqinode = NULL; unsigned int ino[OCFS2_MAXQUOTAS] = { USER_QUOTA_SYSTEM_INODE, GROUP_QUOTA_SYSTEM_INODE }; struct ocfs2_global_disk_dqinfo dinfo; @@ -346,29 +345,31 @@ int ocfs2_global_read_info(struct super_block *sb, int type) u64 pcount; int status; + oinfo->dqi_gi.dqi_sb = sb; + oinfo->dqi_gi.dqi_type = type; + ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo); + oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk); + oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops; + oinfo->dqi_gqi_bh = NULL; + oinfo->dqi_gqi_count = 0; + /* Read global header */ - gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], + oinfo->dqi_gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], OCFS2_INVALID_SLOT); - if (!gqinode) { + if (!oinfo->dqi_gqinode) { mlog(ML_ERROR, "failed to get global quota inode (type=%d)\n", type); status = -EINVAL; goto out_err; } - oinfo->dqi_gi.dqi_sb = sb; - oinfo->dqi_gi.dqi_type = type; - oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk); - oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops; - oinfo->dqi_gqi_bh = NULL; - oinfo->dqi_gqi_count = 0; - oinfo->dqi_gqinode = gqinode; + status = ocfs2_lock_global_qf(oinfo, 0); if (status < 0) { mlog_errno(status); goto out_err; } - status = ocfs2_extent_map_get_blocks(gqinode, 0, &oinfo->dqi_giblk, + status = ocfs2_extent_map_get_blocks(oinfo->dqi_gqinode, 0, &oinfo->dqi_giblk, &pcount, NULL); if (status < 0) goto out_unlock; diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 0e4b16d4c037..b1a8b046f4c2 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -702,8 +702,6 @@ static int ocfs2_local_read_info(struct super_block *sb, int type) info->dqi_priv = oinfo; oinfo->dqi_type = type; INIT_LIST_HEAD(&oinfo->dqi_chunk); - oinfo->dqi_gqinode = NULL; - ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo); oinfo->dqi_rec = NULL; oinfo->dqi_lqi_bh = NULL; oinfo->dqi_libh = NULL; diff --git a/fs/proc/bootconfig.c b/fs/proc/bootconfig.c index 6d8d4bf20837..2e244ada1f97 100644 --- a/fs/proc/bootconfig.c +++ b/fs/proc/bootconfig.c @@ -32,6 +32,8 @@ static int __init copy_xbc_key_value_list(char *dst, size_t size) int ret = 0; key = kzalloc(XBC_KEYLEN_MAX, GFP_KERNEL); + if (!key) + return -ENOMEM; xbc_for_each_key_value(leaf, val) { ret = xbc_node_compose_key(leaf, key, XBC_KEYLEN_MAX); diff --git a/fs/read_write.c b/fs/read_write.c index dc5000173b80..e643aec2b0ef 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -1630,7 +1630,6 @@ int generic_write_checks_count(struct kiocb *iocb, loff_t *count) if (!*count) return 0; - /* FIXME: this is for backwards compatibility with 2.4 */ if (iocb->ki_flags & IOCB_APPEND) iocb->ki_pos = i_size_read(inode); diff --git a/fs/seq_file.c b/fs/seq_file.c index f8e1f4ee87ff..7ab8a58c29b6 100644 --- a/fs/seq_file.c +++ b/fs/seq_file.c @@ -554,9 +554,9 @@ int seq_dentry(struct seq_file *m, struct dentry *dentry, const char *esc) } EXPORT_SYMBOL(seq_dentry); -static void *single_start(struct seq_file *p, loff_t *pos) +void *single_start(struct seq_file *p, loff_t *pos) { - return NULL + (*pos == 0); + return *pos ? NULL : SEQ_START_TOKEN; } static void *single_next(struct seq_file *p, void *v, loff_t *pos) diff --git a/fs/smbfs_common/smb2pdu.h b/fs/smbfs_common/smb2pdu.h index 38b8fc514860..0507aecfc669 100644 --- a/fs/smbfs_common/smb2pdu.h +++ b/fs/smbfs_common/smb2pdu.h @@ -61,6 +61,40 @@ #define NUMBER_OF_SMB2_COMMANDS 0x0013 /* + * Size of the session key (crypto key encrypted with the password + */ +#define SMB2_NTLMV2_SESSKEY_SIZE 16 +#define SMB2_SIGNATURE_SIZE 16 +#define SMB2_HMACSHA256_SIZE 32 +#define SMB2_CMACAES_SIZE 16 +#define SMB3_GCM128_CRYPTKEY_SIZE 16 +#define SMB3_GCM256_CRYPTKEY_SIZE 32 + +/* + * Size of the smb3 encryption/decryption keys + * This size is big enough to store any cipher key types. + */ +#define SMB3_ENC_DEC_KEY_SIZE 32 + +/* + * Size of the smb3 signing key + */ +#define SMB3_SIGN_KEY_SIZE 16 + +#define CIFS_CLIENT_CHALLENGE_SIZE 8 + +/* Maximum buffer size value we can send with 1 credit */ +#define SMB2_MAX_BUFFER_SIZE 65536 + +/* + * The default wsize is 1M for SMB2 (and for some CIFS cases). + * find_get_pages seems to return a maximum of 256 + * pages in a single call. With PAGE_SIZE == 4k, this means we can + * fill a single wsize request with a single call. + */ +#define SMB3_DEFAULT_IOSIZE (4 * 1024 * 1024) + +/* * SMB2 Header Definition * * "MBZ" : Must be Zero @@ -88,6 +122,15 @@ #define SMB2_FLAGS_DFS_OPERATIONS cpu_to_le32(0x10000000) #define SMB2_FLAGS_REPLAY_OPERATION cpu_to_le32(0x20000000) /* SMB3 & up */ +/* + * Definitions for SMB2 Protocol Data Units (network frames) + * + * See MS-SMB2.PDF specification for protocol details. + * The Naming convention is the lower case version of the SMB2 + * command code name for the struct. Note that structures must be packed. + * + */ + /* See MS-SMB2 section 2.2.1 */ struct smb2_hdr { __le32 ProtocolId; /* 0xFE 'S' 'M' 'B' */ @@ -115,6 +158,18 @@ struct smb2_pdu { __le16 StructureSize2; /* size of wct area (varies, request specific) */ } __packed; +#define SMB2_ERROR_STRUCTURE_SIZE2 9 +#define SMB2_ERROR_STRUCTURE_SIZE2_LE cpu_to_le16(SMB2_ERROR_STRUCTURE_SIZE2) + +struct smb2_err_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; + __u8 ErrorContextCount; + __u8 Reserved; + __le32 ByteCount; /* even if zero, at least one byte follows */ + __u8 ErrorData[1]; /* variable length */ +} __packed; + #define SMB3_AES_CCM_NONCE 11 #define SMB3_AES_GCM_NONCE 12 @@ -608,8 +663,8 @@ struct smb2_close_req { __le16 StructureSize; /* Must be 24 */ __le16 Flags; __le32 Reserved; - __le64 PersistentFileId; /* opaque endianness */ - __le64 VolatileFileId; /* opaque endianness */ + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ } __packed; /* @@ -653,8 +708,8 @@ struct smb2_read_req { __u8 Flags; /* MBZ unless SMB3.02 or later */ __le32 Length; __le64 Offset; - __le64 PersistentFileId; - __le64 VolatileFileId; + __u64 PersistentFileId; + __u64 VolatileFileId; __le32 MinimumCount; __le32 Channel; /* MBZ except for SMB3 or later */ __le32 RemainingBytes; @@ -692,8 +747,8 @@ struct smb2_write_req { __le16 DataOffset; /* offset from start of SMB2 header to write data */ __le32 Length; __le64 Offset; - __le64 PersistentFileId; /* opaque endianness */ - __le64 VolatileFileId; /* opaque endianness */ + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ __le32 Channel; /* MBZ unless SMB3.02 or later */ __le32 RemainingBytes; __le16 WriteChannelInfoOffset; @@ -722,8 +777,8 @@ struct smb2_flush_req { __le16 StructureSize; /* Must be 24 */ __le16 Reserved1; __le32 Reserved2; - __le64 PersistentFileId; - __le64 VolatileFileId; + __u64 PersistentFileId; + __u64 VolatileFileId; } __packed; struct smb2_flush_rsp { @@ -732,6 +787,123 @@ struct smb2_flush_rsp { __le16 Reserved; } __packed; +#define SMB2_LOCKFLAG_SHARED 0x0001 +#define SMB2_LOCKFLAG_EXCLUSIVE 0x0002 +#define SMB2_LOCKFLAG_UNLOCK 0x0004 +#define SMB2_LOCKFLAG_FAIL_IMMEDIATELY 0x0010 +#define SMB2_LOCKFLAG_MASK 0x0007 + +struct smb2_lock_element { + __le64 Offset; + __le64 Length; + __le32 Flags; + __le32 Reserved; +} __packed; + +struct smb2_lock_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 48 */ + __le16 LockCount; + /* + * The least significant four bits are the index, the other 28 bits are + * the lock sequence number (0 to 64). See MS-SMB2 2.2.26 + */ + __le32 LockSequenceNumber; + __u64 PersistentFileId; + __u64 VolatileFileId; + /* Followed by at least one */ + struct smb2_lock_element locks[1]; +} __packed; + +struct smb2_lock_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + +struct smb2_echo_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __u16 Reserved; +} __packed; + +struct smb2_echo_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __u16 Reserved; +} __packed; + +/* + * Valid FileInformation classes for query directory + * + * Note that these are a subset of the (file) QUERY_INFO levels defined + * later in this file (but since QUERY_DIRECTORY uses equivalent numbers + * we do not redefine them here) + * + * FileDirectoryInfomation 0x01 + * FileFullDirectoryInformation 0x02 + * FileIdFullDirectoryInformation 0x26 + * FileBothDirectoryInformation 0x03 + * FileIdBothDirectoryInformation 0x25 + * FileNamesInformation 0x0C + * FileIdExtdDirectoryInformation 0x3C + */ + +/* search (query_directory) Flags field */ +#define SMB2_RESTART_SCANS 0x01 +#define SMB2_RETURN_SINGLE_ENTRY 0x02 +#define SMB2_INDEX_SPECIFIED 0x04 +#define SMB2_REOPEN 0x10 + +struct smb2_query_directory_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 33 */ + __u8 FileInformationClass; + __u8 Flags; + __le32 FileIndex; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le16 FileNameOffset; + __le16 FileNameLength; + __le32 OutputBufferLength; + __u8 Buffer[1]; +} __packed; + +struct smb2_query_directory_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 OutputBufferOffset; + __le32 OutputBufferLength; + __u8 Buffer[1]; +} __packed; + +/* + * Maximum number of iovs we need for a set-info request. + * The largest one is rename/hardlink + * [0] : struct smb2_set_info_req + smb2_file_[rename|link]_info + * [1] : path + * [2] : compound padding + */ +#define SMB2_SET_INFO_IOV_SIZE 3 + +struct smb2_set_info_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 33 */ + __u8 InfoType; + __u8 FileInfoClass; + __le32 BufferLength; + __le16 BufferOffset; + __u16 Reserved; + __le32 AdditionalInformation; + __u64 PersistentFileId; + __u64 VolatileFileId; + __u8 Buffer[1]; +} __packed; + +struct smb2_set_info_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 2 */ +} __packed; /* * SMB2_NOTIFY See MS-SMB2 section 2.2.35 @@ -769,8 +941,8 @@ struct smb2_change_notify_req { __le16 StructureSize; __le16 Flags; __le32 OutputBufferLength; - __le64 PersistentFileId; /* opaque endianness */ - __le64 VolatileFileId; /* opaque endianness */ + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ __le32 CompletionFilter; __u32 Reserved; } __packed; @@ -978,12 +1150,455 @@ struct smb2_create_rsp { __le64 EndofFile; __le32 FileAttributes; __le32 Reserved2; - __le64 PersistentFileId; - __le64 VolatileFileId; + __u64 PersistentFileId; + __u64 VolatileFileId; __le32 CreateContextsOffset; __le32 CreateContextsLength; __u8 Buffer[1]; } __packed; +struct create_posix { + struct create_context ccontext; + __u8 Name[16]; + __le32 Mode; + __u32 Reserved; +} __packed; + +#define SMB2_LEASE_NONE_LE cpu_to_le32(0x00) +#define SMB2_LEASE_READ_CACHING_LE cpu_to_le32(0x01) +#define SMB2_LEASE_HANDLE_CACHING_LE cpu_to_le32(0x02) +#define SMB2_LEASE_WRITE_CACHING_LE cpu_to_le32(0x04) + +#define SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE cpu_to_le32(0x02) + +#define SMB2_LEASE_KEY_SIZE 16 + +struct lease_context { + __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; + __le32 LeaseState; + __le32 LeaseFlags; + __le64 LeaseDuration; +} __packed; + +struct lease_context_v2 { + __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; + __le32 LeaseState; + __le32 LeaseFlags; + __le64 LeaseDuration; + __u8 ParentLeaseKey[SMB2_LEASE_KEY_SIZE]; + __le16 Epoch; + __le16 Reserved; +} __packed; + +struct create_lease { + struct create_context ccontext; + __u8 Name[8]; + struct lease_context lcontext; +} __packed; + +struct create_lease_v2 { + struct create_context ccontext; + __u8 Name[8]; + struct lease_context_v2 lcontext; + __u8 Pad[4]; +} __packed; + +/* See MS-SMB2 2.2.31 and 2.2.32 */ +struct smb2_ioctl_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 57 */ + __le16 Reserved; /* offset from start of SMB2 header to write data */ + __le32 CtlCode; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 InputOffset; /* Reserved MBZ */ + __le32 InputCount; + __le32 MaxInputResponse; + __le32 OutputOffset; + __le32 OutputCount; + __le32 MaxOutputResponse; + __le32 Flags; + __le32 Reserved2; + __u8 Buffer[]; +} __packed; + +struct smb2_ioctl_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 49 */ + __le16 Reserved; + __le32 CtlCode; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 InputOffset; /* Reserved MBZ */ + __le32 InputCount; + __le32 OutputOffset; + __le32 OutputCount; + __le32 Flags; + __le32 Reserved2; + __u8 Buffer[]; +} __packed; + +/* this goes in the ioctl buffer when doing FSCTL_SET_ZERO_DATA */ +struct file_zero_data_information { + __le64 FileOffset; + __le64 BeyondFinalZero; +} __packed; + +/* Reparse structures - see MS-FSCC 2.1.2 */ + +/* struct fsctl_reparse_info_req is empty, only response structs (see below) */ +struct reparse_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __u8 DataBuffer[]; /* Variable Length */ +} __packed; + +struct reparse_guid_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __u8 ReparseGuid[16]; + __u8 DataBuffer[]; /* Variable Length */ +} __packed; + +struct reparse_mount_point_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le16 SubstituteNameOffset; + __le16 SubstituteNameLength; + __le16 PrintNameOffset; + __le16 PrintNameLength; + __u8 PathBuffer[]; /* Variable Length */ +} __packed; + +#define SYMLINK_FLAG_RELATIVE 0x00000001 + +struct reparse_symlink_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le16 SubstituteNameOffset; + __le16 SubstituteNameLength; + __le16 PrintNameOffset; + __le16 PrintNameLength; + __le32 Flags; + __u8 PathBuffer[]; /* Variable Length */ +} __packed; + +/* See MS-FSCC 2.1.2.6 and cifspdu.h for struct reparse_posix_data */ + +struct validate_negotiate_info_req { + __le32 Capabilities; + __u8 Guid[SMB2_CLIENT_GUID_SIZE]; + __le16 SecurityMode; + __le16 DialectCount; + __le16 Dialects[4]; /* BB expand this if autonegotiate > 4 dialects */ +} __packed; + +struct validate_negotiate_info_rsp { + __le32 Capabilities; + __u8 Guid[SMB2_CLIENT_GUID_SIZE]; + __le16 SecurityMode; + __le16 Dialect; /* Dialect in use for the connection */ +} __packed; + +struct duplicate_extents_to_file { + __u64 PersistentFileHandle; /* source file handle, opaque endianness */ + __u64 VolatileFileHandle; + __le64 SourceFileOffset; + __le64 TargetFileOffset; + __le64 ByteCount; /* Bytes to be copied */ +} __packed; + +/* Possible InfoType values */ +#define SMB2_O_INFO_FILE 0x01 +#define SMB2_O_INFO_FILESYSTEM 0x02 +#define SMB2_O_INFO_SECURITY 0x03 +#define SMB2_O_INFO_QUOTA 0x04 + +/* SMB2 Query Info see MS-SMB2 (2.2.37) or MS-DTYP */ + +/* List of QUERY INFO levels (those also valid for QUERY_DIR are noted below */ +#define FILE_DIRECTORY_INFORMATION 1 /* also for QUERY_DIR */ +#define FILE_FULL_DIRECTORY_INFORMATION 2 /* also for QUERY_DIR */ +#define FILE_BOTH_DIRECTORY_INFORMATION 3 /* also for QUERY_DIR */ +#define FILE_BASIC_INFORMATION 4 +#define FILE_STANDARD_INFORMATION 5 +#define FILE_INTERNAL_INFORMATION 6 +#define FILE_EA_INFORMATION 7 +#define FILE_ACCESS_INFORMATION 8 +#define FILE_NAME_INFORMATION 9 +#define FILE_RENAME_INFORMATION 10 +#define FILE_LINK_INFORMATION 11 +#define FILE_NAMES_INFORMATION 12 /* also for QUERY_DIR */ +#define FILE_DISPOSITION_INFORMATION 13 +#define FILE_POSITION_INFORMATION 14 +#define FILE_FULL_EA_INFORMATION 15 +#define FILE_MODE_INFORMATION 16 +#define FILE_ALIGNMENT_INFORMATION 17 +#define FILE_ALL_INFORMATION 18 +#define FILE_ALLOCATION_INFORMATION 19 +#define FILE_END_OF_FILE_INFORMATION 20 +#define FILE_ALTERNATE_NAME_INFORMATION 21 +#define FILE_STREAM_INFORMATION 22 +#define FILE_PIPE_INFORMATION 23 +#define FILE_PIPE_LOCAL_INFORMATION 24 +#define FILE_PIPE_REMOTE_INFORMATION 25 +#define FILE_MAILSLOT_QUERY_INFORMATION 26 +#define FILE_MAILSLOT_SET_INFORMATION 27 +#define FILE_COMPRESSION_INFORMATION 28 +#define FILE_OBJECT_ID_INFORMATION 29 +/* Number 30 not defined in documents */ +#define FILE_MOVE_CLUSTER_INFORMATION 31 +#define FILE_QUOTA_INFORMATION 32 +#define FILE_REPARSE_POINT_INFORMATION 33 +#define FILE_NETWORK_OPEN_INFORMATION 34 +#define FILE_ATTRIBUTE_TAG_INFORMATION 35 +#define FILE_TRACKING_INFORMATION 36 +#define FILEID_BOTH_DIRECTORY_INFORMATION 37 /* also for QUERY_DIR */ +#define FILEID_FULL_DIRECTORY_INFORMATION 38 /* also for QUERY_DIR */ +#define FILE_VALID_DATA_LENGTH_INFORMATION 39 +#define FILE_SHORT_NAME_INFORMATION 40 +#define FILE_SFIO_RESERVE_INFORMATION 44 +#define FILE_SFIO_VOLUME_INFORMATION 45 +#define FILE_HARD_LINK_INFORMATION 46 +#define FILE_NORMALIZED_NAME_INFORMATION 48 +#define FILEID_GLOBAL_TX_DIRECTORY_INFORMATION 50 +#define FILE_STANDARD_LINK_INFORMATION 54 +#define FILE_ID_INFORMATION 59 +#define FILE_ID_EXTD_DIRECTORY_INFORMATION 60 /* also for QUERY_DIR */ +/* Used for Query Info and Find File POSIX Info for SMB3.1.1 and SMB1 */ +#define SMB_FIND_FILE_POSIX_INFO 0x064 + +/* Security info type additionalinfo flags. */ +#define OWNER_SECINFO 0x00000001 +#define GROUP_SECINFO 0x00000002 +#define DACL_SECINFO 0x00000004 +#define SACL_SECINFO 0x00000008 +#define LABEL_SECINFO 0x00000010 +#define ATTRIBUTE_SECINFO 0x00000020 +#define SCOPE_SECINFO 0x00000040 +#define BACKUP_SECINFO 0x00010000 +#define UNPROTECTED_SACL_SECINFO 0x10000000 +#define UNPROTECTED_DACL_SECINFO 0x20000000 +#define PROTECTED_SACL_SECINFO 0x40000000 +#define PROTECTED_DACL_SECINFO 0x80000000 + +/* Flags used for FileFullEAinfo */ +#define SL_RESTART_SCAN 0x00000001 +#define SL_RETURN_SINGLE_ENTRY 0x00000002 +#define SL_INDEX_SPECIFIED 0x00000004 + +struct smb2_query_info_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 41 */ + __u8 InfoType; + __u8 FileInfoClass; + __le32 OutputBufferLength; + __le16 InputBufferOffset; + __u16 Reserved; + __le32 InputBufferLength; + __le32 AdditionalInformation; + __le32 Flags; + __u64 PersistentFileId; + __u64 VolatileFileId; + __u8 Buffer[1]; +} __packed; + +struct smb2_query_info_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 OutputBufferOffset; + __le32 OutputBufferLength; + __u8 Buffer[1]; +} __packed; + +/* + * PDU query infolevel structure definitions + */ + +struct file_allocated_range_buffer { + __le64 file_offset; + __le64 length; +} __packed; + +struct smb2_file_internal_info { + __le64 IndexNumber; +} __packed; /* level 6 Query */ + +struct smb2_file_rename_info { /* encoding of request for level 10 */ + __u8 ReplaceIfExists; /* 1 = replace existing target with new */ + /* 0 = fail if target already exists */ + __u8 Reserved[7]; + __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ + __le32 FileNameLength; + char FileName[]; /* New name to be assigned */ + /* padding - overall struct size must be >= 24 so filename + pad >= 6 */ +} __packed; /* level 10 Set */ + +struct smb2_file_link_info { /* encoding of request for level 11 */ + __u8 ReplaceIfExists; /* 1 = replace existing link with new */ + /* 0 = fail if link already exists */ + __u8 Reserved[7]; + __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ + __le32 FileNameLength; + char FileName[]; /* Name to be assigned to new link */ +} __packed; /* level 11 Set */ + +/* + * This level 18, although with struct with same name is different from cifs + * level 0x107. Level 0x107 has an extra u64 between AccessFlags and + * CurrentByteOffset. + */ +struct smb2_file_all_info { /* data block encoding of response to level 18 */ + __le64 CreationTime; /* Beginning of FILE_BASIC_INFO equivalent */ + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + __u32 Pad1; /* End of FILE_BASIC_INFO_INFO equivalent */ + __le64 AllocationSize; /* Beginning of FILE_STANDARD_INFO equivalent */ + __le64 EndOfFile; /* size ie offset to first free byte in file */ + __le32 NumberOfLinks; /* hard links */ + __u8 DeletePending; + __u8 Directory; + __u16 Pad2; /* End of FILE_STANDARD_INFO equivalent */ + __le64 IndexNumber; + __le32 EASize; + __le32 AccessFlags; + __le64 CurrentByteOffset; + __le32 Mode; + __le32 AlignmentRequirement; + __le32 FileNameLength; + char FileName[1]; +} __packed; /* level 18 Query */ + +struct smb2_file_eof_info { /* encoding of request for level 10 */ + __le64 EndOfFile; /* new end of file value */ +} __packed; /* level 20 Set */ + +/* Level 100 query info */ +struct smb311_posix_qinfo { + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 DosAttributes; + __le64 Inode; + __le32 DeviceId; + __le32 Zero; + /* beginning of POSIX Create Context Response */ + __le32 HardLinks; + __le32 ReparseTag; + __le32 Mode; + u8 Sids[]; + /* + * var sized owner SID + * var sized group SID + * le32 filenamelength + * u8 filename[] + */ +} __packed; + +/* File System Information Classes */ +#define FS_VOLUME_INFORMATION 1 /* Query */ +#define FS_LABEL_INFORMATION 2 /* Set */ +#define FS_SIZE_INFORMATION 3 /* Query */ +#define FS_DEVICE_INFORMATION 4 /* Query */ +#define FS_ATTRIBUTE_INFORMATION 5 /* Query */ +#define FS_CONTROL_INFORMATION 6 /* Query, Set */ +#define FS_FULL_SIZE_INFORMATION 7 /* Query */ +#define FS_OBJECT_ID_INFORMATION 8 /* Query, Set */ +#define FS_DRIVER_PATH_INFORMATION 9 /* Query */ +#define FS_SECTOR_SIZE_INFORMATION 11 /* SMB3 or later. Query */ +#define FS_POSIX_INFORMATION 100 /* SMB3.1.1 POSIX. Query */ + +struct smb2_fs_full_size_info { + __le64 TotalAllocationUnits; + __le64 CallerAvailableAllocationUnits; + __le64 ActualAvailableAllocationUnits; + __le32 SectorsPerAllocationUnit; + __le32 BytesPerSector; +} __packed; + +#define SSINFO_FLAGS_ALIGNED_DEVICE 0x00000001 +#define SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE 0x00000002 +#define SSINFO_FLAGS_NO_SEEK_PENALTY 0x00000004 +#define SSINFO_FLAGS_TRIM_ENABLED 0x00000008 + +/* sector size info struct */ +struct smb3_fs_ss_info { + __le32 LogicalBytesPerSector; + __le32 PhysicalBytesPerSectorForAtomicity; + __le32 PhysicalBytesPerSectorForPerf; + __le32 FSEffPhysicalBytesPerSectorForAtomicity; + __le32 Flags; + __le32 ByteOffsetForSectorAlignment; + __le32 ByteOffsetForPartitionAlignment; +} __packed; + +/* File System Control Information */ +struct smb2_fs_control_info { + __le64 FreeSpaceStartFiltering; + __le64 FreeSpaceThreshold; + __le64 FreeSpaceStopFiltering; + __le64 DefaultQuotaThreshold; + __le64 DefaultQuotaLimit; + __le32 FileSystemControlFlags; + __le32 Padding; +} __packed; + +/* volume info struct - see MS-FSCC 2.5.9 */ +#define MAX_VOL_LABEL_LEN 32 +struct smb3_fs_vol_info { + __le64 VolumeCreationTime; + __u32 VolumeSerialNumber; + __le32 VolumeLabelLength; /* includes trailing null */ + __u8 SupportsObjects; /* True if eg like NTFS, supports objects */ + __u8 Reserved; + __u8 VolumeLabel[]; /* variable len */ +} __packed; + +/* See MS-SMB2 2.2.23 through 2.2.25 */ +struct smb2_oplock_break { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 24 */ + __u8 OplockLevel; + __u8 Reserved; + __le32 Reserved2; + __u64 PersistentFid; + __u64 VolatileFid; +} __packed; + +#define SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED cpu_to_le32(0x01) + +struct smb2_lease_break { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 44 */ + __le16 Epoch; + __le32 Flags; + __u8 LeaseKey[16]; + __le32 CurrentLeaseState; + __le32 NewLeaseState; + __le32 BreakReason; + __le32 AccessMaskHint; + __le32 ShareMaskHint; +} __packed; + +struct smb2_lease_ack { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 36 */ + __le16 Reserved; + __le32 Flags; + __u8 LeaseKey[16]; + __le32 LeaseState; + __le64 LeaseDuration; +} __packed; +#define OP_BREAK_STRUCT_SIZE_20 24 +#define OP_BREAK_STRUCT_SIZE_21 36 #endif /* _COMMON_SMB2PDU_H */ diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index dbe72f664abf..86151889548e 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -349,20 +349,97 @@ out_budg: return err; } -static int do_tmpfile(struct inode *dir, struct dentry *dentry, - umode_t mode, struct inode **whiteout) +static struct inode *create_whiteout(struct inode *dir, struct dentry *dentry) +{ + int err; + umode_t mode = S_IFCHR | WHITEOUT_MODE; + struct inode *inode; + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct fscrypt_name nm; + + /* + * Create an inode('nlink = 1') for whiteout without updating journal, + * let ubifs_jnl_rename() store it on flash to complete rename whiteout + * atomically. + */ + + dbg_gen("dent '%pd', mode %#hx in dir ino %lu", + dentry, mode, dir->i_ino); + + err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm); + if (err) + return ERR_PTR(err); + + inode = ubifs_new_inode(c, dir, mode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_free; + } + + init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); + ubifs_assert(c, inode->i_op == &ubifs_file_inode_operations); + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + /* The dir size is updated by do_rename. */ + insert_inode_hash(inode); + + return inode; + +out_inode: + make_bad_inode(inode); + iput(inode); +out_free: + fscrypt_free_filename(&nm); + ubifs_err(c, "cannot create whiteout file, error %d", err); + return ERR_PTR(err); +} + +/** + * lock_2_inodes - a wrapper for locking two UBIFS inodes. + * @inode1: first inode + * @inode2: second inode + * + * We do not implement any tricks to guarantee strict lock ordering, because + * VFS has already done it for us on the @i_mutex. So this is just a simple + * wrapper function. + */ +static void lock_2_inodes(struct inode *inode1, struct inode *inode2) +{ + mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1); + mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2); +} + +/** + * unlock_2_inodes - a wrapper for unlocking two UBIFS inodes. + * @inode1: first inode + * @inode2: second inode + */ +static void unlock_2_inodes(struct inode *inode1, struct inode *inode2) +{ + mutex_unlock(&ubifs_inode(inode2)->ui_mutex); + mutex_unlock(&ubifs_inode(inode1)->ui_mutex); +} + +static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode) { struct inode *inode; struct ubifs_info *c = dir->i_sb->s_fs_info; - struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1}; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1}; struct ubifs_budget_req ino_req = { .dirtied_ino = 1 }; - struct ubifs_inode *ui, *dir_ui = ubifs_inode(dir); + struct ubifs_inode *ui; int err, instantiated = 0; struct fscrypt_name nm; /* - * Budget request settings: new dirty inode, new direntry, - * budget for dirtied inode will be released via writeback. + * Budget request settings: new inode, new direntry, changing the + * parent directory inode. + * Allocate budget separately for new dirtied inode, the budget will + * be released via writeback. */ dbg_gen("dent '%pd', mode %#hx in dir ino %lu", @@ -392,42 +469,30 @@ static int do_tmpfile(struct inode *dir, struct dentry *dentry, } ui = ubifs_inode(inode); - if (whiteout) { - init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); - ubifs_assert(c, inode->i_op == &ubifs_file_inode_operations); - } - err = ubifs_init_security(dir, inode, &dentry->d_name); if (err) goto out_inode; mutex_lock(&ui->ui_mutex); insert_inode_hash(inode); - - if (whiteout) { - mark_inode_dirty(inode); - drop_nlink(inode); - *whiteout = inode; - } else { - d_tmpfile(dentry, inode); - } + d_tmpfile(dentry, inode); ubifs_assert(c, ui->dirty); instantiated = 1; mutex_unlock(&ui->ui_mutex); - mutex_lock(&dir_ui->ui_mutex); + lock_2_inodes(dir, inode); err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0); if (err) goto out_cancel; - mutex_unlock(&dir_ui->ui_mutex); + unlock_2_inodes(dir, inode); ubifs_release_budget(c, &req); return 0; out_cancel: - mutex_unlock(&dir_ui->ui_mutex); + unlock_2_inodes(dir, inode); out_inode: make_bad_inode(inode); if (!instantiated) @@ -441,12 +506,6 @@ out_budg: return err; } -static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) -{ - return do_tmpfile(dir, dentry, mode, NULL); -} - /** * vfs_dent_type - get VFS directory entry type. * @type: UBIFS directory entry type @@ -660,32 +719,6 @@ static int ubifs_dir_release(struct inode *dir, struct file *file) return 0; } -/** - * lock_2_inodes - a wrapper for locking two UBIFS inodes. - * @inode1: first inode - * @inode2: second inode - * - * We do not implement any tricks to guarantee strict lock ordering, because - * VFS has already done it for us on the @i_mutex. So this is just a simple - * wrapper function. - */ -static void lock_2_inodes(struct inode *inode1, struct inode *inode2) -{ - mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1); - mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2); -} - -/** - * unlock_2_inodes - a wrapper for unlocking two UBIFS inodes. - * @inode1: first inode - * @inode2: second inode - */ -static void unlock_2_inodes(struct inode *inode1, struct inode *inode2) -{ - mutex_unlock(&ubifs_inode(inode2)->ui_mutex); - mutex_unlock(&ubifs_inode(inode1)->ui_mutex); -} - static int ubifs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) { @@ -949,7 +982,8 @@ static int ubifs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, struct ubifs_inode *dir_ui = ubifs_inode(dir); struct ubifs_info *c = dir->i_sb->s_fs_info; int err, sz_change; - struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1 }; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1}; struct fscrypt_name nm; /* @@ -1264,17 +1298,19 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry, .dirtied_ino = 3 }; struct ubifs_budget_req ino_req = { .dirtied_ino = 1, .dirtied_ino_d = ALIGN(old_inode_ui->data_len, 8) }; + struct ubifs_budget_req wht_req; struct timespec64 time; unsigned int saved_nlink; struct fscrypt_name old_nm, new_nm; /* - * Budget request settings: deletion direntry, new direntry, removing - * the old inode, and changing old and new parent directory inodes. + * Budget request settings: + * req: deletion direntry, new direntry, removing the old inode, + * and changing old and new parent directory inodes. + * + * wht_req: new whiteout inode for RENAME_WHITEOUT. * - * However, this operation also marks the target inode as dirty and - * does not write it, so we allocate budget for the target inode - * separately. + * ino_req: marks the target inode as dirty and does not write it. */ dbg_gen("dent '%pd' ino %lu in dir ino %lu to dent '%pd' in dir ino %lu flags 0x%x", @@ -1331,20 +1367,44 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry, goto out_release; } - err = do_tmpfile(old_dir, old_dentry, S_IFCHR | WHITEOUT_MODE, &whiteout); - if (err) { + /* + * The whiteout inode without dentry is pinned in memory, + * umount won't happen during rename process because we + * got parent dentry. + */ + whiteout = create_whiteout(old_dir, old_dentry); + if (IS_ERR(whiteout)) { + err = PTR_ERR(whiteout); kfree(dev); goto out_release; } - spin_lock(&whiteout->i_lock); - whiteout->i_state |= I_LINKABLE; - spin_unlock(&whiteout->i_lock); - whiteout_ui = ubifs_inode(whiteout); whiteout_ui->data = dev; whiteout_ui->data_len = ubifs_encode_dev(dev, MKDEV(0, 0)); ubifs_assert(c, !whiteout_ui->dirty); + + memset(&wht_req, 0, sizeof(struct ubifs_budget_req)); + wht_req.new_ino = 1; + wht_req.new_ino_d = ALIGN(whiteout_ui->data_len, 8); + /* + * To avoid deadlock between space budget (holds ui_mutex and + * waits wb work) and writeback work(waits ui_mutex), do space + * budget before ubifs inodes locked. + */ + err = ubifs_budget_space(c, &wht_req); + if (err) { + /* + * Whiteout inode can not be written on flash by + * ubifs_jnl_write_inode(), because it's neither + * dirty nor zero-nlink. + */ + iput(whiteout); + goto out_release; + } + + /* Add the old_dentry size to the old_dir size. */ + old_sz -= CALC_DENT_SIZE(fname_len(&old_nm)); } lock_4_inodes(old_dir, new_dir, new_inode, whiteout); @@ -1416,29 +1476,11 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry, sync = IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir); if (unlink && IS_SYNC(new_inode)) sync = 1; - } - - if (whiteout) { - struct ubifs_budget_req wht_req = { .dirtied_ino = 1, - .dirtied_ino_d = \ - ALIGN(ubifs_inode(whiteout)->data_len, 8) }; - - err = ubifs_budget_space(c, &wht_req); - if (err) { - kfree(whiteout_ui->data); - whiteout_ui->data_len = 0; - iput(whiteout); - goto out_release; - } - - inc_nlink(whiteout); - mark_inode_dirty(whiteout); - - spin_lock(&whiteout->i_lock); - whiteout->i_state &= ~I_LINKABLE; - spin_unlock(&whiteout->i_lock); - - iput(whiteout); + /* + * S_SYNC flag of whiteout inherits from the old_dir, and we + * have already checked the old dir inode. So there is no need + * to check whiteout. + */ } err = ubifs_jnl_rename(c, old_dir, old_inode, &old_nm, new_dir, @@ -1449,6 +1491,11 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry, unlock_4_inodes(old_dir, new_dir, new_inode, whiteout); ubifs_release_budget(c, &req); + if (whiteout) { + ubifs_release_budget(c, &wht_req); + iput(whiteout); + } + mutex_lock(&old_inode_ui->ui_mutex); release = old_inode_ui->dirty; mark_inode_dirty_sync(old_inode); @@ -1457,11 +1504,16 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry, if (release) ubifs_release_budget(c, &ino_req); if (IS_SYNC(old_inode)) - err = old_inode->i_sb->s_op->write_inode(old_inode, NULL); + /* + * Rename finished here. Although old inode cannot be updated + * on flash, old ctime is not a big problem, don't return err + * code to userspace. + */ + old_inode->i_sb->s_op->write_inode(old_inode, NULL); fscrypt_free_filename(&old_nm); fscrypt_free_filename(&new_nm); - return err; + return 0; out_cancel: if (unlink) { @@ -1482,11 +1534,11 @@ out_cancel: inc_nlink(old_dir); } } + unlock_4_inodes(old_dir, new_dir, new_inode, whiteout); if (whiteout) { - drop_nlink(whiteout); + ubifs_release_budget(c, &wht_req); iput(whiteout); } - unlock_4_inodes(old_dir, new_dir, new_inode, whiteout); out_release: ubifs_release_budget(c, &ino_req); ubifs_release_budget(c, &req); diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index 8a9ffc2d4167..0383fbdc95ff 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -570,7 +570,7 @@ static int ubifs_write_end(struct file *file, struct address_space *mapping, } if (!PagePrivate(page)) { - SetPagePrivate(page); + attach_page_private(page, (void *)1); atomic_long_inc(&c->dirty_pg_cnt); __set_page_dirty_nobuffers(page); } @@ -947,7 +947,7 @@ static int do_writepage(struct page *page, int len) release_existing_page_budget(c); atomic_long_dec(&c->dirty_pg_cnt); - ClearPagePrivate(page); + detach_page_private(page); ClearPageChecked(page); kunmap(page); @@ -1304,7 +1304,7 @@ static void ubifs_invalidate_folio(struct folio *folio, size_t offset, release_existing_page_budget(c); atomic_long_dec(&c->dirty_pg_cnt); - folio_clear_private(folio); + folio_detach_private(folio); folio_clear_checked(folio); } @@ -1471,8 +1471,8 @@ static int ubifs_migrate_page(struct address_space *mapping, return rc; if (PagePrivate(page)) { - ClearPagePrivate(page); - SetPagePrivate(newpage); + detach_page_private(page); + attach_page_private(newpage, (void *)1); } if (mode != MIGRATE_SYNC_NO_COPY) @@ -1496,7 +1496,7 @@ static int ubifs_releasepage(struct page *page, gfp_t unused_gfp_flags) return 0; ubifs_assert(c, PagePrivate(page)); ubifs_assert(c, 0); - ClearPagePrivate(page); + detach_page_private(page); ClearPageChecked(page); return 1; } @@ -1567,7 +1567,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf) else { if (!PageChecked(page)) ubifs_convert_page_budget(c); - SetPagePrivate(page); + attach_page_private(page, (void *)1); atomic_long_inc(&c->dirty_pg_cnt); __set_page_dirty_nobuffers(page); } diff --git a/fs/ubifs/io.c b/fs/ubifs/io.c index 789a7813f3fa..1607a3c76681 100644 --- a/fs/ubifs/io.c +++ b/fs/ubifs/io.c @@ -854,16 +854,42 @@ int ubifs_wbuf_write_nolock(struct ubifs_wbuf *wbuf, void *buf, int len) */ n = aligned_len >> c->max_write_shift; if (n) { - n <<= c->max_write_shift; + int m = n - 1; + dbg_io("write %d bytes to LEB %d:%d", n, wbuf->lnum, wbuf->offs); - err = ubifs_leb_write(c, wbuf->lnum, buf + written, - wbuf->offs, n); + + if (m) { + /* '(n-1)<<c->max_write_shift < len' is always true. */ + m <<= c->max_write_shift; + err = ubifs_leb_write(c, wbuf->lnum, buf + written, + wbuf->offs, m); + if (err) + goto out; + wbuf->offs += m; + aligned_len -= m; + len -= m; + written += m; + } + + /* + * The non-written len of buf may be less than 'n' because + * parameter 'len' is not 8 bytes aligned, so here we read + * min(len, n) bytes from buf. + */ + n = 1 << c->max_write_shift; + memcpy(wbuf->buf, buf + written, min(len, n)); + if (n > len) { + ubifs_assert(c, n - len < 8); + ubifs_pad(c, wbuf->buf + len, n - len); + } + + err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, wbuf->offs, n); if (err) goto out; wbuf->offs += n; aligned_len -= n; - len -= n; + len -= min(len, n); written += n; } diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c index c6a863487780..71bcebe45f9c 100644 --- a/fs/ubifs/ioctl.c +++ b/fs/ubifs/ioctl.c @@ -108,7 +108,7 @@ static int setflags(struct inode *inode, int flags) struct ubifs_inode *ui = ubifs_inode(inode); struct ubifs_info *c = inode->i_sb->s_fs_info; struct ubifs_budget_req req = { .dirtied_ino = 1, - .dirtied_ino_d = ui->data_len }; + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; err = ubifs_budget_space(c, &req); if (err) diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c index 8ea680dba61e..75dab0ae3939 100644 --- a/fs/ubifs/journal.c +++ b/fs/ubifs/journal.c @@ -1207,9 +1207,9 @@ out_free: * @sync: non-zero if the write-buffer has to be synchronized * * This function implements the re-name operation which may involve writing up - * to 4 inodes and 2 directory entries. It marks the written inodes as clean - * and returns zero on success. In case of failure, a negative error code is - * returned. + * to 4 inodes(new inode, whiteout inode, old and new parent directory inodes) + * and 2 directory entries. It marks the written inodes as clean and returns + * zero on success. In case of failure, a negative error code is returned. */ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, const struct inode *old_inode, @@ -1222,14 +1222,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, void *p; union ubifs_key key; struct ubifs_dent_node *dent, *dent2; - int err, dlen1, dlen2, ilen, lnum, offs, len, orphan_added = 0; + int err, dlen1, dlen2, ilen, wlen, lnum, offs, len, orphan_added = 0; int aligned_dlen1, aligned_dlen2, plen = UBIFS_INO_NODE_SZ; int last_reference = !!(new_inode && new_inode->i_nlink == 0); int move = (old_dir != new_dir); - struct ubifs_inode *new_ui; + struct ubifs_inode *new_ui, *whiteout_ui; u8 hash_old_dir[UBIFS_HASH_ARR_SZ]; u8 hash_new_dir[UBIFS_HASH_ARR_SZ]; u8 hash_new_inode[UBIFS_HASH_ARR_SZ]; + u8 hash_whiteout_inode[UBIFS_HASH_ARR_SZ]; u8 hash_dent1[UBIFS_HASH_ARR_SZ]; u8 hash_dent2[UBIFS_HASH_ARR_SZ]; @@ -1249,9 +1250,20 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, } else ilen = 0; + if (whiteout) { + whiteout_ui = ubifs_inode(whiteout); + ubifs_assert(c, mutex_is_locked(&whiteout_ui->ui_mutex)); + ubifs_assert(c, whiteout->i_nlink == 1); + ubifs_assert(c, !whiteout_ui->dirty); + wlen = UBIFS_INO_NODE_SZ; + wlen += whiteout_ui->data_len; + } else + wlen = 0; + aligned_dlen1 = ALIGN(dlen1, 8); aligned_dlen2 = ALIGN(dlen2, 8); - len = aligned_dlen1 + aligned_dlen2 + ALIGN(ilen, 8) + ALIGN(plen, 8); + len = aligned_dlen1 + aligned_dlen2 + ALIGN(ilen, 8) + + ALIGN(wlen, 8) + ALIGN(plen, 8); if (move) len += plen; @@ -1313,6 +1325,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, p += ALIGN(ilen, 8); } + if (whiteout) { + pack_inode(c, p, whiteout, 0); + err = ubifs_node_calc_hash(c, p, hash_whiteout_inode); + if (err) + goto out_release; + + p += ALIGN(wlen, 8); + } + if (!move) { pack_inode(c, p, old_dir, 1); err = ubifs_node_calc_hash(c, p, hash_old_dir); @@ -1352,6 +1373,9 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, if (new_inode) ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, new_inode->i_ino); + if (whiteout) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, + whiteout->i_ino); } release_head(c, BASEHD); @@ -1368,8 +1392,6 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, hash_dent2, old_nm); if (err) goto out_ro; - - ubifs_delete_orphan(c, whiteout->i_ino); } else { err = ubifs_add_dirt(c, lnum, dlen2); if (err) @@ -1390,6 +1412,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, offs += ALIGN(ilen, 8); } + if (whiteout) { + ino_key_init(c, &key, whiteout->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, wlen, + hash_whiteout_inode); + if (err) + goto out_ro; + offs += ALIGN(wlen, 8); + } + ino_key_init(c, &key, old_dir->i_ino); err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_old_dir); if (err) @@ -1410,6 +1441,11 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, new_ui->synced_i_size = new_ui->ui_size; spin_unlock(&new_ui->ui_lock); } + /* + * No need to mark whiteout inode clean. + * Whiteout doesn't have non-zero size, no need to update + * synced_i_size for whiteout_ui. + */ mark_inode_clean(c, ubifs_inode(old_dir)); if (move) mark_inode_clean(c, ubifs_inode(new_dir)); diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h index f55828c0a300..008fa46ef61e 100644 --- a/fs/ubifs/ubifs.h +++ b/fs/ubifs/ubifs.h @@ -381,7 +381,7 @@ struct ubifs_gced_idx_leb { * @ui_mutex exists for two main reasons. At first it prevents inodes from * being written back while UBIFS changing them, being in the middle of an VFS * operation. This way UBIFS makes sure the inode fields are consistent. For - * example, in 'ubifs_rename()' we change 3 inodes simultaneously, and + * example, in 'ubifs_rename()' we change 4 inodes simultaneously, and * write-back must not write any of them before we have finished. * * The second reason is budgeting - UBIFS has to budget all operations. If an diff --git a/fs/unicode/Makefile b/fs/unicode/Makefile index 0cc87423de82..0e51c0025a16 100644 --- a/fs/unicode/Makefile +++ b/fs/unicode/Makefile @@ -33,7 +33,7 @@ $(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE else $(obj)/utf8data.c: $(src)/utf8data.c_shipped FORCE - $(call if_changed,shipped) + $(call if_changed,copy) endif diff --git a/fs/verity/verify.c b/fs/verity/verify.c index 0adb970f4e73..14e2fb49cff5 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Data verification functions, i.e. hooks for ->readpages() + * Data verification functions, i.e. hooks for ->readahead() * * Copyright 2019 Google LLC */ @@ -214,7 +214,7 @@ EXPORT_SYMBOL_GPL(fsverity_verify_page); * that fail verification are set to the Error state. Verification is skipped * for pages already in the Error state, e.g. due to fscrypt decryption failure. * - * This is a helper function for use by the ->readpages() method of filesystems + * This is a helper function for use by the ->readahead() method of filesystems * that issue bios to read data directly into the page cache. Filesystems that * populate the page cache without issuing bios (e.g. non block-based * filesystems) must instead call fsverity_verify_page() directly on each page. diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index 353e53b892e6..b52ed339727f 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -82,6 +82,24 @@ xfs_prealloc_blocks( } /* + * The number of blocks per AG that we withhold from xfs_mod_fdblocks to + * guarantee that we can refill the AGFL prior to allocating space in a nearly + * full AG. Although the the space described by the free space btrees, the + * blocks used by the freesp btrees themselves, and the blocks owned by the + * AGFL are counted in the ondisk fdblocks, it's a mistake to let the ondisk + * free space in the AG drop so low that the free space btrees cannot refill an + * empty AGFL up to the minimum level. Rather than grind through empty AGs + * until the fs goes down, we subtract this many AG blocks from the incore + * fdblocks to ensure user allocation does not overcommit the space the + * filesystem needs for the AGFLs. The rmap btree uses a per-AG reservation to + * withhold space from xfs_mod_fdblocks, so we do not account for that here. + */ +#define XFS_ALLOCBT_AGFL_RESERVE 4 + +/* + * Compute the number of blocks that we set aside to guarantee the ability to + * refill the AGFL and handle a full bmap btree split. + * * In order to avoid ENOSPC-related deadlock caused by out-of-order locking of * AGF buffer (PV 947395), we place constraints on the relationship among * actual allocations for data blocks, freelist blocks, and potential file data @@ -93,14 +111,14 @@ xfs_prealloc_blocks( * extents need to be actually allocated. To get around this, we explicitly set * aside a few blocks which will not be reserved in delayed allocation. * - * We need to reserve 4 fsbs _per AG_ for the freelist and 4 more to handle a - * potential split of the file's bmap btree. + * For each AG, we need to reserve enough blocks to replenish a totally empty + * AGFL and 4 more to handle a potential split of the file's bmap btree. */ unsigned int xfs_alloc_set_aside( struct xfs_mount *mp) { - return mp->m_sb.sb_agcount * (XFS_ALLOC_AGFL_RESERVE + 4); + return mp->m_sb.sb_agcount * (XFS_ALLOCBT_AGFL_RESERVE + 4); } /* @@ -124,12 +142,12 @@ xfs_alloc_ag_max_usable( unsigned int blocks; blocks = XFS_BB_TO_FSB(mp, XFS_FSS_TO_BB(mp, 4)); /* ag headers */ - blocks += XFS_ALLOC_AGFL_RESERVE; + blocks += XFS_ALLOCBT_AGFL_RESERVE; blocks += 3; /* AGF, AGI btree root blocks */ if (xfs_has_finobt(mp)) blocks++; /* finobt root block */ if (xfs_has_rmapbt(mp)) - blocks++; /* rmap root block */ + blocks++; /* rmap root block */ if (xfs_has_reflink(mp)) blocks++; /* refcount root block */ diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h index 1c14a0b1abea..d4c057b764f9 100644 --- a/fs/xfs/libxfs/xfs_alloc.h +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -88,7 +88,6 @@ typedef struct xfs_alloc_arg { #define XFS_ALLOC_NOBUSY (1 << 2)/* Busy extents not allowed */ /* freespace limit calculations */ -#define XFS_ALLOC_AGFL_RESERVE 4 unsigned int xfs_alloc_set_aside(struct xfs_mount *mp); unsigned int xfs_alloc_ag_max_usable(struct xfs_mount *mp); diff --git a/fs/xfs/xfs_bio_io.c b/fs/xfs/xfs_bio_io.c index 32fa02945f73..ae4345b37621 100644 --- a/fs/xfs/xfs_bio_io.c +++ b/fs/xfs/xfs_bio_io.c @@ -9,39 +9,6 @@ static inline unsigned int bio_max_vecs(unsigned int count) return bio_max_segs(howmany(count, PAGE_SIZE)); } -static void -xfs_flush_bdev_async_endio( - struct bio *bio) -{ - complete(bio->bi_private); -} - -/* - * Submit a request for an async cache flush to run. If the request queue does - * not require flush operations, just skip it altogether. If the caller needs - * to wait for the flush completion at a later point in time, they must supply a - * valid completion. This will be signalled when the flush completes. The - * caller never sees the bio that is issued here. - */ -void -xfs_flush_bdev_async( - struct bio *bio, - struct block_device *bdev, - struct completion *done) -{ - struct request_queue *q = bdev->bd_disk->queue; - - if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { - complete(done); - return; - } - - bio_init(bio, bdev, NULL, 0, REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); - bio->bi_private = done; - bio->bi_end_io = xfs_flush_bdev_async_endio; - - submit_bio(bio); -} int xfs_rw_bdev( struct block_device *bdev, diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 33e26690a8c4..68f74549fa22 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -17,6 +17,7 @@ #include "xfs_fsops.h" #include "xfs_trans_space.h" #include "xfs_log.h" +#include "xfs_log_priv.h" #include "xfs_ag.h" #include "xfs_ag_resv.h" #include "xfs_trace.h" @@ -347,7 +348,7 @@ xfs_fs_counts( cnt->allocino = percpu_counter_read_positive(&mp->m_icount); cnt->freeino = percpu_counter_read_positive(&mp->m_ifree); cnt->freedata = percpu_counter_read_positive(&mp->m_fdblocks) - - mp->m_alloc_set_aside; + xfs_fdblocks_unavailable(mp); spin_lock(&mp->m_sb_lock); cnt->freertx = mp->m_sb.sb_frextents; @@ -430,46 +431,36 @@ xfs_reserve_blocks( * If the request is larger than the current reservation, reserve the * blocks before we update the reserve counters. Sample m_fdblocks and * perform a partial reservation if the request exceeds free space. + * + * The code below estimates how many blocks it can request from + * fdblocks to stash in the reserve pool. This is a classic TOCTOU + * race since fdblocks updates are not always coordinated via + * m_sb_lock. Set the reserve size even if there's not enough free + * space to fill it because mod_fdblocks will refill an undersized + * reserve when it can. */ - error = -ENOSPC; - do { - free = percpu_counter_sum(&mp->m_fdblocks) - - mp->m_alloc_set_aside; - if (free <= 0) - break; - - delta = request - mp->m_resblks; - lcounter = free - delta; - if (lcounter < 0) - /* We can't satisfy the request, just get what we can */ - fdblks_delta = free; - else - fdblks_delta = delta; - + free = percpu_counter_sum(&mp->m_fdblocks) - + xfs_fdblocks_unavailable(mp); + delta = request - mp->m_resblks; + mp->m_resblks = request; + if (delta > 0 && free > 0) { /* * We'll either succeed in getting space from the free block - * count or we'll get an ENOSPC. If we get a ENOSPC, it means - * things changed while we were calculating fdblks_delta and so - * we should try again to see if there is anything left to - * reserve. + * count or we'll get an ENOSPC. Don't set the reserved flag + * here - we don't want to reserve the extra reserve blocks + * from the reserve. * - * Don't set the reserved flag here - we don't want to reserve - * the extra reserve blocks from the reserve..... + * The desired reserve size can change after we drop the lock. + * Use mod_fdblocks to put the space into the reserve or into + * fdblocks as appropriate. */ + fdblks_delta = min(free, delta); spin_unlock(&mp->m_sb_lock); error = xfs_mod_fdblocks(mp, -fdblks_delta, 0); + if (!error) + xfs_mod_fdblocks(mp, fdblks_delta, 0); spin_lock(&mp->m_sb_lock); - } while (error == -ENOSPC); - - /* - * Update the reserve counters if blocks have been successfully - * allocated. - */ - if (!error && fdblks_delta) { - mp->m_resblks += fdblks_delta; - mp->m_resblks_avail += fdblks_delta; } - out: if (outval) { outval->resblks = mp->m_resblks; @@ -528,8 +519,11 @@ xfs_do_force_shutdown( int tag; const char *why; - if (test_and_set_bit(XFS_OPSTATE_SHUTDOWN, &mp->m_opstate)) + + if (test_and_set_bit(XFS_OPSTATE_SHUTDOWN, &mp->m_opstate)) { + xlog_shutdown_wait(mp->m_log); return; + } if (mp->m_sb_bp) mp->m_sb_bp->b_flags |= XBF_DONE; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 20186c584c7d..bffd6eb0b298 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -883,7 +883,7 @@ xfs_reclaim_inode( */ if (xlog_is_shutdown(ip->i_mount->m_log)) { xfs_iunpin_wait(ip); - xfs_iflush_abort(ip); + xfs_iflush_shutdown_abort(ip); goto reclaim; } if (xfs_ipincount(ip)) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 26227d26f274..9de6205fe134 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3631,7 +3631,7 @@ xfs_iflush_cluster( /* * We must use the safe variant here as on shutdown xfs_iflush_abort() - * can remove itself from the list. + * will remove itself from the list. */ list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) { iip = (struct xfs_inode_log_item *)lip; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 11158fa81a09..9e6ef55cf29e 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -544,10 +544,17 @@ xfs_inode_item_push( uint rval = XFS_ITEM_SUCCESS; int error; - ASSERT(iip->ili_item.li_buf); + if (!bp || (ip->i_flags & XFS_ISTALE)) { + /* + * Inode item/buffer is being being aborted due to cluster + * buffer deletion. Trigger a log force to have that operation + * completed and items removed from the AIL before the next push + * attempt. + */ + return XFS_ITEM_PINNED; + } - if (xfs_ipincount(ip) > 0 || xfs_buf_ispinned(bp) || - (ip->i_flags & XFS_ISTALE)) + if (xfs_ipincount(ip) > 0 || xfs_buf_ispinned(bp)) return XFS_ITEM_PINNED; if (xfs_iflags_test(ip, XFS_IFLUSHING)) @@ -834,46 +841,143 @@ xfs_buf_inode_io_fail( } /* - * This is the inode flushing abort routine. It is called when - * the filesystem is shutting down to clean up the inode state. It is - * responsible for removing the inode item from the AIL if it has not been - * re-logged and clearing the inode's flush state. + * Clear the inode logging fields so no more flushes are attempted. If we are + * on a buffer list, it is now safe to remove it because the buffer is + * guaranteed to be locked. The caller will drop the reference to the buffer + * the log item held. + */ +static void +xfs_iflush_abort_clean( + struct xfs_inode_log_item *iip) +{ + iip->ili_last_fields = 0; + iip->ili_fields = 0; + iip->ili_fsync_fields = 0; + iip->ili_flush_lsn = 0; + iip->ili_item.li_buf = NULL; + list_del_init(&iip->ili_item.li_bio_list); +} + +/* + * Abort flushing the inode from a context holding the cluster buffer locked. + * + * This is the normal runtime method of aborting writeback of an inode that is + * attached to a cluster buffer. It occurs when the inode and the backing + * cluster buffer have been freed (i.e. inode is XFS_ISTALE), or when cluster + * flushing or buffer IO completion encounters a log shutdown situation. + * + * If we need to abort inode writeback and we don't already hold the buffer + * locked, call xfs_iflush_shutdown_abort() instead as this should only ever be + * necessary in a shutdown situation. */ void xfs_iflush_abort( struct xfs_inode *ip) { struct xfs_inode_log_item *iip = ip->i_itemp; - struct xfs_buf *bp = NULL; + struct xfs_buf *bp; - if (iip) { - /* - * Clear the failed bit before removing the item from the AIL so - * xfs_trans_ail_delete() doesn't try to clear and release the - * buffer attached to the log item before we are done with it. - */ - clear_bit(XFS_LI_FAILED, &iip->ili_item.li_flags); - xfs_trans_ail_delete(&iip->ili_item, 0); + if (!iip) { + /* clean inode, nothing to do */ + xfs_iflags_clear(ip, XFS_IFLUSHING); + return; + } + + /* + * Remove the inode item from the AIL before we clear its internal + * state. Whilst the inode is in the AIL, it should have a valid buffer + * pointer for push operations to access - it is only safe to remove the + * inode from the buffer once it has been removed from the AIL. + * + * We also clear the failed bit before removing the item from the AIL + * as xfs_trans_ail_delete()->xfs_clear_li_failed() will release buffer + * references the inode item owns and needs to hold until we've fully + * aborted the inode log item and detached it from the buffer. + */ + clear_bit(XFS_LI_FAILED, &iip->ili_item.li_flags); + xfs_trans_ail_delete(&iip->ili_item, 0); + + /* + * Grab the inode buffer so can we release the reference the inode log + * item holds on it. + */ + spin_lock(&iip->ili_lock); + bp = iip->ili_item.li_buf; + xfs_iflush_abort_clean(iip); + spin_unlock(&iip->ili_lock); + xfs_iflags_clear(ip, XFS_IFLUSHING); + if (bp) + xfs_buf_rele(bp); +} + +/* + * Abort an inode flush in the case of a shutdown filesystem. This can be called + * from anywhere with just an inode reference and does not require holding the + * inode cluster buffer locked. If the inode is attached to a cluster buffer, + * it will grab and lock it safely, then abort the inode flush. + */ +void +xfs_iflush_shutdown_abort( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_buf *bp; + + if (!iip) { + /* clean inode, nothing to do */ + xfs_iflags_clear(ip, XFS_IFLUSHING); + return; + } + + spin_lock(&iip->ili_lock); + bp = iip->ili_item.li_buf; + if (!bp) { + spin_unlock(&iip->ili_lock); + xfs_iflush_abort(ip); + return; + } + + /* + * We have to take a reference to the buffer so that it doesn't get + * freed when we drop the ili_lock and then wait to lock the buffer. + * We'll clean up the extra reference after we pick up the ili_lock + * again. + */ + xfs_buf_hold(bp); + spin_unlock(&iip->ili_lock); + xfs_buf_lock(bp); + + spin_lock(&iip->ili_lock); + if (!iip->ili_item.li_buf) { /* - * Clear the inode logging fields so no more flushes are - * attempted. + * Raced with another removal, hold the only reference + * to bp now. Inode should not be in the AIL now, so just clean + * up and return; */ - spin_lock(&iip->ili_lock); - iip->ili_last_fields = 0; - iip->ili_fields = 0; - iip->ili_fsync_fields = 0; - iip->ili_flush_lsn = 0; - bp = iip->ili_item.li_buf; - iip->ili_item.li_buf = NULL; - list_del_init(&iip->ili_item.li_bio_list); + ASSERT(list_empty(&iip->ili_item.li_bio_list)); + ASSERT(!test_bit(XFS_LI_IN_AIL, &iip->ili_item.li_flags)); + xfs_iflush_abort_clean(iip); spin_unlock(&iip->ili_lock); + xfs_iflags_clear(ip, XFS_IFLUSHING); + xfs_buf_relse(bp); + return; } - xfs_iflags_clear(ip, XFS_IFLUSHING); - if (bp) - xfs_buf_rele(bp); + + /* + * Got two references to bp. The first will get dropped by + * xfs_iflush_abort() when the item is removed from the buffer list, but + * we can't drop our reference until _abort() returns because we have to + * unlock the buffer as well. Hence we abort and then unlock and release + * our reference to the buffer. + */ + ASSERT(iip->ili_item.li_buf == bp); + spin_unlock(&iip->ili_lock); + xfs_iflush_abort(ip); + xfs_buf_relse(bp); } + /* * convert an xfs_inode_log_format struct from the old 32 bit version * (which can have different field alignments) to the native 64 bit version diff --git a/fs/xfs/xfs_inode_item.h b/fs/xfs/xfs_inode_item.h index 1a302000d604..bbd836a44ff0 100644 --- a/fs/xfs/xfs_inode_item.h +++ b/fs/xfs/xfs_inode_item.h @@ -44,6 +44,7 @@ static inline int xfs_inode_clean(struct xfs_inode *ip) extern void xfs_inode_item_init(struct xfs_inode *, struct xfs_mount *); extern void xfs_inode_item_destroy(struct xfs_inode *); extern void xfs_iflush_abort(struct xfs_inode *); +extern void xfs_iflush_shutdown_abort(struct xfs_inode *); extern int xfs_inode_item_format_convert(xfs_log_iovec_t *, struct xfs_inode_log_format *); diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index 09a8fba84ff9..cb9105d667db 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -197,8 +197,6 @@ static inline uint64_t howmany_64(uint64_t x, uint32_t y) int xfs_rw_bdev(struct block_device *bdev, sector_t sector, unsigned int count, char *data, unsigned int op); -void xfs_flush_bdev_async(struct bio *bio, struct block_device *bdev, - struct completion *done); #define ASSERT_ALWAYS(expr) \ (likely(expr) ? (void)0 : assfail(NULL, #expr, __FILE__, __LINE__)) diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index a8034c0afdf2..499e15b24215 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -487,7 +487,10 @@ out_error: * Run all the pending iclog callbacks and wake log force waiters and iclog * space waiters so they can process the newly set shutdown state. We really * don't care what order we process callbacks here because the log is shut down - * and so state cannot change on disk anymore. + * and so state cannot change on disk anymore. However, we cannot wake waiters + * until the callbacks have been processed because we may be in unmount and + * we must ensure that all AIL operations the callbacks perform have completed + * before we tear down the AIL. * * We avoid processing actively referenced iclogs so that we don't run callbacks * while the iclog owner might still be preparing the iclog for IO submssion. @@ -501,7 +504,6 @@ xlog_state_shutdown_callbacks( struct xlog_in_core *iclog; LIST_HEAD(cb_list); - spin_lock(&log->l_icloglock); iclog = log->l_iclog; do { if (atomic_read(&iclog->ic_refcnt)) { @@ -509,26 +511,22 @@ xlog_state_shutdown_callbacks( continue; } list_splice_init(&iclog->ic_callbacks, &cb_list); + spin_unlock(&log->l_icloglock); + + xlog_cil_process_committed(&cb_list); + + spin_lock(&log->l_icloglock); wake_up_all(&iclog->ic_write_wait); wake_up_all(&iclog->ic_force_wait); } while ((iclog = iclog->ic_next) != log->l_iclog); wake_up_all(&log->l_flush_wait); - spin_unlock(&log->l_icloglock); - - xlog_cil_process_committed(&cb_list); } /* * Flush iclog to disk if this is the last reference to the given iclog and the * it is in the WANT_SYNC state. * - * If the caller passes in a non-zero @old_tail_lsn and the current log tail - * does not match, there may be metadata on disk that must be persisted before - * this iclog is written. To satisfy that requirement, set the - * XLOG_ICL_NEED_FLUSH flag as a condition for writing this iclog with the new - * log tail value. - * * If XLOG_ICL_NEED_FUA is already set on the iclog, we need to ensure that the * log tail is updated correctly. NEED_FUA indicates that the iclog will be * written to stable storage, and implies that a commit record is contained @@ -545,12 +543,10 @@ xlog_state_shutdown_callbacks( * always capture the tail lsn on the iclog on the first NEED_FUA release * regardless of the number of active reference counts on this iclog. */ - int xlog_state_release_iclog( struct xlog *log, - struct xlog_in_core *iclog, - xfs_lsn_t old_tail_lsn) + struct xlog_in_core *iclog) { xfs_lsn_t tail_lsn; bool last_ref; @@ -561,18 +557,14 @@ xlog_state_release_iclog( /* * Grabbing the current log tail needs to be atomic w.r.t. the writing * of the tail LSN into the iclog so we guarantee that the log tail does - * not move between deciding if a cache flush is required and writing - * the LSN into the iclog below. + * not move between the first time we know that the iclog needs to be + * made stable and when we eventually submit it. */ - if (old_tail_lsn || iclog->ic_state == XLOG_STATE_WANT_SYNC) { + if ((iclog->ic_state == XLOG_STATE_WANT_SYNC || + (iclog->ic_flags & XLOG_ICL_NEED_FUA)) && + !iclog->ic_header.h_tail_lsn) { tail_lsn = xlog_assign_tail_lsn(log->l_mp); - - if (old_tail_lsn && tail_lsn != old_tail_lsn) - iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; - - if ((iclog->ic_flags & XLOG_ICL_NEED_FUA) && - !iclog->ic_header.h_tail_lsn) - iclog->ic_header.h_tail_lsn = cpu_to_be64(tail_lsn); + iclog->ic_header.h_tail_lsn = cpu_to_be64(tail_lsn); } last_ref = atomic_dec_and_test(&iclog->ic_refcnt); @@ -583,11 +575,8 @@ xlog_state_release_iclog( * pending iclog callbacks that were waiting on the release of * this iclog. */ - if (last_ref) { - spin_unlock(&log->l_icloglock); + if (last_ref) xlog_state_shutdown_callbacks(log); - spin_lock(&log->l_icloglock); - } return -EIO; } @@ -600,8 +589,6 @@ xlog_state_release_iclog( } iclog->ic_state = XLOG_STATE_SYNCING; - if (!iclog->ic_header.h_tail_lsn) - iclog->ic_header.h_tail_lsn = cpu_to_be64(tail_lsn); xlog_verify_tail_lsn(log, iclog); trace_xlog_iclog_syncing(iclog, _RET_IP_); @@ -873,7 +860,7 @@ xlog_force_iclog( iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA; if (iclog->ic_state == XLOG_STATE_ACTIVE) xlog_state_switch_iclogs(iclog->ic_log, iclog, 0); - return xlog_state_release_iclog(iclog->ic_log, iclog, 0); + return xlog_state_release_iclog(iclog->ic_log, iclog); } /* @@ -1373,7 +1360,7 @@ xlog_ioend_work( */ if (XFS_TEST_ERROR(error, log->l_mp, XFS_ERRTAG_IODONE_IOERR)) { xfs_alert(log->l_mp, "log I/O error %d", error); - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); } xlog_state_done_syncing(iclog); @@ -1912,7 +1899,7 @@ xlog_write_iclog( iclog->ic_flags &= ~(XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA); if (xlog_map_iclog_data(&iclog->ic_bio, iclog->ic_data, count)) { - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); return; } if (is_vmalloc_addr(iclog->ic_data)) @@ -2411,7 +2398,7 @@ xlog_write_copy_finish( ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC || xlog_is_shutdown(log)); release_iclog: - error = xlog_state_release_iclog(log, iclog, 0); + error = xlog_state_release_iclog(log, iclog); spin_unlock(&log->l_icloglock); return error; } @@ -2487,7 +2474,7 @@ xlog_write( xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES, "ctx ticket reservation ran out. Need to up reservation"); xlog_print_tic_res(log->l_mp, ticket); - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); } len = xlog_write_calc_vec_length(ticket, log_vector, optype); @@ -2628,7 +2615,7 @@ next_lv: spin_lock(&log->l_icloglock); xlog_state_finish_copy(log, iclog, record_cnt, data_cnt); - error = xlog_state_release_iclog(log, iclog, 0); + error = xlog_state_release_iclog(log, iclog); spin_unlock(&log->l_icloglock); return error; @@ -3052,7 +3039,7 @@ restart: * reference to the iclog. */ if (!atomic_add_unless(&iclog->ic_refcnt, -1, 1)) - error = xlog_state_release_iclog(log, iclog, 0); + error = xlog_state_release_iclog(log, iclog); spin_unlock(&log->l_icloglock); if (error) return error; @@ -3821,9 +3808,10 @@ xlog_verify_iclog( #endif /* - * Perform a forced shutdown on the log. This should be called once and once - * only by the high level filesystem shutdown code to shut the log subsystem - * down cleanly. + * Perform a forced shutdown on the log. + * + * This can be called from low level log code to trigger a shutdown, or from the + * high level mount shutdown code when the mount shuts down. * * Our main objectives here are to make sure that: * a. if the shutdown was not due to a log IO error, flush the logs to @@ -3832,6 +3820,8 @@ xlog_verify_iclog( * parties to find out. Nothing new gets queued after this is done. * c. Tasks sleeping on log reservations, pinned objects and * other resources get woken up. + * d. The mount is also marked as shut down so that log triggered shutdowns + * still behave the same as if they called xfs_forced_shutdown(). * * Return true if the shutdown cause was a log IO error and we actually shut the * log down. @@ -3843,25 +3833,25 @@ xlog_force_shutdown( { bool log_error = (shutdown_flags & SHUTDOWN_LOG_IO_ERROR); - /* - * If this happens during log recovery then we aren't using the runtime - * log mechanisms yet so there's nothing to shut down. - */ - if (!log || xlog_in_recovery(log)) + if (!log) return false; - ASSERT(!xlog_is_shutdown(log)); - /* * Flush all the completed transactions to disk before marking the log * being shut down. We need to do this first as shutting down the log * before the force will prevent the log force from flushing the iclogs * to disk. * - * Re-entry due to a log IO error shutdown during the log force is - * prevented by the atomicity of higher level shutdown code. + * When we are in recovery, there are no transactions to flush, and + * we don't want to touch the log because we don't want to perturb the + * current head/tail for future recovery attempts. Hence we need to + * avoid a log force in this case. + * + * If we are shutting down due to a log IO error, then we must avoid + * trying to write the log as that may just result in more IO errors and + * an endless shutdown/force loop. */ - if (!log_error) + if (!log_error && !xlog_in_recovery(log)) xfs_log_force(log->l_mp, XFS_LOG_SYNC); /* @@ -3878,12 +3868,25 @@ xlog_force_shutdown( spin_lock(&log->l_icloglock); if (test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate)) { spin_unlock(&log->l_icloglock); - ASSERT(0); return false; } spin_unlock(&log->l_icloglock); /* + * If this log shutdown also sets the mount shutdown state, issue a + * shutdown warning message. + */ + if (!test_and_set_bit(XFS_OPSTATE_SHUTDOWN, &log->l_mp->m_opstate)) { + xfs_alert_tag(log->l_mp, XFS_PTAG_SHUTDOWN_LOGERROR, +"Filesystem has been shut down due to log error (0x%x).", + shutdown_flags); + xfs_alert(log->l_mp, +"Please unmount the filesystem and rectify the problem(s)."); + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); + } + + /* * We don't want anybody waiting for log reservations after this. That * means we have to wake up everybody queued up on reserveq as well as * writeq. In addition, we make sure in xlog_{re}grant_log_space that @@ -3903,8 +3906,12 @@ xlog_force_shutdown( wake_up_all(&log->l_cilp->xc_start_wait); wake_up_all(&log->l_cilp->xc_commit_wait); spin_unlock(&log->l_cilp->xc_push_lock); + + spin_lock(&log->l_icloglock); xlog_state_shutdown_callbacks(log); + spin_unlock(&log->l_icloglock); + wake_up_var(&log->l_opstate); return log_error; } diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c index 796e4464f809..ba57323bfdce 100644 --- a/fs/xfs/xfs_log_cil.c +++ b/fs/xfs/xfs_log_cil.c @@ -540,7 +540,7 @@ xlog_cil_insert_items( spin_unlock(&cil->xc_cil_lock); if (tp->t_ticket->t_curr_res < 0) - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); } static void @@ -705,11 +705,21 @@ xlog_cil_set_ctx_write_state( * The LSN we need to pass to the log items on transaction * commit is the LSN reported by the first log vector write, not * the commit lsn. If we use the commit record lsn then we can - * move the tail beyond the grant write head. + * move the grant write head beyond the tail LSN and overwrite + * it. */ ctx->start_lsn = lsn; wake_up_all(&cil->xc_start_wait); spin_unlock(&cil->xc_push_lock); + + /* + * Make sure the metadata we are about to overwrite in the log + * has been flushed to stable storage before this iclog is + * issued. + */ + spin_lock(&cil->xc_log->l_icloglock); + iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; + spin_unlock(&cil->xc_log->l_icloglock); return; } @@ -854,7 +864,7 @@ xlog_cil_write_commit_record( error = xlog_write(log, ctx, &vec, ctx->ticket, XLOG_COMMIT_TRANS); if (error) - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); return error; } @@ -888,10 +898,7 @@ xlog_cil_push_work( struct xfs_trans_header thdr; struct xfs_log_iovec lhdr; struct xfs_log_vec lvhdr = { NULL }; - xfs_lsn_t preflush_tail_lsn; xfs_csn_t push_seq; - struct bio bio; - DECLARE_COMPLETION_ONSTACK(bdev_flush); bool push_commit_stable; new_ctx = xlog_cil_ctx_alloc(); @@ -962,23 +969,6 @@ xlog_cil_push_work( spin_unlock(&cil->xc_push_lock); /* - * The CIL is stable at this point - nothing new will be added to it - * because we hold the flush lock exclusively. Hence we can now issue - * a cache flush to ensure all the completed metadata in the journal we - * are about to overwrite is on stable storage. - * - * Because we are issuing this cache flush before we've written the - * tail lsn to the iclog, we can have metadata IO completions move the - * tail forwards between the completion of this flush and the iclog - * being written. In this case, we need to re-issue the cache flush - * before the iclog write. To detect whether the log tail moves, sample - * the tail LSN *before* we issue the flush. - */ - preflush_tail_lsn = atomic64_read(&log->l_tail_lsn); - xfs_flush_bdev_async(&bio, log->l_mp->m_ddev_targp->bt_bdev, - &bdev_flush); - - /* * Pull all the log vectors off the items in the CIL, and remove the * items from the CIL. We don't need the CIL lock here because it's only * needed on the transaction commit side which is currently locked out @@ -1054,12 +1044,6 @@ xlog_cil_push_work( lvhdr.lv_iovecp = &lhdr; lvhdr.lv_next = ctx->lv_chain; - /* - * Before we format and submit the first iclog, we have to ensure that - * the metadata writeback ordering cache flush is complete. - */ - wait_for_completion(&bdev_flush); - error = xlog_cil_write_chain(ctx, &lvhdr); if (error) goto out_abort_free_ticket; @@ -1118,7 +1102,7 @@ xlog_cil_push_work( if (push_commit_stable && ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE) xlog_state_switch_iclogs(log, ctx->commit_iclog, 0); - xlog_state_release_iclog(log, ctx->commit_iclog, preflush_tail_lsn); + xlog_state_release_iclog(log, ctx->commit_iclog); /* Not safe to reference ctx now! */ @@ -1139,7 +1123,7 @@ out_abort_free_ticket: return; } spin_lock(&log->l_icloglock); - xlog_state_release_iclog(log, ctx->commit_iclog, 0); + xlog_state_release_iclog(log, ctx->commit_iclog); /* Not safe to reference ctx now! */ spin_unlock(&log->l_icloglock); } diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index 23103d68423c..401cdc400980 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -484,6 +484,17 @@ xlog_is_shutdown(struct xlog *log) return test_bit(XLOG_IO_ERROR, &log->l_opstate); } +/* + * Wait until the xlog_force_shutdown() has marked the log as shut down + * so xlog_is_shutdown() will always return true. + */ +static inline void +xlog_shutdown_wait( + struct xlog *log) +{ + wait_var_event(&log->l_opstate, xlog_is_shutdown(log)); +} + /* common routines */ extern int xlog_recover( @@ -524,8 +535,7 @@ void xfs_log_ticket_regrant(struct xlog *log, struct xlog_ticket *ticket); void xlog_state_switch_iclogs(struct xlog *log, struct xlog_in_core *iclog, int eventual_size); -int xlog_state_release_iclog(struct xlog *log, struct xlog_in_core *iclog, - xfs_lsn_t log_tail_lsn); +int xlog_state_release_iclog(struct xlog *log, struct xlog_in_core *iclog); /* * When we crack an atomic LSN, we sample it first so that the value will not diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 96c997ed2ec8..c4ad4296c540 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2485,7 +2485,7 @@ xlog_finish_defer_ops( error = xfs_trans_alloc(mp, &resv, dfc->dfc_blkres, dfc->dfc_rtxres, XFS_TRANS_RESERVE, &tp); if (error) { - xfs_force_shutdown(mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(mp->m_log, SHUTDOWN_LOG_IO_ERROR); return error; } @@ -2519,21 +2519,22 @@ xlog_abort_defer_ops( xfs_defer_ops_capture_free(mp, dfc); } } + /* * When this is called, all of the log intent items which did not have - * corresponding log done items should be in the AIL. What we do now - * is update the data structures associated with each one. + * corresponding log done items should be in the AIL. What we do now is update + * the data structures associated with each one. * - * Since we process the log intent items in normal transactions, they - * will be removed at some point after the commit. This prevents us - * from just walking down the list processing each one. We'll use a - * flag in the intent item to skip those that we've already processed - * and use the AIL iteration mechanism's generation count to try to - * speed this up at least a bit. + * Since we process the log intent items in normal transactions, they will be + * removed at some point after the commit. This prevents us from just walking + * down the list processing each one. We'll use a flag in the intent item to + * skip those that we've already processed and use the AIL iteration mechanism's + * generation count to try to speed this up at least a bit. * - * When we start, we know that the intents are the only things in the - * AIL. As we process them, however, other items are added to the - * AIL. + * When we start, we know that the intents are the only things in the AIL. As we + * process them, however, other items are added to the AIL. Hence we know we + * have started recovery on all the pending intents when we find an non-intent + * item in the AIL. */ STATIC int xlog_recover_process_intents( @@ -2556,17 +2557,8 @@ xlog_recover_process_intents( for (lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); lip != NULL; lip = xfs_trans_ail_cursor_next(ailp, &cur)) { - /* - * We're done when we see something other than an intent. - * There should be no intents left in the AIL now. - */ - if (!xlog_item_is_intent(lip)) { -#ifdef DEBUG - for (; lip; lip = xfs_trans_ail_cursor_next(ailp, &cur)) - ASSERT(!xlog_item_is_intent(lip)); -#endif + if (!xlog_item_is_intent(lip)) break; - } /* * We should never see a redo item with a LSN higher than @@ -2607,8 +2599,9 @@ err: } /* - * A cancel occurs when the mount has failed and we're bailing out. - * Release all pending log intent items so they don't pin the AIL. + * A cancel occurs when the mount has failed and we're bailing out. Release all + * pending log intent items that we haven't started recovery on so they don't + * pin the AIL. */ STATIC void xlog_recover_cancel_intents( @@ -2622,17 +2615,8 @@ xlog_recover_cancel_intents( spin_lock(&ailp->ail_lock); lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); while (lip != NULL) { - /* - * We're done when we see something other than an intent. - * There should be no intents left in the AIL now. - */ - if (!xlog_item_is_intent(lip)) { -#ifdef DEBUG - for (; lip; lip = xfs_trans_ail_cursor_next(ailp, &cur)) - ASSERT(!xlog_item_is_intent(lip)); -#endif + if (!xlog_item_is_intent(lip)) break; - } spin_unlock(&ailp->ail_lock); lip->li_ops->iop_release(lip); @@ -3470,7 +3454,7 @@ xlog_recover_finish( */ xlog_recover_cancel_intents(log); xfs_alert(log->l_mp, "Failed to recover intents"); - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); return error; } @@ -3517,7 +3501,7 @@ xlog_recover_finish( * end of intents processing can be pushed through the CIL * and AIL. */ - xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); } return 0; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index bed73e8002a5..c5f153c3693f 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -21,6 +21,7 @@ #include "xfs_trans.h" #include "xfs_trans_priv.h" #include "xfs_log.h" +#include "xfs_log_priv.h" #include "xfs_error.h" #include "xfs_quota.h" #include "xfs_fsops.h" @@ -1146,7 +1147,7 @@ xfs_mod_fdblocks( * problems (i.e. transaction abort, pagecache discards, etc.) than * slightly premature -ENOSPC. */ - set_aside = mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks); + set_aside = xfs_fdblocks_unavailable(mp); percpu_counter_add_batch(&mp->m_fdblocks, delta, batch); if (__percpu_counter_compare(&mp->m_fdblocks, set_aside, XFS_FDBLOCKS_BATCH) >= 0) { diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 00720a02e761..f6dc19de8322 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -479,6 +479,21 @@ extern void xfs_unmountfs(xfs_mount_t *); */ #define XFS_FDBLOCKS_BATCH 1024 +/* + * Estimate the amount of free space that is not available to userspace and is + * not explicitly reserved from the incore fdblocks. This includes: + * + * - The minimum number of blocks needed to support splitting a bmap btree + * - The blocks currently in use by the freespace btrees because they record + * the actual blocks that will fill per-AG metadata space reservations + */ +static inline uint64_t +xfs_fdblocks_unavailable( + struct xfs_mount *mp) +{ + return mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks); +} + extern int xfs_mod_fdblocks(struct xfs_mount *mp, int64_t delta, bool reserved); extern int xfs_mod_frextents(struct xfs_mount *mp, int64_t delta); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index d84714e4e46a..54be9d64093e 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -815,7 +815,8 @@ xfs_fs_statfs( spin_unlock(&mp->m_sb_lock); /* make sure statp->f_bfree does not underflow */ - statp->f_bfree = max_t(int64_t, fdblocks - mp->m_alloc_set_aside, 0); + statp->f_bfree = max_t(int64_t, 0, + fdblocks - xfs_fdblocks_unavailable(mp)); statp->f_bavail = statp->f_bfree; fakeinos = XFS_FSB_TO_INO(mp, statp->f_bfree); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 917a69f0a6ff..0ac717aad380 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -836,6 +836,7 @@ __xfs_trans_commit( bool regrant) { struct xfs_mount *mp = tp->t_mountp; + struct xlog *log = mp->m_log; xfs_csn_t commit_seq = 0; int error = 0; int sync = tp->t_flags & XFS_TRANS_SYNC; @@ -864,7 +865,13 @@ __xfs_trans_commit( if (!(tp->t_flags & XFS_TRANS_DIRTY)) goto out_unreserve; - if (xfs_is_shutdown(mp)) { + /* + * We must check against log shutdown here because we cannot abort log + * items and leave them dirty, inconsistent and unpinned in memory while + * the log is active. This leaves them open to being written back to + * disk, and that will lead to on-disk corruption. + */ + if (xlog_is_shutdown(log)) { error = -EIO; goto out_unreserve; } @@ -878,7 +885,7 @@ __xfs_trans_commit( xfs_trans_apply_sb_deltas(tp); xfs_trans_apply_dquot_deltas(tp); - xlog_cil_commit(mp->m_log, tp, &commit_seq, regrant); + xlog_cil_commit(log, tp, &commit_seq, regrant); xfs_trans_free(tp); @@ -905,10 +912,10 @@ out_unreserve: */ xfs_trans_unreserve_and_mod_dquots(tp); if (tp->t_ticket) { - if (regrant && !xlog_is_shutdown(mp->m_log)) - xfs_log_ticket_regrant(mp->m_log, tp->t_ticket); + if (regrant && !xlog_is_shutdown(log)) + xfs_log_ticket_regrant(log, tp->t_ticket); else - xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket); + xfs_log_ticket_ungrant(log, tp->t_ticket); tp->t_ticket = NULL; } xfs_trans_free_items(tp, !!error); @@ -926,18 +933,27 @@ xfs_trans_commit( } /* - * Unlock all of the transaction's items and free the transaction. - * The transaction must not have modified any of its items, because - * there is no way to restore them to their previous state. + * Unlock all of the transaction's items and free the transaction. If the + * transaction is dirty, we must shut down the filesystem because there is no + * way to restore them to their previous state. * - * If the transaction has made a log reservation, make sure to release - * it as well. + * If the transaction has made a log reservation, make sure to release it as + * well. + * + * This is a high level function (equivalent to xfs_trans_commit()) and so can + * be called after the transaction has effectively been aborted due to the mount + * being shut down. However, if the mount has not been shut down and the + * transaction is dirty we will shut the mount down and, in doing so, that + * guarantees that the log is shut down, too. Hence we don't need to be as + * careful with shutdown state and dirty items here as we need to be in + * xfs_trans_commit(). */ void xfs_trans_cancel( struct xfs_trans *tp) { struct xfs_mount *mp = tp->t_mountp; + struct xlog *log = mp->m_log; bool dirty = (tp->t_flags & XFS_TRANS_DIRTY); trace_xfs_trans_cancel(tp, _RET_IP_); @@ -955,16 +971,18 @@ xfs_trans_cancel( } /* - * See if the caller is relying on us to shut down the - * filesystem. This happens in paths where we detect - * corruption and decide to give up. + * See if the caller is relying on us to shut down the filesystem. We + * only want an error report if there isn't already a shutdown in + * progress, so we only need to check against the mount shutdown state + * here. */ if (dirty && !xfs_is_shutdown(mp)) { XFS_ERROR_REPORT("xfs_trans_cancel", XFS_ERRLEVEL_LOW, mp); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); } #ifdef DEBUG - if (!dirty && !xfs_is_shutdown(mp)) { + /* Log items need to be consistent until the log is shut down. */ + if (!dirty && !xlog_is_shutdown(log)) { struct xfs_log_item *lip; list_for_each_entry(lip, &tp->t_items, li_trans) @@ -975,7 +993,7 @@ xfs_trans_cancel( xfs_trans_unreserve_and_mod_dquots(tp); if (tp->t_ticket) { - xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket); + xfs_log_ticket_ungrant(log, tp->t_ticket); tp->t_ticket = NULL; } diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index c2ccb98c7bcd..d3a97a028560 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -873,17 +873,17 @@ xfs_trans_ail_delete( int shutdown_type) { struct xfs_ail *ailp = lip->li_ailp; - struct xfs_mount *mp = ailp->ail_log->l_mp; + struct xlog *log = ailp->ail_log; xfs_lsn_t tail_lsn; spin_lock(&ailp->ail_lock); if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { spin_unlock(&ailp->ail_lock); - if (shutdown_type && !xlog_is_shutdown(ailp->ail_log)) { - xfs_alert_tag(mp, XFS_PTAG_AILDELETE, + if (shutdown_type && !xlog_is_shutdown(log)) { + xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE, "%s: attempting to delete a log item that is not in the AIL", __func__); - xfs_force_shutdown(mp, shutdown_type); + xlog_force_shutdown(log, shutdown_type); } return; } diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index c08758b6b364..c05d2ce9b6cd 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -269,6 +269,7 @@ bool hv_isolation_type_snp(void); u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size); void hyperv_cleanup(void); bool hv_query_ext_cap(u64 cap_query); +void hv_setup_dma_ops(struct device *dev, bool coherent); void *hv_map_memory(void *addr, unsigned long size); void hv_unmap_memory(void *addr); #else /* CONFIG_HYPERV */ diff --git a/include/dt-bindings/clock/sun6i-rtc.h b/include/dt-bindings/clock/sun6i-rtc.h new file mode 100644 index 000000000000..c845493e4d37 --- /dev/null +++ b/include/dt-bindings/clock/sun6i-rtc.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: (GPL-2.0+ or MIT) */ + +#ifndef _DT_BINDINGS_CLK_SUN6I_RTC_H_ +#define _DT_BINDINGS_CLK_SUN6I_RTC_H_ + +#define CLK_OSC32K 0 +#define CLK_OSC32K_FANOUT 1 +#define CLK_IOSC 2 + +#endif /* _DT_BINDINGS_CLK_SUN6I_RTC_H_ */ diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h index 338aa27e4773..edb7f6d41faa 100644 --- a/include/linux/balloon_compaction.h +++ b/include/linux/balloon_compaction.h @@ -80,12 +80,6 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon) #ifdef CONFIG_BALLOON_COMPACTION extern const struct address_space_operations balloon_aops; -extern bool balloon_page_isolate(struct page *page, - isolate_mode_t mode); -extern void balloon_page_putback(struct page *page); -extern int balloon_page_migrate(struct address_space *mapping, - struct page *newpage, - struct page *page, enum migrate_mode mode); /* * balloon_page_insert - insert a page into the balloon's page list and make @@ -155,22 +149,6 @@ static inline void balloon_page_delete(struct page *page) list_del(&page->lru); } -static inline bool balloon_page_isolate(struct page *page) -{ - return false; -} - -static inline void balloon_page_putback(struct page *page) -{ - return; -} - -static inline int balloon_page_migrate(struct page *newpage, - struct page *page, enum migrate_mode mode) -{ - return 0; -} - static inline gfp_t balloon_mapping_gfp_mask(void) { return GFP_HIGHUSER; diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index f2ad8ed8f777..652cd05b0924 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -95,7 +95,10 @@ struct blkcg_gq { spinlock_t async_bio_lock; struct bio_list async_bios; - struct work_struct async_bio_work; + union { + struct work_struct async_bio_work; + struct work_struct free_work; + }; atomic_t use_delay; atomic64_t delay_nsec; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index dd0763a1c674..1973ef9bd40f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -85,8 +85,10 @@ struct block_device { */ #if defined(CONFIG_ALPHA) && !defined(__alpha_bwx__) typedef u32 __bitwise blk_status_t; +typedef u32 blk_short_t; #else typedef u8 __bitwise blk_status_t; +typedef u16 blk_short_t; #endif #define BLK_STS_OK 0 #define BLK_STS_NOTSUPP ((__force blk_status_t)1) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c1fc4af47f69..3a9d2d7cc6b7 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -570,9 +570,11 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } +/* only use after check_attach_btf_id() */ static inline enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) { - return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; + return prog->type == BPF_PROG_TYPE_EXT ? + prog->aux->dst_prog->type : prog->type; } #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/clk/sunxi-ng.h b/include/linux/clk/sunxi-ng.h index cf32123b39f5..57c8ec44ab4e 100644 --- a/include/linux/clk/sunxi-ng.h +++ b/include/linux/clk/sunxi-ng.h @@ -9,4 +9,6 @@ int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, bool new_mode); int sunxi_ccu_get_mmc_timing_mode(struct clk *clk); +int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg); + #endif diff --git a/include/linux/cma.h b/include/linux/cma.h index 90fd742fd1ef..a6f637342740 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -12,10 +12,6 @@ */ #ifdef CONFIG_CMA_AREAS #define MAX_CMA_AREAS (1 + CONFIG_CMA_AREAS) - -#else -#define MAX_CMA_AREAS (0) - #endif #define CMA_MAX_NAME 64 diff --git a/include/linux/fs.h b/include/linux/fs.h index 183160872133..bbde95387a23 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -275,7 +275,6 @@ enum positive_aop_returns { AOP_TRUNCATED_PAGE = 0x80001, }; -#define AOP_FLAG_CONT_EXPAND 0x0001 /* called from cont_expand */ #define AOP_FLAG_NOFS 0x0002 /* used by filesystem to direct * helper code (eg buffer layer) * to clear GFP_FS from alloc */ @@ -338,28 +337,6 @@ static inline bool is_sync_kiocb(struct kiocb *kiocb) return kiocb->ki_complete == NULL; } -/* - * "descriptor" for what we're up to with a read. - * This allows us to use the same read code yet - * have multiple different users of the data that - * we read from a file. - * - * The simplest case just copies the data to user - * mode. - */ -typedef struct { - size_t written; - size_t count; - union { - char __user *buf; - void *data; - } arg; - int error; -} read_descriptor_t; - -typedef int (*read_actor_t)(read_descriptor_t *, struct page *, - unsigned long, unsigned long); - struct address_space_operations { int (*writepage)(struct page *page, struct writeback_control *wbc); int (*readpage)(struct file *, struct page *); @@ -370,12 +347,6 @@ struct address_space_operations { /* Mark a folio dirty. Return true if this dirtied it */ bool (*dirty_folio)(struct address_space *, struct folio *); - /* - * Reads in the requested pages. Unlike ->readpage(), this is - * PURELY used for read-ahead!. - */ - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); void (*readahead)(struct readahead_control *); int (*write_begin)(struct file *, struct address_space *mapping, @@ -3027,7 +2998,7 @@ extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *); extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *); -extern ssize_t generic_perform_write(struct file *, struct iov_iter *, loff_t); +ssize_t generic_perform_write(struct kiocb *, struct iov_iter *); ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index d44ff747a657..6727fb0db619 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -457,6 +457,20 @@ int fscache_begin_read_operation(struct netfs_cache_resources *cres, } /** + * fscache_end_operation - End the read operation for the netfs lib + * @cres: The cache resources for the read operation + * + * Clean up the resources at the end of the read request. + */ +static inline void fscache_end_operation(struct netfs_cache_resources *cres) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + + if (ops) + ops->end_operation(cres); +} + +/** * fscache_read - Start a read from the cache. * @cres: The cache resources to use * @start_pos: The beginning file offset in the cache file diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index b568b3c7d095..a7afc800bd8d 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -221,7 +221,7 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work) * * This checks whether ->i_verity_info has been set. * - * Filesystems call this from ->readpages() to check whether the pages need to + * Filesystems call this from ->readahead() to check whether the pages need to * be verified or not. Don't use IS_VERITY() for this purpose; it's subject to * a race condition where the file is being read concurrently with * FS_IOC_ENABLE_VERITY completing. (S_VERITY is set before ->i_verity_info.) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index ed8cf433a46a..4816b7e11047 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -9,6 +9,7 @@ #include <linux/trace_recursion.h> #include <linux/trace_clock.h> +#include <linux/jump_label.h> #include <linux/kallsyms.h> #include <linux/linkage.h> #include <linux/bitops.h> @@ -1018,7 +1019,20 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, extern int register_ftrace_graph(struct fgraph_ops *ops); extern void unregister_ftrace_graph(struct fgraph_ops *ops); -extern bool ftrace_graph_is_dead(void); +/** + * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called + * + * ftrace_graph_stop() is called when a severe error is detected in + * the function graph tracing. This function is called by the critical + * paths of function graph to keep those paths from doing any more harm. + */ +DECLARE_STATIC_KEY_FALSE(kill_ftrace_graph); + +static inline bool ftrace_graph_is_dead(void) +{ + return static_branch_unlikely(&kill_ftrace_graph); +} + extern void ftrace_graph_stop(void); /* The current handlers in use */ diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 0fa17fb85de5..761f8f1885c7 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -264,9 +264,7 @@ struct vm_area_struct; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (24 + \ - 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ - IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index b0728c8ad90c..98c93510640e 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -168,13 +168,16 @@ struct gpio_irq_chip { /** * @parent_handler_data: + * + * If @per_parent_data is false, @parent_handler_data is a single + * pointer used as the data associated with every parent interrupt. + * * @parent_handler_data_array: * - * Data associated, and passed to, the handler for the parent - * interrupt. Can either be a single pointer if @per_parent_data - * is false, or an array of @num_parents pointers otherwise. If - * @per_parent_data is true, @parent_handler_data_array cannot be - * NULL. + * If @per_parent_data is true, @parent_handler_data_array is + * an array of @num_parents pointers, and is used to associate + * different data for each parent. This cannot be NULL if + * @per_parent_data is true. */ union { void *parent_handler_data; diff --git a/include/linux/input.h b/include/linux/input.h index 0354b298d874..49790c1bd2c4 100644 --- a/include/linux/input.h +++ b/include/linux/input.h @@ -475,6 +475,8 @@ static inline void input_set_events_per_packet(struct input_dev *dev, int n_even void input_alloc_absinfo(struct input_dev *dev); void input_set_abs_params(struct input_dev *dev, unsigned int axis, int min, int max, int fuzz, int flat); +void input_copy_abs(struct input_dev *dst, unsigned int dst_axis, + const struct input_dev *src, unsigned int src_axis); #define INPUT_GENERATE_ABS_ACCESSORS(_suffix, _item) \ static inline int input_abs_get_##_suffix(struct input_dev *dev, \ diff --git a/include/linux/input/vivaldi-fmap.h b/include/linux/input/vivaldi-fmap.h new file mode 100644 index 000000000000..7e4b7023bf04 --- /dev/null +++ b/include/linux/input/vivaldi-fmap.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VIVALDI_FMAP_H +#define _VIVALDI_FMAP_H + +#include <linux/types.h> + +#define VIVALDI_MAX_FUNCTION_ROW_KEYS 24 + +/** + * struct vivaldi_data - Function row map data for ChromeOS Vivaldi keyboards + * @function_row_physmap: An array of scancodes or their equivalent (HID usage + * codes, encoded rows/columns, etc) for the top + * row function keys, in an order from left to right + * @num_function_row_keys: The number of top row keys in a custom keyboard + * + * This structure is supposed to be used by ChromeOS keyboards using + * the Vivaldi keyboard function row design. + */ +struct vivaldi_data { + u32 function_row_physmap[VIVALDI_MAX_FUNCTION_ROW_KEYS]; + unsigned int num_function_row_keys; +}; + +ssize_t vivaldi_function_row_physmap_show(const struct vivaldi_data *data, + char *buf); + +#endif /* _VIVALDI_FMAP_H */ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 9536ffa0473b..3f9b22c4983a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -148,6 +148,7 @@ static inline bool is_error_page(struct page *page) #define KVM_REQUEST_MASK GENMASK(7,0) #define KVM_REQUEST_NO_WAKEUP BIT(8) #define KVM_REQUEST_WAIT BIT(9) +#define KVM_REQUEST_NO_ACTION BIT(10) /* * Architecture-independent vcpu->requests bit members * Bits 4-7 are reserved for more arch-independent bits. @@ -156,9 +157,18 @@ static inline bool is_error_page(struct page *page) #define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UNBLOCK 2 #define KVM_REQ_UNHALT 3 -#define KVM_REQ_GPC_INVALIDATE (5 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQUEST_ARCH_BASE 8 +/* + * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to + * OUTSIDE_GUEST_MODE. KVM_REQ_OUTSIDE_GUEST_MODE differs from a vCPU "kick" + * in that it ensures the vCPU has reached OUTSIDE_GUEST_MODE before continuing + * on. A kick only guarantees that the vCPU is on its way out, e.g. a previous + * kick may have set vcpu->mode to EXITING_GUEST_MODE, and so there's no + * guarantee the vCPU received an IPI and has actually exited guest mode. + */ +#define KVM_REQ_OUTSIDE_GUEST_MODE (KVM_REQUEST_NO_ACTION | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \ BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \ (unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \ @@ -1221,27 +1231,27 @@ void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); * @gpc: struct gfn_to_pfn_cache object. * @vcpu: vCPU to be used for marking pages dirty and to be woken on * invalidation. - * @guest_uses_pa: indicates that the resulting host physical PFN is used while - * @vcpu is IN_GUEST_MODE so invalidations should wake it. - * @kernel_map: requests a kernel virtual mapping (kmap / memremap). + * @usage: indicates if the resulting host physical PFN is used while + * the @vcpu is IN_GUEST_MODE (in which case invalidation of + * the cache from MMU notifiers---but not for KVM memslot + * changes!---will also force @vcpu to exit the guest and + * refresh the cache); and/or if the PFN used directly + * by KVM (and thus needs a kernel virtual mapping). * @gpa: guest physical address to map. * @len: sanity check; the range being access must fit a single page. - * @dirty: mark the cache dirty immediately. * * @return: 0 for success. * -EINVAL for a mapping which would cross a page boundary. * -EFAULT for an untranslatable guest physical address. * * This primes a gfn_to_pfn_cache and links it into the @kvm's list for - * invalidations to be processed. Invalidation callbacks to @vcpu using - * %KVM_REQ_GPC_INVALIDATE will occur only for MMU notifiers, not for KVM - * memslot changes. Callers are required to use kvm_gfn_to_pfn_cache_check() - * to ensure that the cache is valid before accessing the target page. + * invalidations to be processed. Callers are required to use + * kvm_gfn_to_pfn_cache_check() to ensure that the cache is valid before + * accessing the target page. */ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - struct kvm_vcpu *vcpu, bool guest_uses_pa, - bool kernel_map, gpa_t gpa, unsigned long len, - bool dirty); + struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, + gpa_t gpa, unsigned long len); /** * kvm_gfn_to_pfn_cache_check - check validity of a gfn_to_pfn_cache. @@ -1250,7 +1260,6 @@ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, * @gpc: struct gfn_to_pfn_cache object. * @gpa: current guest physical address to map. * @len: sanity check; the range being access must fit a single page. - * @dirty: mark the cache dirty immediately. * * @return: %true if the cache is still valid and the address matches. * %false if the cache is not valid. @@ -1272,7 +1281,6 @@ bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, * @gpc: struct gfn_to_pfn_cache object. * @gpa: updated guest physical address to map. * @len: sanity check; the range being access must fit a single page. - * @dirty: mark the cache dirty immediately. * * @return: 0 for success. * -EINVAL for a mapping which would cross a page boundary. @@ -1285,7 +1293,7 @@ bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, * with the lock still held to permit access. */ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - gpa_t gpa, unsigned long len, bool dirty); + gpa_t gpa, unsigned long len); /** * kvm_gfn_to_pfn_cache_unmap - temporarily unmap a gfn_to_pfn_cache. @@ -1293,10 +1301,9 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, * @kvm: pointer to kvm instance. * @gpc: struct gfn_to_pfn_cache object. * - * This unmaps the referenced page and marks it dirty, if appropriate. The - * cache is left in the invalid state but at least the mapping from GPA to - * userspace HVA will remain cached and can be reused on a subsequent - * refresh. + * This unmaps the referenced page. The cache is left in the invalid state + * but at least the mapping from GPA to userspace HVA will remain cached + * and can be reused on a subsequent refresh. */ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); @@ -1984,7 +1991,7 @@ static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) void kvm_arch_irq_routing_update(struct kvm *kvm); -static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) +static inline void __kvm_make_request(int req, struct kvm_vcpu *vcpu) { /* * Ensure the rest of the request is published to kvm_check_request's @@ -1994,6 +2001,19 @@ static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) set_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } +static __always_inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) +{ + /* + * Request that don't require vCPU action should never be logged in + * vcpu->requests. The vCPU won't clear the request, so it will stay + * logged indefinitely and prevent the vCPU from entering the guest. + */ + BUILD_BUG_ON(!__builtin_constant_p(req) || + (req & KVM_REQUEST_NO_ACTION)); + + __kvm_make_request(req, vcpu); +} + static inline bool kvm_request_pending(struct kvm_vcpu *vcpu) { return READ_ONCE(vcpu->requests); diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index dceac12c1ce5..ac1ebb37a0ff 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -18,6 +18,7 @@ struct kvm_memslots; enum kvm_mr_change; +#include <linux/bits.h> #include <linux/types.h> #include <linux/spinlock_types.h> @@ -46,6 +47,12 @@ typedef u64 hfn_t; typedef hfn_t kvm_pfn_t; +enum pfn_cache_usage { + KVM_GUEST_USES_PFN = BIT(0), + KVM_HOST_USES_PFN = BIT(1), + KVM_GUEST_AND_HOST_USE_PFN = KVM_GUEST_USES_PFN | KVM_HOST_USES_PFN, +}; + struct gfn_to_hva_cache { u64 generation; gpa_t gpa; @@ -64,11 +71,9 @@ struct gfn_to_pfn_cache { rwlock_t lock; void *khva; kvm_pfn_t pfn; + enum pfn_cache_usage usage; bool active; bool valid; - bool dirty; - bool kernel_map; - bool guest_uses_pa; }; #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE diff --git a/include/linux/mc146818rtc.h b/include/linux/mc146818rtc.h index 808bb4cee230..b0da04fe087b 100644 --- a/include/linux/mc146818rtc.h +++ b/include/linux/mc146818rtc.h @@ -86,6 +86,8 @@ struct cmos_rtc_board_info { /* 2 values for divider stage reset, others for "testing purposes only" */ # define RTC_DIV_RESET1 0x60 # define RTC_DIV_RESET2 0x70 + /* In AMD BKDG bit 5 and 6 are reserved, bit 4 is for select dv0 bank */ +# define RTC_AMD_BANK_SELECT 0x10 /* Periodic intr. / Square wave rate select. 0=none, 1=32.8kHz,... 15=2Hz */ # define RTC_RATE_SELECT 0x0F diff --git a/include/linux/net.h b/include/linux/net.h index ba736b457a06..12093f4db50c 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -125,6 +125,25 @@ struct socket { struct socket_wq wq; }; +/* + * "descriptor" for what we're up to with a read. + * This allows us to use the same read code yet + * have multiple different users of the data that + * we read from a file. + * + * The simplest case just copies the data to user + * mode. + */ +typedef struct { + size_t written; + size_t count; + union { + char __user *buf; + void *data; + } arg; + int error; +} read_descriptor_t; + struct vm_area_struct; struct page; struct sockaddr; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 59e27a2b7bf0..28ea4f8269d4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -59,7 +59,8 @@ struct dsa_port; struct ip_tunnel_parm; struct macsec_context; struct macsec_ops; - +struct netdev_name_node; +struct sd_flow_limit; struct sfp_bus; /* 802.11 specific */ struct wireless_dev; @@ -202,6 +203,7 @@ struct net_device_core_stats { local_t rx_dropped; local_t tx_dropped; local_t rx_nohandler; + local_t rx_otherhost_dropped; } __aligned(4 * sizeof(local_t)); #include <linux/cache.h> @@ -862,6 +864,7 @@ enum net_device_path_type { DEV_PATH_BRIDGE, DEV_PATH_PPPOE, DEV_PATH_DSA, + DEV_PATH_MTK_WDMA, }; struct net_device_path { @@ -887,6 +890,12 @@ struct net_device_path { int port; u16 proto; } dsa; + struct { + u8 wdma_idx; + u8 queue; + u16 wcid; + u8 bss; + } mtk_wdma; }; }; @@ -1013,16 +1022,6 @@ struct dev_ifalias { struct devlink; struct tlsdev_ops; -struct netdev_name_node { - struct hlist_node hlist; - struct list_head list; - struct net_device *dev; - const char *name; -}; - -int netdev_name_node_alt_create(struct net_device *dev, const char *name); -int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); - struct netdev_net_notifier { struct list_head list; struct notifier_block *nb; @@ -2968,7 +2967,6 @@ struct net_device *dev_get_by_index(struct net *net, int ifindex); struct net_device *__dev_get_by_index(struct net *net, int ifindex); struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); struct net_device *dev_get_by_napi_id(unsigned int napi_id); -int netdev_get_name(struct net *net, char *name, int ifindex); int dev_restart(struct net_device *dev); @@ -3027,19 +3025,6 @@ static inline bool dev_has_header(const struct net_device *dev) return dev->header_ops && dev->header_ops->create; } -#ifdef CONFIG_NET_FLOW_LIMIT -#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ -struct sd_flow_limit { - u64 count; - unsigned int num_buckets; - unsigned int history_head; - u16 history[FLOW_LIMIT_HISTORY]; - u8 buckets[]; -}; - -extern int netdev_flow_limit_table_len; -#endif /* CONFIG_NET_FLOW_LIMIT */ - /* * Incoming packets are placed on per-CPU queues */ @@ -3763,7 +3748,6 @@ int dev_change_flags(struct net_device *dev, unsigned int flags, struct netlink_ext_ack *extack); void __dev_notify_flags(struct net_device *, unsigned int old_flags, unsigned int gchanges); -int dev_change_name(struct net_device *, const char *); int dev_set_alias(struct net_device *, const char *, size_t); int dev_get_alias(const struct net_device *, char *, size_t); int __dev_change_net_namespace(struct net_device *dev, struct net *net, @@ -3775,13 +3759,7 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, return __dev_change_net_namespace(dev, net, pat, 0); } int __dev_set_mtu(struct net_device *, int); -int dev_validate_mtu(struct net_device *dev, int mtu, - struct netlink_ext_ack *extack); -int dev_set_mtu_ext(struct net_device *dev, int mtu, - struct netlink_ext_ack *extack); int dev_set_mtu(struct net_device *, int); -int dev_change_tx_queue_len(struct net_device *, unsigned long); -void dev_set_group(struct net_device *, int); int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr, struct netlink_ext_ack *extack); int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, @@ -3789,24 +3767,13 @@ int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa, struct netlink_ext_ack *extack); int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name); -int dev_change_carrier(struct net_device *, bool new_carrier); -int dev_get_phys_port_id(struct net_device *dev, - struct netdev_phys_item_id *ppid); -int dev_get_phys_port_name(struct net_device *dev, - char *name, size_t len); int dev_get_port_parent_id(struct net_device *dev, struct netdev_phys_item_id *ppid, bool recurse); bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b); -int dev_change_proto_down(struct net_device *dev, bool proto_down); -void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, - u32 value); struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again); struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, struct netdev_queue *txq, int *ret); -typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); -int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, - int fd, int expected_fd, u32 flags); int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); u8 dev_xdp_prog_count(struct net_device *dev); u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode); @@ -3871,6 +3838,7 @@ static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev) \ DEV_CORE_STATS_INC(rx_dropped) DEV_CORE_STATS_INC(tx_dropped) DEV_CORE_STATS_INC(rx_nohandler) +DEV_CORE_STATS_INC(rx_otherhost_dropped) static __always_inline int ____dev_forward_skb(struct net_device *dev, struct sk_buff *skb, @@ -3891,12 +3859,6 @@ static __always_inline int ____dev_forward_skb(struct net_device *dev, bool dev_nit_active(struct net_device *dev); void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev); -extern int netdev_budget; -extern unsigned int netdev_budget_usecs; - -/* Called by rtnetlink.c:rtnl_unlock() */ -void netdev_run_todo(void); - static inline void __dev_put(struct net_device *dev) { if (dev) { @@ -4013,10 +3975,7 @@ static inline void dev_replace_track(struct net_device *odev, * called netif_lowerlayer_*() because they represent the state of any * kind of lower layer not just hardware media. */ - -void linkwatch_init_dev(struct net_device *dev); void linkwatch_fire_event(struct net_device *dev); -void linkwatch_forget_dev(struct net_device *dev); /** * netif_carrier_ok - test if carrier present @@ -4462,9 +4421,6 @@ int dev_addr_add(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); int dev_addr_del(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); -void dev_addr_flush(struct net_device *dev); -int dev_addr_init(struct net_device *dev); -void dev_addr_check(struct net_device *dev); /* Functions used for unicast addresses handling */ int dev_uc_add(struct net_device *dev, const unsigned char *addr); @@ -4554,7 +4510,6 @@ static inline void __dev_mc_unsync(struct net_device *dev, /* Functions used for secondary unicast and multicast support */ void dev_set_rx_mode(struct net_device *dev); -void __dev_set_rx_mode(struct net_device *dev); int dev_set_promiscuity(struct net_device *dev, int inc); int dev_set_allmulti(struct net_device *dev, int inc); void netdev_state_change(struct net_device *dev); @@ -4572,11 +4527,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); extern int netdev_max_backlog; -extern int netdev_tstamp_prequeue; -extern int netdev_unregister_timeout_secs; -extern int weight_p; -extern int dev_weight_rx_bias; -extern int dev_weight_tx_bias; extern int dev_rx_weight; extern int dev_tx_weight; extern int gro_normal_batch; @@ -4764,12 +4714,6 @@ static inline void netdev_rx_csum_fault(struct net_device *dev, void net_enable_timestamp(void); void net_disable_timestamp(void); -#ifdef CONFIG_PROC_FS -int __init dev_proc_init(void); -#else -#define dev_proc_init() 0 -#endif - static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops, struct sk_buff *skb, struct net_device *dev, bool more) @@ -4805,8 +4749,6 @@ extern const struct kobj_ns_type_operations net_ns_type_operations; const char *netdev_drivername(const struct net_device *dev); -void linkwatch_run_queue(void); - static inline netdev_features_t netdev_intersect_features(netdev_features_t f1, netdev_features_t f2) { diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 614f22213e21..c7bf1eaf51d5 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -18,6 +18,8 @@ #include <linux/fs.h> #include <linux/pagemap.h> +enum netfs_sreq_ref_trace; + /* * Overload PG_private_2 to give us PG_fscache - this is used to indicate that * a page is currently backed by a local disk cache @@ -106,7 +108,7 @@ static inline int wait_on_page_fscache_killable(struct page *page) return folio_wait_private_2_killable(page_folio(page)); } -enum netfs_read_source { +enum netfs_io_source { NETFS_FILL_WITH_ZEROES, NETFS_DOWNLOAD_FROM_SERVER, NETFS_READ_FROM_CACHE, @@ -117,6 +119,17 @@ typedef void (*netfs_io_terminated_t)(void *priv, ssize_t transferred_or_error, bool was_async); /* + * Per-inode description. This must be directly after the inode struct. + */ +struct netfs_i_context { + const struct netfs_request_ops *ops; +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cache; +#endif + loff_t remote_i_size; /* Size of the remote file */ +}; + +/* * Resources required to do operations on a cache. */ struct netfs_cache_resources { @@ -130,69 +143,75 @@ struct netfs_cache_resources { /* * Descriptor for a single component subrequest. */ -struct netfs_read_subrequest { - struct netfs_read_request *rreq; /* Supervising read request */ +struct netfs_io_subrequest { + struct netfs_io_request *rreq; /* Supervising I/O request */ struct list_head rreq_link; /* Link in rreq->subrequests */ loff_t start; /* Where to start the I/O */ size_t len; /* Size of the I/O */ size_t transferred; /* Amount of data transferred */ - refcount_t usage; + refcount_t ref; short error; /* 0 or error that occurred */ unsigned short debug_index; /* Index in list (for debugging output) */ - enum netfs_read_source source; /* Where to read from */ + enum netfs_io_source source; /* Where to read from/write to */ unsigned long flags; -#define NETFS_SREQ_WRITE_TO_CACHE 0 /* Set if should write to cache */ +#define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */ #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */ -#define NETFS_SREQ_SHORT_READ 2 /* Set if there was a short read from the cache */ +#define NETFS_SREQ_SHORT_IO 2 /* Set if the I/O was short */ #define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */ #define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */ }; +enum netfs_io_origin { + NETFS_READAHEAD, /* This read was triggered by readahead */ + NETFS_READPAGE, /* This read is a synchronous read */ + NETFS_READ_FOR_WRITE, /* This read is to prepare a write */ +} __mode(byte); + /* - * Descriptor for a read helper request. This is used to make multiple I/O - * requests on a variety of sources and then stitch the result together. + * Descriptor for an I/O helper request. This is used to make multiple I/O + * operations to a variety of data stores and then stitch the result together. */ -struct netfs_read_request { +struct netfs_io_request { struct work_struct work; struct inode *inode; /* The file being accessed */ struct address_space *mapping; /* The mapping being accessed */ struct netfs_cache_resources cache_resources; - struct list_head subrequests; /* Requests to fetch I/O from disk or net */ + struct list_head subrequests; /* Contributory I/O operations */ void *netfs_priv; /* Private data for the netfs */ unsigned int debug_id; - atomic_t nr_rd_ops; /* Number of read ops in progress */ - atomic_t nr_wr_ops; /* Number of write ops in progress */ + atomic_t nr_outstanding; /* Number of ops in progress */ + atomic_t nr_copy_ops; /* Number of copy-to-cache ops in progress */ size_t submitted; /* Amount submitted for I/O so far */ size_t len; /* Length of the request */ short error; /* 0 or error that occurred */ + enum netfs_io_origin origin; /* Origin of the request */ loff_t i_size; /* Size of the file */ loff_t start; /* Start position */ pgoff_t no_unlock_folio; /* Don't unlock this folio after read */ - refcount_t usage; + refcount_t ref; unsigned long flags; #define NETFS_RREQ_INCOMPLETE_IO 0 /* Some ioreqs terminated short or with error */ -#define NETFS_RREQ_WRITE_TO_CACHE 1 /* Need to write to the cache */ +#define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */ #define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */ #define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */ #define NETFS_RREQ_FAILED 4 /* The request failed */ #define NETFS_RREQ_IN_PROGRESS 5 /* Unlocked when the request completes */ - const struct netfs_read_request_ops *netfs_ops; + const struct netfs_request_ops *netfs_ops; }; /* * Operations the network filesystem can/must provide to the helpers. */ -struct netfs_read_request_ops { - bool (*is_cache_enabled)(struct inode *inode); - void (*init_rreq)(struct netfs_read_request *rreq, struct file *file); - int (*begin_cache_operation)(struct netfs_read_request *rreq); - void (*expand_readahead)(struct netfs_read_request *rreq); - bool (*clamp_length)(struct netfs_read_subrequest *subreq); - void (*issue_op)(struct netfs_read_subrequest *subreq); - bool (*is_still_valid)(struct netfs_read_request *rreq); +struct netfs_request_ops { + int (*init_request)(struct netfs_io_request *rreq, struct file *file); + int (*begin_cache_operation)(struct netfs_io_request *rreq); + void (*expand_readahead)(struct netfs_io_request *rreq); + bool (*clamp_length)(struct netfs_io_subrequest *subreq); + void (*issue_read)(struct netfs_io_subrequest *subreq); + bool (*is_still_valid)(struct netfs_io_request *rreq); int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, struct folio *folio, void **_fsdata); - void (*done)(struct netfs_read_request *rreq); + void (*done)(struct netfs_io_request *rreq); void (*cleanup)(struct address_space *mapping, void *netfs_priv); }; @@ -235,7 +254,7 @@ struct netfs_cache_ops { /* Prepare a read operation, shortening it to a cached/uncached * boundary as appropriate. */ - enum netfs_read_source (*prepare_read)(struct netfs_read_subrequest *subreq, + enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq, loff_t i_size); /* Prepare a write operation, working out what part of the write we can @@ -254,20 +273,89 @@ struct netfs_cache_ops { }; struct readahead_control; -extern void netfs_readahead(struct readahead_control *, - const struct netfs_read_request_ops *, - void *); -extern int netfs_readpage(struct file *, - struct folio *, - const struct netfs_read_request_ops *, - void *); +extern void netfs_readahead(struct readahead_control *); +extern int netfs_readpage(struct file *, struct page *); extern int netfs_write_begin(struct file *, struct address_space *, loff_t, unsigned int, unsigned int, struct folio **, - void **, - const struct netfs_read_request_ops *, - void *); + void **); -extern void netfs_subreq_terminated(struct netfs_read_subrequest *, ssize_t, bool); +extern void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool); +extern void netfs_get_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what); +extern void netfs_put_subrequest(struct netfs_io_subrequest *subreq, + bool was_async, enum netfs_sreq_ref_trace what); extern void netfs_stats_show(struct seq_file *); +/** + * netfs_i_context - Get the netfs inode context from the inode + * @inode: The inode to query + * + * Get the netfs lib inode context from the network filesystem's inode. The + * context struct is expected to directly follow on from the VFS inode struct. + */ +static inline struct netfs_i_context *netfs_i_context(struct inode *inode) +{ + return (struct netfs_i_context *)(inode + 1); +} + +/** + * netfs_inode - Get the netfs inode from the inode context + * @ctx: The context to query + * + * Get the netfs inode from the netfs library's inode context. The VFS inode + * is expected to directly precede the context struct. + */ +static inline struct inode *netfs_inode(struct netfs_i_context *ctx) +{ + return ((struct inode *)ctx) - 1; +} + +/** + * netfs_i_context_init - Initialise a netfs lib context + * @inode: The inode with which the context is associated + * @ops: The netfs's operations list + * + * Initialise the netfs library context struct. This is expected to follow on + * directly from the VFS inode struct. + */ +static inline void netfs_i_context_init(struct inode *inode, + const struct netfs_request_ops *ops) +{ + struct netfs_i_context *ctx = netfs_i_context(inode); + + memset(ctx, 0, sizeof(*ctx)); + ctx->ops = ops; + ctx->remote_i_size = i_size_read(inode); +} + +/** + * netfs_resize_file - Note that a file got resized + * @inode: The inode being resized + * @new_i_size: The new file size + * + * Inform the netfs lib that a file got resized so that it can adjust its state. + */ +static inline void netfs_resize_file(struct inode *inode, loff_t new_i_size) +{ + struct netfs_i_context *ctx = netfs_i_context(inode); + + ctx->remote_i_size = new_i_size; +} + +/** + * netfs_i_cookie - Get the cache cookie from the inode + * @inode: The inode to query + * + * Get the caching cookie (if enabled) from the network filesystem's inode. + */ +static inline struct fscache_cookie *netfs_i_cookie(struct inode *inode) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct netfs_i_context *ctx = netfs_i_context(inode); + return ctx->cache; +#else + return NULL; +#endif +} + #endif /* _LINUX_NETFS_H */ diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 4f44f83817a9..f626a445d1a8 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -346,6 +346,7 @@ enum { NVME_CTRL_ONCS_TIMESTAMP = 1 << 6, NVME_CTRL_VWC_PRESENT = 1 << 0, NVME_CTRL_OACS_SEC_SUPP = 1 << 0, + NVME_CTRL_OACS_NS_MNGT_SUPP = 1 << 3, NVME_CTRL_OACS_DIRECTIVES = 1 << 5, NVME_CTRL_OACS_DBBUF_SUPP = 1 << 8, NVME_CTRL_LPA_CMD_EFFECTS_LOG = 1 << 1, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index a8d0b327b066..993994cd943a 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -752,8 +752,6 @@ struct page *read_cache_page(struct address_space *, pgoff_t index, filler_t *filler, void *data); extern struct page * read_cache_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -extern int read_cache_pages(struct address_space *mapping, - struct list_head *pages, filler_t *filler, void *data); static inline struct page *read_mapping_page(struct address_space *mapping, pgoff_t index, struct file *file) diff --git a/include/linux/pci-dma-compat.h b/include/linux/pci-dma-compat.h deleted file mode 100644 index 249d4d7fbf18..000000000000 --- a/include/linux/pci-dma-compat.h +++ /dev/null @@ -1,129 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* include this file if the platform implements the dma_ DMA Mapping API - * and wants to provide the pci_ DMA Mapping API in terms of it */ - -#ifndef _ASM_GENERIC_PCI_DMA_COMPAT_H -#define _ASM_GENERIC_PCI_DMA_COMPAT_H - -#include <linux/dma-mapping.h> - -/* This defines the direction arg to the DMA mapping routines. */ -#define PCI_DMA_BIDIRECTIONAL DMA_BIDIRECTIONAL -#define PCI_DMA_TODEVICE DMA_TO_DEVICE -#define PCI_DMA_FROMDEVICE DMA_FROM_DEVICE -#define PCI_DMA_NONE DMA_NONE - -static inline void * -pci_alloc_consistent(struct pci_dev *hwdev, size_t size, - dma_addr_t *dma_handle) -{ - return dma_alloc_coherent(&hwdev->dev, size, dma_handle, GFP_ATOMIC); -} - -static inline void * -pci_zalloc_consistent(struct pci_dev *hwdev, size_t size, - dma_addr_t *dma_handle) -{ - return dma_alloc_coherent(&hwdev->dev, size, dma_handle, GFP_ATOMIC); -} - -static inline void -pci_free_consistent(struct pci_dev *hwdev, size_t size, - void *vaddr, dma_addr_t dma_handle) -{ - dma_free_coherent(&hwdev->dev, size, vaddr, dma_handle); -} - -static inline dma_addr_t -pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction) -{ - return dma_map_single(&hwdev->dev, ptr, size, (enum dma_data_direction)direction); -} - -static inline void -pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, - size_t size, int direction) -{ - dma_unmap_single(&hwdev->dev, dma_addr, size, (enum dma_data_direction)direction); -} - -static inline dma_addr_t -pci_map_page(struct pci_dev *hwdev, struct page *page, - unsigned long offset, size_t size, int direction) -{ - return dma_map_page(&hwdev->dev, page, offset, size, (enum dma_data_direction)direction); -} - -static inline void -pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address, - size_t size, int direction) -{ - dma_unmap_page(&hwdev->dev, dma_address, size, (enum dma_data_direction)direction); -} - -static inline int -pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, - int nents, int direction) -{ - return dma_map_sg(&hwdev->dev, sg, nents, (enum dma_data_direction)direction); -} - -static inline void -pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, - int nents, int direction) -{ - dma_unmap_sg(&hwdev->dev, sg, nents, (enum dma_data_direction)direction); -} - -static inline void -pci_dma_sync_single_for_cpu(struct pci_dev *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - dma_sync_single_for_cpu(&hwdev->dev, dma_handle, size, (enum dma_data_direction)direction); -} - -static inline void -pci_dma_sync_single_for_device(struct pci_dev *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - dma_sync_single_for_device(&hwdev->dev, dma_handle, size, (enum dma_data_direction)direction); -} - -static inline void -pci_dma_sync_sg_for_cpu(struct pci_dev *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - dma_sync_sg_for_cpu(&hwdev->dev, sg, nelems, (enum dma_data_direction)direction); -} - -static inline void -pci_dma_sync_sg_for_device(struct pci_dev *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - dma_sync_sg_for_device(&hwdev->dev, sg, nelems, (enum dma_data_direction)direction); -} - -static inline int -pci_dma_mapping_error(struct pci_dev *pdev, dma_addr_t dma_addr) -{ - return dma_mapping_error(&pdev->dev, dma_addr); -} - -#ifdef CONFIG_PCI -static inline int pci_set_dma_mask(struct pci_dev *dev, u64 mask) -{ - return dma_set_mask(&dev->dev, mask); -} - -static inline int pci_set_consistent_dma_mask(struct pci_dev *dev, u64 mask) -{ - return dma_set_coherent_mask(&dev->dev, mask); -} -#else -static inline int pci_set_dma_mask(struct pci_dev *dev, u64 mask) -{ return -EIO; } -static inline int pci_set_consistent_dma_mask(struct pci_dev *dev, u64 mask) -{ return -EIO; } -#endif - -#endif diff --git a/include/linux/pci.h b/include/linux/pci.h index b957eeb89c7a..60adf42460ab 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2473,8 +2473,7 @@ static inline bool pci_is_thunderbolt_attached(struct pci_dev *pdev) void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type); #endif -/* Provide the legacy pci_dma_* API */ -#include <linux/pci-dma-compat.h> +#include <linux/dma-mapping.h> #define pci_printk(level, pdev, fmt, arg...) \ dev_printk(level, &(pdev)->dev, fmt, ##arg) diff --git a/include/linux/rtc.h b/include/linux/rtc.h index 47fd1c2d3a57..1fd9c6a21ebe 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -110,8 +110,6 @@ struct rtc_device { struct hrtimer pie_timer; /* sub second exp, so needs hrtimer */ int pie_enabled; struct work_struct irqwork; - /* Some hardware can't support UIE mode */ - int uie_unsupported; /* * This offset specifies the update timing of the RTC. diff --git a/include/linux/rtc/ds1685.h b/include/linux/rtc/ds1685.h index 67ee9d20cc5a..5a41c3bbcbe3 100644 --- a/include/linux/rtc/ds1685.h +++ b/include/linux/rtc/ds1685.h @@ -46,7 +46,6 @@ struct ds1685_priv { u32 regstep; int irq_num; bool bcd_mode; - bool no_irq; u8 (*read)(struct ds1685_priv *, int); void (*write)(struct ds1685_priv *, int, u8); void (*prepare_poweroff)(void); diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index dffeb8281c2d..8f5a86e210b9 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -174,7 +174,7 @@ static inline unsigned int __map_depth(const struct sbitmap *sb, int index) static inline void sbitmap_free(struct sbitmap *sb) { free_percpu(sb->alloc_hint); - kfree(sb->map); + kvfree(sb->map); sb->map = NULL; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 4a6fdd2a679f..d5e3c00b74e1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1090,9 +1090,6 @@ struct task_struct { /* Restored if set_restore_sigmask() was used: */ sigset_t saved_sigmask; struct sigpending pending; -#ifdef CONFIG_RT_DELAYED_SIGNALS - struct kernel_siginfo forced_info; -#endif unsigned long sas_ss_sp; size_t sas_ss_size; unsigned int sas_ss_flags; diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index 88cc16444b43..60820ab511d2 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -162,6 +162,7 @@ int seq_dentry(struct seq_file *, struct dentry *, const char *); int seq_path_root(struct seq_file *m, const struct path *path, const struct path *root, const char *esc); +void *single_start(struct seq_file *, loff_t *); int single_open(struct file *, int (*)(struct seq_file *, void *), void *); int single_open_size(struct file *, int (*)(struct seq_file *, void *), void *, size_t); int single_release(struct inode *, struct file *); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3a30cae8b0a5..2394441fa3dd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3836,8 +3836,7 @@ struct sk_buff *__skb_try_recv_datagram(struct sock *sk, struct sk_buff *__skb_recv_datagram(struct sock *sk, struct sk_buff_head *sk_queue, unsigned int flags, int *off, int *err); -struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock, - int *err); +struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err); __poll_t datagram_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait); int skb_copy_datagram_iter(const struct sk_buff *from, int offset, diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h new file mode 100644 index 000000000000..7e00cca06709 --- /dev/null +++ b/include/linux/soc/mediatek/mtk_wed.h @@ -0,0 +1,131 @@ +#ifndef __MTK_WED_H +#define __MTK_WED_H + +#include <linux/kernel.h> +#include <linux/rcupdate.h> +#include <linux/regmap.h> +#include <linux/pci.h> + +#define MTK_WED_TX_QUEUES 2 + +struct mtk_wed_hw; +struct mtk_wdma_desc; + +struct mtk_wed_ring { + struct mtk_wdma_desc *desc; + dma_addr_t desc_phys; + int size; + + u32 reg_base; + void __iomem *wpdma; +}; + +struct mtk_wed_device { +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + const struct mtk_wed_ops *ops; + struct device *dev; + struct mtk_wed_hw *hw; + bool init_done, running; + int wdma_idx; + int irq; + + struct mtk_wed_ring tx_ring[MTK_WED_TX_QUEUES]; + struct mtk_wed_ring txfree_ring; + struct mtk_wed_ring tx_wdma[MTK_WED_TX_QUEUES]; + + struct { + int size; + void **pages; + struct mtk_wdma_desc *desc; + dma_addr_t desc_phys; + } buf_ring; + + /* filled by driver: */ + struct { + struct pci_dev *pci_dev; + + u32 wpdma_phys; + + u16 token_start; + unsigned int nbuf; + + u32 (*init_buf)(void *ptr, dma_addr_t phys, int token_id); + int (*offload_enable)(struct mtk_wed_device *wed); + void (*offload_disable)(struct mtk_wed_device *wed); + } wlan; +#endif +}; + +struct mtk_wed_ops { + int (*attach)(struct mtk_wed_device *dev); + int (*tx_ring_setup)(struct mtk_wed_device *dev, int ring, + void __iomem *regs); + int (*txfree_ring_setup)(struct mtk_wed_device *dev, + void __iomem *regs); + void (*detach)(struct mtk_wed_device *dev); + + void (*stop)(struct mtk_wed_device *dev); + void (*start)(struct mtk_wed_device *dev, u32 irq_mask); + void (*reset_dma)(struct mtk_wed_device *dev); + + u32 (*reg_read)(struct mtk_wed_device *dev, u32 reg); + void (*reg_write)(struct mtk_wed_device *dev, u32 reg, u32 val); + + u32 (*irq_get)(struct mtk_wed_device *dev, u32 mask); + void (*irq_set_mask)(struct mtk_wed_device *dev, u32 mask); +}; + +extern const struct mtk_wed_ops __rcu *mtk_soc_wed_ops; + +static inline int +mtk_wed_device_attach(struct mtk_wed_device *dev) +{ + int ret = -ENODEV; + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + rcu_read_lock(); + dev->ops = rcu_dereference(mtk_soc_wed_ops); + if (dev->ops) + ret = dev->ops->attach(dev); + else + rcu_read_unlock(); + + if (ret) + dev->ops = NULL; +#endif + + return ret; +} + +#ifdef CONFIG_NET_MEDIATEK_SOC_WED +#define mtk_wed_device_active(_dev) !!(_dev)->ops +#define mtk_wed_device_detach(_dev) (_dev)->ops->detach(_dev) +#define mtk_wed_device_start(_dev, _mask) (_dev)->ops->start(_dev, _mask) +#define mtk_wed_device_tx_ring_setup(_dev, _ring, _regs) \ + (_dev)->ops->tx_ring_setup(_dev, _ring, _regs) +#define mtk_wed_device_txfree_ring_setup(_dev, _regs) \ + (_dev)->ops->txfree_ring_setup(_dev, _regs) +#define mtk_wed_device_reg_read(_dev, _reg) \ + (_dev)->ops->reg_read(_dev, _reg) +#define mtk_wed_device_reg_write(_dev, _reg, _val) \ + (_dev)->ops->reg_write(_dev, _reg, _val) +#define mtk_wed_device_irq_get(_dev, _mask) \ + (_dev)->ops->irq_get(_dev, _mask) +#define mtk_wed_device_irq_set_mask(_dev, _mask) \ + (_dev)->ops->irq_set_mask(_dev, _mask) +#else +static inline bool mtk_wed_device_active(struct mtk_wed_device *dev) +{ + return false; +} +#define mtk_wed_device_detach(_dev) do {} while (0) +#define mtk_wed_device_start(_dev, _mask) do {} while (0) +#define mtk_wed_device_tx_ring_setup(_dev, _ring, _regs) -ENODEV +#define mtk_wed_device_txfree_ring_setup(_dev, _ring, _regs) -ENODEV +#define mtk_wed_device_reg_read(_dev, _reg) 0 +#define mtk_wed_device_reg_write(_dev, _reg, _val) do {} while (0) +#define mtk_wed_device_irq_get(_dev, _mask) 0 +#define mtk_wed_device_irq_set_mask(_dev, _mask) do {} while (0) +#endif + +#endif diff --git a/include/uapi/linux/user_events.h b/include/linux/user_events.h index e570840571e1..736e05603463 100644 --- a/include/uapi/linux/user_events.h +++ b/include/linux/user_events.h @@ -32,9 +32,6 @@ /* Create dynamic location entry within a 32-bit value */ #define DYN_LOC(offset, size) ((size) << 16 | (offset)) -/* Use raw iterator for attached BPF program(s), no affect on ftrace/perf */ -#define FLAG_BPF_ITER (1 << 0) - /* * Describes an event registration and stores the results of the registration. * This structure is passed to the DIAG_IOCSREG ioctl, callers at a minimum @@ -63,54 +60,4 @@ struct user_reg { /* Requests to delete a user_event */ #define DIAG_IOCSDEL _IOW(DIAG_IOC_MAGIC, 1, char*) -/* Data type that was passed to the BPF program */ -enum { - /* Data resides in kernel space */ - USER_BPF_DATA_KERNEL, - - /* Data resides in user space */ - USER_BPF_DATA_USER, - - /* Data is a pointer to a user_bpf_iter structure */ - USER_BPF_DATA_ITER, -}; - -/* - * Describes an iovec iterator that BPF programs can use to access data for - * a given user_event write() / writev() call. - */ -struct user_bpf_iter { - - /* Offset of the data within the first iovec */ - __u32 iov_offset; - - /* Number of iovec structures */ - __u32 nr_segs; - - /* Pointer to iovec structures */ - const struct iovec *iov; -}; - -/* Context that BPF programs receive when attached to a user_event */ -struct user_bpf_context { - - /* Data type being passed (see union below) */ - __u32 data_type; - - /* Length of the data */ - __u32 data_len; - - /* Pointer to data, varies by data type */ - union { - /* Kernel data (data_type == USER_BPF_DATA_KERNEL) */ - void *kdata; - - /* User data (data_type == USER_BPF_DATA_USER) */ - void *udata; - - /* Direct iovec (data_type == USER_BPF_DATA_ITER) */ - struct user_bpf_iter *iter; - }; -}; - #endif /* _UAPI_LINUX_USER_EVENTS_H */ diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 721089bb4c84..8943a209202e 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -83,7 +83,7 @@ struct vdpa_device { unsigned int index; bool features_valid; bool use_va; - int nvqs; + u32 nvqs; struct vdpa_mgmt_dev *mdev; }; @@ -207,7 +207,8 @@ struct vdpa_map_file { * @reset: Reset device * @vdev: vdpa device * Returns integer: success (0) or error (< 0) - * @get_config_size: Get the size of the configuration space + * @get_config_size: Get the size of the configuration space includes + * fields that are conditional on feature bits. * @vdev: vdpa device * Returns size_t: configuration size * @get_config: Read from device specific configuration space @@ -337,10 +338,10 @@ struct vdpa_device *__vdpa_alloc_device(struct device *parent, dev_struct, member)), name, use_va), \ dev_struct, member) -int vdpa_register_device(struct vdpa_device *vdev, int nvqs); +int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs); void vdpa_unregister_device(struct vdpa_device *vdev); -int _vdpa_register_device(struct vdpa_device *vdev, int nvqs); +int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs); void _vdpa_unregister_device(struct vdpa_device *vdev); /** diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index dafdc7f48c01..b341dd62aa4d 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -23,8 +23,6 @@ struct virtio_shm_region { * any of @get/@set, @get_status/@set_status, or @get_features/ * @finalize_features are NOT safe to be called from an atomic * context. - * @enable_cbs: enable the callbacks - * vdev: the virtio_device * @get: read the value of a configuration field * vdev: the virtio_device * offset: the offset of the configuration field @@ -78,7 +76,6 @@ struct virtio_shm_region { */ typedef void vq_callback_t(struct virtqueue *); struct virtio_config_ops { - void (*enable_cbs)(struct virtio_device *vdev); void (*get)(struct virtio_device *vdev, unsigned offset, void *buf, unsigned len); void (*set)(struct virtio_device *vdev, unsigned offset, @@ -233,9 +230,6 @@ void virtio_device_ready(struct virtio_device *dev) { unsigned status = dev->config->get_status(dev); - if (dev->config->enable_cbs) - dev->config->enable_cbs(dev); - BUG_ON(status & VIRTIO_CONFIG_S_DRIVER_OK); dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK); } diff --git a/include/linux/xarray.h b/include/linux/xarray.h index bb52b786be1b..72feab5ea8d4 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -9,6 +9,7 @@ * See Documentation/core-api/xarray.rst for how to use the XArray. */ +#include <linux/bitmap.h> #include <linux/bug.h> #include <linux/compiler.h> #include <linux/gfp.h> diff --git a/include/net/act_api.h b/include/net/act_api.h index 3049cb69c025..9cf6870b526e 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -134,7 +134,8 @@ struct tc_action_ops { (*get_psample_group)(const struct tc_action *a, tc_action_priv_destructor *destructor); int (*offload_act_setup)(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind); + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack); }; struct tc_action_net { diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 4cfdef6ca4f6..c8490729b4ae 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -64,6 +64,14 @@ struct inet6_ifaddr { struct hlist_node addr_lst; struct list_head if_list; + /* + * Used to safely traverse idev->addr_list in process context + * if the idev->lock needed to protect idev->addr_list cannot be held. + * In that case, add the items to this list temporarily and iterate + * without holding idev->lock. + * See addrconf_ifdown and dev_forward_change. + */ + struct list_head if_list_aux; struct list_head tmp_list; struct inet6_ifaddr *ifpub; diff --git a/include/net/mctp.h b/include/net/mctp.h index d37268fe6825..82800d521c3d 100644 --- a/include/net/mctp.h +++ b/include/net/mctp.h @@ -36,8 +36,6 @@ struct mctp_hdr { #define MCTP_HDR_TAG_SHIFT 0 #define MCTP_HDR_TAG_MASK GENMASK(2, 0) -#define MCTP_HEADER_MAXLEN 4 - #define MCTP_INITIAL_DEFAULT_NET 1 static inline bool mctp_address_unicast(mctp_eid_t eid) diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index a3b57a93228a..8cf001aed858 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -547,10 +547,12 @@ tcf_match_indev(struct sk_buff *skb, int ifindex) } int tc_setup_offload_action(struct flow_action *flow_action, - const struct tcf_exts *exts); + const struct tcf_exts *exts, + struct netlink_ext_ack *extack); void tc_cleanup_offload_action(struct flow_action *flow_action); int tc_setup_action(struct flow_action *flow_action, - struct tc_action *actions[]); + struct tc_action *actions[], + struct netlink_ext_ack *extack); int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, void *type_data, bool err_stop, bool rtnl_held); diff --git a/include/net/strparser.h b/include/net/strparser.h index 732b7097d78e..a191486eb1e4 100644 --- a/include/net/strparser.h +++ b/include/net/strparser.h @@ -70,6 +70,10 @@ struct sk_skb_cb { * when dst_reg == src_reg. */ u64 temp_reg; + struct tls_msg { + u8 control; + u8 decrypted; + } tls; }; static inline struct strp_msg *strp_msg(struct sk_buff *skb) diff --git a/include/net/tc_act/tc_gact.h b/include/net/tc_act/tc_gact.h index eb8f01c819e6..832efd40e023 100644 --- a/include/net/tc_act/tc_gact.h +++ b/include/net/tc_act/tc_gact.h @@ -59,4 +59,19 @@ static inline u32 tcf_gact_goto_chain_index(const struct tc_action *a) return READ_ONCE(a->tcfa_action) & TC_ACT_EXT_VAL_MASK; } +static inline bool is_tcf_gact_continue(const struct tc_action *a) +{ + return __is_tcf_gact_act(a, TC_ACT_UNSPEC, false); +} + +static inline bool is_tcf_gact_reclassify(const struct tc_action *a) +{ + return __is_tcf_gact_act(a, TC_ACT_RECLASSIFY, false); +} + +static inline bool is_tcf_gact_pipe(const struct tc_action *a) +{ + return __is_tcf_gact_act(a, TC_ACT_PIPE, false); +} + #endif /* __NET_TC_GACT_H */ diff --git a/include/net/tc_act/tc_skbedit.h b/include/net/tc_act/tc_skbedit.h index 00bfee70609e..cab8229b9bed 100644 --- a/include/net/tc_act/tc_skbedit.h +++ b/include/net/tc_act/tc_skbedit.h @@ -94,4 +94,16 @@ static inline u32 tcf_skbedit_priority(const struct tc_action *a) return priority; } +/* Return true iff action is queue_mapping */ +static inline bool is_tcf_skbedit_queue_mapping(const struct tc_action *a) +{ + return is_tcf_skbedit_with_flag(a, SKBEDIT_F_QUEUE_MAPPING); +} + +/* Return true iff action is inheritdsfield */ +static inline bool is_tcf_skbedit_inheritdsfield(const struct tc_action *a) +{ + return is_tcf_skbedit_with_flag(a, SKBEDIT_F_INHERITDSFIELD); +} + #endif /* __NET_TC_SKBEDIT_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 70ca4a5e330a..6d50a662bf89 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1139,15 +1139,6 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } -static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state) -{ - struct inet_connection_sock *icsk = inet_csk(sk); - - if (icsk->icsk_ca_ops->set_state) - icsk->icsk_ca_ops->set_state(sk, ca_state); - icsk->icsk_ca_state = ca_state; -} - static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1156,6 +1147,9 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) icsk->icsk_ca_ops->cwnd_event(sk, event); } +/* From tcp_cong.c */ +void tcp_set_ca_state(struct sock *sk, const u8 ca_state); + /* From tcp_rate.c */ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, @@ -1207,9 +1201,20 @@ static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) #define TCP_INFINITE_SSTHRESH 0x7fffffff +static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) +{ + return tp->snd_cwnd; +} + +static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) +{ + WARN_ON_ONCE((int)val <= 0); + tp->snd_cwnd = val; +} + static inline bool tcp_in_slow_start(const struct tcp_sock *tp) { - return tp->snd_cwnd < tp->snd_ssthresh; + return tcp_snd_cwnd(tp) < tp->snd_ssthresh; } static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) @@ -1235,8 +1240,8 @@ static inline __u32 tcp_current_ssthresh(const struct sock *sk) return tp->snd_ssthresh; else return max(tp->snd_ssthresh, - ((tp->snd_cwnd >> 1) + - (tp->snd_cwnd >> 2))); + ((tcp_snd_cwnd(tp) >> 1) + + (tcp_snd_cwnd(tp) >> 2))); } /* Use define here intentionally to get WARN_ON location shown at the caller */ @@ -1278,7 +1283,7 @@ static inline bool tcp_is_cwnd_limited(const struct sock *sk) /* If in slow start, ensure cwnd grows to twice what was ACKed. */ if (tcp_in_slow_start(tp)) - return tp->snd_cwnd < 2 * tp->max_packets_out; + return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; return tp->is_cwnd_limited; } diff --git a/include/net/tls.h b/include/net/tls.h index b6968a5b5538..a01c264e5f15 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -64,6 +64,7 @@ #define TLS_AAD_SPACE_SIZE 13 #define MAX_IV_SIZE 16 +#define TLS_TAG_SIZE 16 #define TLS_MAX_REC_SEQ_SIZE 8 /* For CCM mode, the full 16-bytes of IV is made of '4' fields of given sizes. @@ -117,11 +118,6 @@ struct tls_rec { u8 aead_req_ctx[]; }; -struct tls_msg { - struct strp_msg rxm; - u8 control; -}; - struct tx_work { struct delayed_work work; struct sock *sk; @@ -152,9 +148,7 @@ struct tls_sw_context_rx { void (*saved_data_ready)(struct sock *sk); struct sk_buff *recv_pkt; - u8 control; u8 async_capable:1; - u8 decrypted:1; atomic_t decrypt_pending; /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; @@ -411,7 +405,9 @@ void tls_free_partial_record(struct sock *sk, struct tls_context *ctx); static inline struct tls_msg *tls_msg(struct sk_buff *skb) { - return (struct tls_msg *)strp_msg(skb); + struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; + + return &scb->tls; } static inline bool tls_is_partially_sent_record(struct tls_context *ctx) diff --git a/include/sound/pcm.h b/include/sound/pcm.h index 314f2779cab5..6b99310b5b88 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -402,6 +402,7 @@ struct snd_pcm_runtime { struct fasync_struct *fasync; bool stop_operating; /* sync_stop will be called */ struct mutex buffer_mutex; /* protect for buffer changes */ + atomic_t buffer_accessing; /* >0: in r/w operation, <0: blocked */ /* -- private section -- */ void *private_data; diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index 2c530637e10a..311c14a20e70 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -426,8 +426,8 @@ TRACE_EVENT(cachefiles_vol_coherency, ); TRACE_EVENT(cachefiles_prep_read, - TP_PROTO(struct netfs_read_subrequest *sreq, - enum netfs_read_source source, + TP_PROTO(struct netfs_io_subrequest *sreq, + enum netfs_io_source source, enum cachefiles_prepare_read_trace why, ino_t cache_inode), @@ -437,7 +437,7 @@ TRACE_EVENT(cachefiles_prep_read, __field(unsigned int, rreq ) __field(unsigned short, index ) __field(unsigned short, flags ) - __field(enum netfs_read_source, source ) + __field(enum netfs_io_source, source ) __field(enum cachefiles_prepare_read_trace, why ) __field(size_t, len ) __field(loff_t, start ) diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index e6f4ebbb4c69..beec534cbaab 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -15,63 +15,25 @@ /* * Define enums for tracing information. */ -#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY -#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY - -enum netfs_read_trace { - netfs_read_trace_expanded, - netfs_read_trace_readahead, - netfs_read_trace_readpage, - netfs_read_trace_write_begin, -}; - -enum netfs_rreq_trace { - netfs_rreq_trace_assess, - netfs_rreq_trace_done, - netfs_rreq_trace_free, - netfs_rreq_trace_resubmit, - netfs_rreq_trace_unlock, - netfs_rreq_trace_unmark, - netfs_rreq_trace_write, -}; - -enum netfs_sreq_trace { - netfs_sreq_trace_download_instead, - netfs_sreq_trace_free, - netfs_sreq_trace_prepare, - netfs_sreq_trace_resubmit_short, - netfs_sreq_trace_submit, - netfs_sreq_trace_terminated, - netfs_sreq_trace_write, - netfs_sreq_trace_write_skip, - netfs_sreq_trace_write_term, -}; - -enum netfs_failure { - netfs_fail_check_write_begin, - netfs_fail_copy_to_cache, - netfs_fail_read, - netfs_fail_short_readpage, - netfs_fail_short_write_begin, - netfs_fail_prepare_write, -}; - -#endif - #define netfs_read_traces \ EM(netfs_read_trace_expanded, "EXPANDED ") \ EM(netfs_read_trace_readahead, "READAHEAD") \ EM(netfs_read_trace_readpage, "READPAGE ") \ E_(netfs_read_trace_write_begin, "WRITEBEGN") +#define netfs_rreq_origins \ + EM(NETFS_READAHEAD, "RA") \ + EM(NETFS_READPAGE, "RP") \ + E_(NETFS_READ_FOR_WRITE, "RW") + #define netfs_rreq_traces \ - EM(netfs_rreq_trace_assess, "ASSESS") \ - EM(netfs_rreq_trace_done, "DONE ") \ - EM(netfs_rreq_trace_free, "FREE ") \ - EM(netfs_rreq_trace_resubmit, "RESUBM") \ - EM(netfs_rreq_trace_unlock, "UNLOCK") \ - EM(netfs_rreq_trace_unmark, "UNMARK") \ - E_(netfs_rreq_trace_write, "WRITE ") + EM(netfs_rreq_trace_assess, "ASSESS ") \ + EM(netfs_rreq_trace_copy, "COPY ") \ + EM(netfs_rreq_trace_done, "DONE ") \ + EM(netfs_rreq_trace_free, "FREE ") \ + EM(netfs_rreq_trace_resubmit, "RESUBMT") \ + EM(netfs_rreq_trace_unlock, "UNLOCK ") \ + E_(netfs_rreq_trace_unmark, "UNMARK ") #define netfs_sreq_sources \ EM(NETFS_FILL_WITH_ZEROES, "ZERO") \ @@ -94,10 +56,47 @@ enum netfs_failure { EM(netfs_fail_check_write_begin, "check-write-begin") \ EM(netfs_fail_copy_to_cache, "copy-to-cache") \ EM(netfs_fail_read, "read") \ - EM(netfs_fail_short_readpage, "short-readpage") \ - EM(netfs_fail_short_write_begin, "short-write-begin") \ + EM(netfs_fail_short_read, "short-read") \ E_(netfs_fail_prepare_write, "prep-write") +#define netfs_rreq_ref_traces \ + EM(netfs_rreq_trace_get_hold, "GET HOLD ") \ + EM(netfs_rreq_trace_get_subreq, "GET SUBREQ ") \ + EM(netfs_rreq_trace_put_complete, "PUT COMPLT ") \ + EM(netfs_rreq_trace_put_discard, "PUT DISCARD") \ + EM(netfs_rreq_trace_put_failed, "PUT FAILED ") \ + EM(netfs_rreq_trace_put_hold, "PUT HOLD ") \ + EM(netfs_rreq_trace_put_subreq, "PUT SUBREQ ") \ + EM(netfs_rreq_trace_put_zero_len, "PUT ZEROLEN") \ + E_(netfs_rreq_trace_new, "NEW ") + +#define netfs_sreq_ref_traces \ + EM(netfs_sreq_trace_get_copy_to_cache, "GET COPY2C ") \ + EM(netfs_sreq_trace_get_resubmit, "GET RESUBMIT") \ + EM(netfs_sreq_trace_get_short_read, "GET SHORTRD") \ + EM(netfs_sreq_trace_new, "NEW ") \ + EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \ + EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \ + EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \ + EM(netfs_sreq_trace_put_no_copy, "PUT NO COPY") \ + E_(netfs_sreq_trace_put_terminated, "PUT TERM ") + +#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY +#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY + +#undef EM +#undef E_ +#define EM(a, b) a, +#define E_(a, b) a + +enum netfs_read_trace { netfs_read_traces } __mode(byte); +enum netfs_rreq_trace { netfs_rreq_traces } __mode(byte); +enum netfs_sreq_trace { netfs_sreq_traces } __mode(byte); +enum netfs_failure { netfs_failures } __mode(byte); +enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); +enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); + +#endif /* * Export enum symbols via userspace. @@ -108,10 +107,13 @@ enum netfs_failure { #define E_(a, b) TRACE_DEFINE_ENUM(a); netfs_read_traces; +netfs_rreq_origins; netfs_rreq_traces; netfs_sreq_sources; netfs_sreq_traces; netfs_failures; +netfs_rreq_ref_traces; +netfs_sreq_ref_traces; /* * Now redefine the EM() and E_() macros to map the enums to the strings that @@ -123,7 +125,7 @@ netfs_failures; #define E_(a, b) { a, b } TRACE_EVENT(netfs_read, - TP_PROTO(struct netfs_read_request *rreq, + TP_PROTO(struct netfs_io_request *rreq, loff_t start, size_t len, enum netfs_read_trace what), @@ -156,31 +158,34 @@ TRACE_EVENT(netfs_read, ); TRACE_EVENT(netfs_rreq, - TP_PROTO(struct netfs_read_request *rreq, + TP_PROTO(struct netfs_io_request *rreq, enum netfs_rreq_trace what), TP_ARGS(rreq, what), TP_STRUCT__entry( __field(unsigned int, rreq ) - __field(unsigned short, flags ) + __field(unsigned int, flags ) + __field(enum netfs_io_origin, origin ) __field(enum netfs_rreq_trace, what ) ), TP_fast_assign( __entry->rreq = rreq->debug_id; __entry->flags = rreq->flags; + __entry->origin = rreq->origin; __entry->what = what; ), - TP_printk("R=%08x %s f=%02x", + TP_printk("R=%08x %s %s f=%02x", __entry->rreq, + __print_symbolic(__entry->origin, netfs_rreq_origins), __print_symbolic(__entry->what, netfs_rreq_traces), __entry->flags) ); TRACE_EVENT(netfs_sreq, - TP_PROTO(struct netfs_read_subrequest *sreq, + TP_PROTO(struct netfs_io_subrequest *sreq, enum netfs_sreq_trace what), TP_ARGS(sreq, what), @@ -190,7 +195,7 @@ TRACE_EVENT(netfs_sreq, __field(unsigned short, index ) __field(short, error ) __field(unsigned short, flags ) - __field(enum netfs_read_source, source ) + __field(enum netfs_io_source, source ) __field(enum netfs_sreq_trace, what ) __field(size_t, len ) __field(size_t, transferred ) @@ -211,26 +216,26 @@ TRACE_EVENT(netfs_sreq, TP_printk("R=%08x[%u] %s %s f=%02x s=%llx %zx/%zx e=%d", __entry->rreq, __entry->index, - __print_symbolic(__entry->what, netfs_sreq_traces), __print_symbolic(__entry->source, netfs_sreq_sources), + __print_symbolic(__entry->what, netfs_sreq_traces), __entry->flags, __entry->start, __entry->transferred, __entry->len, __entry->error) ); TRACE_EVENT(netfs_failure, - TP_PROTO(struct netfs_read_request *rreq, - struct netfs_read_subrequest *sreq, + TP_PROTO(struct netfs_io_request *rreq, + struct netfs_io_subrequest *sreq, int error, enum netfs_failure what), TP_ARGS(rreq, sreq, error, what), TP_STRUCT__entry( __field(unsigned int, rreq ) - __field(unsigned short, index ) + __field(short, index ) __field(short, error ) __field(unsigned short, flags ) - __field(enum netfs_read_source, source ) + __field(enum netfs_io_source, source ) __field(enum netfs_failure, what ) __field(size_t, len ) __field(size_t, transferred ) @@ -239,17 +244,17 @@ TRACE_EVENT(netfs_failure, TP_fast_assign( __entry->rreq = rreq->debug_id; - __entry->index = sreq ? sreq->debug_index : 0; + __entry->index = sreq ? sreq->debug_index : -1; __entry->error = error; __entry->flags = sreq ? sreq->flags : 0; __entry->source = sreq ? sreq->source : NETFS_INVALID_READ; __entry->what = what; - __entry->len = sreq ? sreq->len : 0; + __entry->len = sreq ? sreq->len : rreq->len; __entry->transferred = sreq ? sreq->transferred : 0; __entry->start = sreq ? sreq->start : 0; ), - TP_printk("R=%08x[%u] %s f=%02x s=%llx %zx/%zx %s e=%d", + TP_printk("R=%08x[%d] %s f=%02x s=%llx %zx/%zx %s e=%d", __entry->rreq, __entry->index, __print_symbolic(__entry->source, netfs_sreq_sources), __entry->flags, @@ -258,6 +263,59 @@ TRACE_EVENT(netfs_failure, __entry->error) ); +TRACE_EVENT(netfs_rreq_ref, + TP_PROTO(unsigned int rreq_debug_id, int ref, + enum netfs_rreq_ref_trace what), + + TP_ARGS(rreq_debug_id, ref, what), + + TP_STRUCT__entry( + __field(unsigned int, rreq ) + __field(int, ref ) + __field(enum netfs_rreq_ref_trace, what ) + ), + + TP_fast_assign( + __entry->rreq = rreq_debug_id; + __entry->ref = ref; + __entry->what = what; + ), + + TP_printk("R=%08x %s r=%u", + __entry->rreq, + __print_symbolic(__entry->what, netfs_rreq_ref_traces), + __entry->ref) + ); + +TRACE_EVENT(netfs_sreq_ref, + TP_PROTO(unsigned int rreq_debug_id, unsigned int subreq_debug_index, + int ref, enum netfs_sreq_ref_trace what), + + TP_ARGS(rreq_debug_id, subreq_debug_index, ref, what), + + TP_STRUCT__entry( + __field(unsigned int, rreq ) + __field(unsigned int, subreq ) + __field(int, ref ) + __field(enum netfs_sreq_ref_trace, what ) + ), + + TP_fast_assign( + __entry->rreq = rreq_debug_id; + __entry->subreq = subreq_debug_index; + __entry->ref = ref; + __entry->what = what; + ), + + TP_printk("R=%08x[%x] %s r=%u", + __entry->rreq, + __entry->subreq, + __print_symbolic(__entry->what, netfs_sreq_ref_traces), + __entry->ref) + ); + +#undef EM +#undef E_ #endif /* _TRACE_NETFS_H */ /* This part must be outside protection */ diff --git a/include/trace/events/tcp.h b/include/trace/events/tcp.h index 521059d8dc0a..901b440238d5 100644 --- a/include/trace/events/tcp.h +++ b/include/trace/events/tcp.h @@ -279,7 +279,7 @@ TRACE_EVENT(tcp_probe, __entry->data_len = skb->len - __tcp_hdrlen(th); __entry->snd_nxt = tp->snd_nxt; __entry->snd_una = tp->snd_una; - __entry->snd_cwnd = tp->snd_cwnd; + __entry->snd_cwnd = tcp_snd_cwnd(tp); __entry->snd_wnd = tp->snd_wnd; __entry->rcv_wnd = tp->rcv_wnd; __entry->ssthresh = tcp_current_ssthresh(sk); @@ -371,6 +371,51 @@ DEFINE_EVENT(tcp_event_skb, tcp_bad_csum, TP_ARGS(skb) ); +TRACE_EVENT(tcp_cong_state_set, + + TP_PROTO(struct sock *sk, const u8 ca_state), + + TP_ARGS(sk, ca_state), + + TP_STRUCT__entry( + __field(const void *, skaddr) + __field(__u16, sport) + __field(__u16, dport) + __array(__u8, saddr, 4) + __array(__u8, daddr, 4) + __array(__u8, saddr_v6, 16) + __array(__u8, daddr_v6, 16) + __field(__u8, cong_state) + ), + + TP_fast_assign( + struct inet_sock *inet = inet_sk(sk); + __be32 *p32; + + __entry->skaddr = sk; + + __entry->sport = ntohs(inet->inet_sport); + __entry->dport = ntohs(inet->inet_dport); + + p32 = (__be32 *) __entry->saddr; + *p32 = inet->inet_saddr; + + p32 = (__be32 *) __entry->daddr; + *p32 = inet->inet_daddr; + + TP_STORE_ADDRS(__entry, inet->inet_saddr, inet->inet_daddr, + sk->sk_v6_rcv_saddr, sk->sk_v6_daddr); + + __entry->cong_state = ca_state; + ), + + TP_printk("sport=%hu dport=%hu saddr=%pI4 daddr=%pI4 saddrv6=%pI6c daddrv6=%pI6c cong_state=%u", + __entry->sport, __entry->dport, + __entry->saddr, __entry->daddr, + __entry->saddr_v6, __entry->daddr_v6, + __entry->cong_state) +); + #endif /* _TRACE_TCP_H */ /* This part must be outside protection */ diff --git a/include/trace/stages/stage1_defines.h b/include/trace/stages/stage1_struct_define.h index a16783419687..a16783419687 100644 --- a/include/trace/stages/stage1_defines.h +++ b/include/trace/stages/stage1_struct_define.h diff --git a/include/trace/stages/stage2_defines.h b/include/trace/stages/stage2_data_offsets.h index 42fd1e8813ec..42fd1e8813ec 100644 --- a/include/trace/stages/stage2_defines.h +++ b/include/trace/stages/stage2_data_offsets.h diff --git a/include/trace/stages/stage3_defines.h b/include/trace/stages/stage3_trace_output.h index e3b183e9d18e..e3b183e9d18e 100644 --- a/include/trace/stages/stage3_defines.h +++ b/include/trace/stages/stage3_trace_output.h diff --git a/include/trace/stages/stage4_defines.h b/include/trace/stages/stage4_event_fields.h index e80cdc397a43..e80cdc397a43 100644 --- a/include/trace/stages/stage4_defines.h +++ b/include/trace/stages/stage4_event_fields.h diff --git a/include/trace/stages/stage5_defines.h b/include/trace/stages/stage5_get_offsets.h index 7ee5931300e6..7ee5931300e6 100644 --- a/include/trace/stages/stage5_defines.h +++ b/include/trace/stages/stage5_get_offsets.h diff --git a/include/trace/stages/stage6_defines.h b/include/trace/stages/stage6_event_callback.h index e1724f73594b..e1724f73594b 100644 --- a/include/trace/stages/stage6_defines.h +++ b/include/trace/stages/stage6_event_callback.h diff --git a/include/trace/stages/stage7_defines.h b/include/trace/stages/stage7_class_define.h index 8a7ec24c246d..8a7ec24c246d 100644 --- a/include/trace/stages/stage7_defines.h +++ b/include/trace/stages/stage7_class_define.h diff --git a/include/trace/trace_custom_events.h b/include/trace/trace_custom_events.h index b567c7202339..6e492dba96bf 100644 --- a/include/trace/trace_custom_events.h +++ b/include/trace/trace_custom_events.h @@ -35,7 +35,7 @@ /* Stage 1 creates the structure of the recorded event layout */ -#include "stages/stage1_defines.h" +#include "stages/stage1_struct_define.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print) \ @@ -56,7 +56,7 @@ /* Stage 2 creates the custom class */ -#include "stages/stage2_defines.h" +#include "stages/stage2_data_offsets.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -71,7 +71,7 @@ /* Stage 3 create the way to print the custom event */ -#include "stages/stage3_defines.h" +#include "stages/stage3_trace_output.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -102,7 +102,7 @@ static struct trace_event_functions trace_custom_event_type_funcs_##call = { \ /* Stage 4 creates the offset layout for the fields */ -#include "stages/stage4_defines.h" +#include "stages/stage4_event_fields.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, func, print) \ @@ -114,7 +114,7 @@ static struct trace_event_fields trace_custom_event_fields_##call[] = { \ /* Stage 5 creates the helper function for dynamic fields */ -#include "stages/stage5_defines.h" +#include "stages/stage5_get_offsets.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -134,7 +134,7 @@ static inline notrace int trace_custom_event_get_offsets_##call( \ /* Stage 6 creates the probe function that records the event */ -#include "stages/stage6_defines.h" +#include "stages/stage6_event_callback.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -182,7 +182,7 @@ static inline void ftrace_test_custom_probe_##call(void) \ /* Stage 7 creates the actual class and event structure for the custom event */ -#include "stages/stage7_defines.h" +#include "stages/stage7_class_define.h" #undef DECLARE_CUSTOM_EVENT_CLASS #define DECLARE_CUSTOM_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ diff --git a/include/trace/trace_events.h b/include/trace/trace_events.h index 8a8cd66cc6d5..c2f9cabf154d 100644 --- a/include/trace/trace_events.h +++ b/include/trace/trace_events.h @@ -45,7 +45,7 @@ PARAMS(print)); \ DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); -#include "stages/stage1_defines.h" +#include "stages/stage1_struct_define.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \ @@ -109,7 +109,7 @@ * The size of an array is also encoded, in the higher 16 bits of <item>. */ -#include "stages/stage2_defines.h" +#include "stages/stage2_data_offsets.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -181,7 +181,7 @@ * in binary. */ -#include "stages/stage3_defines.h" +#include "stages/stage3_trace_output.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -236,7 +236,7 @@ static struct trace_event_functions trace_event_type_funcs_##call = { \ #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -#include "stages/stage4_defines.h" +#include "stages/stage4_event_fields.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ @@ -249,7 +249,7 @@ static struct trace_event_fields trace_event_fields_##call[] = { \ #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -#include "stages/stage5_defines.h" +#include "stages/stage5_get_offsets.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -372,7 +372,7 @@ static inline notrace int trace_event_get_offsets_##call( \ #define _TRACE_PERF_INIT(call) #endif /* CONFIG_PERF_EVENTS */ -#include "stages/stage6_defines.h" +#include "stages/stage6_event_callback.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ @@ -418,7 +418,7 @@ static inline void ftrace_test_probe_##call(void) \ #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -#include "stages/stage7_defines.h" +#include "stages/stage7_class_define.h" #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index cc284c048e69..d1e600816b82 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -211,6 +211,9 @@ struct rtnl_link_stats { * @rx_nohandler: Number of packets received on the interface * but dropped by the networking stack because the device is * not designated to receive packets (e.g. backup link in a bond). + * + * @rx_otherhost_dropped: Number of packets dropped due to mismatch + * in destination MAC address. */ struct rtnl_link_stats64 { __u64 rx_packets; @@ -243,6 +246,8 @@ struct rtnl_link_stats64 { __u64 rx_compressed; __u64 tx_compressed; __u64 rx_nohandler; + + __u64 rx_otherhost_dropped; }; /* Subset of link stats useful for in-HW collection. Meaning of the fields is as diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index d2be4eb22008..784adc6f6ed2 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -201,11 +201,9 @@ struct io_uring_cqe { * * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries - * IORING_CQE_F_MSG If set, CQE was generated with IORING_OP_MSG_RING */ #define IORING_CQE_F_BUFFER (1U << 0) #define IORING_CQE_F_MORE (1U << 1) -#define IORING_CQE_F_MSG (1U << 2) enum { IORING_CQE_BUFFER_SHIFT = 16, diff --git a/include/uapi/linux/loop.h b/include/uapi/linux/loop.h index 24a1c45bd1ae..98e60801195e 100644 --- a/include/uapi/linux/loop.h +++ b/include/uapi/linux/loop.h @@ -45,7 +45,7 @@ struct loop_info { unsigned long lo_inode; /* ioctl r/o */ __kernel_old_dev_t lo_rdevice; /* ioctl r/o */ int lo_offset; - int lo_encrypt_type; + int lo_encrypt_type; /* obsolete, ignored */ int lo_encrypt_key_size; /* ioctl w/o */ int lo_flags; char lo_name[LO_NAME_SIZE]; @@ -61,7 +61,7 @@ struct loop_info64 { __u64 lo_offset; __u64 lo_sizelimit;/* bytes, 0 == max available */ __u32 lo_number; /* ioctl r/o */ - __u32 lo_encrypt_type; + __u32 lo_encrypt_type; /* obsolete, ignored */ __u32 lo_encrypt_key_size; /* ioctl w/o */ __u32 lo_flags; __u8 lo_file_name[LO_NAME_SIZE]; diff --git a/include/uapi/linux/rtc.h b/include/uapi/linux/rtc.h index 03e5b776e597..97aca4503a6a 100644 --- a/include/uapi/linux/rtc.h +++ b/include/uapi/linux/rtc.h @@ -133,7 +133,8 @@ struct rtc_param { #define RTC_FEATURE_UPDATE_INTERRUPT 4 #define RTC_FEATURE_CORRECTION 5 #define RTC_FEATURE_BACKUP_SWITCH_MODE 6 -#define RTC_FEATURE_CNT 7 +#define RTC_FEATURE_ALARM_WAKEUP_ONLY 7 +#define RTC_FEATURE_CNT 8 /* parameter list */ #define RTC_PARAM_FEATURES 0 diff --git a/include/uapi/linux/tipc_config.h b/include/uapi/linux/tipc_config.h index 4dfc05651c98..c00adf2fe868 100644 --- a/include/uapi/linux/tipc_config.h +++ b/include/uapi/linux/tipc_config.h @@ -43,10 +43,6 @@ #include <linux/tipc.h> #include <asm/byteorder.h> -#ifndef __KERNEL__ -#include <arpa/inet.h> /* for ntohs etc. */ -#endif - /* * Configuration * @@ -269,33 +265,33 @@ static inline int TLV_OK(const void *tlv, __u16 space) */ return (space >= TLV_SPACE(0)) && - (ntohs(((struct tlv_desc *)tlv)->tlv_len) <= space); + (__be16_to_cpu(((struct tlv_desc *)tlv)->tlv_len) <= space); } static inline int TLV_CHECK(const void *tlv, __u16 space, __u16 exp_type) { return TLV_OK(tlv, space) && - (ntohs(((struct tlv_desc *)tlv)->tlv_type) == exp_type); + (__be16_to_cpu(((struct tlv_desc *)tlv)->tlv_type) == exp_type); } static inline int TLV_GET_LEN(struct tlv_desc *tlv) { - return ntohs(tlv->tlv_len); + return __be16_to_cpu(tlv->tlv_len); } static inline void TLV_SET_LEN(struct tlv_desc *tlv, __u16 len) { - tlv->tlv_len = htons(len); + tlv->tlv_len = __cpu_to_be16(len); } static inline int TLV_CHECK_TYPE(struct tlv_desc *tlv, __u16 type) { - return (ntohs(tlv->tlv_type) == type); + return (__be16_to_cpu(tlv->tlv_type) == type); } static inline void TLV_SET_TYPE(struct tlv_desc *tlv, __u16 type) { - tlv->tlv_type = htons(type); + tlv->tlv_type = __cpu_to_be16(type); } static inline int TLV_SET(void *tlv, __u16 type, void *data, __u16 len) @@ -305,8 +301,8 @@ static inline int TLV_SET(void *tlv, __u16 type, void *data, __u16 len) tlv_len = TLV_LENGTH(len); tlv_ptr = (struct tlv_desc *)tlv; - tlv_ptr->tlv_type = htons(type); - tlv_ptr->tlv_len = htons(tlv_len); + tlv_ptr->tlv_type = __cpu_to_be16(type); + tlv_ptr->tlv_len = __cpu_to_be16(tlv_len); if (len && data) { memcpy(TLV_DATA(tlv_ptr), data, len); memset((char *)TLV_DATA(tlv_ptr) + len, 0, TLV_SPACE(len) - tlv_len); @@ -348,7 +344,7 @@ static inline void *TLV_LIST_DATA(struct tlv_list_desc *list) static inline void TLV_LIST_STEP(struct tlv_list_desc *list) { - __u16 tlv_space = TLV_ALIGN(ntohs(list->tlv_ptr->tlv_len)); + __u16 tlv_space = TLV_ALIGN(__be16_to_cpu(list->tlv_ptr->tlv_len)); list->tlv_ptr = (struct tlv_desc *)((char *)list->tlv_ptr + tlv_space); list->tlv_space -= tlv_space; @@ -404,9 +400,9 @@ static inline int TCM_SET(void *msg, __u16 cmd, __u16 flags, msg_len = TCM_LENGTH(data_len); tcm_hdr = (struct tipc_cfg_msg_hdr *)msg; - tcm_hdr->tcm_len = htonl(msg_len); - tcm_hdr->tcm_type = htons(cmd); - tcm_hdr->tcm_flags = htons(flags); + tcm_hdr->tcm_len = __cpu_to_be32(msg_len); + tcm_hdr->tcm_type = __cpu_to_be16(cmd); + tcm_hdr->tcm_flags = __cpu_to_be16(flags); if (data_len && data) { memcpy(TCM_DATA(msg), data, data_len); memset((char *)TCM_DATA(msg) + data_len, 0, TCM_SPACE(data_len) - msg_len); diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index c998860d7bbc..5d99e7c242a2 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -150,4 +150,11 @@ /* Get the valid iova range */ #define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \ struct vhost_vdpa_iova_range) + +/* Get the config size */ +#define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32) + +/* Get the count of all virtqueues */ +#define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32) + #endif diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h index b5eda06f0d57..f0fb0ae021c0 100644 --- a/include/uapi/linux/virtio_config.h +++ b/include/uapi/linux/virtio_config.h @@ -83,6 +83,12 @@ #define VIRTIO_F_RING_PACKED 34 /* + * Inorder feature indicates that all buffers are used by the device + * in the same order in which they have been made available. + */ +#define VIRTIO_F_IN_ORDER 35 + +/* * This feature indicates that memory accesses by the driver and the * device are ordered in a way described by the platform. */ diff --git a/include/uapi/linux/virtio_crypto.h b/include/uapi/linux/virtio_crypto.h index a03932f10565..71a54a6849ca 100644 --- a/include/uapi/linux/virtio_crypto.h +++ b/include/uapi/linux/virtio_crypto.h @@ -37,6 +37,7 @@ #define VIRTIO_CRYPTO_SERVICE_HASH 1 #define VIRTIO_CRYPTO_SERVICE_MAC 2 #define VIRTIO_CRYPTO_SERVICE_AEAD 3 +#define VIRTIO_CRYPTO_SERVICE_AKCIPHER 4 #define VIRTIO_CRYPTO_OPCODE(service, op) (((service) << 8) | (op)) @@ -57,6 +58,10 @@ struct virtio_crypto_ctrl_header { VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02) #define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03) +#define VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x04) +#define VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x05) __le32 opcode; __le32 algo; __le32 flag; @@ -180,6 +185,58 @@ struct virtio_crypto_aead_create_session_req { __u8 padding[32]; }; +struct virtio_crypto_rsa_session_para { +#define VIRTIO_CRYPTO_RSA_RAW_PADDING 0 +#define VIRTIO_CRYPTO_RSA_PKCS1_PADDING 1 + __le32 padding_algo; + +#define VIRTIO_CRYPTO_RSA_NO_HASH 0 +#define VIRTIO_CRYPTO_RSA_MD2 1 +#define VIRTIO_CRYPTO_RSA_MD3 2 +#define VIRTIO_CRYPTO_RSA_MD4 3 +#define VIRTIO_CRYPTO_RSA_MD5 4 +#define VIRTIO_CRYPTO_RSA_SHA1 5 +#define VIRTIO_CRYPTO_RSA_SHA256 6 +#define VIRTIO_CRYPTO_RSA_SHA384 7 +#define VIRTIO_CRYPTO_RSA_SHA512 8 +#define VIRTIO_CRYPTO_RSA_SHA224 9 + __le32 hash_algo; +}; + +struct virtio_crypto_ecdsa_session_para { +#define VIRTIO_CRYPTO_CURVE_UNKNOWN 0 +#define VIRTIO_CRYPTO_CURVE_NIST_P192 1 +#define VIRTIO_CRYPTO_CURVE_NIST_P224 2 +#define VIRTIO_CRYPTO_CURVE_NIST_P256 3 +#define VIRTIO_CRYPTO_CURVE_NIST_P384 4 +#define VIRTIO_CRYPTO_CURVE_NIST_P521 5 + __le32 curve_id; + __le32 padding; +}; + +struct virtio_crypto_akcipher_session_para { +#define VIRTIO_CRYPTO_NO_AKCIPHER 0 +#define VIRTIO_CRYPTO_AKCIPHER_RSA 1 +#define VIRTIO_CRYPTO_AKCIPHER_DSA 2 +#define VIRTIO_CRYPTO_AKCIPHER_ECDSA 3 + __le32 algo; + +#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC 1 +#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE 2 + __le32 keytype; + __le32 keylen; + + union { + struct virtio_crypto_rsa_session_para rsa; + struct virtio_crypto_ecdsa_session_para ecdsa; + } u; +}; + +struct virtio_crypto_akcipher_create_session_req { + struct virtio_crypto_akcipher_session_para para; + __u8 padding[36]; +}; + struct virtio_crypto_alg_chain_session_para { #define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER 1 #define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH 2 @@ -247,6 +304,8 @@ struct virtio_crypto_op_ctrl_req { mac_create_session; struct virtio_crypto_aead_create_session_req aead_create_session; + struct virtio_crypto_akcipher_create_session_req + akcipher_create_session; struct virtio_crypto_destroy_session_req destroy_session; __u8 padding[56]; @@ -266,6 +325,14 @@ struct virtio_crypto_op_header { VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00) #define VIRTIO_CRYPTO_AEAD_DECRYPT \ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01) +#define VIRTIO_CRYPTO_AKCIPHER_ENCRYPT \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x00) +#define VIRTIO_CRYPTO_AKCIPHER_DECRYPT \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x01) +#define VIRTIO_CRYPTO_AKCIPHER_SIGN \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x02) +#define VIRTIO_CRYPTO_AKCIPHER_VERIFY \ + VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x03) __le32 opcode; /* algo should be service-specific algorithms */ __le32 algo; @@ -390,6 +457,16 @@ struct virtio_crypto_aead_data_req { __u8 padding[32]; }; +struct virtio_crypto_akcipher_para { + __le32 src_data_len; + __le32 dst_data_len; +}; + +struct virtio_crypto_akcipher_data_req { + struct virtio_crypto_akcipher_para para; + __u8 padding[40]; +}; + /* The request of the data virtqueue's packet */ struct virtio_crypto_op_data_req { struct virtio_crypto_op_header header; @@ -399,6 +476,7 @@ struct virtio_crypto_op_data_req { struct virtio_crypto_hash_data_req hash_req; struct virtio_crypto_mac_data_req mac_req; struct virtio_crypto_aead_data_req aead_req; + struct virtio_crypto_akcipher_data_req akcipher_req; __u8 padding[48]; } u; }; @@ -408,6 +486,8 @@ struct virtio_crypto_op_data_req { #define VIRTIO_CRYPTO_BADMSG 2 #define VIRTIO_CRYPTO_NOTSUPP 3 #define VIRTIO_CRYPTO_INVSESS 4 /* Invalid session id */ +#define VIRTIO_CRYPTO_NOSPC 5 /* no free session ID */ +#define VIRTIO_CRYPTO_KEY_REJECTED 6 /* Signature verification failed */ /* The accelerator hardware is ready */ #define VIRTIO_CRYPTO_S_HW_READY (1 << 0) @@ -438,7 +518,7 @@ struct virtio_crypto_config { __le32 max_cipher_key_len; /* Maximum length of authenticated key */ __le32 max_auth_key_len; - __le32 reserve; + __le32 akcipher_algo; /* Maximum size of each crypto request's content */ __le64 max_size; }; diff --git a/init/Kconfig b/init/Kconfig index 97463a33baa7..ddcbefe535e9 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -62,13 +62,13 @@ config LLD_VERSION config CC_CAN_LINK bool - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag)) if 64BIT - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m32-flag)) + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag)) if 64BIT + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m32-flag)) config CC_CAN_LINK_STATIC bool - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag) -static) if 64BIT - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m32-flag) -static) + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag) -static) if 64BIT + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m32-flag) -static) config CC_HAS_ASM_GOTO def_bool $(success,$(srctree)/scripts/gcc-goto.sh $(CC)) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 8c6de5a9ecc4..c2f1fd95a821 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -133,14 +133,4 @@ config SCHED_CORE which is the likely usage by Linux distributions, there should be no measurable impact on performance. -config ARCH_WANTS_RT_DELAYED_SIGNALS - bool - help - This option is selected by architectures where raising signals - can happen in atomic contexts on PREEMPT_RT enabled kernels. This - option delays raising the signal until the return to user space - loop where it is also delivered. X86 requires this to deliver - signals from trap handlers which run on IST stacks. - -config RT_DELAYED_SIGNALS - def_bool PREEMPT_RT && ARCH_WANTS_RT_DELAYED_SIGNALS + diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 35a1d29d6a2e..9743c6ccce1a 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -277,12 +277,16 @@ void *dma_direct_alloc(struct device *dev, size_t size, } if (remap) { + pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); + + if (force_dma_unencrypted(dev)) + prot = pgprot_decrypted(prot); + /* remove any dirty cache lines on the kernel alias */ arch_dma_prep_coherent(page, size); /* create a coherent mapping */ - ret = dma_common_contiguous_remap(page, size, - dma_pgprot(dev, PAGE_KERNEL, attrs), + ret = dma_common_contiguous_remap(page, size, prot, __builtin_return_address(0)); if (!ret) goto out_free_pages; @@ -535,6 +539,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, int ret = -ENXIO; vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); + if (force_dma_unencrypted(dev)) + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) return ret; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 559461a826ba..db7244291b74 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -407,8 +407,6 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs); */ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) { - if (force_dma_unencrypted(dev)) - prot = pgprot_decrypted(prot); if (dev_is_dma_coherent(dev)) return prot; #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE diff --git a/kernel/entry/common.c b/kernel/entry/common.c index ef8d94a98b7e..e57a224d6b79 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -142,18 +142,6 @@ void noinstr exit_to_user_mode(void) /* Workaround to allow gradual conversion of architecture code */ void __weak arch_do_signal_or_restart(struct pt_regs *regs) { } -#ifdef CONFIG_RT_DELAYED_SIGNALS -static inline void raise_delayed_signal(void) -{ - if (unlikely(current->forced_info.si_signo)) { - force_sig_info(¤t->forced_info); - current->forced_info.si_signo = 0; - } -} -#else -static inline void raise_delayed_signal(void) { } -#endif - static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work) { @@ -168,8 +156,6 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, if (ti_work & _TIF_NEED_RESCHED) schedule(); - raise_delayed_signal(); - if (ti_work & _TIF_UPROBE) uprobe_notify_resume(regs); diff --git a/kernel/signal.c b/kernel/signal.c index 368a34c25bbf..30cd1ca43bcd 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1308,43 +1308,6 @@ enum sig_handler { }; /* - * On some archictectures, PREEMPT_RT has to delay sending a signal from a - * trap since it cannot enable preemption, and the signal code's - * spin_locks turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME - * which will send the signal on exit of the trap. - */ -#ifdef CONFIG_RT_DELAYED_SIGNALS -static inline bool force_sig_delayed(struct kernel_siginfo *info, - struct task_struct *t) -{ - if (!in_atomic()) - return false; - - if (WARN_ON_ONCE(t->forced_info.si_signo)) - return true; - - if (is_si_special(info)) { - WARN_ON_ONCE(info != SEND_SIG_PRIV); - t->forced_info.si_signo = info->si_signo; - t->forced_info.si_errno = 0; - t->forced_info.si_code = SI_KERNEL; - t->forced_info.si_pid = 0; - t->forced_info.si_uid = 0; - } else { - t->forced_info = *info; - } - set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); - return true; -} -#else -static inline bool force_sig_delayed(struct kernel_siginfo *info, - struct task_struct *t) -{ - return false; -} -#endif - -/* * Force a signal that the process can't ignore: if necessary * we unblock the signal and change any SIG_IGN to SIG_DFL. * @@ -1364,9 +1327,6 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, struct k_sigaction *action; int sig = info->si_signo; - if (force_sig_delayed(info, t)) - return 0; - spin_lock_irqsave(&t->sighand->siglock, flags); action = &t->sighand->action[sig-1]; ignored = action->sa.sa_handler == SIG_IGN; diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 9bb54c0b3b2d..2c43e327a619 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -767,6 +767,7 @@ config USER_EVENTS bool "User trace events" select TRACING select DYNAMIC_EVENTS + depends on BROKEN || COMPILE_TEST # API needs to be straighten out help User trace events are user-defined trace events that can be used like an existing kernel trace event. User trace diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 836f021cb609..b26f3da943de 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2347,11 +2347,11 @@ kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip, } static int -kprobe_multi_resolve_syms(const void *usyms, u32 cnt, +kprobe_multi_resolve_syms(const void __user *usyms, u32 cnt, unsigned long *addrs) { unsigned long addr, size; - const char **syms; + const char __user **syms; int err = -ENOMEM; unsigned int i; char *func; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 19028e072cdb..8f4fb328133a 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -7,6 +7,7 @@ * * Highly modified by Steven Rostedt (VMware). */ +#include <linux/jump_label.h> #include <linux/suspend.h> #include <linux/ftrace.h> #include <linux/slab.h> @@ -23,25 +24,13 @@ #define ASSIGN_OPS_HASH(opsname, val) #endif -static bool kill_ftrace_graph; +DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; /* Both enabled by default (can be cleared by function_graph tracer flags */ static bool fgraph_sleep_time = true; /** - * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called - * - * ftrace_graph_stop() is called when a severe error is detected in - * the function graph tracing. This function is called by the critical - * paths of function graph to keep those paths from doing any more harm. - */ -bool ftrace_graph_is_dead(void) -{ - return kill_ftrace_graph; -} - -/** * ftrace_graph_stop - set to permanently disable function graph tracing * * In case of an error int function graph tracing, this is called @@ -51,7 +40,7 @@ bool ftrace_graph_is_dead(void) */ void ftrace_graph_stop(void) { - kill_ftrace_graph = true; + static_branch_enable(&kill_ftrace_graph); } /* Add a function return address to the trace stack on thread info.*/ diff --git a/kernel/trace/rethook.c b/kernel/trace/rethook.c index ab463a4d2b23..b56833700d23 100644 --- a/kernel/trace/rethook.c +++ b/kernel/trace/rethook.c @@ -65,7 +65,7 @@ static void rethook_free_rcu(struct rcu_head *head) */ void rethook_free(struct rethook *rh) { - rcu_assign_pointer(rh->handler, NULL); + WRITE_ONCE(rh->handler, NULL); call_rcu(&rh->rcu, rethook_free_rcu); } diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8b3d241a31c2..706e1686b5eb 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -18,7 +18,12 @@ #include <linux/tracefs.h> #include <linux/types.h> #include <linux/uaccess.h> +/* Reminder to move to uapi when everything works */ +#ifdef CONFIG_COMPILE_TEST +#include <linux/user_events.h> +#else #include <uapi/linux/user_events.h> +#endif #include "trace.h" #include "trace_dynevent.h" @@ -42,9 +47,6 @@ #define MAX_FIELD_ARRAY_SIZE 1024 #define MAX_FIELD_ARG_NAME 256 -#define MAX_BPF_COPY_SIZE PAGE_SIZE -#define MAX_STACK_BPF_DATA 512 - static char *register_page_data; static DEFINE_MUTEX(reg_mutex); @@ -405,19 +407,6 @@ parse: type[0] != 'u', FILTER_OTHER); } -static void user_event_parse_flags(struct user_event *user, char *flags) -{ - char *flag; - - if (flags == NULL) - return; - - while ((flag = strsep(&flags, ",")) != NULL) { - if (strcmp(flag, "BPF_ITER") == 0) - user->flags |= FLAG_BPF_ITER; - } -} - static int user_event_parse_fields(struct user_event *user, char *args) { char *field; @@ -713,64 +702,14 @@ discard: } #ifdef CONFIG_PERF_EVENTS -static void user_event_bpf(struct user_event *user, struct iov_iter *i) -{ - struct user_bpf_context context; - struct user_bpf_iter bpf_i; - char fast_data[MAX_STACK_BPF_DATA]; - void *temp = NULL; - - if ((user->flags & FLAG_BPF_ITER) && iter_is_iovec(i)) { - /* Raw iterator */ - context.data_type = USER_BPF_DATA_ITER; - context.data_len = i->count; - context.iter = &bpf_i; - - bpf_i.iov_offset = i->iov_offset; - bpf_i.iov = i->iov; - bpf_i.nr_segs = i->nr_segs; - } else if (i->nr_segs == 1 && iter_is_iovec(i)) { - /* Single buffer from user */ - context.data_type = USER_BPF_DATA_USER; - context.data_len = i->count; - context.udata = i->iov->iov_base + i->iov_offset; - } else { - /* Multi buffer from user */ - struct iov_iter copy = *i; - size_t copy_size = min_t(size_t, i->count, MAX_BPF_COPY_SIZE); - - context.data_type = USER_BPF_DATA_KERNEL; - context.kdata = fast_data; - - if (unlikely(copy_size > sizeof(fast_data))) { - temp = kmalloc(copy_size, GFP_NOWAIT); - - if (temp) - context.kdata = temp; - else - copy_size = sizeof(fast_data); - } - - context.data_len = copy_nofault(context.kdata, - copy_size, ©); - } - - trace_call_bpf(&user->call, &context); - - kfree(temp); -} - /* - * Writes the user supplied payload out to perf ring buffer or eBPF program. + * Writes the user supplied payload out to perf ring buffer. */ static void user_event_perf(struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted) { struct hlist_head *perf_head; - if (bpf_prog_array_valid(&user->call)) - user_event_bpf(user, i); - perf_head = this_cpu_ptr(user->call.perf_events); if (perf_head && !hlist_empty(perf_head)) { @@ -1136,8 +1075,6 @@ static int user_event_parse(char *name, char *args, char *flags, user->tracepoint.name = name; - user_event_parse_flags(user, flags); - ret = user_event_parse_fields(user, args); if (ret) @@ -1165,11 +1102,11 @@ static int user_event_parse(char *name, char *args, char *flags, #endif mutex_lock(&event_mutex); + ret = user_event_trace_register(user); - mutex_unlock(&event_mutex); if (ret) - goto put_user; + goto put_user_lock; user->index = index; @@ -1181,8 +1118,12 @@ static int user_event_parse(char *name, char *args, char *flags, set_bit(user->index, page_bitmap); hash_add(register_table, &user->node, key); + mutex_unlock(&event_mutex); + *newuser = user; return 0; +put_user_lock: + mutex_unlock(&event_mutex); put_user: user_event_destroy_fields(user); user_event_destroy_validators(user); @@ -1575,9 +1516,6 @@ static int user_seq_show(struct seq_file *m, void *p) busy++; } - if (flags & FLAG_BPF_ITER) - seq_puts(m, " FLAG:BPF_ITER"); - seq_puts(m, "\n"); active++; } diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c index 3990e4df3d7b..230038d4f908 100644 --- a/kernel/watch_queue.c +++ b/kernel/watch_queue.c @@ -370,6 +370,7 @@ static void __put_watch_queue(struct kref *kref) for (i = 0; i < wqueue->nr_pages; i++) __free_page(wqueue->notes[i]); + kfree(wqueue->notes); bitmap_free(wqueue->notes_bitmap); wfilter = rcu_access_pointer(wqueue->filter); diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c index 8c3365f26e51..b247d412ddef 100644 --- a/lib/logic_iomem.c +++ b/lib/logic_iomem.c @@ -68,7 +68,7 @@ int logic_iomem_add_region(struct resource *resource, } EXPORT_SYMBOL(logic_iomem_add_region); -#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK static void __iomem *real_ioremap(phys_addr_t offset, size_t size) { WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n", @@ -81,7 +81,7 @@ static void real_iounmap(volatile void __iomem *addr) WARN(1, "invalid iounmap for addr 0x%llx\n", (unsigned long long)(uintptr_t __force)addr); } -#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ +#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */ void __iomem *ioremap(phys_addr_t offset, size_t size) { @@ -168,7 +168,7 @@ void iounmap(volatile void __iomem *addr) } EXPORT_SYMBOL(iounmap); -#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK #define MAKE_FALLBACK(op, sz) \ static u##sz real_raw_read ## op(const volatile void __iomem *addr) \ { \ @@ -213,7 +213,7 @@ static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer, WARN(1, "Invalid memcpy_toio at address 0x%llx\n", (unsigned long long)(uintptr_t __force)addr); } -#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ +#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */ #define MAKE_OP(op, sz) \ u##sz __raw_read ## op(const volatile void __iomem *addr) \ diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 2eb3de18ded3..ae4fd4de9ebe 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -110,7 +110,7 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, sb->alloc_hint = NULL; } - sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node); + sb->map = kvzalloc_node(sb->map_nr * sizeof(*sb->map), flags, node); if (!sb->map) { free_percpu(sb->alloc_hint); return -ENOMEM; diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 8b1c318189ce..e77d4856442c 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1463,6 +1463,25 @@ unlock: XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_create_range_5(struct xarray *xa, + unsigned long index, unsigned int order) +{ + XA_STATE_ORDER(xas, xa, index, order); + unsigned int i; + + xa_store_order(xa, index, order, xa_mk_index(index), GFP_KERNEL); + + for (i = 0; i < order + 10; i++) { + do { + xas_lock(&xas); + xas_create_range(&xas); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + } + + xa_destroy(xa); +} + static noinline void check_create_range(struct xarray *xa) { unsigned int order; @@ -1490,6 +1509,9 @@ static noinline void check_create_range(struct xarray *xa) check_create_range_4(xa, (3U << order) + 1, order); check_create_range_4(xa, (3U << order) - 1, order); check_create_range_4(xa, (1U << 24) + 1, order); + + check_create_range_5(xa, 0, order); + check_create_range_5(xa, (1U << order), order); } check_create_range_3(); diff --git a/lib/xarray.c b/lib/xarray.c index b95e92598b9c..4acc88ea7c21 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -722,6 +722,8 @@ void xas_create_range(struct xa_state *xas) for (;;) { struct xa_node *node = xas->xa_node; + if (node->shift >= shift) + break; xas->xa_node = xa_parent_locked(xas->xa, node); xas->xa_offset = node->offset - 1; if (node->offset != 0) @@ -1079,6 +1081,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) xa_mk_node(child)); if (xa_is_value(curr)) values--; + xas_update(xas, child); } else { unsigned int canon = offset - xas->xa_sibs; @@ -1093,6 +1096,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) } while (offset-- > xas->xa_offset); node->nr_values += values; + xas_update(xas, node); } EXPORT_SYMBOL_GPL(xas_split); #endif diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c index 907fefde2572..4b8eab4b3f45 100644 --- a/mm/balloon_compaction.c +++ b/mm/balloon_compaction.c @@ -203,7 +203,7 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue); #ifdef CONFIG_BALLOON_COMPACTION -bool balloon_page_isolate(struct page *page, isolate_mode_t mode) +static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) { struct balloon_dev_info *b_dev_info = balloon_page_device(page); @@ -217,7 +217,7 @@ bool balloon_page_isolate(struct page *page, isolate_mode_t mode) return true; } -void balloon_page_putback(struct page *page) +static void balloon_page_putback(struct page *page) { struct balloon_dev_info *b_dev_info = balloon_page_device(page); unsigned long flags; @@ -230,7 +230,7 @@ void balloon_page_putback(struct page *page) /* move_to_new_page() counterpart for a ballooned page */ -int balloon_page_migrate(struct address_space *mapping, +static int balloon_page_migrate(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) { diff --git a/mm/damon/core.c b/mm/damon/core.c index c1e0fed4e877..5ce8d7c867f0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1019,12 +1019,15 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) struct damos *s; unsigned long wait_time; unsigned long min_wait_time = 0; + bool init_wait_time = false; while (!kdamond_need_stop(ctx)) { damon_for_each_scheme(s, ctx) { wait_time = damos_wmark_wait_us(s); - if (!min_wait_time || wait_time < min_wait_time) + if (!init_wait_time || wait_time < min_wait_time) { + init_wait_time = true; min_wait_time = wait_time; + } } if (!min_wait_time) return 0; diff --git a/mm/filemap.c b/mm/filemap.c index 647d72bf23b6..3a5ffb5587cd 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2538,7 +2538,7 @@ static int filemap_create_folio(struct file *file, * the page cache as the locked folio would then be enough to * synchronize with hole punching. But there are code paths * such as filemap_update_page() filling in partially uptodate - * pages or ->readpages() that need to hold invalidate_lock + * pages or ->readahead() that need to hold invalidate_lock * while mapping blocks for IO so let's hold the lock here as * well to keep locking rules simple. */ @@ -3752,9 +3752,10 @@ out: } EXPORT_SYMBOL(generic_file_direct_write); -ssize_t generic_perform_write(struct file *file, - struct iov_iter *i, loff_t pos) +ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) { + struct file *file = iocb->ki_filp; + loff_t pos = iocb->ki_pos; struct address_space *mapping = file->f_mapping; const struct address_space_operations *a_ops = mapping->a_ops; long status = 0; @@ -3884,7 +3885,8 @@ ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (written < 0 || !iov_iter_count(from) || IS_DAX(inode)) goto out; - status = generic_perform_write(file, from, pos = iocb->ki_pos); + pos = iocb->ki_pos; + status = generic_perform_write(iocb, from); /* * If generic_perform_write() returned a synchronous error * then we want to return the number of bytes which were @@ -3916,7 +3918,7 @@ ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) */ } } else { - written = generic_perform_write(file, from, iocb->ki_pos); + written = generic_perform_write(iocb, from); if (likely(written > 0)) iocb->ki_pos += written; } @@ -1404,6 +1404,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1438,8 +1439,10 @@ long populate_vma_page_range(struct vm_area_struct *vma, * We made sure addr is within a VMA, so the following will * not result in a stack expansion that recurses back here. */ - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* @@ -1471,6 +1474,7 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1498,8 +1502,10 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, if (check_vma_flags(vma, gup_flags)) return -EINVAL; - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* diff --git a/mm/internal.h b/mm/internal.h index 58dc6adc19c5..cf16280ce132 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -456,7 +456,8 @@ static inline void munlock_vma_page(struct page *page, } void mlock_new_page(struct page *page); bool need_mlock_page_drain(int cpu); -void mlock_page_drain(int cpu); +void mlock_page_drain_local(void); +void mlock_page_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); @@ -539,7 +540,8 @@ static inline void munlock_vma_page(struct page *page, struct vm_area_struct *vma, bool compound) { } static inline void mlock_new_page(struct page *page) { } static inline bool need_mlock_page_drain(int cpu) { return false; } -static inline void mlock_page_drain(int cpu) { } +static inline void mlock_page_drain_local(void) { } +static inline void mlock_page_drain_remote(int cpu) { } static inline void vunmap_range_noflush(unsigned long start, unsigned long end) { } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 2f9fdfde1941..a203747ad2c0 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -566,6 +566,8 @@ static unsigned long kfence_init_pool(void) * enters __slab_free() slow-path. */ for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { + struct slab *slab = page_slab(&pages[i]); + if (!i || (i % 2)) continue; @@ -573,7 +575,11 @@ static unsigned long kfence_init_pool(void) if (WARN_ON(compound_head(&pages[i]) != &pages[i])) return addr; - __SetPageSlab(&pages[i]); + __folio_set_slab(slab_folio(slab)); +#ifdef CONFIG_MEMCG + slab->memcg_data = (unsigned long)&kfence_metadata[i / 2 - 1].objcg | + MEMCG_DATA_OBJCGS; +#endif } /* @@ -1033,6 +1039,9 @@ void __kfence_free(void *addr) { struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); +#ifdef CONFIG_MEMCG + KFENCE_WARN_ON(meta->objcg); +#endif /* * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing * the object, as the object page may be recycled for other-typed diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 2a2d5de9d379..9a6c4b1b12a8 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -89,6 +89,9 @@ struct kfence_metadata { struct kfence_track free_track; /* For updating alloc_covered on frees. */ u32 alloc_stack_hash; +#ifdef CONFIG_MEMCG + struct obj_cgroup *objcg; +#endif }; extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS]; diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 7580baa76af1..acd7cbb82e16 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -796,6 +796,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) unsigned long flags; struct kmemleak_object *object; struct kmemleak_scan_area *area = NULL; + unsigned long untagged_ptr; + unsigned long untagged_objp; object = find_and_get_object(ptr, 1); if (!object) { @@ -804,6 +806,9 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) return; } + untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); + untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); + if (scan_area_cache) area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp)); @@ -815,8 +820,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) goto out_unlock; } if (size == SIZE_MAX) { - size = object->pointer + object->size - ptr; - } else if (ptr + size > object->pointer + object->size) { + size = untagged_objp + object->size - untagged_ptr; + } else if (untagged_ptr + size > untagged_objp + object->size) { kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); dump_object_info(object); kmem_cache_free(scan_area_cache, area); diff --git a/mm/madvise.c b/mm/madvise.c index b41858ee937b..1873616a37d2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1464,16 +1464,9 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, while (iov_iter_count(&iter)) { iovec = iov_iter_iovec(&iter); - /* - * do_madvise returns ENOMEM if unmapped holes are present - * in the passed VMA. process_madvise() is expected to skip - * unmapped holes passed to it in the 'struct iovec' list - * and not fail because of them. Thus treat -ENOMEM return - * from do_madvise as valid and continue processing. - */ ret = do_madvise(mm, (unsigned long)iovec.iov_base, iovec.iov_len, behavior); - if (ret < 0 && ret != -ENOMEM) + if (ret < 0) break; iov_iter_advance(&iter, iovec.iov_len); } diff --git a/mm/memory.c b/mm/memory.c index be44d0b36b18..76e3af9639d9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3918,14 +3918,18 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) return ret; if (unlikely(PageHWPoison(vmf->page))) { + struct page *page = vmf->page; vm_fault_t poisonret = VM_FAULT_HWPOISON; if (ret & VM_FAULT_LOCKED) { + if (page_mapped(page)) + unmap_mapping_pages(page_mapping(page), + page->index, 1, false); /* Retry if a clean page was removed from the cache. */ - if (invalidate_inode_page(vmf->page)) - poisonret = 0; - unlock_page(vmf->page); + if (invalidate_inode_page(page)) + poisonret = VM_FAULT_NOPAGE; + unlock_page(page); } - put_page(vmf->page); + put_page(page); vmf->page = NULL; return poisonret; } diff --git a/mm/migrate.c b/mm/migrate.c index 3d60823afd2d..de175e2fdba5 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -246,7 +246,7 @@ static bool remove_migration_pte(struct folio *folio, set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); trace_remove_migration_pte(pvmw.address, pte_val(pte), compound_order(new)); diff --git a/mm/mlock.c b/mm/mlock.c index 529fbc1f27c8..716caf851043 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -28,7 +28,14 @@ #include "internal.h" -static DEFINE_PER_CPU(struct pagevec, mlock_pvec); +struct mlock_pvec { + local_lock_t lock; + struct pagevec vec; +}; + +static DEFINE_PER_CPU(struct mlock_pvec, mlock_pvec) = { + .lock = INIT_LOCAL_LOCK(lock), +}; bool can_do_mlock(void) { @@ -203,18 +210,30 @@ static void mlock_pagevec(struct pagevec *pvec) pagevec_reinit(pvec); } -void mlock_page_drain(int cpu) +void mlock_page_drain_local(void) +{ + struct pagevec *pvec; + + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); + if (pagevec_count(pvec)) + mlock_pagevec(pvec); + local_unlock(&mlock_pvec.lock); +} + +void mlock_page_drain_remote(int cpu) { struct pagevec *pvec; - pvec = &per_cpu(mlock_pvec, cpu); + WARN_ON_ONCE(cpu_online(cpu)); + pvec = &per_cpu(mlock_pvec.vec, cpu); if (pagevec_count(pvec)) mlock_pagevec(pvec); } bool need_mlock_page_drain(int cpu) { - return pagevec_count(&per_cpu(mlock_pvec, cpu)); + return pagevec_count(&per_cpu(mlock_pvec.vec, cpu)); } /** @@ -223,7 +242,10 @@ bool need_mlock_page_drain(int cpu) */ void mlock_folio(struct folio *folio) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); if (!folio_test_set_mlocked(folio)) { int nr_pages = folio_nr_pages(folio); @@ -236,7 +258,7 @@ void mlock_folio(struct folio *folio) if (!pagevec_add(pvec, mlock_lru(&folio->page)) || folio_test_large(folio) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -245,9 +267,11 @@ void mlock_folio(struct folio *folio) */ void mlock_new_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; int nr_pages = thp_nr_pages(page); + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); SetPageMlocked(page); mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); @@ -256,7 +280,7 @@ void mlock_new_page(struct page *page) if (!pagevec_add(pvec, mlock_new(page)) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -265,8 +289,10 @@ void mlock_new_page(struct page *page) */ void munlock_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); /* * TestClearPageMlocked(page) must be left to __munlock_page(), * which will check whether the page is multiply mlocked. @@ -276,7 +302,7 @@ void munlock_page(struct page *page) if (!pagevec_add(pvec, page) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } static int mlock_pte_range(pmd_t *pmd, unsigned long addr, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6c6af8658775..2db95780e003 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8367,6 +8367,7 @@ static int page_alloc_cpu_dead(unsigned int cpu) struct zone *zone; lru_add_drain_cpu(cpu); + mlock_page_drain_remote(cpu); drain_pages(cpu); /* diff --git a/mm/readahead.c b/mm/readahead.c index d3a47546d17d..8e3775829513 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -13,29 +13,29 @@ * * Readahead is used to read content into the page cache before it is * explicitly requested by the application. Readahead only ever - * attempts to read pages that are not yet in the page cache. If a - * page is present but not up-to-date, readahead will not try to read + * attempts to read folios that are not yet in the page cache. If a + * folio is present but not up-to-date, readahead will not try to read * it. In that case a simple ->readpage() will be requested. * * Readahead is triggered when an application read request (whether a - * systemcall or a page fault) finds that the requested page is not in + * system call or a page fault) finds that the requested folio is not in * the page cache, or that it is in the page cache and has the - * %PG_readahead flag set. This flag indicates that the page was loaded - * as part of a previous read-ahead request and now that it has been - * accessed, it is time for the next read-ahead. + * readahead flag set. This flag indicates that the folio was read + * as part of a previous readahead request and now that it has been + * accessed, it is time for the next readahead. * * Each readahead request is partly synchronous read, and partly async - * read-ahead. This is reflected in the struct file_ra_state which - * contains ->size being to total number of pages, and ->async_size - * which is the number of pages in the async section. The first page in - * this async section will have %PG_readahead set as a trigger for a - * subsequent read ahead. Once a series of sequential reads has been + * readahead. This is reflected in the struct file_ra_state which + * contains ->size being the total number of pages, and ->async_size + * which is the number of pages in the async section. The readahead + * flag will be set on the first folio in this async section to trigger + * a subsequent readahead. Once a series of sequential reads has been * established, there should be no need for a synchronous component and - * all read ahead request will be fully asynchronous. + * all readahead request will be fully asynchronous. * - * When either of the triggers causes a readahead, three numbers need to - * be determined: the start of the region, the size of the region, and - * the size of the async tail. + * When either of the triggers causes a readahead, three numbers need + * to be determined: the start of the region to read, the size of the + * region, and the size of the async tail. * * The start of the region is simply the first page address at or after * the accessed address, which is not currently populated in the page @@ -45,14 +45,14 @@ * was explicitly requested from the determined request size, unless * this would be less than zero - then zero is used. NOTE THIS * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED - * PAGE. + * PAGE. ALSO THIS CALCULATION IS NOT USED CONSISTENTLY. * * The size of the region is normally determined from the size of the * previous readahead which loaded the preceding pages. This may be * discovered from the struct file_ra_state for simple sequential reads, * or from examining the state of the page cache when multiple * sequential reads are interleaved. Specifically: where the readahead - * was triggered by the %PG_readahead flag, the size of the previous + * was triggered by the readahead flag, the size of the previous * readahead is assumed to be the number of pages from the triggering * page to the start of the new readahead. In these cases, the size of * the previous readahead is scaled, often doubled, for the new @@ -65,52 +65,52 @@ * larger than the current request, and it is not scaled up, unless it * is at the start of file. * - * In general read ahead is accelerated at the start of the file, as + * In general readahead is accelerated at the start of the file, as * reads from there are often sequential. There are other minor - * adjustments to the read ahead size in various special cases and these + * adjustments to the readahead size in various special cases and these * are best discovered by reading the code. * - * The above calculation determines the readahead, to which any requested - * read size may be added. + * The above calculation, based on the previous readahead size, + * determines the size of the readahead, to which any requested read + * size may be added. * * Readahead requests are sent to the filesystem using the ->readahead() * address space operation, for which mpage_readahead() is a canonical * implementation. ->readahead() should normally initiate reads on all - * pages, but may fail to read any or all pages without causing an IO + * folios, but may fail to read any or all folios without causing an I/O * error. The page cache reading code will issue a ->readpage() request - * for any page which ->readahead() does not provided, and only an error + * for any folio which ->readahead() did not read, and only an error * from this will be final. * - * ->readahead() will generally call readahead_page() repeatedly to get - * each page from those prepared for read ahead. It may fail to read a - * page by: + * ->readahead() will generally call readahead_folio() repeatedly to get + * each folio from those prepared for readahead. It may fail to read a + * folio by: * - * * not calling readahead_page() sufficiently many times, effectively - * ignoring some pages, as might be appropriate if the path to + * * not calling readahead_folio() sufficiently many times, effectively + * ignoring some folios, as might be appropriate if the path to * storage is congested. * - * * failing to actually submit a read request for a given page, + * * failing to actually submit a read request for a given folio, * possibly due to insufficient resources, or * * * getting an error during subsequent processing of a request. * - * In the last two cases, the page should be unlocked to indicate that - * the read attempt has failed. In the first case the page will be - * unlocked by the caller. + * In the last two cases, the folio should be unlocked by the filesystem + * to indicate that the read attempt has failed. In the first case the + * folio will be unlocked by the VFS. * - * Those pages not in the final ``async_size`` of the request should be + * Those folios not in the final ``async_size`` of the request should be * considered to be important and ->readahead() should not fail them due * to congestion or temporary resource unavailability, but should wait * for necessary resources (e.g. memory or indexing information) to - * become available. Pages in the final ``async_size`` may be + * become available. Folios in the final ``async_size`` may be * considered less urgent and failure to read them is more acceptable. - * In this case it is best to use delete_from_page_cache() to remove the - * pages from the page cache as is automatically done for pages that - * were not fetched with readahead_page(). This will allow a - * subsequent synchronous read ahead request to try them again. If they + * In this case it is best to use filemap_remove_folio() to remove the + * folios from the page cache as is automatically done for folios that + * were not fetched with readahead_folio(). This will allow a + * subsequent synchronous readahead request to try them again. If they * are left in the page cache, then they will be read individually using - * ->readpage(). - * + * ->readpage() which may be less efficient. */ #include <linux/kernel.h> @@ -142,91 +142,14 @@ file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) } EXPORT_SYMBOL_GPL(file_ra_state_init); -/* - * see if a page needs releasing upon read_cache_pages() failure - * - the caller of read_cache_pages() may have set PG_private or PG_fscache - * before calling, such as the NFS fs marking pages that are cached locally - * on disk, thus we need to give the fs a chance to clean up in the event of - * an error - */ -static void read_cache_pages_invalidate_page(struct address_space *mapping, - struct page *page) -{ - if (page_has_private(page)) { - if (!trylock_page(page)) - BUG(); - page->mapping = mapping; - folio_invalidate(page_folio(page), 0, PAGE_SIZE); - page->mapping = NULL; - unlock_page(page); - } - put_page(page); -} - -/* - * release a list of pages, invalidating them first if need be - */ -static void read_cache_pages_invalidate_pages(struct address_space *mapping, - struct list_head *pages) -{ - struct page *victim; - - while (!list_empty(pages)) { - victim = lru_to_page(pages); - list_del(&victim->lru); - read_cache_pages_invalidate_page(mapping, victim); - } -} - -/** - * read_cache_pages - populate an address space with some pages & start reads against them - * @mapping: the address_space - * @pages: The address of a list_head which contains the target pages. These - * pages have their ->index populated and are otherwise uninitialised. - * @filler: callback routine for filling a single page. - * @data: private data for the callback routine. - * - * Hides the details of the LRU cache etc from the filesystems. - * - * Returns: %0 on success, error return by @filler otherwise - */ -int read_cache_pages(struct address_space *mapping, struct list_head *pages, - int (*filler)(void *, struct page *), void *data) -{ - struct page *page; - int ret = 0; - - while (!list_empty(pages)) { - page = lru_to_page(pages); - list_del(&page->lru); - if (add_to_page_cache_lru(page, mapping, page->index, - readahead_gfp_mask(mapping))) { - read_cache_pages_invalidate_page(mapping, page); - continue; - } - put_page(page); - - ret = filler(data, page); - if (unlikely(ret)) { - read_cache_pages_invalidate_pages(mapping, pages); - break; - } - task_io_account_read(PAGE_SIZE); - } - return ret; -} - -EXPORT_SYMBOL(read_cache_pages); - -static void read_pages(struct readahead_control *rac, struct list_head *pages, - bool skip_page) +static void read_pages(struct readahead_control *rac) { const struct address_space_operations *aops = rac->mapping->a_ops; struct page *page; struct blk_plug plug; if (!readahead_count(rac)) - goto out; + return; blk_start_plug(&plug); @@ -234,7 +157,7 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, aops->readahead(rac); /* * Clean up the remaining pages. The sizes in ->ra - * maybe be used to size next read-ahead, so make sure + * may be used to size the next readahead, so make sure * they accurately reflect what happened. */ while ((page = readahead_page(rac))) { @@ -246,13 +169,6 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, unlock_page(page); put_page(page); } - } else if (aops->readpages) { - aops->readpages(rac->file, rac->mapping, pages, - readahead_count(rac)); - /* Clean up the remaining pages */ - put_pages_list(pages); - rac->_index += rac->_nr_pages; - rac->_nr_pages = 0; } else { while ((page = readahead_page(rac))) { aops->readpage(rac->file, page); @@ -262,12 +178,7 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, blk_finish_plug(&plug); - BUG_ON(pages && !list_empty(pages)); BUG_ON(readahead_count(rac)); - -out: - if (skip_page) - rac->_index++; } /** @@ -289,7 +200,6 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, { struct address_space *mapping = ractl->mapping; unsigned long index = readahead_index(ractl); - LIST_HEAD(page_pool); gfp_t gfp_mask = readahead_gfp_mask(mapping); unsigned long i; @@ -321,7 +231,8 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * have a stable reference to this page, and it's * not worth getting one just for that. */ - read_pages(ractl, &page_pool, true); + read_pages(ractl); + ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -329,13 +240,11 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, folio = filemap_alloc_folio(gfp_mask, 0); if (!folio) break; - if (mapping->a_ops->readpages) { - folio->index = index + i; - list_add(&folio->lru, &page_pool); - } else if (filemap_add_folio(mapping, folio, index + i, + if (filemap_add_folio(mapping, folio, index + i, gfp_mask) < 0) { folio_put(folio); - read_pages(ractl, &page_pool, true); + read_pages(ractl); + ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -349,7 +258,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * uptodate then the caller will launch readpage again, and * will then handle the error. */ - read_pages(ractl, &page_pool, false); + read_pages(ractl); filemap_invalidate_unlock_shared(mapping); memalloc_nofs_restore(nofs); } @@ -394,8 +303,7 @@ void force_page_cache_ra(struct readahead_control *ractl, struct backing_dev_info *bdi = inode_to_bdi(mapping->host); unsigned long max_pages, index; - if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages && - !mapping->a_ops->readahead)) + if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readahead)) return; /* @@ -512,7 +420,7 @@ static pgoff_t count_history_pages(struct address_space *mapping, } /* - * page cache context based read-ahead + * page cache context based readahead */ static int try_context_readahead(struct address_space *mapping, struct file_ra_state *ra, @@ -624,7 +532,7 @@ void page_cache_ra_order(struct readahead_control *ractl, ra->async_size += index - limit - 1; } - read_pages(ractl, NULL, false); + read_pages(ractl); /* * If there were already pages in the page cache, then we may have @@ -763,9 +671,9 @@ void page_cache_sync_ra(struct readahead_control *ractl, bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM); /* - * Even if read-ahead is disabled, issue this request as read-ahead + * Even if readahead is disabled, issue this request as readahead * as we'll need it to satisfy the requested range. The forced - * read-ahead will do the right thing and limit the read to just the + * readahead will do the right thing and limit the read to just the * requested range, which we'll set to 1 page for this case. */ if (!ractl->ra->ra_pages || blk_cgroup_congested()) { @@ -781,7 +689,6 @@ void page_cache_sync_ra(struct readahead_control *ractl, return; } - /* do read-ahead */ ondemand_readahead(ractl, NULL, req_count); } EXPORT_SYMBOL_GPL(page_cache_sync_ra); @@ -789,7 +696,7 @@ EXPORT_SYMBOL_GPL(page_cache_sync_ra); void page_cache_async_ra(struct readahead_control *ractl, struct folio *folio, unsigned long req_count) { - /* no read-ahead */ + /* no readahead */ if (!ractl->ra->ra_pages) return; @@ -804,7 +711,6 @@ void page_cache_async_ra(struct readahead_control *ractl, if (blk_cgroup_congested()) return; - /* do read-ahead */ ondemand_readahead(ractl, folio, req_count); } EXPORT_SYMBOL_GPL(page_cache_async_ra); diff --git a/mm/rmap.c b/mm/rmap.c index 5cb970d51f0a..fedb82371efe 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1683,7 +1683,7 @@ discard: */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } @@ -1961,7 +1961,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } diff --git a/mm/swap.c b/mm/swap.c index bceff0cb559c..7e320ec08c6a 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -624,7 +624,6 @@ void lru_add_drain_cpu(int cpu) pagevec_lru_move_fn(pvec, lru_lazyfree_fn); activate_page_drain(cpu); - mlock_page_drain(cpu); } /** @@ -706,6 +705,7 @@ void lru_add_drain(void) local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } /* @@ -720,6 +720,7 @@ static void lru_add_and_bh_lrus_drain(void) lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); invalidate_bh_lrus_cpu(); + mlock_page_drain_local(); } void lru_add_drain_cpu_zone(struct zone *zone) @@ -728,6 +729,7 @@ void lru_add_drain_cpu_zone(struct zone *zone) lru_add_drain_cpu(smp_processor_id()); drain_local_pages(zone); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } #ifdef CONFIG_SMP diff --git a/net/appletalk/ddp.c b/net/appletalk/ddp.c index bf5736c1d458..a06f4d4a6f47 100644 --- a/net/appletalk/ddp.c +++ b/net/appletalk/ddp.c @@ -1753,8 +1753,7 @@ static int atalk_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, int err = 0; struct sk_buff *skb; - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); lock_sock(sk); if (!skb) diff --git a/net/atm/common.c b/net/atm/common.c index 1cfa9bf1d187..d0c8ab7ff8f6 100644 --- a/net/atm/common.c +++ b/net/atm/common.c @@ -540,7 +540,7 @@ int vcc_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, !test_bit(ATM_VF_READY, &vcc->flags)) return 0; - skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &error); + skb = skb_recv_datagram(sk, flags, &error); if (!skb) return error; diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index 363d47f94532..116481e4da82 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -1669,8 +1669,7 @@ static int ax25_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, } /* Now we can treat all alike */ - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); if (skb == NULL) goto out; diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index a0cb2e3da8d4..62705734343b 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -251,7 +251,6 @@ EXPORT_SYMBOL(bt_accept_dequeue); int bt_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { - int noblock = flags & MSG_DONTWAIT; struct sock *sk = sock->sk; struct sk_buff *skb; size_t copied; @@ -263,7 +262,7 @@ int bt_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (flags & MSG_OOB) return -EOPNOTSUPP; - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) { if (sk->sk_shutdown & RCV_SHUTDOWN) return 0; diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 33b3c0ffc339..189e3115c8c6 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -1453,7 +1453,6 @@ static void hci_sock_cmsg(struct sock *sk, struct msghdr *msg, static int hci_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { - int noblock = flags & MSG_DONTWAIT; struct sock *sk = sock->sk; struct sk_buff *skb; int copied, err; @@ -1470,7 +1469,7 @@ static int hci_sock_recvmsg(struct socket *sock, struct msghdr *msg, if (sk->sk_state == BT_CLOSED) return 0; - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) return err; diff --git a/net/caif/caif_socket.c b/net/caif/caif_socket.c index 2b8892d502f7..251e666ba9a2 100644 --- a/net/caif/caif_socket.c +++ b/net/caif/caif_socket.c @@ -282,7 +282,7 @@ static int caif_seqpkt_recvmsg(struct socket *sock, struct msghdr *m, if (flags & MSG_OOB) goto read_error; - skb = skb_recv_datagram(sk, flags, 0 , &ret); + skb = skb_recv_datagram(sk, flags, &ret); if (!skb) goto read_error; copylen = skb->len; diff --git a/net/can/bcm.c b/net/can/bcm.c index 95d209b52e6a..64c07e650bb4 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -1632,12 +1632,9 @@ static int bcm_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, struct sock *sk = sock->sk; struct sk_buff *skb; int error = 0; - int noblock; int err; - noblock = flags & MSG_DONTWAIT; - flags &= ~MSG_DONTWAIT; - skb = skb_recv_datagram(sk, flags, noblock, &error); + skb = skb_recv_datagram(sk, flags, &error); if (!skb) return error; diff --git a/net/can/isotp.c b/net/can/isotp.c index bafb0fb5f0e0..02d81effaa54 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -1047,7 +1047,6 @@ static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, struct sock *sk = sock->sk; struct sk_buff *skb; struct isotp_sock *so = isotp_sk(sk); - int noblock = flags & MSG_DONTWAIT; int ret = 0; if (flags & ~(MSG_DONTWAIT | MSG_TRUNC | MSG_PEEK)) @@ -1056,8 +1055,7 @@ static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, if (!so->bound) return -EADDRNOTAVAIL; - flags &= ~MSG_DONTWAIT; - skb = skb_recv_datagram(sk, flags, noblock, &ret); + skb = skb_recv_datagram(sk, flags, &ret); if (!skb) return ret; diff --git a/net/can/j1939/socket.c b/net/can/j1939/socket.c index 6dff4510687a..0bb4fd3f6264 100644 --- a/net/can/j1939/socket.c +++ b/net/can/j1939/socket.c @@ -802,7 +802,7 @@ static int j1939_sk_recvmsg(struct socket *sock, struct msghdr *msg, return sock_recv_errqueue(sock->sk, msg, size, SOL_CAN_J1939, SCM_J1939_ERRQUEUE); - skb = skb_recv_datagram(sk, flags, 0, &ret); + skb = skb_recv_datagram(sk, flags, &ret); if (!skb) return ret; diff --git a/net/can/raw.c b/net/can/raw.c index 7105fa4824e4..0cf728dcff36 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -846,16 +846,12 @@ static int raw_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, struct sock *sk = sock->sk; struct sk_buff *skb; int err = 0; - int noblock; - - noblock = flags & MSG_DONTWAIT; - flags &= ~MSG_DONTWAIT; if (flags & MSG_ERRQUEUE) return sock_recv_errqueue(sk, msg, size, SOL_CAN_RAW, SCM_CAN_RAW_ERRQUEUE); - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) return err; diff --git a/net/core/datagram.c b/net/core/datagram.c index ee290776c661..70126d15ca6e 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -310,12 +310,11 @@ struct sk_buff *__skb_recv_datagram(struct sock *sk, EXPORT_SYMBOL(__skb_recv_datagram); struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, - int noblock, int *err) + int *err) { int off = 0; - return __skb_recv_datagram(sk, &sk->sk_receive_queue, - flags | (noblock ? MSG_DONTWAIT : 0), + return __skb_recv_datagram(sk, &sk->sk_receive_queue, flags, &off, err); } EXPORT_SYMBOL(skb_recv_datagram); diff --git a/net/core/dev.c b/net/core/dev.c index 8c6c08446556..e027410e861b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -151,6 +151,7 @@ #include <linux/prandom.h> #include <linux/once_lite.h> +#include "dev.h" #include "net-sysfs.h" @@ -701,6 +702,10 @@ int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr, if (WARN_ON_ONCE(last_dev == ctx.dev)) return -1; } + + if (!ctx.dev) + return ret; + path = dev_fwd_path(stack); if (!path) return -1; @@ -8641,7 +8646,6 @@ void dev_set_group(struct net_device *dev, int new_group) { dev->group = new_group; } -EXPORT_SYMBOL(dev_set_group); /** * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR. @@ -8756,7 +8760,6 @@ int dev_change_carrier(struct net_device *dev, bool new_carrier) return -ENODEV; return ops->ndo_change_carrier(dev, new_carrier); } -EXPORT_SYMBOL(dev_change_carrier); /** * dev_get_phys_port_id - Get device physical port ID @@ -8774,7 +8777,6 @@ int dev_get_phys_port_id(struct net_device *dev, return -EOPNOTSUPP; return ops->ndo_get_phys_port_id(dev, ppid); } -EXPORT_SYMBOL(dev_get_phys_port_id); /** * dev_get_phys_port_name - Get device physical port name @@ -8797,7 +8799,6 @@ int dev_get_phys_port_name(struct net_device *dev, } return devlink_compat_phys_port_name_get(dev, name, len); } -EXPORT_SYMBOL(dev_get_phys_port_name); /** * dev_get_port_parent_id - Get the device's port parent identifier @@ -8879,7 +8880,6 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down) dev->proto_down = proto_down; return 0; } -EXPORT_SYMBOL(dev_change_proto_down); /** * dev_change_proto_down_reason - proto down reason @@ -8904,7 +8904,6 @@ void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, } } } -EXPORT_SYMBOL(dev_change_proto_down_reason); struct bpf_xdp_link { struct bpf_link link; @@ -9431,7 +9430,7 @@ static int dev_new_index(struct net *net) } /* Delayed registration/unregisteration */ -static LIST_HEAD(net_todo_list); +LIST_HEAD(net_todo_list); DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); static void net_set_todo(struct net_device *dev) @@ -10359,6 +10358,7 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, storage->rx_dropped += local_read(&core_stats->rx_dropped); storage->tx_dropped += local_read(&core_stats->tx_dropped); storage->rx_nohandler += local_read(&core_stats->rx_nohandler); + storage->rx_otherhost_dropped += local_read(&core_stats->rx_otherhost_dropped); } } return storage; diff --git a/net/core/dev.h b/net/core/dev.h new file mode 100644 index 000000000000..27923df00637 --- /dev/null +++ b/net/core/dev.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _NET_CORE_DEV_H +#define _NET_CORE_DEV_H + +#include <linux/types.h> + +struct net; +struct net_device; +struct netdev_bpf; +struct netdev_phys_item_id; +struct netlink_ext_ack; + +/* Random bits of netdevice that don't need to be exposed */ +#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ +struct sd_flow_limit { + u64 count; + unsigned int num_buckets; + unsigned int history_head; + u16 history[FLOW_LIMIT_HISTORY]; + u8 buckets[]; +}; + +extern int netdev_flow_limit_table_len; + +#ifdef CONFIG_PROC_FS +int __init dev_proc_init(void); +#else +#define dev_proc_init() 0 +#endif + +void linkwatch_init_dev(struct net_device *dev); +void linkwatch_forget_dev(struct net_device *dev); +void linkwatch_run_queue(void); + +void dev_addr_flush(struct net_device *dev); +int dev_addr_init(struct net_device *dev); +void dev_addr_check(struct net_device *dev); + +/* sysctls not referred to from outside net/core/ */ +extern int netdev_budget; +extern unsigned int netdev_budget_usecs; + +extern int netdev_tstamp_prequeue; +extern int netdev_unregister_timeout_secs; +extern int weight_p; +extern int dev_weight_rx_bias; +extern int dev_weight_tx_bias; + +/* rtnl helpers */ +extern struct list_head net_todo_list; +void netdev_run_todo(void); + +/* netdev management, shared between various uAPI entry points */ +struct netdev_name_node { + struct hlist_node hlist; + struct list_head list; + struct net_device *dev; + const char *name; +}; + +int netdev_get_name(struct net *net, char *name, int ifindex); +int dev_change_name(struct net_device *dev, const char *newname); + +int netdev_name_node_alt_create(struct net_device *dev, const char *name); +int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); + +int dev_validate_mtu(struct net_device *dev, int mtu, + struct netlink_ext_ack *extack); +int dev_set_mtu_ext(struct net_device *dev, int mtu, + struct netlink_ext_ack *extack); + +int dev_get_phys_port_id(struct net_device *dev, + struct netdev_phys_item_id *ppid); +int dev_get_phys_port_name(struct net_device *dev, + char *name, size_t len); + +int dev_change_proto_down(struct net_device *dev, bool proto_down); +void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, + u32 value); + +typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); +int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, + int fd, int expected_fd, u32 flags); + +int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len); +void dev_set_group(struct net_device *dev, int new_group); +int dev_change_carrier(struct net_device *dev, bool new_carrier); + +void __dev_set_rx_mode(struct net_device *dev); + +#endif diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index bead38ca50bd..baa63dee2829 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -12,6 +12,8 @@ #include <linux/export.h> #include <linux/list.h> +#include "dev.h" + /* * General list handling functions */ diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 1b807d119da5..4f6be442ae7e 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -10,6 +10,8 @@ #include <net/dsa.h> #include <net/wext.h> +#include "dev.h" + /* * Map an interface index to its name (SIOCGIFNAME) */ diff --git a/net/core/filter.c b/net/core/filter.c index a7044e98765e..143f442a9505 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5173,7 +5173,7 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname, if (val <= 0 || tp->data_segs_out > tp->syn_data) ret = -EINVAL; else - tp->snd_cwnd = val; + tcp_snd_cwnd_set(tp, val); break; case TCP_BPF_SNDCWND_CLAMP: if (val <= 0) { @@ -7016,24 +7016,33 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len if (!th->ack || th->rst || th->syn) return -ENOENT; + if (unlikely(iph_len < sizeof(struct iphdr))) + return -EINVAL; + if (tcp_synq_no_recent_overflow(sk)) return -ENOENT; cookie = ntohl(th->ack_seq) - 1; - switch (sk->sk_family) { - case AF_INET: - if (unlikely(iph_len < sizeof(struct iphdr))) + /* Both struct iphdr and struct ipv6hdr have the version field at the + * same offset so we can cast to the shorter header (struct iphdr). + */ + switch (((struct iphdr *)iph)->version) { + case 4: + if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk)) return -EINVAL; ret = __cookie_v4_check((struct iphdr *)iph, th, cookie); break; #if IS_BUILTIN(CONFIG_IPV6) - case AF_INET6: + case 6: if (unlikely(iph_len < sizeof(struct ipv6hdr))) return -EINVAL; + if (sk->sk_family != AF_INET6) + return -EINVAL; + ret = __cookie_v6_check((struct ipv6hdr *)iph, th, cookie); break; #endif /* CONFIG_IPV6 */ diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 95098d1a49bd..a244d3bade7d 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -18,6 +18,7 @@ #include <linux/bitops.h> #include <linux/types.h> +#include "dev.h" enum lw_bits { LW_URGENT = 0, diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c index 88cc0ad7d386..1ec23bf8b05c 100644 --- a/net/core/net-procfs.c +++ b/net/core/net-procfs.c @@ -4,6 +4,8 @@ #include <linux/seq_file.h> #include <net/wext.h> +#include "dev.h" + #define BUCKET_SPACE (32 - NETDEV_HASHBITS - 1) #define get_bucket(x) ((x) >> BUCKET_SPACE) diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 9cbc1c8289bc..4980c3a50475 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -24,6 +24,7 @@ #include <linux/of_net.h> #include <linux/cpu.h> +#include "dev.h" #include "net-sysfs.h" #ifdef CONFIG_SYSFS diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 159c9c61e6af..4041b3e2e8ec 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -54,6 +54,8 @@ #include <net/rtnetlink.h> #include <net/net_namespace.h> +#include "dev.h" + #define RTNL_MAX_TYPE 50 #define RTNL_SLAVE_MAX_TYPE 40 @@ -95,6 +97,39 @@ void __rtnl_unlock(void) defer_kfree_skb_list = NULL; + /* Ensure that we didn't actually add any TODO item when __rtnl_unlock() + * is used. In some places, e.g. in cfg80211, we have code that will do + * something like + * rtnl_lock() + * wiphy_lock() + * ... + * rtnl_unlock() + * + * and because netdev_run_todo() acquires the RTNL for items on the list + * we could cause a situation such as this: + * Thread 1 Thread 2 + * rtnl_lock() + * unregister_netdevice() + * __rtnl_unlock() + * rtnl_lock() + * wiphy_lock() + * rtnl_unlock() + * netdev_run_todo() + * __rtnl_unlock() + * + * // list not empty now + * // because of thread 2 + * rtnl_lock() + * while (!list_empty(...)) + * rtnl_lock() + * wiphy_lock() + * **** DEADLOCK **** + * + * However, usage of __rtnl_unlock() is rare, and so we can ensure that + * it's not used in cases where something is added to do the list. + */ + WARN_ON(!list_empty(&net_todo_list)); + mutex_unlock(&rtnl_mutex); while (head) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 10bde7c6db44..30b523fa4ad2 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -5276,11 +5276,18 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, if (skb_cloned(to)) return false; - /* The page pool signature of struct page will eventually figure out - * which pages can be recycled or not but for now let's prohibit slab - * allocated and page_pool allocated SKBs from being coalesced. + /* In general, avoid mixing slab allocated and page_pool allocated + * pages within the same SKB. However when @to is not pp_recycle and + * @from is cloned, we can transition frag pages from page_pool to + * reference counted. + * + * On the other hand, don't allow coalescing two pp_recycle SKBs if + * @from is cloned, in case the SKB is using page_pool fragment + * references (PP_FLAG_PAGE_FRAG). Since we only take full page + * references for cloned SKBs at the moment that would result in + * inconsistent reference counts. */ - if (to->pp_recycle != from->pp_recycle) + if (to->pp_recycle != (from->pp_recycle && !skb_cloned(from))) return false; if (len <= skb_tailroom(to)) { diff --git a/net/core/sock.c b/net/core/sock.c index 1180a0cb0110..7000403eaeb2 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -141,6 +141,8 @@ #include <linux/ethtool.h> +#include "dev.h" + static DEFINE_MUTEX(proto_list_mutex); static LIST_HEAD(proto_list); diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 7123fe7feeac..8295e5877eb3 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -23,6 +23,8 @@ #include <net/busy_poll.h> #include <net/pkt_sched.h> +#include "dev.h" + static int two = 2; static int three = 3; static int int_3600 = 3600; diff --git a/net/dsa/master.c b/net/dsa/master.c index 991c2930d631..2851e44c4cf0 100644 --- a/net/dsa/master.c +++ b/net/dsa/master.c @@ -335,11 +335,24 @@ static const struct attribute_group dsa_group = { .attrs = dsa_slave_attrs, }; +static void dsa_master_reset_mtu(struct net_device *dev) +{ + int err; + + err = dev_set_mtu(dev, ETH_DATA_LEN); + if (err) + netdev_dbg(dev, + "Unable to reset MTU to exclude DSA overheads\n"); +} + int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp) { + const struct dsa_device_ops *tag_ops = cpu_dp->tag_ops; struct dsa_switch *ds = cpu_dp->ds; struct device_link *consumer_link; - int ret; + int mtu, ret; + + mtu = ETH_DATA_LEN + dsa_tag_protocol_overhead(tag_ops); /* The DSA master must use SET_NETDEV_DEV for this to work. */ consumer_link = device_link_add(ds->dev, dev->dev.parent, @@ -349,6 +362,15 @@ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp) "Failed to create a device link to DSA switch %s\n", dev_name(ds->dev)); + /* The switch driver may not implement ->port_change_mtu(), case in + * which dsa_slave_change_mtu() will not update the master MTU either, + * so we need to do that here. + */ + ret = dev_set_mtu(dev, mtu); + if (ret) + netdev_warn(dev, "error %d setting MTU to %d to include DSA overhead\n", + ret, mtu); + /* If we use a tagging format that doesn't have an ethertype * field, make sure that all packets from this point on get * sent to the tag format's receive function. @@ -384,6 +406,7 @@ void dsa_master_teardown(struct net_device *dev) sysfs_remove_group(&dev->dev.kobj, &dsa_group); dsa_netdev_ops_set(dev, NULL); dsa_master_ethtool_teardown(dev); + dsa_master_reset_mtu(dev); dsa_master_set_promiscuity(dev, -1); dev->dsa_ptr = NULL; diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index ebcc812735a4..62b89d6f54fd 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -391,7 +391,7 @@ EXPORT_SYMBOL(ether_setup); struct net_device *alloc_etherdev_mqs(int sizeof_priv, unsigned int txqs, unsigned int rxqs) { - return alloc_netdev_mqs(sizeof_priv, "eth%d", NET_NAME_UNKNOWN, + return alloc_netdev_mqs(sizeof_priv, "eth%d", NET_NAME_ENUM, ether_setup, txqs, rxqs); } EXPORT_SYMBOL(alloc_etherdev_mqs); diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index 3b2366a88c3c..a725dd9bbda8 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -314,7 +314,8 @@ static int raw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int err = -EOPNOTSUPP; struct sk_buff *skb; - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; @@ -703,7 +704,8 @@ static int dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, struct dgram_sock *ro = dgram_sk(sk); DECLARE_SOCKADDR(struct sockaddr_ieee802154 *, saddr, msg->msg_name); - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index cc8e84ef2ae4..ccb62038f6a4 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -889,8 +889,13 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi, } if (cfg->fc_oif || cfg->fc_gw_family) { - struct fib_nh *nh = fib_info_nh(fi, 0); + struct fib_nh *nh; + + /* cannot match on nexthop object attributes */ + if (fi->nh) + return 1; + nh = fib_info_nh(fi, 0); if (cfg->fc_encap) { if (fib_encap_match(net, cfg->fc_encap_type, cfg->fc_encap, nh, cfg, extack)) diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index 95f7bb052784..b1165f717cd1 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -451,6 +451,7 @@ static struct sk_buff *ip_rcv_core(struct sk_buff *skb, struct net *net) * that it receives, do not try to analyse it. */ if (skb->pkt_type == PACKET_OTHERHOST) { + dev_core_stats_rx_otherhost_dropped_inc(skb->dev); drop_reason = SKB_DROP_REASON_OTHERHOST; goto drop; } diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 3ee947557b88..550dc5c795c0 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -861,7 +861,8 @@ int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, if (flags & MSG_ERRQUEUE) return inet_recv_error(sk, msg, len, addr_len); - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index 9f97b9cbf7b3..c9dd9603f2e7 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -769,7 +769,8 @@ static int raw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, goto out; } - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index cf18fbcbf123..e31cf137c614 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -429,7 +429,7 @@ void tcp_init_sock(struct sock *sk) * algorithms that we must have the following bandaid to talk * efficiently to them. -DaveM */ - tp->snd_cwnd = TCP_INIT_CWND; + tcp_snd_cwnd_set(tp, TCP_INIT_CWND); /* There's a bubble in the pipe until at least the first ACK. */ tp->app_limited = ~0U; @@ -3033,7 +3033,7 @@ int tcp_disconnect(struct sock *sk, int flags) icsk->icsk_rto_min = TCP_RTO_MIN; icsk->icsk_delack_max = TCP_DELACK_MAX; tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; - tp->snd_cwnd = TCP_INIT_CWND; + tcp_snd_cwnd_set(tp, TCP_INIT_CWND); tp->snd_cwnd_cnt = 0; tp->window_clamp = 0; tp->delivered = 0; @@ -3744,7 +3744,7 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) info->tcpi_max_pacing_rate = rate64; info->tcpi_reordering = tp->reordering; - info->tcpi_snd_cwnd = tp->snd_cwnd; + info->tcpi_snd_cwnd = tcp_snd_cwnd(tp); if (info->tcpi_state == TCP_LISTEN) { /* listeners aliased fields : @@ -3915,7 +3915,7 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, rate64 = tcp_compute_delivery_rate(tp); nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); - nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd); + nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp)); nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 02e8626ccb27..c7d30a3bbd81 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -276,7 +276,7 @@ static void bbr_init_pacing_rate_from_rtt(struct sock *sk) } else { /* no RTT sample yet */ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ } - bw = (u64)tp->snd_cwnd * BW_UNIT; + bw = (u64)tcp_snd_cwnd(tp) * BW_UNIT; do_div(bw, rtt_us); sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr_high_gain); } @@ -323,9 +323,9 @@ static void bbr_save_cwnd(struct sock *sk) struct bbr *bbr = inet_csk_ca(sk); if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) - bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ + bbr->prior_cwnd = tcp_snd_cwnd(tp); /* this cwnd is good enough */ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ - bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); + bbr->prior_cwnd = max(bbr->prior_cwnd, tcp_snd_cwnd(tp)); } static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) @@ -482,7 +482,7 @@ static bool bbr_set_cwnd_to_recover_or_restore( struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state; - u32 cwnd = tp->snd_cwnd; + u32 cwnd = tcp_snd_cwnd(tp); /* An ACK for P pkts should release at most 2*P packets. We do this * in two steps. First, here we deduct the number of lost packets. @@ -520,7 +520,7 @@ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u32 cwnd = tp->snd_cwnd, target_cwnd = 0; + u32 cwnd = tcp_snd_cwnd(tp), target_cwnd = 0; if (!acked) goto done; /* no packet fully ACKed; just apply caps */ @@ -544,9 +544,9 @@ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, cwnd = max(cwnd, bbr_cwnd_min_target); done: - tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ + tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* apply global cap */ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ - tp->snd_cwnd = min(tp->snd_cwnd, bbr_cwnd_min_target); + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), bbr_cwnd_min_target)); } /* End cycle phase if it's time and/or we hit the phase's in-flight target. */ @@ -856,7 +856,7 @@ static void bbr_update_ack_aggregation(struct sock *sk, bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, bbr->ack_epoch_acked + rs->acked_sacked); extra_acked = bbr->ack_epoch_acked - expected_acked; - extra_acked = min(extra_acked, tp->snd_cwnd); + extra_acked = min(extra_acked, tcp_snd_cwnd(tp)); if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; } @@ -914,7 +914,7 @@ static void bbr_check_probe_rtt_done(struct sock *sk) return; bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */ - tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); bbr_reset_mode(sk); } @@ -1093,7 +1093,7 @@ static u32 bbr_undo_cwnd(struct sock *sk) bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ bbr->full_bw_cnt = 0; bbr_reset_lt_bw_sampling(sk); - return tcp_sk(sk)->snd_cwnd; + return tcp_snd_cwnd(tcp_sk(sk)); } /* Entering loss recovery, so save cwnd for when we exit or undo recovery. */ diff --git a/net/ipv4/tcp_bic.c b/net/ipv4/tcp_bic.c index f5f588b1f6e9..58358bf92e1b 100644 --- a/net/ipv4/tcp_bic.c +++ b/net/ipv4/tcp_bic.c @@ -150,7 +150,7 @@ static void bictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) if (!acked) return; } - bictcp_update(ca, tp->snd_cwnd); + bictcp_update(ca, tcp_snd_cwnd(tp)); tcp_cong_avoid_ai(tp, ca->cnt, acked); } @@ -166,16 +166,16 @@ static u32 bictcp_recalc_ssthresh(struct sock *sk) ca->epoch_start = 0; /* end of epoch */ /* Wmax and fast convergence */ - if (tp->snd_cwnd < ca->last_max_cwnd && fast_convergence) - ca->last_max_cwnd = (tp->snd_cwnd * (BICTCP_BETA_SCALE + beta)) + if (tcp_snd_cwnd(tp) < ca->last_max_cwnd && fast_convergence) + ca->last_max_cwnd = (tcp_snd_cwnd(tp) * (BICTCP_BETA_SCALE + beta)) / (2 * BICTCP_BETA_SCALE); else - ca->last_max_cwnd = tp->snd_cwnd; + ca->last_max_cwnd = tcp_snd_cwnd(tp); - if (tp->snd_cwnd <= low_window) - return max(tp->snd_cwnd >> 1U, 2U); + if (tcp_snd_cwnd(tp) <= low_window) + return max(tcp_snd_cwnd(tp) >> 1U, 2U); else - return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U); + return max((tcp_snd_cwnd(tp) * beta) / BICTCP_BETA_SCALE, 2U); } static void bictcp_state(struct sock *sk, u8 new_state) diff --git a/net/ipv4/tcp_cdg.c b/net/ipv4/tcp_cdg.c index 709d23801823..ddc7ba0554bd 100644 --- a/net/ipv4/tcp_cdg.c +++ b/net/ipv4/tcp_cdg.c @@ -161,8 +161,8 @@ static void tcp_cdg_hystart_update(struct sock *sk) LINUX_MIB_TCPHYSTARTTRAINDETECT); NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTTRAINCWND, - tp->snd_cwnd); - tp->snd_ssthresh = tp->snd_cwnd; + tcp_snd_cwnd(tp)); + tp->snd_ssthresh = tcp_snd_cwnd(tp); return; } } @@ -180,8 +180,8 @@ static void tcp_cdg_hystart_update(struct sock *sk) LINUX_MIB_TCPHYSTARTDELAYDETECT); NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTDELAYCWND, - tp->snd_cwnd); - tp->snd_ssthresh = tp->snd_cwnd; + tcp_snd_cwnd(tp)); + tp->snd_ssthresh = tcp_snd_cwnd(tp); } } } @@ -252,7 +252,7 @@ static bool tcp_cdg_backoff(struct sock *sk, u32 grad) return false; } - ca->shadow_wnd = max(ca->shadow_wnd, tp->snd_cwnd); + ca->shadow_wnd = max(ca->shadow_wnd, tcp_snd_cwnd(tp)); ca->state = CDG_BACKOFF; tcp_enter_cwr(sk); return true; @@ -285,14 +285,14 @@ static void tcp_cdg_cong_avoid(struct sock *sk, u32 ack, u32 acked) } if (!tcp_is_cwnd_limited(sk)) { - ca->shadow_wnd = min(ca->shadow_wnd, tp->snd_cwnd); + ca->shadow_wnd = min(ca->shadow_wnd, tcp_snd_cwnd(tp)); return; } - prior_snd_cwnd = tp->snd_cwnd; + prior_snd_cwnd = tcp_snd_cwnd(tp); tcp_reno_cong_avoid(sk, ack, acked); - incr = tp->snd_cwnd - prior_snd_cwnd; + incr = tcp_snd_cwnd(tp) - prior_snd_cwnd; ca->shadow_wnd = max(ca->shadow_wnd, ca->shadow_wnd + incr); } @@ -331,15 +331,15 @@ static u32 tcp_cdg_ssthresh(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); if (ca->state == CDG_BACKOFF) - return max(2U, (tp->snd_cwnd * min(1024U, backoff_beta)) >> 10); + return max(2U, (tcp_snd_cwnd(tp) * min(1024U, backoff_beta)) >> 10); if (ca->state == CDG_NONFULL && use_tolerance) - return tp->snd_cwnd; + return tcp_snd_cwnd(tp); - ca->shadow_wnd = min(ca->shadow_wnd >> 1, tp->snd_cwnd); + ca->shadow_wnd = min(ca->shadow_wnd >> 1, tcp_snd_cwnd(tp)); if (use_shadow) - return max3(2U, ca->shadow_wnd, tp->snd_cwnd >> 1); - return max(2U, tp->snd_cwnd >> 1); + return max3(2U, ca->shadow_wnd, tcp_snd_cwnd(tp) >> 1); + return max(2U, tcp_snd_cwnd(tp) >> 1); } static void tcp_cdg_cwnd_event(struct sock *sk, const enum tcp_ca_event ev) @@ -357,7 +357,7 @@ static void tcp_cdg_cwnd_event(struct sock *sk, const enum tcp_ca_event ev) ca->gradients = gradients; ca->rtt_seq = tp->snd_nxt; - ca->shadow_wnd = tp->snd_cwnd; + ca->shadow_wnd = tcp_snd_cwnd(tp); break; case CA_EVENT_COMPLETE_CWR: ca->state = CDG_UNKNOWN; @@ -380,7 +380,7 @@ static void tcp_cdg_init(struct sock *sk) ca->gradients = kcalloc(window, sizeof(ca->gradients[0]), GFP_NOWAIT | __GFP_NOWARN); ca->rtt_seq = tp->snd_nxt; - ca->shadow_wnd = tp->snd_cwnd; + ca->shadow_wnd = tcp_snd_cwnd(tp); } static void tcp_cdg_release(struct sock *sk) diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index dc95572163df..d3cae40749e8 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -16,6 +16,7 @@ #include <linux/gfp.h> #include <linux/jhash.h> #include <net/tcp.h> +#include <trace/events/tcp.h> static DEFINE_SPINLOCK(tcp_cong_list_lock); static LIST_HEAD(tcp_cong_list); @@ -33,6 +34,17 @@ struct tcp_congestion_ops *tcp_ca_find(const char *name) return NULL; } +void tcp_set_ca_state(struct sock *sk, const u8 ca_state) +{ + struct inet_connection_sock *icsk = inet_csk(sk); + + trace_tcp_cong_state_set(sk, ca_state); + + if (icsk->icsk_ca_ops->set_state) + icsk->icsk_ca_ops->set_state(sk, ca_state); + icsk->icsk_ca_state = ca_state; +} + /* Must be called with rcu lock held */ static struct tcp_congestion_ops *tcp_ca_find_autoload(struct net *net, const char *name) @@ -393,10 +405,10 @@ int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, */ u32 tcp_slow_start(struct tcp_sock *tp, u32 acked) { - u32 cwnd = min(tp->snd_cwnd + acked, tp->snd_ssthresh); + u32 cwnd = min(tcp_snd_cwnd(tp) + acked, tp->snd_ssthresh); - acked -= cwnd - tp->snd_cwnd; - tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); + acked -= cwnd - tcp_snd_cwnd(tp); + tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); return acked; } @@ -410,7 +422,7 @@ void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked) /* If credits accumulated at a higher w, apply them gently now. */ if (tp->snd_cwnd_cnt >= w) { tp->snd_cwnd_cnt = 0; - tp->snd_cwnd++; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); } tp->snd_cwnd_cnt += acked; @@ -418,9 +430,9 @@ void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked) u32 delta = tp->snd_cwnd_cnt / w; tp->snd_cwnd_cnt -= delta * w; - tp->snd_cwnd += delta; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + delta); } - tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_cwnd_clamp); + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_cwnd_clamp)); } EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai); @@ -445,7 +457,7 @@ void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked) return; } /* In dangerous area, increase slowly. */ - tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked); + tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked); } EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid); @@ -454,7 +466,7 @@ u32 tcp_reno_ssthresh(struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); - return max(tp->snd_cwnd >> 1U, 2U); + return max(tcp_snd_cwnd(tp) >> 1U, 2U); } EXPORT_SYMBOL_GPL(tcp_reno_ssthresh); @@ -462,7 +474,7 @@ u32 tcp_reno_undo_cwnd(struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); - return max(tp->snd_cwnd, tp->prior_cwnd); + return max(tcp_snd_cwnd(tp), tp->prior_cwnd); } EXPORT_SYMBOL_GPL(tcp_reno_undo_cwnd); diff --git a/net/ipv4/tcp_cubic.c b/net/ipv4/tcp_cubic.c index 24d562dd6225..b0918839bee7 100644 --- a/net/ipv4/tcp_cubic.c +++ b/net/ipv4/tcp_cubic.c @@ -334,7 +334,7 @@ static void cubictcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) if (!acked) return; } - bictcp_update(ca, tp->snd_cwnd, acked); + bictcp_update(ca, tcp_snd_cwnd(tp), acked); tcp_cong_avoid_ai(tp, ca->cnt, acked); } @@ -346,13 +346,13 @@ static u32 cubictcp_recalc_ssthresh(struct sock *sk) ca->epoch_start = 0; /* end of epoch */ /* Wmax and fast convergence */ - if (tp->snd_cwnd < ca->last_max_cwnd && fast_convergence) - ca->last_max_cwnd = (tp->snd_cwnd * (BICTCP_BETA_SCALE + beta)) + if (tcp_snd_cwnd(tp) < ca->last_max_cwnd && fast_convergence) + ca->last_max_cwnd = (tcp_snd_cwnd(tp) * (BICTCP_BETA_SCALE + beta)) / (2 * BICTCP_BETA_SCALE); else - ca->last_max_cwnd = tp->snd_cwnd; + ca->last_max_cwnd = tcp_snd_cwnd(tp); - return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U); + return max((tcp_snd_cwnd(tp) * beta) / BICTCP_BETA_SCALE, 2U); } static void cubictcp_state(struct sock *sk, u8 new_state) @@ -413,13 +413,13 @@ static void hystart_update(struct sock *sk, u32 delay) ca->found = 1; pr_debug("hystart_ack_train (%u > %u) delay_min %u (+ ack_delay %u) cwnd %u\n", now - ca->round_start, threshold, - ca->delay_min, hystart_ack_delay(sk), tp->snd_cwnd); + ca->delay_min, hystart_ack_delay(sk), tcp_snd_cwnd(tp)); NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTTRAINDETECT); NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTTRAINCWND, - tp->snd_cwnd); - tp->snd_ssthresh = tp->snd_cwnd; + tcp_snd_cwnd(tp)); + tp->snd_ssthresh = tcp_snd_cwnd(tp); } } } @@ -438,8 +438,8 @@ static void hystart_update(struct sock *sk, u32 delay) LINUX_MIB_TCPHYSTARTDELAYDETECT); NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTDELAYCWND, - tp->snd_cwnd); - tp->snd_ssthresh = tp->snd_cwnd; + tcp_snd_cwnd(tp)); + tp->snd_ssthresh = tcp_snd_cwnd(tp); } } } @@ -469,7 +469,7 @@ static void cubictcp_acked(struct sock *sk, const struct ack_sample *sample) /* hystart triggers when cwnd is larger than some threshold */ if (!ca->found && tcp_in_slow_start(tp) && hystart && - tp->snd_cwnd >= hystart_low_window) + tcp_snd_cwnd(tp) >= hystart_low_window) hystart_update(sk, delay); } diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index 1943a6630341..ab034a4e9324 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -106,8 +106,8 @@ static u32 dctcp_ssthresh(struct sock *sk) struct dctcp *ca = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); - ca->loss_cwnd = tp->snd_cwnd; - return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U); + ca->loss_cwnd = tcp_snd_cwnd(tp); + return max(tcp_snd_cwnd(tp) - ((tcp_snd_cwnd(tp) * ca->dctcp_alpha) >> 11U), 2U); } static void dctcp_update_alpha(struct sock *sk, u32 flags) @@ -148,8 +148,8 @@ static void dctcp_react_to_loss(struct sock *sk) struct dctcp *ca = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); - ca->loss_cwnd = tp->snd_cwnd; - tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U); + ca->loss_cwnd = tcp_snd_cwnd(tp); + tp->snd_ssthresh = max(tcp_snd_cwnd(tp) >> 1U, 2U); } static void dctcp_state(struct sock *sk, u8 new_state) @@ -211,8 +211,9 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr, static u32 dctcp_cwnd_undo(struct sock *sk) { const struct dctcp *ca = inet_csk_ca(sk); + struct tcp_sock *tp = tcp_sk(sk); - return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd); + return max(tcp_snd_cwnd(tp), ca->loss_cwnd); } static struct tcp_congestion_ops dctcp __read_mostly = { diff --git a/net/ipv4/tcp_highspeed.c b/net/ipv4/tcp_highspeed.c index 349069d6cd0a..c6de5ce79ad3 100644 --- a/net/ipv4/tcp_highspeed.c +++ b/net/ipv4/tcp_highspeed.c @@ -127,22 +127,22 @@ static void hstcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) * snd_cwnd <= * hstcp_aimd_vals[ca->ai].cwnd */ - if (tp->snd_cwnd > hstcp_aimd_vals[ca->ai].cwnd) { - while (tp->snd_cwnd > hstcp_aimd_vals[ca->ai].cwnd && + if (tcp_snd_cwnd(tp) > hstcp_aimd_vals[ca->ai].cwnd) { + while (tcp_snd_cwnd(tp) > hstcp_aimd_vals[ca->ai].cwnd && ca->ai < HSTCP_AIMD_MAX - 1) ca->ai++; - } else if (ca->ai && tp->snd_cwnd <= hstcp_aimd_vals[ca->ai-1].cwnd) { - while (ca->ai && tp->snd_cwnd <= hstcp_aimd_vals[ca->ai-1].cwnd) + } else if (ca->ai && tcp_snd_cwnd(tp) <= hstcp_aimd_vals[ca->ai-1].cwnd) { + while (ca->ai && tcp_snd_cwnd(tp) <= hstcp_aimd_vals[ca->ai-1].cwnd) ca->ai--; } /* Do additive increase */ - if (tp->snd_cwnd < tp->snd_cwnd_clamp) { + if (tcp_snd_cwnd(tp) < tp->snd_cwnd_clamp) { /* cwnd = cwnd + a(w) / cwnd */ tp->snd_cwnd_cnt += ca->ai + 1; - if (tp->snd_cwnd_cnt >= tp->snd_cwnd) { - tp->snd_cwnd_cnt -= tp->snd_cwnd; - tp->snd_cwnd++; + if (tp->snd_cwnd_cnt >= tcp_snd_cwnd(tp)) { + tp->snd_cwnd_cnt -= tcp_snd_cwnd(tp); + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); } } } @@ -154,7 +154,7 @@ static u32 hstcp_ssthresh(struct sock *sk) struct hstcp *ca = inet_csk_ca(sk); /* Do multiplicative decrease */ - return max(tp->snd_cwnd - ((tp->snd_cwnd * hstcp_aimd_vals[ca->ai].md) >> 8), 2U); + return max(tcp_snd_cwnd(tp) - ((tcp_snd_cwnd(tp) * hstcp_aimd_vals[ca->ai].md) >> 8), 2U); } static struct tcp_congestion_ops tcp_highspeed __read_mostly = { diff --git a/net/ipv4/tcp_htcp.c b/net/ipv4/tcp_htcp.c index 55adcfcf96fe..52b1f2665dfa 100644 --- a/net/ipv4/tcp_htcp.c +++ b/net/ipv4/tcp_htcp.c @@ -124,7 +124,7 @@ static void measure_achieved_throughput(struct sock *sk, ca->packetcount += sample->pkts_acked; - if (ca->packetcount >= tp->snd_cwnd - (ca->alpha >> 7 ? : 1) && + if (ca->packetcount >= tcp_snd_cwnd(tp) - (ca->alpha >> 7 ? : 1) && now - ca->lasttime >= ca->minRTT && ca->minRTT > 0) { __u32 cur_Bi = ca->packetcount * HZ / (now - ca->lasttime); @@ -225,7 +225,7 @@ static u32 htcp_recalc_ssthresh(struct sock *sk) const struct htcp *ca = inet_csk_ca(sk); htcp_param_update(sk); - return max((tp->snd_cwnd * ca->beta) >> 7, 2U); + return max((tcp_snd_cwnd(tp) * ca->beta) >> 7, 2U); } static void htcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) @@ -242,9 +242,9 @@ static void htcp_cong_avoid(struct sock *sk, u32 ack, u32 acked) /* In dangerous area, increase slowly. * In theory this is tp->snd_cwnd += alpha / tp->snd_cwnd */ - if ((tp->snd_cwnd_cnt * ca->alpha)>>7 >= tp->snd_cwnd) { - if (tp->snd_cwnd < tp->snd_cwnd_clamp) - tp->snd_cwnd++; + if ((tp->snd_cwnd_cnt * ca->alpha)>>7 >= tcp_snd_cwnd(tp)) { + if (tcp_snd_cwnd(tp) < tp->snd_cwnd_clamp) + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); tp->snd_cwnd_cnt = 0; htcp_alpha_update(ca); } else diff --git a/net/ipv4/tcp_hybla.c b/net/ipv4/tcp_hybla.c index be39327e04e6..abd7d91807e5 100644 --- a/net/ipv4/tcp_hybla.c +++ b/net/ipv4/tcp_hybla.c @@ -54,7 +54,7 @@ static void hybla_init(struct sock *sk) ca->rho2_7ls = 0; ca->snd_cwnd_cents = 0; ca->hybla_en = true; - tp->snd_cwnd = 2; + tcp_snd_cwnd_set(tp, 2); tp->snd_cwnd_clamp = 65535; /* 1st Rho measurement based on initial srtt */ @@ -62,7 +62,7 @@ static void hybla_init(struct sock *sk) /* set minimum rtt as this is the 1st ever seen */ ca->minrtt_us = tp->srtt_us; - tp->snd_cwnd = ca->rho; + tcp_snd_cwnd_set(tp, ca->rho); } static void hybla_state(struct sock *sk, u8 ca_state) @@ -137,31 +137,31 @@ static void hybla_cong_avoid(struct sock *sk, u32 ack, u32 acked) * as long as increment is estimated as (rho<<7)/window * it already is <<7 and we can easily count its fractions. */ - increment = ca->rho2_7ls / tp->snd_cwnd; + increment = ca->rho2_7ls / tcp_snd_cwnd(tp); if (increment < 128) tp->snd_cwnd_cnt++; } odd = increment % 128; - tp->snd_cwnd += increment >> 7; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + (increment >> 7)); ca->snd_cwnd_cents += odd; /* check when fractions goes >=128 and increase cwnd by 1. */ while (ca->snd_cwnd_cents >= 128) { - tp->snd_cwnd++; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); ca->snd_cwnd_cents -= 128; tp->snd_cwnd_cnt = 0; } /* check when cwnd has not been incremented for a while */ - if (increment == 0 && odd == 0 && tp->snd_cwnd_cnt >= tp->snd_cwnd) { - tp->snd_cwnd++; + if (increment == 0 && odd == 0 && tp->snd_cwnd_cnt >= tcp_snd_cwnd(tp)) { + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); tp->snd_cwnd_cnt = 0; } /* clamp down slowstart cwnd to ssthresh value. */ if (is_slowstart) - tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_ssthresh); + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_ssthresh)); - tp->snd_cwnd = min_t(u32, tp->snd_cwnd, tp->snd_cwnd_clamp); + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_cwnd_clamp)); } static struct tcp_congestion_ops tcp_hybla __read_mostly = { diff --git a/net/ipv4/tcp_illinois.c b/net/ipv4/tcp_illinois.c index 00e54873213e..c0c81a2c77fa 100644 --- a/net/ipv4/tcp_illinois.c +++ b/net/ipv4/tcp_illinois.c @@ -224,7 +224,7 @@ static void update_params(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); struct illinois *ca = inet_csk_ca(sk); - if (tp->snd_cwnd < win_thresh) { + if (tcp_snd_cwnd(tp) < win_thresh) { ca->alpha = ALPHA_BASE; ca->beta = BETA_BASE; } else if (ca->cnt_rtt > 0) { @@ -284,9 +284,9 @@ static void tcp_illinois_cong_avoid(struct sock *sk, u32 ack, u32 acked) * tp->snd_cwnd += alpha/tp->snd_cwnd */ delta = (tp->snd_cwnd_cnt * ca->alpha) >> ALPHA_SHIFT; - if (delta >= tp->snd_cwnd) { - tp->snd_cwnd = min(tp->snd_cwnd + delta / tp->snd_cwnd, - (u32)tp->snd_cwnd_clamp); + if (delta >= tcp_snd_cwnd(tp)) { + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp) + delta / tcp_snd_cwnd(tp), + (u32)tp->snd_cwnd_clamp)); tp->snd_cwnd_cnt = 0; } } @@ -296,9 +296,11 @@ static u32 tcp_illinois_ssthresh(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); struct illinois *ca = inet_csk_ca(sk); + u32 decr; /* Multiplicative decrease */ - return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->beta) >> BETA_SHIFT), 2U); + decr = (tcp_snd_cwnd(tp) * ca->beta) >> BETA_SHIFT; + return max(tcp_snd_cwnd(tp) - decr, 2U); } /* Extract info for Tcp socket info provided via netlink. */ diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 2088f93fa37b..1595b76ea2be 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -414,7 +414,7 @@ static void tcp_sndbuf_expand(struct sock *sk) per_mss = roundup_pow_of_two(per_mss) + SKB_DATA_ALIGN(sizeof(struct sk_buff)); - nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd); + nr_segs = max_t(u32, TCP_INIT_CWND, tcp_snd_cwnd(tp)); nr_segs = max_t(u32, nr_segs, tp->reordering + 1); /* Fast Recovery (RFC 5681 3.2) : @@ -909,12 +909,12 @@ static void tcp_update_pacing_rate(struct sock *sk) * If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching * end of slow start and should slow down. */ - if (tp->snd_cwnd < tp->snd_ssthresh / 2) + if (tcp_snd_cwnd(tp) < tp->snd_ssthresh / 2) rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio; else rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio; - rate *= max(tp->snd_cwnd, tp->packets_out); + rate *= max(tcp_snd_cwnd(tp), tp->packets_out); if (likely(tp->srtt_us)) do_div(rate, tp->srtt_us); @@ -2147,12 +2147,12 @@ void tcp_enter_loss(struct sock *sk) !after(tp->high_seq, tp->snd_una) || (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) { tp->prior_ssthresh = tcp_current_ssthresh(sk); - tp->prior_cwnd = tp->snd_cwnd; + tp->prior_cwnd = tcp_snd_cwnd(tp); tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk); tcp_ca_event(sk, CA_EVENT_LOSS); tcp_init_undo(tp); } - tp->snd_cwnd = tcp_packets_in_flight(tp) + 1; + tcp_snd_cwnd_set(tp, tcp_packets_in_flight(tp) + 1); tp->snd_cwnd_cnt = 0; tp->snd_cwnd_stamp = tcp_jiffies32; @@ -2458,7 +2458,7 @@ static void DBGUNDO(struct sock *sk, const char *msg) pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n", msg, &inet->inet_daddr, ntohs(inet->inet_dport), - tp->snd_cwnd, tcp_left_out(tp), + tcp_snd_cwnd(tp), tcp_left_out(tp), tp->snd_ssthresh, tp->prior_ssthresh, tp->packets_out); } @@ -2467,7 +2467,7 @@ static void DBGUNDO(struct sock *sk, const char *msg) pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n", msg, &sk->sk_v6_daddr, ntohs(inet->inet_dport), - tp->snd_cwnd, tcp_left_out(tp), + tcp_snd_cwnd(tp), tcp_left_out(tp), tp->snd_ssthresh, tp->prior_ssthresh, tp->packets_out); } @@ -2492,7 +2492,7 @@ static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss) if (tp->prior_ssthresh) { const struct inet_connection_sock *icsk = inet_csk(sk); - tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk); + tcp_snd_cwnd_set(tp, icsk->icsk_ca_ops->undo_cwnd(sk)); if (tp->prior_ssthresh > tp->snd_ssthresh) { tp->snd_ssthresh = tp->prior_ssthresh; @@ -2599,7 +2599,7 @@ static void tcp_init_cwnd_reduction(struct sock *sk) tp->high_seq = tp->snd_nxt; tp->tlp_high_seq = 0; tp->snd_cwnd_cnt = 0; - tp->prior_cwnd = tp->snd_cwnd; + tp->prior_cwnd = tcp_snd_cwnd(tp); tp->prr_delivered = 0; tp->prr_out = 0; tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk); @@ -2629,7 +2629,7 @@ void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, } /* Force a fast retransmit upon entering fast recovery */ sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1)); - tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt; + tcp_snd_cwnd_set(tp, tcp_packets_in_flight(tp) + sndcnt); } static inline void tcp_end_cwnd_reduction(struct sock *sk) @@ -2642,7 +2642,7 @@ static inline void tcp_end_cwnd_reduction(struct sock *sk) /* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */ if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH && (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) { - tp->snd_cwnd = tp->snd_ssthresh; + tcp_snd_cwnd_set(tp, tp->snd_ssthresh); tp->snd_cwnd_stamp = tcp_jiffies32; } tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR); @@ -2709,9 +2709,9 @@ static void tcp_mtup_probe_success(struct sock *sk) /* FIXME: breaks with very large cwnd */ tp->prior_ssthresh = tcp_current_ssthresh(sk); - tp->snd_cwnd = tp->snd_cwnd * - tcp_mss_to_mtu(sk, tp->mss_cache) / - icsk->icsk_mtup.probe_size; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) * + tcp_mss_to_mtu(sk, tp->mss_cache) / + icsk->icsk_mtup.probe_size); tp->snd_cwnd_cnt = 0; tp->snd_cwnd_stamp = tcp_jiffies32; tp->snd_ssthresh = tcp_current_ssthresh(sk); @@ -3034,7 +3034,7 @@ static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una, tp->snd_una == tp->mtu_probe.probe_seq_start) { tcp_mtup_probe_failed(sk); /* Restores the reduction we did in tcp_mtup_probe() */ - tp->snd_cwnd++; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); tcp_simple_retransmit(sk); return; } @@ -5436,7 +5436,7 @@ static bool tcp_should_expand_sndbuf(struct sock *sk) return false; /* If we filled the congestion window, do not expand. */ - if (tcp_packets_in_flight(tp) >= tp->snd_cwnd) + if (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp)) return false; return true; @@ -5998,9 +5998,9 @@ void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb) * retransmission has occurred. */ if (tp->total_retrans > 1 && tp->undo_marker) - tp->snd_cwnd = 1; + tcp_snd_cwnd_set(tp, 1); else - tp->snd_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk)); + tcp_snd_cwnd_set(tp, tcp_init_cwnd(tp, __sk_dst_get(sk))); tp->snd_cwnd_stamp = tcp_jiffies32; bpf_skops_established(sk, bpf_op, skb); diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index f9cec624068d..157265aecbed 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2621,7 +2621,7 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i) jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk), - tp->snd_cwnd, + tcp_snd_cwnd(tp), state == TCP_LISTEN ? fastopenq->max_qlen : (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)); diff --git a/net/ipv4/tcp_lp.c b/net/ipv4/tcp_lp.c index 82b36ec3f2f8..ae36780977d2 100644 --- a/net/ipv4/tcp_lp.c +++ b/net/ipv4/tcp_lp.c @@ -297,7 +297,7 @@ static void tcp_lp_pkts_acked(struct sock *sk, const struct ack_sample *sample) lp->flag &= ~LP_WITHIN_THR; pr_debug("TCP-LP: %05o|%5u|%5u|%15u|%15u|%15u\n", lp->flag, - tp->snd_cwnd, lp->remote_hz, lp->owd_min, lp->owd_max, + tcp_snd_cwnd(tp), lp->remote_hz, lp->owd_min, lp->owd_max, lp->sowd >> 3); if (lp->flag & LP_WITHIN_THR) @@ -313,12 +313,12 @@ static void tcp_lp_pkts_acked(struct sock *sk, const struct ack_sample *sample) /* happened within inference * drop snd_cwnd into 1 */ if (lp->flag & LP_WITHIN_INF) - tp->snd_cwnd = 1U; + tcp_snd_cwnd_set(tp, 1U); /* happened after inference * cut snd_cwnd into half */ else - tp->snd_cwnd = max(tp->snd_cwnd >> 1U, 1U); + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp) >> 1U, 1U)); /* record this drop time */ lp->last_drop = now; diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c index 0588b004ddac..7029b0e98edb 100644 --- a/net/ipv4/tcp_metrics.c +++ b/net/ipv4/tcp_metrics.c @@ -388,15 +388,15 @@ void tcp_update_metrics(struct sock *sk) if (!net->ipv4.sysctl_tcp_no_ssthresh_metrics_save && !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); - if (val && (tp->snd_cwnd >> 1) > val) + if (val && (tcp_snd_cwnd(tp) >> 1) > val) tcp_metric_set(tm, TCP_METRIC_SSTHRESH, - tp->snd_cwnd >> 1); + tcp_snd_cwnd(tp) >> 1); } if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { val = tcp_metric_get(tm, TCP_METRIC_CWND); - if (tp->snd_cwnd > val) + if (tcp_snd_cwnd(tp) > val) tcp_metric_set(tm, TCP_METRIC_CWND, - tp->snd_cwnd); + tcp_snd_cwnd(tp)); } } else if (!tcp_in_slow_start(tp) && icsk->icsk_ca_state == TCP_CA_Open) { @@ -404,10 +404,10 @@ void tcp_update_metrics(struct sock *sk) if (!net->ipv4.sysctl_tcp_no_ssthresh_metrics_save && !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) tcp_metric_set(tm, TCP_METRIC_SSTHRESH, - max(tp->snd_cwnd >> 1, tp->snd_ssthresh)); + max(tcp_snd_cwnd(tp) >> 1, tp->snd_ssthresh)); if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { val = tcp_metric_get(tm, TCP_METRIC_CWND); - tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1); + tcp_metric_set(tm, TCP_METRIC_CWND, (val + tcp_snd_cwnd(tp)) >> 1); } } else { /* Else slow start did not finish, cwnd is non-sense, diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c index ab552356bdba..a60662f4bdf9 100644 --- a/net/ipv4/tcp_nv.c +++ b/net/ipv4/tcp_nv.c @@ -197,10 +197,10 @@ static void tcpnv_cong_avoid(struct sock *sk, u32 ack, u32 acked) } if (ca->cwnd_growth_factor < 0) { - cnt = tp->snd_cwnd << -ca->cwnd_growth_factor; + cnt = tcp_snd_cwnd(tp) << -ca->cwnd_growth_factor; tcp_cong_avoid_ai(tp, cnt, acked); } else { - cnt = max(4U, tp->snd_cwnd >> ca->cwnd_growth_factor); + cnt = max(4U, tcp_snd_cwnd(tp) >> ca->cwnd_growth_factor); tcp_cong_avoid_ai(tp, cnt, acked); } } @@ -209,7 +209,7 @@ static u32 tcpnv_recalc_ssthresh(struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); - return max((tp->snd_cwnd * nv_loss_dec_factor) >> 10, 2U); + return max((tcp_snd_cwnd(tp) * nv_loss_dec_factor) >> 10, 2U); } static void tcpnv_state(struct sock *sk, u8 new_state) @@ -257,7 +257,7 @@ static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample) return; /* Stop cwnd growth if we were in catch up mode */ - if (ca->nv_catchup && tp->snd_cwnd >= nv_min_cwnd) { + if (ca->nv_catchup && tcp_snd_cwnd(tp) >= nv_min_cwnd) { ca->nv_catchup = 0; ca->nv_allow_cwnd_growth = 0; } @@ -371,7 +371,7 @@ static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample) * if cwnd < max_win, grow cwnd * else leave the same */ - if (tp->snd_cwnd > max_win) { + if (tcp_snd_cwnd(tp) > max_win) { /* there is congestion, check that it is ok * to make a CA decision * 1. We should have at least nv_dec_eval_min_calls @@ -398,20 +398,20 @@ static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample) ca->nv_allow_cwnd_growth = 0; tp->snd_ssthresh = (nv_ssthresh_factor * max_win) >> 3; - if (tp->snd_cwnd - max_win > 2) { + if (tcp_snd_cwnd(tp) - max_win > 2) { /* gap > 2, we do exponential cwnd decrease */ int dec; - dec = max(2U, ((tp->snd_cwnd - max_win) * + dec = max(2U, ((tcp_snd_cwnd(tp) - max_win) * nv_cong_dec_mult) >> 7); - tp->snd_cwnd -= dec; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - dec); } else if (nv_cong_dec_mult > 0) { - tp->snd_cwnd = max_win; + tcp_snd_cwnd_set(tp, max_win); } if (ca->cwnd_growth_factor > 0) ca->cwnd_growth_factor = 0; ca->nv_no_cong_cnt = 0; - } else if (tp->snd_cwnd <= max_win - nv_pad_buffer) { + } else if (tcp_snd_cwnd(tp) <= max_win - nv_pad_buffer) { /* There is no congestion, grow cwnd if allowed*/ if (ca->nv_eval_call_cnt < nv_inc_eval_min_calls) return; @@ -444,8 +444,8 @@ static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample) * (it wasn't before, if it is now is because nv * decreased it). */ - if (tp->snd_cwnd < nv_min_cwnd) - tp->snd_cwnd = nv_min_cwnd; + if (tcp_snd_cwnd(tp) < nv_min_cwnd) + tcp_snd_cwnd_set(tp, nv_min_cwnd); } } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 9ede847f4199..c221f3bce975 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -142,7 +142,7 @@ void tcp_cwnd_restart(struct sock *sk, s32 delta) { struct tcp_sock *tp = tcp_sk(sk); u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk)); - u32 cwnd = tp->snd_cwnd; + u32 cwnd = tcp_snd_cwnd(tp); tcp_ca_event(sk, CA_EVENT_CWND_RESTART); @@ -151,7 +151,7 @@ void tcp_cwnd_restart(struct sock *sk, s32 delta) while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) cwnd >>= 1; - tp->snd_cwnd = max(cwnd, restart_cwnd); + tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd)); tp->snd_cwnd_stamp = tcp_jiffies32; tp->snd_cwnd_used = 0; } @@ -1013,7 +1013,7 @@ static void tcp_tsq_write(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); if (tp->lost_out > tp->retrans_out && - tp->snd_cwnd > tcp_packets_in_flight(tp)) { + tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) { tcp_mstamp_refresh(tp); tcp_xmit_retransmit_queue(sk); } @@ -1860,9 +1860,9 @@ static void tcp_cwnd_application_limited(struct sock *sk) /* Limited by application or receiver window. */ u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); u32 win_used = max(tp->snd_cwnd_used, init_win); - if (win_used < tp->snd_cwnd) { + if (win_used < tcp_snd_cwnd(tp)) { tp->snd_ssthresh = tcp_current_ssthresh(sk); - tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1; + tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1); } tp->snd_cwnd_used = 0; } @@ -2043,7 +2043,7 @@ static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp, return 1; in_flight = tcp_packets_in_flight(tp); - cwnd = tp->snd_cwnd; + cwnd = tcp_snd_cwnd(tp); if (in_flight >= cwnd) return 0; @@ -2196,12 +2196,12 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, in_flight = tcp_packets_in_flight(tp); BUG_ON(tcp_skb_pcount(skb) <= 1); - BUG_ON(tp->snd_cwnd <= in_flight); + BUG_ON(tcp_snd_cwnd(tp) <= in_flight); send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq; /* From in_flight test above, we know that cwnd > in_flight. */ - cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache; + cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache; limit = min(send_win, cong_win); @@ -2215,7 +2215,7 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb, win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor); if (win_divisor) { - u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache); + u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache); /* If at least some fraction of a window is available, * just use it. @@ -2345,7 +2345,7 @@ static int tcp_mtu_probe(struct sock *sk) if (likely(!icsk->icsk_mtup.enabled || icsk->icsk_mtup.probe_size || inet_csk(sk)->icsk_ca_state != TCP_CA_Open || - tp->snd_cwnd < 11 || + tcp_snd_cwnd(tp) < 11 || tp->rx_opt.num_sacks || tp->rx_opt.dsack)) return -1; @@ -2381,7 +2381,7 @@ static int tcp_mtu_probe(struct sock *sk) return 0; /* Do we need to wait to drain cwnd? With none in flight, don't stall */ - if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) { + if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) { if (!tcp_packets_in_flight(tp)) return -1; else @@ -2450,7 +2450,7 @@ static int tcp_mtu_probe(struct sock *sk) if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) { /* Decrement cwnd here because we are sending * effectively two packets. */ - tp->snd_cwnd--; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1); tcp_event_new_data_sent(sk, nskb); icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len); @@ -2708,7 +2708,7 @@ repair: else tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED); - is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd); + is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp)); if (likely(sent_pkts || is_cwnd_limited)) tcp_cwnd_validate(sk, is_cwnd_limited); @@ -2818,7 +2818,7 @@ void tcp_send_loss_probe(struct sock *sk) if (unlikely(!skb)) { WARN_ONCE(tp->packets_out, "invalid inflight: %u state %u cwnd %u mss %d\n", - tp->packets_out, sk->sk_state, tp->snd_cwnd, mss); + tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss); inet_csk(sk)->icsk_pending = 0; return; } @@ -3302,7 +3302,7 @@ void tcp_xmit_retransmit_queue(struct sock *sk) if (!hole) tp->retransmit_skb_hint = skb; - segs = tp->snd_cwnd - tcp_packets_in_flight(tp); + segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp); if (segs <= 0) break; sacked = TCP_SKB_CB(skb)->sacked; diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c index fbab921670cc..617b8187c03d 100644 --- a/net/ipv4/tcp_rate.c +++ b/net/ipv4/tcp_rate.c @@ -195,7 +195,7 @@ void tcp_rate_check_app_limited(struct sock *sk) /* Nothing in sending host's qdisc queues or NIC tx queue. */ sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) && /* We are not limited by CWND. */ - tcp_packets_in_flight(tp) < tp->snd_cwnd && + tcp_packets_in_flight(tp) < tcp_snd_cwnd(tp) && /* All lost packets have been retransmitted. */ tp->lost_out <= tp->retrans_out) tp->app_limited = diff --git a/net/ipv4/tcp_scalable.c b/net/ipv4/tcp_scalable.c index 5842081bc8a2..862b96248a92 100644 --- a/net/ipv4/tcp_scalable.c +++ b/net/ipv4/tcp_scalable.c @@ -27,7 +27,7 @@ static void tcp_scalable_cong_avoid(struct sock *sk, u32 ack, u32 acked) if (!acked) return; } - tcp_cong_avoid_ai(tp, min(tp->snd_cwnd, TCP_SCALABLE_AI_CNT), + tcp_cong_avoid_ai(tp, min(tcp_snd_cwnd(tp), TCP_SCALABLE_AI_CNT), acked); } @@ -35,7 +35,7 @@ static u32 tcp_scalable_ssthresh(struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); - return max(tp->snd_cwnd - (tp->snd_cwnd>>TCP_SCALABLE_MD_SCALE), 2U); + return max(tcp_snd_cwnd(tp) - (tcp_snd_cwnd(tp)>>TCP_SCALABLE_MD_SCALE), 2U); } static struct tcp_congestion_ops tcp_scalable __read_mostly = { diff --git a/net/ipv4/tcp_vegas.c b/net/ipv4/tcp_vegas.c index c8003c8aad2c..786848ad37ea 100644 --- a/net/ipv4/tcp_vegas.c +++ b/net/ipv4/tcp_vegas.c @@ -159,7 +159,7 @@ EXPORT_SYMBOL_GPL(tcp_vegas_cwnd_event); static inline u32 tcp_vegas_ssthresh(struct tcp_sock *tp) { - return min(tp->snd_ssthresh, tp->snd_cwnd); + return min(tp->snd_ssthresh, tcp_snd_cwnd(tp)); } static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) @@ -217,14 +217,14 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) * This is: * (actual rate in segments) * baseRTT */ - target_cwnd = (u64)tp->snd_cwnd * vegas->baseRTT; + target_cwnd = (u64)tcp_snd_cwnd(tp) * vegas->baseRTT; do_div(target_cwnd, rtt); /* Calculate the difference between the window we had, * and the window we would like to have. This quantity * is the "Diff" from the Arizona Vegas papers. */ - diff = tp->snd_cwnd * (rtt-vegas->baseRTT) / vegas->baseRTT; + diff = tcp_snd_cwnd(tp) * (rtt-vegas->baseRTT) / vegas->baseRTT; if (diff > gamma && tcp_in_slow_start(tp)) { /* Going too fast. Time to slow down @@ -238,7 +238,8 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) * truncation robs us of full link * utilization. */ - tp->snd_cwnd = min(tp->snd_cwnd, (u32)target_cwnd+1); + tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), + (u32)target_cwnd + 1)); tp->snd_ssthresh = tcp_vegas_ssthresh(tp); } else if (tcp_in_slow_start(tp)) { @@ -254,14 +255,14 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) /* The old window was too fast, so * we slow down. */ - tp->snd_cwnd--; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1); tp->snd_ssthresh = tcp_vegas_ssthresh(tp); } else if (diff < alpha) { /* We don't have enough extra packets * in the network, so speed up. */ - tp->snd_cwnd++; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); } else { /* Sending just as fast as we * should be. @@ -269,10 +270,10 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) } } - if (tp->snd_cwnd < 2) - tp->snd_cwnd = 2; - else if (tp->snd_cwnd > tp->snd_cwnd_clamp) - tp->snd_cwnd = tp->snd_cwnd_clamp; + if (tcp_snd_cwnd(tp) < 2) + tcp_snd_cwnd_set(tp, 2); + else if (tcp_snd_cwnd(tp) > tp->snd_cwnd_clamp) + tcp_snd_cwnd_set(tp, tp->snd_cwnd_clamp); tp->snd_ssthresh = tcp_current_ssthresh(sk); } diff --git a/net/ipv4/tcp_veno.c b/net/ipv4/tcp_veno.c index cd50a61c9976..366ff6f214b2 100644 --- a/net/ipv4/tcp_veno.c +++ b/net/ipv4/tcp_veno.c @@ -146,11 +146,11 @@ static void tcp_veno_cong_avoid(struct sock *sk, u32 ack, u32 acked) rtt = veno->minrtt; - target_cwnd = (u64)tp->snd_cwnd * veno->basertt; + target_cwnd = (u64)tcp_snd_cwnd(tp) * veno->basertt; target_cwnd <<= V_PARAM_SHIFT; do_div(target_cwnd, rtt); - veno->diff = (tp->snd_cwnd << V_PARAM_SHIFT) - target_cwnd; + veno->diff = (tcp_snd_cwnd(tp) << V_PARAM_SHIFT) - target_cwnd; if (tcp_in_slow_start(tp)) { /* Slow start. */ @@ -164,15 +164,15 @@ static void tcp_veno_cong_avoid(struct sock *sk, u32 ack, u32 acked) /* In the "non-congestive state", increase cwnd * every rtt. */ - tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked); + tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked); } else { /* In the "congestive state", increase cwnd * every other rtt. */ - if (tp->snd_cwnd_cnt >= tp->snd_cwnd) { + if (tp->snd_cwnd_cnt >= tcp_snd_cwnd(tp)) { if (veno->inc && - tp->snd_cwnd < tp->snd_cwnd_clamp) { - tp->snd_cwnd++; + tcp_snd_cwnd(tp) < tp->snd_cwnd_clamp) { + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1); veno->inc = 0; } else veno->inc = 1; @@ -181,10 +181,10 @@ static void tcp_veno_cong_avoid(struct sock *sk, u32 ack, u32 acked) tp->snd_cwnd_cnt += acked; } done: - if (tp->snd_cwnd < 2) - tp->snd_cwnd = 2; - else if (tp->snd_cwnd > tp->snd_cwnd_clamp) - tp->snd_cwnd = tp->snd_cwnd_clamp; + if (tcp_snd_cwnd(tp) < 2) + tcp_snd_cwnd_set(tp, 2); + else if (tcp_snd_cwnd(tp) > tp->snd_cwnd_clamp) + tcp_snd_cwnd_set(tp, tp->snd_cwnd_clamp); } /* Wipe the slate clean for the next rtt. */ /* veno->cntrtt = 0; */ @@ -199,10 +199,10 @@ static u32 tcp_veno_ssthresh(struct sock *sk) if (veno->diff < beta) /* in "non-congestive state", cut cwnd by 1/5 */ - return max(tp->snd_cwnd * 4 / 5, 2U); + return max(tcp_snd_cwnd(tp) * 4 / 5, 2U); else /* in "congestive state", cut cwnd by 1/2 */ - return max(tp->snd_cwnd >> 1U, 2U); + return max(tcp_snd_cwnd(tp) >> 1U, 2U); } static struct tcp_congestion_ops tcp_veno __read_mostly = { diff --git a/net/ipv4/tcp_westwood.c b/net/ipv4/tcp_westwood.c index b2e05c4cea00..c6e97141eef2 100644 --- a/net/ipv4/tcp_westwood.c +++ b/net/ipv4/tcp_westwood.c @@ -244,7 +244,8 @@ static void tcp_westwood_event(struct sock *sk, enum tcp_ca_event event) switch (event) { case CA_EVENT_COMPLETE_CWR: - tp->snd_cwnd = tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk); + tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk); + tcp_snd_cwnd_set(tp, tp->snd_ssthresh); break; case CA_EVENT_LOSS: tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk); diff --git a/net/ipv4/tcp_yeah.c b/net/ipv4/tcp_yeah.c index 07c4c93b9fdb..18b07ff5d20e 100644 --- a/net/ipv4/tcp_yeah.c +++ b/net/ipv4/tcp_yeah.c @@ -71,11 +71,11 @@ static void tcp_yeah_cong_avoid(struct sock *sk, u32 ack, u32 acked) if (!yeah->doing_reno_now) { /* Scalable */ - tcp_cong_avoid_ai(tp, min(tp->snd_cwnd, TCP_SCALABLE_AI_CNT), + tcp_cong_avoid_ai(tp, min(tcp_snd_cwnd(tp), TCP_SCALABLE_AI_CNT), acked); } else { /* Reno */ - tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked); + tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked); } /* The key players are v_vegas.beg_snd_una and v_beg_snd_nxt. @@ -130,7 +130,7 @@ do_vegas: /* Compute excess number of packets above bandwidth * Avoid doing full 64 bit divide. */ - bw = tp->snd_cwnd; + bw = tcp_snd_cwnd(tp); bw *= rtt - yeah->vegas.baseRTT; do_div(bw, rtt); queue = bw; @@ -138,20 +138,20 @@ do_vegas: if (queue > TCP_YEAH_ALPHA || rtt - yeah->vegas.baseRTT > (yeah->vegas.baseRTT / TCP_YEAH_PHY)) { if (queue > TCP_YEAH_ALPHA && - tp->snd_cwnd > yeah->reno_count) { + tcp_snd_cwnd(tp) > yeah->reno_count) { u32 reduction = min(queue / TCP_YEAH_GAMMA , - tp->snd_cwnd >> TCP_YEAH_EPSILON); + tcp_snd_cwnd(tp) >> TCP_YEAH_EPSILON); - tp->snd_cwnd -= reduction; + tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - reduction); - tp->snd_cwnd = max(tp->snd_cwnd, - yeah->reno_count); + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), + yeah->reno_count)); - tp->snd_ssthresh = tp->snd_cwnd; + tp->snd_ssthresh = tcp_snd_cwnd(tp); } if (yeah->reno_count <= 2) - yeah->reno_count = max(tp->snd_cwnd>>1, 2U); + yeah->reno_count = max(tcp_snd_cwnd(tp)>>1, 2U); else yeah->reno_count++; @@ -176,7 +176,7 @@ do_vegas: */ yeah->vegas.beg_snd_una = yeah->vegas.beg_snd_nxt; yeah->vegas.beg_snd_nxt = tp->snd_nxt; - yeah->vegas.beg_snd_cwnd = tp->snd_cwnd; + yeah->vegas.beg_snd_cwnd = tcp_snd_cwnd(tp); /* Wipe the slate clean for the next RTT. */ yeah->vegas.cntRTT = 0; @@ -193,16 +193,16 @@ static u32 tcp_yeah_ssthresh(struct sock *sk) if (yeah->doing_reno_now < TCP_YEAH_RHO) { reduction = yeah->lastQ; - reduction = min(reduction, max(tp->snd_cwnd>>1, 2U)); + reduction = min(reduction, max(tcp_snd_cwnd(tp)>>1, 2U)); - reduction = max(reduction, tp->snd_cwnd >> TCP_YEAH_DELTA); + reduction = max(reduction, tcp_snd_cwnd(tp) >> TCP_YEAH_DELTA); } else - reduction = max(tp->snd_cwnd>>1, 2U); + reduction = max(tcp_snd_cwnd(tp)>>1, 2U); yeah->fast_count = 0; yeah->reno_count = max(yeah->reno_count>>1, 2U); - return max_t(int, tp->snd_cwnd - reduction, 2); + return max_t(int, tcp_snd_cwnd(tp) - reduction, 2); } static struct tcp_congestion_ops tcp_yeah __read_mostly = { diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index b22504176588..1afc4c024981 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -797,6 +797,7 @@ static void dev_forward_change(struct inet6_dev *idev) { struct net_device *dev; struct inet6_ifaddr *ifa; + LIST_HEAD(tmp_addr_list); if (!idev) return; @@ -815,14 +816,24 @@ static void dev_forward_change(struct inet6_dev *idev) } } + read_lock_bh(&idev->lock); list_for_each_entry(ifa, &idev->addr_list, if_list) { if (ifa->flags&IFA_F_TENTATIVE) continue; + list_add_tail(&ifa->if_list_aux, &tmp_addr_list); + } + read_unlock_bh(&idev->lock); + + while (!list_empty(&tmp_addr_list)) { + ifa = list_first_entry(&tmp_addr_list, + struct inet6_ifaddr, if_list_aux); + list_del(&ifa->if_list_aux); if (idev->cnf.forwarding) addrconf_join_anycast(ifa); else addrconf_leave_anycast(ifa); } + inet6_netconf_notify_devconf(dev_net(dev), RTM_NEWNETCONF, NETCONFA_FORWARDING, dev->ifindex, &idev->cnf); @@ -3728,7 +3739,8 @@ static int addrconf_ifdown(struct net_device *dev, bool unregister) unsigned long event = unregister ? NETDEV_UNREGISTER : NETDEV_DOWN; struct net *net = dev_net(dev); struct inet6_dev *idev; - struct inet6_ifaddr *ifa, *tmp; + struct inet6_ifaddr *ifa; + LIST_HEAD(tmp_addr_list); bool keep_addr = false; bool was_ready; int state, i; @@ -3820,16 +3832,23 @@ restart: write_lock_bh(&idev->lock); } - list_for_each_entry_safe(ifa, tmp, &idev->addr_list, if_list) { + list_for_each_entry(ifa, &idev->addr_list, if_list) + list_add_tail(&ifa->if_list_aux, &tmp_addr_list); + write_unlock_bh(&idev->lock); + + while (!list_empty(&tmp_addr_list)) { struct fib6_info *rt = NULL; bool keep; + ifa = list_first_entry(&tmp_addr_list, + struct inet6_ifaddr, if_list_aux); + list_del(&ifa->if_list_aux); + addrconf_del_dad_work(ifa); keep = keep_addr && (ifa->flags & IFA_F_PERMANENT) && !addr_is_local(&ifa->addr); - write_unlock_bh(&idev->lock); spin_lock_bh(&ifa->lock); if (keep) { @@ -3860,15 +3879,14 @@ restart: addrconf_leave_solict(ifa->idev, &ifa->addr); } - write_lock_bh(&idev->lock); if (!keep) { + write_lock_bh(&idev->lock); list_del_rcu(&ifa->if_list); + write_unlock_bh(&idev->lock); in6_ifa_put(ifa); } } - write_unlock_bh(&idev->lock); - /* Step 5: Discard anycast and multicast list */ if (unregister) { ipv6_ac_destroy_dev(idev); diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index 5b5ea35635f9..b4880c7c84eb 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -150,6 +150,7 @@ static struct sk_buff *ip6_rcv_core(struct sk_buff *skb, struct net_device *dev, struct inet6_dev *idev; if (skb->pkt_type == PACKET_OTHERHOST) { + dev_core_stats_rx_otherhost_dropped_inc(skb->dev); kfree_skb(skb); return NULL; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 53f632a560ec..19325b7600bb 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -257,8 +257,6 @@ static int ip6_tnl_create2(struct net_device *dev) struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id); int err; - t = netdev_priv(dev); - dev->rtnl_link_ops = &ip6_link_ops; err = register_netdevice(dev); if (err < 0) diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index a9775c830194..4e74bc61a3db 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1653,7 +1653,6 @@ int ip6_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, mifi_t mifi; struct net *net = sock_net(sk); struct mr_table *mrt; - bool do_wrmifwhole; if (sk->sk_type != SOCK_RAW || inet_sk(sk)->inet_num != IPPROTO_ICMPV6) @@ -1761,6 +1760,7 @@ int ip6_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, #ifdef CONFIG_IPV6_PIMSM_V2 case MRT6_PIM: { + bool do_wrmifwhole; int v; if (optlen != sizeof(v)) diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index c51d5ce3711c..8bb41f3b246a 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -477,7 +477,8 @@ static int rawv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (np->rxpmtu && np->rxopt.bits.rxpmtu) return ipv6_recv_rxpmtu(sk, msg, len, addr_len); - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 2fa10e60cccd..169e9df6d172 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4484,7 +4484,7 @@ static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes) struct inet6_dev *idev; int type; - if (netif_is_l3_master(skb->dev) && + if (netif_is_l3_master(skb->dev) || dst->dev == net->loopback_dev) idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif)); else diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 13678d3908fa..782df529ff69 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -2044,7 +2044,7 @@ static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) jiffies_to_clock_t(icsk->icsk_rto), jiffies_to_clock_t(icsk->icsk_ack.ato), (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp), - tp->snd_cwnd, + tcp_snd_cwnd(tp), state == TCP_LISTEN ? fastopenq->max_qlen : (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh) diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index a1760add5bf1..a0385ddbffcf 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -1223,7 +1223,6 @@ static void iucv_process_message_q(struct sock *sk) static int iucv_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { - int noblock = flags & MSG_DONTWAIT; struct sock *sk = sock->sk; struct iucv_sock *iucv = iucv_sk(sk); unsigned int copied, rlen; @@ -1242,7 +1241,7 @@ static int iucv_sock_recvmsg(struct socket *sock, struct msghdr *msg, /* receive/dequeue next skb: * the function understands MSG_PEEK and, thus, does not dequeue skb */ - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) { if (sk->sk_shutdown & RCV_SHUTDOWN) return 0; diff --git a/net/key/af_key.c b/net/key/af_key.c index fd51db3be91c..d09ec26b1081 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -3696,7 +3696,7 @@ static int pfkey_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT)) goto out; - skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); if (skb == NULL) goto out; diff --git a/net/l2tp/l2tp_ip.c b/net/l2tp/l2tp_ip.c index b3edafa5fba4..c6a5cc2d88e7 100644 --- a/net/l2tp/l2tp_ip.c +++ b/net/l2tp/l2tp_ip.c @@ -526,7 +526,8 @@ static int l2tp_ip_recvmsg(struct sock *sk, struct msghdr *msg, if (flags & MSG_OOB) goto out; - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index 96f975777438..97fde8a9209b 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -671,7 +671,8 @@ static int l2tp_ip6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, if (flags & MSG_ERRQUEUE) return ipv6_recv_error(sk, msg, len, addr_len); - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto out; diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c index bf35710127dd..8be1fdc68a0b 100644 --- a/net/l2tp/l2tp_ppp.c +++ b/net/l2tp/l2tp_ppp.c @@ -191,8 +191,7 @@ static int pppol2tp_recvmsg(struct socket *sock, struct msghdr *msg, goto end; err = 0; - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) goto end; diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c index f0702d920d8d..221863afc4b1 100644 --- a/net/mctp/af_mctp.c +++ b/net/mctp/af_mctp.c @@ -93,13 +93,13 @@ out_release: static int mctp_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { DECLARE_SOCKADDR(struct sockaddr_mctp *, addr, msg->msg_name); - const int hlen = MCTP_HEADER_MAXLEN + sizeof(struct mctp_hdr); int rc, addrlen = msg->msg_namelen; struct sock *sk = sock->sk; struct mctp_sock *msk = container_of(sk, struct mctp_sock, sk); struct mctp_skb_cb *cb; struct mctp_route *rt; - struct sk_buff *skb; + struct sk_buff *skb = NULL; + int hlen; if (addr) { const u8 tagbits = MCTP_TAG_MASK | MCTP_TAG_OWNER | @@ -129,6 +129,34 @@ static int mctp_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) if (addr->smctp_network == MCTP_NET_ANY) addr->smctp_network = mctp_default_net(sock_net(sk)); + /* direct addressing */ + if (msk->addr_ext && addrlen >= sizeof(struct sockaddr_mctp_ext)) { + DECLARE_SOCKADDR(struct sockaddr_mctp_ext *, + extaddr, msg->msg_name); + struct net_device *dev; + + rc = -EINVAL; + rcu_read_lock(); + dev = dev_get_by_index_rcu(sock_net(sk), extaddr->smctp_ifindex); + /* check for correct halen */ + if (dev && extaddr->smctp_halen == dev->addr_len) { + hlen = LL_RESERVED_SPACE(dev) + sizeof(struct mctp_hdr); + rc = 0; + } + rcu_read_unlock(); + if (rc) + goto err_free; + rt = NULL; + } else { + rt = mctp_route_lookup(sock_net(sk), addr->smctp_network, + addr->smctp_addr.s_addr); + if (!rt) { + rc = -EHOSTUNREACH; + goto err_free; + } + hlen = LL_RESERVED_SPACE(rt->dev->dev) + sizeof(struct mctp_hdr); + } + skb = sock_alloc_send_skb(sk, hlen + 1 + len, msg->msg_flags & MSG_DONTWAIT, &rc); if (!skb) @@ -147,8 +175,8 @@ static int mctp_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) cb = __mctp_cb(skb); cb->net = addr->smctp_network; - /* direct addressing */ - if (msk->addr_ext && addrlen >= sizeof(struct sockaddr_mctp_ext)) { + if (!rt) { + /* fill extended address in cb */ DECLARE_SOCKADDR(struct sockaddr_mctp_ext *, extaddr, msg->msg_name); @@ -159,17 +187,9 @@ static int mctp_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) } cb->ifindex = extaddr->smctp_ifindex; + /* smctp_halen is checked above */ cb->halen = extaddr->smctp_halen; memcpy(cb->haddr, extaddr->smctp_haddr, cb->halen); - - rt = NULL; - } else { - rt = mctp_route_lookup(sock_net(sk), addr->smctp_network, - addr->smctp_addr.s_addr); - if (!rt) { - rc = -EHOSTUNREACH; - goto err_free; - } } rc = mctp_local_output(sk, rt, skb, addr->smctp_addr.s_addr, @@ -196,7 +216,7 @@ static int mctp_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (flags & ~(MSG_DONTWAIT | MSG_TRUNC | MSG_PEEK)) return -EOPNOTSUPP; - skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &rc); + skb = skb_recv_datagram(sk, flags, &rc); if (!skb) return rc; diff --git a/net/mctp/route.c b/net/mctp/route.c index d5e7db83fe9d..3b24b8d18b5b 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -503,6 +503,11 @@ static int mctp_route_output(struct mctp_route *route, struct sk_buff *skb) if (cb->ifindex) { /* direct route; use the hwaddr we stashed in sendmsg */ + if (cb->halen != skb->dev->addr_len) { + /* sanity check, sendmsg should have already caught this */ + kfree_skb(skb); + return -EMSGSIZE; + } daddr = cb->haddr; } else { /* If lookup fails let the device handle daddr==NULL */ @@ -512,7 +517,7 @@ static int mctp_route_output(struct mctp_route *route, struct sk_buff *skb) rc = dev_hard_header(skb, skb->dev, ntohs(skb->protocol), daddr, skb->dev->dev_addr, skb->len); - if (rc) { + if (rc < 0) { kfree_skb(skb); return -EHOSTUNREACH; } @@ -756,7 +761,7 @@ static int mctp_do_fragment_route(struct mctp_route *rt, struct sk_buff *skb, { const unsigned int hlen = sizeof(struct mctp_hdr); struct mctp_hdr *hdr, *hdr2; - unsigned int pos, size; + unsigned int pos, size, headroom; struct sk_buff *skb2; int rc; u8 seq; @@ -770,6 +775,9 @@ static int mctp_do_fragment_route(struct mctp_route *rt, struct sk_buff *skb, return -EMSGSIZE; } + /* keep same headroom as the original skb */ + headroom = skb_headroom(skb); + /* we've got the header */ skb_pull(skb, hlen); @@ -777,7 +785,7 @@ static int mctp_do_fragment_route(struct mctp_route *rt, struct sk_buff *skb, /* size of message payload */ size = min(mtu - hlen, skb->len - pos); - skb2 = alloc_skb(MCTP_HEADER_MAXLEN + hlen + size, GFP_KERNEL); + skb2 = alloc_skb(headroom + hlen + size, GFP_KERNEL); if (!skb2) { rc = -ENOMEM; break; @@ -793,7 +801,7 @@ static int mctp_do_fragment_route(struct mctp_route *rt, struct sk_buff *skb, skb_set_owner_w(skb2, skb->sk); /* establish packet */ - skb_reserve(skb2, MCTP_HEADER_MAXLEN); + skb_reserve(skb2, headroom); skb_reset_network_header(skb2); skb_put(skb2, hlen + size); skb2->transport_header = skb2->network_header + hlen; diff --git a/net/mctp/test/route-test.c b/net/mctp/test/route-test.c index 61205cf40074..24df29e135ed 100644 --- a/net/mctp/test/route-test.c +++ b/net/mctp/test/route-test.c @@ -352,7 +352,7 @@ static void mctp_test_route_input_sk(struct kunit *test) if (params->deliver) { KUNIT_EXPECT_EQ(test, rc, 0); - skb2 = skb_recv_datagram(sock->sk, 0, 1, &rc); + skb2 = skb_recv_datagram(sock->sk, MSG_DONTWAIT, &rc); KUNIT_EXPECT_NOT_ERR_OR_NULL(test, skb2); KUNIT_EXPECT_EQ(test, skb->len, 1); @@ -360,7 +360,7 @@ static void mctp_test_route_input_sk(struct kunit *test) } else { KUNIT_EXPECT_NE(test, rc, 0); - skb2 = skb_recv_datagram(sock->sk, 0, 1, &rc); + skb2 = skb_recv_datagram(sock->sk, MSG_DONTWAIT, &rc); KUNIT_EXPECT_PTR_EQ(test, skb2, NULL); } @@ -423,7 +423,7 @@ static void mctp_test_route_input_sk_reasm(struct kunit *test) rc = mctp_route_input(&rt->rt, skb); } - skb2 = skb_recv_datagram(sock->sk, 0, 1, &rc); + skb2 = skb_recv_datagram(sock->sk, MSG_DONTWAIT, &rc); if (params->rx_len) { KUNIT_EXPECT_NOT_ERR_OR_NULL(test, skb2); @@ -582,7 +582,7 @@ static void mctp_test_route_input_sk_keys(struct kunit *test) rc = mctp_route_input(&rt->rt, skb); /* (potentially) receive message */ - skb2 = skb_recv_datagram(sock->sk, 0, 1, &rc); + skb2 = skb_recv_datagram(sock->sk, MSG_DONTWAIT, &rc); if (params->deliver) KUNIT_EXPECT_NOT_ERR_OR_NULL(test, skb2); diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index d6fdc5782d33..35b5f806fdda 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -1527,10 +1527,9 @@ static int mpls_ifdown(struct net_device *dev, int event) rt->rt_nh_size; struct mpls_route *orig = rt; - rt = kmalloc(size, GFP_KERNEL); + rt = kmemdup(orig, size, GFP_KERNEL); if (!rt) return -ENOMEM; - memcpy(rt, orig, size); } } diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 5ddfdb2adaf1..128ee3b300d6 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -5526,7 +5526,7 @@ int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, int err, i, k; for (i = 0; i < set->num_exprs; i++) { - expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL); + expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT); if (!expr) goto err_expr; diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c index 38caa66632b4..f590ee1c8a1b 100644 --- a/net/netfilter/nft_bitwise.c +++ b/net/netfilter/nft_bitwise.c @@ -290,7 +290,7 @@ static bool nft_bitwise_reduce(struct nft_regs_track *track, if (!track->regs[priv->sreg].selector) return false; - bitwise = nft_expr_priv(expr); + bitwise = nft_expr_priv(track->regs[priv->dreg].selector); if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && track->regs[priv->sreg].num_reg == 0 && track->regs[priv->dreg].bitwise && @@ -442,7 +442,7 @@ static bool nft_bitwise_fast_reduce(struct nft_regs_track *track, if (!track->regs[priv->sreg].selector) return false; - bitwise = nft_expr_priv(expr); + bitwise = nft_expr_priv(track->regs[priv->dreg].selector); if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && track->regs[priv->dreg].bitwise && track->regs[priv->dreg].bitwise->ops == expr->ops && diff --git a/net/netfilter/nft_connlimit.c b/net/netfilter/nft_connlimit.c index 9de1462e4ac4..d657f999a11b 100644 --- a/net/netfilter/nft_connlimit.c +++ b/net/netfilter/nft_connlimit.c @@ -77,7 +77,7 @@ static int nft_connlimit_do_init(const struct nft_ctx *ctx, invert = true; } - priv->list = kmalloc(sizeof(*priv->list), GFP_KERNEL); + priv->list = kmalloc(sizeof(*priv->list), GFP_KERNEL_ACCOUNT); if (!priv->list) return -ENOMEM; diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index da9083605a61..f4d3573e8782 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -62,7 +62,7 @@ static int nft_counter_do_init(const struct nlattr * const tb[], struct nft_counter __percpu *cpu_stats; struct nft_counter *this_cpu; - cpu_stats = alloc_percpu(struct nft_counter); + cpu_stats = alloc_percpu_gfp(struct nft_counter, GFP_KERNEL_ACCOUNT); if (cpu_stats == NULL) return -ENOMEM; diff --git a/net/netfilter/nft_last.c b/net/netfilter/nft_last.c index 43d0d4aadb1f..bb15a55dad5c 100644 --- a/net/netfilter/nft_last.c +++ b/net/netfilter/nft_last.c @@ -30,7 +30,7 @@ static int nft_last_init(const struct nft_ctx *ctx, const struct nft_expr *expr, u64 last_jiffies; int err; - last = kzalloc(sizeof(*last), GFP_KERNEL); + last = kzalloc(sizeof(*last), GFP_KERNEL_ACCOUNT); if (!last) return -ENOMEM; diff --git a/net/netfilter/nft_limit.c b/net/netfilter/nft_limit.c index d4a6cf3cd697..04ea8b9bf202 100644 --- a/net/netfilter/nft_limit.c +++ b/net/netfilter/nft_limit.c @@ -90,7 +90,7 @@ static int nft_limit_init(struct nft_limit_priv *priv, priv->rate); } - priv->limit = kmalloc(sizeof(*priv->limit), GFP_KERNEL); + priv->limit = kmalloc(sizeof(*priv->limit), GFP_KERNEL_ACCOUNT); if (!priv->limit) return -ENOMEM; diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index d7db57ed3bc1..e6b0df68feea 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -90,7 +90,7 @@ static int nft_quota_do_init(const struct nlattr * const tb[], return -EOPNOTSUPP; } - priv->consumed = kmalloc(sizeof(*priv->consumed), GFP_KERNEL); + priv->consumed = kmalloc(sizeof(*priv->consumed), GFP_KERNEL_ACCOUNT); if (!priv->consumed) return -ENOMEM; diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 47a876ccd288..9fa85bb36c0e 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -1931,7 +1931,6 @@ static int netlink_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, struct scm_cookie scm; struct sock *sk = sock->sk; struct netlink_sock *nlk = nlk_sk(sk); - int noblock = flags & MSG_DONTWAIT; size_t copied; struct sk_buff *skb, *data_skb; int err, ret; @@ -1941,7 +1940,7 @@ static int netlink_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, copied = 0; - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (skb == NULL) goto out; diff --git a/net/netrom/af_netrom.c b/net/netrom/af_netrom.c index fa9dc2ba3941..6f7f4392cffb 100644 --- a/net/netrom/af_netrom.c +++ b/net/netrom/af_netrom.c @@ -1159,7 +1159,8 @@ static int nr_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, } /* Now we can treat all alike */ - if ((skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, flags & MSG_DONTWAIT, &er)) == NULL) { + skb = skb_recv_datagram(sk, flags, &er); + if (!skb) { release_sock(sk); return er; } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index 4ca35791c93b..77642d18a3b4 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -821,7 +821,6 @@ static int llcp_sock_sendmsg(struct socket *sock, struct msghdr *msg, static int llcp_sock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { - int noblock = flags & MSG_DONTWAIT; struct sock *sk = sock->sk; unsigned int copied, rlen; struct sk_buff *skb, *cskb; @@ -842,7 +841,7 @@ static int llcp_sock_recvmsg(struct socket *sock, struct msghdr *msg, if (flags & (MSG_OOB)) return -EOPNOTSUPP; - skb = skb_recv_datagram(sk, flags, noblock, &err); + skb = skb_recv_datagram(sk, flags, &err); if (!skb) { pr_err("Recv datagram failed state %d %d %d", sk->sk_state, err, sock_error(sk)); diff --git a/net/nfc/rawsock.c b/net/nfc/rawsock.c index 0ca214ab5aef..8dd569765f96 100644 --- a/net/nfc/rawsock.c +++ b/net/nfc/rawsock.c @@ -238,7 +238,6 @@ static int rawsock_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) static int rawsock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { - int noblock = flags & MSG_DONTWAIT; struct sock *sk = sock->sk; struct sk_buff *skb; int copied; @@ -246,7 +245,7 @@ static int rawsock_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, pr_debug("sock=%p sk=%p len=%zu flags=%d\n", sock, sk, len, flags); - skb = skb_recv_datagram(sk, flags, noblock, &rc); + skb = skb_recv_datagram(sk, flags, &rc); if (!skb) return rc; diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 7056cb1b8ba0..1b5d73079dc9 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -1051,7 +1051,7 @@ static int clone(struct datapath *dp, struct sk_buff *skb, int rem = nla_len(attr); bool dont_clone_flow_key; - /* The first action is always 'OVS_CLONE_ATTR_ARG'. */ + /* The first action is always 'OVS_CLONE_ATTR_EXEC'. */ clone_arg = nla_data(attr); dont_clone_flow_key = nla_get_u32(clone_arg); actions = nla_next(clone_arg, &rem); diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index cc282a58b75b..7176156d3844 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -2317,6 +2317,62 @@ static struct sw_flow_actions *nla_alloc_flow_actions(int size) return sfa; } +static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len); + +static void ovs_nla_free_check_pkt_len_action(const struct nlattr *action) +{ + const struct nlattr *a; + int rem; + + nla_for_each_nested(a, action, rem) { + switch (nla_type(a)) { + case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL: + case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER: + ovs_nla_free_nested_actions(nla_data(a), nla_len(a)); + break; + } + } +} + +static void ovs_nla_free_clone_action(const struct nlattr *action) +{ + const struct nlattr *a = nla_data(action); + int rem = nla_len(action); + + switch (nla_type(a)) { + case OVS_CLONE_ATTR_EXEC: + /* The real list of actions follows this attribute. */ + a = nla_next(a, &rem); + ovs_nla_free_nested_actions(a, rem); + break; + } +} + +static void ovs_nla_free_dec_ttl_action(const struct nlattr *action) +{ + const struct nlattr *a = nla_data(action); + + switch (nla_type(a)) { + case OVS_DEC_TTL_ATTR_ACTION: + ovs_nla_free_nested_actions(nla_data(a), nla_len(a)); + break; + } +} + +static void ovs_nla_free_sample_action(const struct nlattr *action) +{ + const struct nlattr *a = nla_data(action); + int rem = nla_len(action); + + switch (nla_type(a)) { + case OVS_SAMPLE_ATTR_ARG: + /* The real list of actions follows this attribute. */ + a = nla_next(a, &rem); + ovs_nla_free_nested_actions(a, rem); + break; + } +} + static void ovs_nla_free_set_action(const struct nlattr *a) { const struct nlattr *ovs_key = nla_data(a); @@ -2330,25 +2386,54 @@ static void ovs_nla_free_set_action(const struct nlattr *a) } } -void ovs_nla_free_flow_actions(struct sw_flow_actions *sf_acts) +static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len) { const struct nlattr *a; int rem; - if (!sf_acts) + /* Whenever new actions are added, the need to update this + * function should be considered. + */ + BUILD_BUG_ON(OVS_ACTION_ATTR_MAX != 23); + + if (!actions) return; - nla_for_each_attr(a, sf_acts->actions, sf_acts->actions_len, rem) { + nla_for_each_attr(a, actions, len, rem) { switch (nla_type(a)) { - case OVS_ACTION_ATTR_SET: - ovs_nla_free_set_action(a); + case OVS_ACTION_ATTR_CHECK_PKT_LEN: + ovs_nla_free_check_pkt_len_action(a); + break; + + case OVS_ACTION_ATTR_CLONE: + ovs_nla_free_clone_action(a); break; + case OVS_ACTION_ATTR_CT: ovs_ct_free_action(a); break; + + case OVS_ACTION_ATTR_DEC_TTL: + ovs_nla_free_dec_ttl_action(a); + break; + + case OVS_ACTION_ATTR_SAMPLE: + ovs_nla_free_sample_action(a); + break; + + case OVS_ACTION_ATTR_SET: + ovs_nla_free_set_action(a); + break; } } +} + +void ovs_nla_free_flow_actions(struct sw_flow_actions *sf_acts) +{ + if (!sf_acts) + return; + ovs_nla_free_nested_actions(sf_acts->actions, sf_acts->actions_len); kfree(sf_acts); } @@ -3458,7 +3543,9 @@ static int clone_action_to_attr(const struct nlattr *attr, if (!start) return -EMSGSIZE; - err = ovs_nla_put_actions(nla_data(attr), rem, skb); + /* Skipping the OVS_CLONE_ATTR_EXEC that is always the first attribute. */ + attr = nla_next(nla_data(attr), &rem); + err = ovs_nla_put_actions(attr, rem, skb); if (err) nla_nest_cancel(skb, start); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index c39c09899fd0..d3caaf4d4b3e 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3421,7 +3421,7 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, * but then it will block. */ - skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err); + skb = skb_recv_datagram(sk, flags, &err); /* * An error occurred so return it. Because skb_recv_datagram() diff --git a/net/phonet/datagram.c b/net/phonet/datagram.c index 393e6aa7a592..3f2e62b63dd4 100644 --- a/net/phonet/datagram.c +++ b/net/phonet/datagram.c @@ -123,7 +123,8 @@ static int pn_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, MSG_CMSG_COMPAT)) goto out_nofree; - skb = skb_recv_datagram(sk, flags, noblock, &rval); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &rval); if (skb == NULL) goto out_nofree; diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 65d463ad8770..441a26706592 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -772,7 +772,8 @@ static struct sock *pep_sock_accept(struct sock *sk, int flags, int *errp, u8 pipe_handle, enabled, n_sb; u8 aligned = 0; - skb = skb_recv_datagram(sk, 0, flags & O_NONBLOCK, errp); + skb = skb_recv_datagram(sk, (flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, + errp); if (!skb) return NULL; @@ -1267,7 +1268,8 @@ static int pep_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, return -EINVAL; } - skb = skb_recv_datagram(sk, flags, noblock, &err); + flags |= (noblock ? MSG_DONTWAIT : 0); + skb = skb_recv_datagram(sk, flags, &err); lock_sock(sk); if (skb == NULL) { if (err == -ENOTCONN && sk->sk_state == TCP_CLOSE_WAIT) diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index ec2322529727..5c2fb992803b 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -1035,8 +1035,7 @@ static int qrtr_recvmsg(struct socket *sock, struct msghdr *msg, return -EADDRNOTAVAIL; } - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &rc); + skb = skb_recv_datagram(sk, flags, &rc); if (!skb) { release_sock(sk); return rc; diff --git a/net/rose/af_rose.c b/net/rose/af_rose.c index 30a1cf4c16c6..bf2d986a6bc3 100644 --- a/net/rose/af_rose.c +++ b/net/rose/af_rose.c @@ -1230,7 +1230,8 @@ static int rose_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, return -ENOTCONN; /* Now we can treat all alike */ - if ((skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, flags & MSG_DONTWAIT, &er)) == NULL) + skb = skb_recv_datagram(sk, flags, &er); + if (!skb) return er; qbit = (skb->data[0] & ROSE_Q_BIT) == ROSE_Q_BIT; diff --git a/net/rxrpc/net_ns.c b/net/rxrpc/net_ns.c index 25bbc4cc8b13..f15d6942da45 100644 --- a/net/rxrpc/net_ns.c +++ b/net/rxrpc/net_ns.c @@ -113,8 +113,8 @@ static __net_exit void rxrpc_exit_net(struct net *net) struct rxrpc_net *rxnet = rxrpc_net(net); rxnet->live = false; - del_timer_sync(&rxnet->peer_keepalive_timer); cancel_work_sync(&rxnet->peer_keepalive_work); + del_timer_sync(&rxnet->peer_keepalive_timer); rxrpc_destroy_all_calls(rxnet); rxrpc_destroy_all_connections(rxnet); rxrpc_destroy_all_peers(rxnet); diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 4f51094da9da..da9733da9868 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -195,7 +195,7 @@ static int offload_action_init(struct flow_offload_action *fl_action, if (act->ops->offload_act_setup) { spin_lock_bh(&act->tcfa_lock); err = act->ops->offload_act_setup(act, fl_action, NULL, - false); + false, extack); spin_unlock_bh(&act->tcfa_lock); return err; } @@ -271,7 +271,7 @@ static int tcf_action_offload_add_ex(struct tc_action *action, if (err) goto fl_err; - err = tc_setup_action(&fl_action->action, actions); + err = tc_setup_action(&fl_action->action, actions, extack); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed to setup tc actions for offload"); diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index e0f515b774ca..22847ee009ef 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -696,7 +696,8 @@ static size_t tcf_csum_get_fill_size(const struct tc_action *act) } static int tcf_csum_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index b1f502fce595..8af9d6e5ba61 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1584,7 +1584,8 @@ static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets, } static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index bde6a6c01e64..ac29d1065232 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -253,7 +253,8 @@ static size_t tcf_gact_get_fill_size(const struct tc_action *act) } static int tcf_gact_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -267,7 +268,17 @@ static int tcf_gact_offload_act_setup(struct tc_action *act, void *entry_data, } else if (is_tcf_gact_goto_chain(act)) { entry->id = FLOW_ACTION_GOTO; entry->chain_index = tcf_gact_goto_chain_index(act); + } else if (is_tcf_gact_continue(act)) { + NL_SET_ERR_MSG_MOD(extack, "Offload of \"continue\" action is not supported"); + return -EOPNOTSUPP; + } else if (is_tcf_gact_reclassify(act)) { + NL_SET_ERR_MSG_MOD(extack, "Offload of \"reclassify\" action is not supported"); + return -EOPNOTSUPP; + } else if (is_tcf_gact_pipe(act)) { + NL_SET_ERR_MSG_MOD(extack, "Offload of \"pipe\" action is not supported"); + return -EOPNOTSUPP; } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported generic action offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index d56e73843a4b..fd5155274733 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -619,7 +619,8 @@ static int tcf_gate_get_entries(struct flow_action_entry *entry, } static int tcf_gate_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { int err; diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index 39acd1d18609..ebb92fb072ab 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -460,7 +460,8 @@ static void tcf_offload_mirred_get_dev(struct flow_action_entry *entry, } static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -478,6 +479,7 @@ static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data, entry->id = FLOW_ACTION_MIRRED_INGRESS; tcf_offload_mirred_get_dev(entry, act); } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported mirred offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index b9ff3459fdab..adabeccb63e1 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -385,7 +385,8 @@ static int tcf_mpls_search(struct net *net, struct tc_action **a, u32 index) } static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -410,7 +411,14 @@ static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data, entry->mpls_mangle.bos = tcf_mpls_bos(act); entry->mpls_mangle.ttl = tcf_mpls_ttl(act); break; + case TCA_MPLS_ACT_DEC_TTL: + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"dec_ttl\" option is used"); + return -EOPNOTSUPP; + case TCA_MPLS_ACT_MAC_PUSH: + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"mac_push\" option is used"); + return -EOPNOTSUPP; default: + NL_SET_ERR_MSG_MOD(extack, "Unsupported MPLS mode offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 31fcd279c177..e01ef7f109f4 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -488,7 +488,8 @@ static int tcf_pedit_search(struct net *net, struct tc_action **a, u32 index) } static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -503,6 +504,7 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, entry->id = FLOW_ACTION_ADD; break; default: + NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); return -EOPNOTSUPP; } entry->mangle.htype = tcf_pedit_htype(act, k); diff --git a/net/sched/act_police.c b/net/sched/act_police.c index f4d917705263..79c8901f66ab 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -419,7 +419,8 @@ static int tcf_police_search(struct net *net, struct tc_action **a, u32 index) return tcf_idr_search(tn, a, index); } -static int tcf_police_act_to_flow_act(int tc_act, u32 *extval) +static int tcf_police_act_to_flow_act(int tc_act, u32 *extval, + struct netlink_ext_ack *extack) { int act_id = -EOPNOTSUPP; @@ -430,19 +431,28 @@ static int tcf_police_act_to_flow_act(int tc_act, u32 *extval) act_id = FLOW_ACTION_DROP; else if (tc_act == TC_ACT_PIPE) act_id = FLOW_ACTION_PIPE; + else if (tc_act == TC_ACT_RECLASSIFY) + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when conform/exceed action is \"reclassify\""); + else + NL_SET_ERR_MSG_MOD(extack, "Unsupported conform/exceed action offload"); } else if (TC_ACT_EXT_CMP(tc_act, TC_ACT_GOTO_CHAIN)) { act_id = FLOW_ACTION_GOTO; *extval = tc_act & TC_ACT_EXT_VAL_MASK; } else if (TC_ACT_EXT_CMP(tc_act, TC_ACT_JUMP)) { act_id = FLOW_ACTION_JUMP; *extval = tc_act & TC_ACT_EXT_VAL_MASK; + } else if (tc_act == TC_ACT_UNSPEC) { + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when conform/exceed action is \"continue\""); + } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported conform/exceed action offload"); } return act_id; } static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -466,14 +476,16 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, entry->police.mtu = tcf_police_tcfp_mtu(act); act_id = tcf_police_act_to_flow_act(police->tcf_action, - &entry->police.exceed.extval); + &entry->police.exceed.extval, + extack); if (act_id < 0) return act_id; entry->police.exceed.act_id = act_id; act_id = tcf_police_act_to_flow_act(p->tcfp_result, - &entry->police.notexceed.extval); + &entry->police.notexceed.extval, + extack); if (act_id < 0) return act_id; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 9a22cdda6bbd..2f7f5e44d28c 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -291,7 +291,8 @@ static void tcf_offload_sample_get_group(struct flow_action_entry *entry, } static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index ceba11b198bb..92d0dc754207 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -328,7 +328,8 @@ static size_t tcf_skbedit_get_fill_size(const struct tc_action *act) } static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -342,7 +343,14 @@ static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data } else if (is_tcf_skbedit_priority(act)) { entry->id = FLOW_ACTION_PRIORITY; entry->priority = tcf_skbedit_priority(act); + } else if (is_tcf_skbedit_queue_mapping(act)) { + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used"); + return -EOPNOTSUPP; + } else if (is_tcf_skbedit_inheritdsfield(act)) { + NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used"); + return -EOPNOTSUPP; } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 23aba03d26a8..856dc23cef8c 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -808,7 +808,8 @@ static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry, static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, void *entry_data, u32 *index_inc, - bool bind) + bool bind, + struct netlink_ext_ack *extack) { int err; @@ -823,6 +824,7 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, } else if (is_tcf_tunnel_release(act)) { entry->id = FLOW_ACTION_TUNNEL_DECAP; } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel key mode offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c index 883454c4f921..68b5e772386a 100644 --- a/net/sched/act_vlan.c +++ b/net/sched/act_vlan.c @@ -369,7 +369,8 @@ static size_t tcf_vlan_get_fill_size(const struct tc_action *act) } static int tcf_vlan_offload_act_setup(struct tc_action *act, void *entry_data, - u32 *index_inc, bool bind) + u32 *index_inc, bool bind, + struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; @@ -398,6 +399,7 @@ static int tcf_vlan_offload_act_setup(struct tc_action *act, void *entry_data, tcf_vlan_push_eth(entry->vlan_push_eth.src, entry->vlan_push_eth.dst, act); break; default: + NL_SET_ERR_MSG_MOD(extack, "Unsupported vlan action mode offload"); return -EOPNOTSUPP; } *index_inc = 1; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 2957f8f5cea7..d8a22bedd026 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3513,20 +3513,25 @@ EXPORT_SYMBOL(tc_cleanup_offload_action); static int tc_setup_offload_act(struct tc_action *act, struct flow_action_entry *entry, - u32 *index_inc) + u32 *index_inc, + struct netlink_ext_ack *extack) { #ifdef CONFIG_NET_CLS_ACT - if (act->ops->offload_act_setup) - return act->ops->offload_act_setup(act, entry, index_inc, true); - else + if (act->ops->offload_act_setup) { + return act->ops->offload_act_setup(act, entry, index_inc, true, + extack); + } else { + NL_SET_ERR_MSG(extack, "Action does not support offload"); return -EOPNOTSUPP; + } #else return 0; #endif } int tc_setup_action(struct flow_action *flow_action, - struct tc_action *actions[]) + struct tc_action *actions[], + struct netlink_ext_ack *extack) { int i, j, index, err = 0; struct tc_action *act; @@ -3551,7 +3556,7 @@ int tc_setup_action(struct flow_action *flow_action, entry->hw_stats = tc_act_hw_stats(act->hw_stats); entry->hw_index = act->tcfa_index; index = 0; - err = tc_setup_offload_act(act, entry, &index); + err = tc_setup_offload_act(act, entry, &index, extack); if (!err) j += index; else @@ -3570,13 +3575,14 @@ err_out_locked: } int tc_setup_offload_action(struct flow_action *flow_action, - const struct tcf_exts *exts) + const struct tcf_exts *exts, + struct netlink_ext_ack *extack) { #ifdef CONFIG_NET_CLS_ACT if (!exts) return 0; - return tc_setup_action(flow_action, exts->actions); + return tc_setup_action(flow_action, exts->actions, extack); #else return 0; #endif diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index c80fc49c0da1..87e030dfc328 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -464,14 +464,12 @@ static int fl_hw_replace_filter(struct tcf_proto *tp, cls_flower.rule->match.key = &f->mkey; cls_flower.classid = f->res.classid; - err = tc_setup_offload_action(&cls_flower.rule->action, &f->exts); + err = tc_setup_offload_action(&cls_flower.rule->action, &f->exts, + cls_flower.common.extack); if (err) { kfree(cls_flower.rule); - if (skip_sw) { - NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); - return err; - } - return 0; + + return skip_sw ? err : 0; } err = tc_setup_cb_add(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, @@ -2354,11 +2352,11 @@ static int fl_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, cls_flower.rule->match.mask = &f->mask->key; cls_flower.rule->match.key = &f->mkey; - err = tc_setup_offload_action(&cls_flower.rule->action, &f->exts); + err = tc_setup_offload_action(&cls_flower.rule->action, &f->exts, + cls_flower.common.extack); if (err) { kfree(cls_flower.rule); if (tc_skip_sw(f->flags)) { - NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); __fl_put(f); return err; } diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index ca5670fd5228..06cf22adbab7 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -97,16 +97,13 @@ static int mall_replace_hw_filter(struct tcf_proto *tp, cls_mall.command = TC_CLSMATCHALL_REPLACE; cls_mall.cookie = cookie; - err = tc_setup_offload_action(&cls_mall.rule->action, &head->exts); + err = tc_setup_offload_action(&cls_mall.rule->action, &head->exts, + cls_mall.common.extack); if (err) { kfree(cls_mall.rule); mall_destroy_hw_filter(tp, head, cookie, NULL); - if (skip_sw) - NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); - else - err = 0; - return err; + return skip_sw ? err : 0; } err = tc_setup_cb_add(block, tp, TC_SETUP_CLSMATCHALL, &cls_mall, @@ -302,14 +299,12 @@ static int mall_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, TC_CLSMATCHALL_REPLACE : TC_CLSMATCHALL_DESTROY; cls_mall.cookie = (unsigned long)head; - err = tc_setup_offload_action(&cls_mall.rule->action, &head->exts); + err = tc_setup_offload_action(&cls_mall.rule->action, &head->exts, + cls_mall.common.extack); if (err) { kfree(cls_mall.rule); - if (add && tc_skip_sw(head->flags)) { - NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); - return err; - } - return 0; + + return add && tc_skip_sw(head->flags) ? err : 0; } err = tc_setup_cb_reoffload(block, tp, add, cb, TC_SETUP_CLSMATCHALL, diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index a18609f608fb..e213aaf45d67 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -914,6 +914,7 @@ static void sctp_outq_flush_ctrl(struct sctp_flush_ctx *ctx) ctx->asoc->base.sk->sk_err = -error; return; } + ctx->asoc->stats.octrlchunks++; break; case SCTP_CID_ABORT: @@ -938,7 +939,10 @@ static void sctp_outq_flush_ctrl(struct sctp_flush_ctx *ctx) case SCTP_CID_HEARTBEAT: if (chunk->pmtu_probe) { - sctp_packet_singleton(ctx->transport, chunk, ctx->gfp); + error = sctp_packet_singleton(ctx->transport, + chunk, ctx->gfp); + if (!error) + ctx->asoc->stats.octrlchunks++; break; } fallthrough; diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 12f7b56771d9..5e4bd8ba48d3 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -962,11 +962,9 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, tls_ctx->rx.rec_seq, rxm->full_len, is_encrypted, is_decrypted); - ctx->sw.decrypted |= is_decrypted; - if (unlikely(test_bit(TLS_RX_DEV_DEGRADED, &tls_ctx->flags))) { if (likely(is_encrypted || is_decrypted)) - return 0; + return is_decrypted; /* After tls_device_down disables the offload, the next SKB will * likely have initial fragments decrypted, and final ones not @@ -981,7 +979,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, */ if (is_decrypted) { ctx->resync_nh_reset = 1; - return 0; + return is_decrypted; } if (is_encrypted) { tls_device_core_ctrl_rx_resync(tls_ctx, ctx, sk, skb); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 0024a692f0f8..3a0a120f9c56 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -128,32 +128,31 @@ static int skb_nsg(struct sk_buff *skb, int offset, int len) return __skb_nsg(skb, offset, len, 0); } -static int padding_length(struct tls_sw_context_rx *ctx, - struct tls_prot_info *prot, struct sk_buff *skb) +static int padding_length(struct tls_prot_info *prot, struct sk_buff *skb) { struct strp_msg *rxm = strp_msg(skb); + struct tls_msg *tlm = tls_msg(skb); int sub = 0; /* Determine zero-padding length */ if (prot->version == TLS_1_3_VERSION) { + int offset = rxm->full_len - TLS_TAG_SIZE - 1; char content_type = 0; int err; - int back = 17; while (content_type == 0) { - if (back > rxm->full_len - prot->prepend_size) + if (offset < prot->prepend_size) return -EBADMSG; - err = skb_copy_bits(skb, - rxm->offset + rxm->full_len - back, + err = skb_copy_bits(skb, rxm->offset + offset, &content_type, 1); if (err) return err; if (content_type) break; sub++; - back++; + offset--; } - ctx->control = content_type; + tlm->control = content_type; } return sub; } @@ -187,7 +186,7 @@ static void tls_decrypt_done(struct crypto_async_request *req, int err) struct strp_msg *rxm = strp_msg(skb); int pad; - pad = padding_length(ctx, prot, skb); + pad = padding_length(prot, skb); if (pad < 0) { ctx->async_wait.err = pad; tls_err_abort(skb->sk, pad); @@ -1421,6 +1420,7 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; struct strp_msg *rxm = strp_msg(skb); + struct tls_msg *tlm = tls_msg(skb); int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0; struct aead_request *aead_req; struct sk_buff *unused; @@ -1496,7 +1496,7 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, if (prot->version == TLS_1_3_VERSION || prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) memcpy(iv + iv_offset, tls_ctx->rx.iv, - crypto_aead_ivsize(ctx->aead_recv)); + prot->iv_size + prot->salt_size); else memcpy(iv + iv_offset, tls_ctx->rx.iv, prot->salt_size); @@ -1505,7 +1505,7 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, /* Prepare AAD */ tls_make_aad(aad, rxm->full_len - prot->overhead_size + prot->tail_size, - tls_ctx->rx.rec_seq, ctx->control, prot); + tls_ctx->rx.rec_seq, tlm->control, prot); /* Prepare sgin */ sg_init_table(sgin, n_sgin); @@ -1561,50 +1561,48 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, bool async) { struct tls_context *tls_ctx = tls_get_ctx(sk); - struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; struct strp_msg *rxm = strp_msg(skb); - int pad, err = 0; + struct tls_msg *tlm = tls_msg(skb); + int pad, err; - if (!ctx->decrypted) { - if (tls_ctx->rx_conf == TLS_HW) { - err = tls_device_decrypted(sk, tls_ctx, skb, rxm); - if (err < 0) - return err; - } + if (tlm->decrypted) { + *zc = false; + return 0; + } - /* Still not decrypted after tls_device */ - if (!ctx->decrypted) { - err = decrypt_internal(sk, skb, dest, NULL, chunk, zc, - async); - if (err < 0) { - if (err == -EINPROGRESS) - tls_advance_record_sn(sk, prot, - &tls_ctx->rx); - else if (err == -EBADMSG) - TLS_INC_STATS(sock_net(sk), - LINUX_MIB_TLSDECRYPTERROR); - return err; - } - } else { + if (tls_ctx->rx_conf == TLS_HW) { + err = tls_device_decrypted(sk, tls_ctx, skb, rxm); + if (err < 0) + return err; + if (err > 0) { + tlm->decrypted = 1; *zc = false; + goto decrypt_done; } + } - pad = padding_length(ctx, prot, skb); - if (pad < 0) - return pad; - - rxm->full_len -= pad; - rxm->offset += prot->prepend_size; - rxm->full_len -= prot->overhead_size; - tls_advance_record_sn(sk, prot, &tls_ctx->rx); - ctx->decrypted = 1; - ctx->saved_data_ready(sk); - } else { - *zc = false; + err = decrypt_internal(sk, skb, dest, NULL, chunk, zc, async); + if (err < 0) { + if (err == -EINPROGRESS) + tls_advance_record_sn(sk, prot, &tls_ctx->rx); + else if (err == -EBADMSG) + TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR); + return err; } - return err; +decrypt_done: + pad = padding_length(prot, skb); + if (pad < 0) + return pad; + + rxm->full_len -= pad; + rxm->offset += prot->prepend_size; + rxm->full_len -= prot->overhead_size; + tls_advance_record_sn(sk, prot, &tls_ctx->rx); + tlm->decrypted = 1; + + return 0; } int decrypt_skb(struct sock *sk, struct sk_buff *skb, @@ -1760,6 +1758,7 @@ int tls_sw_recvmsg(struct sock *sk, struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; struct sk_psock *psock; + int num_async, pending; unsigned char control = 0; ssize_t decrypted = 0; struct strp_msg *rxm; @@ -1772,8 +1771,6 @@ int tls_sw_recvmsg(struct sock *sk, bool is_kvec = iov_iter_is_kvec(&msg->msg_iter); bool is_peek = flags & MSG_PEEK; bool bpf_strp_enabled; - int num_async = 0; - int pending; flags |= nonblock; @@ -1790,17 +1787,18 @@ int tls_sw_recvmsg(struct sock *sk, if (err < 0) { tls_err_abort(sk, err); goto end; - } else { - copied = err; } + copied = err; if (len <= copied) - goto recv_end; + goto end; target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); len = len - copied; timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + decrypted = 0; + num_async = 0; while (len && (decrypted + copied < target || ctx->recv_pkt)) { bool retain_skb = false; bool zc = false; @@ -1822,26 +1820,21 @@ int tls_sw_recvmsg(struct sock *sk, } } goto recv_end; - } else { - tlm = tls_msg(skb); - if (prot->version == TLS_1_3_VERSION) - tlm->control = 0; - else - tlm->control = ctx->control; } rxm = strp_msg(skb); + tlm = tls_msg(skb); to_decrypt = rxm->full_len - prot->overhead_size; if (to_decrypt <= len && !is_kvec && !is_peek && - ctx->control == TLS_RECORD_TYPE_DATA && + tlm->control == TLS_RECORD_TYPE_DATA && prot->version != TLS_1_3_VERSION && !bpf_strp_enabled) zc = true; /* Do not use async mode if record is non-data */ - if (ctx->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled) + if (tlm->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled) async_capable = ctx->async_capable; else async_capable = false; @@ -1856,8 +1849,6 @@ int tls_sw_recvmsg(struct sock *sk, if (err == -EINPROGRESS) { async = true; num_async++; - } else if (prot->version == TLS_1_3_VERSION) { - tlm->control = ctx->control; } /* If the type of records being processed is not known yet, @@ -2005,6 +1996,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct strp_msg *rxm = NULL; struct sock *sk = sock->sk; + struct tls_msg *tlm; struct sk_buff *skb; ssize_t copied = 0; bool from_queue; @@ -2033,14 +2025,15 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, } } + rxm = strp_msg(skb); + tlm = tls_msg(skb); + /* splice does not support reading control messages */ - if (ctx->control != TLS_RECORD_TYPE_DATA) { + if (tlm->control != TLS_RECORD_TYPE_DATA) { err = -EINVAL; goto splice_read_end; } - rxm = strp_msg(skb); - chunk = min_t(unsigned int, rxm->full_len, len); copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); if (copied < 0) @@ -2084,10 +2077,10 @@ bool tls_sw_sock_is_readable(struct sock *sk) static int tls_read_size(struct strparser *strp, struct sk_buff *skb) { struct tls_context *tls_ctx = tls_get_ctx(strp->sk); - struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; char header[TLS_HEADER_SIZE + MAX_IV_SIZE]; struct strp_msg *rxm = strp_msg(skb); + struct tls_msg *tlm = tls_msg(skb); size_t cipher_overhead; size_t data_len = 0; int ret; @@ -2104,11 +2097,11 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb) /* Linearize header to local buffer */ ret = skb_copy_bits(skb, rxm->offset, header, prot->prepend_size); - if (ret < 0) goto read_failure; - ctx->control = header[0]; + tlm->decrypted = 0; + tlm->control = header[0]; data_len = ((header[4] & 0xFF) | (header[3] << 8)); @@ -2149,8 +2142,6 @@ static void tls_queue(struct strparser *strp, struct sk_buff *skb) struct tls_context *tls_ctx = tls_get_ctx(strp->sk); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); - ctx->decrypted = 0; - ctx->recv_pkt = skb; strp_pause(strp); @@ -2501,7 +2492,7 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) /* Sanity-check the sizes for stack allocations. */ if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE || - rec_seq_size > TLS_MAX_REC_SEQ_SIZE) { + rec_seq_size > TLS_MAX_REC_SEQ_SIZE || tag_size != TLS_TAG_SIZE) { rc = -EINVAL; goto free_priv; } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index e71a312faa1e..fecbd95da918 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1643,7 +1643,8 @@ static int unix_accept(struct socket *sock, struct socket *newsock, int flags, * so that no locks are necessary. */ - skb = skb_recv_datagram(sk, 0, flags&O_NONBLOCK, &err); + skb = skb_recv_datagram(sk, (flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, + &err); if (!skb) { /* This means receive shutdown. */ if (err == 0) @@ -2500,7 +2501,7 @@ static int unix_read_sock(struct sock *sk, read_descriptor_t *desc, int used, err; mutex_lock(&u->iolock); - skb = skb_recv_datagram(sk, 0, 1, &err); + skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err); mutex_unlock(&u->iolock); if (!skb) return err; diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index b17dc9745188..b14f0ed7427b 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -1732,19 +1732,16 @@ static int vmci_transport_dgram_dequeue(struct vsock_sock *vsk, int flags) { int err; - int noblock; struct vmci_datagram *dg; size_t payload_len; struct sk_buff *skb; - noblock = flags & MSG_DONTWAIT; - if (flags & MSG_OOB || flags & MSG_ERRQUEUE) return -EOPNOTSUPP; /* Retrieve the head sk_buff from the socket's receive queue. */ err = 0; - skb = skb_recv_datagram(&vsk->sk, flags, noblock, &err); + skb = skb_recv_datagram(&vsk->sk, flags, &err); if (!skb) return err; diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index 3a171828638b..6bc2ac8d8146 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -1315,8 +1315,7 @@ static int x25_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, } else { /* Now we can treat all alike */ release_sock(sk); - skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT, - flags & MSG_DONTWAIT, &rc); + skb = skb_recv_datagram(sk, flags, &rc); lock_sock(sk); if (!skb) goto out; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 2173a6729f30..9717e6f6fb31 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -40,8 +40,7 @@ include $(srctree)/scripts/Makefile.compiler # The filename Kbuild has precedence over Makefile kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) -kbuild-file := $(if $(wildcard $(kbuild-dir)/Kbuild),$(kbuild-dir)/Kbuild,$(kbuild-dir)/Makefile) -include $(kbuild-file) +include $(or $(wildcard $(kbuild-dir)/Kbuild),$(kbuild-dir)/Makefile) include $(srctree)/scripts/Makefile.lib diff --git a/scripts/Makefile.clean b/scripts/Makefile.clean index fd6175322470..74cb1c5c3658 100644 --- a/scripts/Makefile.clean +++ b/scripts/Makefile.clean @@ -12,7 +12,7 @@ include $(srctree)/scripts/Kbuild.include # The filename Kbuild has precedence over Makefile kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) -include $(if $(wildcard $(kbuild-dir)/Kbuild), $(kbuild-dir)/Kbuild, $(kbuild-dir)/Makefile) +include $(or $(wildcard $(kbuild-dir)/Kbuild),$(kbuild-dir)/Makefile) # Figure out what we need to build from the various variables # ========================================================================== diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index c593475d4a93..9f69ecdd7977 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -106,7 +106,7 @@ subdir-ym := $(addprefix $(obj)/,$(subdir-ym)) modname-multi = $(sort $(foreach m,$(multi-obj-ym),\ $(if $(filter $*.o, $(call suffix-search, $m, .o, -objs -y -m)),$(m:.o=)))) -__modname = $(if $(modname-multi),$(modname-multi),$(basetarget)) +__modname = $(or $(modname-multi),$(basetarget)) modname = $(subst $(space),:,$(__modname)) modfile = $(addprefix $(obj)/,$(__modname)) @@ -241,20 +241,16 @@ $(foreach m, $(notdir $1), \ $(addprefix $(obj)/, $(foreach s, $3, $($(m:%$(strip $2)=%$(s))))))) endef -quiet_cmd_copy = COPY $@ - cmd_copy = cp $< $@ - -# Shipped files +# Copy a file # =========================================================================== # 'cp' preserves permissions. If you use it to copy a file in read-only srctree, # the copy would be read-only as well, leading to an error when executing the # rule next time. Use 'cat' instead in order to generate a writable file. - -quiet_cmd_shipped = SHIPPED $@ -cmd_shipped = cat $< > $@ +quiet_cmd_copy = COPY $@ + cmd_copy = cat $< > $@ $(obj)/%: $(src)/%_shipped - $(call cmd,shipped) + $(call cmd,copy) # Commands useful for building a boot image # =========================================================================== @@ -431,7 +427,7 @@ MKIMAGE := $(srctree)/scripts/mkuboot.sh # SRCARCH just happens to match slightly more than ARCH (on sparc), so reduces # the number of overrides in arch makefiles UIMAGE_ARCH ?= $(SRCARCH) -UIMAGE_COMPRESSION ?= $(if $(2),$(2),none) +UIMAGE_COMPRESSION ?= $(or $(2),none) UIMAGE_OPTS-y ?= UIMAGE_TYPE ?= kernel UIMAGE_LOADADDR ?= arch_must_set_this diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 44e887cff49b..2328f9a641da 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -105,25 +105,6 @@ static void usage(void) exit(1); } -/* - * In the intended usage of this program, the stdout is redirected to .*.cmd - * files. The return value of printf() must be checked to catch any error, - * e.g. "No space left on device". - */ -static void xprintf(const char *format, ...) -{ - va_list ap; - int ret; - - va_start(ap, format); - ret = vprintf(format, ap); - if (ret < 0) { - perror("fixdep"); - exit(1); - } - va_end(ap); -} - struct item { struct item *next; unsigned int len; @@ -189,7 +170,7 @@ static void use_config(const char *m, int slen) define_config(m, slen, hash); /* Print out a dependency path from a symbol name. */ - xprintf(" $(wildcard include/config/%.*s) \\\n", slen, m); + printf(" $(wildcard include/config/%.*s) \\\n", slen, m); } /* test if s ends in sub */ @@ -318,13 +299,13 @@ static void parse_dep_file(char *m, const char *target) */ if (!saw_any_target) { saw_any_target = 1; - xprintf("source_%s := %s\n\n", - target, m); - xprintf("deps_%s := \\\n", target); + printf("source_%s := %s\n\n", + target, m); + printf("deps_%s := \\\n", target); } is_first_dep = 0; } else { - xprintf(" %s \\\n", m); + printf(" %s \\\n", m); } buf = read_file(m); @@ -347,8 +328,8 @@ static void parse_dep_file(char *m, const char *target) exit(1); } - xprintf("\n%s: $(deps_%s)\n\n", target, target); - xprintf("$(deps_%s):\n", target); + printf("\n%s: $(deps_%s)\n\n", target, target); + printf("$(deps_%s):\n", target); } int main(int argc, char *argv[]) @@ -363,11 +344,22 @@ int main(int argc, char *argv[]) target = argv[2]; cmdline = argv[3]; - xprintf("cmd_%s := %s\n\n", target, cmdline); + printf("cmd_%s := %s\n\n", target, cmdline); buf = read_file(depfile); parse_dep_file(buf, target); free(buf); + fflush(stdout); + + /* + * In the intended usage, the stdout is redirected to .*.cmd files. + * Call ferror() to catch errors such as "No space left on device". + */ + if (ferror(stdout)) { + fprintf(stderr, "fixdep: not all data was written to the output\n"); + exit(1); + } + return 0; } diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl index 7437e19ba3ac..1389db76cff3 100755 --- a/scripts/get_abi.pl +++ b/scripts/get_abi.pl @@ -327,7 +327,7 @@ sub output_rest { my @filepath = split / /, $data{$what}->{filepath}; if ($enable_lineno) { - printf "#define LINENO %s%s#%s\n\n", + printf ".. LINENO %s%s#%s\n\n", $prefix, $file[0], $data{$what}->{line_no}; } @@ -1023,7 +1023,7 @@ logic (B<--no-rst-source>). =item B<--enable-lineno> -Enable output of #define LINENO lines. +Enable output of .. LINENO lines. =item B<--debug> I<debug level> diff --git a/scripts/get_feat.pl b/scripts/get_feat.pl index 457712355676..76cfb96b59b6 100755 --- a/scripts/get_feat.pl +++ b/scripts/get_feat.pl @@ -13,6 +13,7 @@ my $man; my $debug; my $arch; my $feat; +my $enable_fname; my $basename = abs_path($0); $basename =~ s,/[^/]+$,/,; @@ -31,6 +32,7 @@ GetOptions( 'arch=s' => \$arch, 'feat=s' => \$feat, 'feature=s' => \$feat, + "enable-fname" => \$enable_fname, man => \$man ) or pod2usage(2); @@ -95,6 +97,10 @@ sub parse_feat { return if ($file =~ m,($prefix)/arch-support.txt,); return if (!($file =~ m,arch-support.txt$,)); + if ($enable_fname) { + printf ".. FILE %s\n", abs_path($file); + } + my $subsys = ""; $subsys = $2 if ( m,.*($prefix)/([^/]+).*,); @@ -580,6 +586,11 @@ Output features for a single specific feature. Changes the location of the Feature files. By default, it uses the Documentation/features directory. +=item B<--enable-fname> + +Prints the file name of the feature files. This can be used in order to +track dependencies during documentation build. + =item B<--debug> Put the script in verbose mode, useful for debugging. Can be called multiple diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 54ad86d13784..8caabddf817c 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -108,7 +108,7 @@ static bool is_ignored_symbol(const char *name, char type) /* Symbol names that begin with the following are ignored.*/ static const char * const ignored_prefixes[] = { "$", /* local symbols for ARM, MIPS, etc. */ - ".LASANPC", /* s390 kasan local symbols */ + ".L", /* local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. */ "__crc_", /* modversions */ "__efistub_", /* arm64 EFI stub namespace */ "__kvm_nvhe_", /* arm64 non-VHE KVM namespace */ diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index d3c3a61308ad..c4340c90e172 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -658,13 +658,6 @@ static char *escape_string_value(const char *in) return out; } -/* - * Kconfig configuration printer - * - * This printer is used when generating the resulting configuration after - * kconfig invocation and `defconfig' files. Unset symbol might be omitted by - * passing a non-NULL argument to the printer. - */ enum output_n { OUTPUT_N, OUTPUT_N_AS_UNSET, OUTPUT_N_NONE }; static void __print_symbol(FILE *fp, struct symbol *sym, enum output_n output_n, @@ -903,19 +896,20 @@ next: menu = menu->list; continue; } - if (menu->next) + +end_check: + if (!menu->sym && menu_is_visible(menu) && menu != &rootmenu && + menu->prompt->type == P_MENU) { + fprintf(out, "# end of %s\n", menu_get_prompt(menu)); + need_newline = true; + } + + if (menu->next) { menu = menu->next; - else while ((menu = menu->parent)) { - if (!menu->sym && menu_is_visible(menu) && - menu != &rootmenu) { - str = menu_get_prompt(menu); - fprintf(out, "# end of %s\n", str); - need_newline = true; - } - if (menu->next) { - menu = menu->next; - break; - } + } else { + menu = menu->parent; + if (menu) + goto end_check; } } fclose(out); @@ -979,6 +973,7 @@ static int conf_write_autoconf_cmd(const char *autoconf_name) fprintf(out, "\n$(deps_config): ;\n"); + fflush(out); ret = ferror(out); /* error check for all fprintf() calls */ fclose(out); if (ret) @@ -1097,6 +1092,7 @@ static int __conf_write_autoconf(const char *filename, if ((sym->flags & SYMBOL_WRITE) && sym->name) print_symbol(file, sym); + fflush(file); /* check possible errors in conf_write_heading() and print_symbol() */ ret = ferror(file); fclose(file); diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 9c084a2ba3b0..7516949bb049 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -424,7 +424,7 @@ sub get_kernel_version() { sub print_lineno { my $lineno = shift; if ($enable_lineno && defined($lineno)) { - print "#define LINENO " . $lineno . "\n"; + print ".. LINENO " . $lineno . "\n"; } } ## @@ -2478,7 +2478,7 @@ May be specified multiple times. =item -enable-lineno -Enable output of #define LINENO lines. +Enable output of .. LINENO lines. =back diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index f704034ebbe6..20f44504a644 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -50,7 +50,7 @@ gen_initcalls() { info GEN .tmp_initcalls.lds - ${PYTHON} ${srctree}/scripts/jobserver-exec \ + ${PYTHON3} ${srctree}/scripts/jobserver-exec \ ${PERL} ${srctree}/scripts/generate_initcall_order.pl \ ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS} \ > .tmp_initcalls.lds diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index d10f93aac1c8..ed9d056d2108 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -674,7 +674,7 @@ static void handle_modversion(const struct module *mod, unsigned int crc; if (sym->st_shndx == SHN_UNDEF) { - warn("EXPORT symbol \"%s\" [%s%s] version ...\n" + warn("EXPORT symbol \"%s\" [%s%s] version generation failed, symbol will not be versioned.\n" "Is \"%s\" prototyped in <asm/asm-prototypes.h>?\n", symname, mod->name, mod->is_vmlinux ? "" : ".ko", symname); diff --git a/security/Kconfig b/security/Kconfig index 1d2d71cc1f36..9b2c4925585a 100644 --- a/security/Kconfig +++ b/security/Kconfig @@ -166,7 +166,7 @@ config HARDENED_USERCOPY config HARDENED_USERCOPY_PAGESPAN bool "Refuse to copy allocations that span multiple pages" depends on HARDENED_USERCOPY - depends on EXPERT + depends on BROKEN help When a multi-page allocation is done without __GFP_COMP, hardened usercopy will reject attempts to copy it. There are, diff --git a/sound/core/pcm.c b/sound/core/pcm.c index edd9849210f2..977d54320a5c 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -970,6 +970,7 @@ int snd_pcm_attach_substream(struct snd_pcm *pcm, int stream, runtime->status->state = SNDRV_PCM_STATE_OPEN; mutex_init(&runtime->buffer_mutex); + atomic_set(&runtime->buffer_accessing, 0); substream->runtime = runtime; substream->private_data = pcm->private_data; diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index a40a35e51fad..1fc7c50ffa62 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -1906,11 +1906,9 @@ static int wait_for_avail(struct snd_pcm_substream *substream, if (avail >= runtime->twake) break; snd_pcm_stream_unlock_irq(substream); - mutex_unlock(&runtime->buffer_mutex); tout = schedule_timeout(wait_time); - mutex_lock(&runtime->buffer_mutex); snd_pcm_stream_lock_irq(substream); set_current_state(TASK_INTERRUPTIBLE); switch (runtime->status->state) { @@ -2221,7 +2219,6 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, nonblock = !!(substream->f_flags & O_NONBLOCK); - mutex_lock(&runtime->buffer_mutex); snd_pcm_stream_lock_irq(substream); err = pcm_accessible_state(runtime); if (err < 0) @@ -2276,6 +2273,10 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, err = -EINVAL; goto _end_unlock; } + if (!atomic_inc_unless_negative(&runtime->buffer_accessing)) { + err = -EBUSY; + goto _end_unlock; + } snd_pcm_stream_unlock_irq(substream); if (!is_playback) snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_CPU); @@ -2284,6 +2285,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, if (is_playback) snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE); snd_pcm_stream_lock_irq(substream); + atomic_dec(&runtime->buffer_accessing); if (err < 0) goto _end_unlock; err = pcm_accessible_state(runtime); @@ -2313,7 +2315,6 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, if (xfer > 0 && err >= 0) snd_pcm_update_state(substream, runtime); snd_pcm_stream_unlock_irq(substream); - mutex_unlock(&runtime->buffer_mutex); return xfer > 0 ? (snd_pcm_sframes_t)xfer : err; } EXPORT_SYMBOL(__snd_pcm_lib_xfer); diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 704fdc9ebf91..4adaee62ef33 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -685,6 +685,24 @@ static int snd_pcm_hw_params_choose(struct snd_pcm_substream *pcm, return 0; } +/* acquire buffer_mutex; if it's in r/w operation, return -EBUSY, otherwise + * block the further r/w operations + */ +static int snd_pcm_buffer_access_lock(struct snd_pcm_runtime *runtime) +{ + if (!atomic_dec_unless_positive(&runtime->buffer_accessing)) + return -EBUSY; + mutex_lock(&runtime->buffer_mutex); + return 0; /* keep buffer_mutex, unlocked by below */ +} + +/* release buffer_mutex and clear r/w access flag */ +static void snd_pcm_buffer_access_unlock(struct snd_pcm_runtime *runtime) +{ + mutex_unlock(&runtime->buffer_mutex); + atomic_inc(&runtime->buffer_accessing); +} + #if IS_ENABLED(CONFIG_SND_PCM_OSS) #define is_oss_stream(substream) ((substream)->oss.oss) #else @@ -695,14 +713,16 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_pcm_runtime *runtime; - int err = 0, usecs; + int err, usecs; unsigned int bits; snd_pcm_uframes_t frames; if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - mutex_lock(&runtime->buffer_mutex); + err = snd_pcm_buffer_access_lock(runtime); + if (err < 0) + return err; snd_pcm_stream_lock_irq(substream); switch (runtime->status->state) { case SNDRV_PCM_STATE_OPEN: @@ -820,7 +840,7 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, snd_pcm_lib_free_pages(substream); } unlock: - mutex_unlock(&runtime->buffer_mutex); + snd_pcm_buffer_access_unlock(runtime); return err; } @@ -865,7 +885,9 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - mutex_lock(&runtime->buffer_mutex); + result = snd_pcm_buffer_access_lock(runtime); + if (result < 0) + return result; snd_pcm_stream_lock_irq(substream); switch (runtime->status->state) { case SNDRV_PCM_STATE_SETUP: @@ -884,7 +906,7 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN); cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); unlock: - mutex_unlock(&runtime->buffer_mutex); + snd_pcm_buffer_access_unlock(runtime); return result; } @@ -1369,12 +1391,15 @@ static int snd_pcm_action_nonatomic(const struct action_ops *ops, /* Guarantee the group members won't change during non-atomic action */ down_read(&snd_pcm_link_rwsem); - mutex_lock(&substream->runtime->buffer_mutex); + res = snd_pcm_buffer_access_lock(substream->runtime); + if (res < 0) + goto unlock; if (snd_pcm_stream_linked(substream)) res = snd_pcm_action_group(ops, substream, state, false); else res = snd_pcm_action_single(ops, substream, state); - mutex_unlock(&substream->runtime->buffer_mutex); + snd_pcm_buffer_access_unlock(substream->runtime); + unlock: up_read(&snd_pcm_link_rwsem); return res; } diff --git a/sound/isa/cs423x/cs4236.c b/sound/isa/cs423x/cs4236.c index b6bdebd9ef27..10112e1bb25d 100644 --- a/sound/isa/cs423x/cs4236.c +++ b/sound/isa/cs423x/cs4236.c @@ -494,7 +494,7 @@ static int snd_cs423x_pnpbios_detect(struct pnp_dev *pdev, static int dev; int err; struct snd_card *card; - struct pnp_dev *cdev; + struct pnp_dev *cdev, *iter; char cid[PNP_ID_LEN]; if (pnp_device_is_isapnp(pdev)) @@ -510,9 +510,11 @@ static int snd_cs423x_pnpbios_detect(struct pnp_dev *pdev, strcpy(cid, pdev->id[0].id); cid[5] = '1'; cdev = NULL; - list_for_each_entry(cdev, &(pdev->protocol->devices), protocol_list) { - if (!strcmp(cdev->id[0].id, cid)) + list_for_each_entry(iter, &(pdev->protocol->devices), protocol_list) { + if (!strcmp(iter->id[0].id, cid)) { + cdev = iter; break; + } } err = snd_cs423x_card_new(&pdev->dev, dev, &card); if (err < 0) diff --git a/sound/pci/hda/patch_cs8409-tables.c b/sound/pci/hda/patch_cs8409-tables.c index 2d1fa706327b..74c50ec040d9 100644 --- a/sound/pci/hda/patch_cs8409-tables.c +++ b/sound/pci/hda/patch_cs8409-tables.c @@ -478,28 +478,29 @@ const struct snd_pci_quirk cs8409_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0A29, "Bullseye", CS8409_BULLSEYE), SND_PCI_QUIRK(0x1028, 0x0A2A, "Bullseye", CS8409_BULLSEYE), SND_PCI_QUIRK(0x1028, 0x0A2B, "Bullseye", CS8409_BULLSEYE), + SND_PCI_QUIRK(0x1028, 0x0A77, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A78, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A79, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A7A, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A7D, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A7E, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A7F, "Cyborg", CS8409_CYBORG), + SND_PCI_QUIRK(0x1028, 0x0A80, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0AB0, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0AB2, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0AB1, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0AB3, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0AB4, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0AB5, "Warlock", CS8409_WARLOCK), + SND_PCI_QUIRK(0x1028, 0x0ACF, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0AD0, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0AD1, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0AD2, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0AD3, "Dolphin", CS8409_DOLPHIN), SND_PCI_QUIRK(0x1028, 0x0AD9, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0ADA, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0ADB, "Warlock", CS8409_WARLOCK), SND_PCI_QUIRK(0x1028, 0x0ADC, "Warlock", CS8409_WARLOCK), - SND_PCI_QUIRK(0x1028, 0x0AF4, "Warlock", CS8409_WARLOCK), - SND_PCI_QUIRK(0x1028, 0x0AF5, "Warlock", CS8409_WARLOCK), - SND_PCI_QUIRK(0x1028, 0x0BB5, "Warlock N3 15 TGL-U Nuvoton EC", CS8409_WARLOCK), - SND_PCI_QUIRK(0x1028, 0x0BB6, "Warlock V3 15 TGL-U Nuvoton EC", CS8409_WARLOCK), - SND_PCI_QUIRK(0x1028, 0x0A77, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A78, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A79, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A7A, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A7D, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A7E, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A7F, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0A80, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0ADF, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0AE0, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0AE1, "Cyborg", CS8409_CYBORG), @@ -512,11 +513,30 @@ const struct snd_pci_quirk cs8409_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0AEE, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0AEF, "Cyborg", CS8409_CYBORG), SND_PCI_QUIRK(0x1028, 0x0AF0, "Cyborg", CS8409_CYBORG), - SND_PCI_QUIRK(0x1028, 0x0AD0, "Dolphin", CS8409_DOLPHIN), - SND_PCI_QUIRK(0x1028, 0x0AD1, "Dolphin", CS8409_DOLPHIN), - SND_PCI_QUIRK(0x1028, 0x0AD2, "Dolphin", CS8409_DOLPHIN), - SND_PCI_QUIRK(0x1028, 0x0AD3, "Dolphin", CS8409_DOLPHIN), - SND_PCI_QUIRK(0x1028, 0x0ACF, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0AF4, "Warlock", CS8409_WARLOCK), + SND_PCI_QUIRK(0x1028, 0x0AF5, "Warlock", CS8409_WARLOCK), + SND_PCI_QUIRK(0x1028, 0x0B92, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0B93, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0B94, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0B95, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0B96, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0B97, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0BB2, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BB3, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BB4, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BB5, "Warlock N3 15 TGL-U Nuvoton EC", CS8409_WARLOCK), + SND_PCI_QUIRK(0x1028, 0x0BB6, "Warlock V3 15 TGL-U Nuvoton EC", CS8409_WARLOCK), + SND_PCI_QUIRK(0x1028, 0x0BB8, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BB9, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0BBA, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BBB, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0BBC, "Warlock MLK", CS8409_WARLOCK_MLK), + SND_PCI_QUIRK(0x1028, 0x0BBD, "Warlock MLK Dual Mic", CS8409_WARLOCK_MLK_DUAL_MIC), + SND_PCI_QUIRK(0x1028, 0x0BD4, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0BD5, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0BD6, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0BD7, "Dolphin", CS8409_DOLPHIN), + SND_PCI_QUIRK(0x1028, 0x0BD8, "Dolphin", CS8409_DOLPHIN), {} /* terminator */ }; @@ -524,6 +544,8 @@ const struct snd_pci_quirk cs8409_fixup_tbl[] = { const struct hda_model_fixup cs8409_models[] = { { .id = CS8409_BULLSEYE, .name = "bullseye" }, { .id = CS8409_WARLOCK, .name = "warlock" }, + { .id = CS8409_WARLOCK_MLK, .name = "warlock mlk" }, + { .id = CS8409_WARLOCK_MLK_DUAL_MIC, .name = "warlock mlk dual mic" }, { .id = CS8409_CYBORG, .name = "cyborg" }, { .id = CS8409_DOLPHIN, .name = "dolphin" }, {} @@ -542,6 +564,18 @@ const struct hda_fixup cs8409_fixups[] = { .chained = true, .chain_id = CS8409_FIXUPS, }, + [CS8409_WARLOCK_MLK] = { + .type = HDA_FIXUP_PINS, + .v.pins = cs8409_cs42l42_pincfgs, + .chained = true, + .chain_id = CS8409_FIXUPS, + }, + [CS8409_WARLOCK_MLK_DUAL_MIC] = { + .type = HDA_FIXUP_PINS, + .v.pins = cs8409_cs42l42_pincfgs, + .chained = true, + .chain_id = CS8409_FIXUPS, + }, [CS8409_CYBORG] = { .type = HDA_FIXUP_PINS, .v.pins = cs8409_cs42l42_pincfgs, diff --git a/sound/pci/hda/patch_cs8409.c b/sound/pci/hda/patch_cs8409.c index aff2b5abb81e..343fabc4387d 100644 --- a/sound/pci/hda/patch_cs8409.c +++ b/sound/pci/hda/patch_cs8409.c @@ -733,6 +733,7 @@ static void cs42l42_resume(struct sub_codec *cs42l42) { 0x130A, 0x00 }, { 0x130F, 0x00 }, }; + int fsv_old, fsv_new; /* Bring CS42L42 out of Reset */ gpio_data = snd_hda_codec_read(codec, CS8409_PIN_AFG, 0, AC_VERB_GET_GPIO_DATA, 0); @@ -749,8 +750,13 @@ static void cs42l42_resume(struct sub_codec *cs42l42) /* Clear interrupts, by reading interrupt status registers */ cs8409_i2c_bulk_read(cs42l42, irq_regs, ARRAY_SIZE(irq_regs)); - if (cs42l42->full_scale_vol) - cs8409_i2c_write(cs42l42, 0x2001, 0x01); + fsv_old = cs8409_i2c_read(cs42l42, 0x2001); + if (cs42l42->full_scale_vol == CS42L42_FULL_SCALE_VOL_0DB) + fsv_new = fsv_old & ~CS42L42_FULL_SCALE_VOL_MASK; + else + fsv_new = fsv_old & CS42L42_FULL_SCALE_VOL_MASK; + if (fsv_new != fsv_old) + cs8409_i2c_write(cs42l42, 0x2001, fsv_new); /* we have to explicitly allow unsol event handling even during the * resume phase so that the jack event is processed properly @@ -906,9 +912,15 @@ static void cs8409_cs42l42_hw_init(struct hda_codec *codec) cs8409_vendor_coef_set(codec, seq_bullseye->cir, seq_bullseye->coeff); } - /* DMIC1_MO=00b, DMIC1/2_SR=1 */ - if (codec->fixup_id == CS8409_WARLOCK || codec->fixup_id == CS8409_CYBORG) - cs8409_vendor_coef_set(codec, 0x09, 0x0003); + switch (codec->fixup_id) { + case CS8409_CYBORG: + case CS8409_WARLOCK_MLK_DUAL_MIC: + /* DMIC1_MO=00b, DMIC1/2_SR=1 */ + cs8409_vendor_coef_set(codec, CS8409_DMIC_CFG, 0x0003); + break; + default: + break; + } cs42l42_resume(cs42l42); @@ -993,25 +1005,17 @@ void cs8409_cs42l42_fixups(struct hda_codec *codec, const struct hda_fixup *fix, cs8409_fix_caps(codec, CS8409_CS42L42_HP_PIN_NID); cs8409_fix_caps(codec, CS8409_CS42L42_AMIC_PIN_NID); - /* Set TIP_SENSE_EN for analog front-end of tip sense. - * Additionally set HSBIAS_SENSE_EN and Full Scale volume for some variants. - */ + /* Set HSBIAS_SENSE_EN and Full Scale volume for some variants. */ switch (codec->fixup_id) { - case CS8409_WARLOCK: + case CS8409_WARLOCK_MLK: + case CS8409_WARLOCK_MLK_DUAL_MIC: spec->scodecs[CS8409_CODEC0]->hsbias_hiz = 0x0020; - spec->scodecs[CS8409_CODEC0]->full_scale_vol = 1; - break; - case CS8409_BULLSEYE: - spec->scodecs[CS8409_CODEC0]->hsbias_hiz = 0x0020; - spec->scodecs[CS8409_CODEC0]->full_scale_vol = 0; - break; - case CS8409_CYBORG: - spec->scodecs[CS8409_CODEC0]->hsbias_hiz = 0x00a0; - spec->scodecs[CS8409_CODEC0]->full_scale_vol = 1; + spec->scodecs[CS8409_CODEC0]->full_scale_vol = CS42L42_FULL_SCALE_VOL_0DB; break; default: - spec->scodecs[CS8409_CODEC0]->hsbias_hiz = 0x0003; - spec->scodecs[CS8409_CODEC0]->full_scale_vol = 1; + spec->scodecs[CS8409_CODEC0]->hsbias_hiz = 0x0020; + spec->scodecs[CS8409_CODEC0]->full_scale_vol = + CS42L42_FULL_SCALE_VOL_MINUS6DB; break; } @@ -1222,6 +1226,9 @@ void dolphin_fixups(struct hda_codec *codec, const struct hda_fixup *fix, int ac cs8409_fix_caps(codec, DOLPHIN_LO_PIN_NID); cs8409_fix_caps(codec, DOLPHIN_AMIC_PIN_NID); + spec->scodecs[CS8409_CODEC0]->full_scale_vol = CS42L42_FULL_SCALE_VOL_MINUS6DB; + spec->scodecs[CS8409_CODEC1]->full_scale_vol = CS42L42_FULL_SCALE_VOL_MINUS6DB; + break; case HDA_FIXUP_ACT_PROBE: /* Fix Sample Rate to 48kHz */ diff --git a/sound/pci/hda/patch_cs8409.h b/sound/pci/hda/patch_cs8409.h index d0b725c7285b..7df46bd8d2da 100644 --- a/sound/pci/hda/patch_cs8409.h +++ b/sound/pci/hda/patch_cs8409.h @@ -235,6 +235,9 @@ enum cs8409_coefficient_index_registers { #define CS42L42_I2C_SLEEP_US (2000) #define CS42L42_PDN_TIMEOUT_US (250000) #define CS42L42_PDN_SLEEP_US (2000) +#define CS42L42_FULL_SCALE_VOL_MASK (2) +#define CS42L42_FULL_SCALE_VOL_0DB (1) +#define CS42L42_FULL_SCALE_VOL_MINUS6DB (0) /* Dell BULLSEYE / WARLOCK / CYBORG Specific Definitions */ @@ -264,6 +267,8 @@ enum cs8409_coefficient_index_registers { enum { CS8409_BULLSEYE, CS8409_WARLOCK, + CS8409_WARLOCK_MLK, + CS8409_WARLOCK_MLK_DUAL_MIC, CS8409_CYBORG, CS8409_FIXUPS, CS8409_DOLPHIN, diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index c85ed7bc121e..3e086eebf88d 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -1625,6 +1625,7 @@ static void hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, struct hda_codec *codec = per_pin->codec; struct hdmi_spec *spec = codec->spec; struct hdmi_eld *eld = &spec->temp_eld; + struct device *dev = hda_codec_dev(codec); hda_nid_t pin_nid = per_pin->pin_nid; int dev_id = per_pin->dev_id; /* @@ -1638,8 +1639,13 @@ static void hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin, int present; int ret; +#ifdef CONFIG_PM + if (dev->power.runtime_status == RPM_SUSPENDING) + return; +#endif + ret = snd_hda_power_up_pm(codec); - if (ret < 0 && pm_runtime_suspended(hda_codec_dev(codec))) + if (ret < 0 && pm_runtime_suspended(dev)) goto out; present = snd_hda_jack_pin_sense(codec, pin_nid, dev_id); diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index c78f16944f43..4e12af24b4d3 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -3617,8 +3617,8 @@ static void alc256_shutup(struct hda_codec *codec) /* If disable 3k pulldown control for alc257, the Mic detection will not work correctly * when booting with headset plugged. So skip setting it for the codec alc257 */ - if (spec->codec_variant != ALC269_TYPE_ALC257 && - spec->codec_variant != ALC269_TYPE_ALC256) + if (codec->core.vendor_id != 0x10ec0236 && + codec->core.vendor_id != 0x10ec0257) alc_update_coef_idx(codec, 0x46, 0, 3 << 12); if (!spec->no_shutup_pins) @@ -7006,6 +7006,7 @@ enum { ALC287_FIXUP_LEGION_16ACHG6, ALC287_FIXUP_CS35L41_I2C_2, ALC245_FIXUP_CS35L41_SPI_2, + ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED, ALC245_FIXUP_CS35L41_SPI_4, ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED, ALC285_FIXUP_HP_SPEAKERS_MICMUTE_LED, @@ -8771,6 +8772,12 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = cs35l41_fixup_spi_two, }, + [ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = cs35l41_fixup_spi_two, + .chained = true, + .chain_id = ALC285_FIXUP_HP_GPIO_LED, + }, [ALC245_FIXUP_CS35L41_SPI_4] = { .type = HDA_FIXUP_FUNC, .v.func = cs35l41_fixup_spi_four, @@ -9026,7 +9033,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x89ac, "HP EliteBook 640 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89ae, "HP EliteBook 650 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89c3, "Zbook Studio G9", ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED), - SND_PCI_QUIRK(0x103c, 0x89c6, "Zbook Fury 17 G9", ALC245_FIXUP_CS35L41_SPI_2), + SND_PCI_QUIRK(0x103c, 0x89c6, "Zbook Fury 17 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89ca, "HP", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x103f, "ASUS TX300", ALC282_FIXUP_ASUS_TX300), @@ -11140,6 +11147,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc051, "Samsung R720", ALC662_FIXUP_IDEAPAD), SND_PCI_QUIRK(0x14cd, 0x5003, "USI", ALC662_FIXUP_USI_HEADSET_MODE), SND_PCI_QUIRK(0x17aa, 0x1036, "Lenovo P520", ALC662_FIXUP_LENOVO_MULTI_CODECS), + SND_PCI_QUIRK(0x17aa, 0x1057, "Lenovo P360", ALC897_FIXUP_HEADSET_MIC_PIN), SND_PCI_QUIRK(0x17aa, 0x32ca, "Lenovo ThinkCentre M80", ALC897_FIXUP_HEADSET_MIC_PIN), SND_PCI_QUIRK(0x17aa, 0x32cb, "Lenovo ThinkCentre M70", ALC897_FIXUP_HEADSET_MIC_PIN), SND_PCI_QUIRK(0x17aa, 0x32cf, "Lenovo ThinkCentre M950", ALC897_FIXUP_HEADSET_MIC_PIN), diff --git a/sound/soc/codecs/mt6358.c b/sound/soc/codecs/mt6358.c index 9b263a9a669d..4c7b5d940799 100644 --- a/sound/soc/codecs/mt6358.c +++ b/sound/soc/codecs/mt6358.c @@ -107,6 +107,7 @@ int mt6358_set_mtkaif_protocol(struct snd_soc_component *cmpnt, priv->mtkaif_protocol = mtkaif_protocol; return 0; } +EXPORT_SYMBOL_GPL(mt6358_set_mtkaif_protocol); static void playback_gpio_set(struct mt6358_priv *priv) { @@ -273,6 +274,7 @@ int mt6358_mtkaif_calibration_enable(struct snd_soc_component *cmpnt) 1 << RG_AUD_PAD_TOP_DAT_MISO_LOOPBACK_SFT); return 0; } +EXPORT_SYMBOL_GPL(mt6358_mtkaif_calibration_enable); int mt6358_mtkaif_calibration_disable(struct snd_soc_component *cmpnt) { @@ -296,6 +298,7 @@ int mt6358_mtkaif_calibration_disable(struct snd_soc_component *cmpnt) capture_gpio_reset(priv); return 0; } +EXPORT_SYMBOL_GPL(mt6358_mtkaif_calibration_disable); int mt6358_set_mtkaif_calibration_phase(struct snd_soc_component *cmpnt, int phase_1, int phase_2) @@ -310,6 +313,7 @@ int mt6358_set_mtkaif_calibration_phase(struct snd_soc_component *cmpnt, phase_2 << RG_AUD_PAD_TOP_PHASE_MODE2_SFT); return 0; } +EXPORT_SYMBOL_GPL(mt6358_set_mtkaif_calibration_phase); /* dl pga gain */ enum { diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index 370bc790c6ba..d9a0d4768c4d 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -462,11 +462,9 @@ static int hp_jack_event(struct notifier_block *nb, unsigned long event, if (event & SND_JACK_HEADPHONE) /* Disable speaker if headphone is plugged in */ - snd_soc_dapm_disable_pin(dapm, "Ext Spk"); + return snd_soc_dapm_disable_pin(dapm, "Ext Spk"); else - snd_soc_dapm_enable_pin(dapm, "Ext Spk"); - - return 0; + return snd_soc_dapm_enable_pin(dapm, "Ext Spk"); } static struct notifier_block hp_jack_nb = { @@ -481,11 +479,9 @@ static int mic_jack_event(struct notifier_block *nb, unsigned long event, if (event & SND_JACK_MICROPHONE) /* Disable dmic if microphone is plugged in */ - snd_soc_dapm_disable_pin(dapm, "DMIC"); + return snd_soc_dapm_disable_pin(dapm, "DMIC"); else - snd_soc_dapm_enable_pin(dapm, "DMIC"); - - return 0; + return snd_soc_dapm_enable_pin(dapm, "DMIC"); } static struct notifier_block mic_jack_nb = { diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index d3b710406941..98700e75b82a 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -469,14 +469,14 @@ static int rockchip_i2s_tdm_set_fmt(struct snd_soc_dai *cpu_dai, txcr_val = I2S_TXCR_IBM_NORMAL; rxcr_val = I2S_RXCR_IBM_NORMAL; break; - case SND_SOC_DAIFMT_DSP_A: /* PCM no delay mode */ - txcr_val = I2S_TXCR_TFS_PCM; - rxcr_val = I2S_RXCR_TFS_PCM; - break; - case SND_SOC_DAIFMT_DSP_B: /* PCM delay 1 mode */ + case SND_SOC_DAIFMT_DSP_A: /* PCM delay 1 mode */ txcr_val = I2S_TXCR_TFS_PCM | I2S_TXCR_PBM_MODE(1); rxcr_val = I2S_RXCR_TFS_PCM | I2S_RXCR_PBM_MODE(1); break; + case SND_SOC_DAIFMT_DSP_B: /* PCM no delay mode */ + txcr_val = I2S_TXCR_TFS_PCM; + rxcr_val = I2S_RXCR_TFS_PCM; + break; default: ret = -EINVAL; goto err_pm_put; diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index b53f216d4ecc..172419392b33 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -84,6 +84,7 @@ if SND_SOC_SOF_PCI config SND_SOC_SOF_MERRIFIELD tristate "SOF support for Tangier/Merrifield" default SND_SOC_SOF_PCI + select SND_SOC_SOF_PCI_DEV select SND_SOC_SOF_INTEL_ATOM_HIFI_EP help This adds support for Sound Open Firmware for Intel(R) platforms diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 490d489d0ee8..c1b6ddc02d2f 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -419,6 +419,16 @@ struct kvm_arm_copy_mte_tags { #define KVM_PSCI_RET_INVAL PSCI_RET_INVALID_PARAMS #define KVM_PSCI_RET_DENIED PSCI_RET_DENIED +/* arm64-specific kvm_run::system_event flags */ +/* + * Reset caused by a PSCI v1.1 SYSTEM_RESET2 call. + * Valid only when the system event has a type of KVM_SYSTEM_EVENT_RESET. + */ +#define KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2 (1ULL << 0) + +/* run->fail_entry.hardware_entry_failure_reason codes. */ +#define KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED (1ULL << 0) + #endif #endif /* __ARM_KVM_H__ */ diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 3edf05e98e58..73e643ae94b6 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -388,6 +388,7 @@ #define X86_FEATURE_TSXLDTRK (18*32+16) /* TSX Suspend Load Address Tracking */ #define X86_FEATURE_PCONFIG (18*32+18) /* Intel PCONFIG */ #define X86_FEATURE_ARCH_LBR (18*32+19) /* Intel ARCH LBR */ +#define X86_FEATURE_IBT (18*32+20) /* Indirect Branch Tracking */ #define X86_FEATURE_AMX_BF16 (18*32+22) /* AMX bf16 Support */ #define X86_FEATURE_AVX512_FP16 (18*32+23) /* AVX512 FP16 */ #define X86_FEATURE_AMX_TILE (18*32+24) /* AMX tile Support */ diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 0e7f303542bf..0eb90d21049e 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -205,6 +205,8 @@ #define RTIT_CTL_DISRETC BIT(11) #define RTIT_CTL_PTW_EN BIT(12) #define RTIT_CTL_BRANCH_EN BIT(13) +#define RTIT_CTL_EVENT_EN BIT(31) +#define RTIT_CTL_NOTNT BIT_ULL(55) #define RTIT_CTL_MTC_RANGE_OFFSET 14 #define RTIT_CTL_MTC_RANGE (0x0full << RTIT_CTL_MTC_RANGE_OFFSET) #define RTIT_CTL_CYC_THRESH_OFFSET 19 @@ -360,11 +362,29 @@ #define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c #define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d - #define MSR_CORE_PERF_LIMIT_REASONS 0x00000690 #define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0 #define MSR_RING_PERF_LIMIT_REASONS 0x000006B1 +/* Control-flow Enforcement Technology MSRs */ +#define MSR_IA32_U_CET 0x000006a0 /* user mode cet */ +#define MSR_IA32_S_CET 0x000006a2 /* kernel mode cet */ +#define CET_SHSTK_EN BIT_ULL(0) +#define CET_WRSS_EN BIT_ULL(1) +#define CET_ENDBR_EN BIT_ULL(2) +#define CET_LEG_IW_EN BIT_ULL(3) +#define CET_NO_TRACK_EN BIT_ULL(4) +#define CET_SUPPRESS_DISABLE BIT_ULL(5) +#define CET_RESERVED (BIT_ULL(6) | BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) +#define CET_SUPPRESS BIT_ULL(10) +#define CET_WAIT_ENDBR BIT_ULL(11) + +#define MSR_IA32_PL0_SSP 0x000006a4 /* ring-0 shadow stack pointer */ +#define MSR_IA32_PL1_SSP 0x000006a5 /* ring-1 shadow stack pointer */ +#define MSR_IA32_PL2_SSP 0x000006a6 /* ring-2 shadow stack pointer */ +#define MSR_IA32_PL3_SSP 0x000006a7 /* ring-3 shadow stack pointer */ +#define MSR_IA32_INT_SSP_TAB 0x000006a8 /* exception shadow stack table */ + /* Hardware P state interface */ #define MSR_PPERF 0x0000064e #define MSR_PERF_LIMIT_REASONS 0x0000064f diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 9800f966fd51..c6d2c77d0252 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -74,7 +74,7 @@ CFLAGS += -O2 CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers CFLAGS += $(filter-out -Wswitch-enum -Wnested-externs,$(EXTRA_WARNINGS)) CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \ - -I$(if $(OUTPUT),$(OUTPUT),.) \ + -I$(or $(OUTPUT),.) \ -I$(LIBBPF_INCLUDE) \ -I$(srctree)/kernel/bpf/ \ -I$(srctree)/tools/include \ @@ -180,7 +180,7 @@ endif $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF_BOOTSTRAP) $(QUIET_CLANG)$(CLANG) \ - -I$(if $(OUTPUT),$(OUTPUT),.) \ + -I$(or $(OUTPUT),.) \ -I$(srctree)/tools/include/uapi/ \ -I$(LIBBPF_BOOTSTRAP_INCLUDE) \ -g -O2 -Wall -target bpf -c $< -o $@ diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 91af2850b505..7678af364793 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -828,8 +828,10 @@ codegen_maps_skeleton(struct bpf_object *obj, size_t map_cnt, bool mmaped) s->map_cnt = %zu; \n\ s->map_skel_sz = sizeof(*s->maps); \n\ s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz);\n\ - if (!s->maps) \n\ + if (!s->maps) { \n\ + err = -ENOMEM; \n\ goto err; \n\ + } \n\ ", map_cnt ); @@ -870,8 +872,10 @@ codegen_progs_skeleton(struct bpf_object *obj, size_t prog_cnt, bool populate_li s->prog_cnt = %zu; \n\ s->prog_skel_sz = sizeof(*s->progs); \n\ s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz);\n\ - if (!s->progs) \n\ + if (!s->progs) { \n\ + err = -ENOMEM; \n\ goto err; \n\ + } \n\ ", prog_cnt ); @@ -1182,10 +1186,13 @@ static int do_skeleton(int argc, char **argv) %1$s__create_skeleton(struct %1$s *obj) \n\ { \n\ struct bpf_object_skeleton *s; \n\ + int err; \n\ \n\ s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s));\n\ - if (!s) \n\ + if (!s) { \n\ + err = -ENOMEM; \n\ goto err; \n\ + } \n\ \n\ s->sz = sizeof(*s); \n\ s->name = \"%1$s\"; \n\ @@ -1206,7 +1213,7 @@ static int do_skeleton(int argc, char **argv) return 0; \n\ err: \n\ bpf_object__destroy_skeleton(s); \n\ - return -ENOMEM; \n\ + return err; \n\ } \n\ \n\ static inline const void *%2$s__elf_bytes(size_t *sz) \n\ @@ -1466,12 +1473,12 @@ static int do_subskeleton(int argc, char **argv) \n\ obj = (struct %1$s *)calloc(1, sizeof(*obj)); \n\ if (!obj) { \n\ - errno = ENOMEM; \n\ + err = -ENOMEM; \n\ goto err; \n\ } \n\ s = (struct bpf_object_subskeleton *)calloc(1, sizeof(*s));\n\ if (!s) { \n\ - errno = ENOMEM; \n\ + err = -ENOMEM; \n\ goto err; \n\ } \n\ s->sz = sizeof(*s); \n\ @@ -1483,7 +1490,7 @@ static int do_subskeleton(int argc, char **argv) s->var_cnt = %2$d; \n\ s->vars = (struct bpf_var_skeleton *)calloc(%2$d, sizeof(*s->vars));\n\ if (!s->vars) { \n\ - errno = ENOMEM; \n\ + err = -ENOMEM; \n\ goto err; \n\ } \n\ ", @@ -1538,6 +1545,7 @@ static int do_subskeleton(int argc, char **argv) return obj; \n\ err: \n\ %1$s__destroy(obj); \n\ + errno = -err; \n\ return NULL; \n\ } \n\ \n\ diff --git a/tools/build/Makefile b/tools/build/Makefile index 6f11e6fc9ffe..17cdf01e29a0 100644 --- a/tools/build/Makefile +++ b/tools/build/Makefile @@ -36,7 +36,7 @@ TMP_O := $(if $(OUTPUT),$(OUTPUT)feature/,./) clean: $(call QUIET_CLEAN, fixdep) - $(Q)find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete + $(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete $(Q)rm -f $(OUTPUT)fixdep $(call QUIET_CLEAN, feature-detect) ifneq ($(wildcard $(TMP_O)),) diff --git a/tools/counter/Makefile b/tools/counter/Makefile index 5ebc195fd9c0..8843f0fa6119 100644 --- a/tools/counter/Makefile +++ b/tools/counter/Makefile @@ -40,7 +40,7 @@ $(OUTPUT)counter_example: $(COUNTER_EXAMPLE) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/linux/counter.h - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/gpio/Makefile b/tools/gpio/Makefile index 440434027557..d29c9c49e251 100644 --- a/tools/gpio/Makefile +++ b/tools/gpio/Makefile @@ -78,7 +78,7 @@ $(OUTPUT)gpio-watch: $(GPIO_WATCH_IN) clean: rm -f $(ALL_PROGRAMS) rm -f $(OUTPUT)include/linux/gpio.h - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/hv/Makefile b/tools/hv/Makefile index b57143d9459c..fe770e679ae8 100644 --- a/tools/hv/Makefile +++ b/tools/hv/Makefile @@ -47,7 +47,7 @@ $(OUTPUT)hv_fcopy_daemon: $(HV_FCOPY_DAEMON_IN) clean: rm -f $(ALL_PROGRAMS) - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(sbindir); \ diff --git a/tools/iio/Makefile b/tools/iio/Makefile index 5d12ac4e7f8f..fa720f062229 100644 --- a/tools/iio/Makefile +++ b/tools/iio/Makefile @@ -58,7 +58,7 @@ $(OUTPUT)iio_generic_buffer: $(IIO_GENERIC_BUFFER_IN) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/linux/iio - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h index 1567a3294c3d..6c1aa92a92e4 100644 --- a/tools/include/uapi/asm-generic/mman-common.h +++ b/tools/include/uapi/asm-generic/mman-common.h @@ -75,6 +75,8 @@ #define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ #define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ +#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/tools/include/uapi/drm/i915_drm.h b/tools/include/uapi/drm/i915_drm.h index 914ebd9290e5..05c3642aaece 100644 --- a/tools/include/uapi/drm/i915_drm.h +++ b/tools/include/uapi/drm/i915_drm.h @@ -1118,10 +1118,16 @@ struct drm_i915_gem_exec_object2 { /** * When the EXEC_OBJECT_PINNED flag is specified this is populated by * the user with the GTT offset at which this object will be pinned. + * * When the I915_EXEC_NO_RELOC flag is specified this must contain the * presumed_offset of the object. + * * During execbuffer2 the kernel populates it with the value of the * current GTT offset of the object, for future presumed_offset writes. + * + * See struct drm_i915_gem_create_ext for the rules when dealing with + * alignment restrictions with I915_MEMORY_CLASS_DEVICE, on devices with + * minimum page sizes, like DG2. */ __u64 offset; @@ -3144,11 +3150,40 @@ struct drm_i915_gem_create_ext { * * The (page-aligned) allocated size for the object will be returned. * - * Note that for some devices we have might have further minimum - * page-size restrictions(larger than 4K), like for device local-memory. - * However in general the final size here should always reflect any - * rounding up, if for example using the I915_GEM_CREATE_EXT_MEMORY_REGIONS - * extension to place the object in device local-memory. + * + * DG2 64K min page size implications: + * + * On discrete platforms, starting from DG2, we have to contend with GTT + * page size restrictions when dealing with I915_MEMORY_CLASS_DEVICE + * objects. Specifically the hardware only supports 64K or larger GTT + * page sizes for such memory. The kernel will already ensure that all + * I915_MEMORY_CLASS_DEVICE memory is allocated using 64K or larger page + * sizes underneath. + * + * Note that the returned size here will always reflect any required + * rounding up done by the kernel, i.e 4K will now become 64K on devices + * such as DG2. + * + * Special DG2 GTT address alignment requirement: + * + * The GTT alignment will also need to be at least 2M for such objects. + * + * Note that due to how the hardware implements 64K GTT page support, we + * have some further complications: + * + * 1) The entire PDE (which covers a 2MB virtual address range), must + * contain only 64K PTEs, i.e mixing 4K and 64K PTEs in the same + * PDE is forbidden by the hardware. + * + * 2) We still need to support 4K PTEs for I915_MEMORY_CLASS_SYSTEM + * objects. + * + * To keep things simple for userland, we mandate that any GTT mappings + * must be aligned to and rounded up to 2MB. The kernel will internally + * pad them out to the next 2MB boundary. As this only wastes virtual + * address space and avoids userland having to copy any needlessly + * complicated PDE sharing scheme (coloring) and only affects DG2, this + * is deemed to be a good compromise. */ __u64 size; /** diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index bbc6b7c2dc1b..91a6fe4e02c0 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -562,9 +562,12 @@ struct kvm_s390_mem_op { __u32 op; /* type of operation */ __u64 buf; /* buffer in userspace */ union { - __u8 ar; /* the access register number */ + struct { + __u8 ar; /* the access register number */ + __u8 key; /* access key, ignored if flag unset */ + }; __u32 sida_offset; /* offset into the sida */ - __u8 reserved[32]; /* should be set to 0 */ + __u8 reserved[32]; /* ignored */ }; }; /* types for kvm_s390_mem_op->op */ @@ -572,9 +575,12 @@ struct kvm_s390_mem_op { #define KVM_S390_MEMOP_LOGICAL_WRITE 1 #define KVM_S390_MEMOP_SIDA_READ 2 #define KVM_S390_MEMOP_SIDA_WRITE 3 +#define KVM_S390_MEMOP_ABSOLUTE_READ 4 +#define KVM_S390_MEMOP_ABSOLUTE_WRITE 5 /* flags for kvm_s390_mem_op->flags */ #define KVM_S390_MEMOP_F_CHECK_ONLY (1ULL << 0) #define KVM_S390_MEMOP_F_INJECT_EXCEPTION (1ULL << 1) +#define KVM_S390_MEMOP_F_SKEY_PROTECTION (1ULL << 2) /* for KVM_INTERRUPT */ struct kvm_interrupt { @@ -1137,6 +1143,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_PPC_AIL_MODE_3 210 #define KVM_CAP_S390_MEM_OP_EXTENSION 211 #define KVM_CAP_PMU_CAPABILITY 212 +#define KVM_CAP_DISABLE_QUIRKS2 213 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/tools/lib/api/Makefile b/tools/lib/api/Makefile index a13e9c7f1fc5..e21e1b40b525 100644 --- a/tools/lib/api/Makefile +++ b/tools/lib/api/Makefile @@ -60,7 +60,7 @@ $(LIBFILE): $(API_IN) clean: $(call QUIET_CLEAN, libapi) $(RM) $(LIBFILE); \ - find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM) + find $(or $(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM) FORCE: diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index b4fbe8bed555..64741c55b8e3 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -60,7 +60,7 @@ ifndef VERBOSE VERBOSE = 0 endif -INCLUDES = -I$(if $(OUTPUT),$(OUTPUT),.) \ +INCLUDES = -I$(or $(OUTPUT),.) \ -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi export prefix libdir src obj diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile index 08fe6e3c4089..21df023a2103 100644 --- a/tools/lib/perf/Makefile +++ b/tools/lib/perf/Makefile @@ -153,7 +153,7 @@ $(TESTS_STATIC): $(TESTS_IN) $(LIBPERF_A) $(LIBAPI) $(QUIET_LINK)$(CC) -o $@ $^ $(TESTS_SHARED): $(TESTS_IN) $(LIBAPI) - $(QUIET_LINK)$(CC) -o $@ -L$(if $(OUTPUT),$(OUTPUT),.) $^ -lperf + $(QUIET_LINK)$(CC) -o $@ -L$(or $(OUTPUT),.) $^ -lperf make-tests: libs $(TESTS_SHARED) $(TESTS_STATIC) diff --git a/tools/lib/perf/cpumap.c b/tools/lib/perf/cpumap.c index ee66760f1e63..384d5e076ee4 100644 --- a/tools/lib/perf/cpumap.c +++ b/tools/lib/perf/cpumap.c @@ -319,6 +319,26 @@ struct perf_cpu perf_cpu_map__max(struct perf_cpu_map *map) return map->nr > 0 ? map->map[map->nr - 1] : result; } +/** Is 'b' a subset of 'a'. */ +bool perf_cpu_map__is_subset(const struct perf_cpu_map *a, const struct perf_cpu_map *b) +{ + if (a == b || !b) + return true; + if (!a || b->nr > a->nr) + return false; + + for (int i = 0, j = 0; i < a->nr; i++) { + if (a->map[i].cpu > b->map[j].cpu) + return false; + if (a->map[i].cpu == b->map[j].cpu) { + j++; + if (j == b->nr) + return true; + } + } + return false; +} + /* * Merge two cpumaps * @@ -335,17 +355,12 @@ struct perf_cpu_map *perf_cpu_map__merge(struct perf_cpu_map *orig, int i, j, k; struct perf_cpu_map *merged; - if (!orig && !other) - return NULL; - if (!orig) { - perf_cpu_map__get(other); - return other; - } - if (!other) - return orig; - if (orig->nr == other->nr && - !memcmp(orig->map, other->map, orig->nr * sizeof(struct perf_cpu))) + if (perf_cpu_map__is_subset(orig, other)) return orig; + if (perf_cpu_map__is_subset(other, orig)) { + perf_cpu_map__put(orig); + return perf_cpu_map__get(other); + } tmp_len = orig->nr + other->nr; tmp_cpus = malloc(tmp_len * sizeof(struct perf_cpu)); diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c index 9a770bfdc804..1b15ba13c477 100644 --- a/tools/lib/perf/evlist.c +++ b/tools/lib/perf/evlist.c @@ -41,10 +41,10 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, */ if (!evsel->own_cpus || evlist->has_user_cpus) { perf_cpu_map__put(evsel->cpus); - evsel->cpus = perf_cpu_map__get(evlist->cpus); - } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->cpus)) { + evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus); + } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->user_requested_cpus)) { perf_cpu_map__put(evsel->cpus); - evsel->cpus = perf_cpu_map__get(evlist->cpus); + evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus); } else if (evsel->cpus != evsel->own_cpus) { perf_cpu_map__put(evsel->cpus); evsel->cpus = perf_cpu_map__get(evsel->own_cpus); @@ -123,10 +123,10 @@ static void perf_evlist__purge(struct perf_evlist *evlist) void perf_evlist__exit(struct perf_evlist *evlist) { - perf_cpu_map__put(evlist->cpus); + perf_cpu_map__put(evlist->user_requested_cpus); perf_cpu_map__put(evlist->all_cpus); perf_thread_map__put(evlist->threads); - evlist->cpus = NULL; + evlist->user_requested_cpus = NULL; evlist->all_cpus = NULL; evlist->threads = NULL; fdarray__exit(&evlist->pollfd); @@ -155,9 +155,9 @@ void perf_evlist__set_maps(struct perf_evlist *evlist, * original reference count of 1. If that is not the case it is up to * the caller to increase the reference count. */ - if (cpus != evlist->cpus) { - perf_cpu_map__put(evlist->cpus); - evlist->cpus = perf_cpu_map__get(cpus); + if (cpus != evlist->user_requested_cpus) { + perf_cpu_map__put(evlist->user_requested_cpus); + evlist->user_requested_cpus = perf_cpu_map__get(cpus); } if (threads != evlist->threads) { @@ -294,7 +294,7 @@ add: int perf_evlist__alloc_pollfd(struct perf_evlist *evlist) { - int nr_cpus = perf_cpu_map__nr(evlist->cpus); + int nr_cpus = perf_cpu_map__nr(evlist->user_requested_cpus); int nr_threads = perf_thread_map__nr(evlist->threads); int nfds = 0; struct perf_evsel *evsel; @@ -426,7 +426,7 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, int idx, struct perf_mmap_param *mp, int cpu_idx, int thread, int *_output, int *_output_overwrite) { - struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->cpus, cpu_idx); + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->user_requested_cpus, cpu_idx); struct perf_evsel *evsel; int revent; @@ -536,7 +536,7 @@ mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, struct perf_mmap_param *mp) { int nr_threads = perf_thread_map__nr(evlist->threads); - int nr_cpus = perf_cpu_map__nr(evlist->cpus); + int nr_cpus = perf_cpu_map__nr(evlist->user_requested_cpus); int cpu, thread; for (cpu = 0; cpu < nr_cpus; cpu++) { @@ -564,8 +564,8 @@ static int perf_evlist__nr_mmaps(struct perf_evlist *evlist) { int nr_mmaps; - nr_mmaps = perf_cpu_map__nr(evlist->cpus); - if (perf_cpu_map__empty(evlist->cpus)) + nr_mmaps = perf_cpu_map__nr(evlist->user_requested_cpus); + if (perf_cpu_map__empty(evlist->user_requested_cpus)) nr_mmaps = perf_thread_map__nr(evlist->threads); return nr_mmaps; @@ -576,7 +576,7 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist, struct perf_mmap_param *mp) { struct perf_evsel *evsel; - const struct perf_cpu_map *cpus = evlist->cpus; + const struct perf_cpu_map *cpus = evlist->user_requested_cpus; const struct perf_thread_map *threads = evlist->threads; if (!ops || !ops->get || !ops->mmap) diff --git a/tools/lib/perf/include/internal/cpumap.h b/tools/lib/perf/include/internal/cpumap.h index 1973a18c096b..35dd29642296 100644 --- a/tools/lib/perf/include/internal/cpumap.h +++ b/tools/lib/perf/include/internal/cpumap.h @@ -25,5 +25,6 @@ struct perf_cpu_map { #endif int perf_cpu_map__idx(const struct perf_cpu_map *cpus, struct perf_cpu cpu); +bool perf_cpu_map__is_subset(const struct perf_cpu_map *a, const struct perf_cpu_map *b); #endif /* __LIBPERF_INTERNAL_CPUMAP_H */ diff --git a/tools/lib/perf/include/internal/evlist.h b/tools/lib/perf/include/internal/evlist.h index 4cefade540bd..e3e64f37db7b 100644 --- a/tools/lib/perf/include/internal/evlist.h +++ b/tools/lib/perf/include/internal/evlist.h @@ -19,7 +19,12 @@ struct perf_evlist { int nr_entries; int nr_groups; bool has_user_cpus; - struct perf_cpu_map *cpus; + /** + * The cpus passed from the command line or all online CPUs by + * default. + */ + struct perf_cpu_map *user_requested_cpus; + /** The union of all evsel cpu maps. */ struct perf_cpu_map *all_cpus; struct perf_thread_map *threads; int nr_mmaps; diff --git a/tools/lib/subcmd/Makefile b/tools/lib/subcmd/Makefile index 1c777a72bb39..8f1a09cdfd17 100644 --- a/tools/lib/subcmd/Makefile +++ b/tools/lib/subcmd/Makefile @@ -63,7 +63,7 @@ $(LIBFILE): $(SUBCMD_IN) clean: $(call QUIET_CLEAN, libsubcmd) $(RM) $(LIBFILE); \ - find $(if $(OUTPUT),$(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM) + find $(or $(OUTPUT),.) -name \*.o -or -name \*.o.cmd -or -name \*.o.d | xargs $(RM) FORCE: diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 92ce4fce7bc7..0dbd397f319d 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -13,7 +13,7 @@ srctree := $(patsubst %/,%,$(dir $(srctree))) endif SUBCMD_SRCDIR = $(srctree)/tools/lib/subcmd/ -LIBSUBCMD_OUTPUT = $(if $(OUTPUT),$(OUTPUT),$(CURDIR)/) +LIBSUBCMD_OUTPUT = $(or $(OUTPUT),$(CURDIR)/) LIBSUBCMD = $(LIBSUBCMD_OUTPUT)libsubcmd.a OBJTOOL := $(OUTPUT)objtool diff --git a/tools/pci/Makefile b/tools/pci/Makefile index 4b95a5176355..57744778b518 100644 --- a/tools/pci/Makefile +++ b/tools/pci/Makefile @@ -42,7 +42,7 @@ $(OUTPUT)pcitest: $(PCITEST_IN) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/ - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 9c935f86d172..69473a836bae 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -691,9 +691,8 @@ $(OUTPUT)common-cmds.h: $(wildcard Documentation/perf-*.txt) $(SCRIPTS) : % : %.sh $(QUIET_GEN)$(INSTALL) '$@.sh' '$(OUTPUT)$@' -$(OUTPUT)PERF-VERSION-FILE: ../../.git/HEAD ../../.git/ORIG_HEAD +$(OUTPUT)PERF-VERSION-FILE: .FORCE-PERF-VERSION-FILE $(Q)$(SHELL_PATH) util/PERF-VERSION-GEN $(OUTPUT) - $(Q)touch $(OUTPUT)PERF-VERSION-FILE # These can record PERF_VERSION perf.spec $(SCRIPTS) \ @@ -724,7 +723,7 @@ endif # get relative building directory (to $(OUTPUT)) # and '.' if it's $(OUTPUT) itself __build-dir = $(subst $(OUTPUT),,$(dir $@)) -build-dir = $(if $(__build-dir),$(__build-dir),.) +build-dir = $(or $(__build-dir),.) prepare: $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)common-cmds.h archheaders $(drm_ioctl_array) \ $(fadvise_advice_array) \ @@ -1090,7 +1089,7 @@ bpf-skel-clean: clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBPERF)-clean fixdep-clean python-clean bpf-skel-clean $(call QUIET_CLEAN, core-objs) $(RM) $(LIBPERF_A) $(OUTPUT)perf-archive $(OUTPUT)perf-with-kcore $(OUTPUT)perf-iostat $(LANG_BINDINGS) - $(Q)find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete + $(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete $(Q)$(RM) $(OUTPUT).config-detected $(call QUIET_CLEAN, core-progs) $(RM) $(ALL_PROGRAMS) perf perf-read-vdso32 perf-read-vdsox32 $(OUTPUT)pmu-events/jevents $(OUTPUT)$(LIBJVMTI).so $(call QUIET_CLEAN, core-gen) $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)FEATURE-DUMP $(OUTPUT)util/*-bison* $(OUTPUT)util/*-flex* \ @@ -1139,21 +1138,12 @@ else @echo "FEATURE-DUMP file available in $(OUTPUT)FEATURE-DUMP" endif -# -# Trick: if ../../.git does not exist - we are building out of tree for example, -# then force version regeneration: -# -ifeq ($(wildcard ../../.git/HEAD),) - GIT-HEAD-PHONY = ../../.git/HEAD ../../.git/ORIG_HEAD -else - GIT-HEAD-PHONY = -endif FORCE: .PHONY: all install clean config-clean strip install-gtk .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell -.PHONY: $(GIT-HEAD-PHONY) TAGS tags cscope FORCE prepare +.PHONY: .FORCE-PERF-VERSION-FILE TAGS tags cscope FORCE prepare .PHONY: libtraceevent_plugins archheaders endif # force_fixdep diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index cbc555245959..11c71aa219f7 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -199,7 +199,7 @@ static int cs_etm_set_option(struct auxtrace_record *itr, struct evsel *evsel, u32 option) { int i, err = -EINVAL; - struct perf_cpu_map *event_cpus = evsel->evlist->core.cpus; + struct perf_cpu_map *event_cpus = evsel->evlist->core.user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL); /* Set option of each CPU we have */ @@ -299,7 +299,7 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, container_of(itr, struct cs_etm_recording, itr); struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu; struct evsel *evsel, *cs_etm_evsel = NULL; - struct perf_cpu_map *cpus = evlist->core.cpus; + struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); int err = 0; @@ -522,7 +522,7 @@ cs_etm_info_priv_size(struct auxtrace_record *itr __maybe_unused, { int i; int etmv3 = 0, etmv4 = 0, ete = 0; - struct perf_cpu_map *event_cpus = evlist->core.cpus; + struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL); /* cpu map is not empty, we have specific CPUs to work with */ @@ -713,7 +713,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, u32 offset; u64 nr_cpu, type; struct perf_cpu_map *cpu_map; - struct perf_cpu_map *event_cpus = session->evlist->core.cpus; + struct perf_cpu_map *event_cpus = session->evlist->core.user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL); struct cs_etm_recording *ptr = container_of(itr, struct cs_etm_recording, itr); diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 5860bbaea95a..86e2e926aa0e 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -144,7 +144,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, container_of(itr, struct arm_spe_recording, itr); struct perf_pmu *arm_spe_pmu = sper->arm_spe_pmu; struct evsel *evsel, *arm_spe_evsel = NULL; - struct perf_cpu_map *cpus = evlist->core.cpus; + struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); struct evsel *tracking_evsel; int err; diff --git a/tools/perf/arch/x86/util/intel-bts.c b/tools/perf/arch/x86/util/intel-bts.c index 4a76d49d25d6..d68a0f48e41e 100644 --- a/tools/perf/arch/x86/util/intel-bts.c +++ b/tools/perf/arch/x86/util/intel-bts.c @@ -110,7 +110,7 @@ static int intel_bts_recording_options(struct auxtrace_record *itr, container_of(itr, struct intel_bts_recording, itr); struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; struct evsel *evsel, *intel_bts_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.cpus; + const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); if (opts->auxtrace_sample_mode) { diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 8c31578d6f4a..38ec2666ec12 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -382,7 +382,7 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, ui__warning("Intel Processor Trace: TSC not available\n"); } - per_cpu_mmaps = !perf_cpu_map__empty(session->evlist->core.cpus); + per_cpu_mmaps = !perf_cpu_map__empty(session->evlist->core.user_requested_cpus); auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; @@ -632,7 +632,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr, struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; bool have_timing_info, need_immediate = false; struct evsel *evsel, *intel_pt_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.cpus; + const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); u64 tsc_bit; int err; diff --git a/tools/perf/bench/evlist-open-close.c b/tools/perf/bench/evlist-open-close.c index de56601f69ee..5a27691469ed 100644 --- a/tools/perf/bench/evlist-open-close.c +++ b/tools/perf/bench/evlist-open-close.c @@ -151,7 +151,7 @@ static int bench_evlist_open_close__run(char *evstr) init_stats(&time_stats); - printf(" Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.cpus)); + printf(" Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.user_requested_cpus)); printf(" Number of threads:\t%d\n", evlist->core.threads->nr); printf(" Number of events:\t%d (%d fds)\n", evlist->core.nr_entries, evlist__count_evsel_fds(evlist)); diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index ad9ce1bfffa1..7de07bb16d23 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -301,7 +301,7 @@ static int set_tracing_cpumask(struct perf_cpu_map *cpumap) static int set_tracing_cpu(struct perf_ftrace *ftrace) { - struct perf_cpu_map *cpumap = ftrace->evlist->core.cpus; + struct perf_cpu_map *cpumap = ftrace->evlist->core.user_requested_cpus; if (!target__has_cpu(&ftrace->target)) return 0; diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 0b4abed555d8..ba74fab02e62 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -987,7 +987,7 @@ static int record__thread_data_init_maps(struct record_thread *thread_data, stru int m, tm, nr_mmaps = evlist->core.nr_mmaps; struct mmap *mmap = evlist->mmap; struct mmap *overwrite_mmap = evlist->overwrite_mmap; - struct perf_cpu_map *cpus = evlist->core.cpus; + struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits, thread_data->mask->maps.nbits); @@ -1881,7 +1881,7 @@ static int record__synthesize(struct record *rec, bool tail) return err; } - err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.cpus, + err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.user_requested_cpus, process_synthesized_event, NULL); if (err < 0) { pr_err("Couldn't synthesize cpu map.\n"); @@ -3675,7 +3675,7 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu static int record__init_thread_masks(struct record *rec) { int ret = 0; - struct perf_cpu_map *cpus = rec->evlist->core.cpus; + struct perf_cpu_map *cpus = rec->evlist->core.user_requested_cpus; if (!record__threads_enabled(rec)) return record__init_thread_default_masks(rec, cpus); diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 4ee40de698a4..a96f106dc93a 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -804,7 +804,7 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) if (group) evlist__set_leader(evsel_list); - if (!cpu_map__is_dummy(evsel_list->core.cpus)) { + if (!cpu_map__is_dummy(evsel_list->core.user_requested_cpus)) { if (affinity__setup(&saved_affinity) < 0) return -1; affinity = &saved_affinity; @@ -1458,7 +1458,7 @@ static int perf_stat_init_aggr_mode(void) aggr_cpu_id_get_t get_id = aggr_mode__get_aggr(stat_config.aggr_mode); if (get_id) { - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.cpus, + stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, get_id, /*data=*/NULL); if (!stat_config.aggr_map) { pr_err("cannot build %s map", aggr_mode__string[stat_config.aggr_mode]); @@ -1472,7 +1472,10 @@ static int perf_stat_init_aggr_mode(void) * taking the highest cpu number to be the size of * the aggregation translate cpumap. */ - nr = perf_cpu_map__max(evsel_list->core.cpus).cpu; + if (evsel_list->core.user_requested_cpus) + nr = perf_cpu_map__max(evsel_list->core.user_requested_cpus).cpu; + else + nr = 0; stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr + 1); return stat_config.cpus_aggr_map ? 0 : -ENOMEM; } @@ -1627,7 +1630,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) if (!get_id) return 0; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.cpus, get_id, env); + stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, get_id, env); if (!stat_config.aggr_map) { pr_err("cannot build %s map", aggr_mode__string[stat_config.aggr_mode]); return -1; diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index 9b08e44a31d9..fd8fd913c533 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -1021,7 +1021,7 @@ static int perf_top__start_counters(struct perf_top *top) evlist__for_each_entry(evlist, counter) { try_again: - if (evsel__open(counter, top->evlist->core.cpus, + if (evsel__open(counter, top->evlist->core.user_requested_cpus, top->evlist->core.threads) < 0) { /* diff --git a/tools/perf/python/tracepoint.py b/tools/perf/python/tracepoint.py index 461848c7f57d..bba68a6d4515 100755 --- a/tools/perf/python/tracepoint.py +++ b/tools/perf/python/tracepoint.py @@ -34,7 +34,7 @@ def main(): if not isinstance(event, perf.sample_event): continue - print "time %u prev_comm=%s prev_pid=%d prev_prio=%d prev_state=0x%x ==> next_comm=%s next_pid=%d next_prio=%d" % ( + print("time %u prev_comm=%s prev_pid=%d prev_prio=%d prev_state=0x%x ==> next_comm=%s next_pid=%d next_prio=%d" % ( event.sample_time, event.prev_comm, event.prev_pid, @@ -42,7 +42,7 @@ def main(): event.prev_state, event.next_comm, event.next_pid, - event.next_prio) + event.next_prio)) if __name__ == '__main__': main() diff --git a/tools/perf/trace/beauty/include/linux/socket.h b/tools/perf/trace/beauty/include/linux/socket.h index 8ef26d89ef49..6f85f5d957ef 100644 --- a/tools/perf/trace/beauty/include/linux/socket.h +++ b/tools/perf/trace/beauty/include/linux/socket.h @@ -366,6 +366,7 @@ struct ucred { #define SOL_XDP 283 #define SOL_MPTCP 284 #define SOL_MCTP 285 +#define SOL_SMC 286 /* IPX options */ #define IPX_TYPE 1 diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 9e48652662d4..df1c5bbbaa0d 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -174,7 +174,7 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, mp->idx = idx; if (per_cpu) { - mp->cpu = perf_cpu_map__cpu(evlist->core.cpus, idx); + mp->cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, idx); if (evlist->core.threads) mp->tid = perf_thread_map__pid(evlist->core.threads, 0); else diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c index 4f4d3aaff37c..7a4297d8fd2c 100644 --- a/tools/perf/util/bpf_ftrace.c +++ b/tools/perf/util/bpf_ftrace.c @@ -38,7 +38,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) /* don't need to set cpu filter for system-wide mode */ if (ftrace->target.cpu_list) { - ncpus = perf_cpu_map__nr(ftrace->evlist->core.cpus); + ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); } @@ -63,7 +63,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(ftrace->evlist->core.cpus, i).cpu; + cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 9bb79e049957..52ea004ba01e 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -440,7 +440,7 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name) bool has_imm = false; // See explanation in evlist__close() - if (!cpu_map__is_dummy(evlist->core.cpus)) { + if (!cpu_map__is_dummy(evlist->core.user_requested_cpus)) { if (affinity__setup(&saved_affinity) < 0) return; affinity = &saved_affinity; @@ -500,7 +500,7 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name) struct affinity saved_affinity, *affinity = NULL; // See explanation in evlist__close() - if (!cpu_map__is_dummy(evlist->core.cpus)) { + if (!cpu_map__is_dummy(evlist->core.user_requested_cpus)) { if (affinity__setup(&saved_affinity) < 0) return; affinity = &saved_affinity; @@ -565,7 +565,7 @@ static int evlist__enable_event_cpu(struct evlist *evlist, struct evsel *evsel, static int evlist__enable_event_thread(struct evlist *evlist, struct evsel *evsel, int thread) { int cpu; - int nr_cpus = perf_cpu_map__nr(evlist->core.cpus); + int nr_cpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); if (!evsel->core.fd) return -EINVAL; @@ -580,7 +580,7 @@ static int evlist__enable_event_thread(struct evlist *evlist, struct evsel *evse int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu_mmaps = !perf_cpu_map__empty(evlist->core.cpus); + bool per_cpu_mmaps = !perf_cpu_map__empty(evlist->core.user_requested_cpus); if (per_cpu_mmaps) return evlist__enable_event_cpu(evlist, evsel, idx); @@ -1301,10 +1301,11 @@ void evlist__close(struct evlist *evlist) struct affinity affinity; /* - * With perf record core.cpus is usually NULL. + * With perf record core.user_requested_cpus is usually NULL. * Use the old method to handle this for now. */ - if (!evlist->core.cpus || cpu_map__is_dummy(evlist->core.cpus)) { + if (!evlist->core.user_requested_cpus || + cpu_map__is_dummy(evlist->core.user_requested_cpus)) { evlist__for_each_entry_reverse(evlist, evsel) evsel__close(evsel); return; @@ -1330,7 +1331,6 @@ static int evlist__create_syswide_maps(struct evlist *evlist) { struct perf_cpu_map *cpus; struct perf_thread_map *threads; - int err = -ENOMEM; /* * Try reading /sys/devices/system/cpu/online to get @@ -1355,7 +1355,7 @@ static int evlist__create_syswide_maps(struct evlist *evlist) out_put: perf_cpu_map__put(cpus); out: - return err; + return -ENOMEM; } int evlist__open(struct evlist *evlist) @@ -1367,7 +1367,7 @@ int evlist__open(struct evlist *evlist) * Default: one fd per CPU, all threads, aka systemwide * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL */ - if (evlist->core.threads == NULL && evlist->core.cpus == NULL) { + if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) { err = evlist__create_syswide_maps(evlist); if (err < 0) goto out_err; diff --git a/tools/perf/util/hashmap.c b/tools/perf/util/hashmap.c index 3c20b126d60d..aeb09c288716 100644 --- a/tools/perf/util/hashmap.c +++ b/tools/perf/util/hashmap.c @@ -75,7 +75,7 @@ void hashmap__clear(struct hashmap *map) void hashmap__free(struct hashmap *map) { - if (!map) + if (IS_ERR_OR_NULL(map)) return; hashmap__clear(map); @@ -238,4 +238,3 @@ bool hashmap__delete(struct hashmap *map, const void *key, return true; } - diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c index 007a64681416..5b09ecbb05dc 100644 --- a/tools/perf/util/record.c +++ b/tools/perf/util/record.c @@ -106,7 +106,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call if (opts->group) evlist__set_leader(evlist); - if (perf_cpu_map__cpu(evlist->core.cpus, 0).cpu < 0) + if (perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0).cpu < 0) opts->no_inherit = true; use_comm_exec = perf_can_comm_exec(); @@ -244,7 +244,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) evsel = evlist__last(temp_evlist); - if (!evlist || perf_cpu_map__empty(evlist->core.cpus)) { + if (!evlist || perf_cpu_map__empty(evlist->core.user_requested_cpus)) { struct perf_cpu_map *cpus = perf_cpu_map__new(NULL); if (cpus) @@ -252,7 +252,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) perf_cpu_map__put(cpus); } else { - cpu = perf_cpu_map__cpu(evlist->core.cpus, 0); + cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0); } while (1) { diff --git a/tools/perf/util/sideband_evlist.c b/tools/perf/util/sideband_evlist.c index 748371ac22be..388846f17bc1 100644 --- a/tools/perf/util/sideband_evlist.c +++ b/tools/perf/util/sideband_evlist.c @@ -114,7 +114,8 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) } evlist__for_each_entry(evlist, counter) { - if (evsel__open(counter, evlist->core.cpus, evlist->core.threads) < 0) + if (evsel__open(counter, evlist->core.user_requested_cpus, + evlist->core.threads) < 0) goto out_delete_evlist; } diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 9cbe351b141f..138e3ab9d638 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -929,7 +929,7 @@ static void print_no_aggr_metric(struct perf_stat_config *config, int all_idx; struct perf_cpu cpu; - perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.cpus) { + perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.user_requested_cpus) { struct evsel *counter; bool first = true; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index b654de0841f8..27acdc5e5723 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -2127,7 +2127,7 @@ int perf_event__synthesize_stat_events(struct perf_stat_config *config, struct p return err; } - err = perf_event__synthesize_cpu_map(tool, evlist->core.cpus, process, NULL); + err = perf_event__synthesize_cpu_map(tool, evlist->core.user_requested_cpus, process, NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; diff --git a/tools/perf/util/top.c b/tools/perf/util/top.c index c1ebfc5d2e0c..b8b32431d2f7 100644 --- a/tools/perf/util/top.c +++ b/tools/perf/util/top.c @@ -95,15 +95,17 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) if (target->cpu_list) ret += SNPRINTF(bf + ret, size - ret, ", CPU%s: %s)", - perf_cpu_map__nr(top->evlist->core.cpus) > 1 ? "s" : "", + perf_cpu_map__nr(top->evlist->core.user_requested_cpus) > 1 + ? "s" : "", target->cpu_list); else { if (target->tid) ret += SNPRINTF(bf + ret, size - ret, ")"); else ret += SNPRINTF(bf + ret, size - ret, ", %d CPU%s)", - perf_cpu_map__nr(top->evlist->core.cpus), - perf_cpu_map__nr(top->evlist->core.cpus) > 1 ? "s" : ""); + perf_cpu_map__nr(top->evlist->core.user_requested_cpus), + perf_cpu_map__nr(top->evlist->core.user_requested_cpus) > 1 + ? "s" : ""); } perf_top__reset_sample_counters(top); diff --git a/tools/power/x86/intel-speed-select/Makefile b/tools/power/x86/intel-speed-select/Makefile index d2fba1297d96..846f785e278d 100644 --- a/tools/power/x86/intel-speed-select/Makefile +++ b/tools/power/x86/intel-speed-select/Makefile @@ -47,7 +47,7 @@ $(OUTPUT)intel-speed-select: $(ISST_IN) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/linux/isst_if.h - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index a2335e402145..0efb8f2b33ce 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -52,11 +52,17 @@ define allow-override endef ifneq ($(LLVM),) -$(call allow-override,CC,clang) -$(call allow-override,AR,llvm-ar) -$(call allow-override,LD,ld.lld) -$(call allow-override,CXX,clang++) -$(call allow-override,STRIP,llvm-strip) +ifneq ($(filter %/,$(LLVM)),) +LLVM_PREFIX := $(LLVM) +else ifneq ($(filter -%,$(LLVM)),) +LLVM_SUFFIX := $(LLVM) +endif + +$(call allow-override,CC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,AR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) +$(call allow-override,LD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) +$(call allow-override,CXX,$(LLVM_PREFIX)clang++$(LLVM_SUFFIX)) +$(call allow-override,STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) else # Allow setting various cross-compile vars or setting CROSS_COMPILE as a prefix. $(call allow-override,CC,$(CROSS_COMPILE)gcc) @@ -69,9 +75,9 @@ endif CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) ifneq ($(LLVM),) -HOSTAR ?= llvm-ar -HOSTCC ?= clang -HOSTLD ?= ld.lld +HOSTAR ?= $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) +HOSTCC ?= $(LLVM_PREFIX)clang$(LLVM_SUFFIX) +HOSTLD ?= $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) else HOSTAR ?= ar HOSTCC ?= gcc diff --git a/tools/scripts/utilities.mak b/tools/scripts/utilities.mak index c16ce833079c..172e47273b5d 100644 --- a/tools/scripts/utilities.mak +++ b/tools/scripts/utilities.mak @@ -175,5 +175,5 @@ _ge-abspath = $(if $(is-executable),$(1)) define get-executable-or-default $(if $($(1)),$(call _ge_attempt,$($(1)),$(1)),$(call _ge_attempt,$(2))) endef -_ge_attempt = $(if $(get-executable),$(get-executable),$(call _gea_err,$(2))) +_ge_attempt = $(or $(get-executable),$(call _gea_err,$(2))) _gea_err = $(if $(1),$(error Please set '$(1)' appropriately)) diff --git a/tools/spi/Makefile b/tools/spi/Makefile index 0aa6dbd31fb8..7fccd245a535 100644 --- a/tools/spi/Makefile +++ b/tools/spi/Makefile @@ -53,9 +53,9 @@ $(OUTPUT)spidev_fdx: $(SPIDEV_FDX_IN) clean: rm -f $(ALL_PROGRAMS) rm -rf $(OUTPUT)include/ - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete - find $(if $(OUTPUT),$(OUTPUT),.) -name '\.*.o.d' -delete - find $(if $(OUTPUT),$(OUTPUT),.) -name '\.*.o.cmd' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete + find $(or $(OUTPUT),.) -name '\.*.o.d' -delete + find $(or $(OUTPUT),.) -name '\.*.o.cmd' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c b/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c index 5aa52cc31dc2..c11832657d2b 100644 --- a/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c +++ b/tools/testing/selftests/bpf/prog_tests/dummy_st_ops.c @@ -2,6 +2,7 @@ /* Copyright (C) 2021. Huawei Technologies Co., Ltd */ #include <test_progs.h> #include "dummy_st_ops.skel.h" +#include "trace_dummy_st_ops.skel.h" /* Need to keep consistent with definition in include/linux/bpf.h */ struct bpf_dummy_ops_state { @@ -56,6 +57,7 @@ static void test_dummy_init_ptr_arg(void) .ctx_in = args, .ctx_size_in = sizeof(args), ); + struct trace_dummy_st_ops *trace_skel; struct dummy_st_ops *skel; int fd, err; @@ -64,12 +66,33 @@ static void test_dummy_init_ptr_arg(void) return; fd = bpf_program__fd(skel->progs.test_1); + + trace_skel = trace_dummy_st_ops__open(); + if (!ASSERT_OK_PTR(trace_skel, "trace_dummy_st_ops__open")) + goto done; + + err = bpf_program__set_attach_target(trace_skel->progs.fentry_test_1, + fd, "test_1"); + if (!ASSERT_OK(err, "set_attach_target(fentry_test_1)")) + goto done; + + err = trace_dummy_st_ops__load(trace_skel); + if (!ASSERT_OK(err, "load(trace_skel)")) + goto done; + + err = trace_dummy_st_ops__attach(trace_skel); + if (!ASSERT_OK(err, "attach(trace_skel)")) + goto done; + err = bpf_prog_test_run_opts(fd, &attr); ASSERT_OK(err, "test_run"); ASSERT_EQ(in_state.val, 0x5a, "test_ptr_ret"); ASSERT_EQ(attr.retval, exp_retval, "test_ret"); + ASSERT_EQ(trace_skel->bss->val, exp_retval, "fentry_val"); +done: dummy_st_ops__destroy(skel); + trace_dummy_st_ops__destroy(trace_skel); } static void test_dummy_multiple_args(void) diff --git a/tools/testing/selftests/bpf/progs/map_ptr_kern.c b/tools/testing/selftests/bpf/progs/map_ptr_kern.c index b64df94ec476..db388f593d0a 100644 --- a/tools/testing/selftests/bpf/progs/map_ptr_kern.c +++ b/tools/testing/selftests/bpf/progs/map_ptr_kern.c @@ -367,7 +367,7 @@ static inline int check_array_of_maps(void) VERIFY(check_default(&array_of_maps->map, map)); inner_map = bpf_map_lookup_elem(array_of_maps, &key); - VERIFY(inner_map != 0); + VERIFY(inner_map != NULL); VERIFY(inner_map->map.max_entries == INNER_MAX_ENTRIES); return 1; @@ -394,7 +394,7 @@ static inline int check_hash_of_maps(void) VERIFY(check_default(&hash_of_maps->map, map)); inner_map = bpf_map_lookup_elem(hash_of_maps, &key); - VERIFY(inner_map != 0); + VERIFY(inner_map != NULL); VERIFY(inner_map->map.max_entries == INNER_MAX_ENTRIES); return 1; diff --git a/tools/testing/selftests/bpf/progs/trace_dummy_st_ops.c b/tools/testing/selftests/bpf/progs/trace_dummy_st_ops.c new file mode 100644 index 000000000000..00a4be9d3074 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/trace_dummy_st_ops.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +int val = 0; + +SEC("fentry/test_1") +int BPF_PROG(fentry_test_1, __u64 *st_ops_ctx) +{ + __u64 state; + + /* Read the traced st_ops arg1 which is a pointer */ + bpf_probe_read_kernel(&state, sizeof(__u64), (void *)st_ops_ctx); + /* Read state->val */ + bpf_probe_read_kernel(&val, sizeof(__u32), (void *)state); + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_tcp_check_syncookie_user.c b/tools/testing/selftests/bpf/test_tcp_check_syncookie_user.c index b9e991d43155..e7775d3bbe08 100644 --- a/tools/testing/selftests/bpf/test_tcp_check_syncookie_user.c +++ b/tools/testing/selftests/bpf/test_tcp_check_syncookie_user.c @@ -18,8 +18,9 @@ #include "bpf_rlimit.h" #include "cgroup_helpers.h" -static int start_server(const struct sockaddr *addr, socklen_t len) +static int start_server(const struct sockaddr *addr, socklen_t len, bool dual) { + int mode = !dual; int fd; fd = socket(addr->sa_family, SOCK_STREAM, 0); @@ -28,6 +29,14 @@ static int start_server(const struct sockaddr *addr, socklen_t len) goto out; } + if (addr->sa_family == AF_INET6) { + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, (char *)&mode, + sizeof(mode)) == -1) { + log_err("Failed to set the dual-stack mode"); + goto close_out; + } + } + if (bind(fd, addr, len) == -1) { log_err("Failed to bind server socket"); goto close_out; @@ -47,24 +56,17 @@ out: return fd; } -static int connect_to_server(int server_fd) +static int connect_to_server(const struct sockaddr *addr, socklen_t len) { - struct sockaddr_storage addr; - socklen_t len = sizeof(addr); int fd = -1; - if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { - log_err("Failed to get server addr"); - goto out; - } - - fd = socket(addr.ss_family, SOCK_STREAM, 0); + fd = socket(addr->sa_family, SOCK_STREAM, 0); if (fd == -1) { log_err("Failed to create client socket"); goto out; } - if (connect(fd, (const struct sockaddr *)&addr, len) == -1) { + if (connect(fd, (const struct sockaddr *)addr, len) == -1) { log_err("Fail to connect to server"); goto close_out; } @@ -116,7 +118,8 @@ err: return map_fd; } -static int run_test(int server_fd, int results_fd, bool xdp) +static int run_test(int server_fd, int results_fd, bool xdp, + const struct sockaddr *addr, socklen_t len) { int client = -1, srv_client = -1; int ret = 0; @@ -142,7 +145,7 @@ static int run_test(int server_fd, int results_fd, bool xdp) goto err; } - client = connect_to_server(server_fd); + client = connect_to_server(addr, len); if (client == -1) goto err; @@ -199,12 +202,30 @@ out: return ret; } +static bool get_port(int server_fd, in_port_t *port) +{ + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { + log_err("Failed to get server addr"); + return false; + } + + /* sin_port and sin6_port are located at the same offset. */ + *port = addr.sin_port; + return true; +} + int main(int argc, char **argv) { struct sockaddr_in addr4; struct sockaddr_in6 addr6; + struct sockaddr_in addr4dual; + struct sockaddr_in6 addr6dual; int server = -1; int server_v6 = -1; + int server_dual = -1; int results = -1; int err = 0; bool xdp; @@ -224,25 +245,43 @@ int main(int argc, char **argv) addr4.sin_family = AF_INET; addr4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr4.sin_port = 0; + memcpy(&addr4dual, &addr4, sizeof(addr4dual)); memset(&addr6, 0, sizeof(addr6)); addr6.sin6_family = AF_INET6; addr6.sin6_addr = in6addr_loopback; addr6.sin6_port = 0; - server = start_server((const struct sockaddr *)&addr4, sizeof(addr4)); - if (server == -1) + memset(&addr6dual, 0, sizeof(addr6dual)); + addr6dual.sin6_family = AF_INET6; + addr6dual.sin6_addr = in6addr_any; + addr6dual.sin6_port = 0; + + server = start_server((const struct sockaddr *)&addr4, sizeof(addr4), + false); + if (server == -1 || !get_port(server, &addr4.sin_port)) goto err; server_v6 = start_server((const struct sockaddr *)&addr6, - sizeof(addr6)); - if (server_v6 == -1) + sizeof(addr6), false); + if (server_v6 == -1 || !get_port(server_v6, &addr6.sin6_port)) + goto err; + + server_dual = start_server((const struct sockaddr *)&addr6dual, + sizeof(addr6dual), true); + if (server_dual == -1 || !get_port(server_dual, &addr4dual.sin_port)) + goto err; + + if (run_test(server, results, xdp, + (const struct sockaddr *)&addr4, sizeof(addr4))) goto err; - if (run_test(server, results, xdp)) + if (run_test(server_v6, results, xdp, + (const struct sockaddr *)&addr6, sizeof(addr6))) goto err; - if (run_test(server_v6, results, xdp)) + if (run_test(server_dual, results, xdp, + (const struct sockaddr *)&addr4dual, sizeof(addr4dual))) goto err; printf("ok\n"); @@ -252,6 +291,7 @@ err: out: close(server); close(server_v6); + close(server_dual); close(results); return err; } diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index a40add31a2e3..2a2d240cdc1b 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -1,7 +1,13 @@ # This mimics the top-level Makefile. We do it explicitly here so that this # Makefile can operate with or without the kbuild infrastructure. ifneq ($(LLVM),) -CC := clang +ifneq ($(filter %/,$(LLVM)),) +LLVM_PREFIX := $(LLVM) +else ifneq ($(filter -%,$(LLVM)),) +LLVM_SUFFIX := $(LLVM) +endif + +CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) else CC := $(CROSS_COMPILE)gcc endif diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index d444ee6aa3cb..b3bf5319bb0e 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -1208,6 +1208,20 @@ ipv4_fcnal() set +e check_nexthop "dev veth1" "" log_test $? 0 "Nexthops removed on admin down" + + # nexthop route delete warning: route add with nhid and delete + # using device + run_cmd "$IP li set dev veth1 up" + run_cmd "$IP nexthop add id 12 via 172.16.1.3 dev veth1" + out1=`dmesg | grep "WARNING:.*fib_nh_match.*" | wc -l` + run_cmd "$IP route add 172.16.101.1/32 nhid 12" + run_cmd "$IP route delete 172.16.101.1/32 dev veth1" + out2=`dmesg | grep "WARNING:.*fib_nh_match.*" | wc -l` + [ $out1 -eq $out2 ] + rc=$? + log_test $rc 0 "Delete nexthop route warning" + run_cmd "$IP route delete 172.16.101.1/32 nhid 12" + run_cmd "$IP nexthop del id 12" } ipv4_grp_fcnal() diff --git a/tools/testing/selftests/x86/amx.c b/tools/testing/selftests/x86/amx.c index 3615ef4a48bb..2189f0322d8b 100644 --- a/tools/testing/selftests/x86/amx.c +++ b/tools/testing/selftests/x86/amx.c @@ -368,9 +368,16 @@ static void req_xtiledata_perm(void) static void validate_req_xcomp_perm(enum expected_result exp) { - unsigned long bitmask; + unsigned long bitmask, expected_bitmask; long rc; + rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_PERM, &bitmask); + if (rc) { + fatal_error("prctl(ARCH_GET_XCOMP_PERM) error: %ld", rc); + } else if (!(bitmask & XFEATURE_MASK_XTILECFG)) { + fatal_error("ARCH_GET_XCOMP_PERM returns XFEATURE_XTILECFG off."); + } + rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA); if (exp == FAIL_EXPECTED) { if (rc) { @@ -383,10 +390,15 @@ static void validate_req_xcomp_perm(enum expected_result exp) fatal_error("ARCH_REQ_XCOMP_PERM saw unexpected failure.\n"); } + expected_bitmask = bitmask | XFEATURE_MASK_XTILEDATA; + rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_PERM, &bitmask); if (rc) { fatal_error("prctl(ARCH_GET_XCOMP_PERM) error: %ld", rc); - } else if (bitmask & XFEATURE_MASK_XTILE) { + } else if (bitmask != expected_bitmask) { + fatal_error("ARCH_REQ_XCOMP_PERM set a wrong bitmask: %lx, expected: %lx.\n", + bitmask, expected_bitmask); + } else { printf("\tARCH_REQ_XCOMP_PERM is successful.\n"); } } diff --git a/tools/tracing/rtla/Makefile b/tools/tracing/rtla/Makefile index 5a1eda617992..11fb417abb42 100644 --- a/tools/tracing/rtla/Makefile +++ b/tools/tracing/rtla/Makefile @@ -46,7 +46,7 @@ DATADIR := /usr/share DOCDIR := $(DATADIR)/doc MANDIR := $(DATADIR)/man LICDIR := $(DATADIR)/licenses -SRCTREE := $(if $(BUILD_SRC),$(BUILD_SRC),$(CURDIR)) +SRCTREE := $(or $(BUILD_SRC),$(CURDIR)) # If running from the tarball, man pages are stored in the Documentation # dir. If running from the kernel source, man pages are stored in diff --git a/tools/usb/Makefile b/tools/usb/Makefile index 1b128e551b2e..c6235667dd46 100644 --- a/tools/usb/Makefile +++ b/tools/usb/Makefile @@ -38,7 +38,7 @@ $(OUTPUT)ffs-test: $(FFS_TEST_IN) clean: rm -f $(ALL_PROGRAMS) - find $(if $(OUTPUT),$(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete -o -name '\.*.o.cmd' -delete + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete -o -name '\.*.o.cmd' -delete install: $(ALL_PROGRAMS) install -d -m 755 $(DESTDIR)$(bindir); \ diff --git a/tools/virtio/Makefile b/tools/virtio/Makefile index 0d7bbe49359d..1b25cc7c64bb 100644 --- a/tools/virtio/Makefile +++ b/tools/virtio/Makefile @@ -5,7 +5,8 @@ virtio_test: virtio_ring.o virtio_test.o vringh_test: vringh_test.o vringh.o virtio_ring.o CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h -LDFLAGS += -lpthread +CFLAGS += -pthread +LDFLAGS += -pthread vpath %.c ../../drivers/virtio ../../drivers/vhost mod: ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V} diff --git a/tools/virtio/linux/dma-mapping.h b/tools/virtio/linux/dma-mapping.h index 8f41cd6bd5c0..834a90bd3270 100644 --- a/tools/virtio/linux/dma-mapping.h +++ b/tools/virtio/linux/dma-mapping.h @@ -26,8 +26,8 @@ enum dma_data_direction { #define dma_map_single(d, p, s, dir) (virt_to_phys(p)) #define dma_mapping_error(...) (0) -#define dma_unmap_single(...) do { } while (0) -#define dma_unmap_page(...) do { } while (0) +#define dma_unmap_single(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0) +#define dma_unmap_page(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0) #define dma_max_mapping_size(...) SIZE_MAX diff --git a/tools/vm/page_owner_sort.c b/tools/vm/page_owner_sort.c index 7679335fce5b..7d98e76c2291 100644 --- a/tools/vm/page_owner_sort.c +++ b/tools/vm/page_owner_sort.c @@ -441,7 +441,6 @@ static void usage(void) "-n\t\tSort by task command name.\n" "-a\t\tSort by memory allocate time.\n" "-r\t\tSort by memory release time.\n" - "-c\t\tCull by comparing stacktrace instead of total block.\n" "-f\t\tFilter out the information of blocks whose memory has been released.\n" "--pid <PID>\tSelect by pid. This selects the information of blocks whose process ID number equals to <PID>.\n" "--tgid <TGID>\tSelect by tgid. This selects the information of blocks whose Thread Group ID number equals to <TGID>.\n" @@ -466,14 +465,11 @@ int main(int argc, char **argv) { 0, 0, 0, 0}, }; - while ((opt = getopt_long(argc, argv, "acfmnprstP", longopts, NULL)) != -1) + while ((opt = getopt_long(argc, argv, "afmnprstP", longopts, NULL)) != -1) switch (opt) { case 'a': cmp = compare_ts; break; - case 'c': - cull = cull | CULL_STACKTRACE; - break; case 'f': filter = filter | FILTER_UNRELEASE; break; diff --git a/usr/Makefile b/usr/Makefile index cc0d2824e100..59d9e8b07a01 100644 --- a/usr/Makefile +++ b/usr/Makefile @@ -3,7 +3,7 @@ # kbuild file for usr/ - including initramfs image # -compress-y := shipped +compress-y := copy compress-$(CONFIG_INITRAMFS_COMPRESSION_GZIP) := gzip compress-$(CONFIG_INITRAMFS_COMPRESSION_BZIP2) := bzip2 compress-$(CONFIG_INITRAMFS_COMPRESSION_LZMA) := lzma @@ -37,7 +37,7 @@ endif # .cpio.*, use it directly as an initramfs, and avoid double compression. ifeq ($(words $(subst .cpio.,$(space),$(ramfs-input))),2) cpio-data := $(ramfs-input) -compress-y := shipped +compress-y := copy endif endif diff --git a/usr/include/Makefile b/usr/include/Makefile index 7b283d43f00d..fa9819e022b7 100644 --- a/usr/include/Makefile +++ b/usr/include/Makefile @@ -10,7 +10,10 @@ UAPI_CFLAGS := -std=c90 -Wall -Werror=implicit-function-declaration # In theory, we do not care -m32 or -m64 for header compile tests. # It is here just because CONFIG_CC_CAN_LINK is tested with -m32 or -m64. -UAPI_CFLAGS += $(filter -m32 -m64, $(KBUILD_CFLAGS)) +UAPI_CFLAGS += $(filter -m32 -m64 --target=%, $(KBUILD_CFLAGS)) + +# USERCFLAGS might contain sysroot location for CC. +UAPI_CFLAGS += $(USERCFLAGS) override c_flags = $(UAPI_CFLAGS) -Wp,-MMD,$(depfile) -I$(objtree)/usr/include @@ -84,7 +87,7 @@ endif # asm-generic/*.h is used by asm/*.h, and should not be included directly no-header-test += asm-generic/% -extra-y := $(patsubst $(obj)/%.h,%.hdrtest, $(shell find $(obj) -name '*.h' 2>/dev/null)) +always-y := $(patsubst $(obj)/%.h,%.hdrtest, $(shell find $(obj) -name '*.h' 2>/dev/null)) # Include the header twice to detect missing include guard. quiet_cmd_hdrtest = HDRTEST $< diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 69c318fdff61..70e05af5ebea 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -117,6 +117,8 @@ EXPORT_SYMBOL_GPL(kvm_debugfs_dir); static const struct file_operations stat_fops_per_vm; +static struct file_operations kvm_chardev_ops; + static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, unsigned long arg); #ifdef CONFIG_KVM_COMPAT @@ -251,7 +253,8 @@ static void kvm_make_vcpu_request(struct kvm_vcpu *vcpu, unsigned int req, { int cpu; - kvm_make_request(req, vcpu); + if (likely(!(req & KVM_REQUEST_NO_ACTION))) + __kvm_make_request(req, vcpu); if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu)) return; @@ -1131,6 +1134,16 @@ static struct kvm *kvm_create_vm(unsigned long type) preempt_notifier_inc(); kvm_init_pm_notifier(kvm); + /* + * When the fd passed to this ioctl() is opened it pins the module, + * but try_module_get() also prevents getting a reference if the module + * is in MODULE_STATE_GOING (e.g. if someone ran "rmmod --wait"). + */ + if (!try_module_get(kvm_chardev_ops.owner)) { + r = -ENODEV; + goto out_err; + } + return kvm; out_err: @@ -1220,6 +1233,7 @@ static void kvm_destroy_vm(struct kvm *kvm) preempt_notifier_dec(); hardware_disable_all(); mmdrop(mm); + module_put(kvm_chardev_ops.owner); } void kvm_get_kvm(struct kvm *kvm) @@ -3663,7 +3677,7 @@ static int kvm_vcpu_release(struct inode *inode, struct file *filp) return 0; } -static struct file_operations kvm_vcpu_fops = { +static const struct file_operations kvm_vcpu_fops = { .release = kvm_vcpu_release, .unlocked_ioctl = kvm_vcpu_ioctl, .mmap = kvm_vcpu_mmap, @@ -4714,7 +4728,7 @@ static long kvm_vm_compat_ioctl(struct file *filp, } #endif -static struct file_operations kvm_vm_fops = { +static const struct file_operations kvm_vm_fops = { .release = kvm_vm_release, .unlocked_ioctl = kvm_vm_ioctl, .llseek = noop_llseek, @@ -5721,8 +5735,6 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, goto out_free_5; kvm_chardev_ops.owner = module; - kvm_vm_fops.owner = module; - kvm_vcpu_fops.owner = module; r = misc_register(&kvm_dev); if (r) { diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index ce878f4be4da..dd84676615f1 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -27,7 +27,7 @@ void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start, { DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS); struct gfn_to_pfn_cache *gpc; - bool wake_vcpus = false; + bool evict_vcpus = false; spin_lock(&kvm->gpc_lock); list_for_each_entry(gpc, &kvm->gpc_list, list) { @@ -40,41 +40,32 @@ void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, unsigned long start, /* * If a guest vCPU could be using the physical address, - * it needs to be woken. + * it needs to be forced out of guest mode. */ - if (gpc->guest_uses_pa) { - if (!wake_vcpus) { - wake_vcpus = true; + if (gpc->usage & KVM_GUEST_USES_PFN) { + if (!evict_vcpus) { + evict_vcpus = true; bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS); } __set_bit(gpc->vcpu->vcpu_idx, vcpu_bitmap); } - - /* - * We cannot call mark_page_dirty() from here because - * this physical CPU might not have an active vCPU - * with which to do the KVM dirty tracking. - * - * Neither is there any point in telling the kernel MM - * that the underlying page is dirty. A vCPU in guest - * mode might still be writing to it up to the point - * where we wake them a few lines further down anyway. - * - * So all the dirty marking happens on the unmap. - */ } write_unlock_irq(&gpc->lock); } spin_unlock(&kvm->gpc_lock); - if (wake_vcpus) { - unsigned int req = KVM_REQ_GPC_INVALIDATE; + if (evict_vcpus) { + /* + * KVM needs to ensure the vCPU is fully out of guest context + * before allowing the invalidation to continue. + */ + unsigned int req = KVM_REQ_OUTSIDE_GUEST_MODE; bool called; /* * If the OOM reaper is active, then all vCPUs should have * been stopped already, so perform the request without - * KVM_REQUEST_WAIT and be sad if any needed to be woken. + * KVM_REQUEST_WAIT and be sad if any needed to be IPI'd. */ if (!may_block) req &= ~KVM_REQUEST_WAIT; @@ -104,8 +95,7 @@ bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, } EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_check); -static void __release_gpc(struct kvm *kvm, kvm_pfn_t pfn, void *khva, - gpa_t gpa, bool dirty) +static void __release_gpc(struct kvm *kvm, kvm_pfn_t pfn, void *khva, gpa_t gpa) { /* Unmap the old page if it was mapped before, and release it */ if (!is_error_noslot_pfn(pfn)) { @@ -118,9 +108,7 @@ static void __release_gpc(struct kvm *kvm, kvm_pfn_t pfn, void *khva, #endif } - kvm_release_pfn(pfn, dirty); - if (dirty) - mark_page_dirty(kvm, gpa); + kvm_release_pfn(pfn, false); } } @@ -152,7 +140,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct kvm *kvm, unsigned long uhva) } int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - gpa_t gpa, unsigned long len, bool dirty) + gpa_t gpa, unsigned long len) { struct kvm_memslots *slots = kvm_memslots(kvm); unsigned long page_offset = gpa & ~PAGE_MASK; @@ -160,7 +148,7 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, unsigned long old_uhva; gpa_t old_gpa; void *old_khva; - bool old_valid, old_dirty; + bool old_valid; int ret = 0; /* @@ -177,20 +165,19 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, old_khva = gpc->khva - offset_in_page(gpc->khva); old_uhva = gpc->uhva; old_valid = gpc->valid; - old_dirty = gpc->dirty; /* If the userspace HVA is invalid, refresh that first */ if (gpc->gpa != gpa || gpc->generation != slots->generation || kvm_is_error_hva(gpc->uhva)) { gfn_t gfn = gpa_to_gfn(gpa); - gpc->dirty = false; gpc->gpa = gpa; gpc->generation = slots->generation; gpc->memslot = __gfn_to_memslot(slots, gfn); gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn); if (kvm_is_error_hva(gpc->uhva)) { + gpc->pfn = KVM_PFN_ERR_FAULT; ret = -EFAULT; goto out; } @@ -219,7 +206,7 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, goto map_done; } - if (gpc->kernel_map) { + if (gpc->usage & KVM_HOST_USES_PFN) { if (new_pfn == old_pfn) { new_khva = old_khva; old_pfn = KVM_PFN_ERR_FAULT; @@ -255,14 +242,9 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, } out: - if (ret) - gpc->dirty = false; - else - gpc->dirty = dirty; - write_unlock_irq(&gpc->lock); - __release_gpc(kvm, old_pfn, old_khva, old_gpa, old_dirty); + __release_gpc(kvm, old_pfn, old_khva, old_gpa); return ret; } @@ -272,7 +254,6 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) { void *old_khva; kvm_pfn_t old_pfn; - bool old_dirty; gpa_t old_gpa; write_lock_irq(&gpc->lock); @@ -280,7 +261,6 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) gpc->valid = false; old_khva = gpc->khva - offset_in_page(gpc->khva); - old_dirty = gpc->dirty; old_gpa = gpc->gpa; old_pfn = gpc->pfn; @@ -293,16 +273,17 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) write_unlock_irq(&gpc->lock); - __release_gpc(kvm, old_pfn, old_khva, old_gpa, old_dirty); + __release_gpc(kvm, old_pfn, old_khva, old_gpa); } EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_unmap); int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - struct kvm_vcpu *vcpu, bool guest_uses_pa, - bool kernel_map, gpa_t gpa, unsigned long len, - bool dirty) + struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, + gpa_t gpa, unsigned long len) { + WARN_ON_ONCE(!usage || (usage & KVM_GUEST_AND_HOST_USE_PFN) != usage); + if (!gpc->active) { rwlock_init(&gpc->lock); @@ -310,8 +291,7 @@ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, gpc->pfn = KVM_PFN_ERR_FAULT; gpc->uhva = KVM_HVA_ERR_BAD; gpc->vcpu = vcpu; - gpc->kernel_map = kernel_map; - gpc->guest_uses_pa = guest_uses_pa; + gpc->usage = usage; gpc->valid = false; gpc->active = true; @@ -319,7 +299,7 @@ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, list_add(&gpc->list, &kvm->gpc_list); spin_unlock(&kvm->gpc_lock); } - return kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpa, len, dirty); + return kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpa, len); } EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_init); |