diff options
Diffstat (limited to 'arch/s390')
63 files changed, 1142 insertions, 901 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index b9df8d11d7a9..de0fcc08dff5 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -4,6 +4,9 @@ config MMU config ZONE_DMA def_bool y +config CPU_BIG_ENDIAN + def_bool y + config LOCKDEP_SUPPORT def_bool y @@ -59,6 +62,9 @@ config PCI_QUIRKS config ARCH_SUPPORTS_UPROBES def_bool y +config DEBUG_RODATA + def_bool y + config S390 def_bool y select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE @@ -101,6 +107,7 @@ config S390 select ARCH_SUPPORTS_NUMA_BALANCING select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF + select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANTS_PROT_NUMA_PROT_NONE select ARCH_WANT_IPC_PARSE_VERSION select BUILDTIME_EXTABLE_SORT @@ -120,7 +127,7 @@ config S390 select HAVE_ARCH_SOFT_DIRTY select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE - select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z196_FEATURES + select HAVE_EBPF_JIT if PACK_STACK && HAVE_MARCH_Z196_FEATURES select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_KMEMLEAK @@ -204,7 +211,7 @@ config HAVE_MARCH_Z13_FEATURES choice prompt "Processor type" - default MARCH_Z900 + default MARCH_Z196 config MARCH_Z900 bool "IBM zSeries model z800 and z900" diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 48e1a2d3e318..7554a8bb2adc 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -28,7 +28,7 @@ #include <linux/init.h> #include <linux/spinlock.h> #include <crypto/xts.h> -#include "crypt_s390.h" +#include <asm/cpacf.h> #define AES_KEYLEN_128 1 #define AES_KEYLEN_192 2 @@ -145,16 +145,16 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) switch (sctx->key_len) { case 16: - crypt_s390_km(KM_AES_128_ENCRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_128_ENC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; case 24: - crypt_s390_km(KM_AES_192_ENCRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_192_ENC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; case 32: - crypt_s390_km(KM_AES_256_ENCRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_256_ENC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; } } @@ -170,16 +170,16 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) switch (sctx->key_len) { case 16: - crypt_s390_km(KM_AES_128_DECRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_128_DEC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; case 24: - crypt_s390_km(KM_AES_192_DECRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_192_DEC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; case 32: - crypt_s390_km(KM_AES_256_DECRYPT, &sctx->key, out, in, - AES_BLOCK_SIZE); + cpacf_km(CPACF_KM_AES_256_DEC, &sctx->key, out, in, + AES_BLOCK_SIZE); break; } } @@ -212,7 +212,7 @@ static void fallback_exit_cip(struct crypto_tfm *tfm) static struct crypto_alg aes_alg = { .cra_name = "aes", .cra_driver_name = "aes-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, @@ -298,16 +298,16 @@ static int ecb_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, switch (key_len) { case 16: - sctx->enc = KM_AES_128_ENCRYPT; - sctx->dec = KM_AES_128_DECRYPT; + sctx->enc = CPACF_KM_AES_128_ENC; + sctx->dec = CPACF_KM_AES_128_DEC; break; case 24: - sctx->enc = KM_AES_192_ENCRYPT; - sctx->dec = KM_AES_192_DECRYPT; + sctx->enc = CPACF_KM_AES_192_ENC; + sctx->dec = CPACF_KM_AES_192_DEC; break; case 32: - sctx->enc = KM_AES_256_ENCRYPT; - sctx->dec = KM_AES_256_DECRYPT; + sctx->enc = CPACF_KM_AES_256_ENC; + sctx->dec = CPACF_KM_AES_256_DEC; break; } @@ -326,7 +326,7 @@ static int ecb_aes_crypt(struct blkcipher_desc *desc, long func, void *param, u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_km(func, param, out, in, n); + ret = cpacf_km(func, param, out, in, n); if (ret < 0 || ret != n) return -EIO; @@ -393,7 +393,7 @@ static void fallback_exit_blk(struct crypto_tfm *tfm) static struct crypto_alg ecb_aes_alg = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: aes + ecb */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, @@ -427,16 +427,16 @@ static int cbc_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, switch (key_len) { case 16: - sctx->enc = KMC_AES_128_ENCRYPT; - sctx->dec = KMC_AES_128_DECRYPT; + sctx->enc = CPACF_KMC_AES_128_ENC; + sctx->dec = CPACF_KMC_AES_128_DEC; break; case 24: - sctx->enc = KMC_AES_192_ENCRYPT; - sctx->dec = KMC_AES_192_DECRYPT; + sctx->enc = CPACF_KMC_AES_192_ENC; + sctx->dec = CPACF_KMC_AES_192_DEC; break; case 32: - sctx->enc = KMC_AES_256_ENCRYPT; - sctx->dec = KMC_AES_256_DECRYPT; + sctx->enc = CPACF_KMC_AES_256_ENC; + sctx->dec = CPACF_KMC_AES_256_DEC; break; } @@ -465,7 +465,7 @@ static int cbc_aes_crypt(struct blkcipher_desc *desc, long func, u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_kmc(func, ¶m, out, in, n); + ret = cpacf_kmc(func, ¶m, out, in, n); if (ret < 0 || ret != n) return -EIO; @@ -509,7 +509,7 @@ static int cbc_aes_decrypt(struct blkcipher_desc *desc, static struct crypto_alg cbc_aes_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: aes + cbc */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, @@ -596,8 +596,8 @@ static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, switch (key_len) { case 32: - xts_ctx->enc = KM_XTS_128_ENCRYPT; - xts_ctx->dec = KM_XTS_128_DECRYPT; + xts_ctx->enc = CPACF_KM_XTS_128_ENC; + xts_ctx->dec = CPACF_KM_XTS_128_DEC; memcpy(xts_ctx->key + 16, in_key, 16); memcpy(xts_ctx->pcc_key + 16, in_key + 16, 16); break; @@ -607,8 +607,8 @@ static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, xts_fallback_setkey(tfm, in_key, key_len); break; case 64: - xts_ctx->enc = KM_XTS_256_ENCRYPT; - xts_ctx->dec = KM_XTS_256_DECRYPT; + xts_ctx->enc = CPACF_KM_XTS_256_ENC; + xts_ctx->dec = CPACF_KM_XTS_256_DEC; memcpy(xts_ctx->key, in_key, 32); memcpy(xts_ctx->pcc_key, in_key + 32, 32); break; @@ -643,7 +643,8 @@ static int xts_aes_crypt(struct blkcipher_desc *desc, long func, memset(pcc_param.xts, 0, sizeof(pcc_param.xts)); memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak)); memcpy(pcc_param.key, xts_ctx->pcc_key, 32); - ret = crypt_s390_pcc(func, &pcc_param.key[offset]); + /* remove decipher modifier bit from 'func' and call PCC */ + ret = cpacf_pcc(func & 0x7f, &pcc_param.key[offset]); if (ret < 0) return -EIO; @@ -655,7 +656,7 @@ static int xts_aes_crypt(struct blkcipher_desc *desc, long func, out = walk->dst.virt.addr; in = walk->src.virt.addr; - ret = crypt_s390_km(func, &xts_param.key[offset], out, in, n); + ret = cpacf_km(func, &xts_param.key[offset], out, in, n); if (ret < 0 || ret != n) return -EIO; @@ -721,7 +722,7 @@ static void xts_fallback_exit(struct crypto_tfm *tfm) static struct crypto_alg xts_aes_alg = { .cra_name = "xts(aes)", .cra_driver_name = "xts-aes-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: aes + xts */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = AES_BLOCK_SIZE, @@ -751,16 +752,16 @@ static int ctr_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, switch (key_len) { case 16: - sctx->enc = KMCTR_AES_128_ENCRYPT; - sctx->dec = KMCTR_AES_128_DECRYPT; + sctx->enc = CPACF_KMCTR_AES_128_ENC; + sctx->dec = CPACF_KMCTR_AES_128_DEC; break; case 24: - sctx->enc = KMCTR_AES_192_ENCRYPT; - sctx->dec = KMCTR_AES_192_DECRYPT; + sctx->enc = CPACF_KMCTR_AES_192_ENC; + sctx->dec = CPACF_KMCTR_AES_192_DEC; break; case 32: - sctx->enc = KMCTR_AES_256_ENCRYPT; - sctx->dec = KMCTR_AES_256_DECRYPT; + sctx->enc = CPACF_KMCTR_AES_256_ENC; + sctx->dec = CPACF_KMCTR_AES_256_DEC; break; } @@ -804,8 +805,7 @@ static int ctr_aes_crypt(struct blkcipher_desc *desc, long func, n = __ctrblk_init(ctrptr, nbytes); else n = AES_BLOCK_SIZE; - ret = crypt_s390_kmctr(func, sctx->key, out, in, - n, ctrptr); + ret = cpacf_kmctr(func, sctx->key, out, in, n, ctrptr); if (ret < 0 || ret != n) { if (ctrptr == ctrblk) spin_unlock(&ctrblk_lock); @@ -837,8 +837,8 @@ static int ctr_aes_crypt(struct blkcipher_desc *desc, long func, if (nbytes) { out = walk->dst.virt.addr; in = walk->src.virt.addr; - ret = crypt_s390_kmctr(func, sctx->key, buf, in, - AES_BLOCK_SIZE, ctrbuf); + ret = cpacf_kmctr(func, sctx->key, buf, in, + AES_BLOCK_SIZE, ctrbuf); if (ret < 0 || ret != AES_BLOCK_SIZE) return -EIO; memcpy(out, buf, nbytes); @@ -875,7 +875,7 @@ static int ctr_aes_decrypt(struct blkcipher_desc *desc, static struct crypto_alg ctr_aes_alg = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: aes + ctr */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct s390_aes_ctx), @@ -899,11 +899,11 @@ static int __init aes_s390_init(void) { int ret; - if (crypt_s390_func_available(KM_AES_128_ENCRYPT, CRYPT_S390_MSA)) + if (cpacf_query(CPACF_KM, CPACF_KM_AES_128_ENC)) keylen_flag |= AES_KEYLEN_128; - if (crypt_s390_func_available(KM_AES_192_ENCRYPT, CRYPT_S390_MSA)) + if (cpacf_query(CPACF_KM, CPACF_KM_AES_192_ENC)) keylen_flag |= AES_KEYLEN_192; - if (crypt_s390_func_available(KM_AES_256_ENCRYPT, CRYPT_S390_MSA)) + if (cpacf_query(CPACF_KM, CPACF_KM_AES_256_ENC)) keylen_flag |= AES_KEYLEN_256; if (!keylen_flag) @@ -926,22 +926,17 @@ static int __init aes_s390_init(void) if (ret) goto cbc_aes_err; - if (crypt_s390_func_available(KM_XTS_128_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4) && - crypt_s390_func_available(KM_XTS_256_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + if (cpacf_query(CPACF_KM, CPACF_KM_XTS_128_ENC) && + cpacf_query(CPACF_KM, CPACF_KM_XTS_256_ENC)) { ret = crypto_register_alg(&xts_aes_alg); if (ret) goto xts_aes_err; xts_aes_alg_reg = 1; } - if (crypt_s390_func_available(KMCTR_AES_128_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4) && - crypt_s390_func_available(KMCTR_AES_192_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4) && - crypt_s390_func_available(KMCTR_AES_256_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + if (cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_128_ENC) && + cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_192_ENC) && + cpacf_query(CPACF_KMCTR, CPACF_KMCTR_AES_256_ENC)) { ctrblk = (u8 *) __get_free_page(GFP_KERNEL); if (!ctrblk) { ret = -ENOMEM; diff --git a/arch/s390/crypto/crypt_s390.h b/arch/s390/crypto/crypt_s390.h deleted file mode 100644 index d9c4c313fbc6..000000000000 --- a/arch/s390/crypto/crypt_s390.h +++ /dev/null @@ -1,493 +0,0 @@ -/* - * Cryptographic API. - * - * Support for s390 cryptographic instructions. - * - * Copyright IBM Corp. 2003, 2015 - * Author(s): Thomas Spatzier - * Jan Glauber (jan.glauber@de.ibm.com) - * Harald Freudenberger (freude@de.ibm.com) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ -#ifndef _CRYPTO_ARCH_S390_CRYPT_S390_H -#define _CRYPTO_ARCH_S390_CRYPT_S390_H - -#include <asm/errno.h> -#include <asm/facility.h> - -#define CRYPT_S390_OP_MASK 0xFF00 -#define CRYPT_S390_FUNC_MASK 0x00FF - -#define CRYPT_S390_PRIORITY 300 -#define CRYPT_S390_COMPOSITE_PRIORITY 400 - -#define CRYPT_S390_MSA 0x1 -#define CRYPT_S390_MSA3 0x2 -#define CRYPT_S390_MSA4 0x4 -#define CRYPT_S390_MSA5 0x8 - -/* s390 cryptographic operations */ -enum crypt_s390_operations { - CRYPT_S390_KM = 0x0100, - CRYPT_S390_KMC = 0x0200, - CRYPT_S390_KIMD = 0x0300, - CRYPT_S390_KLMD = 0x0400, - CRYPT_S390_KMAC = 0x0500, - CRYPT_S390_KMCTR = 0x0600, - CRYPT_S390_PPNO = 0x0700 -}; - -/* - * function codes for KM (CIPHER MESSAGE) instruction - * 0x80 is the decipher modifier bit - */ -enum crypt_s390_km_func { - KM_QUERY = CRYPT_S390_KM | 0x0, - KM_DEA_ENCRYPT = CRYPT_S390_KM | 0x1, - KM_DEA_DECRYPT = CRYPT_S390_KM | 0x1 | 0x80, - KM_TDEA_128_ENCRYPT = CRYPT_S390_KM | 0x2, - KM_TDEA_128_DECRYPT = CRYPT_S390_KM | 0x2 | 0x80, - KM_TDEA_192_ENCRYPT = CRYPT_S390_KM | 0x3, - KM_TDEA_192_DECRYPT = CRYPT_S390_KM | 0x3 | 0x80, - KM_AES_128_ENCRYPT = CRYPT_S390_KM | 0x12, - KM_AES_128_DECRYPT = CRYPT_S390_KM | 0x12 | 0x80, - KM_AES_192_ENCRYPT = CRYPT_S390_KM | 0x13, - KM_AES_192_DECRYPT = CRYPT_S390_KM | 0x13 | 0x80, - KM_AES_256_ENCRYPT = CRYPT_S390_KM | 0x14, - KM_AES_256_DECRYPT = CRYPT_S390_KM | 0x14 | 0x80, - KM_XTS_128_ENCRYPT = CRYPT_S390_KM | 0x32, - KM_XTS_128_DECRYPT = CRYPT_S390_KM | 0x32 | 0x80, - KM_XTS_256_ENCRYPT = CRYPT_S390_KM | 0x34, - KM_XTS_256_DECRYPT = CRYPT_S390_KM | 0x34 | 0x80, -}; - -/* - * function codes for KMC (CIPHER MESSAGE WITH CHAINING) - * instruction - */ -enum crypt_s390_kmc_func { - KMC_QUERY = CRYPT_S390_KMC | 0x0, - KMC_DEA_ENCRYPT = CRYPT_S390_KMC | 0x1, - KMC_DEA_DECRYPT = CRYPT_S390_KMC | 0x1 | 0x80, - KMC_TDEA_128_ENCRYPT = CRYPT_S390_KMC | 0x2, - KMC_TDEA_128_DECRYPT = CRYPT_S390_KMC | 0x2 | 0x80, - KMC_TDEA_192_ENCRYPT = CRYPT_S390_KMC | 0x3, - KMC_TDEA_192_DECRYPT = CRYPT_S390_KMC | 0x3 | 0x80, - KMC_AES_128_ENCRYPT = CRYPT_S390_KMC | 0x12, - KMC_AES_128_DECRYPT = CRYPT_S390_KMC | 0x12 | 0x80, - KMC_AES_192_ENCRYPT = CRYPT_S390_KMC | 0x13, - KMC_AES_192_DECRYPT = CRYPT_S390_KMC | 0x13 | 0x80, - KMC_AES_256_ENCRYPT = CRYPT_S390_KMC | 0x14, - KMC_AES_256_DECRYPT = CRYPT_S390_KMC | 0x14 | 0x80, - KMC_PRNG = CRYPT_S390_KMC | 0x43, -}; - -/* - * function codes for KMCTR (CIPHER MESSAGE WITH COUNTER) - * instruction - */ -enum crypt_s390_kmctr_func { - KMCTR_QUERY = CRYPT_S390_KMCTR | 0x0, - KMCTR_DEA_ENCRYPT = CRYPT_S390_KMCTR | 0x1, - KMCTR_DEA_DECRYPT = CRYPT_S390_KMCTR | 0x1 | 0x80, - KMCTR_TDEA_128_ENCRYPT = CRYPT_S390_KMCTR | 0x2, - KMCTR_TDEA_128_DECRYPT = CRYPT_S390_KMCTR | 0x2 | 0x80, - KMCTR_TDEA_192_ENCRYPT = CRYPT_S390_KMCTR | 0x3, - KMCTR_TDEA_192_DECRYPT = CRYPT_S390_KMCTR | 0x3 | 0x80, - KMCTR_AES_128_ENCRYPT = CRYPT_S390_KMCTR | 0x12, - KMCTR_AES_128_DECRYPT = CRYPT_S390_KMCTR | 0x12 | 0x80, - KMCTR_AES_192_ENCRYPT = CRYPT_S390_KMCTR | 0x13, - KMCTR_AES_192_DECRYPT = CRYPT_S390_KMCTR | 0x13 | 0x80, - KMCTR_AES_256_ENCRYPT = CRYPT_S390_KMCTR | 0x14, - KMCTR_AES_256_DECRYPT = CRYPT_S390_KMCTR | 0x14 | 0x80, -}; - -/* - * function codes for KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) - * instruction - */ -enum crypt_s390_kimd_func { - KIMD_QUERY = CRYPT_S390_KIMD | 0, - KIMD_SHA_1 = CRYPT_S390_KIMD | 1, - KIMD_SHA_256 = CRYPT_S390_KIMD | 2, - KIMD_SHA_512 = CRYPT_S390_KIMD | 3, - KIMD_GHASH = CRYPT_S390_KIMD | 65, -}; - -/* - * function codes for KLMD (COMPUTE LAST MESSAGE DIGEST) - * instruction - */ -enum crypt_s390_klmd_func { - KLMD_QUERY = CRYPT_S390_KLMD | 0, - KLMD_SHA_1 = CRYPT_S390_KLMD | 1, - KLMD_SHA_256 = CRYPT_S390_KLMD | 2, - KLMD_SHA_512 = CRYPT_S390_KLMD | 3, -}; - -/* - * function codes for KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) - * instruction - */ -enum crypt_s390_kmac_func { - KMAC_QUERY = CRYPT_S390_KMAC | 0, - KMAC_DEA = CRYPT_S390_KMAC | 1, - KMAC_TDEA_128 = CRYPT_S390_KMAC | 2, - KMAC_TDEA_192 = CRYPT_S390_KMAC | 3 -}; - -/* - * function codes for PPNO (PERFORM PSEUDORANDOM NUMBER - * OPERATION) instruction - */ -enum crypt_s390_ppno_func { - PPNO_QUERY = CRYPT_S390_PPNO | 0, - PPNO_SHA512_DRNG_GEN = CRYPT_S390_PPNO | 3, - PPNO_SHA512_DRNG_SEED = CRYPT_S390_PPNO | 0x83 -}; - -/** - * crypt_s390_km: - * @func: the function code passed to KM; see crypt_s390_km_func - * @param: address of parameter block; see POP for details on each func - * @dest: address of destination memory area - * @src: address of source memory area - * @src_len: length of src operand in bytes - * - * Executes the KM (CIPHER MESSAGE) operation of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for encryption/decryption funcs - */ -static inline int crypt_s390_km(long func, void *param, - u8 *dest, const u8 *src, long src_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - register u8 *__dest asm("4") = dest; - int ret; - - asm volatile( - "0: .insn rre,0xb92e0000,%3,%1\n" /* KM opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest) - : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_kmc: - * @func: the function code passed to KM; see crypt_s390_kmc_func - * @param: address of parameter block; see POP for details on each func - * @dest: address of destination memory area - * @src: address of source memory area - * @src_len: length of src operand in bytes - * - * Executes the KMC (CIPHER MESSAGE WITH CHAINING) operation of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for encryption/decryption funcs - */ -static inline int crypt_s390_kmc(long func, void *param, - u8 *dest, const u8 *src, long src_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - register u8 *__dest asm("4") = dest; - int ret; - - asm volatile( - "0: .insn rre,0xb92f0000,%3,%1\n" /* KMC opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest) - : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_kimd: - * @func: the function code passed to KM; see crypt_s390_kimd_func - * @param: address of parameter block; see POP for details on each func - * @src: address of source memory area - * @src_len: length of src operand in bytes - * - * Executes the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) operation - * of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for digest funcs - */ -static inline int crypt_s390_kimd(long func, void *param, - const u8 *src, long src_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - int ret; - - asm volatile( - "0: .insn rre,0xb93e0000,%1,%1\n" /* KIMD opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "=d" (ret), "+a" (__src), "+d" (__src_len) - : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_klmd: - * @func: the function code passed to KM; see crypt_s390_klmd_func - * @param: address of parameter block; see POP for details on each func - * @src: address of source memory area - * @src_len: length of src operand in bytes - * - * Executes the KLMD (COMPUTE LAST MESSAGE DIGEST) operation of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for digest funcs - */ -static inline int crypt_s390_klmd(long func, void *param, - const u8 *src, long src_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - int ret; - - asm volatile( - "0: .insn rre,0xb93f0000,%1,%1\n" /* KLMD opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "=d" (ret), "+a" (__src), "+d" (__src_len) - : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_kmac: - * @func: the function code passed to KM; see crypt_s390_klmd_func - * @param: address of parameter block; see POP for details on each func - * @src: address of source memory area - * @src_len: length of src operand in bytes - * - * Executes the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) operation - * of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for digest funcs - */ -static inline int crypt_s390_kmac(long func, void *param, - const u8 *src, long src_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - int ret; - - asm volatile( - "0: .insn rre,0xb91e0000,%1,%1\n" /* KLAC opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "=d" (ret), "+a" (__src), "+d" (__src_len) - : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_kmctr: - * @func: the function code passed to KMCTR; see crypt_s390_kmctr_func - * @param: address of parameter block; see POP for details on each func - * @dest: address of destination memory area - * @src: address of source memory area - * @src_len: length of src operand in bytes - * @counter: address of counter value - * - * Executes the KMCTR (CIPHER MESSAGE WITH COUNTER) operation of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of processed - * bytes for encryption/decryption funcs - */ -static inline int crypt_s390_kmctr(long func, void *param, u8 *dest, - const u8 *src, long src_len, u8 *counter) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; - register const u8 *__src asm("2") = src; - register long __src_len asm("3") = src_len; - register u8 *__dest asm("4") = dest; - register u8 *__ctr asm("6") = counter; - int ret = -1; - - asm volatile( - "0: .insn rrf,0xb92d0000,%3,%1,%4,0\n" /* KMCTR opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "+d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest), - "+a" (__ctr) - : "d" (__func), "a" (__param) : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? src_len - __src_len : __src_len; -} - -/** - * crypt_s390_ppno: - * @func: the function code passed to PPNO; see crypt_s390_ppno_func - * @param: address of parameter block; see POP for details on each func - * @dest: address of destination memory area - * @dest_len: size of destination memory area in bytes - * @seed: address of seed data - * @seed_len: size of seed data in bytes - * - * Executes the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION) - * operation of the CPU. - * - * Returns -1 for failure, 0 for the query func, number of random - * bytes stored in dest buffer for generate function - */ -static inline int crypt_s390_ppno(long func, void *param, - u8 *dest, long dest_len, - const u8 *seed, long seed_len) -{ - register long __func asm("0") = func & CRYPT_S390_FUNC_MASK; - register void *__param asm("1") = param; /* param block (240 bytes) */ - register u8 *__dest asm("2") = dest; /* buf for recv random bytes */ - register long __dest_len asm("3") = dest_len; /* requested random bytes */ - register const u8 *__seed asm("4") = seed; /* buf with seed data */ - register long __seed_len asm("5") = seed_len; /* bytes in seed buf */ - int ret = -1; - - asm volatile ( - "0: .insn rre,0xb93c0000,%1,%5\n" /* PPNO opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "+d" (ret), "+a"(__dest), "+d"(__dest_len) - : "d"(__func), "a"(__param), "a"(__seed), "d"(__seed_len) - : "cc", "memory"); - if (ret < 0) - return ret; - return (func & CRYPT_S390_FUNC_MASK) ? dest_len - __dest_len : 0; -} - -/** - * crypt_s390_func_available: - * @func: the function code of the specific function; 0 if op in general - * - * Tests if a specific crypto function is implemented on the machine. - * - * Returns 1 if func available; 0 if func or op in general not available - */ -static inline int crypt_s390_func_available(int func, - unsigned int facility_mask) -{ - unsigned char status[16]; - int ret; - - if (facility_mask & CRYPT_S390_MSA && !test_facility(17)) - return 0; - if (facility_mask & CRYPT_S390_MSA3 && !test_facility(76)) - return 0; - if (facility_mask & CRYPT_S390_MSA4 && !test_facility(77)) - return 0; - if (facility_mask & CRYPT_S390_MSA5 && !test_facility(57)) - return 0; - - switch (func & CRYPT_S390_OP_MASK) { - case CRYPT_S390_KM: - ret = crypt_s390_km(KM_QUERY, &status, NULL, NULL, 0); - break; - case CRYPT_S390_KMC: - ret = crypt_s390_kmc(KMC_QUERY, &status, NULL, NULL, 0); - break; - case CRYPT_S390_KIMD: - ret = crypt_s390_kimd(KIMD_QUERY, &status, NULL, 0); - break; - case CRYPT_S390_KLMD: - ret = crypt_s390_klmd(KLMD_QUERY, &status, NULL, 0); - break; - case CRYPT_S390_KMAC: - ret = crypt_s390_kmac(KMAC_QUERY, &status, NULL, 0); - break; - case CRYPT_S390_KMCTR: - ret = crypt_s390_kmctr(KMCTR_QUERY, &status, - NULL, NULL, 0, NULL); - break; - case CRYPT_S390_PPNO: - ret = crypt_s390_ppno(PPNO_QUERY, &status, - NULL, 0, NULL, 0); - break; - default: - return 0; - } - if (ret < 0) - return 0; - func &= CRYPT_S390_FUNC_MASK; - func &= 0x7f; /* mask modifier bit */ - return (status[func >> 3] & (0x80 >> (func & 7))) != 0; -} - -/** - * crypt_s390_pcc: - * @func: the function code passed to KM; see crypt_s390_km_func - * @param: address of parameter block; see POP for details on each func - * - * Executes the PCC (PERFORM CRYPTOGRAPHIC COMPUTATION) operation of the CPU. - * - * Returns -1 for failure, 0 for success. - */ -static inline int crypt_s390_pcc(long func, void *param) -{ - register long __func asm("0") = func & 0x7f; /* encrypt or decrypt */ - register void *__param asm("1") = param; - int ret = -1; - - asm volatile( - "0: .insn rre,0xb92c0000,0,0\n" /* PCC opcode */ - "1: brc 1,0b\n" /* handle partial completion */ - " la %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) - : "+d" (ret) - : "d" (__func), "a" (__param) : "cc", "memory"); - return ret; -} - -#endif /* _CRYPTO_ARCH_S390_CRYPT_S390_H */ diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index fba1c10a2dd0..697e71a75fc2 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -20,8 +20,7 @@ #include <linux/crypto.h> #include <crypto/algapi.h> #include <crypto/des.h> - -#include "crypt_s390.h" +#include <asm/cpacf.h> #define DES3_KEY_SIZE (3 * DES_KEY_SIZE) @@ -54,20 +53,20 @@ static void des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_DEA_ENCRYPT, ctx->key, out, in, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_DEA_ENC, ctx->key, out, in, DES_BLOCK_SIZE); } static void des_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_DEA_DECRYPT, ctx->key, out, in, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_DEA_DEC, ctx->key, out, in, DES_BLOCK_SIZE); } static struct crypto_alg des_alg = { .cra_name = "des", .cra_driver_name = "des-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -95,7 +94,7 @@ static int ecb_desall_crypt(struct blkcipher_desc *desc, long func, u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_km(func, key, out, in, n); + ret = cpacf_km(func, key, out, in, n); if (ret < 0 || ret != n) return -EIO; @@ -128,7 +127,7 @@ static int cbc_desall_crypt(struct blkcipher_desc *desc, long func, u8 *out = walk->dst.virt.addr; u8 *in = walk->src.virt.addr; - ret = crypt_s390_kmc(func, ¶m, out, in, n); + ret = cpacf_kmc(func, ¶m, out, in, n); if (ret < 0 || ret != n) return -EIO; @@ -149,7 +148,7 @@ static int ecb_des_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_DEA_ENCRYPT, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_DEA_ENC, ctx->key, &walk); } static int ecb_des_decrypt(struct blkcipher_desc *desc, @@ -160,13 +159,13 @@ static int ecb_des_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_DEA_DECRYPT, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_DEA_DEC, ctx->key, &walk); } static struct crypto_alg ecb_des_alg = { .cra_name = "ecb(des)", .cra_driver_name = "ecb-des-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des + ecb */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -190,7 +189,7 @@ static int cbc_des_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_DEA_ENCRYPT, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_DEA_ENC, &walk); } static int cbc_des_decrypt(struct blkcipher_desc *desc, @@ -200,13 +199,13 @@ static int cbc_des_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_DEA_DECRYPT, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_DEA_DEC, &walk); } static struct crypto_alg cbc_des_alg = { .cra_name = "cbc(des)", .cra_driver_name = "cbc-des-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des + cbc */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -258,20 +257,20 @@ static void des3_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_TDEA_192_ENCRYPT, ctx->key, dst, src, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_TDEA_192_ENC, ctx->key, dst, src, DES_BLOCK_SIZE); } static void des3_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct s390_des_ctx *ctx = crypto_tfm_ctx(tfm); - crypt_s390_km(KM_TDEA_192_DECRYPT, ctx->key, dst, src, DES_BLOCK_SIZE); + cpacf_km(CPACF_KM_TDEA_192_DEC, ctx->key, dst, src, DES_BLOCK_SIZE); } static struct crypto_alg des3_alg = { .cra_name = "des3_ede", .cra_driver_name = "des3_ede-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -295,7 +294,7 @@ static int ecb_des3_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_192_ENCRYPT, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_TDEA_192_ENC, ctx->key, &walk); } static int ecb_des3_decrypt(struct blkcipher_desc *desc, @@ -306,13 +305,13 @@ static int ecb_des3_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_192_DECRYPT, ctx->key, &walk); + return ecb_desall_crypt(desc, CPACF_KM_TDEA_192_DEC, ctx->key, &walk); } static struct crypto_alg ecb_des3_alg = { .cra_name = "ecb(des3_ede)", .cra_driver_name = "ecb-des3_ede-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des3 + ecb */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -336,7 +335,7 @@ static int cbc_des3_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_192_ENCRYPT, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192_ENC, &walk); } static int cbc_des3_decrypt(struct blkcipher_desc *desc, @@ -346,13 +345,13 @@ static int cbc_des3_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_192_DECRYPT, &walk); + return cbc_desall_crypt(desc, CPACF_KMC_TDEA_192_DEC, &walk); } static struct crypto_alg cbc_des3_alg = { .cra_name = "cbc(des3_ede)", .cra_driver_name = "cbc-des3_ede-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des3 + cbc */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -407,8 +406,7 @@ static int ctr_desall_crypt(struct blkcipher_desc *desc, long func, n = __ctrblk_init(ctrptr, nbytes); else n = DES_BLOCK_SIZE; - ret = crypt_s390_kmctr(func, ctx->key, out, in, - n, ctrptr); + ret = cpacf_kmctr(func, ctx->key, out, in, n, ctrptr); if (ret < 0 || ret != n) { if (ctrptr == ctrblk) spin_unlock(&ctrblk_lock); @@ -438,8 +436,8 @@ static int ctr_desall_crypt(struct blkcipher_desc *desc, long func, if (nbytes) { out = walk->dst.virt.addr; in = walk->src.virt.addr; - ret = crypt_s390_kmctr(func, ctx->key, buf, in, - DES_BLOCK_SIZE, ctrbuf); + ret = cpacf_kmctr(func, ctx->key, buf, in, + DES_BLOCK_SIZE, ctrbuf); if (ret < 0 || ret != DES_BLOCK_SIZE) return -EIO; memcpy(out, buf, nbytes); @@ -458,7 +456,7 @@ static int ctr_des_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, KMCTR_DEA_ENCRYPT, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_DEA_ENC, ctx, &walk); } static int ctr_des_decrypt(struct blkcipher_desc *desc, @@ -469,13 +467,13 @@ static int ctr_des_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, KMCTR_DEA_DECRYPT, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_DEA_DEC, ctx, &walk); } static struct crypto_alg ctr_des_alg = { .cra_name = "ctr(des)", .cra_driver_name = "ctr-des-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des + ctr */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -501,7 +499,7 @@ static int ctr_des3_encrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, KMCTR_TDEA_192_ENCRYPT, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192_ENC, ctx, &walk); } static int ctr_des3_decrypt(struct blkcipher_desc *desc, @@ -512,13 +510,13 @@ static int ctr_des3_decrypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; blkcipher_walk_init(&walk, dst, src, nbytes); - return ctr_desall_crypt(desc, KMCTR_TDEA_192_DECRYPT, ctx, &walk); + return ctr_desall_crypt(desc, CPACF_KMCTR_TDEA_192_DEC, ctx, &walk); } static struct crypto_alg ctr_des3_alg = { .cra_name = "ctr(des3_ede)", .cra_driver_name = "ctr-des3_ede-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, + .cra_priority = 400, /* combo: des3 + ede */ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct s390_des_ctx), @@ -540,8 +538,8 @@ static int __init des_s390_init(void) { int ret; - if (!crypt_s390_func_available(KM_DEA_ENCRYPT, CRYPT_S390_MSA) || - !crypt_s390_func_available(KM_TDEA_192_ENCRYPT, CRYPT_S390_MSA)) + if (!cpacf_query(CPACF_KM, CPACF_KM_DEA_ENC) || + !cpacf_query(CPACF_KM, CPACF_KM_TDEA_192_ENC)) return -EOPNOTSUPP; ret = crypto_register_alg(&des_alg); @@ -563,10 +561,8 @@ static int __init des_s390_init(void) if (ret) goto cbc_des3_err; - if (crypt_s390_func_available(KMCTR_DEA_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4) && - crypt_s390_func_available(KMCTR_TDEA_192_ENCRYPT, - CRYPT_S390_MSA | CRYPT_S390_MSA4)) { + if (cpacf_query(CPACF_KMCTR, CPACF_KMCTR_DEA_ENC) && + cpacf_query(CPACF_KMCTR, CPACF_KMCTR_TDEA_192_ENC)) { ret = crypto_register_alg(&ctr_des_alg); if (ret) goto ctr_des_err; diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c index 26e14efd30a7..ab68de72e795 100644 --- a/arch/s390/crypto/ghash_s390.c +++ b/arch/s390/crypto/ghash_s390.c @@ -10,8 +10,7 @@ #include <crypto/internal/hash.h> #include <linux/module.h> #include <linux/cpufeature.h> - -#include "crypt_s390.h" +#include <asm/cpacf.h> #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 @@ -72,8 +71,8 @@ static int ghash_update(struct shash_desc *desc, src += n; if (!dctx->bytes) { - ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, - GHASH_BLOCK_SIZE); + ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, + GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) return -EIO; } @@ -81,7 +80,7 @@ static int ghash_update(struct shash_desc *desc, n = srclen & ~(GHASH_BLOCK_SIZE - 1); if (n) { - ret = crypt_s390_kimd(KIMD_GHASH, dctx, src, n); + ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); if (ret != n) return -EIO; src += n; @@ -106,7 +105,7 @@ static int ghash_flush(struct ghash_desc_ctx *dctx) memset(pos, 0, dctx->bytes); - ret = crypt_s390_kimd(KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); + ret = cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); if (ret != GHASH_BLOCK_SIZE) return -EIO; @@ -137,7 +136,7 @@ static struct shash_alg ghash_alg = { .base = { .cra_name = "ghash", .cra_driver_name = "ghash-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct ghash_ctx), @@ -147,8 +146,7 @@ static struct shash_alg ghash_alg = { static int __init ghash_mod_init(void) { - if (!crypt_s390_func_available(KIMD_GHASH, - CRYPT_S390_MSA | CRYPT_S390_MSA4)) + if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_GHASH)) return -EOPNOTSUPP; return crypto_register_shash(&ghash_alg); diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index b8045b97f4fb..41527b113f5a 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -23,8 +23,7 @@ #include <asm/debug.h> #include <asm/uaccess.h> #include <asm/timex.h> - -#include "crypt_s390.h" +#include <asm/cpacf.h> MODULE_LICENSE("GPL"); MODULE_AUTHOR("IBM Corporation"); @@ -136,8 +135,8 @@ static int generate_entropy(u8 *ebuf, size_t nbytes) else h = ebuf; /* generate sha256 from this page */ - if (crypt_s390_kimd(KIMD_SHA_256, h, - pg, PAGE_SIZE) != PAGE_SIZE) { + if (cpacf_kimd(CPACF_KIMD_SHA_256, h, + pg, PAGE_SIZE) != PAGE_SIZE) { prng_errorflag = PRNG_GEN_ENTROPY_FAILED; ret = -EIO; goto out; @@ -164,9 +163,9 @@ static void prng_tdes_add_entropy(void) int ret; for (i = 0; i < 16; i++) { - ret = crypt_s390_kmc(KMC_PRNG, prng_data->prngws.parm_block, - (char *)entropy, (char *)entropy, - sizeof(entropy)); + ret = cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, + (char *)entropy, (char *)entropy, + sizeof(entropy)); BUG_ON(ret < 0 || ret != sizeof(entropy)); memcpy(prng_data->prngws.parm_block, entropy, sizeof(entropy)); } @@ -311,9 +310,8 @@ static int __init prng_sha512_selftest(void) memset(&ws, 0, sizeof(ws)); /* initial seed */ - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED, - &ws, NULL, 0, - seed, sizeof(seed)); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, &ws, NULL, 0, + seed, sizeof(seed)); if (ret < 0) { pr_err("The prng self test seed operation for the " "SHA-512 mode failed with rc=%d\n", ret); @@ -331,18 +329,16 @@ static int __init prng_sha512_selftest(void) } /* generate random bytes */ - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN, - &ws, buf, sizeof(buf), - NULL, 0); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &ws, buf, sizeof(buf), NULL, 0); if (ret < 0) { pr_err("The prng self test generate operation for " "the SHA-512 mode failed with rc=%d\n", ret); prng_errorflag = PRNG_SELFTEST_FAILED; return -EIO; } - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN, - &ws, buf, sizeof(buf), - NULL, 0); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &ws, buf, sizeof(buf), NULL, 0); if (ret < 0) { pr_err("The prng self test generate operation for " "the SHA-512 mode failed with rc=%d\n", ret); @@ -396,9 +392,8 @@ static int __init prng_sha512_instantiate(void) get_tod_clock_ext(seed + 48); /* initial seed of the ppno drng */ - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED, - &prng_data->ppnows, NULL, 0, - seed, sizeof(seed)); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, + &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); if (ret < 0) { prng_errorflag = PRNG_SEED_FAILED; ret = -EIO; @@ -409,11 +404,9 @@ static int __init prng_sha512_instantiate(void) bytes for the FIPS 140-2 Conditional Self Test */ if (fips_enabled) { prng_data->prev = prng_data->buf + prng_chunk_size; - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN, - &prng_data->ppnows, - prng_data->prev, - prng_chunk_size, - NULL, 0); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &prng_data->ppnows, + prng_data->prev, prng_chunk_size, NULL, 0); if (ret < 0 || ret != prng_chunk_size) { prng_errorflag = PRNG_GEN_FAILED; ret = -EIO; @@ -447,9 +440,8 @@ static int prng_sha512_reseed(void) return ret; /* do a reseed of the ppno drng with this bytestring */ - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED, - &prng_data->ppnows, NULL, 0, - seed, sizeof(seed)); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, + &prng_data->ppnows, NULL, 0, seed, sizeof(seed)); if (ret) { prng_errorflag = PRNG_RESEED_FAILED; return -EIO; @@ -471,9 +463,8 @@ static int prng_sha512_generate(u8 *buf, size_t nbytes) } /* PPNO generate */ - ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN, - &prng_data->ppnows, buf, nbytes, - NULL, 0); + ret = cpacf_ppno(CPACF_PPNO_SHA512_DRNG_GEN, + &prng_data->ppnows, buf, nbytes, NULL, 0); if (ret < 0 || ret != nbytes) { prng_errorflag = PRNG_GEN_FAILED; return -EIO; @@ -555,8 +546,8 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, * Note: you can still get strict X9.17 conformity by setting * prng_chunk_size to 8 bytes. */ - tmp = crypt_s390_kmc(KMC_PRNG, prng_data->prngws.parm_block, - prng_data->buf, prng_data->buf, n); + tmp = cpacf_kmc(CPACF_KMC_PRNG, prng_data->prngws.parm_block, + prng_data->buf, prng_data->buf, n); if (tmp < 0 || tmp != n) { ret = -EIO; break; @@ -669,11 +660,13 @@ static const struct file_operations prng_tdes_fops = { static struct miscdevice prng_sha512_dev = { .name = "prandom", .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, .fops = &prng_sha512_fops, }; static struct miscdevice prng_tdes_dev = { .name = "prandom", .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, .fops = &prng_tdes_fops, }; @@ -813,14 +806,13 @@ static int __init prng_init(void) int ret; /* check if the CPU has a PRNG */ - if (!crypt_s390_func_available(KMC_PRNG, CRYPT_S390_MSA)) + if (!cpacf_query(CPACF_KMC, CPACF_KMC_PRNG)) return -EOPNOTSUPP; /* choose prng mode */ if (prng_mode != PRNG_MODE_TDES) { /* check for MSA5 support for PPNO operations */ - if (!crypt_s390_func_available(PPNO_SHA512_DRNG_GEN, - CRYPT_S390_MSA5)) { + if (!cpacf_query(CPACF_PPNO, CPACF_PPNO_SHA512_DRNG_GEN)) { if (prng_mode == PRNG_MODE_SHA512) { pr_err("The prng module cannot " "start in SHA-512 mode\n"); diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 9208eadae9f0..5fbf91bbb478 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -28,8 +28,8 @@ #include <linux/module.h> #include <linux/cpufeature.h> #include <crypto/sha.h> +#include <asm/cpacf.h> -#include "crypt_s390.h" #include "sha.h" static int sha1_init(struct shash_desc *desc) @@ -42,7 +42,7 @@ static int sha1_init(struct shash_desc *desc) sctx->state[3] = SHA1_H3; sctx->state[4] = SHA1_H4; sctx->count = 0; - sctx->func = KIMD_SHA_1; + sctx->func = CPACF_KIMD_SHA_1; return 0; } @@ -66,7 +66,7 @@ static int sha1_import(struct shash_desc *desc, const void *in) sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); memcpy(sctx->buf, ictx->buffer, sizeof(ictx->buffer)); - sctx->func = KIMD_SHA_1; + sctx->func = CPACF_KIMD_SHA_1; return 0; } @@ -82,7 +82,7 @@ static struct shash_alg alg = { .base = { .cra_name = "sha1", .cra_driver_name= "sha1-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -91,7 +91,7 @@ static struct shash_alg alg = { static int __init sha1_s390_init(void) { - if (!crypt_s390_func_available(KIMD_SHA_1, CRYPT_S390_MSA)) + if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_1)) return -EOPNOTSUPP; return crypto_register_shash(&alg); } diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index 667888f5c964..10aac0b11988 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -18,8 +18,8 @@ #include <linux/module.h> #include <linux/cpufeature.h> #include <crypto/sha.h> +#include <asm/cpacf.h> -#include "crypt_s390.h" #include "sha.h" static int sha256_init(struct shash_desc *desc) @@ -35,7 +35,7 @@ static int sha256_init(struct shash_desc *desc) sctx->state[6] = SHA256_H6; sctx->state[7] = SHA256_H7; sctx->count = 0; - sctx->func = KIMD_SHA_256; + sctx->func = CPACF_KIMD_SHA_256; return 0; } @@ -59,7 +59,7 @@ static int sha256_import(struct shash_desc *desc, const void *in) sctx->count = ictx->count; memcpy(sctx->state, ictx->state, sizeof(ictx->state)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->func = KIMD_SHA_256; + sctx->func = CPACF_KIMD_SHA_256; return 0; } @@ -75,7 +75,7 @@ static struct shash_alg sha256_alg = { .base = { .cra_name = "sha256", .cra_driver_name= "sha256-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -95,7 +95,7 @@ static int sha224_init(struct shash_desc *desc) sctx->state[6] = SHA224_H6; sctx->state[7] = SHA224_H7; sctx->count = 0; - sctx->func = KIMD_SHA_256; + sctx->func = CPACF_KIMD_SHA_256; return 0; } @@ -112,7 +112,7 @@ static struct shash_alg sha224_alg = { .base = { .cra_name = "sha224", .cra_driver_name= "sha224-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -123,7 +123,7 @@ static int __init sha256_s390_init(void) { int ret; - if (!crypt_s390_func_available(KIMD_SHA_256, CRYPT_S390_MSA)) + if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_256)) return -EOPNOTSUPP; ret = crypto_register_shash(&sha256_alg); if (ret < 0) diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 2ba66b1518f0..ea85757be407 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -19,9 +19,9 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/cpufeature.h> +#include <asm/cpacf.h> #include "sha.h" -#include "crypt_s390.h" static int sha512_init(struct shash_desc *desc) { @@ -36,7 +36,7 @@ static int sha512_init(struct shash_desc *desc) *(__u64 *)&ctx->state[12] = 0x1f83d9abfb41bd6bULL; *(__u64 *)&ctx->state[14] = 0x5be0cd19137e2179ULL; ctx->count = 0; - ctx->func = KIMD_SHA_512; + ctx->func = CPACF_KIMD_SHA_512; return 0; } @@ -64,7 +64,7 @@ static int sha512_import(struct shash_desc *desc, const void *in) memcpy(sctx->state, ictx->state, sizeof(ictx->state)); memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); - sctx->func = KIMD_SHA_512; + sctx->func = CPACF_KIMD_SHA_512; return 0; } @@ -80,7 +80,7 @@ static struct shash_alg sha512_alg = { .base = { .cra_name = "sha512", .cra_driver_name= "sha512-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, @@ -102,7 +102,7 @@ static int sha384_init(struct shash_desc *desc) *(__u64 *)&ctx->state[12] = 0xdb0c2e0d64f98fa7ULL; *(__u64 *)&ctx->state[14] = 0x47b5481dbefa4fa4ULL; ctx->count = 0; - ctx->func = KIMD_SHA_512; + ctx->func = CPACF_KIMD_SHA_512; return 0; } @@ -119,7 +119,7 @@ static struct shash_alg sha384_alg = { .base = { .cra_name = "sha384", .cra_driver_name= "sha384-s390", - .cra_priority = CRYPT_S390_PRIORITY, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_SHASH, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct s390_sha_ctx), @@ -133,7 +133,7 @@ static int __init init(void) { int ret; - if (!crypt_s390_func_available(KIMD_SHA_512, CRYPT_S390_MSA)) + if (!cpacf_query(CPACF_KIMD, CPACF_KIMD_SHA_512)) return -EOPNOTSUPP; if ((ret = crypto_register_shash(&sha512_alg)) < 0) goto out; diff --git a/arch/s390/crypto/sha_common.c b/arch/s390/crypto/sha_common.c index 8620b0ec9c42..8e908166c3ee 100644 --- a/arch/s390/crypto/sha_common.c +++ b/arch/s390/crypto/sha_common.c @@ -15,8 +15,8 @@ #include <crypto/internal/hash.h> #include <linux/module.h> +#include <asm/cpacf.h> #include "sha.h" -#include "crypt_s390.h" int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -35,7 +35,7 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) /* process one stored block */ if (index) { memcpy(ctx->buf + index, data, bsize - index); - ret = crypt_s390_kimd(ctx->func, ctx->state, ctx->buf, bsize); + ret = cpacf_kimd(ctx->func, ctx->state, ctx->buf, bsize); if (ret != bsize) return -EIO; data += bsize - index; @@ -45,8 +45,8 @@ int s390_sha_update(struct shash_desc *desc, const u8 *data, unsigned int len) /* process as many blocks as possible */ if (len >= bsize) { - ret = crypt_s390_kimd(ctx->func, ctx->state, data, - len & ~(bsize - 1)); + ret = cpacf_kimd(ctx->func, ctx->state, data, + len & ~(bsize - 1)); if (ret != (len & ~(bsize - 1))) return -EIO; data += ret; @@ -89,7 +89,7 @@ int s390_sha_final(struct shash_desc *desc, u8 *out) bits = ctx->count * 8; memcpy(ctx->buf + end - 8, &bits, sizeof(bits)); - ret = crypt_s390_kimd(ctx->func, ctx->state, ctx->buf, end); + ret = cpacf_kimd(ctx->func, ctx->state, ctx->buf, end); if (ret != end) return -EIO; diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index 0f3da2cb2bd6..255c7eec4481 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c @@ -278,8 +278,8 @@ static int hypfs_fill_super(struct super_block *sb, void *data, int silent) sbi->uid = current_uid(); sbi->gid = current_gid(); sb->s_fs_info = sbi; - sb->s_blocksize = PAGE_CACHE_SIZE; - sb->s_blocksize_bits = PAGE_CACHE_SHIFT; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; sb->s_magic = HYPFS_MAGIC; sb->s_op = &hypfs_s_ops; if (hypfs_parse_options(data, sb)) diff --git a/arch/s390/include/asm/cache.h b/arch/s390/include/asm/cache.h index 4d7ccac5fd1d..22da3b34c655 100644 --- a/arch/s390/include/asm/cache.h +++ b/arch/s390/include/asm/cache.h @@ -15,4 +15,7 @@ #define __read_mostly __attribute__((__section__(".data..read_mostly"))) +/* Read-only memory is marked before mark_rodata_ro() is called. */ +#define __ro_after_init __read_mostly + #endif diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h new file mode 100644 index 000000000000..1a82cf26ee11 --- /dev/null +++ b/arch/s390/include/asm/cpacf.h @@ -0,0 +1,410 @@ +/* + * CP Assist for Cryptographic Functions (CPACF) + * + * Copyright IBM Corp. 2003, 2016 + * Author(s): Thomas Spatzier + * Jan Glauber + * Harald Freudenberger (freude@de.ibm.com) + * Martin Schwidefsky <schwidefsky@de.ibm.com> + */ +#ifndef _ASM_S390_CPACF_H +#define _ASM_S390_CPACF_H + +#include <asm/facility.h> + +/* + * Instruction opcodes for the CPACF instructions + */ +#define CPACF_KMAC 0xb91e /* MSA */ +#define CPACF_KM 0xb92e /* MSA */ +#define CPACF_KMC 0xb92f /* MSA */ +#define CPACF_KIMD 0xb93e /* MSA */ +#define CPACF_KLMD 0xb93f /* MSA */ +#define CPACF_PCC 0xb92c /* MSA4 */ +#define CPACF_KMCTR 0xb92d /* MSA4 */ +#define CPACF_PPNO 0xb93c /* MSA5 */ + +/* + * Function codes for the KM (CIPHER MESSAGE) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KM_QUERY 0x00 +#define CPACF_KM_DEA_ENC 0x01 +#define CPACF_KM_DEA_DEC 0x81 +#define CPACF_KM_TDEA_128_ENC 0x02 +#define CPACF_KM_TDEA_128_DEC 0x82 +#define CPACF_KM_TDEA_192_ENC 0x03 +#define CPACF_KM_TDEA_192_DEC 0x83 +#define CPACF_KM_AES_128_ENC 0x12 +#define CPACF_KM_AES_128_DEC 0x92 +#define CPACF_KM_AES_192_ENC 0x13 +#define CPACF_KM_AES_192_DEC 0x93 +#define CPACF_KM_AES_256_ENC 0x14 +#define CPACF_KM_AES_256_DEC 0x94 +#define CPACF_KM_XTS_128_ENC 0x32 +#define CPACF_KM_XTS_128_DEC 0xb2 +#define CPACF_KM_XTS_256_ENC 0x34 +#define CPACF_KM_XTS_256_DEC 0xb4 + +/* + * Function codes for the KMC (CIPHER MESSAGE WITH CHAINING) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KMC_QUERY 0x00 +#define CPACF_KMC_DEA_ENC 0x01 +#define CPACF_KMC_DEA_DEC 0x81 +#define CPACF_KMC_TDEA_128_ENC 0x02 +#define CPACF_KMC_TDEA_128_DEC 0x82 +#define CPACF_KMC_TDEA_192_ENC 0x03 +#define CPACF_KMC_TDEA_192_DEC 0x83 +#define CPACF_KMC_AES_128_ENC 0x12 +#define CPACF_KMC_AES_128_DEC 0x92 +#define CPACF_KMC_AES_192_ENC 0x13 +#define CPACF_KMC_AES_192_DEC 0x93 +#define CPACF_KMC_AES_256_ENC 0x14 +#define CPACF_KMC_AES_256_DEC 0x94 +#define CPACF_KMC_PRNG 0x43 + +/* + * Function codes for the KMCTR (CIPHER MESSAGE WITH COUNTER) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KMCTR_QUERY 0x00 +#define CPACF_KMCTR_DEA_ENC 0x01 +#define CPACF_KMCTR_DEA_DEC 0x81 +#define CPACF_KMCTR_TDEA_128_ENC 0x02 +#define CPACF_KMCTR_TDEA_128_DEC 0x82 +#define CPACF_KMCTR_TDEA_192_ENC 0x03 +#define CPACF_KMCTR_TDEA_192_DEC 0x83 +#define CPACF_KMCTR_AES_128_ENC 0x12 +#define CPACF_KMCTR_AES_128_DEC 0x92 +#define CPACF_KMCTR_AES_192_ENC 0x13 +#define CPACF_KMCTR_AES_192_DEC 0x93 +#define CPACF_KMCTR_AES_256_ENC 0x14 +#define CPACF_KMCTR_AES_256_DEC 0x94 + +/* + * Function codes for the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KIMD_QUERY 0x00 +#define CPACF_KIMD_SHA_1 0x01 +#define CPACF_KIMD_SHA_256 0x02 +#define CPACF_KIMD_SHA_512 0x03 +#define CPACF_KIMD_GHASH 0x41 + +/* + * Function codes for the KLMD (COMPUTE LAST MESSAGE DIGEST) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KLMD_QUERY 0x00 +#define CPACF_KLMD_SHA_1 0x01 +#define CPACF_KLMD_SHA_256 0x02 +#define CPACF_KLMD_SHA_512 0x03 + +/* + * function codes for the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_KMAC_QUERY 0x00 +#define CPACF_KMAC_DEA 0x01 +#define CPACF_KMAC_TDEA_128 0x02 +#define CPACF_KMAC_TDEA_192 0x03 + +/* + * Function codes for the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION) + * instruction (0x80 is the decipher modifier bit) + */ +#define CPACF_PPNO_QUERY 0x00 +#define CPACF_PPNO_SHA512_DRNG_GEN 0x03 +#define CPACF_PPNO_SHA512_DRNG_SEED 0x83 + +/** + * cpacf_query() - check if a specific CPACF function is available + * @opcode: the opcode of the crypto instruction + * @func: the function code to test for + * + * Executes the query function for the given crypto instruction @opcode + * and checks if @func is available + * + * Returns 1 if @func is available for @opcode, 0 otherwise + */ +static inline void __cpacf_query(unsigned int opcode, unsigned char *status) +{ + typedef struct { unsigned char _[16]; } status_type; + register unsigned long r0 asm("0") = 0; /* query function */ + register unsigned long r1 asm("1") = (unsigned long) status; + + asm volatile( + /* Parameter registers are ignored, but may not be 0 */ + "0: .insn rrf,%[opc] << 16,2,2,2,0\n" + " brc 1,0b\n" /* handle partial completion */ + : "=m" (*(status_type *) status) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (opcode) + : "cc"); +} + +static inline int cpacf_query(unsigned int opcode, unsigned int func) +{ + unsigned char status[16]; + + switch (opcode) { + case CPACF_KMAC: + case CPACF_KM: + case CPACF_KMC: + case CPACF_KIMD: + case CPACF_KLMD: + if (!test_facility(17)) /* check for MSA */ + return 0; + break; + case CPACF_PCC: + case CPACF_KMCTR: + if (!test_facility(77)) /* check for MSA4 */ + return 0; + break; + case CPACF_PPNO: + if (!test_facility(57)) /* check for MSA5 */ + return 0; + break; + default: + BUG(); + } + __cpacf_query(opcode, status); + return (status[func >> 3] & (0x80 >> (func & 7))) != 0; +} + +/** + * cpacf_km() - executes the KM (CIPHER MESSAGE) instruction + * @func: the function code passed to KM; see CPACF_KM_xxx defines + * @param: address of parameter block; see POP for details on each func + * @dest: address of destination memory area + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for + * encryption/decryption funcs + */ +static inline int cpacf_km(long func, void *param, + u8 *dest, const u8 *src, long src_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + register unsigned long r4 asm("4") = (unsigned long) dest; + + asm volatile( + "0: .insn rre,%[opc] << 16,%[dst],%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3), [dst] "+a" (r4) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KM) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_kmc() - executes the KMC (CIPHER MESSAGE WITH CHAINING) instruction + * @func: the function code passed to KM; see CPACF_KMC_xxx defines + * @param: address of parameter block; see POP for details on each func + * @dest: address of destination memory area + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for + * encryption/decryption funcs + */ +static inline int cpacf_kmc(long func, void *param, + u8 *dest, const u8 *src, long src_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + register unsigned long r4 asm("4") = (unsigned long) dest; + + asm volatile( + "0: .insn rre,%[opc] << 16,%[dst],%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3), [dst] "+a" (r4) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KMC) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_kimd() - executes the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) + * instruction + * @func: the function code passed to KM; see CPACF_KIMD_xxx defines + * @param: address of parameter block; see POP for details on each func + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for digest funcs + */ +static inline int cpacf_kimd(long func, void *param, + const u8 *src, long src_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + + asm volatile( + "0: .insn rre,%[opc] << 16,0,%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KIMD) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_klmd() - executes the KLMD (COMPUTE LAST MESSAGE DIGEST) instruction + * @func: the function code passed to KM; see CPACF_KLMD_xxx defines + * @param: address of parameter block; see POP for details on each func + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for digest funcs + */ +static inline int cpacf_klmd(long func, void *param, + const u8 *src, long src_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + + asm volatile( + "0: .insn rre,%[opc] << 16,0,%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KLMD) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_kmac() - executes the KMAC (COMPUTE MESSAGE AUTHENTICATION CODE) + * instruction + * @func: the function code passed to KM; see CPACF_KMAC_xxx defines + * @param: address of parameter block; see POP for details on each func + * @src: address of source memory area + * @src_len: length of src operand in bytes + * + * Returns 0 for the query func, number of processed bytes for digest funcs + */ +static inline int cpacf_kmac(long func, void *param, + const u8 *src, long src_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + + asm volatile( + "0: .insn rre,%[opc] << 16,0,%[src]\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KMAC) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_kmctr() - executes the KMCTR (CIPHER MESSAGE WITH COUNTER) instruction + * @func: the function code passed to KMCTR; see CPACF_KMCTR_xxx defines + * @param: address of parameter block; see POP for details on each func + * @dest: address of destination memory area + * @src: address of source memory area + * @src_len: length of src operand in bytes + * @counter: address of counter value + * + * Returns 0 for the query func, number of processed bytes for + * encryption/decryption funcs + */ +static inline int cpacf_kmctr(long func, void *param, u8 *dest, + const u8 *src, long src_len, u8 *counter) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) src; + register unsigned long r3 asm("3") = (unsigned long) src_len; + register unsigned long r4 asm("4") = (unsigned long) dest; + register unsigned long r6 asm("6") = (unsigned long) counter; + + asm volatile( + "0: .insn rrf,%[opc] << 16,%[dst],%[src],%[ctr],0\n" + " brc 1,0b\n" /* handle partial completion */ + : [src] "+a" (r2), [len] "+d" (r3), + [dst] "+a" (r4), [ctr] "+a" (r6) + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_KMCTR) + : "cc", "memory"); + + return src_len - r3; +} + +/** + * cpacf_ppno() - executes the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION) + * instruction + * @func: the function code passed to PPNO; see CPACF_PPNO_xxx defines + * @param: address of parameter block; see POP for details on each func + * @dest: address of destination memory area + * @dest_len: size of destination memory area in bytes + * @seed: address of seed data + * @seed_len: size of seed data in bytes + * + * Returns 0 for the query func, number of random bytes stored in + * dest buffer for generate function + */ +static inline int cpacf_ppno(long func, void *param, + u8 *dest, long dest_len, + const u8 *seed, long seed_len) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + register unsigned long r2 asm("2") = (unsigned long) dest; + register unsigned long r3 asm("3") = (unsigned long) dest_len; + register unsigned long r4 asm("4") = (unsigned long) seed; + register unsigned long r5 asm("5") = (unsigned long) seed_len; + + asm volatile ( + "0: .insn rre,%[opc] << 16,%[dst],%[seed]\n" + " brc 1,0b\n" /* handle partial completion */ + : [dst] "+a" (r2), [dlen] "+d" (r3) + : [fc] "d" (r0), [pba] "a" (r1), + [seed] "a" (r4), [slen] "d" (r5), [opc] "i" (CPACF_PPNO) + : "cc", "memory"); + + return dest_len - r3; +} + +/** + * cpacf_pcc() - executes the PCC (PERFORM CRYPTOGRAPHIC COMPUTATION) + * instruction + * @func: the function code passed to PCC; see CPACF_KM_xxx defines + * @param: address of parameter block; see POP for details on each func + * + * Returns 0. + */ +static inline int cpacf_pcc(long func, void *param) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + + asm volatile( + "0: .insn rre,%[opc] << 16,0,0\n" /* PCC opcode */ + " brc 1,0b\n" /* handle partial completion */ + : + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_PCC) + : "cc", "memory"); + + return 0; +} + +#endif /* _ASM_S390_CPACF_H */ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu/types.h index 14a8b0c14f87..fe937c9b6471 100644 --- a/arch/s390/include/asm/fpu/types.h +++ b/arch/s390/include/asm/fpu/types.h @@ -11,11 +11,13 @@ #include <asm/sigcontext.h> struct fpu { - __u32 fpc; /* Floating-point control */ + __u32 fpc; /* Floating-point control */ + void *regs; /* Pointer to the current save area */ union { - void *regs; - freg_t *fprs; /* Floating-point register save area */ - __vector128 *vxrs; /* Vector register save area */ + /* Floating-point register save area */ + freg_t fprs[__NUM_FPRS]; + /* Vector register save area */ + __vector128 vxrs[__NUM_VXRS]; }; }; diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 836c56290499..64053d9ac3f2 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -12,7 +12,9 @@ #ifndef __ASSEMBLY__ -#define ftrace_return_address(n) __builtin_return_address(n) +unsigned long return_address(int depth); + +#define ftrace_return_address(n) return_address(n) void _mcount(void); void ftrace_caller(void); diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 6da41fab70fb..37b9017c6a96 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -38,7 +38,7 @@ */ #define KVM_NR_IRQCHIPS 1 #define KVM_IRQCHIP_NUM_PINS 4096 -#define KVM_HALT_POLL_NS_DEFAULT 0 +#define KVM_HALT_POLL_NS_DEFAULT 80000 /* s390-specific vcpu->requests bit members */ #define KVM_REQ_ENABLE_IBS 8 @@ -247,6 +247,7 @@ struct kvm_vcpu_stat { u32 exit_instruction; u32 halt_successful_poll; u32 halt_attempted_poll; + u32 halt_poll_invalid; u32 halt_wakeup; u32 instruction_lctl; u32 instruction_lctlg; @@ -544,10 +545,6 @@ struct kvm_vcpu_arch { struct kvm_s390_local_interrupt local_int; struct hrtimer ckc_timer; struct kvm_s390_pgm_info pgm; - union { - struct cpuid cpu_id; - u64 stidp_data; - }; struct gmap *gmap; struct kvm_guestdbg_info_arch guestdbg; unsigned long pfault_token; @@ -605,7 +602,7 @@ struct kvm_s390_cpu_model { __u64 fac_mask[S390_ARCH_FAC_LIST_SIZE_U64]; /* facility list requested by guest (in dma page) */ __u64 *fac_list; - struct cpuid cpu_id; + u64 cpuid; unsigned short ibc; }; @@ -700,4 +697,6 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu); + #endif diff --git a/arch/s390/include/asm/livepatch.h b/arch/s390/include/asm/livepatch.h index d5427c78b1b3..2c1213785892 100644 --- a/arch/s390/include/asm/livepatch.h +++ b/arch/s390/include/asm/livepatch.h @@ -24,13 +24,6 @@ static inline int klp_check_compiler_support(void) return 0; } -static inline int klp_write_module_reloc(struct module *mod, unsigned long - type, unsigned long loc, unsigned long value) -{ - /* not supported yet */ - return -ENOSYS; -} - static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) { regs->psw.addr = ip; diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index d29ad9545b41..081b2ad99d73 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -11,7 +11,7 @@ typedef struct { spinlock_t list_lock; struct list_head pgtable_list; struct list_head gmap_list; - unsigned long asce_bits; + unsigned long asce; unsigned long asce_limit; unsigned long vdso_base; /* The mmu context allocates 4K page tables. */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index d321469eeda7..c837b79b455d 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -26,12 +26,28 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.has_pgste = 0; mm->context.use_skey = 0; #endif - if (mm->context.asce_limit == 0) { + switch (mm->context.asce_limit) { + case 1UL << 42: + /* + * forked 3-level task, fall through to set new asce with new + * mm->pgd + */ + case 0: /* context created by exec, set asce limit to 4TB */ - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION3; mm->context.asce_limit = STACK_TOP_MAX; - } else if (mm->context.asce_limit == (1UL << 31)) { + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION3; + break; + case 1UL << 53: + /* forked 4-level task, set new asce with new mm->pgd */ + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + break; + case 1UL << 31: + /* forked 2-level compat task, set new asce with new mm->pgd */ + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; + /* pgd_alloc() did not increase mm->nr_pmds */ mm_inc_nr_pmds(mm); } crst_table_init((unsigned long *) mm->pgd, pgd_entry_type(mm)); @@ -42,7 +58,7 @@ static inline int init_new_context(struct task_struct *tsk, static inline void set_user_asce(struct mm_struct *mm) { - S390_lowcore.user_asce = mm->context.asce_bits | __pa(mm->pgd); + S390_lowcore.user_asce = mm->context.asce; if (current->thread.mm_segment.ar4) __ctl_load(S390_lowcore.user_asce, 7, 7); set_cpu_flag(CIF_ASCE); @@ -71,7 +87,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, { int cpu = smp_processor_id(); - S390_lowcore.user_asce = next->context.asce_bits | __pa(next->pgd); + S390_lowcore.user_asce = next->context.asce; if (prev == next) return; if (MACHINE_HAS_TLB_LC) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index b6bfa169a002..0da91c4d30fd 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -31,20 +31,42 @@ int pci_proc_domain(struct pci_bus *); #define ZPCI_FC_BLOCKED 0x20 #define ZPCI_FC_DMA_ENABLED 0x10 +#define ZPCI_FMB_DMA_COUNTER_VALID (1 << 23) + +struct zpci_fmb_fmt0 { + u64 dma_rbytes; + u64 dma_wbytes; +}; + +struct zpci_fmb_fmt1 { + u64 rx_bytes; + u64 rx_packets; + u64 tx_bytes; + u64 tx_packets; +}; + +struct zpci_fmb_fmt2 { + u64 consumed_work_units; + u64 max_work_units; +}; + struct zpci_fmb { - u32 format : 8; - u32 dma_valid : 1; - u32 : 23; + u32 format : 8; + u32 fmt_ind : 24; u32 samples; u64 last_update; - /* hardware counters */ + /* common counters */ u64 ld_ops; u64 st_ops; u64 stb_ops; u64 rpcit_ops; - u64 dma_rbytes; - u64 dma_wbytes; -} __packed __aligned(64); + /* format specific counters */ + union { + struct zpci_fmb_fmt0 fmt0; + struct zpci_fmb_fmt1 fmt1; + struct zpci_fmb_fmt2 fmt2; + }; +} __packed __aligned(128); enum zpci_state { ZPCI_FN_STATE_RESERVED, diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index 9b3d9b6099f2..da34cb6b1f3b 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -52,8 +52,8 @@ static inline unsigned long pgd_entry_type(struct mm_struct *mm) return _REGION2_ENTRY_EMPTY; } -int crst_table_upgrade(struct mm_struct *, unsigned long limit); -void crst_table_downgrade(struct mm_struct *, unsigned long limit); +int crst_table_upgrade(struct mm_struct *); +void crst_table_downgrade(struct mm_struct *); static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) { diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index d6fd22ea270d..9d4d311d7e52 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -105,7 +105,6 @@ typedef struct { * Thread structure */ struct thread_struct { - struct fpu fpu; /* FP and VX register save area */ unsigned int acrs[NUM_ACRS]; unsigned long ksp; /* kernel stack pointer */ mm_segment_t mm_segment; @@ -120,6 +119,11 @@ struct thread_struct { /* cpu runtime instrumentation */ struct runtime_instr_cb *ri_cb; unsigned char trap_tdb[256]; /* Transaction abort diagnose block */ + /* + * Warning: 'fpu' is dynamically-sized. It *MUST* be at + * the end. + */ + struct fpu fpu; /* FP and VX register save area */ }; /* Flag to disable transactions. */ @@ -155,10 +159,9 @@ struct stack_frame { #define ARCH_MIN_TASKALIGN 8 -extern __vector128 init_task_fpu_regs[__NUM_VXRS]; #define INIT_THREAD { \ .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ - .fpu.regs = (void *)&init_task_fpu_regs, \ + .fpu.regs = (void *) init_task.thread.fpu.fprs, \ } /* @@ -175,7 +178,7 @@ extern __vector128 init_task_fpu_regs[__NUM_VXRS]; regs->psw.mask = PSW_USER_BITS | PSW_MASK_BA; \ regs->psw.addr = new_psw; \ regs->gprs[15] = new_stackp; \ - crst_table_downgrade(current->mm, 1UL << 31); \ + crst_table_downgrade(current->mm); \ execve_tail(); \ } while (0) diff --git a/arch/s390/include/asm/rwsem.h b/arch/s390/include/asm/rwsem.h index fead491dfc28..c75e4471e618 100644 --- a/arch/s390/include/asm/rwsem.h +++ b/arch/s390/include/asm/rwsem.h @@ -90,7 +90,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) /* * lock for writing */ -static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) +static inline long ___down_write(struct rw_semaphore *sem) { signed long old, new, tmp; @@ -104,13 +104,23 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "m" (tmp) : "cc", "memory"); - if (old != 0) - rwsem_down_write_failed(sem); + + return old; } static inline void __down_write(struct rw_semaphore *sem) { - __down_write_nested(sem, 0); + if (___down_write(sem)) + rwsem_down_write_failed(sem); +} + +static inline int __down_write_killable(struct rw_semaphore *sem) +{ + if (___down_write(sem)) + if (IS_ERR(rwsem_down_write_failed_killable(sem))) + return -EINTR; + + return 0; } /* diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index bab456be9a4f..e4f6f73afe2f 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -69,9 +69,22 @@ struct sclp_info { unsigned int max_cores; unsigned long hsa_size; unsigned long facilities; + unsigned int hmfai; }; extern struct sclp_info sclp; +struct zpci_report_error_header { + u8 version; /* Interface version byte */ + u8 action; /* Action qualifier byte + * 1: Deconfigure and repair action requested + * (OpenCrypto Problem Call Home) + * 2: Informational Report + * (OpenCrypto Successful Diagnostics Execution) + */ + u16 length; /* Length of Subsequent Data (up to 4K – SCLP header */ + u8 data[0]; /* Subsequent Data passed verbatim to SCLP ET 24 */ +} __packed; + int sclp_get_core_info(struct sclp_core_info *info); int sclp_core_configure(u8 core); int sclp_core_deconfigure(u8 core); @@ -83,6 +96,7 @@ int sclp_chp_read_info(struct sclp_chp_info *info); void sclp_get_ipl_info(struct sclp_ipl_info *info); int sclp_pci_configure(u32 fid); int sclp_pci_deconfigure(u32 fid); +int sclp_pci_report(struct zpci_report_error_header *report, u32 fh, u32 fid); int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count); int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count); void sclp_early_detect(void); diff --git a/arch/s390/include/asm/seccomp.h b/arch/s390/include/asm/seccomp.h index 781a9cf9b002..e10f8337367b 100644 --- a/arch/s390/include/asm/seccomp.h +++ b/arch/s390/include/asm/seccomp.h @@ -13,4 +13,6 @@ #define __NR_seccomp_exit_32 __NR_exit #define __NR_seccomp_sigreturn_32 __NR_sigreturn +#include <asm-generic/seccomp.h> + #endif /* _ASM_S390_SECCOMP_H */ diff --git a/arch/s390/include/asm/sigp.h b/arch/s390/include/asm/sigp.h index ec60cf7fa0a2..1c8f33fca356 100644 --- a/arch/s390/include/asm/sigp.h +++ b/arch/s390/include/asm/sigp.h @@ -27,6 +27,7 @@ /* SIGP cpu status bits */ +#define SIGP_STATUS_INVALID_ORDER 0x00000002UL #define SIGP_STATUS_CHECK_STOP 0x00000010UL #define SIGP_STATUS_STOPPED 0x00000040UL #define SIGP_STATUS_EXT_CALL_PENDING 0x00000080UL diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index 2fffc2c27581..f15c0398c363 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -62,6 +62,7 @@ static inline struct thread_info *current_thread_info(void) } void arch_release_task_struct(struct task_struct *tsk); +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); #define THREAD_SIZE_ORDER THREAD_ORDER diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index ca148f7c3eaa..a2e6ef32e054 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -110,8 +110,7 @@ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) static inline void __tlb_flush_kernel(void) { if (MACHINE_HAS_IDTE) - __tlb_flush_idte((unsigned long) init_mm.pgd | - init_mm.context.asce_bits); + __tlb_flush_idte(init_mm.context.asce); else __tlb_flush_global(); } @@ -133,8 +132,7 @@ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) static inline void __tlb_flush_kernel(void) { if (MACHINE_HAS_TLB_LC) - __tlb_flush_idte_local((unsigned long) init_mm.pgd | - init_mm.context.asce_bits); + __tlb_flush_idte_local(init_mm.context.asce); else __tlb_flush_local(); } @@ -148,8 +146,7 @@ static inline void __tlb_flush_mm(struct mm_struct * mm) * only ran on the local cpu. */ if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) - __tlb_flush_asce(mm, (unsigned long) mm->pgd | - mm->context.asce_bits); + __tlb_flush_asce(mm, mm->context.asce); else __tlb_flush_full(mm); } diff --git a/arch/s390/include/uapi/asm/dasd.h b/arch/s390/include/uapi/asm/dasd.h index 5812a3b2df9e..1340311dab77 100644 --- a/arch/s390/include/uapi/asm/dasd.h +++ b/arch/s390/include/uapi/asm/dasd.h @@ -187,6 +187,36 @@ typedef struct format_data_t { #define DASD_FMT_INT_INVAL 4 /* invalidate tracks */ #define DASD_FMT_INT_COMPAT 8 /* use OS/390 compatible disk layout */ +/* + * struct format_check_t + * represents all data necessary to evaluate the format of + * different tracks of a dasd + */ +typedef struct format_check_t { + /* Input */ + struct format_data_t expect; + + /* Output */ + unsigned int result; /* Error indication (DASD_FMT_ERR_*) */ + unsigned int unit; /* Track that is in error */ + unsigned int rec; /* Record that is in error */ + unsigned int num_records; /* Records in the track in error */ + unsigned int blksize; /* Blocksize of first record in error */ + unsigned int key_length; /* Key length of first record in error */ +} format_check_t; + +/* Values returned in format_check_t when a format error is detected: */ +/* Too few records were found on a single track */ +#define DASD_FMT_ERR_TOO_FEW_RECORDS 1 +/* Too many records were found on a single track */ +#define DASD_FMT_ERR_TOO_MANY_RECORDS 2 +/* Blocksize/data-length of a record was wrong */ +#define DASD_FMT_ERR_BLKSIZE 3 +/* A record ID is defined by cylinder, head, and record number (CHR). */ +/* On mismatch, this error is set */ +#define DASD_FMT_ERR_RECORD_ID 4 +/* If key-length was != 0 */ +#define DASD_FMT_ERR_KEY_LENGTH 5 /* * struct attrib_data_t @@ -288,6 +318,8 @@ struct dasd_snid_ioctl_data { /* Get Sense Path Group ID (SNID) data */ #define BIODASDSNID _IOWR(DASD_IOCTL_LETTER, 1, struct dasd_snid_ioctl_data) +/* Check device format according to format_check_t */ +#define BIODASDCHECKFMT _IOWR(DASD_IOCTL_LETTER, 2, format_check_t) #define BIODASDSYMMIO _IOWR(DASD_IOCTL_LETTER, 240, dasd_symmio_parms_t) diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h index 347fe5afa419..3b8e99ef9d58 100644 --- a/arch/s390/include/uapi/asm/kvm.h +++ b/arch/s390/include/uapi/asm/kvm.h @@ -25,6 +25,7 @@ #define KVM_DEV_FLIC_APF_DISABLE_WAIT 5 #define KVM_DEV_FLIC_ADAPTER_REGISTER 6 #define KVM_DEV_FLIC_ADAPTER_MODIFY 7 +#define KVM_DEV_FLIC_CLEAR_IO_IRQ 8 /* * We can have up to 4*64k pending subchannels + 8 adapter interrupts, * as well as up to ASYNC_PF_PER_VCPU*KVM_MAX_VCPUS pfault done interrupts. diff --git a/arch/s390/include/uapi/asm/sie.h b/arch/s390/include/uapi/asm/sie.h index 5dbaa72baa64..8fb5d4a6dd25 100644 --- a/arch/s390/include/uapi/asm/sie.h +++ b/arch/s390/include/uapi/asm/sie.h @@ -16,14 +16,19 @@ { 0x01, "SIGP sense" }, \ { 0x02, "SIGP external call" }, \ { 0x03, "SIGP emergency signal" }, \ + { 0x04, "SIGP start" }, \ { 0x05, "SIGP stop" }, \ { 0x06, "SIGP restart" }, \ { 0x09, "SIGP stop and store status" }, \ { 0x0b, "SIGP initial cpu reset" }, \ + { 0x0c, "SIGP cpu reset" }, \ { 0x0d, "SIGP set prefix" }, \ { 0x0e, "SIGP store status at address" }, \ { 0x12, "SIGP set architecture" }, \ - { 0x15, "SIGP sense running" } + { 0x13, "SIGP conditional emergency signal" }, \ + { 0x15, "SIGP sense running" }, \ + { 0x16, "SIGP set multithreading"}, \ + { 0x17, "SIGP store additional status ait address"} #define icpt_prog_codes \ { 0x0001, "Prog Operation" }, \ diff --git a/arch/s390/include/uapi/asm/unistd.h b/arch/s390/include/uapi/asm/unistd.h index ab3aa6875a59..4384bc797a54 100644 --- a/arch/s390/include/uapi/asm/unistd.h +++ b/arch/s390/include/uapi/asm/unistd.h @@ -311,7 +311,9 @@ #define __NR_shutdown 373 #define __NR_mlock2 374 #define __NR_copy_file_range 375 -#define NR_syscalls 376 +#define __NR_preadv2 376 +#define __NR_pwritev2 377 +#define NR_syscalls 378 /* * There are some system calls that are not present on 64 bit, some diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 8ba32436effe..77a84bd78be2 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -72,7 +72,6 @@ void show_cacheinfo(struct seq_file *m) if (!test_facility(34)) return; - get_online_cpus(); this_cpu_ci = get_cpu_cacheinfo(cpumask_any(cpu_online_mask)); for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { cache = this_cpu_ci->info_list + idx; @@ -86,7 +85,6 @@ void show_cacheinfo(struct seq_file *m) seq_printf(m, "associativity=%d", cache->ways_of_associativity); seq_puts(m, "\n"); } - put_online_cpus(); } static inline enum cache_type get_cache_type(struct cache_info *ci, int level) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 3986c9f62191..29df8484282b 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -173,7 +173,7 @@ int copy_oldmem_kernel(void *dst, void *src, size_t count) /* * Copy memory of the old, dumped system to a user space virtual address */ -int copy_oldmem_user(void __user *dst, void *src, size_t count) +static int copy_oldmem_user(void __user *dst, void *src, size_t count) { unsigned long from, len; int rc; diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index 1b6081c0aff9..69f9908ac44c 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -89,6 +89,30 @@ void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, } EXPORT_SYMBOL_GPL(dump_trace); +struct return_address_data { + unsigned long address; + int depth; +}; + +static int __return_address(void *data, unsigned long address) +{ + struct return_address_data *rd = data; + + if (rd->depth--) + return 0; + rd->address = address; + return 1; +} + +unsigned long return_address(int depth) +{ + struct return_address_data rd = { .depth = depth + 2 }; + + dump_trace(__return_address, &rd, NULL, current_stack_pointer()); + return rd.address; +} +EXPORT_SYMBOL_GPL(return_address); + static int show_address(void *data, unsigned long address) { printk("([<%016lx>] %pSR)\n", address, (void *)address); diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index b7019ab74070..bedd2f55d860 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -1,6 +1,7 @@ #ifndef _ENTRY_H #define _ENTRY_H +#include <linux/percpu.h> #include <linux/types.h> #include <linux/signal.h> #include <asm/ptrace.h> @@ -75,4 +76,7 @@ long sys_s390_personality(unsigned int personality); long sys_s390_runtime_instr(int command, int signum); long sys_s390_pci_mmio_write(unsigned long, const void __user *, size_t); long sys_s390_pci_mmio_read(unsigned long, void __user *, size_t); + +DECLARE_PER_CPU(u64, mt_cycles[8]); + #endif /* _ENTRY_H */ diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index 7873e171457c..fbc07891f9e7 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -51,6 +51,10 @@ void *module_alloc(unsigned long size) void module_arch_freeing_init(struct module *mod) { + if (is_livepatch_module(mod) && + mod->state == MODULE_STATE_LIVE) + return; + vfree(mod->arch.syminfo); mod->arch.syminfo = NULL; } @@ -425,7 +429,5 @@ int module_finalize(const Elf_Ehdr *hdr, struct module *me) { jump_label_apply_nops(me); - vfree(me->arch.syminfo); - me->arch.syminfo = NULL; return 0; } diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 58bf4572d457..59215c518f37 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -665,17 +665,21 @@ static struct pmu cpumf_pmu = { static int cpumf_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { - unsigned int cpu = (long) hcpu; int flags; switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + case CPU_DOWN_FAILED: flags = PMC_INIT; - smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1); + local_irq_disable(); + setup_pmc_cpu(&flags); + local_irq_enable(); break; case CPU_DOWN_PREPARE: flags = PMC_RELEASE; - smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1); + local_irq_disable(); + setup_pmc_cpu(&flags); + local_irq_enable(); break; default: break; diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 1a43474df541..a8e832166417 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1510,7 +1510,6 @@ static void cpumf_measurement_alert(struct ext_code ext_code, static int cpumf_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) { - unsigned int cpu = (long) hcpu; int flags; /* Ignore the notification if no events are scheduled on the PMU. @@ -1521,13 +1520,17 @@ static int cpumf_pmu_notifier(struct notifier_block *self, switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: - case CPU_ONLINE_FROZEN: + case CPU_DOWN_FAILED: flags = PMC_INIT; - smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1); + local_irq_disable(); + setup_pmc_cpu(&flags); + local_irq_enable(); break; case CPU_DOWN_PREPARE: flags = PMC_RELEASE; - smp_call_function_single(cpu, setup_pmc_cpu, &flags, 1); + local_irq_disable(); + setup_pmc_cpu(&flags); + local_irq_enable(); break; default: break; diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 2bba7df4ac51..481d7a83efc6 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -7,6 +7,7 @@ * Denis Joseph Barrow, */ +#include <linux/elf-randomize.h> #include <linux/compiler.h> #include <linux/cpu.h> #include <linux/sched.h> @@ -37,9 +38,6 @@ asmlinkage void ret_from_fork(void) asm ("ret_from_fork"); -/* FPU save area for the init task */ -__vector128 init_task_fpu_regs[__NUM_VXRS] __init_task_data; - /* * Return saved PC of a blocked thread. used in kernel/sched. * resume in entry.S does not create a new stack frame, it @@ -85,35 +83,19 @@ void release_thread(struct task_struct *dead_task) void arch_release_task_struct(struct task_struct *tsk) { - /* Free either the floating-point or the vector register save area */ - kfree(tsk->thread.fpu.regs); } int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - size_t fpu_regs_size; - - *dst = *src; - - /* - * If the vector extension is available, it is enabled for all tasks, - * and, thus, the FPU register save area must be allocated accordingly. - */ - fpu_regs_size = MACHINE_HAS_VX ? sizeof(__vector128) * __NUM_VXRS - : sizeof(freg_t) * __NUM_FPRS; - dst->thread.fpu.regs = kzalloc(fpu_regs_size, GFP_KERNEL|__GFP_REPEAT); - if (!dst->thread.fpu.regs) - return -ENOMEM; - /* * Save the floating-point or vector register state of the current * task and set the CIF_FPU flag to lazy restore the FPU register * state when returning to user space. */ save_fpu_regs(); - dst->thread.fpu.fpc = current->thread.fpu.fpc; - memcpy(dst->thread.fpu.regs, current->thread.fpu.regs, fpu_regs_size); + memcpy(dst, src, arch_task_struct_size); + dst->thread.fpu.regs = dst->thread.fpu.fprs; return 0; } diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 647128d5b983..de7451065c34 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -6,6 +6,7 @@ #define KMSG_COMPONENT "cpu" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include <linux/cpufeature.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/seq_file.h> @@ -84,7 +85,6 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_puts(m, "\n"); show_cacheinfo(m); } - get_online_cpus(); if (cpu_online(n)) { struct cpuid *id = &per_cpu(cpu_id, n); seq_printf(m, "processor %li: " @@ -93,23 +93,31 @@ static int show_cpuinfo(struct seq_file *m, void *v) "machine = %04X\n", n, id->version, id->ident, id->machine); } - put_online_cpus(); return 0; } +static inline void *c_update(loff_t *pos) +{ + if (*pos) + *pos = cpumask_next(*pos - 1, cpu_online_mask); + return *pos < nr_cpu_ids ? (void *)*pos + 1 : NULL; +} + static void *c_start(struct seq_file *m, loff_t *pos) { - return *pos < nr_cpu_ids ? (void *)((unsigned long) *pos + 1) : NULL; + get_online_cpus(); + return c_update(pos); } static void *c_next(struct seq_file *m, void *v, loff_t *pos) { ++*pos; - return c_start(m, pos); + return c_update(pos); } static void c_stop(struct seq_file *m, void *v) { + put_online_cpus(); } const struct seq_operations cpuinfo_op = { diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d3f9688f26b5..f31939147ccd 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -809,6 +809,22 @@ static void __init setup_randomness(void) } /* + * Find the correct size for the task_struct. This depends on + * the size of the struct fpu at the end of the thread_struct + * which is embedded in the task_struct. + */ +static void __init setup_task_size(void) +{ + int task_size = sizeof(struct task_struct); + + if (!MACHINE_HAS_VX) { + task_size -= sizeof(__vector128) * __NUM_VXRS; + task_size += sizeof(freg_t) * __NUM_FPRS; + } + arch_task_struct_size = task_size; +} + +/* * Setup function called from init/main.c just after the banner * was printed. */ @@ -846,6 +862,7 @@ void __init setup_arch(char **cmdline_p) os_info_init(); setup_ipl(); + setup_task_size(); /* Do some memory reservations *before* memory is added to memblock */ reserve_memory_end(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 40a6b4f9c36c..7b89a7572100 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -832,7 +832,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); /* Wait until cpu puts itself in the online & active maps */ - while (!cpu_online(cpu) || !cpu_active(cpu)) + while (!cpu_online(cpu)) cpu_relax(); return 0; } diff --git a/arch/s390/kernel/syscalls.S b/arch/s390/kernel/syscalls.S index 293d8b98fd52..9b59e6212d8f 100644 --- a/arch/s390/kernel/syscalls.S +++ b/arch/s390/kernel/syscalls.S @@ -384,3 +384,5 @@ SYSCALL(sys_recvmsg,compat_sys_recvmsg) SYSCALL(sys_shutdown,sys_shutdown) SYSCALL(sys_mlock2,compat_sys_mlock2) SYSCALL(sys_copy_file_range,compat_sys_copy_file_range) /* 375 */ +SYSCALL(sys_preadv2,compat_sys_preadv2) +SYSCALL(sys_pwritev2,compat_sys_pwritev2) diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index dafc44f519c3..856e30d8463f 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -18,6 +18,8 @@ #include <asm/cpu_mf.h> #include <asm/smp.h> +#include "entry.h" + static void virt_timer_expire(void); static LIST_HEAD(virt_timer_list); diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 5ea5af3c7db7..b1900239b0ab 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -28,6 +28,7 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_INVALID_WAKEUPS select SRCU select KVM_VFIO ---help--- diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 84efc2ba6a90..5a80af740d3e 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -977,6 +977,11 @@ no_timer: void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) { + /* + * We cannot move this into the if, as the CPU might be already + * in kvm_vcpu_block without having the waitqueue set (polling) + */ + vcpu->valid_wakeup = true; if (swait_active(&vcpu->wq)) { /* * The vcpu gave up the cpu voluntarily, mark it as a good @@ -2034,6 +2039,27 @@ static int modify_io_adapter(struct kvm_device *dev, return ret; } +static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) + +{ + const u64 isc_mask = 0xffUL << 24; /* all iscs set */ + u32 schid; + + if (attr->flags) + return -EINVAL; + if (attr->attr != sizeof(schid)) + return -EINVAL; + if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) + return -EFAULT; + kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); + /* + * If userspace is conforming to the architecture, we can have at most + * one pending I/O interrupt per subchannel, so this is effectively a + * clear all. + */ + return 0; +} + static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { int r = 0; @@ -2067,6 +2093,9 @@ static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) case KVM_DEV_FLIC_ADAPTER_MODIFY: r = modify_io_adapter(dev, attr); break; + case KVM_DEV_FLIC_CLEAR_IO_IRQ: + r = clear_io_irq(dev->kvm, attr); + break; default: r = -EINVAL; } @@ -2074,6 +2103,23 @@ static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) return r; } +static int flic_has_attr(struct kvm_device *dev, + struct kvm_device_attr *attr) +{ + switch (attr->group) { + case KVM_DEV_FLIC_GET_ALL_IRQS: + case KVM_DEV_FLIC_ENQUEUE: + case KVM_DEV_FLIC_CLEAR_IRQS: + case KVM_DEV_FLIC_APF_ENABLE: + case KVM_DEV_FLIC_APF_DISABLE_WAIT: + case KVM_DEV_FLIC_ADAPTER_REGISTER: + case KVM_DEV_FLIC_ADAPTER_MODIFY: + case KVM_DEV_FLIC_CLEAR_IO_IRQ: + return 0; + } + return -ENXIO; +} + static int flic_create(struct kvm_device *dev, u32 type) { if (!dev) @@ -2095,6 +2141,7 @@ struct kvm_device_ops kvm_flic_ops = { .name = "kvm-flic", .get_attr = flic_get_attr, .set_attr = flic_set_attr, + .has_attr = flic_has_attr, .create = flic_create, .destroy = flic_destroy, }; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 668c087513e5..6d8ec3ac9dd8 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -65,6 +65,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, + { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, { "halt_wakeup", VCPU_STAT(halt_wakeup) }, { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, { "instruction_lctl", VCPU_STAT(instruction_lctl) }, @@ -118,9 +119,9 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { }; /* upper facilities limit for kvm */ -unsigned long kvm_s390_fac_list_mask[] = { - 0xffe6fffbfcfdfc40UL, - 0x005e800000000000UL, +unsigned long kvm_s390_fac_list_mask[16] = { + 0xffe6000000000000UL, + 0x005e000000000000UL, }; unsigned long kvm_s390_fac_list_mask_size(void) @@ -638,6 +639,7 @@ static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr) static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_vm_cpu_processor *proc; + u16 lowest_ibc, unblocked_ibc; int ret = 0; mutex_lock(&kvm->lock); @@ -652,9 +654,17 @@ static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) } if (!copy_from_user(proc, (void __user *)attr->addr, sizeof(*proc))) { - memcpy(&kvm->arch.model.cpu_id, &proc->cpuid, - sizeof(struct cpuid)); - kvm->arch.model.ibc = proc->ibc; + kvm->arch.model.cpuid = proc->cpuid; + lowest_ibc = sclp.ibc >> 16 & 0xfff; + unblocked_ibc = sclp.ibc & 0xfff; + if (lowest_ibc) { + if (proc->ibc > unblocked_ibc) + kvm->arch.model.ibc = unblocked_ibc; + else if (proc->ibc < lowest_ibc) + kvm->arch.model.ibc = lowest_ibc; + else + kvm->arch.model.ibc = proc->ibc; + } memcpy(kvm->arch.model.fac_list, proc->fac_list, S390_ARCH_FAC_LIST_SIZE_BYTE); } else @@ -687,7 +697,7 @@ static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) ret = -ENOMEM; goto out; } - memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid)); + proc->cpuid = kvm->arch.model.cpuid; proc->ibc = kvm->arch.model.ibc; memcpy(&proc->fac_list, kvm->arch.model.fac_list, S390_ARCH_FAC_LIST_SIZE_BYTE); @@ -1081,10 +1091,13 @@ static void kvm_s390_set_crycb_format(struct kvm *kvm) kvm->arch.crypto.crycbd |= CRYCB_FORMAT1; } -static void kvm_s390_get_cpu_id(struct cpuid *cpu_id) +static u64 kvm_s390_get_initial_cpuid(void) { - get_cpu_id(cpu_id); - cpu_id->version = 0xff; + struct cpuid cpuid; + + get_cpu_id(&cpuid); + cpuid.version = 0xff; + return *((u64 *) &cpuid); } static void kvm_s390_crypto_init(struct kvm *kvm) @@ -1175,7 +1188,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask, S390_ARCH_FAC_LIST_SIZE_BYTE); - kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id); + kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid(); kvm->arch.model.ibc = sclp.ibc & 0x0fff; kvm_s390_crypto_init(kvm); @@ -1624,7 +1637,6 @@ static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu) { struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model; - vcpu->arch.cpu_id = model->cpu_id; vcpu->arch.sie_block->ibc = model->ibc; if (test_kvm_facility(vcpu->kvm, 7)) vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list; @@ -1645,11 +1657,14 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) kvm_s390_vcpu_setup_model(vcpu); - vcpu->arch.sie_block->ecb = 6; + vcpu->arch.sie_block->ecb = 0x02; + if (test_kvm_facility(vcpu->kvm, 9)) + vcpu->arch.sie_block->ecb |= 0x04; if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73)) vcpu->arch.sie_block->ecb |= 0x10; - vcpu->arch.sie_block->ecb2 = 8; + if (test_kvm_facility(vcpu->kvm, 8)) + vcpu->arch.sie_block->ecb2 |= 0x08; vcpu->arch.sie_block->eca = 0xC1002000U; if (sclp.has_siif) vcpu->arch.sie_block->eca |= 1; @@ -2971,13 +2986,31 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, return; } +static inline unsigned long nonhyp_mask(int i) +{ + unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30; + + return 0x0000ffffffffffffUL >> (nonhyp_fai << 4); +} + +void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) +{ + vcpu->valid_wakeup = false; +} + static int __init kvm_s390_init(void) { + int i; + if (!sclp.has_sief2) { pr_info("SIE not available\n"); return -ENODEV; } + for (i = 0; i < 16; i++) + kvm_s390_fac_list_mask[i] |= + S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i); + return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index 0a1591d3d25d..95916fa7c670 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -439,7 +439,7 @@ static int handle_lpswe(struct kvm_vcpu *vcpu) static int handle_stidp(struct kvm_vcpu *vcpu) { - u64 stidp_data = vcpu->arch.stidp_data; + u64 stidp_data = vcpu->kvm->arch.model.cpuid; u64 operand2; int rc; ar_t ar; @@ -670,8 +670,9 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - /* Only provide non-quiescing support if the host supports it */ - if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && !test_facility(14)) + /* Only provide non-quiescing support if enabled for the guest */ + if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && + !test_kvm_facility(vcpu->kvm, 14)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); /* No support for conditional-SSKE */ @@ -744,7 +745,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) { /* entries expected to be 1FF */ int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; - unsigned long *cbrlo, cbrle; + unsigned long *cbrlo; struct gmap *gmap; int i; @@ -765,17 +766,9 @@ static int handle_essa(struct kvm_vcpu *vcpu) vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); down_read(&gmap->mm->mmap_sem); - for (i = 0; i < entries; ++i) { - cbrle = cbrlo[i]; - if (unlikely(cbrle & ~PAGE_MASK || cbrle < 2 * PAGE_SIZE)) - /* invalid entry */ - break; - /* try to free backing */ - __gmap_zap(gmap, cbrle); - } + for (i = 0; i < entries; ++i) + __gmap_zap(gmap, cbrlo[i]); up_read(&gmap->mm->mmap_sem); - if (i < entries) - return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); return 0; } diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c index 77c22d685c7a..28ea0cab1f1b 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/sigp.c @@ -240,6 +240,12 @@ static int __sigp_sense_running(struct kvm_vcpu *vcpu, struct kvm_s390_local_interrupt *li; int rc; + if (!test_kvm_facility(vcpu->kvm, 9)) { + *reg &= 0xffffffff00000000UL; + *reg |= SIGP_STATUS_INVALID_ORDER; + return SIGP_CC_STATUS_STORED; + } + li = &dst_vcpu->arch.local_int; if (atomic_read(li->cpuflags) & CPUSTAT_RUNNING) { /* running */ diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index d4549c964589..e5f50a7d2f4e 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -105,6 +105,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) if (_raw_compare_and_swap(&lp->lock, 0, cpu)) return; local_irq_restore(flags); + continue; } /* Check if the lock owner is running. */ if (first_diag && cpu_is_preempted(~owner)) { diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index cce577feab1e..7a3144017301 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -631,6 +631,29 @@ void pfault_fini(void) static DEFINE_SPINLOCK(pfault_lock); static LIST_HEAD(pfault_list); +#define PF_COMPLETE 0x0080 + +/* + * The mechanism of our pfault code: if Linux is running as guest, runs a user + * space process and the user space process accesses a page that the host has + * paged out we get a pfault interrupt. + * + * This allows us, within the guest, to schedule a different process. Without + * this mechanism the host would have to suspend the whole virtual cpu until + * the page has been paged in. + * + * So when we get such an interrupt then we set the state of the current task + * to uninterruptible and also set the need_resched flag. Both happens within + * interrupt context(!). If we later on want to return to user space we + * recognize the need_resched flag and then call schedule(). It's not very + * obvious how this works... + * + * Of course we have a lot of additional fun with the completion interrupt (-> + * host signals that a page of a process has been paged in and the process can + * continue to run). This interrupt can arrive on any cpu and, since we have + * virtual cpus, actually appear before the interrupt that signals that a page + * is missing. + */ static void pfault_interrupt(struct ext_code ext_code, unsigned int param32, unsigned long param64) { @@ -639,10 +662,9 @@ static void pfault_interrupt(struct ext_code ext_code, pid_t pid; /* - * Get the external interruption subcode & pfault - * initial/completion signal bit. VM stores this - * in the 'cpu address' field associated with the - * external interrupt. + * Get the external interruption subcode & pfault initial/completion + * signal bit. VM stores this in the 'cpu address' field associated + * with the external interrupt. */ subcode = ext_code.subcode; if ((subcode & 0xff00) != __SUBCODE_MASK) @@ -658,7 +680,7 @@ static void pfault_interrupt(struct ext_code ext_code, if (!tsk) return; spin_lock(&pfault_lock); - if (subcode & 0x0080) { + if (subcode & PF_COMPLETE) { /* signal bit is set -> a page has been swapped in by VM */ if (tsk->thread.pfault_wait == 1) { /* Initial interrupt was faster than the completion @@ -687,8 +709,7 @@ static void pfault_interrupt(struct ext_code ext_code, goto out; if (tsk->thread.pfault_wait == 1) { /* Already on the list with a reference: put to sleep */ - __set_task_state(tsk, TASK_UNINTERRUPTIBLE); - set_tsk_need_resched(tsk); + goto block; } else if (tsk->thread.pfault_wait == -1) { /* Completion interrupt was faster than the initial * interrupt (pfault_wait == -1). Set pfault_wait @@ -703,7 +724,11 @@ static void pfault_interrupt(struct ext_code ext_code, get_task_struct(tsk); tsk->thread.pfault_wait = 1; list_add(&tsk->thread.list, &pfault_list); - __set_task_state(tsk, TASK_UNINTERRUPTIBLE); +block: + /* Since this must be a userspace fault, there + * is no kernel task state to trample. Rely on the + * return to userspace schedule() to block. */ + __set_current_state(TASK_UNINTERRUPTIBLE); set_tsk_need_resched(tsk); } } diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 69247b4dcc43..cace818d86eb 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -23,7 +23,7 @@ /** * gmap_alloc - allocate a guest address space * @mm: pointer to the parent mm_struct - * @limit: maximum size of the gmap address space + * @limit: maximum address of the gmap address space * * Returns a guest address space structure. */ @@ -292,7 +292,7 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, if ((from | to | len) & (PMD_SIZE - 1)) return -EINVAL; if (len == 0 || from + len < from || to + len < to || - from + len > TASK_MAX_SIZE || to + len > gmap->asce_end) + from + len - 1 > TASK_MAX_SIZE || to + len - 1 > gmap->asce_end) return -EINVAL; flush = 0; diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c index 49a1c84ed266..a8a6765f1a51 100644 --- a/arch/s390/mm/gup.c +++ b/arch/s390/mm/gup.c @@ -20,9 +20,9 @@ static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, unsigned long end, int write, struct page **pages, int *nr) { + struct page *head, *page; unsigned long mask; pte_t *ptep, pte; - struct page *page; mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL; @@ -37,12 +37,14 @@ static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, return 0; VM_BUG_ON(!pfn_valid(pte_pfn(pte))); page = pte_page(pte); - if (!page_cache_get_speculative(page)) + head = compound_head(page); + if (!page_cache_get_speculative(head)) return 0; if (unlikely(pte_val(pte) != pte_val(*ptep))) { - put_page(page); + put_page(head); return 0; } + VM_BUG_ON_PAGE(compound_head(page) != head, page); pages[*nr] = page; (*nr)++; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 73e290337092..2489b2e917c8 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -89,7 +89,8 @@ void __init paging_init(void) asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; pgd_type = _REGION3_ENTRY_EMPTY; } - S390_lowcore.kernel_asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; + init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; + S390_lowcore.kernel_asce = init_mm.context.asce; clear_table((unsigned long *) init_mm.pgd, pgd_type, sizeof(unsigned long)*2048); vmem_map_init(); @@ -108,6 +109,13 @@ void __init paging_init(void) free_area_init_nodes(max_zone_pfns); } +void mark_rodata_ro(void) +{ + /* Text and rodata are already protected. Nothing to do here. */ + pr_info("Write protecting the kernel read-only data: %luk\n", + ((unsigned long)&_eshared - (unsigned long)&_stext) >> 10); +} + void __init mem_init(void) { if (MACHINE_HAS_TLB_LC) @@ -126,9 +134,6 @@ void __init mem_init(void) setup_zero_pages(); /* Setup zeroed pages. */ mem_init_print_info(NULL); - printk("Write protected kernel read-only data: %#lx - %#lx\n", - (unsigned long)&_stext, - PFN_ALIGN((unsigned long)&_eshared) - 1); } void free_initmem(void) diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 45c4daa49930..eb9df2822da1 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -22,6 +22,7 @@ * Started by Ingo Molnar <mingo@elte.hu> */ +#include <linux/elf-randomize.h> #include <linux/personality.h> #include <linux/mm.h> #include <linux/mman.h> @@ -174,7 +175,7 @@ int s390_mmap_check(unsigned long addr, unsigned long len, unsigned long flags) if (!(flags & MAP_FIXED)) addr = 0; if ((addr + len) >= TASK_SIZE) - return crst_table_upgrade(current->mm, TASK_MAX_SIZE); + return crst_table_upgrade(current->mm); return 0; } @@ -191,7 +192,7 @@ s390_get_unmapped_area(struct file *filp, unsigned long addr, return area; if (area == -ENOMEM && !is_compat_task() && TASK_SIZE < TASK_MAX_SIZE) { /* Upgrade the page table to 4 levels and retry. */ - rc = crst_table_upgrade(mm, TASK_MAX_SIZE); + rc = crst_table_upgrade(mm); if (rc) return (unsigned long) rc; area = arch_get_unmapped_area(filp, addr, len, pgoff, flags); @@ -213,7 +214,7 @@ s390_get_unmapped_area_topdown(struct file *filp, const unsigned long addr, return area; if (area == -ENOMEM && !is_compat_task() && TASK_SIZE < TASK_MAX_SIZE) { /* Upgrade the page table to 4 levels and retry. */ - rc = crst_table_upgrade(mm, TASK_MAX_SIZE); + rc = crst_table_upgrade(mm); if (rc) return (unsigned long) rc; area = arch_get_unmapped_area_topdown(filp, addr, len, diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index f6c3de26cda8..e8b5962ac12a 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -76,81 +76,52 @@ static void __crst_table_upgrade(void *arg) __tlb_flush_local(); } -int crst_table_upgrade(struct mm_struct *mm, unsigned long limit) +int crst_table_upgrade(struct mm_struct *mm) { unsigned long *table, *pgd; - unsigned long entry; - int flush; - BUG_ON(limit > TASK_MAX_SIZE); - flush = 0; -repeat: + /* upgrade should only happen from 3 to 4 levels */ + BUG_ON(mm->context.asce_limit != (1UL << 42)); + table = crst_table_alloc(mm); if (!table) return -ENOMEM; + spin_lock_bh(&mm->page_table_lock); - if (mm->context.asce_limit < limit) { - pgd = (unsigned long *) mm->pgd; - if (mm->context.asce_limit <= (1UL << 31)) { - entry = _REGION3_ENTRY_EMPTY; - mm->context.asce_limit = 1UL << 42; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION3; - } else { - entry = _REGION2_ENTRY_EMPTY; - mm->context.asce_limit = 1UL << 53; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION2; - } - crst_table_init(table, entry); - pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); - mm->pgd = (pgd_t *) table; - mm->task_size = mm->context.asce_limit; - table = NULL; - flush = 1; - } + pgd = (unsigned long *) mm->pgd; + crst_table_init(table, _REGION2_ENTRY_EMPTY); + pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); + mm->pgd = (pgd_t *) table; + mm->context.asce_limit = 1UL << 53; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + mm->task_size = mm->context.asce_limit; spin_unlock_bh(&mm->page_table_lock); - if (table) - crst_table_free(mm, table); - if (mm->context.asce_limit < limit) - goto repeat; - if (flush) - on_each_cpu(__crst_table_upgrade, mm, 0); + + on_each_cpu(__crst_table_upgrade, mm, 0); return 0; } -void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) +void crst_table_downgrade(struct mm_struct *mm) { pgd_t *pgd; + /* downgrade should only happen from 3 to 2 levels (compat only) */ + BUG_ON(mm->context.asce_limit != (1UL << 42)); + if (current->active_mm == mm) { clear_user_asce(); __tlb_flush_mm(mm); } - while (mm->context.asce_limit > limit) { - pgd = mm->pgd; - switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) { - case _REGION_ENTRY_TYPE_R2: - mm->context.asce_limit = 1UL << 42; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION3; - break; - case _REGION_ENTRY_TYPE_R3: - mm->context.asce_limit = 1UL << 31; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_SEGMENT; - break; - default: - BUG(); - } - mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); - mm->task_size = mm->context.asce_limit; - crst_table_free(mm, (unsigned long *) pgd); - } + + pgd = mm->pgd; + mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); + mm->context.asce_limit = 1UL << 31; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; + mm->task_size = mm->context.asce_limit; + crst_table_free(mm, (unsigned long *) pgd); + if (current->active_mm == mm) set_user_asce(mm); } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index d27fccbad7c1..d48cf25cfe99 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -56,7 +56,7 @@ static inline pmd_t *vmem_pmd_alloc(void) return pmd; } -static pte_t __ref *vmem_pte_alloc(unsigned long address) +static pte_t __ref *vmem_pte_alloc(void) { pte_t *pte; @@ -121,7 +121,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) continue; } if (pmd_none(*pm_dir)) { - pt_dir = vmem_pte_alloc(address); + pt_dir = vmem_pte_alloc(); if (!pt_dir) goto out; pmd_populate(&init_mm, pm_dir, pt_dir); @@ -233,7 +233,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) address = (address + PMD_SIZE) & PMD_MASK; continue; } - pt_dir = vmem_pte_alloc(address); + pt_dir = vmem_pte_alloc(); if (!pt_dir) goto out; pmd_populate(&init_mm, pm_dir, pt_dir); @@ -370,7 +370,7 @@ void __init vmem_map_init(void) ro_end = (unsigned long)&_eshared & PAGE_MASK; for_each_memblock(memory, reg) { start = reg->base; - end = reg->base + reg->size - 1; + end = reg->base + reg->size; if (start >= ro_end || end <= ro_start) vmem_add_mem(start, end - start, 0); else if (start >= ro_start && end <= ro_end) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 3c0bfc1f2694..9133b0ec000b 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -54,16 +54,17 @@ struct bpf_jit { #define SEEN_FUNC 16 /* calls C functions */ #define SEEN_TAIL_CALL 32 /* code uses tail calls */ #define SEEN_SKB_CHANGE 64 /* code changes skb data */ +#define SEEN_REG_AX 128 /* code uses constant blinding */ #define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB) /* * s390 registers */ -#define REG_W0 (__MAX_BPF_REG+0) /* Work register 1 (even) */ -#define REG_W1 (__MAX_BPF_REG+1) /* Work register 2 (odd) */ -#define REG_SKB_DATA (__MAX_BPF_REG+2) /* SKB data register */ -#define REG_L (__MAX_BPF_REG+3) /* Literal pool register */ -#define REG_15 (__MAX_BPF_REG+4) /* Register 15 */ +#define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ +#define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ +#define REG_SKB_DATA (MAX_BPF_JIT_REG + 2) /* SKB data register */ +#define REG_L (MAX_BPF_JIT_REG + 3) /* Literal pool register */ +#define REG_15 (MAX_BPF_JIT_REG + 4) /* Register 15 */ #define REG_0 REG_W0 /* Register 0 */ #define REG_1 REG_W1 /* Register 1 */ #define REG_2 BPF_REG_1 /* Register 2 */ @@ -88,6 +89,8 @@ static const int reg2hex[] = { [BPF_REG_9] = 10, /* BPF stack pointer */ [BPF_REG_FP] = 13, + /* Register for blinding (shared with REG_SKB_DATA) */ + [BPF_REG_AX] = 12, /* SKB data pointer */ [REG_SKB_DATA] = 12, /* Work registers for s390x backend */ @@ -385,7 +388,7 @@ static void save_restore_regs(struct bpf_jit *jit, int op) /* * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" * we store the SKB header length on the stack and the SKB data - * pointer in REG_SKB_DATA. + * pointer in REG_SKB_DATA if BPF_REG_AX is not used. */ static void emit_load_skb_data_hlen(struct bpf_jit *jit) { @@ -397,9 +400,10 @@ static void emit_load_skb_data_hlen(struct bpf_jit *jit) offsetof(struct sk_buff, data_len)); /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN); - /* lg %skb_data,data_off(%b1) */ - EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, - BPF_REG_1, offsetof(struct sk_buff, data)); + if (!(jit->seen & SEEN_REG_AX)) + /* lg %skb_data,data_off(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, + BPF_REG_1, offsetof(struct sk_buff, data)); } /* @@ -487,6 +491,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i s32 imm = insn->imm; s16 off = insn->off; + if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX) + jit->seen |= SEEN_REG_AX; switch (insn->code) { /* * BPF_MOV @@ -1188,7 +1194,7 @@ call_fn: /* * Implicit input: * BPF_REG_6 (R7) : skb pointer - * REG_SKB_DATA (R12): skb data pointer + * REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX) * * Calculated input: * BPF_REG_2 (R3) : offset of byte(s) to fetch in skb @@ -1209,6 +1215,11 @@ call_fn: /* agfr %b2,%src (%src is s32 here) */ EMIT4(0xb9180000, BPF_REG_2, src_reg); + /* Reload REG_SKB_DATA if BPF_REG_AX is used */ + if (jit->seen & SEEN_REG_AX) + /* lg %skb_data,data_off(%b6) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, + BPF_REG_6, offsetof(struct sk_buff, data)); /* basr %b5,%w1 (%b5 is call saved) */ EMIT2(0x0d00, BPF_REG_5, REG_W1); @@ -1262,37 +1273,62 @@ void bpf_jit_compile(struct bpf_prog *fp) /* * Compile eBPF program "fp" */ -void bpf_int_jit_compile(struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) { + struct bpf_prog *tmp, *orig_fp = fp; struct bpf_binary_header *header; + bool tmp_blinded = false; struct bpf_jit jit; int pass; if (!bpf_jit_enable) - return; + return orig_fp; + + tmp = bpf_jit_blind_constants(fp); + /* + * If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. + */ + if (IS_ERR(tmp)) + return orig_fp; + if (tmp != fp) { + tmp_blinded = true; + fp = tmp; + } + memset(&jit, 0, sizeof(jit)); jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); - if (jit.addrs == NULL) - return; + if (jit.addrs == NULL) { + fp = orig_fp; + goto out; + } /* * Three initial passes: * - 1/2: Determine clobbered registers * - 3: Calculate program size and addrs arrray */ for (pass = 1; pass <= 3; pass++) { - if (bpf_jit_prog(&jit, fp)) + if (bpf_jit_prog(&jit, fp)) { + fp = orig_fp; goto free_addrs; + } } /* * Final pass: Allocate and generate program */ - if (jit.size >= BPF_SIZE_MAX) + if (jit.size >= BPF_SIZE_MAX) { + fp = orig_fp; goto free_addrs; + } header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole); - if (!header) + if (!header) { + fp = orig_fp; goto free_addrs; - if (bpf_jit_prog(&jit, fp)) + } + if (bpf_jit_prog(&jit, fp)) { + fp = orig_fp; goto free_addrs; + } if (bpf_jit_enable > 1) { bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); if (jit.prg_buf) @@ -1305,6 +1341,11 @@ void bpf_int_jit_compile(struct bpf_prog *fp) } free_addrs: kfree(jit.addrs); +out: + if (tmp_blinded) + bpf_jit_prog_release_other(fp, fp == orig_fp ? + tmp : orig_fp); + return fp; } /* diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 21591ddb4c1f..1a4512c8544a 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -176,8 +176,7 @@ static int clp_query_pci_fn(struct zpci_dev *zdev, u32 fh) rc = clp_store_query_pci_fn(zdev, &rrb->response); if (rc) goto out; - if (rrb->response.pfgid) - rc = clp_query_pci_fngrp(zdev, rrb->response.pfgid); + rc = clp_query_pci_fngrp(zdev, rrb->response.pfgid); } else { zpci_err("Q PCI FN:\n"); zpci_err_clp(rrb->response.hdr.rsp, rc); diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index c555de3d12d6..38993b156924 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -1,5 +1,5 @@ /* - * Copyright IBM Corp. 2012 + * Copyright IBM Corp. 2012,2015 * * Author(s): * Jan Glauber <jang@linux.vnet.ibm.com> @@ -23,22 +23,45 @@ EXPORT_SYMBOL_GPL(pci_debug_msg_id); debug_info_t *pci_debug_err_id; EXPORT_SYMBOL_GPL(pci_debug_err_id); -static char *pci_perf_names[] = { - /* hardware counters */ +static char *pci_common_names[] = { "Load operations", "Store operations", "Store block operations", "Refresh operations", +}; + +static char *pci_fmt0_names[] = { "DMA read bytes", "DMA write bytes", }; +static char *pci_fmt1_names[] = { + "Received bytes", + "Received packets", + "Transmitted bytes", + "Transmitted packets", +}; + +static char *pci_fmt2_names[] = { + "Consumed work units", + "Maximum work units", +}; + static char *pci_sw_names[] = { "Allocated pages", "Mapped pages", "Unmapped pages", }; +static void pci_fmb_show(struct seq_file *m, char *name[], int length, + u64 *data) +{ + int i; + + for (i = 0; i < length; i++, data++) + seq_printf(m, "%26s:\t%llu\n", name[i], *data); +} + static void pci_sw_counter_show(struct seq_file *m) { struct zpci_dev *zdev = m->private; @@ -53,8 +76,6 @@ static void pci_sw_counter_show(struct seq_file *m) static int pci_perf_show(struct seq_file *m, void *v) { struct zpci_dev *zdev = m->private; - u64 *stat; - int i; if (!zdev) return 0; @@ -72,15 +93,27 @@ static int pci_perf_show(struct seq_file *m, void *v) seq_printf(m, "Samples: %u\n", zdev->fmb->samples); seq_printf(m, "Last update TOD: %Lx\n", zdev->fmb->last_update); - /* hardware counters */ - stat = (u64 *) &zdev->fmb->ld_ops; - for (i = 0; i < 4; i++) - seq_printf(m, "%26s:\t%llu\n", - pci_perf_names[i], *(stat + i)); - if (zdev->fmb->dma_valid) - for (i = 4; i < 6; i++) - seq_printf(m, "%26s:\t%llu\n", - pci_perf_names[i], *(stat + i)); + pci_fmb_show(m, pci_common_names, ARRAY_SIZE(pci_common_names), + &zdev->fmb->ld_ops); + + switch (zdev->fmb->format) { + case 0: + if (!(zdev->fmb->fmt_ind & ZPCI_FMB_DMA_COUNTER_VALID)) + break; + pci_fmb_show(m, pci_fmt0_names, ARRAY_SIZE(pci_fmt0_names), + &zdev->fmb->fmt0.dma_rbytes); + break; + case 1: + pci_fmb_show(m, pci_fmt1_names, ARRAY_SIZE(pci_fmt1_names), + &zdev->fmb->fmt1.rx_bytes); + break; + case 2: + pci_fmb_show(m, pci_fmt2_names, ARRAY_SIZE(pci_fmt2_names), + &zdev->fmb->fmt2.consumed_work_units); + break; + default: + seq_puts(m, "Unknown format\n"); + } pci_sw_counter_show(m); mutex_unlock(&zdev->lock); diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index e595e89eac65..1ea8c07eab84 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -457,7 +457,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev) zdev->dma_table = dma_alloc_cpu_table(); if (!zdev->dma_table) { rc = -ENOMEM; - goto out_clean; + goto out; } /* @@ -477,18 +477,22 @@ int zpci_dma_init_device(struct zpci_dev *zdev) zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8); if (!zdev->iommu_bitmap) { rc = -ENOMEM; - goto out_reg; + goto free_dma_table; } rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, (u64) zdev->dma_table); if (rc) - goto out_reg; - return 0; + goto free_bitmap; -out_reg: + return 0; +free_bitmap: + vfree(zdev->iommu_bitmap); + zdev->iommu_bitmap = NULL; +free_dma_table: dma_free_cpu_table(zdev->dma_table); -out_clean: + zdev->dma_table = NULL; +out: return rc; } diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index f37a5808883d..ed484dc84d14 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -12,6 +12,8 @@ #include <linux/stat.h> #include <linux/pci.h> +#include <asm/sclp.h> + #define zpci_attr(name, fmt, member) \ static ssize_t name##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ @@ -77,8 +79,29 @@ static ssize_t util_string_read(struct file *filp, struct kobject *kobj, sizeof(zdev->util_str)); } static BIN_ATTR_RO(util_string, CLP_UTIL_STR_LEN); + +static ssize_t report_error_write(struct file *filp, struct kobject *kobj, + struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + struct zpci_report_error_header *report = (void *) buf; + struct device *dev = kobj_to_dev(kobj); + struct pci_dev *pdev = to_pci_dev(dev); + struct zpci_dev *zdev = to_zpci(pdev); + int ret; + + if (off || (count < sizeof(*report))) + return -EINVAL; + + ret = sclp_pci_report(report, zdev->fh, zdev->fid); + + return ret ? ret : count; +} +static BIN_ATTR(report_error, S_IWUSR, NULL, report_error_write, PAGE_SIZE); + static struct bin_attribute *zpci_bin_attrs[] = { &bin_attr_util_string, + &bin_attr_report_error, NULL, }; |