From a440943e68cd1b5a853a6f60865967b7cc2539eb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Sep 2021 08:59:58 +0200 Subject: unicode: remove the charset field from struct unicode_map It is hardcoded and only used for a f2fs sysfs file where it can be hardcoded just as easily. Signed-off-by: Christoph Hellwig Reviewed-by: Gabriel Krisman Bertazi Signed-off-by: Gabriel Krisman Bertazi --- include/linux/unicode.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 74484d44c755..6a392cd9f076 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -6,7 +6,6 @@ #include struct unicode_map { - const char *charset; int version; }; -- cgit v1.2.3-59-g8ed1b From f3a9c82396006a5664f6e398d6928799d29de76e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Sep 2021 08:59:59 +0200 Subject: unicode: mark the version field in struct unicode_map unsigned unicode version tripplets are always unsigned. Signed-off-by: Christoph Hellwig Reviewed-by: Gabriel Krisman Bertazi Signed-off-by: Gabriel Krisman Bertazi --- include/linux/unicode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 6a392cd9f076..0744f81c4b5f 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -6,7 +6,7 @@ #include struct unicode_map { - int version; + unsigned int version; }; int utf8_validate(const struct unicode_map *um, const struct qstr *str); -- cgit v1.2.3-59-g8ed1b From 49bd03cc7e95cb78420305ca2f5ef67497b6fa80 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Sep 2021 09:00:00 +0200 Subject: unicode: pass a UNICODE_AGE() tripple to utf8_load Don't bother with pointless string parsing when the caller can just pass the version in the format that the core expects. Also remove the fallback to the latest version that none of the callers actually uses. Signed-off-by: Christoph Hellwig Signed-off-by: Gabriel Krisman Bertazi --- fs/ext4/super.c | 18 +++++++++++------ fs/f2fs/super.c | 18 +++++++++++------ fs/unicode/utf8-core.c | 50 +++++----------------------------------------- fs/unicode/utf8-norm.c | 11 ++-------- fs/unicode/utf8-selftest.c | 15 +++++++------- fs/unicode/utf8n.h | 14 ++----------- include/linux/unicode.h | 25 ++++++++++++++++++++++- 7 files changed, 65 insertions(+), 86 deletions(-) (limited to 'include/linux') diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 26c0e6fc5bce..85e73093c61a 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -2018,9 +2018,9 @@ static const struct mount_opts { static const struct ext4_sb_encodings { __u16 magic; char *name; - char *version; + unsigned int version; } ext4_sb_encoding_map[] = { - {EXT4_ENC_UTF8_12_1, "utf8", "12.1.0"}, + {EXT4_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)}, }; static const struct ext4_sb_encodings * @@ -4166,15 +4166,21 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) encoding = utf8_load(encoding_info->version); if (IS_ERR(encoding)) { ext4_msg(sb, KERN_ERR, - "can't mount with superblock charset: %s-%s " + "can't mount with superblock charset: %s-%u.%u.%u " "not supported by the kernel. flags: 0x%x.", - encoding_info->name, encoding_info->version, + encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), encoding_flags); goto failed_mount; } ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " - "%s-%s with flags 0x%hx", encoding_info->name, - encoding_info->version?:"\b", encoding_flags); + "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); sb->s_encoding = encoding; sb->s_encoding_flags = encoding_flags; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 4c457100f18e..3029f71bf2b7 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -259,9 +259,9 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...) static const struct f2fs_sb_encodings { __u16 magic; char *name; - char *version; + unsigned int version; } f2fs_sb_encoding_map[] = { - {F2FS_ENC_UTF8_12_1, "utf8", "12.1.0"}, + {F2FS_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)}, }; static const struct f2fs_sb_encodings * @@ -3847,15 +3847,21 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi) encoding = utf8_load(encoding_info->version); if (IS_ERR(encoding)) { f2fs_err(sbi, - "can't mount with superblock charset: %s-%s " + "can't mount with superblock charset: %s-%u.%u.%u " "not supported by the kernel. flags: 0x%x.", - encoding_info->name, encoding_info->version, + encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), encoding_flags); return PTR_ERR(encoding); } f2fs_info(sbi, "Using encoding defined by superblock: " - "%s-%s with flags 0x%hx", encoding_info->name, - encoding_info->version?:"\b", encoding_flags); + "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); sbi->sb->s_encoding = encoding; sbi->sb->s_encoding_flags = encoding_flags; diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c index 86f42a078d99..dca2865c3bee 100644 --- a/fs/unicode/utf8-core.c +++ b/fs/unicode/utf8-core.c @@ -167,59 +167,19 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str, } return -EINVAL; } - EXPORT_SYMBOL(utf8_normalize); -static int utf8_parse_version(const char *version, unsigned int *maj, - unsigned int *min, unsigned int *rev) +struct unicode_map *utf8_load(unsigned int version) { - substring_t args[3]; - char version_string[12]; - static const struct match_token token[] = { - {1, "%d.%d.%d"}, - {0, NULL} - }; - - strncpy(version_string, version, sizeof(version_string)); - - if (match_token(version_string, token, args) != 1) - return -EINVAL; - - if (match_int(&args[0], maj) || match_int(&args[1], min) || - match_int(&args[2], rev)) - return -EINVAL; + struct unicode_map *um; - return 0; -} - -struct unicode_map *utf8_load(const char *version) -{ - struct unicode_map *um = NULL; - int unicode_version; - - if (version) { - unsigned int maj, min, rev; - - if (utf8_parse_version(version, &maj, &min, &rev) < 0) - return ERR_PTR(-EINVAL); - - if (!utf8version_is_supported(maj, min, rev)) - return ERR_PTR(-EINVAL); - - unicode_version = UNICODE_AGE(maj, min, rev); - } else { - unicode_version = utf8version_latest(); - printk(KERN_WARNING"UTF-8 version not specified. " - "Assuming latest supported version (%d.%d.%d).", - (unicode_version >> 16) & 0xff, - (unicode_version >> 8) & 0xff, - (unicode_version & 0xff)); - } + if (!utf8version_is_supported(version)) + return ERR_PTR(-EINVAL); um = kzalloc(sizeof(struct unicode_map), GFP_KERNEL); if (!um) return ERR_PTR(-ENOMEM); - um->version = unicode_version; + um->version = version; return um; } EXPORT_SYMBOL(utf8_load); diff --git a/fs/unicode/utf8-norm.c b/fs/unicode/utf8-norm.c index 1d2d2e5b906a..12abf89ae6ec 100644 --- a/fs/unicode/utf8-norm.c +++ b/fs/unicode/utf8-norm.c @@ -15,13 +15,12 @@ struct utf8data { #include "utf8data.h" #undef __INCLUDED_FROM_UTF8NORM_C__ -int utf8version_is_supported(u8 maj, u8 min, u8 rev) +int utf8version_is_supported(unsigned int version) { int i = ARRAY_SIZE(utf8agetab) - 1; - unsigned int sb_utf8version = UNICODE_AGE(maj, min, rev); while (i >= 0 && utf8agetab[i] != 0) { - if (sb_utf8version == utf8agetab[i]) + if (version == utf8agetab[i]) return 1; i--; } @@ -29,12 +28,6 @@ int utf8version_is_supported(u8 maj, u8 min, u8 rev) } EXPORT_SYMBOL(utf8version_is_supported); -int utf8version_latest(void) -{ - return utf8vers; -} -EXPORT_SYMBOL(utf8version_latest); - /* * UTF-8 valid ranges. * diff --git a/fs/unicode/utf8-selftest.c b/fs/unicode/utf8-selftest.c index 6fe8af7edccb..37f33890e012 100644 --- a/fs/unicode/utf8-selftest.c +++ b/fs/unicode/utf8-selftest.c @@ -235,7 +235,7 @@ static void check_utf8_nfdicf(void) static void check_utf8_comparisons(void) { int i; - struct unicode_map *table = utf8_load("12.1.0"); + struct unicode_map *table = utf8_load(UNICODE_AGE(12, 1, 0)); if (IS_ERR(table)) { pr_err("%s: Unable to load utf8 %d.%d.%d. Skipping.\n", @@ -269,18 +269,19 @@ static void check_utf8_comparisons(void) static void check_supported_versions(void) { /* Unicode 7.0.0 should be supported. */ - test(utf8version_is_supported(7, 0, 0)); + test(utf8version_is_supported(UNICODE_AGE(7, 0, 0))); /* Unicode 9.0.0 should be supported. */ - test(utf8version_is_supported(9, 0, 0)); + test(utf8version_is_supported(UNICODE_AGE(9, 0, 0))); /* Unicode 1x.0.0 (the latest version) should be supported. */ - test(utf8version_is_supported(latest_maj, latest_min, latest_rev)); + test(utf8version_is_supported( + UNICODE_AGE(latest_maj, latest_min, latest_rev))); /* Next versions don't exist. */ - test(!utf8version_is_supported(13, 0, 0)); - test(!utf8version_is_supported(0, 0, 0)); - test(!utf8version_is_supported(-1, -1, -1)); + test(!utf8version_is_supported(UNICODE_AGE(13, 0, 0))); + test(!utf8version_is_supported(UNICODE_AGE(0, 0, 0))); + test(!utf8version_is_supported(UNICODE_AGE(-1, -1, -1))); } static int __init init_test_ucd(void) diff --git a/fs/unicode/utf8n.h b/fs/unicode/utf8n.h index 0acd530c2c79..85a7bebf6927 100644 --- a/fs/unicode/utf8n.h +++ b/fs/unicode/utf8n.h @@ -11,19 +11,9 @@ #include #include #include +#include -/* Encoding a unicode version number as a single unsigned int. */ -#define UNICODE_MAJ_SHIFT (16) -#define UNICODE_MIN_SHIFT (8) - -#define UNICODE_AGE(MAJ, MIN, REV) \ - (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ - ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ - ((unsigned int)(REV))) - -/* Highest unicode version supported by the data tables. */ -extern int utf8version_is_supported(u8 maj, u8 min, u8 rev); -extern int utf8version_latest(void); +int utf8version_is_supported(unsigned int version); /* * Look for the correct const struct utf8data for a unicode version. diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 0744f81c4b5f..77bb915fd1f0 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -5,6 +5,29 @@ #include #include +#define UNICODE_MAJ_SHIFT 16 +#define UNICODE_MIN_SHIFT 8 + +#define UNICODE_AGE(MAJ, MIN, REV) \ + (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ + ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ + ((unsigned int)(REV))) + +static inline u8 unicode_major(unsigned int age) +{ + return (age >> UNICODE_MAJ_SHIFT) & 0xff; +} + +static inline u8 unicode_minor(unsigned int age) +{ + return (age >> UNICODE_MIN_SHIFT) & 0xff; +} + +static inline u8 unicode_rev(unsigned int age) +{ + return age & 0xff; +} + struct unicode_map { unsigned int version; }; @@ -29,7 +52,7 @@ int utf8_casefold(const struct unicode_map *um, const struct qstr *str, int utf8_casefold_hash(const struct unicode_map *um, const void *salt, struct qstr *str); -struct unicode_map *utf8_load(const char *version); +struct unicode_map *utf8_load(unsigned int version); void utf8_unload(struct unicode_map *um); #endif /* _LINUX_UNICODE_H */ -- cgit v1.2.3-59-g8ed1b From 6ca99ce756c27852d1ea1e555045de1c920f30ed Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Sep 2021 09:00:04 +0200 Subject: unicode: cache the normalization tables in struct unicode_map Instead of repeatedly looking up the version add pointers to the NFD and NFD+CF tables to struct unicode_map, and pass a unicode_map plus index to the functions using the normalization tables. Signed-off-by: Christoph Hellwig Signed-off-by: Gabriel Krisman Bertazi --- fs/unicode/utf8-core.c | 37 ++++++++++----------- fs/unicode/utf8-norm.c | 45 ++++++++++++-------------- fs/unicode/utf8-selftest.c | 80 +++++++++++++++++++--------------------------- fs/unicode/utf8n.h | 10 +++--- include/linux/unicode.h | 19 +++++++++++ 5 files changed, 97 insertions(+), 94 deletions(-) (limited to 'include/linux') diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c index dca2865c3bee..d9f713d38c0a 100644 --- a/fs/unicode/utf8-core.c +++ b/fs/unicode/utf8-core.c @@ -5,16 +5,13 @@ #include #include #include -#include #include #include "utf8n.h" int utf8_validate(const struct unicode_map *um, const struct qstr *str) { - const struct utf8data *data = utf8nfdi(um->version); - - if (utf8nlen(data, str->name, str->len) < 0) + if (utf8nlen(um, UTF8_NFDI, str->name, str->len) < 0) return -1; return 0; } @@ -23,14 +20,13 @@ EXPORT_SYMBOL(utf8_validate); int utf8_strncmp(const struct unicode_map *um, const struct qstr *s1, const struct qstr *s2) { - const struct utf8data *data = utf8nfdi(um->version); struct utf8cursor cur1, cur2; int c1, c2; - if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0) + if (utf8ncursor(&cur1, um, UTF8_NFDI, s1->name, s1->len) < 0) return -EINVAL; - if (utf8ncursor(&cur2, data, s2->name, s2->len) < 0) + if (utf8ncursor(&cur2, um, UTF8_NFDI, s2->name, s2->len) < 0) return -EINVAL; do { @@ -50,14 +46,13 @@ EXPORT_SYMBOL(utf8_strncmp); int utf8_strncasecmp(const struct unicode_map *um, const struct qstr *s1, const struct qstr *s2) { - const struct utf8data *data = utf8nfdicf(um->version); struct utf8cursor cur1, cur2; int c1, c2; - if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0) + if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0) return -EINVAL; - if (utf8ncursor(&cur2, data, s2->name, s2->len) < 0) + if (utf8ncursor(&cur2, um, UTF8_NFDICF, s2->name, s2->len) < 0) return -EINVAL; do { @@ -81,12 +76,11 @@ int utf8_strncasecmp_folded(const struct unicode_map *um, const struct qstr *cf, const struct qstr *s1) { - const struct utf8data *data = utf8nfdicf(um->version); struct utf8cursor cur1; int c1, c2; int i = 0; - if (utf8ncursor(&cur1, data, s1->name, s1->len) < 0) + if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0) return -EINVAL; do { @@ -105,11 +99,10 @@ EXPORT_SYMBOL(utf8_strncasecmp_folded); int utf8_casefold(const struct unicode_map *um, const struct qstr *str, unsigned char *dest, size_t dlen) { - const struct utf8data *data = utf8nfdicf(um->version); struct utf8cursor cur; size_t nlen = 0; - if (utf8ncursor(&cur, data, str->name, str->len) < 0) + if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0) return -EINVAL; for (nlen = 0; nlen < dlen; nlen++) { @@ -128,12 +121,11 @@ EXPORT_SYMBOL(utf8_casefold); int utf8_casefold_hash(const struct unicode_map *um, const void *salt, struct qstr *str) { - const struct utf8data *data = utf8nfdicf(um->version); struct utf8cursor cur; int c; unsigned long hash = init_name_hash(salt); - if (utf8ncursor(&cur, data, str->name, str->len) < 0) + if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0) return -EINVAL; while ((c = utf8byte(&cur))) { @@ -149,11 +141,10 @@ EXPORT_SYMBOL(utf8_casefold_hash); int utf8_normalize(const struct unicode_map *um, const struct qstr *str, unsigned char *dest, size_t dlen) { - const struct utf8data *data = utf8nfdi(um->version); struct utf8cursor cur; ssize_t nlen = 0; - if (utf8ncursor(&cur, data, str->name, str->len) < 0) + if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0) return -EINVAL; for (nlen = 0; nlen < dlen; nlen++) { @@ -180,7 +171,17 @@ struct unicode_map *utf8_load(unsigned int version) if (!um) return ERR_PTR(-ENOMEM); um->version = version; + um->ntab[UTF8_NFDI] = utf8nfdi(version); + if (!um->ntab[UTF8_NFDI]) + goto out_free_um; + um->ntab[UTF8_NFDICF] = utf8nfdicf(version); + if (!um->ntab[UTF8_NFDICF]) + goto out_free_um; return um; + +out_free_um: + kfree(um); + return ERR_PTR(-EINVAL); } EXPORT_SYMBOL(utf8_load); diff --git a/fs/unicode/utf8-norm.c b/fs/unicode/utf8-norm.c index 1ac90fa00070..7c1f28ab31a8 100644 --- a/fs/unicode/utf8-norm.c +++ b/fs/unicode/utf8-norm.c @@ -309,21 +309,19 @@ utf8hangul(const char *str, unsigned char *hangul) * is well-formed and corresponds to a known unicode code point. The * shorthand for this will be "is valid UTF-8 unicode". */ -static utf8leaf_t *utf8nlookup(const struct utf8data *data, - unsigned char *hangul, const char *s, size_t len) +static utf8leaf_t *utf8nlookup(const struct unicode_map *um, + enum utf8_normalization n, unsigned char *hangul, const char *s, + size_t len) { - utf8trie_t *trie = NULL; + utf8trie_t *trie = utf8data + um->ntab[n]->offset; int offlen; int offset; int mask; int node; - if (!data) - return NULL; if (len == 0) return NULL; - trie = utf8data + data->offset; node = 1; while (node) { offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT; @@ -385,29 +383,28 @@ static utf8leaf_t *utf8nlookup(const struct utf8data *data, * * Forwards to utf8nlookup(). */ -static utf8leaf_t *utf8lookup(const struct utf8data *data, - unsigned char *hangul, const char *s) +static utf8leaf_t *utf8lookup(const struct unicode_map *um, + enum utf8_normalization n, unsigned char *hangul, const char *s) { - return utf8nlookup(data, hangul, s, (size_t)-1); + return utf8nlookup(um, n, hangul, s, (size_t)-1); } /* * Length of the normalization of s, touch at most len bytes. * Return -1 if s is not valid UTF-8 unicode. */ -ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len) +ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, + const char *s, size_t len) { utf8leaf_t *leaf; size_t ret = 0; unsigned char hangul[UTF8HANGULLEAF]; - if (!data) - return -1; while (len && *s) { - leaf = utf8nlookup(data, hangul, s, len); + leaf = utf8nlookup(um, n, hangul, s, len); if (!leaf) return -1; - if (utf8agetab[LEAF_GEN(leaf)] > data->maxage) + if (utf8agetab[LEAF_GEN(leaf)] > um->ntab[n]->maxage) ret += utf8clen(s); else if (LEAF_CCC(leaf) == DECOMPOSE) ret += strlen(LEAF_STR(leaf)); @@ -430,14 +427,13 @@ EXPORT_SYMBOL(utf8nlen); * * Returns -1 on error, 0 on success. */ -int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, - const char *s, size_t len) +int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s, size_t len) { - if (!data) - return -1; if (!s) return -1; - u8c->data = data; + u8c->um = um; + u8c->n = n; u8c->s = s; u8c->p = NULL; u8c->ss = NULL; @@ -512,9 +508,9 @@ int utf8byte(struct utf8cursor *u8c) /* Look up the data for the current character. */ if (u8c->p) { - leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); + leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s); } else { - leaf = utf8nlookup(u8c->data, u8c->hangul, + leaf = utf8nlookup(u8c->um, u8c->n, u8c->hangul, u8c->s, u8c->len); } @@ -524,7 +520,8 @@ int utf8byte(struct utf8cursor *u8c) ccc = LEAF_CCC(leaf); /* Characters that are too new have CCC 0. */ - if (utf8agetab[LEAF_GEN(leaf)] > u8c->data->maxage) { + if (utf8agetab[LEAF_GEN(leaf)] > + u8c->um->ntab[u8c->n]->maxage) { ccc = STOPPER; } else if (ccc == DECOMPOSE) { u8c->len -= utf8clen(u8c->s); @@ -538,7 +535,7 @@ int utf8byte(struct utf8cursor *u8c) goto ccc_mismatch; } - leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); + leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s); if (!leaf) return -1; ccc = LEAF_CCC(leaf); @@ -611,7 +608,6 @@ const struct utf8data *utf8nfdi(unsigned int maxage) return NULL; return &utf8nfdidata[i]; } -EXPORT_SYMBOL(utf8nfdi); const struct utf8data *utf8nfdicf(unsigned int maxage) { @@ -623,4 +619,3 @@ const struct utf8data *utf8nfdicf(unsigned int maxage) return NULL; return &utf8nfdicfdata[i]; } -EXPORT_SYMBOL(utf8nfdicf); diff --git a/fs/unicode/utf8-selftest.c b/fs/unicode/utf8-selftest.c index 04628b50351d..cfa3832b75f4 100644 --- a/fs/unicode/utf8-selftest.c +++ b/fs/unicode/utf8-selftest.c @@ -18,9 +18,7 @@ unsigned int failed_tests; unsigned int total_tests; /* Tests will be based on this version. */ -#define latest_maj 12 -#define latest_min 1 -#define latest_rev 0 +#define UTF8_LATEST UNICODE_AGE(12, 1, 0) #define _test(cond, func, line, fmt, ...) do { \ total_tests++; \ @@ -160,29 +158,22 @@ static const struct { } }; -static ssize_t utf8len(const struct utf8data *data, const char *s) +static ssize_t utf8len(const struct unicode_map *um, enum utf8_normalization n, + const char *s) { - return utf8nlen(data, s, (size_t)-1); + return utf8nlen(um, n, s, (size_t)-1); } -static int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data, - const char *s) +static int utf8cursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s) { - return utf8ncursor(u8c, data, s, (unsigned int)-1); + return utf8ncursor(u8c, um, n, s, (unsigned int)-1); } -static void check_utf8_nfdi(void) +static void check_utf8_nfdi(struct unicode_map *um) { int i; struct utf8cursor u8c; - const struct utf8data *data; - - data = utf8nfdi(UNICODE_AGE(latest_maj, latest_min, latest_rev)); - if (!data) { - pr_err("%s: Unable to load utf8-%d.%d.%d. Skipping.\n", - __func__, latest_maj, latest_min, latest_rev); - return; - } for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) { int len = strlen(nfdi_test_data[i].str); @@ -190,10 +181,11 @@ static void check_utf8_nfdi(void) int j = 0; unsigned char c; - test((utf8len(data, nfdi_test_data[i].str) == nlen)); - test((utf8nlen(data, nfdi_test_data[i].str, len) == nlen)); + test((utf8len(um, UTF8_NFDI, nfdi_test_data[i].str) == nlen)); + test((utf8nlen(um, UTF8_NFDI, nfdi_test_data[i].str, len) == + nlen)); - if (utf8cursor(&u8c, data, nfdi_test_data[i].str) < 0) + if (utf8cursor(&u8c, um, UTF8_NFDI, nfdi_test_data[i].str) < 0) pr_err("can't create cursor\n"); while ((c = utf8byte(&u8c)) > 0) { @@ -207,18 +199,10 @@ static void check_utf8_nfdi(void) } } -static void check_utf8_nfdicf(void) +static void check_utf8_nfdicf(struct unicode_map *um) { int i; struct utf8cursor u8c; - const struct utf8data *data; - - data = utf8nfdicf(UNICODE_AGE(latest_maj, latest_min, latest_rev)); - if (!data) { - pr_err("%s: Unable to load utf8-%d.%d.%d. Skipping.\n", - __func__, latest_maj, latest_min, latest_rev); - return; - } for (i = 0; i < ARRAY_SIZE(nfdicf_test_data); i++) { int len = strlen(nfdicf_test_data[i].str); @@ -226,10 +210,13 @@ static void check_utf8_nfdicf(void) int j = 0; unsigned char c; - test((utf8len(data, nfdicf_test_data[i].str) == nlen)); - test((utf8nlen(data, nfdicf_test_data[i].str, len) == nlen)); + test((utf8len(um, UTF8_NFDICF, nfdicf_test_data[i].str) == + nlen)); + test((utf8nlen(um, UTF8_NFDICF, nfdicf_test_data[i].str, len) == + nlen)); - if (utf8cursor(&u8c, data, nfdicf_test_data[i].str) < 0) + if (utf8cursor(&u8c, um, UTF8_NFDICF, + nfdicf_test_data[i].str) < 0) pr_err("can't create cursor\n"); while ((c = utf8byte(&u8c)) > 0) { @@ -243,16 +230,9 @@ static void check_utf8_nfdicf(void) } } -static void check_utf8_comparisons(void) +static void check_utf8_comparisons(struct unicode_map *table) { int i; - struct unicode_map *table = utf8_load(UNICODE_AGE(12, 1, 0)); - - if (IS_ERR(table)) { - pr_err("%s: Unable to load utf8 %d.%d.%d. Skipping.\n", - __func__, latest_maj, latest_min, latest_rev); - return; - } for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) { const struct qstr s1 = {.name = nfdi_test_data[i].str, @@ -273,8 +253,6 @@ static void check_utf8_comparisons(void) test_f(!utf8_strncasecmp(table, &s1, &s2), "%s %s comparison mismatch\n", s1.name, s2.name); } - - utf8_unload(table); } static void check_supported_versions(void) @@ -286,8 +264,7 @@ static void check_supported_versions(void) test(utf8version_is_supported(UNICODE_AGE(9, 0, 0))); /* Unicode 1x.0.0 (the latest version) should be supported. */ - test(utf8version_is_supported( - UNICODE_AGE(latest_maj, latest_min, latest_rev))); + test(utf8version_is_supported(UTF8_LATEST)); /* Next versions don't exist. */ test(!utf8version_is_supported(UNICODE_AGE(13, 0, 0))); @@ -297,19 +274,28 @@ static void check_supported_versions(void) static int __init init_test_ucd(void) { + struct unicode_map *um; + failed_tests = 0; total_tests = 0; + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + pr_err("%s: Unable to load utf8 table.\n", __func__); + return PTR_ERR(um); + } + check_supported_versions(); - check_utf8_nfdi(); - check_utf8_nfdicf(); - check_utf8_comparisons(); + check_utf8_nfdi(um); + check_utf8_nfdicf(um); + check_utf8_comparisons(um); if (!failed_tests) pr_info("All %u tests passed\n", total_tests); else pr_err("%u out of %u tests failed\n", failed_tests, total_tests); + utf8_unload(um); return 0; } diff --git a/fs/unicode/utf8n.h b/fs/unicode/utf8n.h index 736b6460a38c..206c89f0dbf7 100644 --- a/fs/unicode/utf8n.h +++ b/fs/unicode/utf8n.h @@ -39,7 +39,8 @@ extern const struct utf8data *utf8nfdicf(unsigned int maxage); * Returns 0 if only ignorable code points are present. * Returns -1 if the input is not valid UTF-8. */ -extern ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len); +ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, + const char *s, size_t len); /* Needed in struct utf8cursor below. */ #define UTF8HANGULLEAF (12) @@ -48,7 +49,8 @@ extern ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len); * Cursor structure used by the normalizer. */ struct utf8cursor { - const struct utf8data *data; + const struct unicode_map *um; + enum utf8_normalization n; const char *s; const char *p; const char *ss; @@ -65,8 +67,8 @@ struct utf8cursor { * Returns 0 on success. * Returns -1 on failure. */ -extern int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, - const char *s, size_t len); +int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s, size_t len); /* * Get the next byte in the normalization. diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 77bb915fd1f0..526ca8b8391a 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -5,6 +5,8 @@ #include #include +struct utf8data; + #define UNICODE_MAJ_SHIFT 16 #define UNICODE_MIN_SHIFT 8 @@ -28,8 +30,25 @@ static inline u8 unicode_rev(unsigned int age) return age & 0xff; } +/* + * Two normalization forms are supported: + * 1) NFDI + * - Apply unicode normalization form NFD. + * - Remove any Default_Ignorable_Code_Point. + * 2) NFDICF + * - Apply unicode normalization form NFD. + * - Remove any Default_Ignorable_Code_Point. + * - Apply a full casefold (C + F). + */ +enum utf8_normalization { + UTF8_NFDI = 0, + UTF8_NFDICF, + UTF8_NMAX, +}; + struct unicode_map { unsigned int version; + const struct utf8data *ntab[UTF8_NMAX]; }; int utf8_validate(const struct unicode_map *um, const struct qstr *str); -- cgit v1.2.3-59-g8ed1b From 2b3d047870120bcd46d7cc257d19ff49328fd585 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Sep 2021 09:00:05 +0200 Subject: unicode: Add utf8-data module utf8data.h contains a large database table which is an auto-generated decodification trie for the unicode normalization functions. Allow building it into a separate module. Based on a patch from Shreeya Patel . Signed-off-by: Christoph Hellwig Signed-off-by: Gabriel Krisman Bertazi --- fs/unicode/Kconfig | 13 +- fs/unicode/Makefile | 13 +- fs/unicode/mkutf8data.c | 24 +- fs/unicode/utf8-core.c | 39 +- fs/unicode/utf8-norm.c | 48 +- fs/unicode/utf8-selftest.c | 16 +- fs/unicode/utf8data.c_shipped | 4123 +++++++++++++++++++++++++++++++++++++++++ fs/unicode/utf8data.h_shipped | 4109 ---------------------------------------- fs/unicode/utf8n.h | 40 +- include/linux/unicode.h | 2 + 10 files changed, 4231 insertions(+), 4196 deletions(-) create mode 100644 fs/unicode/utf8data.c_shipped delete mode 100644 fs/unicode/utf8data.h_shipped (limited to 'include/linux') diff --git a/fs/unicode/Kconfig b/fs/unicode/Kconfig index 2c27b9a5cd6c..610d7bc05d6e 100644 --- a/fs/unicode/Kconfig +++ b/fs/unicode/Kconfig @@ -8,7 +8,16 @@ config UNICODE Say Y here to enable UTF-8 NFD normalization and NFD+CF casefolding support. +config UNICODE_UTF8_DATA + tristate "UTF-8 normalization and casefolding tables" + depends on UNICODE + default UNICODE + help + This contains a large table of case foldings, which can be loaded as + a separate module if you say M here. To be on the safe side stick + to the default of Y. Saying N here makes no sense, if you do not want + utf8 casefolding support, disable CONFIG_UNICODE instead. + config UNICODE_NORMALIZATION_SELFTEST tristate "Test UTF-8 normalization support" - depends on UNICODE - default n + depends on UNICODE_UTF8_DATA diff --git a/fs/unicode/Makefile b/fs/unicode/Makefile index b88aecc86550..2f9d9188852b 100644 --- a/fs/unicode/Makefile +++ b/fs/unicode/Makefile @@ -2,14 +2,15 @@ obj-$(CONFIG_UNICODE) += unicode.o obj-$(CONFIG_UNICODE_NORMALIZATION_SELFTEST) += utf8-selftest.o +obj-$(CONFIG_UNICODE_UTF8_DATA) += utf8data.o unicode-y := utf8-norm.o utf8-core.o -$(obj)/utf8-norm.o: $(obj)/utf8data.h +$(obj)/utf8-data.o: $(obj)/utf8data.c -# In the normal build, the checked-in utf8data.h is just shipped. +# In the normal build, the checked-in utf8data.c is just shipped. # -# To generate utf8data.h from UCD, put *.txt files in this directory +# To generate utf8data.c from UCD, put *.txt files in this directory # and pass REGENERATE_UTF8DATA=1 from the command line. ifdef REGENERATE_UTF8DATA @@ -24,15 +25,15 @@ quiet_cmd_utf8data = GEN $@ -t $(srctree)/$(src)/NormalizationTest.txt \ -o $@ -$(obj)/utf8data.h: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE +$(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE $(call if_changed,utf8data) else -$(obj)/utf8data.h: $(src)/utf8data.h_shipped FORCE +$(obj)/utf8data.c: $(src)/utf8data.c_shipped FORCE $(call if_changed,shipped) endif -targets += utf8data.h +targets += utf8data.c hostprogs += mkutf8data diff --git a/fs/unicode/mkutf8data.c b/fs/unicode/mkutf8data.c index ff2025ac5a32..bc1a7c8b5c8d 100644 --- a/fs/unicode/mkutf8data.c +++ b/fs/unicode/mkutf8data.c @@ -3287,12 +3287,10 @@ static void write_file(void) open_fail(utf8_name, errno); fprintf(file, "/* This file is generated code, do not edit. */\n"); - fprintf(file, "#ifndef __INCLUDED_FROM_UTF8NORM_C__\n"); - fprintf(file, "#error Only nls_utf8-norm.c should include this file.\n"); - fprintf(file, "#endif\n"); fprintf(file, "\n"); - fprintf(file, "static const unsigned int utf8vers = %#x;\n", - unicode_maxage); + fprintf(file, "#include \n"); + fprintf(file, "#include \n"); + fprintf(file, "#include \"utf8n.h\"\n"); fprintf(file, "\n"); fprintf(file, "static const unsigned int utf8agetab[] = {\n"); for (i = 0; i != ages_count; i++) @@ -3339,6 +3337,22 @@ static void write_file(void) fprintf(file, "\n"); } fprintf(file, "};\n"); + fprintf(file, "\n"); + fprintf(file, "struct utf8data_table utf8_data_table = {\n"); + fprintf(file, "\t.utf8agetab = utf8agetab,\n"); + fprintf(file, "\t.utf8agetab_size = ARRAY_SIZE(utf8agetab),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8nfdicfdata = utf8nfdicfdata,\n"); + fprintf(file, "\t.utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8nfdidata = utf8nfdidata,\n"); + fprintf(file, "\t.utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8data = utf8data,\n"); + fprintf(file, "};\n"); + fprintf(file, "EXPORT_SYMBOL_GPL(utf8_data_table);"); + fprintf(file, "\n"); + fprintf(file, "MODULE_LICENSE(\"GPL v2\");\n"); fclose(file); } diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c index d9f713d38c0a..67aaadc3ab07 100644 --- a/fs/unicode/utf8-core.c +++ b/fs/unicode/utf8-core.c @@ -160,25 +160,45 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str, } EXPORT_SYMBOL(utf8_normalize); +static const struct utf8data *find_table_version(const struct utf8data *table, + size_t nr_entries, unsigned int version) +{ + size_t i = nr_entries - 1; + + while (version < table[i].maxage) + i--; + if (version > table[i].maxage) + return NULL; + return &table[i]; +} + struct unicode_map *utf8_load(unsigned int version) { struct unicode_map *um; - if (!utf8version_is_supported(version)) - return ERR_PTR(-EINVAL); - um = kzalloc(sizeof(struct unicode_map), GFP_KERNEL); if (!um) return ERR_PTR(-ENOMEM); um->version = version; - um->ntab[UTF8_NFDI] = utf8nfdi(version); - if (!um->ntab[UTF8_NFDI]) + + um->tables = symbol_request(utf8_data_table); + if (!um->tables) goto out_free_um; - um->ntab[UTF8_NFDICF] = utf8nfdicf(version); + + if (!utf8version_is_supported(um, version)) + goto out_symbol_put; + um->ntab[UTF8_NFDI] = find_table_version(um->tables->utf8nfdidata, + um->tables->utf8nfdidata_size, um->version); + if (!um->ntab[UTF8_NFDI]) + goto out_symbol_put; + um->ntab[UTF8_NFDICF] = find_table_version(um->tables->utf8nfdicfdata, + um->tables->utf8nfdicfdata_size, um->version); if (!um->ntab[UTF8_NFDICF]) - goto out_free_um; + goto out_symbol_put; return um; +out_symbol_put: + symbol_put(um->tables); out_free_um: kfree(um); return ERR_PTR(-EINVAL); @@ -187,7 +207,10 @@ EXPORT_SYMBOL(utf8_load); void utf8_unload(struct unicode_map *um) { - kfree(um); + if (um) { + symbol_put(utf8_data_table); + kfree(um); + } } EXPORT_SYMBOL(utf8_unload); diff --git a/fs/unicode/utf8-norm.c b/fs/unicode/utf8-norm.c index 7c1f28ab31a8..829c7e2ad764 100644 --- a/fs/unicode/utf8-norm.c +++ b/fs/unicode/utf8-norm.c @@ -6,21 +6,12 @@ #include "utf8n.h" -struct utf8data { - unsigned int maxage; - unsigned int offset; -}; - -#define __INCLUDED_FROM_UTF8NORM_C__ -#include "utf8data.h" -#undef __INCLUDED_FROM_UTF8NORM_C__ - -int utf8version_is_supported(unsigned int version) +int utf8version_is_supported(const struct unicode_map *um, unsigned int version) { - int i = ARRAY_SIZE(utf8agetab) - 1; + int i = um->tables->utf8agetab_size - 1; - while (i >= 0 && utf8agetab[i] != 0) { - if (version == utf8agetab[i]) + while (i >= 0 && um->tables->utf8agetab[i] != 0) { + if (version == um->tables->utf8agetab[i]) return 1; i--; } @@ -161,7 +152,7 @@ typedef const unsigned char utf8trie_t; * underlying datatype: unsigned char. * * leaf[0]: The unicode version, stored as a generation number that is - * an index into utf8agetab[]. With this we can filter code + * an index into ->utf8agetab[]. With this we can filter code * points based on the unicode version in which they were * defined. The CCC of a non-defined code point is 0. * leaf[1]: Canonical Combining Class. During normalization, we need @@ -313,7 +304,7 @@ static utf8leaf_t *utf8nlookup(const struct unicode_map *um, enum utf8_normalization n, unsigned char *hangul, const char *s, size_t len) { - utf8trie_t *trie = utf8data + um->ntab[n]->offset; + utf8trie_t *trie = um->tables->utf8data + um->ntab[n]->offset; int offlen; int offset; int mask; @@ -404,7 +395,8 @@ ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, leaf = utf8nlookup(um, n, hangul, s, len); if (!leaf) return -1; - if (utf8agetab[LEAF_GEN(leaf)] > um->ntab[n]->maxage) + if (um->tables->utf8agetab[LEAF_GEN(leaf)] > + um->ntab[n]->maxage) ret += utf8clen(s); else if (LEAF_CCC(leaf) == DECOMPOSE) ret += strlen(LEAF_STR(leaf)); @@ -520,7 +512,7 @@ int utf8byte(struct utf8cursor *u8c) ccc = LEAF_CCC(leaf); /* Characters that are too new have CCC 0. */ - if (utf8agetab[LEAF_GEN(leaf)] > + if (u8c->um->tables->utf8agetab[LEAF_GEN(leaf)] > u8c->um->ntab[u8c->n]->maxage) { ccc = STOPPER; } else if (ccc == DECOMPOSE) { @@ -597,25 +589,3 @@ ccc_mismatch: } } EXPORT_SYMBOL(utf8byte); - -const struct utf8data *utf8nfdi(unsigned int maxage) -{ - int i = ARRAY_SIZE(utf8nfdidata) - 1; - - while (maxage < utf8nfdidata[i].maxage) - i--; - if (maxage > utf8nfdidata[i].maxage) - return NULL; - return &utf8nfdidata[i]; -} - -const struct utf8data *utf8nfdicf(unsigned int maxage) -{ - int i = ARRAY_SIZE(utf8nfdicfdata) - 1; - - while (maxage < utf8nfdicfdata[i].maxage) - i--; - if (maxage > utf8nfdicfdata[i].maxage) - return NULL; - return &utf8nfdicfdata[i]; -} diff --git a/fs/unicode/utf8-selftest.c b/fs/unicode/utf8-selftest.c index cfa3832b75f4..eb2bbdd688d7 100644 --- a/fs/unicode/utf8-selftest.c +++ b/fs/unicode/utf8-selftest.c @@ -255,21 +255,21 @@ static void check_utf8_comparisons(struct unicode_map *table) } } -static void check_supported_versions(void) +static void check_supported_versions(struct unicode_map *um) { /* Unicode 7.0.0 should be supported. */ - test(utf8version_is_supported(UNICODE_AGE(7, 0, 0))); + test(utf8version_is_supported(um, UNICODE_AGE(7, 0, 0))); /* Unicode 9.0.0 should be supported. */ - test(utf8version_is_supported(UNICODE_AGE(9, 0, 0))); + test(utf8version_is_supported(um, UNICODE_AGE(9, 0, 0))); /* Unicode 1x.0.0 (the latest version) should be supported. */ - test(utf8version_is_supported(UTF8_LATEST)); + test(utf8version_is_supported(um, UTF8_LATEST)); /* Next versions don't exist. */ - test(!utf8version_is_supported(UNICODE_AGE(13, 0, 0))); - test(!utf8version_is_supported(UNICODE_AGE(0, 0, 0))); - test(!utf8version_is_supported(UNICODE_AGE(-1, -1, -1))); + test(!utf8version_is_supported(um, UNICODE_AGE(13, 0, 0))); + test(!utf8version_is_supported(um, UNICODE_AGE(0, 0, 0))); + test(!utf8version_is_supported(um, UNICODE_AGE(-1, -1, -1))); } static int __init init_test_ucd(void) @@ -285,7 +285,7 @@ static int __init init_test_ucd(void) return PTR_ERR(um); } - check_supported_versions(); + check_supported_versions(um); check_utf8_nfdi(um); check_utf8_nfdicf(um); check_utf8_comparisons(um); diff --git a/fs/unicode/utf8data.c_shipped b/fs/unicode/utf8data.c_shipped new file mode 100644 index 000000000000..d9b62901aa96 --- /dev/null +++ b/fs/unicode/utf8data.c_shipped @@ -0,0 +1,4123 @@ +/* This file is generated code, do not edit. */ + +#include +#include +#include "utf8n.h" + +static const unsigned int utf8agetab[] = { + 0, + 0x10100, + 0x20000, + 0x20100, + 0x30000, + 0x30100, + 0x30200, + 0x40000, + 0x40100, + 0x50000, + 0x50100, + 0x50200, + 0x60000, + 0x60100, + 0x60200, + 0x60300, + 0x70000, + 0x80000, + 0x90000, + 0xa0000, + 0xb0000, + 0xc0000, + 0xc0100 +}; + +static const struct utf8data utf8nfdicfdata[] = { + { 0, 0 }, + { 0x10100, 0 }, + { 0x20000, 0 }, + { 0x20100, 0 }, + { 0x30000, 0 }, + { 0x30100, 0 }, + { 0x30200, 1792 }, + { 0x40000, 3200 }, + { 0x40100, 3200 }, + { 0x50000, 3200 }, + { 0x50100, 3200 }, + { 0x50200, 3200 }, + { 0x60000, 3200 }, + { 0x60100, 3200 }, + { 0x60200, 3200 }, + { 0x60300, 3200 }, + { 0x70000, 3200 }, + { 0x80000, 3200 }, + { 0x90000, 3200 }, + { 0xa0000, 3200 }, + { 0xb0000, 3200 }, + { 0xc0000, 3200 }, + { 0xc0100, 3200 } +}; + +static const struct utf8data utf8nfdidata[] = { + { 0, 896 }, + { 0x10100, 896 }, + { 0x20000, 896 }, + { 0x20100, 896 }, + { 0x30000, 896 }, + { 0x30100, 896 }, + { 0x30200, 2496 }, + { 0x40000, 20736 }, + { 0x40100, 20736 }, + { 0x50000, 20736 }, + { 0x50100, 20736 }, + { 0x50200, 20736 }, + { 0x60000, 20736 }, + { 0x60100, 20736 }, + { 0x60200, 20736 }, + { 0x60300, 20736 }, + { 0x70000, 20736 }, + { 0x80000, 20736 }, + { 0x90000, 20736 }, + { 0xa0000, 20736 }, + { 0xb0000, 20736 }, + { 0xc0000, 20736 }, + { 0xc0100, 20736 } +}; + +static const unsigned char utf8data[64256] = { + /* nfdicf_30100 */ + 0xd7,0x07,0x66,0x84,0x0c,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x1a,0xe3,0x63,0x15, + 0xe2,0x4c,0x0e,0xc1,0xe0,0x4e,0x0d,0xcf,0x86,0x65,0x2d,0x0d,0x01,0x00,0xd4,0xb8, + 0xd3,0x27,0xe2,0x89,0xa3,0xe1,0xce,0x35,0xe0,0x2c,0x22,0xcf,0x86,0xc5,0xe4,0x15, + 0x6d,0xe3,0x60,0x68,0xe2,0xf6,0x65,0xe1,0x29,0x65,0xe0,0xee,0x64,0xcf,0x86,0xe5, + 0xb3,0x64,0x64,0x96,0x64,0x0b,0x00,0xd2,0x0e,0xe1,0xb5,0x3c,0xe0,0xba,0xa3,0xcf, + 0x86,0xcf,0x06,0x01,0x00,0xd1,0x0c,0xe0,0x1e,0xa9,0xcf,0x86,0xcf,0x06,0x02,0xff, + 0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01, + 0x00,0xe4,0xe1,0x45,0xe3,0x3b,0x45,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x87,0xad, + 0xd0,0x21,0xcf,0x86,0xe5,0x81,0xaa,0xe4,0x00,0xaa,0xe3,0xbf,0xa9,0xe2,0x9e,0xa9, + 0xe1,0x8d,0xa9,0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4, + 0x00,0xcf,0x86,0xe5,0x63,0xac,0xd4,0x19,0xe3,0xa2,0xab,0xe2,0x81,0xab,0xe1,0x70, + 0xab,0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3, + 0x09,0xac,0xe2,0xe8,0xab,0xe1,0xd7,0xab,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00, + 0x01,0xff,0xe9,0x9b,0xbb,0x00,0x83,0xe2,0x19,0xfa,0xe1,0xf2,0xf6,0xe0,0x6f,0xf5, + 0xcf,0x86,0xd5,0x31,0xc4,0xe3,0x54,0x4e,0xe2,0xf5,0x4c,0xe1,0xa4,0xcc,0xe0,0x9c, + 0x4b,0xcf,0x86,0xe5,0x8e,0x49,0xe4,0xaf,0x46,0xe3,0x11,0xbd,0xe2,0x68,0xbc,0xe1, + 0x43,0xbc,0xe0,0x1c,0xbc,0xcf,0x86,0xe5,0xe9,0xbb,0x94,0x07,0x63,0xd4,0xbb,0x07, + 0x00,0x07,0x00,0xe4,0xdb,0xf4,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b, + 0xe1,0xea,0xe1,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xd9,0xe2,0xcf,0x86, + 0xe5,0x9e,0xe2,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xd9,0xe2,0xcf,0x06, + 0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x74,0xf4,0xe3,0x5d,0xf3, + 0xd2,0xa0,0xe1,0x13,0xe7,0xd0,0x21,0xcf,0x86,0xe5,0x14,0xe4,0xe4,0x90,0xe3,0xe3, + 0x4e,0xe3,0xe2,0x2d,0xe3,0xe1,0x1b,0xe3,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00, + 0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x70,0xe5,0xe3,0x2f,0xe5, + 0xe2,0x0e,0xe5,0xe1,0xfd,0xe4,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff, + 0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xf7,0xe5,0xe1,0xe6,0xe5,0x10,0x09, + 0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x17, + 0xe6,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac, + 0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x5d,0xe6,0xd2,0x14,0xe1,0x2c,0xe6, + 0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1, + 0x38,0xe6,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00, + 0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x8d,0xeb,0xd4,0x19,0xe3,0xc6,0xea,0xe2,0xa4, + 0xea,0xe1,0x93,0xea,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5, + 0xb7,0x00,0xd3,0x18,0xe2,0x10,0xeb,0xe1,0xff,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3, + 0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0x28,0xeb,0x10, + 0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08, + 0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0x2a, + 0xed,0xd4,0x1a,0xe3,0x62,0xec,0xe2,0x48,0xec,0xe1,0x35,0xec,0x10,0x08,0x05,0xff, + 0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0xaa,0xec, + 0xe1,0x98,0xec,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3, + 0x00,0xd2,0x13,0xe1,0xc6,0xec,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff, + 0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05, + 0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05, + 0xff,0xe7,0xaa,0xae,0x00,0xe0,0xdc,0xef,0xcf,0x86,0xd5,0x1d,0xe4,0x51,0xee,0xe3, + 0x0d,0xee,0xe2,0xeb,0xed,0xe1,0xda,0xed,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f, + 0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xf8,0xee,0xe2,0xd4,0xee,0xe1, + 0xc3,0xee,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00, + 0xd3,0x18,0xe2,0x43,0xef,0xe1,0x32,0xef,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1, + 0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x5b,0xef,0x10,0x08,0x05, + 0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8, + 0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_30100 */ + 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0xc2,0x59,0xe3,0xfb,0x54,0xe2,0x74,0x4f, + 0xc1,0xe0,0xa0,0x4d,0xcf,0x86,0x65,0x84,0x4d,0x01,0x00,0xd4,0xb8,0xd3,0x27,0xe2, + 0x0c,0xa0,0xe1,0xdf,0x8d,0xe0,0x39,0x71,0xcf,0x86,0xc5,0xe4,0x98,0x69,0xe3,0xe3, + 0x64,0xe2,0x79,0x62,0xe1,0xac,0x61,0xe0,0x71,0x61,0xcf,0x86,0xe5,0x36,0x61,0x64, + 0x19,0x61,0x0b,0x00,0xd2,0x0e,0xe1,0xc2,0xa0,0xe0,0x3d,0xa0,0xcf,0x86,0xcf,0x06, + 0x01,0x00,0xd1,0x0c,0xe0,0xa1,0xa5,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd0,0x08, + 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xe4,0x9e, + 0xb6,0xe3,0x18,0xae,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x0a,0xaa,0xd0,0x21,0xcf, + 0x86,0xe5,0x04,0xa7,0xe4,0x83,0xa6,0xe3,0x42,0xa6,0xe2,0x21,0xa6,0xe1,0x10,0xa6, + 0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0xcf,0x86, + 0xe5,0xe6,0xa8,0xd4,0x19,0xe3,0x25,0xa8,0xe2,0x04,0xa8,0xe1,0xf3,0xa7,0x10,0x08, + 0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3,0x8c,0xa8,0xe2, + 0x6b,0xa8,0xe1,0x5a,0xa8,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9, + 0x9b,0xbb,0x00,0x83,0xe2,0x9c,0xf6,0xe1,0x75,0xf3,0xe0,0xf2,0xf1,0xcf,0x86,0xd5, + 0x31,0xc4,0xe3,0x6d,0xcc,0xe2,0x46,0xca,0xe1,0x27,0xc9,0xe0,0xb7,0xbf,0xcf,0x86, + 0xe5,0xaa,0xbb,0xe4,0xa3,0xba,0xe3,0x94,0xb9,0xe2,0xeb,0xb8,0xe1,0xc6,0xb8,0xe0, + 0x9f,0xb8,0xcf,0x86,0xe5,0x6c,0xb8,0x94,0x07,0x63,0x57,0xb8,0x07,0x00,0x07,0x00, + 0xe4,0x5e,0xf1,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1,0x6d,0xde, + 0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x5c,0xdf,0xcf,0x86,0xe5,0x21,0xdf, + 0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x5c,0xdf,0xcf,0x06,0x13,0x00,0xcf, + 0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0xf7,0xf0,0xe3,0xe0,0xef,0xd2,0xa0,0xe1, + 0x96,0xe3,0xd0,0x21,0xcf,0x86,0xe5,0x97,0xe0,0xe4,0x13,0xe0,0xe3,0xd1,0xdf,0xe2, + 0xb0,0xdf,0xe1,0x9e,0xdf,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4, + 0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xf3,0xe1,0xe3,0xb2,0xe1,0xe2,0x91,0xe1, + 0xe1,0x80,0xe1,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6, + 0x00,0xd4,0x34,0xd3,0x18,0xe2,0x7a,0xe2,0xe1,0x69,0xe2,0x10,0x09,0x05,0xff,0xf0, + 0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x9a,0xe2,0x91,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, + 0xff,0xe5,0xac,0xbe,0x00,0xe3,0xe0,0xe2,0xd2,0x14,0xe1,0xaf,0xe2,0x10,0x08,0x05, + 0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0xbb,0xe2,0x10, + 0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1,0xd5,0xd0, + 0x6a,0xcf,0x86,0xe5,0x10,0xe8,0xd4,0x19,0xe3,0x49,0xe7,0xe2,0x27,0xe7,0xe1,0x16, + 0xe7,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7,0x00,0xd3, + 0x18,0xe2,0x93,0xe7,0xe1,0x82,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00, + 0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xab,0xe7,0x10,0x08,0x05,0xff, + 0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, + 0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7, + 0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xad,0xe9,0xd4,0x1a, + 0xe3,0xe5,0xe8,0xe2,0xcb,0xe8,0xe1,0xb8,0xe8,0x10,0x08,0x05,0xff,0xe7,0x9b,0xb4, + 0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x2d,0xe9,0xe1,0x1b,0xe9, + 0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0xd2,0x13, + 0xe1,0x49,0xe9,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80, + 0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff,0xf0,0xa5, + 0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff,0xe7,0xaa, + 0xae,0x00,0xe0,0x5f,0xec,0xcf,0x86,0xd5,0x1d,0xe4,0xd4,0xea,0xe3,0x90,0xea,0xe2, + 0x6e,0xea,0xe1,0x5d,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff, + 0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x7b,0xeb,0xe2,0x57,0xeb,0xe1,0x46,0xeb,0x10, + 0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3,0x18,0xe2, + 0xc6,0xeb,0xe1,0xb5,0xeb,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00,0x05,0xff, + 0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xde,0xeb,0x10,0x08,0x05,0xff,0xe8,0x9a, + 0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c, + 0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00, + 0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdicf_30200 */ + 0xd7,0x07,0x66,0x84,0x05,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x13,0xe3,0x63,0x0e, + 0xe2,0x4c,0x07,0xc1,0xe0,0x4e,0x06,0xcf,0x86,0x65,0x2d,0x06,0x01,0x00,0xd4,0x2a, + 0xe3,0xd0,0x35,0xe2,0x88,0x9c,0xe1,0xcd,0x2e,0xe0,0x2b,0x1b,0xcf,0x86,0xc5,0xe4, + 0x14,0x66,0xe3,0x5f,0x61,0xe2,0xf5,0x5e,0xe1,0x28,0x5e,0xe0,0xed,0x5d,0xcf,0x86, + 0xe5,0xb2,0x5d,0x64,0x95,0x5d,0x0b,0x00,0x83,0xe2,0xa7,0xf3,0xe1,0x80,0xf0,0xe0, + 0xfd,0xee,0xcf,0x86,0xd5,0x31,0xc4,0xe3,0xe2,0x47,0xe2,0x83,0x46,0xe1,0x32,0xc6, + 0xe0,0x2a,0x45,0xcf,0x86,0xe5,0x1c,0x43,0xe4,0x3d,0x40,0xe3,0x9f,0xb6,0xe2,0xf6, + 0xb5,0xe1,0xd1,0xb5,0xe0,0xaa,0xb5,0xcf,0x86,0xe5,0x77,0xb5,0x94,0x07,0x63,0x62, + 0xb5,0x07,0x00,0x07,0x00,0xe4,0x69,0xee,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, + 0xd2,0x0b,0xe1,0x78,0xdb,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x67,0xdc, + 0xcf,0x86,0xe5,0x2c,0xdc,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x67,0xdc, + 0xcf,0x06,0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x02,0xee,0xe3, + 0xeb,0xec,0xd2,0xa0,0xe1,0xa1,0xe0,0xd0,0x21,0xcf,0x86,0xe5,0xa2,0xdd,0xe4,0x1e, + 0xdd,0xe3,0xdc,0xdc,0xe2,0xbb,0xdc,0xe1,0xa9,0xdc,0x10,0x08,0x05,0xff,0xe4,0xb8, + 0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xfe,0xde,0xe3, + 0xbd,0xde,0xe2,0x9c,0xde,0xe1,0x8b,0xde,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00, + 0x05,0xff,0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0x85,0xdf,0xe1,0x74,0xdf, + 0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00, + 0xe2,0xa5,0xdf,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff, + 0xe5,0xac,0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0xeb,0xdf,0xd2,0x14,0xe1, + 0xba,0xdf,0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98, + 0x00,0xe1,0xc6,0xdf,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0, + 0xa2,0x00,0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x1b,0xe5,0xd4,0x19,0xe3,0x54,0xe4, + 0xe2,0x32,0xe4,0xe1,0x21,0xe4,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, + 0xe6,0xb5,0xb7,0x00,0xd3,0x18,0xe2,0x9e,0xe4,0xe1,0x8d,0xe4,0x10,0x09,0x05,0xff, + 0xf0,0xa3,0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xb6, + 0xe4,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00, + 0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86, + 0xe5,0xb8,0xe6,0xd4,0x1a,0xe3,0xf0,0xe5,0xe2,0xd6,0xe5,0xe1,0xc3,0xe5,0x10,0x08, + 0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2, + 0x38,0xe6,0xe1,0x26,0xe6,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4, + 0x83,0xa3,0x00,0xd2,0x13,0xe1,0x54,0xe6,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00, + 0x05,0xff,0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc, + 0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7, + 0x00,0x05,0xff,0xe7,0xaa,0xae,0x00,0xe0,0x6a,0xe9,0xcf,0x86,0xd5,0x1d,0xe4,0xdf, + 0xe7,0xe3,0x9b,0xe7,0xe2,0x79,0xe7,0xe1,0x68,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3, + 0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x86,0xe8,0xe2,0x62, + 0xe8,0xe1,0x51,0xe8,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, + 0x8a,0x00,0xd3,0x18,0xe2,0xd1,0xe8,0xe1,0xc0,0xe8,0x10,0x09,0x05,0xff,0xf0,0xa6, + 0xbe,0xb1,0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xe9,0xe8,0x10, + 0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05, + 0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_30200 */ + 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x82,0x53,0xe3,0xbb,0x4e,0xe2,0x34,0x49, + 0xc1,0xe0,0x60,0x47,0xcf,0x86,0x65,0x44,0x47,0x01,0x00,0xd4,0x2a,0xe3,0x1c,0x9a, + 0xe2,0xcb,0x99,0xe1,0x9e,0x87,0xe0,0xf8,0x6a,0xcf,0x86,0xc5,0xe4,0x57,0x63,0xe3, + 0xa2,0x5e,0xe2,0x38,0x5c,0xe1,0x6b,0x5b,0xe0,0x30,0x5b,0xcf,0x86,0xe5,0xf5,0x5a, + 0x64,0xd8,0x5a,0x0b,0x00,0x83,0xe2,0xea,0xf0,0xe1,0xc3,0xed,0xe0,0x40,0xec,0xcf, + 0x86,0xd5,0x31,0xc4,0xe3,0xbb,0xc6,0xe2,0x94,0xc4,0xe1,0x75,0xc3,0xe0,0x05,0xba, + 0xcf,0x86,0xe5,0xf8,0xb5,0xe4,0xf1,0xb4,0xe3,0xe2,0xb3,0xe2,0x39,0xb3,0xe1,0x14, + 0xb3,0xe0,0xed,0xb2,0xcf,0x86,0xe5,0xba,0xb2,0x94,0x07,0x63,0xa5,0xb2,0x07,0x00, + 0x07,0x00,0xe4,0xac,0xeb,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1, + 0xbb,0xd8,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xaa,0xd9,0xcf,0x86,0xe5, + 0x6f,0xd9,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xaa,0xd9,0xcf,0x06,0x13, + 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x45,0xeb,0xe3,0x2e,0xea,0xd2, + 0xa0,0xe1,0xe4,0xdd,0xd0,0x21,0xcf,0x86,0xe5,0xe5,0xda,0xe4,0x61,0xda,0xe3,0x1f, + 0xda,0xe2,0xfe,0xd9,0xe1,0xec,0xd9,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05, + 0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x41,0xdc,0xe3,0x00,0xdc,0xe2, + 0xdf,0xdb,0xe1,0xce,0xdb,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5, + 0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xc8,0xdc,0xe1,0xb7,0xdc,0x10,0x09,0x05, + 0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0xe8,0xdc, + 0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88, + 0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x2e,0xdd,0xd2,0x14,0xe1,0xfd,0xdc,0x10, + 0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0x09, + 0xdd,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1, + 0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x5e,0xe2,0xd4,0x19,0xe3,0x97,0xe1,0xe2,0x75,0xe1, + 0xe1,0x64,0xe1,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7, + 0x00,0xd3,0x18,0xe2,0xe1,0xe1,0xe1,0xd0,0xe1,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd, + 0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xf9,0xe1,0x10,0x08, + 0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05, + 0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xfb,0xe3, + 0xd4,0x1a,0xe3,0x33,0xe3,0xe2,0x19,0xe3,0xe1,0x06,0xe3,0x10,0x08,0x05,0xff,0xe7, + 0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x7b,0xe3,0xe1, + 0x69,0xe3,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00, + 0xd2,0x13,0xe1,0x97,0xe3,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7, + 0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff, + 0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff, + 0xe7,0xaa,0xae,0x00,0xe0,0xad,0xe6,0xcf,0x86,0xd5,0x1d,0xe4,0x22,0xe5,0xe3,0xde, + 0xe4,0xe2,0xbc,0xe4,0xe1,0xab,0xe4,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00, + 0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xc9,0xe5,0xe2,0xa5,0xe5,0xe1,0x94, + 0xe5,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3, + 0x18,0xe2,0x14,0xe6,0xe1,0x03,0xe6,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, + 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x2c,0xe6,0x10,0x08,0x05,0xff, + 0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e, + 0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdicf_c0100 */ + 0xd7,0xb0,0x56,0x04,0x01,0x00,0x95,0xa8,0xd4,0x5e,0xd3,0x2e,0xd2,0x16,0xd1,0x0a, + 0x10,0x04,0x01,0x00,0x01,0xff,0x61,0x00,0x10,0x06,0x01,0xff,0x62,0x00,0x01,0xff, + 0x63,0x00,0xd1,0x0c,0x10,0x06,0x01,0xff,0x64,0x00,0x01,0xff,0x65,0x00,0x10,0x06, + 0x01,0xff,0x66,0x00,0x01,0xff,0x67,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, + 0x68,0x00,0x01,0xff,0x69,0x00,0x10,0x06,0x01,0xff,0x6a,0x00,0x01,0xff,0x6b,0x00, + 0xd1,0x0c,0x10,0x06,0x01,0xff,0x6c,0x00,0x01,0xff,0x6d,0x00,0x10,0x06,0x01,0xff, + 0x6e,0x00,0x01,0xff,0x6f,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, + 0x70,0x00,0x01,0xff,0x71,0x00,0x10,0x06,0x01,0xff,0x72,0x00,0x01,0xff,0x73,0x00, + 0xd1,0x0c,0x10,0x06,0x01,0xff,0x74,0x00,0x01,0xff,0x75,0x00,0x10,0x06,0x01,0xff, + 0x76,0x00,0x01,0xff,0x77,0x00,0x92,0x16,0xd1,0x0c,0x10,0x06,0x01,0xff,0x78,0x00, + 0x01,0xff,0x79,0x00,0x10,0x06,0x01,0xff,0x7a,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0xc6,0xe5,0xf9,0x14,0xe4,0x6f,0x0d,0xe3,0x39,0x08,0xe2,0x22,0x01,0xc1,0xd0,0x24, + 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x07,0x63,0xd8,0x43,0x01,0x00,0x93,0x13,0x52, + 0x04,0x01,0x00,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xce,0xbc,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xe5,0xb3,0x44,0xd4,0x7f,0xd3,0x3f,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x07,0x01,0xff,0xc3, + 0xa6,0x00,0x01,0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0x80,0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x82,0x00,0x01,0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc, + 0x80,0x00,0x01,0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00, + 0x01,0xff,0x69,0xcc,0x88,0x00,0xd3,0x3b,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff, + 0xc3,0xb0,0x00,0x01,0xff,0x6e,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80, + 0x00,0x01,0xff,0x6f,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82, + 0x00,0x01,0xff,0x6f,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01, + 0x00,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff,0xc3,0xb8,0x00,0x01,0xff,0x75,0xcc, + 0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x82,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88,0x00,0x01,0xff,0x79,0xcc,0x81,0x00, + 0x10,0x07,0x01,0xff,0xc3,0xbe,0x00,0x01,0xff,0x73,0x73,0x00,0xe1,0xd4,0x03,0xe0, + 0xeb,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x61,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84,0x00,0x10,0x08,0x01,0xff, + 0x61,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x61,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x63,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x8c,0x00, + 0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x3b,0xd2,0x1b,0xd1,0x0b,0x10,0x07,0x01,0xff, + 0xc4,0x91,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x84,0x00,0x01,0xff,0x65, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x86,0x00,0x01,0xff,0x65, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0xa8,0x00,0x01,0xff,0x65, + 0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82, + 0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4, + 0x7b,0xd3,0x3b,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x87,0x00,0x01, + 0xff,0x67,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0xa7,0x00,0x01,0xff,0x67, + 0xcc,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x68,0xcc,0x82,0x00,0x01,0xff,0x68, + 0xcc,0x82,0x00,0x10,0x07,0x01,0xff,0xc4,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x69,0xcc,0x83,0x00,0x01,0xff,0x69,0xcc,0x83,0x00,0x10,0x08, + 0x01,0xff,0x69,0xcc,0x84,0x00,0x01,0xff,0x69,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x69,0xcc,0x86,0x00,0x01,0xff,0x69,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, + 0x69,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8,0x00,0xd3,0x37,0xd2,0x17,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0x69,0xcc,0x87,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc4,0xb3, + 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6a,0xcc,0x82,0x00,0x01,0xff,0x6a, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, + 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6c,0xcc,0x81,0x00,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x6c,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x6c,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x6c,0xcc,0x8c,0x00,0x01,0xff,0xc5,0x80,0x00,0xcf,0x86,0xd5,0xed,0xd4,0x72, + 0xd3,0x37,0xd2,0x17,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc5,0x82,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0x6e,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x81,0x00,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa7, + 0x00,0x01,0xff,0x6e,0xcc,0x8c,0x00,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x8c,0x00,0x01,0xff,0xca,0xbc,0x6e,0x00,0x10,0x07,0x01,0xff,0xc5,0x8b,0x00, + 0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc, + 0x84,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00, + 0xd3,0x3b,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01,0xff, + 0x6f,0xcc,0x8b,0x00,0x10,0x07,0x01,0xff,0xc5,0x93,0x00,0x01,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x72,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x72,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x72,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x73,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x73,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x73, + 0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x7b,0xd3,0x3b,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x73,0xcc,0x8c,0x00,0x01,0xff,0x73,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x74,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x74,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c,0x00,0x10,0x07,0x01, + 0xff,0xc5,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, + 0x83,0x00,0x01,0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x84,0x00, + 0x01,0xff,0x75,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x86,0x00, + 0x01,0xff,0x75,0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x8a,0x00,0x01,0xff, + 0x75,0xcc,0x8a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, + 0x8b,0x00,0x01,0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa8,0x00, + 0x01,0xff,0x75,0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x82,0x00, + 0x01,0xff,0x77,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x82,0x00,0x01,0xff, + 0x79,0xcc,0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x79,0xcc,0x88,0x00, + 0x01,0xff,0x7a,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff, + 0x7a,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff, + 0x7a,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0xff,0x73,0x00, + 0xe0,0x65,0x01,0xcf,0x86,0xd5,0xb4,0xd4,0x5a,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10, + 0x04,0x01,0x00,0x01,0xff,0xc9,0x93,0x00,0x10,0x07,0x01,0xff,0xc6,0x83,0x00,0x01, + 0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xc9,0x94,0x00,0x01,0xff,0xc6,0x88,0x00,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x01,0x00, + 0x01,0xff,0xc9,0x96,0x00,0x10,0x07,0x01,0xff,0xc9,0x97,0x00,0x01,0xff,0xc6,0x8c, + 0x00,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xc7,0x9d,0x00,0x01,0xff,0xc9,0x99, + 0x00,0xd3,0x32,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0x9b,0x00,0x01,0xff, + 0xc6,0x92,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xa0,0x00,0xd1,0x0b,0x10,0x07, + 0x01,0xff,0xc9,0xa3,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc9,0xa9,0x00,0x01,0xff, + 0xc9,0xa8,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0x99,0x00,0x01,0x00, + 0x01,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0xaf,0x00,0x01,0xff,0xc9,0xb2,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xb5,0x00,0xd4,0x5d,0xd3,0x34,0xd2,0x1b,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x10, + 0x07,0x01,0xff,0xc6,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0xa5, + 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x80,0x00,0x01,0xff,0xc6,0xa8,0x00,0xd2, + 0x0f,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x83,0x00,0x01,0x00,0xd1,0x0b, + 0x10,0x07,0x01,0xff,0xc6,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x88,0x00, + 0x01,0xff,0x75,0xcc,0x9b,0x00,0xd3,0x33,0xd2,0x1d,0xd1,0x0f,0x10,0x08,0x01,0xff, + 0x75,0xcc,0x9b,0x00,0x01,0xff,0xca,0x8a,0x00,0x10,0x07,0x01,0xff,0xca,0x8b,0x00, + 0x01,0xff,0xc6,0xb4,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc6,0xb6,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x92,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01, + 0xff,0xc6,0xb9,0x00,0x01,0x00,0x01,0x00,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0xbd, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x44,0xd3,0x16,0x52,0x04,0x01, + 0x00,0x51,0x07,0x01,0xff,0xc7,0x86,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc7,0x89, + 0x00,0xd2,0x12,0x91,0x0b,0x10,0x07,0x01,0xff,0xc7,0x89,0x00,0x01,0x00,0x01,0xff, + 0xc7,0x8c,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x61,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x69,0xcc,0x8c,0x00,0xd3,0x46,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x6f,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x8c,0x00,0xd1, + 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84, + 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x88, + 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x87,0xd3,0x41,0xd2, + 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, + 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x84,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x84,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc7,0xa5,0x00,0x01,0x00, + 0x10,0x08,0x01,0xff,0x67,0xcc,0x8c,0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00, + 0x10,0x08,0x01,0xff,0x6f,0xcc,0xa8,0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14, + 0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc, + 0x84,0x00,0x10,0x09,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc, + 0x8c,0x00,0xd3,0x38,0xd2,0x1a,0xd1,0x0f,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00, + 0x01,0xff,0xc7,0xb3,0x00,0x10,0x07,0x01,0xff,0xc7,0xb3,0x00,0x01,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x67,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x10,0x07, + 0x04,0xff,0xc6,0x95,0x00,0x04,0xff,0xc6,0xbf,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08, + 0x04,0xff,0x6e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, + 0x61,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a,0xcc,0x81,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00, + 0x10,0x09,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00, + 0xe2,0x31,0x02,0xe1,0xc3,0x44,0xe0,0xc8,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0x8f,0x00,0x01,0xff,0x61, + 0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x91,0x00,0x01,0xff,0x61,0xcc,0x91, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x8f,0x00,0x01,0xff,0x65,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x91,0x00,0x01,0xff,0x65,0xcc,0x91,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8f,0x00,0x01,0xff,0x69,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x91,0x00,0x01,0xff,0x69,0xcc,0x91,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x10, + 0x08,0x01,0xff,0x6f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc,0x91,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x8f,0x00,0x01,0xff,0x72,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0x91,0x00,0x01,0xff,0x72,0xcc,0x91,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x8f,0x00,0x01,0xff,0x75,0xcc,0x8f,0x00,0x10, + 0x08,0x01,0xff,0x75,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc,0x91,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x04,0xff,0x73,0xcc,0xa6,0x00,0x04,0xff,0x73,0xcc,0xa6,0x00,0x10, + 0x08,0x04,0xff,0x74,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc,0xa6,0x00,0xd1,0x0b,0x10, + 0x07,0x04,0xff,0xc8,0x9d,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x68,0xcc,0x8c,0x00, + 0x04,0xff,0x68,0xcc,0x8c,0x00,0xd4,0x79,0xd3,0x31,0xd2,0x16,0xd1,0x0b,0x10,0x07, + 0x06,0xff,0xc6,0x9e,0x00,0x07,0x00,0x10,0x07,0x04,0xff,0xc8,0xa3,0x00,0x04,0x00, + 0xd1,0x0b,0x10,0x07,0x04,0xff,0xc8,0xa5,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x61, + 0xcc,0x87,0x00,0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04, + 0xff,0x65,0xcc,0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x6f, + 0xcc,0x88,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10, + 0x0a,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84, + 0x00,0x10,0x08,0x04,0xff,0x6f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0xd3, + 0x27,0xe2,0x21,0x43,0xd1,0x14,0x10,0x0a,0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00, + 0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x79,0xcc,0x84,0x00, + 0x04,0xff,0x79,0xcc,0x84,0x00,0xd2,0x13,0x51,0x04,0x08,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0xa5,0x00,0x08,0xff,0xc8,0xbc,0x00,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08, + 0xff,0xc6,0x9a,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0xa6,0x00,0x08,0x00,0xcf,0x86, + 0x95,0x5f,0x94,0x5b,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08,0xff, + 0xc9,0x82,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xc6,0x80,0x00,0xd1,0x0e,0x10,0x07, + 0x09,0xff,0xca,0x89,0x00,0x09,0xff,0xca,0x8c,0x00,0x10,0x07,0x09,0xff,0xc9,0x87, + 0x00,0x09,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9,0x89,0x00,0x09,0x00, + 0x10,0x07,0x09,0xff,0xc9,0x8b,0x00,0x09,0x00,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9, + 0x8d,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xc9,0x8f,0x00,0x09,0x00,0x01,0x00,0x01, + 0x00,0xd1,0x8b,0xd0,0x0c,0xcf,0x86,0xe5,0x10,0x43,0x64,0xef,0x42,0x01,0xe6,0xcf, + 0x86,0xd5,0x2a,0xe4,0x99,0x43,0xe3,0x7f,0x43,0xd2,0x11,0xe1,0x5e,0x43,0x10,0x07, + 0x01,0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0xe1,0x65,0x43,0x10,0x09,0x01, + 0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0x00,0xd4,0x0f,0x93,0x0b,0x92, + 0x07,0x61,0xab,0x43,0x01,0xea,0x06,0xe6,0x06,0xe6,0xd3,0x2c,0xd2,0x16,0xd1,0x0b, + 0x10,0x07,0x0a,0xff,0xcd,0xb1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xcd,0xb3,0x00, + 0x0a,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x10,0x07,0x0a, + 0xff,0xcd,0xb7,0x00,0x0a,0x00,0xd2,0x07,0x61,0x97,0x43,0x00,0x00,0x51,0x04,0x09, + 0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10,0xff,0xcf,0xb3,0x00,0xe0,0x31,0x01,0xcf, + 0x86,0xd5,0xd3,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x01, + 0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x81, + 0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9, + 0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00, + 0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x81,0x00,0xd3,0x3c,0xd2,0x20,0xd1,0x12,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x88, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0x00,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01, + 0xff,0xce,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb4,0x00,0x01,0xff,0xce, + 0xb5,0x00,0x10,0x07,0x01,0xff,0xce,0xb6,0x00,0x01,0xff,0xce,0xb7,0x00,0xd2,0x1c, + 0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb8,0x00,0x01,0xff,0xce,0xb9,0x00,0x10,0x07, + 0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xce,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff, + 0xce,0xbc,0x00,0x01,0xff,0xce,0xbd,0x00,0x10,0x07,0x01,0xff,0xce,0xbe,0x00,0x01, + 0xff,0xce,0xbf,0x00,0xe4,0x85,0x43,0xd3,0x35,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, + 0xff,0xcf,0x80,0x00,0x01,0xff,0xcf,0x81,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xcf,0x84,0x00,0x01,0xff,0xcf,0x85,0x00, + 0x10,0x07,0x01,0xff,0xcf,0x86,0x00,0x01,0xff,0xcf,0x87,0x00,0xe2,0x2b,0x43,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xcf,0x88,0x00,0x01,0xff,0xcf,0x89,0x00,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc,0x88,0x00,0xcf,0x86,0xd5, + 0x94,0xd4,0x3c,0xd3,0x13,0x92,0x0f,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xcf, + 0x83,0x00,0x01,0x00,0x01,0x00,0xd2,0x07,0x61,0x3a,0x43,0x01,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x10, + 0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0xff,0xcf,0x97,0x00,0xd3,0x2c,0xd2, + 0x11,0xe1,0x46,0x43,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01,0xff,0xce,0xb8,0x00, + 0xd1,0x10,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0xff,0xcf,0x86,0x00, + 0x10,0x07,0x01,0xff,0xcf,0x80,0x00,0x04,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06, + 0xff,0xcf,0x99,0x00,0x06,0x00,0x10,0x07,0x01,0xff,0xcf,0x9b,0x00,0x04,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xcf,0x9d,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0x9f, + 0x00,0x04,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf, + 0xa1,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xcf,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xa7,0x00,0x01, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xa9,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xcf,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xad,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xaf,0x00,0x01,0x00,0xd3,0x2b,0xd2,0x12,0x91, + 0x0e,0x10,0x07,0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xcf,0x81,0x00,0x01,0x00,0xd1, + 0x0e,0x10,0x07,0x05,0xff,0xce,0xb8,0x00,0x05,0xff,0xce,0xb5,0x00,0x10,0x04,0x06, + 0x00,0x07,0xff,0xcf,0xb8,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x07,0x00,0x07,0xff, + 0xcf,0xb2,0x00,0x10,0x07,0x07,0xff,0xcf,0xbb,0x00,0x07,0x00,0xd1,0x0b,0x10,0x04, + 0x08,0x00,0x08,0xff,0xcd,0xbb,0x00,0x10,0x07,0x08,0xff,0xcd,0xbc,0x00,0x08,0xff, + 0xcd,0xbd,0x00,0xe3,0xed,0x46,0xe2,0x3d,0x05,0xe1,0x27,0x02,0xe0,0x66,0x01,0xcf, + 0x86,0xd5,0xf0,0xd4,0x7e,0xd3,0x40,0xd2,0x22,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, + 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x07,0x01,0xff,0xd1, + 0x92,0x00,0x01,0xff,0xd0,0xb3,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1, + 0x94,0x00,0x01,0xff,0xd1,0x95,0x00,0x10,0x07,0x01,0xff,0xd1,0x96,0x00,0x01,0xff, + 0xd1,0x96,0xcc,0x88,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x98,0x00, + 0x01,0xff,0xd1,0x99,0x00,0x10,0x07,0x01,0xff,0xd1,0x9a,0x00,0x01,0xff,0xd1,0x9b, + 0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8, + 0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x86,0x00,0x01,0xff,0xd1,0x9f, + 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xb0,0x00,0x01,0xff, + 0xd0,0xb1,0x00,0x10,0x07,0x01,0xff,0xd0,0xb2,0x00,0x01,0xff,0xd0,0xb3,0x00,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xd0,0xb4,0x00,0x01,0xff,0xd0,0xb5,0x00,0x10,0x07,0x01, + 0xff,0xd0,0xb6,0x00,0x01,0xff,0xd0,0xb7,0x00,0xd2,0x1e,0xd1,0x10,0x10,0x07,0x01, + 0xff,0xd0,0xb8,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x10,0x07,0x01,0xff,0xd0, + 0xba,0x00,0x01,0xff,0xd0,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xbc,0x00, + 0x01,0xff,0xd0,0xbd,0x00,0x10,0x07,0x01,0xff,0xd0,0xbe,0x00,0x01,0xff,0xd0,0xbf, + 0x00,0xe4,0x25,0x42,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x80, + 0x00,0x01,0xff,0xd1,0x81,0x00,0x10,0x07,0x01,0xff,0xd1,0x82,0x00,0x01,0xff,0xd1, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x84,0x00,0x01,0xff,0xd1,0x85,0x00, + 0x10,0x07,0x01,0xff,0xd1,0x86,0x00,0x01,0xff,0xd1,0x87,0x00,0xd2,0x1c,0xd1,0x0e, + 0x10,0x07,0x01,0xff,0xd1,0x88,0x00,0x01,0xff,0xd1,0x89,0x00,0x10,0x07,0x01,0xff, + 0xd1,0x8a,0x00,0x01,0xff,0xd1,0x8b,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x8c, + 0x00,0x01,0xff,0xd1,0x8d,0x00,0x10,0x07,0x01,0xff,0xd1,0x8e,0x00,0x01,0xff,0xd1, + 0x8f,0x00,0xcf,0x86,0xd5,0x07,0x64,0xcf,0x41,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xd1,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa5,0x00,0x01,0x00, + 0x10,0x07,0x01,0xff,0xd1,0xa7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd1,0xa9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xab,0x00,0x01,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd1,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xaf, + 0x00,0x01,0x00,0xd3,0x33,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb1,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xb3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd1,0xb5,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb5,0xcc,0x8f,0x00,0x01, + 0xff,0xd1,0xb5,0xcc,0x8f,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb9, + 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07, + 0x01,0xff,0xd1,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbf,0x00,0x01,0x00, + 0xe0,0x41,0x01,0xcf,0x86,0xd5,0x8e,0xd4,0x36,0xd3,0x11,0xe2,0x91,0x41,0xe1,0x88, + 0x41,0x10,0x07,0x01,0xff,0xd2,0x81,0x00,0x01,0x00,0xd2,0x0f,0x51,0x04,0x04,0x00, + 0x10,0x07,0x06,0xff,0xd2,0x8b,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x04,0xff,0xd2, + 0x8d,0x00,0x04,0x00,0x10,0x07,0x04,0xff,0xd2,0x8f,0x00,0x04,0x00,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x91,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xd2,0x93,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x95,0x00,0x01,0x00, + 0x10,0x07,0x01,0xff,0xd2,0x97,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd2,0x99,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9b,0x00,0x01,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd2,0x9d,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9f, + 0x00,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, + 0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xd2,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa7,0x00,0x01, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xa9,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xd2,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xad,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xaf,0x00,0x01,0x00,0xd3,0x2c,0xd2,0x16,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd2,0xb1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xb3, + 0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xb5,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xd2,0xb7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, + 0xb9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xd2,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbf,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0xdc,0xd4,0x5a,0xd3,0x36,0xd2,0x20,0xd1,0x10,0x10,0x07,0x01, + 0xff,0xd3,0x8f,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, + 0xb6,0xcc,0x86,0x00,0x01,0xff,0xd3,0x84,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x06, + 0xff,0xd3,0x86,0x00,0x10,0x04,0x06,0x00,0x01,0xff,0xd3,0x88,0x00,0xd2,0x16,0xd1, + 0x0b,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8a,0x00,0x10,0x04,0x06,0x00,0x01,0xff, + 0xd3,0x8c,0x00,0xe1,0x69,0x40,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8e,0x00,0xd3, + 0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x01,0xff, + 0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x01,0xff, + 0xd0,0xb0,0xcc,0x88,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x95,0x00,0x01,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00, + 0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x99,0x00,0x01,0x00,0x10,0x09,0x01, + 0xff,0xd3,0x99,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc,0x88,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x10, + 0x09,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0xd4, + 0x82,0xd3,0x41,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa1,0x00,0x01,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc, + 0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc, + 0x88,0x00,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa9,0x00,0x01,0x00,0x10, + 0x09,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1, + 0x12,0x10,0x09,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88, + 0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0x01,0xff,0xd1,0x83,0xcc,0x84, + 0x00,0xd3,0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00, + 0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00, + 0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x87,0xcc, + 0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00,0x10,0x07,0x08,0xff,0xd3,0xb7,0x00, + 0x08,0x00,0xd2,0x1d,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x8b,0xcc,0x88,0x00,0x01, + 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x10,0x07,0x09,0xff,0xd3,0xbb,0x00,0x09,0x00,0xd1, + 0x0b,0x10,0x07,0x09,0xff,0xd3,0xbd,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd3,0xbf, + 0x00,0x09,0x00,0xe1,0x26,0x02,0xe0,0x78,0x01,0xcf,0x86,0xd5,0xb0,0xd4,0x58,0xd3, + 0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x81,0x00,0x06,0x00,0x10,0x07, + 0x06,0xff,0xd4,0x83,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x85,0x00, + 0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x87,0x00,0x06,0x00,0xd2,0x16,0xd1,0x0b,0x10, + 0x07,0x06,0xff,0xd4,0x89,0x00,0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x8b,0x00,0x06, + 0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x8d,0x00,0x06,0x00,0x10,0x07,0x06,0xff, + 0xd4,0x8f,0x00,0x06,0x00,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xd4, + 0x91,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd4,0x93,0x00,0x09,0x00,0xd1,0x0b,0x10, + 0x07,0x0a,0xff,0xd4,0x95,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x97,0x00,0x0a, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x99,0x00,0x0a,0x00,0x10,0x07, + 0x0a,0xff,0xd4,0x9b,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x9d,0x00, + 0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x9f,0x00,0x0a,0x00,0xd4,0x58,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0xa1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff, + 0xd4,0xa3,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xd4,0xa5,0x00,0x0b,0x00, + 0x10,0x07,0x0c,0xff,0xd4,0xa7,0x00,0x0c,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x10, + 0xff,0xd4,0xa9,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xab,0x00,0x10,0x00,0xd1, + 0x0b,0x10,0x07,0x10,0xff,0xd4,0xad,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xaf, + 0x00,0x10,0x00,0xd3,0x35,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x00,0x00,0x01,0xff,0xd5, + 0xa1,0x00,0x10,0x07,0x01,0xff,0xd5,0xa2,0x00,0x01,0xff,0xd5,0xa3,0x00,0xd1,0x0e, + 0x10,0x07,0x01,0xff,0xd5,0xa4,0x00,0x01,0xff,0xd5,0xa5,0x00,0x10,0x07,0x01,0xff, + 0xd5,0xa6,0x00,0x01,0xff,0xd5,0xa7,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff, + 0xd5,0xa8,0x00,0x01,0xff,0xd5,0xa9,0x00,0x10,0x07,0x01,0xff,0xd5,0xaa,0x00,0x01, + 0xff,0xd5,0xab,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xac,0x00,0x01,0xff,0xd5, + 0xad,0x00,0x10,0x07,0x01,0xff,0xd5,0xae,0x00,0x01,0xff,0xd5,0xaf,0x00,0xcf,0x86, + 0xe5,0x08,0x3f,0xd4,0x70,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, + 0xb0,0x00,0x01,0xff,0xd5,0xb1,0x00,0x10,0x07,0x01,0xff,0xd5,0xb2,0x00,0x01,0xff, + 0xd5,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xb4,0x00,0x01,0xff,0xd5,0xb5, + 0x00,0x10,0x07,0x01,0xff,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb7,0x00,0xd2,0x1c,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xd5,0xb8,0x00,0x01,0xff,0xd5,0xb9,0x00,0x10,0x07,0x01, + 0xff,0xd5,0xba,0x00,0x01,0xff,0xd5,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, + 0xbc,0x00,0x01,0xff,0xd5,0xbd,0x00,0x10,0x07,0x01,0xff,0xd5,0xbe,0x00,0x01,0xff, + 0xd5,0xbf,0x00,0xe3,0x87,0x3e,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x80, + 0x00,0x01,0xff,0xd6,0x81,0x00,0x10,0x07,0x01,0xff,0xd6,0x82,0x00,0x01,0xff,0xd6, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x84,0x00,0x01,0xff,0xd6,0x85,0x00, + 0x10,0x07,0x01,0xff,0xd6,0x86,0x00,0x00,0x00,0xe0,0x2f,0x3f,0xcf,0x86,0xe5,0xc0, + 0x3e,0xe4,0x97,0x3e,0xe3,0x76,0x3e,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0xd5,0xa5,0xd6,0x82,0x00,0xe4,0x3e,0x25,0xe3,0xc3,0x1a, + 0xe2,0x7b,0x81,0xe1,0xc0,0x13,0xd0,0x1e,0xcf,0x86,0xc5,0xe4,0x08,0x4b,0xe3,0x53, + 0x46,0xe2,0xe9,0x43,0xe1,0x1c,0x43,0xe0,0xe1,0x42,0xcf,0x86,0xe5,0xa6,0x42,0x64, + 0x89,0x42,0x0b,0x00,0xcf,0x86,0xe5,0xfa,0x01,0xe4,0x03,0x56,0xe3,0x76,0x01,0xe2, + 0x8e,0x53,0xd1,0x0c,0xe0,0xef,0x52,0xcf,0x86,0x65,0x8d,0x52,0x04,0x00,0xe0,0x0d, + 0x01,0xcf,0x86,0xd5,0x0a,0xe4,0x10,0x53,0x63,0xff,0x52,0x0a,0x00,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x80,0x00,0x01,0xff,0xe2, + 0xb4,0x81,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x82,0x00,0x01,0xff,0xe2,0xb4,0x83, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x84,0x00,0x01,0xff,0xe2,0xb4,0x85, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x86,0x00,0x01,0xff,0xe2,0xb4,0x87,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x88,0x00,0x01,0xff,0xe2,0xb4,0x89, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8a,0x00,0x01,0xff,0xe2,0xb4,0x8b,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8c,0x00,0x01,0xff,0xe2,0xb4,0x8d,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x8e,0x00,0x01,0xff,0xe2,0xb4,0x8f,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x90,0x00,0x01,0xff,0xe2,0xb4,0x91, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x92,0x00,0x01,0xff,0xe2,0xb4,0x93,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x94,0x00,0x01,0xff,0xe2,0xb4,0x95,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x96,0x00,0x01,0xff,0xe2,0xb4,0x97,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x98,0x00,0x01,0xff,0xe2,0xb4,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x9a,0x00,0x01,0xff,0xe2,0xb4,0x9b,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x9c,0x00,0x01,0xff,0xe2,0xb4,0x9d,0x00,0x10,0x08,0x01, + 0xff,0xe2,0xb4,0x9e,0x00,0x01,0xff,0xe2,0xb4,0x9f,0x00,0xcf,0x86,0xe5,0x42,0x52, + 0x94,0x50,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa0,0x00, + 0x01,0xff,0xe2,0xb4,0xa1,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa2,0x00,0x01,0xff, + 0xe2,0xb4,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa4,0x00,0x01,0xff, + 0xe2,0xb4,0xa5,0x00,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xa7,0x00,0x52,0x04, + 0x00,0x00,0x91,0x0c,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xad,0x00,0x00,0x00, + 0x01,0x00,0xd2,0x1b,0xe1,0xfc,0x52,0xe0,0xad,0x52,0xcf,0x86,0x95,0x0f,0x94,0x0b, + 0x93,0x07,0x62,0x92,0x52,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xd1,0x13,0xe0, + 0xd3,0x53,0xcf,0x86,0x95,0x0a,0xe4,0xa8,0x53,0x63,0x97,0x53,0x04,0x00,0x04,0x00, + 0xd0,0x0d,0xcf,0x86,0x95,0x07,0x64,0x22,0x54,0x08,0x00,0x04,0x00,0xcf,0x86,0x55, + 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x07,0x62,0x2f,0x54,0x04,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0xb0,0x00,0x11,0xff,0xe1,0x8f,0xb1,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb2,0x00,0x11,0xff,0xe1,0x8f,0xb3,0x00,0x91,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb4,0x00,0x11,0xff,0xe1,0x8f,0xb5,0x00,0x00,0x00, + 0xd4,0x1c,0xe3,0xe0,0x56,0xe2,0x17,0x56,0xe1,0xda,0x55,0xe0,0xbb,0x55,0xcf,0x86, + 0x95,0x0a,0xe4,0xa4,0x55,0x63,0x88,0x55,0x04,0x00,0x04,0x00,0xe3,0xd2,0x01,0xe2, + 0x2b,0x5a,0xd1,0x0c,0xe0,0x4c,0x59,0xcf,0x86,0x65,0x25,0x59,0x0a,0x00,0xe0,0x9c, + 0x59,0xcf,0x86,0xd5,0xc5,0xd4,0x45,0xd3,0x31,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x12, + 0xff,0xd0,0xb2,0x00,0x12,0xff,0xd0,0xb4,0x00,0x10,0x07,0x12,0xff,0xd0,0xbe,0x00, + 0x12,0xff,0xd1,0x81,0x00,0x51,0x07,0x12,0xff,0xd1,0x82,0x00,0x10,0x07,0x12,0xff, + 0xd1,0x8a,0x00,0x12,0xff,0xd1,0xa3,0x00,0x92,0x10,0x91,0x0c,0x10,0x08,0x12,0xff, + 0xea,0x99,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x14,0xff,0xe1,0x83,0x90,0x00,0x14,0xff,0xe1,0x83,0x91,0x00,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x92,0x00,0x14,0xff,0xe1,0x83,0x93,0x00,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x94,0x00,0x14,0xff,0xe1,0x83,0x95,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x96,0x00,0x14,0xff,0xe1,0x83,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x98,0x00,0x14,0xff,0xe1,0x83,0x99,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x9a,0x00,0x14,0xff,0xe1,0x83,0x9b,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x9c,0x00,0x14,0xff,0xe1,0x83,0x9d,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0x9e,0x00,0x14,0xff,0xe1,0x83,0x9f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x14,0xff,0xe1,0x83,0xa0,0x00,0x14,0xff,0xe1,0x83,0xa1,0x00,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa2,0x00,0x14,0xff,0xe1,0x83,0xa3,0x00,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa4,0x00,0x14,0xff,0xe1,0x83,0xa5,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xa6,0x00,0x14,0xff,0xe1,0x83,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa8,0x00,0x14,0xff,0xe1,0x83,0xa9,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xaa,0x00,0x14,0xff,0xe1,0x83,0xab,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xac,0x00,0x14,0xff,0xe1,0x83,0xad,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xae,0x00,0x14,0xff,0xe1,0x83,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xb0,0x00,0x14,0xff,0xe1,0x83,0xb1,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb2,0x00,0x14,0xff,0xe1,0x83,0xb3,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb4,0x00,0x14,0xff,0xe1,0x83,0xb5,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xb6,0x00,0x14,0xff,0xe1,0x83,0xb7,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb8,0x00,0x14,0xff,0xe1,0x83,0xb9,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xba,0x00,0x00,0x00,0xd1,0x0c,0x10,0x04,0x00,0x00,0x14,0xff,0xe1,0x83,0xbd,0x00, + 0x10,0x08,0x14,0xff,0xe1,0x83,0xbe,0x00,0x14,0xff,0xe1,0x83,0xbf,0x00,0xe2,0x9d, + 0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa5,0x00,0x01,0xff,0x61,0xcc, + 0xa5,0x00,0x10,0x08,0x01,0xff,0x62,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc,0x87,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x62,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc,0xa3,0x00, + 0x10,0x08,0x01,0xff,0x62,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x63,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc, + 0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x87,0x00,0x01,0xff,0x64,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa3,0x00,0x01,0xff,0x64,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc,0xb1,0x00, + 0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa7,0x00,0x01,0xff, + 0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xad,0x00,0x01,0xff,0x64,0xcc, + 0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff, + 0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00, + 0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc, + 0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, + 0x66,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc,0x84,0x00, + 0x10,0x08,0x01,0xff,0x68,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x68,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00,0x10,0x08, + 0x01,0xff,0x68,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x68,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00,0x10,0x08, + 0x01,0xff,0x68,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x69,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a,0x01,0xff, + 0x69,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x81,0x00,0x01,0xff,0x6b,0xcc, + 0x81,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc,0xa3,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc,0xb1,0x00, + 0x10,0x08,0x01,0xff,0x6c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff,0x6c,0xcc, + 0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x6c,0xcc,0xb1,0x00,0x01,0xff,0x6c,0xcc, + 0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6c,0xcc,0xad,0x00,0x01,0xff,0x6c,0xcc, + 0xad,0x00,0x10,0x08,0x01,0xff,0x6d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc,0x81,0x00, + 0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x6d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6d, + 0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa3, + 0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xad, + 0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x83, + 0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3,0x48,0xd2, + 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff,0x6f, + 0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x70,0xcc,0x81, + 0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x70,0xcc,0x87,0x00,0x01, + 0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x87, + 0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xa3,0x00,0x01, + 0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84, + 0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xb1, + 0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x73,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10,0x08,0x01, + 0xff,0x73,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01, + 0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x10, + 0x0a,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87, + 0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x01, + 0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0x87,0x00,0x01, + 0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xa3,0x00,0x01, + 0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0xb1,0x00,0x01,0xff,0x74, + 0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xad, + 0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa4,0x00,0x01, + 0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0xb0,0x00,0x01, + 0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xad,0x00,0x01,0xff,0x75, + 0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x84, + 0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x76,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x76, + 0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x11,0x02,0xcf,0x86,0xd5,0xe2, + 0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x80,0x00, + 0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x81,0x00,0x01,0xff, + 0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x88,0x00,0x01,0xff, + 0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x87,0x00,0x01,0xff,0x77,0xcc, + 0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0xa3,0x00,0x01,0xff, + 0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x78,0xcc,0x87,0x00,0x01,0xff,0x78,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x78,0xcc,0x88,0x00,0x01,0xff,0x78,0xcc, + 0x88,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc,0x87,0x00, + 0xd3,0x33,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x82,0x00,0x01,0xff, + 0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0xa3,0x00,0x01,0xff,0x7a,0xcc, + 0xa3,0x00,0xe1,0x12,0x59,0x10,0x08,0x01,0xff,0x7a,0xcc,0xb1,0x00,0x01,0xff,0x7a, + 0xcc,0xb1,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01, + 0xff,0x79,0xcc,0x8a,0x00,0x10,0x08,0x01,0xff,0x61,0xca,0xbe,0x00,0x02,0xff,0x73, + 0xcc,0x87,0x00,0x51,0x04,0x0a,0x00,0x10,0x07,0x0a,0xff,0x73,0x73,0x00,0x0a,0x00, + 0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa3,0x00, + 0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x89,0x00,0x01,0xff, + 0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00, + 0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00, + 0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc, + 0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, + 0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00, + 0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00, + 0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc, + 0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x01,0xff, + 0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00, + 0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc, + 0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01,0xd4, + 0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80, + 0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82, + 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01, + 0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x10, + 0x0a,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x89,0x00,0x01,0xff,0x69, + 0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc,0xa3, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc,0xa3, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00,0xd3, + 0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80, + 0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a,0x01, + 0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0xd2, + 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x6f, + 0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b, + 0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98,0xd3, + 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x01, + 0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3, + 0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75, + 0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x89, + 0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0xd1, + 0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc,0x9b, + 0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08,0x01, + 0xff,0x79,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x79,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x79, + 0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x79,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x10,0x08,0x0a,0xff,0xe1, + 0xbb,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbd,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbf,0x00,0x0a,0x00,0xe1,0xbf,0x02,0xe0,0xa1, + 0x01,0xcf,0x86,0xd5,0xc6,0xd4,0x6c,0xd3,0x18,0xe2,0x0e,0x59,0xe1,0xf7,0x58,0x10, + 0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0x00,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, + 0x00,0xd3,0x18,0xe2,0x4a,0x59,0xe1,0x33,0x59,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xb5,0xcc,0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb5,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80, + 0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, + 0xce,0xb5,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd4,0x6c,0xd3,0x18,0xe2,0x74,0x59, + 0xe1,0x5d,0x59,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7, + 0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00, + 0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc, + 0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7, + 0xcc,0x94,0xcd,0x82,0x00,0xd3,0x18,0xe2,0xb0,0x59,0xe1,0x99,0x59,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0x00,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce, + 0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82,0x00,0xcf, + 0x86,0xd5,0xac,0xd4,0x5a,0xd3,0x18,0xe2,0xed,0x59,0xe1,0xd6,0x59,0x10,0x09,0x01, + 0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf, + 0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x18,0xe2, + 0x17,0x5a,0xe1,0x00,0x5a,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff, + 0xcf,0x85,0xcc,0x94,0x00,0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x94,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x80, + 0x00,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00, + 0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00,0xe4,0xd3,0x5a, + 0xd3,0x18,0xe2,0x52,0x5a,0xe1,0x3b,0x5a,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0x00, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xcf, + 0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xe0,0xd9,0x02,0xcf,0x86,0xe5, + 0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1, + 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93, + 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff, + 0xce,0xb1,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1, + 0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce, + 0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01, + 0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x64,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, + 0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0xce,0xb9, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, + 0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82, + 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb7,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80, + 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a, + 0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb7,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93, + 0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00, + 0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, + 0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, + 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89, + 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf, + 0x89,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x49,0xd2,0x26,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce,0xb1,0xcc,0x84,0x00,0x10,0x0b, + 0x01,0xff,0xce,0xb1,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xce,0xb9,0x00, + 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10, + 0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcd,0x82,0xce,0xb9, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x81,0x00,0xe1,0xf3,0x5a,0x10,0x09,0x01,0xff,0xce,0xb1,0xce,0xb9, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0xbd,0xd4,0x7e,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10, + 0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7, + 0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0xd1,0x0f,0x10,0x0b, + 0x01,0xff,0xce,0xb7,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce, + 0xb7,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81, + 0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81, + 0x00,0xe1,0x02,0x5b,0x10,0x09,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0x01,0xff,0xe1, + 0xbe,0xbf,0xcc,0x80,0x00,0xd3,0x18,0xe2,0x28,0x5b,0xe1,0x11,0x5b,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0xe2,0x4c,0x5b, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc, + 0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc, + 0x81,0x00,0xd4,0x51,0xd3,0x18,0xe2,0x6f,0x5b,0xe1,0x58,0x5b,0x10,0x09,0x01,0xff, + 0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00, + 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00, + 0xe1,0x8f,0x5b,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8, + 0xcc,0x80,0x00,0xd3,0x3b,0xd2,0x18,0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf, + 0x89,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0xd1,0x0f,0x10, + 0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff, + 0xcf,0x89,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, + 0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x81,0x00,0xe1,0x99,0x5b,0x10,0x09,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0x01,0xff, + 0xc2,0xb4,0x00,0xe0,0x0c,0x68,0xcf,0x86,0xe5,0x23,0x02,0xe4,0x25,0x01,0xe3,0x85, + 0x5e,0xd2,0x2a,0xe1,0x5f,0x5c,0xe0,0xdd,0x5b,0xcf,0x86,0xe5,0xbb,0x5b,0x94,0x1b, + 0xe3,0xa4,0x5b,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01, + 0xff,0xe2,0x80,0x83,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd1,0xd6,0xd0,0x46,0xcf, + 0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04,0x01, + 0x00,0x10,0x07,0x01,0xff,0xcf,0x89,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01,0x00, + 0x10,0x06,0x01,0xff,0x6b,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x01,0x00,0xe3,0x25, + 0x5d,0x92,0x10,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0x8e,0x00,0x01, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x0a,0xe4,0x42,0x5d,0x63,0x2d,0x5d,0x06,0x00,0x94, + 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb0,0x00,0x01, + 0xff,0xe2,0x85,0xb1,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb2,0x00,0x01,0xff,0xe2, + 0x85,0xb3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb4,0x00,0x01,0xff,0xe2, + 0x85,0xb5,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb6,0x00,0x01,0xff,0xe2,0x85,0xb7, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb8,0x00,0x01,0xff,0xe2, + 0x85,0xb9,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xba,0x00,0x01,0xff,0xe2,0x85,0xbb, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xbc,0x00,0x01,0xff,0xe2,0x85,0xbd, + 0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xbe,0x00,0x01,0xff,0xe2,0x85,0xbf,0x00,0x01, + 0x00,0xe0,0x34,0x5d,0xcf,0x86,0xe5,0x13,0x5d,0xe4,0xf2,0x5c,0xe3,0xe1,0x5c,0xe2, + 0xd4,0x5c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0xff,0xe2,0x86,0x84,0x00, + 0xe3,0x23,0x61,0xe2,0xf0,0x60,0xd1,0x0c,0xe0,0x9d,0x60,0xcf,0x86,0x65,0x7e,0x60, + 0x01,0x00,0xd0,0x62,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x18, + 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x90,0x00, + 0x01,0xff,0xe2,0x93,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93, + 0x92,0x00,0x01,0xff,0xe2,0x93,0x93,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x94,0x00, + 0x01,0xff,0xe2,0x93,0x95,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x96,0x00, + 0x01,0xff,0xe2,0x93,0x97,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x98,0x00,0x01,0xff, + 0xe2,0x93,0x99,0x00,0xcf,0x86,0xe5,0x57,0x60,0x94,0x80,0xd3,0x40,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x9a,0x00,0x01,0xff,0xe2,0x93,0x9b,0x00,0x10, + 0x08,0x01,0xff,0xe2,0x93,0x9c,0x00,0x01,0xff,0xe2,0x93,0x9d,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0x93,0x9e,0x00,0x01,0xff,0xe2,0x93,0x9f,0x00,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa0,0x00,0x01,0xff,0xe2,0x93,0xa1,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0x93,0xa2,0x00,0x01,0xff,0xe2,0x93,0xa3,0x00,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa4,0x00,0x01,0xff,0xe2,0x93,0xa5,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa6,0x00,0x01,0xff,0xe2,0x93,0xa7,0x00,0x10,0x08,0x01,0xff,0xe2, + 0x93,0xa8,0x00,0x01,0xff,0xe2,0x93,0xa9,0x00,0x01,0x00,0xd4,0x0c,0xe3,0x33,0x62, + 0xe2,0x2c,0x62,0xcf,0x06,0x04,0x00,0xe3,0x0c,0x65,0xe2,0xff,0x63,0xe1,0x2e,0x02, + 0xe0,0x84,0x01,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe2,0xb0,0xb0,0x00,0x08,0xff,0xe2,0xb0,0xb1,0x00,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb2,0x00,0x08,0xff,0xe2,0xb0,0xb3,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb4,0x00,0x08,0xff,0xe2,0xb0,0xb5,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xb6,0x00,0x08,0xff,0xe2,0xb0,0xb7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb8,0x00,0x08,0xff,0xe2,0xb0,0xb9,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xba,0x00,0x08,0xff,0xe2,0xb0,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xbc,0x00,0x08,0xff,0xe2,0xb0,0xbd,0x00,0x10,0x08,0x08,0xff,0xe2,0xb0, + 0xbe,0x00,0x08,0xff,0xe2,0xb0,0xbf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb1,0x80,0x00,0x08,0xff,0xe2,0xb1,0x81,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x82,0x00,0x08,0xff,0xe2,0xb1,0x83,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x84,0x00,0x08,0xff,0xe2,0xb1,0x85,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x86,0x00,0x08,0xff,0xe2,0xb1,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x88,0x00,0x08,0xff,0xe2,0xb1,0x89,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x8a,0x00,0x08,0xff,0xe2,0xb1,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x8c,0x00,0x08,0xff,0xe2,0xb1,0x8d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x8e,0x00, + 0x08,0xff,0xe2,0xb1,0x8f,0x00,0x94,0x7c,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb1,0x90,0x00,0x08,0xff,0xe2,0xb1,0x91,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x92,0x00,0x08,0xff,0xe2,0xb1,0x93,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x94,0x00,0x08,0xff,0xe2,0xb1,0x95,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x96,0x00,0x08,0xff,0xe2,0xb1,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x98,0x00,0x08,0xff,0xe2,0xb1,0x99,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x9a,0x00,0x08,0xff,0xe2,0xb1,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x9c,0x00,0x08,0xff,0xe2,0xb1,0x9d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x9e,0x00, + 0x00,0x00,0x08,0x00,0xcf,0x86,0xd5,0x07,0x64,0xef,0x61,0x08,0x00,0xd4,0x63,0xd3, + 0x32,0xd2,0x1b,0xd1,0x0c,0x10,0x08,0x09,0xff,0xe2,0xb1,0xa1,0x00,0x09,0x00,0x10, + 0x07,0x09,0xff,0xc9,0xab,0x00,0x09,0xff,0xe1,0xb5,0xbd,0x00,0xd1,0x0b,0x10,0x07, + 0x09,0xff,0xc9,0xbd,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xa8, + 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xaa,0x00,0x10, + 0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xac,0x00,0xd1,0x0b,0x10,0x04,0x09,0x00,0x0a, + 0xff,0xc9,0x91,0x00,0x10,0x07,0x0a,0xff,0xc9,0xb1,0x00,0x0a,0xff,0xc9,0x90,0x00, + 0xd3,0x27,0xd2,0x17,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xc9,0x92,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xe2,0xb1,0xb3,0x00,0x0a,0x00,0x91,0x0c,0x10,0x04,0x09,0x00,0x09, + 0xff,0xe2,0xb1,0xb6,0x00,0x09,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10, + 0x07,0x0b,0xff,0xc8,0xbf,0x00,0x0b,0xff,0xc9,0x80,0x00,0xe0,0x83,0x01,0xcf,0x86, + 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x83,0x00,0x08,0x00,0xd1,0x0c, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x89,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8f,0x00, + 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x91,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x97,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x99,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x9f,0x00,0x08,0x00, + 0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa1,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0xa5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa7,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa9,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0xab,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0xad,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xaf,0x00,0x08,0x00, + 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb1,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0xb3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0xb5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb7,0x00,0x08,0x00, + 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb9,0x00,0x08,0x00,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0xbb,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0xbd,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xbf,0x00,0x08,0x00,0xcf,0x86, + 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3, + 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x83,0x00,0x08,0x00,0xd1,0x0c, + 0x10,0x08,0x08,0xff,0xe2,0xb3,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3, + 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x89,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb3,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8f,0x00, + 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x91,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb3,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x97,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x99,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb3,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb3,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x9f,0x00,0x08,0x00, + 0xd4,0x3b,0xd3,0x1c,0x92,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa1,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa3,0x00,0x08,0x00,0x08,0x00,0xd2,0x10, + 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x0b,0xff,0xe2,0xb3,0xac,0x00,0xe1,0x3b, + 0x5f,0x10,0x04,0x0b,0x00,0x0b,0xff,0xe2,0xb3,0xae,0x00,0xe3,0x40,0x5f,0x92,0x10, + 0x51,0x04,0x0b,0xe6,0x10,0x08,0x0d,0xff,0xe2,0xb3,0xb3,0x00,0x0d,0x00,0x00,0x00, + 0xe2,0x98,0x08,0xd1,0x0b,0xe0,0x11,0x67,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe0,0x65, + 0x6c,0xcf,0x86,0xe5,0xa7,0x05,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x0c,0xe2,0xf8, + 0x67,0xe1,0x8f,0x67,0xcf,0x06,0x04,0x00,0xe2,0xdb,0x01,0xe1,0x26,0x01,0xd0,0x09, + 0xcf,0x86,0x65,0xf4,0x67,0x0a,0x00,0xcf,0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2, + 0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a, + 0xff,0xea,0x99,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x85, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x99,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x8d,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x99,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x95,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x97,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x99,0x99,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x9b, + 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x9d,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x99,0x9f,0x00,0x0a,0x00,0xe4,0x5d,0x67,0xd3,0x30,0xd2,0x18, + 0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x99,0xa1,0x00,0x0c,0x00,0x10,0x08,0x0a,0xff, + 0xea,0x99,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0xa5,0x00, + 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x0a,0xff,0xea,0x99,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99, + 0xab,0x00,0x0a,0x00,0xe1,0x0c,0x67,0x10,0x08,0x0a,0xff,0xea,0x99,0xad,0x00,0x0a, + 0x00,0xe0,0x35,0x67,0xcf,0x86,0x95,0xab,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x0a,0xff,0xea,0x9a,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a, + 0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x85,0x00,0x0a,0x00, + 0x10,0x08,0x0a,0xff,0xea,0x9a,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8b,0x00, + 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8d,0x00,0x0a,0x00,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x93,0x00, + 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x95,0x00,0x0a,0x00,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x97,0x00,0x0a,0x00,0xe2,0x92,0x66,0xd1,0x0c,0x10,0x08,0x10, + 0xff,0xea,0x9a,0x99,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9a,0x9b,0x00,0x10, + 0x00,0x0b,0x00,0xe1,0x10,0x02,0xd0,0xb9,0xcf,0x86,0xd5,0x07,0x64,0x9e,0x66,0x08, + 0x00,0xd4,0x58,0xd3,0x28,0xd2,0x10,0x51,0x04,0x09,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x9c,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa5,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xab, + 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xad,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xaf,0x00,0x0a,0x00,0xd3,0x28,0xd2,0x10,0x51,0x04,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9c,0xb5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb7,0x00,0x0a, + 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb9,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, + 0x9c,0xbd,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xbf,0x00,0x0a,0x00,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, + 0x9d,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x83,0x00,0x0a,0x00,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x85,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x9d,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x89, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0x8d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8f, + 0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x91, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0x95,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x97, + 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x99,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9d,0x9d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9f,0x00,0x0a, + 0x00,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa1, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0xa5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa7, + 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa9,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xab,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9d,0xad,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xaf,0x00,0x0a, + 0x00,0x53,0x04,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea, + 0x9d,0xba,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9d,0xbc,0x00,0xd1,0x0c,0x10, + 0x04,0x0a,0x00,0x0a,0xff,0xe1,0xb5,0xb9,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xbf, + 0x00,0x0a,0x00,0xe0,0x71,0x01,0xcf,0x86,0xd5,0xa6,0xd4,0x4e,0xd3,0x30,0xd2,0x18, + 0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff, + 0xea,0x9e,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x85,0x00, + 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9e,0x87,0x00,0x0a,0x00,0xd2,0x10,0x51,0x04, + 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9e,0x8c,0x00,0xe1,0x9a,0x64,0x10, + 0x04,0x0a,0x00,0x0c,0xff,0xc9,0xa5,0x00,0xd3,0x28,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0c,0xff,0xea,0x9e,0x91,0x00,0x0c,0x00,0x10,0x08,0x0d,0xff,0xea,0x9e,0x93,0x00, + 0x0d,0x00,0x51,0x04,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x97,0x00,0x10,0x00, + 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e,0x99,0x00,0x10,0x00,0x10,0x08, + 0x10,0xff,0xea,0x9e,0x9b,0x00,0x10,0x00,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e, + 0x9d,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x9f,0x00,0x10,0x00,0xd4,0x63, + 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa1,0x00,0x0c,0x00, + 0x10,0x08,0x0c,0xff,0xea,0x9e,0xa3,0x00,0x0c,0x00,0xd1,0x0c,0x10,0x08,0x0c,0xff, + 0xea,0x9e,0xa5,0x00,0x0c,0x00,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa7,0x00,0x0c,0x00, + 0xd2,0x1a,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa9,0x00,0x0c,0x00,0x10,0x07, + 0x0d,0xff,0xc9,0xa6,0x00,0x10,0xff,0xc9,0x9c,0x00,0xd1,0x0e,0x10,0x07,0x10,0xff, + 0xc9,0xa1,0x00,0x10,0xff,0xc9,0xac,0x00,0x10,0x07,0x12,0xff,0xc9,0xaa,0x00,0x14, + 0x00,0xd3,0x35,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x10,0xff,0xca,0x9e,0x00,0x10,0xff, + 0xca,0x87,0x00,0x10,0x07,0x11,0xff,0xca,0x9d,0x00,0x11,0xff,0xea,0xad,0x93,0x00, + 0xd1,0x0c,0x10,0x08,0x11,0xff,0xea,0x9e,0xb5,0x00,0x11,0x00,0x10,0x08,0x11,0xff, + 0xea,0x9e,0xb7,0x00,0x11,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x14,0xff,0xea,0x9e, + 0xb9,0x00,0x14,0x00,0x10,0x08,0x15,0xff,0xea,0x9e,0xbb,0x00,0x15,0x00,0xd1,0x0c, + 0x10,0x08,0x15,0xff,0xea,0x9e,0xbd,0x00,0x15,0x00,0x10,0x08,0x15,0xff,0xea,0x9e, + 0xbf,0x00,0x15,0x00,0xcf,0x86,0xe5,0xd4,0x63,0x94,0x2f,0x93,0x2b,0xd2,0x10,0x51, + 0x04,0x00,0x00,0x10,0x08,0x15,0xff,0xea,0x9f,0x83,0x00,0x15,0x00,0xd1,0x0f,0x10, + 0x08,0x15,0xff,0xea,0x9e,0x94,0x00,0x15,0xff,0xca,0x82,0x00,0x10,0x08,0x15,0xff, + 0xe1,0xb6,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe4,0xb4,0x66,0xd3,0x1d,0xe2, + 0x5b,0x64,0xe1,0x0a,0x64,0xe0,0xf7,0x63,0xcf,0x86,0xe5,0xd8,0x63,0x94,0x0b,0x93, + 0x07,0x62,0xc3,0x63,0x08,0x00,0x08,0x00,0x08,0x00,0xd2,0x0f,0xe1,0x5a,0x65,0xe0, + 0x27,0x65,0xcf,0x86,0x65,0x0c,0x65,0x0a,0x00,0xd1,0xab,0xd0,0x1a,0xcf,0x86,0xe5, + 0x17,0x66,0xe4,0xfa,0x65,0xe3,0xe1,0x65,0xe2,0xd4,0x65,0x91,0x08,0x10,0x04,0x00, + 0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x0b,0x93,0x07,0x62, + 0x27,0x66,0x11,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff, + 0xe1,0x8e,0xa0,0x00,0x11,0xff,0xe1,0x8e,0xa1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa2,0x00,0x11,0xff,0xe1,0x8e,0xa3,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa4,0x00,0x11,0xff,0xe1,0x8e,0xa5,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xa6,0x00, + 0x11,0xff,0xe1,0x8e,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa8,0x00,0x11,0xff,0xe1,0x8e,0xa9,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xaa,0x00, + 0x11,0xff,0xe1,0x8e,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xac,0x00, + 0x11,0xff,0xe1,0x8e,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xae,0x00,0x11,0xff, + 0xe1,0x8e,0xaf,0x00,0xe0,0xb2,0x65,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb0,0x00,0x11,0xff,0xe1,0x8e, + 0xb1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb2,0x00,0x11,0xff,0xe1,0x8e,0xb3,0x00, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb4,0x00,0x11,0xff,0xe1,0x8e,0xb5,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xb6,0x00,0x11,0xff,0xe1,0x8e,0xb7,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb8,0x00,0x11,0xff,0xe1,0x8e,0xb9,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xba,0x00,0x11,0xff,0xe1,0x8e,0xbb,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xbc,0x00,0x11,0xff,0xe1,0x8e,0xbd,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8e,0xbe,0x00,0x11,0xff,0xe1,0x8e,0xbf,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x80,0x00,0x11,0xff,0xe1,0x8f,0x81,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x82,0x00,0x11,0xff,0xe1,0x8f,0x83,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x84,0x00,0x11,0xff,0xe1,0x8f,0x85,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x86,0x00,0x11,0xff,0xe1,0x8f,0x87,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x88,0x00,0x11,0xff,0xe1,0x8f,0x89,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x8a,0x00,0x11,0xff,0xe1,0x8f,0x8b,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x8c,0x00,0x11,0xff,0xe1,0x8f,0x8d,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0x8e,0x00,0x11,0xff,0xe1,0x8f,0x8f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x90,0x00,0x11,0xff,0xe1,0x8f,0x91,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x92,0x00,0x11,0xff,0xe1,0x8f,0x93,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x94,0x00,0x11,0xff,0xe1,0x8f,0x95,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x96,0x00,0x11,0xff,0xe1,0x8f,0x97,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x98,0x00,0x11,0xff,0xe1,0x8f,0x99,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x9a,0x00,0x11,0xff,0xe1,0x8f,0x9b,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x9c,0x00,0x11,0xff,0xe1,0x8f,0x9d,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0x9e,0x00,0x11,0xff,0xe1,0x8f,0x9f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xa0,0x00,0x11,0xff,0xe1,0x8f,0xa1,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa2,0x00,0x11,0xff,0xe1,0x8f,0xa3,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa4,0x00,0x11,0xff,0xe1,0x8f,0xa5,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xa6,0x00,0x11,0xff,0xe1,0x8f,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa8,0x00,0x11,0xff,0xe1,0x8f,0xa9,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xaa,0x00,0x11,0xff,0xe1,0x8f,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xac,0x00,0x11,0xff,0xe1,0x8f,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8f, + 0xae,0x00,0x11,0xff,0xe1,0x8f,0xaf,0x00,0xd1,0x0c,0xe0,0xeb,0x63,0xcf,0x86,0xcf, + 0x06,0x02,0xff,0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x01,0x00,0xd4,0xae,0xd3,0x09,0xe2,0x54,0x64,0xcf,0x06,0x01,0x00,0xd2, + 0x27,0xe1,0x1f,0x70,0xe0,0x26,0x6e,0xcf,0x86,0xe5,0x3f,0x6d,0xe4,0xce,0x6c,0xe3, + 0x99,0x6c,0xe2,0x78,0x6c,0xe1,0x67,0x6c,0x10,0x08,0x01,0xff,0xe5,0x88,0x87,0x00, + 0x01,0xff,0xe5,0xba,0xa6,0x00,0xe1,0x74,0x74,0xe0,0xe8,0x73,0xcf,0x86,0xe5,0x22, + 0x73,0xd4,0x3b,0x93,0x37,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x01,0xff,0x66,0x66,0x00, + 0x01,0xff,0x66,0x69,0x00,0x10,0x07,0x01,0xff,0x66,0x6c,0x00,0x01,0xff,0x66,0x66, + 0x69,0x00,0xd1,0x0f,0x10,0x08,0x01,0xff,0x66,0x66,0x6c,0x00,0x01,0xff,0x73,0x74, + 0x00,0x10,0x07,0x01,0xff,0x73,0x74,0x00,0x00,0x00,0x00,0x00,0xe3,0xc8,0x72,0xd2, + 0x11,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xd5,0xb4,0xd5,0xb6,0x00, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xd5,0xb4,0xd5,0xa5,0x00,0x01,0xff,0xd5,0xb4,0xd5, + 0xab,0x00,0x10,0x09,0x01,0xff,0xd5,0xbe,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb4,0xd5, + 0xad,0x00,0xd3,0x09,0xe2,0x40,0x74,0xcf,0x06,0x01,0x00,0xd2,0x13,0xe1,0x30,0x75, + 0xe0,0xc1,0x74,0xcf,0x86,0xe5,0x9e,0x74,0x64,0x8d,0x74,0x06,0xff,0x00,0xe1,0x96, + 0x75,0xe0,0x63,0x75,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x7c, + 0xd3,0x3c,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xef,0xbd,0x81,0x00, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x82,0x00,0x01,0xff,0xef,0xbd,0x83,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x84,0x00,0x01,0xff,0xef,0xbd,0x85,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x86,0x00,0x01,0xff,0xef,0xbd,0x87,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x88,0x00,0x01,0xff,0xef,0xbd,0x89,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x8a,0x00,0x01,0xff,0xef,0xbd,0x8b,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x8c,0x00,0x01,0xff,0xef,0xbd,0x8d,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x8e,0x00,0x01,0xff,0xef,0xbd,0x8f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x90,0x00,0x01,0xff,0xef,0xbd,0x91,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x92,0x00,0x01,0xff,0xef,0xbd,0x93,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x94,0x00,0x01,0xff,0xef,0xbd,0x95,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x96,0x00,0x01,0xff,0xef,0xbd,0x97,0x00,0x92,0x1c,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x98,0x00,0x01,0xff,0xef,0xbd,0x99,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x9a,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x87,0xb3,0xe1,0x60,0xb0,0xe0, + 0xdd,0xae,0xcf,0x86,0xe5,0x81,0x9b,0xc4,0xe3,0xc1,0x07,0xe2,0x62,0x06,0xe1,0x11, + 0x86,0xe0,0x09,0x05,0xcf,0x86,0xe5,0xfb,0x02,0xd4,0x1c,0xe3,0x7f,0x76,0xe2,0xd6, + 0x75,0xe1,0xb1,0x75,0xe0,0x8a,0x75,0xcf,0x86,0xe5,0x57,0x75,0x94,0x07,0x63,0x42, + 0x75,0x07,0x00,0x07,0x00,0xe3,0x2b,0x78,0xe2,0xf0,0x77,0xe1,0x77,0x01,0xe0,0x88, + 0x77,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xa8,0x00,0x05,0xff,0xf0,0x90,0x90,0xa9,0x00,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xaa,0x00,0x05,0xff,0xf0,0x90,0x90,0xab,0x00,0xd1,0x12, + 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xac,0x00,0x05,0xff,0xf0,0x90,0x90,0xad,0x00, + 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xae,0x00,0x05,0xff,0xf0,0x90,0x90,0xaf,0x00, + 0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb0,0x00,0x05,0xff,0xf0, + 0x90,0x90,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb2,0x00,0x05,0xff,0xf0, + 0x90,0x90,0xb3,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb4,0x00,0x05, + 0xff,0xf0,0x90,0x90,0xb5,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb6,0x00,0x05, + 0xff,0xf0,0x90,0x90,0xb7,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x90,0xb8,0x00,0x05,0xff,0xf0,0x90,0x90,0xb9,0x00,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x90,0xba,0x00,0x05,0xff,0xf0,0x90,0x90,0xbb,0x00,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xbc,0x00,0x05,0xff,0xf0,0x90,0x90,0xbd,0x00,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xbe,0x00,0x05,0xff,0xf0,0x90,0x90,0xbf,0x00,0xd2,0x24, + 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x80,0x00,0x05,0xff,0xf0,0x90,0x91, + 0x81,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x82,0x00,0x05,0xff,0xf0,0x90,0x91, + 0x83,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x84,0x00,0x05,0xff,0xf0, + 0x90,0x91,0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x86,0x00,0x05,0xff,0xf0, + 0x90,0x91,0x87,0x00,0x94,0x4c,0x93,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x91,0x88,0x00,0x05,0xff,0xf0,0x90,0x91,0x89,0x00,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x91,0x8a,0x00,0x05,0xff,0xf0,0x90,0x91,0x8b,0x00,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x91,0x8c,0x00,0x05,0xff,0xf0,0x90,0x91,0x8d,0x00,0x10,0x09, + 0x07,0xff,0xf0,0x90,0x91,0x8e,0x00,0x07,0xff,0xf0,0x90,0x91,0x8f,0x00,0x05,0x00, + 0x05,0x00,0xd0,0xa0,0xcf,0x86,0xd5,0x07,0x64,0x30,0x76,0x07,0x00,0xd4,0x07,0x63, + 0x3d,0x76,0x07,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90, + 0x93,0x98,0x00,0x12,0xff,0xf0,0x90,0x93,0x99,0x00,0x10,0x09,0x12,0xff,0xf0,0x90, + 0x93,0x9a,0x00,0x12,0xff,0xf0,0x90,0x93,0x9b,0x00,0xd1,0x12,0x10,0x09,0x12,0xff, + 0xf0,0x90,0x93,0x9c,0x00,0x12,0xff,0xf0,0x90,0x93,0x9d,0x00,0x10,0x09,0x12,0xff, + 0xf0,0x90,0x93,0x9e,0x00,0x12,0xff,0xf0,0x90,0x93,0x9f,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa0,0x00,0x12,0xff,0xf0,0x90,0x93,0xa1,0x00, + 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa2,0x00,0x12,0xff,0xf0,0x90,0x93,0xa3,0x00, + 0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa4,0x00,0x12,0xff,0xf0,0x90,0x93, + 0xa5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa6,0x00,0x12,0xff,0xf0,0x90,0x93, + 0xa7,0x00,0xcf,0x86,0xe5,0xc6,0x75,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x90,0x93,0xa8,0x00,0x12,0xff,0xf0,0x90,0x93,0xa9,0x00,0x10, + 0x09,0x12,0xff,0xf0,0x90,0x93,0xaa,0x00,0x12,0xff,0xf0,0x90,0x93,0xab,0x00,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xac,0x00,0x12,0xff,0xf0,0x90,0x93,0xad, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xae,0x00,0x12,0xff,0xf0,0x90,0x93,0xaf, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb0,0x00,0x12,0xff, + 0xf0,0x90,0x93,0xb1,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb2,0x00,0x12,0xff, + 0xf0,0x90,0x93,0xb3,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb4,0x00, + 0x12,0xff,0xf0,0x90,0x93,0xb5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb6,0x00, + 0x12,0xff,0xf0,0x90,0x93,0xb7,0x00,0x93,0x28,0x92,0x24,0xd1,0x12,0x10,0x09,0x12, + 0xff,0xf0,0x90,0x93,0xb8,0x00,0x12,0xff,0xf0,0x90,0x93,0xb9,0x00,0x10,0x09,0x12, + 0xff,0xf0,0x90,0x93,0xba,0x00,0x12,0xff,0xf0,0x90,0x93,0xbb,0x00,0x00,0x00,0x12, + 0x00,0xd4,0x1f,0xe3,0xdf,0x76,0xe2,0x6a,0x76,0xe1,0x09,0x76,0xe0,0xea,0x75,0xcf, + 0x86,0xe5,0xb7,0x75,0x94,0x0a,0xe3,0xa2,0x75,0x62,0x99,0x75,0x07,0x00,0x07,0x00, + 0xe3,0xde,0x78,0xe2,0xaf,0x78,0xd1,0x09,0xe0,0x4c,0x78,0xcf,0x06,0x0b,0x00,0xe0, + 0x7f,0x78,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x80,0x00,0x11,0xff,0xf0,0x90,0xb3,0x81,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x82,0x00,0x11,0xff,0xf0,0x90,0xb3,0x83,0x00,0xd1, + 0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x84,0x00,0x11,0xff,0xf0,0x90,0xb3,0x85, + 0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x86,0x00,0x11,0xff,0xf0,0x90,0xb3,0x87, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x88,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x89,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8a,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x8b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8c,0x00, + 0x11,0xff,0xf0,0x90,0xb3,0x8d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8e,0x00, + 0x11,0xff,0xf0,0x90,0xb3,0x8f,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0x90,0x00,0x11,0xff,0xf0,0x90,0xb3,0x91,0x00,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0x92,0x00,0x11,0xff,0xf0,0x90,0xb3,0x93,0x00,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x94,0x00,0x11,0xff,0xf0,0x90,0xb3,0x95,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x96,0x00,0x11,0xff,0xf0,0x90,0xb3,0x97,0x00,0xd2, + 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x98,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0x99,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9a,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9c,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x9d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9e,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x9f,0x00,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0xa0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa1,0x00,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0xa2,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa3,0x00,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa4,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa5,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa6,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa7,0x00,0xd2, + 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xa8,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0xa9,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xaa,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0xab,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xac,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0xad,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xae,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0xaf,0x00,0x93,0x23,0x92,0x1f,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0, + 0x90,0xb3,0xb0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xb1,0x00,0x10,0x09,0x11,0xff,0xf0, + 0x90,0xb3,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x15,0xe4,0x91, + 0x7b,0xe3,0x9b,0x79,0xe2,0x94,0x78,0xe1,0xe4,0x77,0xe0,0x9d,0x77,0xcf,0x06,0x0c, + 0x00,0xe4,0xeb,0x7e,0xe3,0x44,0x7e,0xe2,0xed,0x7d,0xd1,0x0c,0xe0,0xb2,0x7d,0xcf, + 0x86,0x65,0x93,0x7d,0x14,0x00,0xe0,0xb6,0x7d,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4, + 0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x80,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x81,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x82,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x83,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, + 0x84,0x00,0x10,0xff,0xf0,0x91,0xa3,0x85,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, + 0x86,0x00,0x10,0xff,0xf0,0x91,0xa3,0x87,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x88,0x00,0x10,0xff,0xf0,0x91,0xa3,0x89,0x00,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x8a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8b,0x00,0xd1,0x12,0x10, + 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8d,0x00,0x10, + 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8f,0x00,0xd3, + 0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x90,0x00,0x10,0xff, + 0xf0,0x91,0xa3,0x91,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x92,0x00,0x10,0xff, + 0xf0,0x91,0xa3,0x93,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x94,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x95,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x96,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x97,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0, + 0x91,0xa3,0x98,0x00,0x10,0xff,0xf0,0x91,0xa3,0x99,0x00,0x10,0x09,0x10,0xff,0xf0, + 0x91,0xa3,0x9a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x9c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9d,0x00,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x9e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9f,0x00,0xd1,0x11,0xe0, + 0x12,0x81,0xcf,0x86,0xe5,0x09,0x81,0xe4,0xd2,0x80,0xcf,0x06,0x00,0x00,0xe0,0xdb, + 0x82,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x09,0xe3,0x10,0x81,0xcf,0x06, + 0x0c,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xe2,0x3b,0x82,0xe1,0x16,0x82,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0xa5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa1, + 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa3, + 0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa4,0x00,0x14,0xff,0xf0,0x96, + 0xb9,0xa5,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa6,0x00,0x14,0xff,0xf0,0x96, + 0xb9,0xa7,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa8,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xa9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xaa,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xab,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, + 0xac,0x00,0x14,0xff,0xf0,0x96,0xb9,0xad,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, + 0xae,0x00,0x14,0xff,0xf0,0x96,0xb9,0xaf,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb1,0x00,0x10, + 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb3,0x00,0xd1, + 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb4,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb5, + 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb6,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb7, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb8,0x00,0x14,0xff, + 0xf0,0x96,0xb9,0xb9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xba,0x00,0x14,0xff, + 0xf0,0x96,0xb9,0xbb,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbc,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xbd,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbe,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xbf,0x00,0x14,0x00,0xd2,0x14,0xe1,0x25,0x82,0xe0,0x1c, + 0x82,0xcf,0x86,0xe5,0xdd,0x81,0xe4,0x9a,0x81,0xcf,0x06,0x12,0x00,0xd1,0x0b,0xe0, + 0x51,0x83,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0x95,0x8b,0xcf,0x86,0xd5,0x22,0xe4, + 0xd0,0x88,0xe3,0x93,0x88,0xe2,0x38,0x88,0xe1,0x31,0x88,0xe0,0x2a,0x88,0xcf,0x86, + 0xe5,0xfb,0x87,0xe4,0xe2,0x87,0x93,0x07,0x62,0xd1,0x87,0x12,0xe6,0x12,0xe6,0xe4, + 0x36,0x89,0xe3,0x2f,0x89,0xd2,0x09,0xe1,0xb8,0x88,0xcf,0x06,0x10,0x00,0xe1,0x1f, + 0x89,0xe0,0xec,0x88,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa3, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa5, + 0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa6,0x00,0x12,0xff,0xf0,0x9e, + 0xa4,0xa7,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa8,0x00,0x12,0xff,0xf0,0x9e, + 0xa4,0xa9,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xaa,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xab,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xac,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xad,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, + 0xae,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xaf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, + 0xb0,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb1,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb3,0x00,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb5,0x00,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb6,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb7, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb8,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb9, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xba,0x00,0x12,0xff, + 0xf0,0x9e,0xa4,0xbb,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbc,0x00,0x12,0xff, + 0xf0,0x9e,0xa4,0xbd,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbe,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xbf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa5,0x80,0x00, + 0x12,0xff,0xf0,0x9e,0xa5,0x81,0x00,0x94,0x1e,0x93,0x1a,0x92,0x16,0x91,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa5,0x82,0x00,0x12,0xff,0xf0,0x9e,0xa5,0x83,0x00,0x12, + 0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_c0100 */ + 0x57,0x04,0x01,0x00,0xc6,0xe5,0xac,0x13,0xe4,0x41,0x0c,0xe3,0x7a,0x07,0xe2,0xf3, + 0x01,0xc1,0xd0,0x1f,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x53,0x04,0x01,0x00, + 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x04,0x01,0x00,0x01,0xff,0x00,0x01,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0xe4,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x41,0xcc,0x80,0x00,0x01,0xff,0x41,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x41, + 0xcc,0x82,0x00,0x01,0xff,0x41,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x41, + 0xcc,0x88,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x43, + 0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x80,0x00,0x01, + 0xff,0x45,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x82,0x00,0x01,0xff,0x45, + 0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x80,0x00,0x01,0xff,0x49, + 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x82,0x00,0x01,0xff,0x49,0xcc,0x88, + 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x83, + 0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x80,0x00,0x01,0xff,0x4f,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x82,0x00,0x01,0xff,0x4f,0xcc,0x83,0x00,0x10, + 0x08,0x01,0xff,0x4f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01, + 0x00,0x01,0xff,0x55,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x81,0x00,0x01, + 0xff,0x55,0xcc,0x82,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x88,0x00,0x01, + 0xff,0x59,0xcc,0x81,0x00,0x01,0x00,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x80, + 0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x82,0x00,0x01, + 0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x80,0x00,0x01, + 0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00,0x01,0xff,0x69, + 0xcc,0x88,0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6e, + 0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x81, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82,0x00,0x01,0xff,0x6f,0xcc,0x83, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10, + 0x04,0x01,0x00,0x01,0xff,0x75,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88, + 0x00,0x01,0xff,0x79,0xcc,0x81,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x79,0xcc,0x88, + 0x00,0xe1,0x9a,0x03,0xe0,0xd3,0x01,0xcf,0x86,0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84, + 0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x43,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x44,0xcc,0x8c,0x00,0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x34,0xd2,0x14,0x51, + 0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x84,0x00,0x01,0xff,0x65,0xcc,0x84, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0x86, + 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0xa8,0x00,0x01,0xff,0x65,0xcc,0xa8, + 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x47,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4,0x74,0xd3, + 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x87,0x00,0x01,0xff,0x67, + 0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0xa7,0x00,0x01,0xff,0x67,0xcc,0xa7, + 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0x82,0x00,0x01,0xff,0x68,0xcc,0x82, + 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x83,0x00,0x01, + 0xff,0x69,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x84,0x00,0x01,0xff,0x69, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x86,0x00,0x01,0xff,0x69, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8, + 0x00,0xd3,0x30,0xd2,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x49,0xcc,0x87,0x00,0x01, + 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4a,0xcc,0x82,0x00,0x01,0xff,0x6a, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, + 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4c,0xcc,0x81,0x00,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x4c,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x4c,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x6c,0xcc,0x8c,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x60,0xd3,0x30,0xd2, + 0x10,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6e,0xcc,0x81,0x00,0x01,0xff,0x4e,0xcc,0xa7,0x00,0x10, + 0x08,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x01,0xff,0x4e,0xcc,0x8c,0x00,0xd2,0x10,0x91, + 0x0c,0x10,0x08,0x01,0xff,0x6e,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x4f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0x84,0x00,0x10,0x08,0x01, + 0xff,0x4f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00,0xd3,0x34,0xd2,0x14,0x91, + 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8b,0x00,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81, + 0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x53,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x53,0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x74,0xd3, + 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x8c,0x00,0x01,0xff,0x73, + 0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7, + 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c, + 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x83,0x00,0x01, + 0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x84,0x00,0x01,0xff,0x75, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x86,0x00,0x01,0xff,0x75, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x8a,0x00,0x01,0xff,0x75,0xcc,0x8a, + 0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8b,0x00,0x01, + 0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa8,0x00,0x01,0xff,0x75, + 0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x82,0x00,0x01,0xff,0x77, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x82,0x00,0x01,0xff,0x79,0xcc,0x82, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x59,0xcc,0x88,0x00,0x01,0xff,0x5a, + 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff,0x5a,0xcc,0x87, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff,0x5a,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0x00,0xd0,0x4a,0xcf,0x86,0x55, + 0x04,0x01,0x00,0xd4,0x2c,0xd3,0x18,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0x4f, + 0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01, + 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x55,0xcc,0x9b,0x00,0x93, + 0x14,0x92,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x75,0xcc,0x9b,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xb4,0xd4,0x24,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x41,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x49,0xcc,0x8c,0x00,0xd3,0x46,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x4f,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x8c,0x00,0xd1, + 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x84, + 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x55,0xcc,0x88, + 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, + 0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x88, + 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x80,0xd3,0x3a,0xd2, + 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, + 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x84,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x84,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0x8c, + 0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b, + 0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa8, + 0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa8, + 0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc6, + 0xb7,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0xd3,0x24,0xd2,0x10,0x91, + 0x0c,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0x91,0x10,0x10, + 0x08,0x01,0xff,0x47,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x04,0x00,0xd2, + 0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x4e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a, + 0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x81,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xc3,0x98,0xcc,0x81,0x00,0x01,0xff, + 0xc3,0xb8,0xcc,0x81,0x00,0xe2,0x07,0x02,0xe1,0xae,0x01,0xe0,0x93,0x01,0xcf,0x86, + 0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc, + 0x8f,0x00,0x01,0xff,0x61,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x91,0x00, + 0x01,0xff,0x61,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x8f,0x00, + 0x01,0xff,0x65,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x91,0x00,0x01,0xff, + 0x65,0xcc,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x8f,0x00, + 0x01,0xff,0x69,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x91,0x00,0x01,0xff, + 0x69,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8f,0x00,0x01,0xff, + 0x6f,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc, + 0x91,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8f,0x00, + 0x01,0xff,0x72,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0x91,0x00,0x01,0xff, + 0x72,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8f,0x00,0x01,0xff, + 0x75,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc, + 0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x04,0xff,0x53,0xcc,0xa6,0x00,0x04,0xff, + 0x73,0xcc,0xa6,0x00,0x10,0x08,0x04,0xff,0x54,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc, + 0xa6,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x48,0xcc,0x8c,0x00,0x04,0xff, + 0x68,0xcc,0x8c,0x00,0xd4,0x68,0xd3,0x20,0xd2,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, + 0x07,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x41,0xcc,0x87,0x00, + 0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x45,0xcc, + 0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x88,0xcc, + 0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10,0x0a,0x04,0xff, + 0x4f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x10,0x08, + 0x04,0xff,0x4f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0x93,0x30,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x87,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc, + 0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x59,0xcc,0x84,0x00,0x04,0xff,0x79,0xcc, + 0x84,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0xcf,0x86, + 0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04,0x08,0x00,0x09,0x00,0x09,0x00, + 0x09,0x00,0x01,0x00,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18, + 0x53,0x04,0x01,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00, + 0x11,0x04,0x04,0x00,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00, + 0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x04,0x00,0x94,0x18,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xe1,0x35,0x01,0xd0, + 0x72,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0xe6,0xd3,0x10,0x52,0x04,0x01,0xe6,0x91, + 0x08,0x10,0x04,0x01,0xe6,0x01,0xe8,0x01,0xdc,0x92,0x0c,0x51,0x04,0x01,0xdc,0x10, + 0x04,0x01,0xe8,0x01,0xd8,0x01,0xdc,0xd4,0x2c,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10, + 0x04,0x01,0xdc,0x01,0xca,0x10,0x04,0x01,0xca,0x01,0xdc,0x51,0x04,0x01,0xdc,0x10, + 0x04,0x01,0xdc,0x01,0xca,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0xca,0x01,0xdc,0x01, + 0xdc,0x01,0xdc,0xd3,0x08,0x12,0x04,0x01,0xdc,0x01,0x01,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x01,0x01,0x01,0xdc,0x01,0xdc,0x91,0x08,0x10,0x04,0x01,0xdc,0x01,0xe6,0x01, + 0xe6,0xcf,0x86,0xd5,0x7f,0xd4,0x47,0xd3,0x2e,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, + 0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0x10,0x04,0x01,0xe6,0x01,0xff,0xcc, + 0x93,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xf0,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0xd2,0x08,0x11,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x04,0xdc,0x06,0xff,0x00,0xd3,0x18,0xd2,0x0c, + 0x51,0x04,0x07,0xe6,0x10,0x04,0x07,0xe6,0x07,0xdc,0x51,0x04,0x07,0xdc,0x10,0x04, + 0x07,0xdc,0x07,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe8,0x08,0xdc,0x10,0x04, + 0x08,0xdc,0x08,0xe6,0xd1,0x08,0x10,0x04,0x08,0xe9,0x07,0xea,0x10,0x04,0x07,0xea, + 0x07,0xe9,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0xea,0x10,0x04,0x04,0xe9, + 0x06,0xe6,0x06,0xe6,0x06,0xe6,0xd3,0x13,0x52,0x04,0x0a,0x00,0x91,0x0b,0x10,0x07, + 0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10, + 0x04,0x01,0x00,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10, + 0x00,0xd0,0xe1,0xcf,0x86,0xd5,0x7a,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1, + 0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, + 0xce,0x91,0xcc,0x81,0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xce,0x95,0xcc,0x81,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00,0x10,0x09, + 0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce, + 0x9f,0xcc,0x81,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0x01, + 0xff,0xce,0xa9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff, + 0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4, + 0x4a,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01, + 0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x88,0x00, + 0x01,0xff,0xce,0xa5,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10, + 0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x39,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04, + 0x01,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc, + 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0x00,0xd3, + 0x26,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xcf,0x92,0xcc, + 0x81,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0x00,0x10, + 0x04,0x01,0x00,0x04,0x00,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x01,0x00,0x04, + 0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x01,0x00,0x04,0x00,0xd4, + 0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x06, + 0x00,0x07,0x00,0x12,0x04,0x07,0x00,0x08,0x00,0xe3,0x47,0x04,0xe2,0xbe,0x02,0xe1, + 0x07,0x01,0xd0,0x8b,0xcf,0x86,0xd5,0x6c,0xd4,0x53,0xd3,0x30,0xd2,0x1f,0xd1,0x12, + 0x10,0x09,0x04,0xff,0xd0,0x95,0xcc,0x80,0x00,0x01,0xff,0xd0,0x95,0xcc,0x88,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x93,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0xd0,0x86,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xd0,0x9a,0xcc,0x81,0x00,0x04,0xff,0xd0,0x98,0xcc,0x80,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x86,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0x92, + 0x11,0x91,0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x98,0xcc,0x86,0x00,0x01,0x00, + 0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x11,0x91,0x0d,0x10,0x04, + 0x01,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5, + 0x57,0x54,0x04,0x01,0x00,0xd3,0x30,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, + 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xd0,0xb3,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0xd1,0x96,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0, + 0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8,0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1, + 0x83,0xcc,0x86,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x1a,0x52,0x04,0x01,0x00, + 0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb4,0xcc,0x8f,0x00,0x01,0xff,0xd1, + 0xb5,0xcc,0x8f,0x00,0x01,0x00,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x94,0x24,0xd3,0x18, + 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0x51,0x04,0x01,0xe6, + 0x10,0x04,0x01,0xe6,0x0a,0xe6,0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xbe,0xd4,0x4a,0xd3,0x2a,0xd2,0x1a,0xd1,0x0d, + 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x96,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, + 0xb6,0xcc,0x86,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, + 0x06,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, + 0x06,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00, + 0x09,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x86, + 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x88, + 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, + 0xd0,0x95,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0xd2,0x16,0x51,0x04, + 0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0x98,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc, + 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x96,0xcc,0x88,0x00,0x01,0xff,0xd0, + 0xb6,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0x97,0xcc,0x88,0x00,0x01,0xff,0xd0, + 0xb7,0xcc,0x88,0x00,0xd4,0x74,0xd3,0x3a,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09, + 0x01,0xff,0xd0,0x98,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xd0,0x98,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00, + 0x10,0x09,0x01,0xff,0xd0,0x9e,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00, + 0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0xa8,0xcc,0x88,0x00,0x01, + 0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0,0xad,0xcc,0x88, + 0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x84, + 0x00,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xd0,0xa3,0xcc,0x88,0x00,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09, + 0x01,0xff,0xd0,0xa3,0xcc,0x8b,0x00,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0x91,0x12, + 0x10,0x09,0x01,0xff,0xd0,0xa7,0xcc,0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00, + 0x08,0x00,0x92,0x16,0x91,0x12,0x10,0x09,0x01,0xff,0xd0,0xab,0xcc,0x88,0x00,0x01, + 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x09,0x00,0x09,0x00,0xd1,0x74,0xd0,0x36,0xcf,0x86, + 0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00, + 0xd4,0x10,0x93,0x0c,0x52,0x04,0x0a,0x00,0x11,0x04,0x0b,0x00,0x0c,0x00,0x10,0x00, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0xba, + 0xcf,0x86,0xd5,0x4c,0xd4,0x24,0x53,0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x14,0x00,0x01,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x10,0x00,0x10,0x04,0x10,0x00,0x0d,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x02,0xdc,0x02,0xe6,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6, + 0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xde,0x02,0xdc,0x02,0xe6,0xd4,0x2c, + 0xd3,0x10,0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x08,0xdc,0x02,0xdc,0x02,0xdc, + 0xd2,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6,0xd1,0x08,0x10,0x04, + 0x02,0xe6,0x02,0xde,0x10,0x04,0x02,0xe4,0x02,0xe6,0xd3,0x20,0xd2,0x10,0xd1,0x08, + 0x10,0x04,0x01,0x0a,0x01,0x0b,0x10,0x04,0x01,0x0c,0x01,0x0d,0xd1,0x08,0x10,0x04, + 0x01,0x0e,0x01,0x0f,0x10,0x04,0x01,0x10,0x01,0x11,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x01,0x12,0x01,0x13,0x10,0x04,0x09,0x13,0x01,0x14,0xd1,0x08,0x10,0x04,0x01,0x15, + 0x01,0x16,0x10,0x04,0x01,0x00,0x01,0x17,0xcf,0x86,0xd5,0x28,0x94,0x24,0x93,0x20, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x18,0x10,0x04,0x01,0x19,0x01,0x00, + 0xd1,0x08,0x10,0x04,0x02,0xe6,0x08,0xdc,0x10,0x04,0x08,0x00,0x08,0x12,0x00,0x00, + 0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, + 0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x93,0x10, + 0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe2,0xfb,0x01,0xe1,0x2b,0x01,0xd0,0xa8,0xcf,0x86,0xd5,0x55,0xd4,0x28,0xd3,0x10, + 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x10,0x00,0x0a,0x00,0xd2,0x0c, + 0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00,0x08,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x07,0x00,0x07,0x00,0xd3,0x0c,0x52,0x04,0x07,0xe6,0x11,0x04,0x07,0xe6,0x0a,0xe6, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0a,0x1e,0x0a,0x1f,0x10,0x04,0x0a,0x20,0x01,0x00, + 0xd1,0x09,0x10,0x05,0x0f,0xff,0x00,0x00,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd4, + 0x3d,0x93,0x39,0xd2,0x1a,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x09,0x01, + 0xff,0xd8,0xa7,0xd9,0x93,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x94,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xd9,0x88,0xd9,0x94,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x95,0x00,0x10, + 0x09,0x01,0xff,0xd9,0x8a,0xd9,0x94,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00, + 0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86, + 0xd5,0x5c,0xd4,0x20,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, + 0x01,0x00,0x01,0x1b,0xd1,0x08,0x10,0x04,0x01,0x1c,0x01,0x1d,0x10,0x04,0x01,0x1e, + 0x01,0x1f,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x20,0x01,0x21,0x10,0x04, + 0x01,0x22,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x07,0xdc, + 0x07,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x07,0xe6,0x08,0xe6,0x08,0xe6,0xd1,0x08, + 0x10,0x04,0x08,0xdc,0x08,0xe6,0x10,0x04,0x08,0xe6,0x0c,0xdc,0xd4,0x10,0x53,0x04, + 0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x01,0x23,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x22, + 0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x08, + 0x11,0x04,0x04,0x00,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0x00, + 0xcf,0x86,0xd5,0x5b,0xd4,0x2e,0xd3,0x1e,0x92,0x1a,0xd1,0x0d,0x10,0x09,0x01,0xff, + 0xdb,0x95,0xd9,0x94,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xdb,0x81,0xd9,0x94,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x04,0x00,0xd3,0x19,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0xdb,0x92,0xd9,0x94,0x00,0x11,0x04,0x01,0x00,0x01,0xe6,0x52,0x04,0x01,0xe6,0xd1, + 0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd4,0x38,0xd3, + 0x1c,0xd2,0x0c,0x51,0x04,0x01,0xe6,0x10,0x04,0x01,0xe6,0x01,0xdc,0xd1,0x08,0x10, + 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd2,0x10,0xd1,0x08,0x10, + 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0xdc,0x01,0xe6,0x91,0x08,0x10,0x04,0x01, + 0xe6,0x01,0xdc,0x07,0x00,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04, + 0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0xd1,0xc8,0xd0,0x76,0xcf, + 0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04, + 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04, + 0x00,0x04,0x24,0x04,0x00,0x04,0x00,0x04,0x00,0xd4,0x14,0x53,0x04,0x04,0x00,0x52, + 0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xd3,0x1c,0xd2, + 0x0c,0x91,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04, + 0xdc,0x04,0xe6,0x10,0x04,0x04,0xe6,0x04,0xdc,0xd2,0x0c,0x51,0x04,0x04,0xdc,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10,0x04,0x04, + 0xdc,0x04,0xe6,0xcf,0x86,0xd5,0x3c,0x94,0x38,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x04, + 0xe6,0x10,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, + 0x04,0x04,0xdc,0x04,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, + 0x04,0x04,0xe6,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x08, + 0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0a, + 0x00,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93, + 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x92,0x0c,0x51, + 0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xe6,0x09,0xe6,0xd3,0x10,0x92,0x0c,0x51, + 0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, + 0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x14,0xdc,0x14, + 0x00,0xe4,0xf8,0x57,0xe3,0x45,0x3f,0xe2,0xf4,0x3e,0xe1,0xc7,0x2c,0xe0,0x21,0x10, + 0xcf,0x86,0xc5,0xe4,0x80,0x08,0xe3,0xcb,0x03,0xe2,0x61,0x01,0xd1,0x94,0xd0,0x5a, + 0xcf,0x86,0xd5,0x20,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04,0x0b,0x00,0x11,0x04, + 0x0b,0x00,0x0b,0xe6,0x92,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04,0x0b,0x00,0x0b,0xe6, + 0x0b,0xe6,0xd4,0x24,0xd3,0x10,0x52,0x04,0x0b,0xe6,0x91,0x08,0x10,0x04,0x0b,0x00, + 0x0b,0xe6,0x0b,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6,0x0b,0xe6, + 0x11,0x04,0x0b,0xe6,0x00,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04, + 0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0x54,0x04,0x0c,0x00, + 0x53,0x04,0x0c,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0c,0xdc,0x0c,0xdc, + 0x51,0x04,0x00,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x13,0x00, + 0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x4a,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x10,0x00,0x0d,0x00,0x0d,0x00,0x52,0x04,0x0d,0x00,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x10,0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00, + 0x10,0x04,0x10,0x00,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x12,0x00, + 0x52,0x04,0x12,0x00,0x11,0x04,0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04, + 0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0xdc, + 0x12,0xe6,0x12,0xe6,0xd4,0x30,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04, + 0x12,0x00,0x11,0xdc,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x0d,0xe6,0x91,0x08,0x10,0x04,0x0d,0xe6, + 0x0d,0xdc,0x0d,0xdc,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0x1b,0x0d,0x1c, + 0x10,0x04,0x0d,0x1d,0x0d,0xe6,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x10,0x04,0x0d,0xdc,0x0d,0xe6, + 0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x10,0xe6,0xe1,0x3a,0x01,0xd0,0x77,0xcf, + 0x86,0xd5,0x20,0x94,0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x01, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0xd4,0x1b,0x53,0x04,0x01,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe0,0xa4,0xa8,0xe0,0xa4,0xbc,0x00,0x01,0x00,0x01,0x00,0xd3,0x26,0xd2,0x13, + 0x91,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa4,0xb0,0xe0,0xa4,0xbc,0x00,0x01, + 0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xa4,0xb3,0xe0,0xa4,0xbc,0x00,0x01,0x00, + 0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x91,0x08,0x10,0x04,0x01,0x07, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x8c,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04, + 0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x10,0x04,0x0b,0x00,0x0c,0x00, + 0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x10,0x04,0x01,0xdc, + 0x01,0xe6,0x91,0x08,0x10,0x04,0x01,0xe6,0x0b,0x00,0x0c,0x00,0xd2,0x2c,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xe0,0xa4,0x95,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0x96, + 0xe0,0xa4,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa4,0x97,0xe0,0xa4,0xbc,0x00,0x01, + 0xff,0xe0,0xa4,0x9c,0xe0,0xa4,0xbc,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa4, + 0xa1,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xa2,0xe0,0xa4,0xbc,0x00,0x10,0x0b, + 0x01,0xff,0xe0,0xa4,0xab,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xaf,0xe0,0xa4, + 0xbc,0x00,0x54,0x04,0x01,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0c,0x00,0x0c,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x10,0x00,0x0b,0x00,0x10,0x04,0x0b,0x00,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, + 0x08,0x00,0x09,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, + 0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, + 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd3,0x18,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00, + 0x91,0x08,0x10,0x04,0x01,0x07,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x42, + 0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0x00,0xd2,0x17,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0xff,0xe0,0xa7,0x87,0xe0,0xa6,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01, + 0xff,0xe0,0xa7,0x87,0xe0,0xa7,0x97,0x00,0x01,0x09,0x10,0x04,0x08,0x00,0x00,0x00, + 0xd3,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa6,0xa1,0xe0,0xa6,0xbc, + 0x00,0x01,0xff,0xe0,0xa6,0xa2,0xe0,0xa6,0xbc,0x00,0x10,0x04,0x00,0x00,0x01,0xff, + 0xe0,0xa6,0xaf,0xe0,0xa6,0xbc,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x01,0x00,0x11, + 0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x14,0xe6,0x00, + 0x00,0xe2,0x48,0x02,0xe1,0x4f,0x01,0xd0,0xa4,0xcf,0x86,0xd5,0x4c,0xd4,0x34,0xd3, + 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x10,0x04,0x01,0x00,0x07, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01, + 0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x2e,0xd2,0x17,0xd1, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa8,0xb2, + 0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x0b,0x01,0xff, + 0xe0,0xa8,0xb8,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00, + 0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x00,0x00,0x01,0x00,0xcf,0x86,0xd5,0x80,0xd4, + 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01, + 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01, + 0x09,0x00,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x00, + 0x00,0x00,0x00,0xd2,0x25,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xe0,0xa8,0x96, + 0xe0,0xa8,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa8,0x97,0xe0,0xa8,0xbc,0x00,0x01, + 0xff,0xe0,0xa8,0x9c,0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, + 0x10,0x0b,0x01,0xff,0xe0,0xa8,0xab,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd4,0x10,0x93, + 0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04,0x14,0x00,0x00, + 0x00,0x00,0x00,0xd0,0x82,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x10, + 0x04,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01, + 0x07,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10,0x52,0x04,0x01, + 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x00, + 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x07, + 0x00,0x07,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x0d,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x11,0x00,0x13,0x00,0x13,0x00,0xe1,0x24,0x01,0xd0,0x86,0xcf,0x86, + 0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, + 0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x73,0xd4,0x45,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08, + 0x10,0x04,0x0a,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x0f, + 0x10,0x0b,0x01,0xff,0xe0,0xad,0x87,0xe0,0xad,0x96,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0xe0,0xad,0x87,0xe0,0xac,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff, + 0xe0,0xad,0x87,0xe0,0xad,0x97,0x00,0x01,0x09,0x00,0x00,0xd3,0x0c,0x52,0x04,0x00, + 0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01, + 0xff,0xe0,0xac,0xa1,0xe0,0xac,0xbc,0x00,0x01,0xff,0xe0,0xac,0xa2,0xe0,0xac,0xbc, + 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01, + 0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x0c,0x00,0x0c,0x00,0x00,0x00,0xd0,0xb1,0xcf, + 0x86,0xd5,0x63,0xd4,0x28,0xd3,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd3,0x1f,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, + 0xae,0x92,0xe0,0xaf,0x97,0x00,0x01,0x00,0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x00,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x01,0x00,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00, + 0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x08,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, + 0xd5,0x61,0xd4,0x45,0xd3,0x14,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0,0xae,0xbe,0x00,0x01,0xff,0xe0, + 0xaf,0x87,0xe0,0xae,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0, + 0xaf,0x97,0x00,0x01,0x09,0x00,0x00,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0a, + 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x00, + 0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x08, + 0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00, + 0x00,0x00,0x00,0xe3,0x1c,0x04,0xe2,0x1a,0x02,0xd1,0xf3,0xd0,0x76,0xcf,0x86,0xd5, + 0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01, + 0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0xd2, + 0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x53,0xd4,0x2f,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, + 0xb1,0x86,0xe0,0xb1,0x96,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x01,0x54,0x10,0x04,0x01,0x5b,0x00,0x00,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x11,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, + 0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x52,0x04,0x00,0x00, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0a,0x00,0xd0,0x76,0xcf,0x86, + 0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x10,0x00, + 0x01,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, + 0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, + 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x07,0x07,0x07,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x82,0xd4,0x5e,0xd3,0x2a,0xd2,0x13,0x91,0x0f,0x10,0x0b, + 0x01,0xff,0xe0,0xb2,0xbf,0xe0,0xb3,0x95,0x00,0x01,0x00,0x01,0x00,0xd1,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3, + 0x95,0x00,0xd2,0x28,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x96, + 0x00,0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x82,0x00,0x01,0xff, + 0xe0,0xb3,0x86,0xe0,0xb3,0x82,0xe0,0xb3,0x95,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00, + 0x10,0x04,0x01,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, + 0x09,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08, + 0x10,0x04,0x00,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe1,0x06,0x01,0xd0,0x6e,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x13,0x00,0x10,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01, + 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, + 0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x0c,0x00,0x13,0x09,0x91,0x08,0x10,0x04,0x13,0x09,0x0a,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x65,0xd4,0x45,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, + 0x04,0x0a,0x00,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00,0x00, + 0x00,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb4,0xbe,0x00,0x01,0xff,0xe0,0xb5, + 0x87,0xe0,0xb4,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb5, + 0x97,0x00,0x01,0x09,0x10,0x04,0x0c,0x00,0x12,0x00,0xd3,0x10,0x52,0x04,0x00,0x00, + 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x01,0x00,0x52,0x04,0x12,0x00,0x51,0x04, + 0x12,0x00,0x10,0x04,0x12,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04, + 0x01,0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x0c,0x52,0x04, + 0x0a,0x00,0x11,0x04,0x0a,0x00,0x12,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00, + 0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x5a,0xcf,0x86,0xd5,0x34,0xd4,0x18,0x93,0x14, + 0xd2,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00, + 0x04,0x00,0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x54,0x04, + 0x04,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x04,0x00,0x10,0x04,0x00,0x00,0x04,0x00, + 0x04,0x00,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00,0x00,0x00, + 0xcf,0x86,0xd5,0x77,0xd4,0x28,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00, + 0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x04,0x09, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x04,0x00,0xd3,0x14,0x52,0x04, + 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x10,0x04,0x04,0x00,0x00,0x00, + 0xd2,0x13,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8a, + 0x00,0x04,0x00,0xd1,0x19,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0x00, + 0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0xe0,0xb7,0x8a,0x00,0x10,0x0b,0x04,0xff, + 0xe0,0xb7,0x99,0xe0,0xb7,0x9f,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x00, + 0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x14,0xd2,0x08,0x11,0x04,0x00, + 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe2, + 0x31,0x01,0xd1,0x58,0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x67,0x10,0x04, + 0x01,0x09,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, + 0x95,0x18,0xd4,0x0c,0x53,0x04,0x01,0x00,0x12,0x04,0x01,0x6b,0x01,0x00,0x53,0x04, + 0x01,0x00,0x12,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd0,0x9e,0xcf,0x86,0xd5,0x54, + 0xd4,0x3c,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04, + 0x01,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x15,0x00, + 0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x15,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x15,0x00,0xd3,0x08,0x12,0x04, + 0x15,0x00,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, + 0x01,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x15,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, + 0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x76,0x10,0x04,0x15,0x09, + 0x01,0x00,0x11,0x04,0x01,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4,0x20,0xd3,0x14, + 0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x52,0x04,0x01,0x7a,0x11,0x04,0x01,0x00,0x00,0x00,0x53,0x04,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x01,0x00,0x0d,0x00,0x00,0x00, + 0xe1,0x2b,0x01,0xd0,0x3e,0xcf,0x86,0xd5,0x14,0x54,0x04,0x02,0x00,0x53,0x04,0x02, + 0x00,0x92,0x08,0x11,0x04,0x02,0xdc,0x02,0x00,0x02,0x00,0x54,0x04,0x02,0x00,0xd3, + 0x14,0x52,0x04,0x02,0x00,0xd1,0x08,0x10,0x04,0x02,0x00,0x02,0xdc,0x10,0x04,0x02, + 0x00,0x02,0xdc,0x92,0x0c,0x91,0x08,0x10,0x04,0x02,0x00,0x02,0xd8,0x02,0x00,0x02, + 0x00,0xcf,0x86,0xd5,0x73,0xd4,0x36,0xd3,0x17,0x92,0x13,0x51,0x04,0x02,0x00,0x10, + 0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x82,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x02,0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00, + 0x02,0xff,0xe0,0xbd,0x8c,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd3,0x26,0xd2,0x13,0x51, + 0x04,0x02,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x91,0xe0,0xbe,0xb7,0x00,0x02,0x00, + 0x51,0x04,0x02,0x00,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x96,0xe0,0xbe,0xb7, + 0x00,0x52,0x04,0x02,0x00,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x9b,0xe0,0xbe, + 0xb7,0x00,0x02,0x00,0x02,0x00,0xd4,0x27,0x53,0x04,0x02,0x00,0xd2,0x17,0xd1,0x0f, + 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x80,0xe0,0xbe,0xb5,0x00,0x10,0x04,0x04, + 0x00,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd3,0x35,0xd2, + 0x17,0xd1,0x08,0x10,0x04,0x00,0x00,0x02,0x81,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, + 0xbd,0xb1,0xe0,0xbd,0xb2,0x00,0xd1,0x0f,0x10,0x04,0x02,0x84,0x02,0xff,0xe0,0xbd, + 0xb1,0xe0,0xbd,0xb4,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb2,0xe0,0xbe,0x80,0x00, + 0x02,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb3,0xe0,0xbe,0x80, + 0x00,0x02,0x00,0x02,0x82,0x11,0x04,0x02,0x82,0x02,0x00,0xd0,0xd3,0xcf,0x86,0xd5, + 0x65,0xd4,0x27,0xd3,0x1f,0xd2,0x13,0x91,0x0f,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, + 0xbd,0xb1,0xe0,0xbe,0x80,0x00,0x02,0xe6,0x91,0x08,0x10,0x04,0x02,0x09,0x02,0x00, + 0x02,0xe6,0x12,0x04,0x02,0x00,0x0c,0x00,0xd3,0x1f,0xd2,0x13,0x51,0x04,0x02,0x00, + 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x92,0xe0,0xbe,0xb7,0x00,0x51,0x04,0x02, + 0x00,0x10,0x04,0x04,0x00,0x02,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x02, + 0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x9c,0xe0,0xbe, + 0xb7,0x00,0x02,0x00,0xd4,0x3d,0xd3,0x26,0xd2,0x13,0x51,0x04,0x02,0x00,0x10,0x0b, + 0x02,0xff,0xe0,0xbe,0xa1,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x51,0x04,0x02,0x00,0x10, + 0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0xa6,0xe0,0xbe,0xb7,0x00,0x52,0x04,0x02,0x00, + 0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xab,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x04, + 0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x02,0x00,0x02,0x00,0x02, + 0x00,0xd2,0x13,0x91,0x0f,0x10,0x04,0x04,0x00,0x02,0xff,0xe0,0xbe,0x90,0xe0,0xbe, + 0xb5,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xcf,0x86, + 0x95,0x4c,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0xdc,0x04,0x00,0x52,0x04,0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00, + 0x10,0x04,0x0a,0x00,0x04,0x00,0xd3,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x0a,0x00, + 0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04, + 0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, + 0xe5,0xf7,0x04,0xe4,0x79,0x03,0xe3,0x7b,0x01,0xe2,0x04,0x01,0xd1,0x7f,0xd0,0x65, + 0xcf,0x86,0x55,0x04,0x04,0x00,0xd4,0x33,0xd3,0x1f,0xd2,0x0c,0x51,0x04,0x04,0x00, + 0x10,0x04,0x0a,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe1,0x80, + 0xa5,0xe1,0x80,0xae,0x00,0x04,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0a,0x00,0x04, + 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x04,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x04, + 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x04,0x00,0x04, + 0x07,0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0x09,0x10,0x04,0x0a,0x09,0x0a, + 0x00,0x0a,0x00,0xcf,0x86,0x95,0x14,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92, + 0x08,0x11,0x04,0x04,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x2e,0xcf,0x86,0x95, + 0x28,0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a, + 0x00,0x0a,0xdc,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00,0x0b, + 0x00,0x11,0x04,0x0b,0x00,0x0a,0x00,0x01,0x00,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x52, + 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x54, + 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x06,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x06,0x00,0x08,0x00,0x10,0x04,0x08, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x3e,0xd0, + 0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x1d,0x54,0x04,0x01,0x00,0x53,0x04,0x01, + 0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, + 0x00,0x01,0xff,0x00,0x94,0x15,0x93,0x11,0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x0b,0x00,0x0b,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, + 0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x0b, + 0x00,0xe2,0x21,0x01,0xd1,0x6c,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, + 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00, + 0x04,0x00,0x04,0x00,0xcf,0x86,0x95,0x48,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, + 0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08, + 0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00, + 0xd0,0x62,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04, + 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00, + 0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xd4,0x14,0x53,0x04, + 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04, + 0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00, + 0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x93,0x10,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x94,0x14,0x53,0x04, + 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0x04,0x00,0xd1,0x9c,0xd0,0x3e,0xcf,0x86,0x95,0x38,0xd4,0x14,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd3,0x14, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00, + 0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0x04,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04, + 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00,0x53,0x04,0x04,0x00,0xd2,0x0c, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x0c,0xe6,0x10,0x04,0x0c,0xe6,0x08,0xe6,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x08,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a, + 0xcf,0x86,0x95,0x14,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04, + 0x08,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04, + 0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x11,0x00, + 0x00,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0xd3,0x30,0xd2,0x2a, + 0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0b,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00, + 0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd2,0x6c,0xd1,0x24, + 0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, + 0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x0b,0x00, + 0x0b,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00, + 0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x04,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x46,0xcf,0x86,0xd5,0x28, + 0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x00, + 0x00,0x00,0x06,0x00,0x93,0x10,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x09, + 0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x06,0x00,0x93,0x14,0x52,0x04,0x06,0x00, + 0xd1,0x08,0x10,0x04,0x06,0x09,0x06,0x00,0x10,0x04,0x06,0x00,0x00,0x00,0x00,0x00, + 0xcf,0x86,0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x06,0x00,0x00,0x00, + 0x00,0x00,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04, + 0x06,0x00,0x00,0x00,0x06,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, + 0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0xd0,0x1b,0xcf,0x86,0x55,0x04,0x04,0x00, + 0x54,0x04,0x04,0x00,0x93,0x0d,0x52,0x04,0x04,0x00,0x11,0x05,0x04,0xff,0x00,0x04, + 0x00,0x04,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x09,0x04,0x00,0x04,0x00,0x52,0x04,0x04,0x00,0x91, + 0x08,0x10,0x04,0x04,0x00,0x07,0xe6,0x00,0x00,0xd4,0x10,0x53,0x04,0x04,0x00,0x92, + 0x08,0x11,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x08,0x11, + 0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xe4,0xb7,0x03,0xe3,0x58,0x01,0xd2,0x8f,0xd1, + 0x53,0xd0,0x35,0xcf,0x86,0x95,0x2f,0xd4,0x1f,0x53,0x04,0x04,0x00,0xd2,0x0d,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x04,0xff,0x00,0x51,0x05,0x04,0xff,0x00,0x10, + 0x05,0x04,0xff,0x00,0x00,0x00,0x53,0x04,0x04,0x00,0x92,0x08,0x11,0x04,0x04,0x00, + 0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, + 0x53,0x04,0x04,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x04,0x00,0x94,0x18,0x53,0x04,0x04,0x00, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0xe4,0x10,0x04,0x0a,0x00,0x00,0x00, + 0x00,0x00,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x0c, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x42, + 0xcf,0x86,0xd5,0x1c,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, + 0xd1,0x08,0x10,0x04,0x07,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0xd4,0x0c, + 0x53,0x04,0x07,0x00,0x12,0x04,0x07,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x10, + 0xd1,0x08,0x10,0x04,0x07,0x00,0x07,0xde,0x10,0x04,0x07,0xe6,0x07,0xdc,0x00,0x00, + 0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, + 0x00,0x00,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd4,0x10,0x53,0x04,0x07,0x00, + 0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x07,0x00, + 0x91,0x08,0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a,0xcf,0x86, + 0x55,0x04,0x08,0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00, + 0x0b,0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x95,0x28,0xd4,0x10,0x53,0x04,0x08,0x00, + 0x92,0x08,0x11,0x04,0x08,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0xd2,0x0c, + 0x51,0x04,0x08,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x08,0x00, + 0x07,0x00,0xd2,0xe4,0xd1,0x80,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x54,0x04,0x08,0x00, + 0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x08,0xe6, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x08,0xdc,0x08,0x00,0x08,0x00,0x11,0x04,0x00,0x00, + 0x08,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xd4,0x14, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00, + 0x0b,0x00,0xd3,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6, + 0x0b,0xe6,0x52,0x04,0x0b,0xe6,0xd1,0x08,0x10,0x04,0x0b,0xe6,0x00,0x00,0x10,0x04, + 0x00,0x00,0x0b,0xdc,0xd0,0x5e,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0b,0x00, + 0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0b,0x00,0x92,0x08, + 0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53,0x04,0x0b,0x00,0x52,0x04, + 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x10,0xe6,0x91,0x08, + 0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0xdc,0xd2,0x0c,0x51,0x04,0x10,0xdc,0x10,0x04, + 0x10,0xdc,0x10,0xe6,0xd1,0x08,0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0x04,0x10,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xe1,0x1e,0x01,0xd0,0xaa,0xcf,0x86,0xd5,0x6e,0xd4, + 0x53,0xd3,0x17,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1, + 0xac,0x85,0xe1,0xac,0xb5,0x00,0x09,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x09,0xff, + 0xe1,0xac,0x87,0xe1,0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x89, + 0xe1,0xac,0xb5,0x00,0x09,0x00,0xd1,0x0f,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8b,0xe1, + 0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8d,0xe1,0xac,0xb5,0x00, + 0x09,0x00,0x93,0x17,0x92,0x13,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac, + 0x91,0xe1,0xac,0xb5,0x00,0x09,0x00,0x09,0x00,0x09,0x00,0x54,0x04,0x09,0x00,0xd3, + 0x10,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x07,0x09,0x00,0x09,0x00,0xd2, + 0x13,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xba,0xe1,0xac, + 0xb5,0x00,0x91,0x0f,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xbc,0xe1,0xac,0xb5, + 0x00,0x09,0x00,0xcf,0x86,0xd5,0x3d,0x94,0x39,0xd3,0x31,0xd2,0x25,0xd1,0x16,0x10, + 0x0b,0x09,0xff,0xe1,0xac,0xbe,0xe1,0xac,0xb5,0x00,0x09,0xff,0xe1,0xac,0xbf,0xe1, + 0xac,0xb5,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xad,0x82,0xe1,0xac,0xb5,0x00, + 0x91,0x08,0x10,0x04,0x09,0x09,0x09,0x00,0x09,0x00,0x12,0x04,0x09,0x00,0x00,0x00, + 0x09,0x00,0xd4,0x1c,0x53,0x04,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09,0x00,0x10,0x04, + 0x09,0x00,0x09,0xe6,0x91,0x08,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0xe6,0xd3,0x08, + 0x12,0x04,0x09,0xe6,0x09,0x00,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, + 0x00,0x00,0x00,0x00,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x18,0x53,0x04, + 0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x09,0x0d,0x09,0x11,0x04, + 0x0d,0x00,0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x0d,0x00, + 0x0d,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x0c,0x00, + 0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x07,0x0c,0x00,0x0c,0x00,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0c,0x09,0x00,0x00,0x12,0x04,0x00,0x00,0x0c,0x00,0xe3,0xb2, + 0x01,0xe2,0x09,0x01,0xd1,0x4c,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x0a,0x00,0x54,0x04, + 0x0a,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00, + 0x0a,0x07,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00, + 0xcf,0x86,0x95,0x1c,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, + 0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00, + 0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x54,0x04,0x14,0x00, + 0x53,0x04,0x14,0x00,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00, + 0x91,0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x08, + 0x13,0x04,0x0d,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04, + 0x0b,0xe6,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x01,0x0b,0xdc,0x0b,0xdc,0x92,0x08, + 0x11,0x04,0x0b,0xdc,0x0b,0xe6,0x0b,0xdc,0xd4,0x28,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x01,0x0b,0x01,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x0b,0x01,0x0b,0x00,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xdc,0x0b,0x00, + 0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0d,0x00,0xd1,0x08, + 0x10,0x04,0x0d,0xe6,0x0d,0x00,0x10,0x04,0x0d,0x00,0x13,0x00,0x92,0x0c,0x51,0x04, + 0x10,0xe6,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06, + 0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x94,0x0c,0x53,0x04,0x07,0x00,0x12,0x04, + 0x07,0x00,0x08,0x00,0x08,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0xd5,0x40, + 0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x51,0x04,0x08,0xe6,0x10,0x04,0x08,0xdc,0x08,0xe6, + 0x09,0xe6,0xd2,0x0c,0x51,0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x0a,0xe6,0xd1,0x08, + 0x10,0x04,0x0a,0xe6,0x0a,0xea,0x10,0x04,0x0a,0xd6,0x0a,0xdc,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0a,0xca,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0xd4,0x14, + 0x93,0x10,0x52,0x04,0x0a,0xe6,0x51,0x04,0x0a,0xe6,0x10,0x04,0x0a,0xe6,0x10,0xe6, + 0x10,0xe6,0xd3,0x10,0x52,0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x13,0xe8, + 0x13,0xe4,0xd2,0x10,0xd1,0x08,0x10,0x04,0x13,0xe4,0x13,0xdc,0x10,0x04,0x00,0x00, + 0x12,0xe6,0xd1,0x08,0x10,0x04,0x0c,0xe9,0x0b,0xdc,0x10,0x04,0x09,0xe6,0x09,0xdc, + 0xe2,0x80,0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa5,0x00,0x01,0xff, + 0x61,0xcc,0xa5,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x42,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00, + 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x43,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff, + 0x63,0xcc,0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0x87,0x00,0x01,0xff, + 0x64,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa3,0x00,0x01,0xff, + 0x64,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc, + 0xb1,0x00,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa7,0x00, + 0x01,0xff,0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xad,0x00,0x01,0xff, + 0x64,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc,0x80,0x00, + 0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc, + 0x81,0x00,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x45,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff, + 0x45,0xcc,0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x45,0xcc,0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08, + 0x01,0xff,0x46,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc, + 0x84,0x00,0x10,0x08,0x01,0xff,0x48,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00, + 0x10,0x08,0x01,0xff,0x48,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00, + 0x10,0x08,0x01,0xff,0x48,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x49,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a, + 0x01,0xff,0x49,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0x81,0x00,0x01,0xff, + 0x6b,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc, + 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc, + 0xb1,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00, + 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff, + 0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xb1,0x00,0x01,0xff, + 0x6c,0xcc,0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4c,0xcc,0xad,0x00,0x01,0xff, + 0x6c,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x4d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc, + 0x81,0x00,0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x4d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01, + 0xff,0x4d,0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x4e,0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x4e, + 0xcc,0xa3,0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x4e,0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x4e, + 0xcc,0xad,0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f, + 0xcc,0x83,0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01, + 0xff,0x4f,0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3, + 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x80,0x00,0x01, + 0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x81, + 0x00,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x50, + 0xcc,0x81,0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x50,0xcc,0x87, + 0x00,0x01,0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52, + 0xcc,0x87,0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa3, + 0x00,0x01,0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x52,0xcc,0xa3, + 0xcc,0x84,0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x52, + 0xcc,0xb1,0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10, + 0x08,0x01,0xff,0x53,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x53,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87, + 0x00,0x10,0x0a,0x01,0xff,0x53,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c, + 0xcc,0x87,0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x53,0xcc,0xa3,0xcc,0x87, + 0x00,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0x87, + 0x00,0x01,0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0xa3, + 0x00,0x01,0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xb1,0x00,0x01, + 0xff,0x74,0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x54, + 0xcc,0xad,0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa4, + 0x00,0x01,0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0xb0, + 0x00,0x01,0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xad,0x00,0x01, + 0xff,0x75,0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x83, + 0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x55, + 0xcc,0x84,0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x56,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01, + 0xff,0x56,0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x10,0x02,0xcf,0x86, + 0xd5,0xe1,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc, + 0x80,0x00,0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x81,0x00, + 0x01,0xff,0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x88,0x00, + 0x01,0xff,0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x87,0x00,0x01,0xff, + 0x77,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0xa3,0x00, + 0x01,0xff,0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x58,0xcc,0x87,0x00,0x01,0xff, + 0x78,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x58,0xcc,0x88,0x00,0x01,0xff, + 0x78,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc, + 0x87,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0x82,0x00, + 0x01,0xff,0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x5a,0xcc,0xa3,0x00,0x01,0xff, + 0x7a,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0xb1,0x00,0x01,0xff, + 0x7a,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x68,0xcc,0xb1,0x00,0x01,0xff,0x74,0xcc, + 0x88,0x00,0x92,0x1d,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01,0xff, + 0x79,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x02,0xff,0xc5,0xbf,0xcc,0x87,0x00,0x0a, + 0x00,0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa3, + 0x00,0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x89,0x00,0x01, + 0xff,0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x81, + 0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82, + 0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82, + 0xcc,0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x82,0x00,0x01, + 0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x81, + 0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86, + 0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x83,0x00,0x01, + 0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x86, + 0x00,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x45,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x45, + 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45, + 0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82, + 0xcc,0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01, + 0xd4,0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc, + 0x82,0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x45,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00, + 0x10,0x0a,0x01,0xff,0x45,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc, + 0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x89,0x00,0x01,0xff, + 0x69,0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc, + 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00, + 0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc,0x81,0x00, + 0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x4f,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a, + 0x01,0xff,0x4f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00, + 0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, + 0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x81,0x00, + 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc, + 0x9b,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, + 0x4f,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98, + 0xd3,0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x83,0x00, + 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc, + 0xa3,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x55,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x55,0xcc, + 0x89,0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x55,0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a, + 0x01,0xff,0x55,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc, + 0x9b,0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff, + 0x75,0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x55,0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08, + 0x01,0xff,0x59,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x59,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff, + 0x59,0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0x92,0x14,0x91,0x10,0x10,0x08, + 0x01,0xff,0x59,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x0a,0x00,0x0a,0x00, + 0xe1,0xc0,0x04,0xe0,0x80,0x02,0xcf,0x86,0xe5,0x2d,0x01,0xd4,0xa8,0xd3,0x54,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, + 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x91,0xcc,0x93,0x00,0x01,0xff, + 0xce,0x91,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80,0x00, + 0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, + 0x91,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x81,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x91,0xcc,0x94, + 0xcd,0x82,0x00,0xd3,0x42,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, + 0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94, + 0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x95,0xcc, + 0x93,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x95,0xcc, + 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, + 0x0b,0x01,0xff,0xce,0x95,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x95,0xcc,0x94, + 0xcc,0x81,0x00,0x00,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80, + 0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, + 0xce,0xb7,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd, + 0x82,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0x97,0xcc,0x93,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x94, + 0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81,0x00, + 0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc, + 0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x82,0x00,0xd3,0x54,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82, + 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x93,0x00,0x01,0xff, + 0xce,0x99,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcc,0x80,0x00, + 0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, + 0x99,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x81,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x99,0xcc,0x94, + 0xcd,0x82,0x00,0xcf,0x86,0xe5,0x13,0x01,0xd4,0x84,0xd3,0x42,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, + 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0x9f,0xcc,0x93,0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc, + 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x54,0xd2,0x28, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff,0xcf,0x85,0xcc, + 0x94,0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93, + 0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, + 0xcf,0x85,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00, + 0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0x00,0x10, + 0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x0f,0x10,0x04, + 0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x81,0x00,0x10,0x04,0x00,0x00,0x01, + 0xff,0xce,0xa5,0xcc,0x94,0xcd,0x82,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89, + 0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xa9,0xcc,0x93,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce, + 0xa9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, + 0xce,0xa9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0x00, + 0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01, + 0xff,0xce,0xb1,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01, + 0xff,0xce,0xb5,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80, + 0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff, + 0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, + 0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x91,0x12,0x10,0x09, + 0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x00,0x00, + 0xe0,0xe1,0x02,0xcf,0x86,0xe5,0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82, + 0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce, + 0x91,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a, + 0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce, + 0x91,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93, + 0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00, + 0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7, + 0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xcd, + 0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, + 0xce,0xb7,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd, + 0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, + 0xce,0x97,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc, + 0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d, + 0x01,0xff,0xce,0x97,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc, + 0x94,0xcd,0x82,0xcd,0x85,0x00,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd, + 0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, + 0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xcd,0x85, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x82, + 0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd3,0x49, + 0xd2,0x26,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xce,0xb1,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81, + 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff, + 0xce,0xb1,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0x91,0xcc,0x86,0x00,0x01,0xff,0xce,0x91,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce, + 0x91,0xcc,0x80,0x00,0x01,0xff,0xce,0x91,0xcc,0x81,0x00,0xd1,0x0d,0x10,0x09,0x01, + 0xff,0xce,0x91,0xcd,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xb9,0x00,0x01, + 0x00,0xcf,0x86,0xe5,0x16,0x01,0xd4,0x8f,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10,0x04, + 0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc, + 0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x81,0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb7, + 0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0x95,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x81,0x00, + 0x10,0x09,0x01,0xff,0xce,0x97,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00, + 0xd1,0x13,0x10,0x09,0x01,0xff,0xce,0x97,0xcd,0x85,0x00,0x01,0xff,0xe1,0xbe,0xbf, + 0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbe,0xbf,0xcc,0x81,0x00,0x01,0xff,0xe1, + 0xbe,0xbf,0xcd,0x82,0x00,0xd3,0x40,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce, + 0xb9,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x51, + 0x04,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, + 0x86,0x00,0x01,0xff,0xce,0x99,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, + 0x80,0x00,0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x04,0x00,0x00,0x01, + 0xff,0xe1,0xbf,0xbe,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbf,0xbe,0xcc,0x81, + 0x00,0x01,0xff,0xe1,0xbf,0xbe,0xcd,0x82,0x00,0xd4,0x93,0xd3,0x4e,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84, + 0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85, + 0xcc,0x88,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x93,0x00, + 0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcd,0x82,0x00, + 0x01,0xff,0xcf,0x85,0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xa5,0xcc,0x86,0x00,0x01,0xff,0xce,0xa5,0xcc,0x84,0x00,0x10,0x09,0x01, + 0xff,0xce,0xa5,0xcc,0x80,0x00,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0xa1,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x80,0x00,0x10, + 0x09,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x01,0xff,0x60,0x00,0xd3,0x3b,0xd2,0x18, + 0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xcf,0x89,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81, + 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcd,0x82,0x00,0x01,0xff, + 0xcf,0x89,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0x9f,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce, + 0xa9,0xcc,0x80,0x00,0x01,0xff,0xce,0xa9,0xcc,0x81,0x00,0xd1,0x10,0x10,0x09,0x01, + 0xff,0xce,0xa9,0xcd,0x85,0x00,0x01,0xff,0xc2,0xb4,0x00,0x10,0x04,0x01,0x00,0x00, + 0x00,0xe0,0x7e,0x0c,0xcf,0x86,0xe5,0xbb,0x08,0xe4,0x14,0x06,0xe3,0xf7,0x02,0xe2, + 0xbd,0x01,0xd1,0xd0,0xd0,0x4f,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0xd3,0x18,0x92,0x14, + 0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01,0xff,0xe2,0x80,0x83,0x00, + 0x01,0x00,0x01,0x00,0x92,0x0d,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0x00,0x01,0xff,0x00,0x01,0x00,0x94,0x1b,0x53,0x04,0x01,0x00,0xd2,0x09,0x11,0x04, + 0x01,0x00,0x01,0xff,0x00,0x51,0x05,0x01,0xff,0x00,0x10,0x05,0x01,0xff,0x00,0x04, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x48,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x01,0x00,0x51, + 0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x52,0x04,0x04,0x00,0x11,0x04,0x04, + 0x00,0x06,0x00,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x06,0x00,0x07, + 0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0x52, + 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0xd4,0x23,0xd3, + 0x14,0x52,0x05,0x06,0xff,0x00,0x91,0x0a,0x10,0x05,0x0a,0xff,0x00,0x00,0xff,0x00, + 0x0f,0xff,0x00,0x92,0x0a,0x11,0x05,0x0f,0xff,0x00,0x01,0xff,0x00,0x01,0xff,0x00, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x00,0x00,0x01,0x00, + 0x01,0x00,0xd0,0x7e,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x53,0x04,0x01,0x00,0x52,0x04, + 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, + 0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04,0x0c,0x00, + 0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00, + 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x02,0x00,0x91,0x08,0x10,0x04, + 0x03,0x00,0x04,0x00,0x04,0x00,0xd3,0x10,0xd2,0x08,0x11,0x04,0x06,0x00,0x08,0x00, + 0x11,0x04,0x08,0x00,0x0b,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00, + 0x10,0x04,0x0e,0x00,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x11,0x00,0x13,0x00, + 0xcf,0x86,0xd5,0x28,0x54,0x04,0x00,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x01,0xe6, + 0x01,0x01,0x01,0xe6,0xd2,0x0c,0x51,0x04,0x01,0x01,0x10,0x04,0x01,0x01,0x01,0xe6, + 0x91,0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x04,0x00,0xd1,0x08,0x10,0x04,0x06,0x00, + 0x06,0x01,0x10,0x04,0x06,0x01,0x06,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x06,0xdc, + 0x06,0xe6,0x10,0x04,0x06,0x01,0x08,0x01,0x09,0xdc,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0a,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x81,0xd0,0x4f, + 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04, + 0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xa9,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01, + 0x00,0x10,0x06,0x01,0xff,0x4b,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x01,0x00,0x53, + 0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04, + 0x00,0x07,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0x95, + 0x2c,0xd4,0x18,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0xd1,0x08,0x10,0x04,0x08, + 0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b, + 0x00,0x10,0x04,0x0b,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x68,0xcf, + 0x86,0xd5,0x48,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x11,0x00,0x00,0x00,0x53,0x04,0x01,0x00,0x92, + 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x90,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x86,0x92,0xcc,0xb8,0x00,0x01,0x00,0x94,0x1a,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x94,0xcc,0xb8, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x87,0x90,0xcc,0xb8, + 0x00,0x10,0x0a,0x01,0xff,0xe2,0x87,0x94,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x87,0x92, + 0xcc,0xb8,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x06, + 0x00,0x06,0x00,0xe2,0x38,0x02,0xe1,0x3f,0x01,0xd0,0x68,0xcf,0x86,0xd5,0x3e,0x94, + 0x3a,0xd3,0x16,0x52,0x04,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0x83, + 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe2,0x88,0x88,0xcc,0xb8,0x00,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2, + 0x88,0x8b,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x24,0x93,0x20,0x52, + 0x04,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa3,0xcc,0xb8,0x00,0x01, + 0x00,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa5,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x48,0x94,0x44,0xd3,0x2e,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01, + 0x00,0x01,0xff,0xe2,0x88,0xbc,0xcc,0xb8,0x00,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0xe2,0x89,0x83,0xcc,0xb8,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, + 0x89,0x85,0xcc,0xb8,0x00,0x92,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, + 0x89,0x88,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x40,0xd3,0x1e,0x92, + 0x1a,0xd1,0x0c,0x10,0x08,0x01,0xff,0x3d,0xcc,0xb8,0x00,0x01,0x00,0x10,0x0a,0x01, + 0xff,0xe2,0x89,0xa1,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1, + 0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x89,0x8d,0xcc,0xb8,0x00,0x10,0x08,0x01, + 0xff,0x3c,0xcc,0xb8,0x00,0x01,0xff,0x3e,0xcc,0xb8,0x00,0xd3,0x30,0xd2,0x18,0x91, + 0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xa4,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xa5, + 0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xb2,0xcc,0xb8, + 0x00,0x01,0xff,0xe2,0x89,0xb3,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91,0x14,0x10, + 0x0a,0x01,0xff,0xe2,0x89,0xb6,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xb7,0xcc,0xb8, + 0x00,0x01,0x00,0x01,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x50,0x94,0x4c,0xd3,0x30,0xd2, + 0x18,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xba,0xcc,0xb8,0x00,0x01,0xff,0xe2, + 0x89,0xbb,0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x82, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x83,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91, + 0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x86,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x87, + 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x30,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa2,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x8a,0xa8,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa9,0xcc,0xb8, + 0x00,0x01,0xff,0xe2,0x8a,0xab,0xcc,0xb8,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, + 0x00,0xd4,0x5c,0xd3,0x2c,0x92,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xbc, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xbd,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2, + 0x8a,0x91,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x92,0xcc,0xb8,0x00,0x01,0x00,0xd2, + 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb2,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x8a,0xb3,0xcc,0xb8,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb4, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0xb5,0xcc,0xb8,0x00,0x01,0x00,0x93,0x0c,0x92, + 0x08,0x11,0x04,0x01,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xd1,0x64,0xd0,0x3e,0xcf, + 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x20,0x53,0x04,0x01,0x00,0x92, + 0x18,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x80,0x88,0x00,0x10,0x08,0x01, + 0xff,0xe3,0x80,0x89,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, + 0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0xd0, + 0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5, + 0x2c,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10, + 0x04,0x06,0x00,0x07,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x08, + 0x00,0x08,0x00,0x08,0x00,0x12,0x04,0x08,0x00,0x09,0x00,0xd4,0x14,0x53,0x04,0x09, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd3, + 0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10, + 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd3,0xa6,0xd2, + 0x74,0xd1,0x40,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18,0x93,0x14,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x92, + 0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x14,0x53, + 0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x06, + 0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x06, + 0x00,0x07,0x00,0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54, + 0x04,0x01,0x00,0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x06, + 0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x13,0x04,0x04, + 0x00,0x06,0x00,0xd2,0xdc,0xd1,0x48,0xd0,0x26,0xcf,0x86,0x95,0x20,0x54,0x04,0x01, + 0x00,0xd3,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x07,0x00,0x06,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x08,0x00,0x04,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x04,0x00,0x06, + 0x00,0x06,0x00,0x52,0x04,0x06,0x00,0x11,0x04,0x06,0x00,0x08,0x00,0xd0,0x5e,0xcf, + 0x86,0xd5,0x2c,0xd4,0x10,0x53,0x04,0x06,0x00,0x92,0x08,0x11,0x04,0x06,0x00,0x07, + 0x00,0x07,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x52, + 0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0a,0x00,0x0b,0x00,0xd4,0x10,0x93, + 0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd3,0x10,0x92, + 0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a, + 0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x1c,0x94, + 0x18,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b, + 0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0b,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51, + 0x04,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00,0x0b,0x00,0x0b,0x00,0xd1, + 0xa8,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x01, + 0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x53, + 0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04,0x01, + 0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x04,0x0c,0x00,0x01,0x00,0xd3, + 0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0c,0x00,0x51,0x04,0x0c, + 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x0c,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x06,0x00,0x93,0x0c,0x52,0x04,0x06,0x00,0x11, + 0x04,0x06,0x00,0x01,0x00,0x01,0x00,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01, + 0x00,0x93,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x0c, + 0x00,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0x94,0x28,0xd3,0x10,0x52,0x04,0x08, + 0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, + 0x00,0x10,0x04,0x09,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0d,0x00,0x0c, + 0x00,0x06,0x00,0x94,0x0c,0x53,0x04,0x06,0x00,0x12,0x04,0x06,0x00,0x0a,0x00,0x06, + 0x00,0xe4,0x39,0x01,0xd3,0x0c,0xd2,0x06,0xcf,0x06,0x04,0x00,0xcf,0x06,0x06,0x00, + 0xd2,0x30,0xd1,0x06,0xcf,0x06,0x06,0x00,0xd0,0x06,0xcf,0x06,0x06,0x00,0xcf,0x86, + 0x95,0x1e,0x54,0x04,0x06,0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x0e, + 0x10,0x0a,0x06,0xff,0xe2,0xab,0x9d,0xcc,0xb8,0x00,0x06,0x00,0x06,0x00,0x06,0x00, + 0xd1,0x80,0xd0,0x3a,0xcf,0x86,0xd5,0x28,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04, + 0x07,0x00,0x11,0x04,0x07,0x00,0x08,0x00,0xd3,0x08,0x12,0x04,0x08,0x00,0x09,0x00, + 0x92,0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x0c, + 0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x30, + 0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, + 0x10,0x00,0x10,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, + 0x0b,0x00,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x10,0x00,0x10,0x00,0x54,0x04, + 0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00, + 0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, + 0x11,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, + 0xd2,0x08,0x11,0x04,0x10,0x00,0x14,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x10,0x00, + 0x10,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x10,0x00,0x15,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, + 0x10,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd4,0x0c,0x53,0x04, + 0x14,0x00,0x12,0x04,0x14,0x00,0x11,0x00,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00, + 0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0xe3,0xb9,0x01,0xd2,0xac,0xd1, + 0x68,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x08,0x00,0x94,0x14,0x53,0x04,0x08,0x00,0x52, + 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x08,0x00,0xcf, + 0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x51, + 0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x52, + 0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0xd3,0x10,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0a,0x00,0x0a,0x00,0x09,0x00,0x52,0x04,0x0a, + 0x00,0x11,0x04,0x0a,0x00,0x0b,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x55, + 0x04,0x08,0x00,0xd4,0x1c,0x53,0x04,0x08,0x00,0xd2,0x0c,0x51,0x04,0x08,0x00,0x10, + 0x04,0x08,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd3, + 0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0d,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd1,0x6c,0xd0,0x2a,0xcf,0x86,0x55, + 0x04,0x08,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, + 0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x55,0x04,0x08,0x00,0xd4,0x1c,0xd3,0x0c,0x52, + 0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x08,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10, + 0x04,0x00,0x00,0x0c,0x09,0xd0,0x5a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x93, + 0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x00, + 0x00,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xcf, + 0x86,0x95,0x40,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, + 0x04,0x08,0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0x0a,0xe6,0xd2,0x9c,0xd1,0x68,0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x08, + 0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x0a,0x00,0x11,0x04,0x08,0x00,0x0a,0x00,0x54, + 0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0d, + 0x00,0x0d,0x00,0x12,0x04,0x0d,0x00,0x10,0x00,0xcf,0x86,0x95,0x30,0x94,0x2c,0xd3, + 0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x12,0x00,0x91,0x08,0x10, + 0x04,0x12,0x00,0x13,0x00,0x13,0x00,0xd2,0x08,0x11,0x04,0x13,0x00,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf, + 0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x04, + 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04, + 0x00,0x54,0x04,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x04,0x00,0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0xd5,0x14,0x54, + 0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x04,0x00,0x12,0x04,0x04,0x00,0x00,0x00,0xcf, + 0x86,0xe5,0xa6,0x05,0xe4,0x9f,0x05,0xe3,0x96,0x04,0xe2,0xe4,0x03,0xe1,0xc0,0x01, + 0xd0,0x3e,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0xda,0x01,0xe4,0x91,0x08,0x10,0x04,0x01,0xe8, + 0x01,0xde,0x01,0xe0,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0x00,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x04,0x00,0x01,0x00,0xcf,0x86, + 0xd5,0xaa,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, + 0x8b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x8d,0xe3,0x82, + 0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, + 0x8f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x91,0xe3,0x82, + 0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x93,0xe3,0x82,0x99, + 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x95,0xe3,0x82,0x99,0x00,0x01,0x00, + 0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x97,0xe3,0x82,0x99,0x00,0x01, + 0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x99,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f, + 0x10,0x0b,0x01,0xff,0xe3,0x81,0x9b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01, + 0xff,0xe3,0x81,0x9d,0xe3,0x82,0x99,0x00,0x01,0x00,0xd4,0x53,0xd3,0x3c,0xd2,0x1e, + 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x9f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10, + 0x0b,0x01,0xff,0xe3,0x81,0xa1,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x04, + 0x01,0x00,0x01,0xff,0xe3,0x81,0xa4,0xe3,0x82,0x99,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x81,0xa6,0xe3,0x82,0x99,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00, + 0x01,0xff,0xe3,0x81,0xa8,0xe3,0x82,0x99,0x00,0x01,0x00,0x01,0x00,0xd3,0x4a,0xd2, + 0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xaf,0xe3,0x82,0x99,0x00,0x01,0xff, + 0xe3,0x81,0xaf,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x81,0xb2, + 0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb2,0xe3,0x82,0x9a, + 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb5,0xe3,0x82,0x99,0x00,0x01,0xff, + 0xe3,0x81,0xb5,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x81,0xb8,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb8,0xe3, + 0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82, + 0x99,0x00,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82,0x9a,0x00,0x01,0x00,0xd0,0xee,0xcf, + 0x86,0xd5,0x42,0x54,0x04,0x01,0x00,0xd3,0x1b,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10, + 0x0b,0x01,0xff,0xe3,0x81,0x86,0xe3,0x82,0x99,0x00,0x06,0x00,0x10,0x04,0x06,0x00, + 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x08,0x10,0x04,0x01,0x08, + 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0x9d,0xe3,0x82,0x99, + 0x00,0x06,0x00,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01, + 0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x82,0xab,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xad,0xe3, + 0x82,0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x82,0xaf,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb1,0xe3, + 0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb3,0xe3,0x82, + 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb5,0xe3,0x82,0x99,0x00,0x01, + 0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb7,0xe3,0x82,0x99,0x00, + 0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb9,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1, + 0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbb,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b, + 0x01,0xff,0xe3,0x82,0xbd,0xe3,0x82,0x99,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd5,0xd4, + 0x53,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbf,0xe3,0x82, + 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x81,0xe3,0x82,0x99,0x00,0x01, + 0x00,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x84,0xe3,0x82,0x99,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x86,0xe3,0x82,0x99,0x00,0x92,0x13,0x91, + 0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x88,0xe3,0x82,0x99,0x00,0x01,0x00, + 0x01,0x00,0xd3,0x4a,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x83,0x8f,0xe3, + 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x8f,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00, + 0x01,0xff,0xe3,0x83,0x92,0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x83,0x92,0xe3,0x82,0x9a,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x95,0xe3, + 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x95,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f, + 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x98,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01, + 0xff,0xe3,0x83,0x98,0xe3,0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff, + 0xe3,0x83,0x9b,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x9b,0xe3,0x82,0x9a,0x00, + 0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x22,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b, + 0x01,0xff,0xe3,0x82,0xa6,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x83,0xaf,0xe3,0x82,0x99,0x00,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff, + 0xe3,0x83,0xb0,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0xb1,0xe3,0x82,0x99,0x00, + 0x10,0x0b,0x01,0xff,0xe3,0x83,0xb2,0xe3,0x82,0x99,0x00,0x01,0x00,0x51,0x04,0x01, + 0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0xbd,0xe3,0x82,0x99,0x00,0x06,0x00,0xd1,0x65, + 0xd0,0x46,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x91,0x08, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0x53,0x04, + 0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04, + 0x13,0x00,0x14,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x93,0x11, + 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x54, + 0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x08,0x00,0x0a,0x00,0x94, + 0x0c,0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0xd2,0xa4,0xd1, + 0x5c,0xd0,0x22,0xcf,0x86,0x95,0x1c,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x10,0x04,0x07,0x00,0x00, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x93,0x08,0x12,0x04,0x01,0x00,0x0b, + 0x00,0x0b,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x06,0x00,0x06, + 0x00,0x06,0x00,0x06,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01, + 0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, + 0x00,0x06,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5,0x10,0x94,0x0c,0x53, + 0x04,0x01,0x00,0x12,0x04,0x01,0x00,0x07,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x53, + 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x16, + 0x00,0xd1,0x30,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, + 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x01,0x00,0x01, + 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x01,0x00,0x53, + 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x07,0x00,0x54,0x04,0x01, + 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd1,0x48,0xd0,0x40,0xcf, + 0x86,0xd5,0x06,0xcf,0x06,0x04,0x00,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x2c,0xd2, + 0x06,0xcf,0x06,0x04,0x00,0xd1,0x06,0xcf,0x06,0x04,0x00,0xd0,0x1a,0xcf,0x86,0x55, + 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x07,0x00,0xcf,0x06,0x01,0x00,0xcf,0x86,0xcf, + 0x06,0x01,0x00,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe2,0x71,0x05,0xd1,0x8c,0xd0,0x08, + 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xd4,0x06, + 0xcf,0x06,0x01,0x00,0xd3,0x06,0xcf,0x06,0x01,0x00,0xd2,0x06,0xcf,0x06,0x01,0x00, + 0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x10, + 0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x08,0x00,0x08,0x00,0x53,0x04, + 0x08,0x00,0x12,0x04,0x08,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0xd3,0x08, + 0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0d,0x00, + 0x11,0x00,0x11,0x00,0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x13,0x00, + 0x13,0x00,0x94,0x14,0x53,0x04,0x13,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04, + 0x13,0x00,0x14,0x00,0x14,0x00,0x00,0x00,0xe0,0xdb,0x04,0xcf,0x86,0xe5,0xdf,0x01, + 0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x74,0xd2,0x6e,0xd1,0x06,0xcf,0x06,0x04,0x00, + 0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00, + 0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c, + 0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00,0x04,0x00,0x93,0x10,0x52,0x04, + 0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86, + 0x95,0x24,0x94,0x20,0x93,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00, + 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x10,0x04,0x04,0x00,0x00,0x00, + 0x00,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x06,0x0a,0x00,0xd2,0x84,0xd1,0x4c,0xd0,0x16, + 0xcf,0x86,0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x1c,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00, + 0x10,0x04,0x0a,0x00,0x0a,0xe6,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0d,0xe6,0x52,0x04, + 0x0d,0xe6,0x11,0x04,0x0a,0xe6,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x11,0xe6,0x0d,0xe6,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00, + 0x93,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x00,0x00,0xd1,0x40, + 0xd0,0x3a,0xcf,0x86,0xd5,0x24,0x54,0x04,0x08,0x00,0xd3,0x10,0x52,0x04,0x08,0x00, + 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0x92,0x0c,0x51,0x04,0x09,0x00, + 0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, + 0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x5e, + 0xcf,0x86,0xd5,0x28,0xd4,0x18,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0xd1,0x08, + 0x10,0x04,0x0a,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x11,0x00,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0d,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0x53,0x04,0x0c,0x00, + 0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0d,0x00,0x10,0x00,0x51,0x04,0x10,0x00, + 0x10,0x04,0x12,0x00,0x14,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00, + 0x11,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x1c, + 0x94,0x18,0x93,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x51,0x04,0x15,0x00, + 0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x00,0x00,0xd3,0x10, + 0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x92,0x0c, + 0x51,0x04,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0xe4,0xf2,0x02,0xe3, + 0x65,0x01,0xd2,0x98,0xd1,0x48,0xd0,0x36,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10, + 0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x09,0x08,0x00,0x08,0x00, + 0x08,0x00,0xd4,0x0c,0x53,0x04,0x08,0x00,0x12,0x04,0x08,0x00,0x00,0x00,0x53,0x04, + 0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, + 0x09,0x00,0x54,0x04,0x09,0x00,0x13,0x04,0x09,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06, + 0x0a,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08, + 0x10,0x04,0x0a,0x09,0x12,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00, + 0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x00,0x00,0x00,0x00, + 0x54,0x04,0x0b,0xe6,0xd3,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x11,0x00,0x14,0x00,0xd1,0x60,0xd0,0x22,0xcf,0x86, + 0x55,0x04,0x0a,0x00,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00, + 0x10,0x04,0x0a,0x00,0x0a,0xdc,0x11,0x04,0x0a,0xdc,0x0a,0x00,0x0a,0x00,0xcf,0x86, + 0xd5,0x24,0x54,0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x0a,0x00,0x0a,0x09,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04, + 0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x0b,0x07,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x20,0xd3,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00,0x52,0x04, + 0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0xd2,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x0b,0x00,0x54,0x04, + 0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0xd2,0xd0,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0a,0x00, + 0x54,0x04,0x0a,0x00,0x93,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x0a,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0a,0x00, + 0x52,0x04,0x0a,0x00,0x11,0x04,0x0a,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x92,0x08, + 0x11,0x04,0x0a,0x00,0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0x12,0x04,0x0b,0x00,0x10,0x00,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, + 0x0b,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0xe6, + 0xd1,0x08,0x10,0x04,0x0b,0xdc,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x11,0x04,0x0b,0x00,0x0b,0xe6, + 0xcf,0x86,0xd5,0x2c,0xd4,0x18,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00, + 0x0b,0xe6,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x00,0x00, + 0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00,0x54,0x04, + 0x0d,0x00,0x93,0x10,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x09, + 0x00,0x00,0x00,0x00,0xd1,0x8c,0xd0,0x72,0xcf,0x86,0xd5,0x4c,0xd4,0x30,0xd3,0x18, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00, + 0x10,0x04,0x0c,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, + 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x93,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, + 0x0c,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04, + 0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00, + 0x10,0x04,0x0c,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0x94,0x10, + 0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x10,0x00,0x15,0x00,0x00,0x00,0x11,0x00, + 0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0xd4,0x14,0x53,0x04, + 0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x00,0x00, + 0x53,0x04,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x06, + 0x02,0xff,0xff,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd1,0x76,0xd0,0x09,0xcf,0x86, + 0xcf,0x06,0x02,0xff,0xff,0xcf,0x86,0x85,0xd4,0x07,0xcf,0x06,0x02,0xff,0xff,0xd3, + 0x07,0xcf,0x06,0x02,0xff,0xff,0xd2,0x07,0xcf,0x06,0x02,0xff,0xff,0xd1,0x07,0xcf, + 0x06,0x02,0xff,0xff,0xd0,0x18,0xcf,0x86,0x55,0x05,0x02,0xff,0xff,0x94,0x0d,0x93, + 0x09,0x12,0x05,0x02,0xff,0xff,0x00,0x00,0x00,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x24, + 0x94,0x20,0xd3,0x10,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, + 0x0b,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x12,0x04,0x0b,0x00,0x00,0x00, + 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00, + 0xe4,0x9c,0x10,0xe3,0x16,0x08,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x08,0x04,0xe0, + 0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0x10,0x08,0x01, + 0xff,0xe8,0xbb,0x8a,0x00,0x01,0xff,0xe8,0xb3,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe6,0xbb,0x91,0x00,0x01,0xff,0xe4,0xb8,0xb2,0x00,0x10,0x08,0x01,0xff,0xe5, + 0x8f,0xa5,0x00,0x01,0xff,0xe9,0xbe,0x9c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe9,0xbe,0x9c,0x00,0x01,0xff,0xe5,0xa5,0x91,0x00,0x10,0x08,0x01,0xff,0xe9, + 0x87,0x91,0x00,0x01,0xff,0xe5,0x96,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, + 0xa5,0x88,0x00,0x01,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x01,0xff,0xe7,0x99,0xa9, + 0x00,0x01,0xff,0xe7,0xbe,0x85,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe8,0x98,0xbf,0x00,0x01,0xff,0xe8,0x9e,0xba,0x00,0x10,0x08,0x01,0xff,0xe8, + 0xa3,0xb8,0x00,0x01,0xff,0xe9,0x82,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, + 0xa8,0x82,0x00,0x01,0xff,0xe6,0xb4,0x9b,0x00,0x10,0x08,0x01,0xff,0xe7,0x83,0x99, + 0x00,0x01,0xff,0xe7,0x8f,0x9e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, + 0x90,0xbd,0x00,0x01,0xff,0xe9,0x85,0xaa,0x00,0x10,0x08,0x01,0xff,0xe9,0xa7,0xb1, + 0x00,0x01,0xff,0xe4,0xba,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x8d,0xb5, + 0x00,0x01,0xff,0xe6,0xac,0x84,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x9b,0x00,0x01, + 0xff,0xe8,0x98,0xad,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe9,0xb8,0x9e,0x00,0x01,0xff,0xe5,0xb5,0x90,0x00,0x10,0x08,0x01,0xff,0xe6, + 0xbf,0xab,0x00,0x01,0xff,0xe8,0x97,0x8d,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, + 0xa5,0xa4,0x00,0x01,0xff,0xe6,0x8b,0x89,0x00,0x10,0x08,0x01,0xff,0xe8,0x87,0x98, + 0x00,0x01,0xff,0xe8,0xa0,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, + 0xbb,0x8a,0x00,0x01,0xff,0xe6,0x9c,0x97,0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0xaa, + 0x00,0x01,0xff,0xe7,0x8b,0xbc,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x83,0x8e, + 0x00,0x01,0xff,0xe4,0xbe,0x86,0x00,0x10,0x08,0x01,0xff,0xe5,0x86,0xb7,0x00,0x01, + 0xff,0xe5,0x8b,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, + 0x93,0x84,0x00,0x01,0xff,0xe6,0xab,0x93,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x90, + 0x00,0x01,0xff,0xe7,0x9b,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x80,0x81, + 0x00,0x01,0xff,0xe8,0x98,0x86,0x00,0x10,0x08,0x01,0xff,0xe8,0x99,0x9c,0x00,0x01, + 0xff,0xe8,0xb7,0xaf,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x9c,0xb2, + 0x00,0x01,0xff,0xe9,0xad,0xaf,0x00,0x10,0x08,0x01,0xff,0xe9,0xb7,0xba,0x00,0x01, + 0xff,0xe7,0xa2,0x8c,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa5,0xbf,0x00,0x01, + 0xff,0xe7,0xb6,0xa0,0x00,0x10,0x08,0x01,0xff,0xe8,0x8f,0x89,0x00,0x01,0xff,0xe9, + 0x8c,0x84,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xa3,0x9f,0x00,0x01,0xff,0xe5,0xbc,0x84,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe7,0xb1,0xa0,0x00,0x01,0xff,0xe8,0x81,0xbe,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x89,0xa2,0x00,0x01,0xff,0xe7,0xa3,0x8a,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe8,0xb3,0x82,0x00,0x01,0xff,0xe9,0x9b,0xb7,0x00,0x10,0x08,0x01,0xff, + 0xe5,0xa3,0x98,0x00,0x01,0xff,0xe5,0xb1,0xa2,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xa8,0x93,0x00,0x01,0xff,0xe6,0xb7,0x9a,0x00,0x10,0x08,0x01,0xff,0xe6,0xbc, + 0x8f,0x00,0x01,0xff,0xe7,0xb4,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9,0x99,0x8b,0x00,0x10,0x08,0x01,0xff, + 0xe5,0x8b,0x92,0x00,0x01,0xff,0xe8,0x82,0x8b,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0x87,0x9c,0x00,0x01,0xff,0xe5,0x87,0x8c,0x00,0x10,0x08,0x01,0xff,0xe7,0xa8, + 0x9c,0x00,0x01,0xff,0xe7,0xb6,0xbe,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe8,0x8f,0xb1,0x00,0x01,0xff,0xe9,0x99,0xb5,0x00,0x10,0x08,0x01,0xff,0xe8,0xae, + 0x80,0x00,0x01,0xff,0xe6,0x8b,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xa8, + 0x82,0x00,0x01,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x01,0xff,0xe4,0xb8,0xb9,0x00, + 0x01,0xff,0xe5,0xaf,0xa7,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe6,0x80,0x92,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x95,0xb0,0x00,0x01,0xff,0xe5,0x8c,0x97,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe7,0xa3,0xbb,0x00,0x01,0xff,0xe4,0xbe,0xbf,0x00,0x10,0x08,0x01,0xff,0xe5,0xbe, + 0xa9,0x00,0x01,0xff,0xe4,0xb8,0x8d,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xb3,0x8c,0x00,0x01,0xff,0xe6,0x95,0xb8,0x00,0x10,0x08,0x01,0xff,0xe7,0xb4, + 0xa2,0x00,0x01,0xff,0xe5,0x8f,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0xa1, + 0x9e,0x00,0x01,0xff,0xe7,0x9c,0x81,0x00,0x10,0x08,0x01,0xff,0xe8,0x91,0x89,0x00, + 0x01,0xff,0xe8,0xaa,0xaa,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xae,0xba,0x00,0x01,0xff,0xe8,0xbe,0xb0,0x00,0x10,0x08,0x01,0xff,0xe6,0xb2, + 0x88,0x00,0x01,0xff,0xe6,0x8b,0xbe,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x8b, + 0xa5,0x00,0x01,0xff,0xe6,0x8e,0xa0,0x00,0x10,0x08,0x01,0xff,0xe7,0x95,0xa5,0x00, + 0x01,0xff,0xe4,0xba,0xae,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, + 0xa9,0x00,0x01,0xff,0xe5,0x87,0x89,0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0x81,0x00, + 0x01,0xff,0xe7,0xb3,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x89,0xaf,0x00, + 0x01,0xff,0xe8,0xab,0x92,0x00,0x10,0x08,0x01,0xff,0xe9,0x87,0x8f,0x00,0x01,0xff, + 0xe5,0x8b,0xb5,0x00,0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x91,0x82,0x00,0x01,0xff,0xe5,0xa5, + 0xb3,0x00,0x10,0x08,0x01,0xff,0xe5,0xbb,0xac,0x00,0x01,0xff,0xe6,0x97,0x85,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xbf,0xbe,0x00,0x01,0xff,0xe7,0xa4,0xaa,0x00, + 0x10,0x08,0x01,0xff,0xe9,0x96,0xad,0x00,0x01,0xff,0xe9,0xa9,0xaa,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xba,0x97,0x00,0x01,0xff,0xe9,0xbb,0x8e,0x00, + 0x10,0x08,0x01,0xff,0xe5,0x8a,0x9b,0x00,0x01,0xff,0xe6,0x9b,0x86,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe6,0xad,0xb7,0x00,0x01,0xff,0xe8,0xbd,0xa2,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xb9,0xb4,0x00,0x01,0xff,0xe6,0x86,0x90,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x88,0x80,0x00,0x01,0xff,0xe6,0x92,0x9a,0x00, + 0x10,0x08,0x01,0xff,0xe6,0xbc,0xa3,0x00,0x01,0xff,0xe7,0x85,0x89,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe7,0x92,0x89,0x00,0x01,0xff,0xe7,0xa7,0x8a,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xb7,0xb4,0x00,0x01,0xff,0xe8,0x81,0xaf,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe8,0xbc,0xa6,0x00,0x01,0xff,0xe8,0x93,0xae,0x00,0x10,0x08, + 0x01,0xff,0xe9,0x80,0xa3,0x00,0x01,0xff,0xe9,0x8d,0x8a,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe5,0x88,0x97,0x00,0x01,0xff,0xe5,0x8a,0xa3,0x00,0x10,0x08,0x01,0xff, + 0xe5,0x92,0xbd,0x00,0x01,0xff,0xe7,0x83,0x88,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0x82,0x00,0x01,0xff,0xe8,0xaa,0xaa,0x00, + 0x10,0x08,0x01,0xff,0xe5,0xbb,0x89,0x00,0x01,0xff,0xe5,0xbf,0xb5,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe6,0x8d,0xbb,0x00,0x01,0xff,0xe6,0xae,0xae,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xb0,0xbe,0x00,0x01,0xff,0xe7,0x8d,0xb5,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe4,0xbb,0xa4,0x00,0x01,0xff,0xe5,0x9b,0xb9,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xaf,0xa7,0x00,0x01,0xff,0xe5,0xb6,0xba,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe6,0x80,0x9c,0x00,0x01,0xff,0xe7,0x8e,0xb2,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x91,0xa9,0x00,0x01,0xff,0xe7,0xbe,0x9a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe8,0x81,0x86,0x00,0x01,0xff,0xe9,0x88,0xb4,0x00,0x10,0x08, + 0x01,0xff,0xe9,0x9b,0xb6,0x00,0x01,0xff,0xe9,0x9d,0x88,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe9,0xa0,0x98,0x00,0x01,0xff,0xe4,0xbe,0x8b,0x00,0x10,0x08,0x01,0xff, + 0xe7,0xa6,0xae,0x00,0x01,0xff,0xe9,0x86,0xb4,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe9,0x9a,0xb8,0x00,0x01,0xff,0xe6,0x83,0xa1,0x00,0x10,0x08,0x01,0xff, + 0xe4,0xba,0x86,0x00,0x01,0xff,0xe5,0x83,0x9a,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0xaf,0xae,0x00,0x01,0xff,0xe5,0xb0,0xbf,0x00,0x10,0x08,0x01,0xff,0xe6,0x96, + 0x99,0x00,0x01,0xff,0xe6,0xa8,0x82,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0x87,0x8e,0x00,0x01,0xff,0xe7, + 0x99,0x82,0x00,0x10,0x08,0x01,0xff,0xe8,0x93,0xbc,0x00,0x01,0xff,0xe9,0x81,0xbc, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xbe,0x8d,0x00,0x01,0xff,0xe6,0x9a,0x88, + 0x00,0x10,0x08,0x01,0xff,0xe9,0x98,0xae,0x00,0x01,0xff,0xe5,0x8a,0x89,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x9d,0xbb,0x00,0x01,0xff,0xe6,0x9f,0xb3, + 0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0x81,0x00,0x01,0xff,0xe6,0xba,0x9c,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x89,0x00,0x01,0xff,0xe7,0x95,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe7,0xa1,0xab,0x00,0x01,0xff,0xe7,0xb4,0x90,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xa1,0x9e,0x00,0x01,0xff,0xe5,0x85,0xad, + 0x00,0x10,0x08,0x01,0xff,0xe6,0x88,0xae,0x00,0x01,0xff,0xe9,0x99,0xb8,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe5,0x80,0xab,0x00,0x01,0xff,0xe5,0xb4,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe6,0xb7,0xaa,0x00,0x01,0xff,0xe8,0xbc,0xaa,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe5,0xbe,0x8b,0x00,0x01,0xff,0xe6,0x85,0x84,0x00,0x10, + 0x08,0x01,0xff,0xe6,0xa0,0x97,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe9,0x9a,0x86,0x00,0x01,0xff,0xe5,0x88,0xa9,0x00,0x10,0x08,0x01, + 0xff,0xe5,0x90,0x8f,0x00,0x01,0xff,0xe5,0xb1,0xa5,0x00,0xd4,0x80,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x98,0x93,0x00,0x01,0xff,0xe6,0x9d,0x8e, + 0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0xa8,0x00,0x01,0xff,0xe6,0xb3,0xa5,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x86,0x00,0x01,0xff,0xe7,0x97,0xa2,0x00,0x10, + 0x08,0x01,0xff,0xe7,0xbd,0xb9,0x00,0x01,0xff,0xe8,0xa3,0x8f,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0xa1,0x00,0x01,0xff,0xe9,0x87,0x8c,0x00,0x10, + 0x08,0x01,0xff,0xe9,0x9b,0xa2,0x00,0x01,0xff,0xe5,0x8c,0xbf,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe6,0xba,0xba,0x00,0x01,0xff,0xe5,0x90,0x9d,0x00,0x10,0x08,0x01, + 0xff,0xe7,0x87,0x90,0x00,0x01,0xff,0xe7,0x92,0x98,0x00,0xd3,0x40,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe8,0x97,0xba,0x00,0x01,0xff,0xe9,0x9a,0xa3,0x00,0x10, + 0x08,0x01,0xff,0xe9,0xb1,0x97,0x00,0x01,0xff,0xe9,0xba,0x9f,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe6,0x9e,0x97,0x00,0x01,0xff,0xe6,0xb7,0x8b,0x00,0x10,0x08,0x01, + 0xff,0xe8,0x87,0xa8,0x00,0x01,0xff,0xe7,0xab,0x8b,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe7,0xac,0xa0,0x00,0x01,0xff,0xe7,0xb2,0x92,0x00,0x10,0x08,0x01, + 0xff,0xe7,0x8b,0x80,0x00,0x01,0xff,0xe7,0x82,0x99,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe8,0xad,0x98,0x00,0x01,0xff,0xe4,0xbb,0x80,0x00,0x10,0x08,0x01,0xff,0xe8, + 0x8c,0xb6,0x00,0x01,0xff,0xe5,0x88,0xba,0x00,0xe2,0xad,0x06,0xe1,0xc4,0x03,0xe0, + 0xcb,0x01,0xcf,0x86,0xd5,0xe4,0xd4,0x74,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe5,0x88,0x87,0x00,0x01,0xff,0xe5,0xba,0xa6,0x00,0x10,0x08,0x01,0xff, + 0xe6,0x8b,0x93,0x00,0x01,0xff,0xe7,0xb3,0x96,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0xae,0x85,0x00,0x01,0xff,0xe6,0xb4,0x9e,0x00,0x10,0x08,0x01,0xff,0xe6,0x9a, + 0xb4,0x00,0x01,0xff,0xe8,0xbc,0xbb,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe8,0xa1,0x8c,0x00,0x01,0xff,0xe9,0x99,0x8d,0x00,0x10,0x08,0x01,0xff,0xe8,0xa6, + 0x8b,0x00,0x01,0xff,0xe5,0xbb,0x93,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, + 0x80,0x00,0x01,0xff,0xe5,0x97,0x80,0x00,0x01,0x00,0xd3,0x34,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0xe5,0xa1,0x9a,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe6,0x99, + 0xb4,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe5,0x87,0x9e,0x00, + 0x10,0x08,0x01,0xff,0xe7,0x8c,0xaa,0x00,0x01,0xff,0xe7,0x9b,0x8a,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa4,0xbc,0x00,0x01,0xff,0xe7,0xa5,0x9e,0x00, + 0x10,0x08,0x01,0xff,0xe7,0xa5,0xa5,0x00,0x01,0xff,0xe7,0xa6,0x8f,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0x9d,0x96,0x00,0x01,0xff,0xe7,0xb2,0xbe,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xbe,0xbd,0x00,0x01,0x00,0xd4,0x64,0xd3,0x30,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0xe8,0x98,0x92,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe8,0xab, + 0xb8,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe9,0x80,0xb8,0x00, + 0x10,0x08,0x01,0xff,0xe9,0x83,0xbd,0x00,0x01,0x00,0xd2,0x14,0x51,0x04,0x01,0x00, + 0x10,0x08,0x01,0xff,0xe9,0xa3,0xaf,0x00,0x01,0xff,0xe9,0xa3,0xbc,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0xa4,0xa8,0x00,0x01,0xff,0xe9,0xb6,0xb4,0x00,0x10,0x08, + 0x0d,0xff,0xe9,0x83,0x9e,0x00,0x0d,0xff,0xe9,0x9a,0xb7,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x06,0xff,0xe4,0xbe,0xae,0x00,0x06,0xff,0xe5,0x83,0xa7,0x00, + 0x10,0x08,0x06,0xff,0xe5,0x85,0x8d,0x00,0x06,0xff,0xe5,0x8b,0x89,0x00,0xd1,0x10, + 0x10,0x08,0x06,0xff,0xe5,0x8b,0xa4,0x00,0x06,0xff,0xe5,0x8d,0x91,0x00,0x10,0x08, + 0x06,0xff,0xe5,0x96,0x9d,0x00,0x06,0xff,0xe5,0x98,0x86,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x06,0xff,0xe5,0x99,0xa8,0x00,0x06,0xff,0xe5,0xa1,0x80,0x00,0x10,0x08, + 0x06,0xff,0xe5,0xa2,0xa8,0x00,0x06,0xff,0xe5,0xb1,0xa4,0x00,0xd1,0x10,0x10,0x08, + 0x06,0xff,0xe5,0xb1,0xae,0x00,0x06,0xff,0xe6,0x82,0x94,0x00,0x10,0x08,0x06,0xff, + 0xe6,0x85,0xa8,0x00,0x06,0xff,0xe6,0x86,0x8e,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4, + 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0x87,0xb2,0x00,0x06, + 0xff,0xe6,0x95,0x8f,0x00,0x10,0x08,0x06,0xff,0xe6,0x97,0xa2,0x00,0x06,0xff,0xe6, + 0x9a,0x91,0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0xa2,0x85,0x00,0x06,0xff,0xe6, + 0xb5,0xb7,0x00,0x10,0x08,0x06,0xff,0xe6,0xb8,0x9a,0x00,0x06,0xff,0xe6,0xbc,0xa2, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0x85,0xae,0x00,0x06,0xff,0xe7, + 0x88,0xab,0x00,0x10,0x08,0x06,0xff,0xe7,0x90,0xa2,0x00,0x06,0xff,0xe7,0xa2,0x91, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa4,0xbe,0x00,0x06,0xff,0xe7,0xa5,0x89, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xa5,0x88,0x00,0x06,0xff,0xe7,0xa5,0x90,0x00,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa5,0x96,0x00,0x06,0xff,0xe7, + 0xa5,0x9d,0x00,0x10,0x08,0x06,0xff,0xe7,0xa6,0x8d,0x00,0x06,0xff,0xe7,0xa6,0x8e, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa9,0x80,0x00,0x06,0xff,0xe7,0xaa,0x81, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xaf,0x80,0x00,0x06,0xff,0xe7,0xb7,0xb4,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xb8,0x89,0x00,0x06,0xff,0xe7,0xb9,0x81, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xbd,0xb2,0x00,0x06,0xff,0xe8,0x80,0x85,0x00,0xd1, + 0x10,0x10,0x08,0x06,0xff,0xe8,0x87,0xad,0x00,0x06,0xff,0xe8,0x89,0xb9,0x00,0x10, + 0x08,0x06,0xff,0xe8,0x89,0xb9,0x00,0x06,0xff,0xe8,0x91,0x97,0x00,0xd4,0x75,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xa4,0x90,0x00,0x06,0xff,0xe8, + 0xa6,0x96,0x00,0x10,0x08,0x06,0xff,0xe8,0xac,0x81,0x00,0x06,0xff,0xe8,0xac,0xb9, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xb3,0x93,0x00,0x06,0xff,0xe8,0xb4,0x88, + 0x00,0x10,0x08,0x06,0xff,0xe8,0xbe,0xb6,0x00,0x06,0xff,0xe9,0x80,0xb8,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe9,0x9b,0xa3,0x00,0x06,0xff,0xe9,0x9f,0xbf, + 0x00,0x10,0x08,0x06,0xff,0xe9,0xa0,0xbb,0x00,0x0b,0xff,0xe6,0x81,0xb5,0x00,0x91, + 0x11,0x10,0x09,0x0b,0xff,0xf0,0xa4,0x8b,0xae,0x00,0x0b,0xff,0xe8,0x88,0x98,0x00, + 0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe4,0xb8,0xa6,0x00, + 0x08,0xff,0xe5,0x86,0xb5,0x00,0x10,0x08,0x08,0xff,0xe5,0x85,0xa8,0x00,0x08,0xff, + 0xe4,0xbe,0x80,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x85,0x85,0x00,0x08,0xff, + 0xe5,0x86,0x80,0x00,0x10,0x08,0x08,0xff,0xe5,0x8b,0x87,0x00,0x08,0xff,0xe5,0x8b, + 0xba,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x96,0x9d,0x00,0x08,0xff, + 0xe5,0x95,0x95,0x00,0x10,0x08,0x08,0xff,0xe5,0x96,0x99,0x00,0x08,0xff,0xe5,0x97, + 0xa2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0xa1,0x9a,0x00,0x08,0xff,0xe5,0xa2, + 0xb3,0x00,0x10,0x08,0x08,0xff,0xe5,0xa5,0x84,0x00,0x08,0xff,0xe5,0xa5,0x94,0x00, + 0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe5,0xa9,0xa2,0x00,0x08,0xff,0xe5,0xac,0xa8,0x00,0x10,0x08, + 0x08,0xff,0xe5,0xbb,0x92,0x00,0x08,0xff,0xe5,0xbb,0x99,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe5,0xbd,0xa9,0x00,0x08,0xff,0xe5,0xbe,0xad,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x83,0x98,0x00,0x08,0xff,0xe6,0x85,0x8e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe6,0x84,0x88,0x00,0x08,0xff,0xe6,0x86,0x8e,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x85,0xa0,0x00,0x08,0xff,0xe6,0x87,0xb2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0x88,0xb4,0x00,0x08,0xff,0xe6,0x8f,0x84,0x00,0x10,0x08,0x08,0xff,0xe6,0x90, + 0x9c,0x00,0x08,0xff,0xe6,0x91,0x92,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe6,0x95,0x96,0x00,0x08,0xff,0xe6,0x99,0xb4,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x9c,0x97,0x00,0x08,0xff,0xe6,0x9c,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0x9d,0x96,0x00,0x08,0xff,0xe6,0xad,0xb9,0x00,0x10,0x08,0x08,0xff,0xe6,0xae, + 0xba,0x00,0x08,0xff,0xe6,0xb5,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0xbb,0x9b,0x00,0x08,0xff,0xe6,0xbb,0x8b,0x00,0x10,0x08,0x08,0xff,0xe6,0xbc, + 0xa2,0x00,0x08,0xff,0xe7,0x80,0x9e,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0x85, + 0xae,0x00,0x08,0xff,0xe7,0x9e,0xa7,0x00,0x10,0x08,0x08,0xff,0xe7,0x88,0xb5,0x00, + 0x08,0xff,0xe7,0x8a,0xaf,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe7,0x8c,0xaa,0x00,0x08,0xff,0xe7,0x91,0xb1,0x00,0x10,0x08,0x08,0xff, + 0xe7,0x94,0x86,0x00,0x08,0xff,0xe7,0x94,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0x98,0x9d,0x00,0x08,0xff,0xe7,0x98,0x9f,0x00,0x10,0x08,0x08,0xff,0xe7,0x9b, + 0x8a,0x00,0x08,0xff,0xe7,0x9b,0x9b,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0x9b,0xb4,0x00,0x08,0xff,0xe7,0x9d,0x8a,0x00,0x10,0x08,0x08,0xff,0xe7,0x9d, + 0x80,0x00,0x08,0xff,0xe7,0xa3,0x8c,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0xaa, + 0xb1,0x00,0x08,0xff,0xe7,0xaf,0x80,0x00,0x10,0x08,0x08,0xff,0xe7,0xb1,0xbb,0x00, + 0x08,0xff,0xe7,0xb5,0x9b,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0xb7,0xb4,0x00,0x08,0xff,0xe7,0xbc,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0x80, + 0x85,0x00,0x08,0xff,0xe8,0x8d,0x92,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0x8f, + 0xaf,0x00,0x08,0xff,0xe8,0x9d,0xb9,0x00,0x10,0x08,0x08,0xff,0xe8,0xa5,0x81,0x00, + 0x08,0xff,0xe8,0xa6,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xa6, + 0x96,0x00,0x08,0xff,0xe8,0xaa,0xbf,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xb8,0x00, + 0x08,0xff,0xe8,0xab,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xac,0x81,0x00, + 0x08,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xad,0x00,0x08,0xff, + 0xe8,0xac,0xb9,0x00,0xcf,0x86,0x95,0xde,0xd4,0x81,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe8,0xae,0x8a,0x00,0x08,0xff,0xe8,0xb4,0x88,0x00,0x10,0x08, + 0x08,0xff,0xe8,0xbc,0xb8,0x00,0x08,0xff,0xe9,0x81,0xb2,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe9,0x86,0x99,0x00,0x08,0xff,0xe9,0x89,0xb6,0x00,0x10,0x08,0x08,0xff, + 0xe9,0x99,0xbc,0x00,0x08,0xff,0xe9,0x9b,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe9,0x9d,0x96,0x00,0x08,0xff,0xe9,0x9f,0x9b,0x00,0x10,0x08,0x08,0xff, + 0xe9,0x9f,0xbf,0x00,0x08,0xff,0xe9,0xa0,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe9,0xa0,0xbb,0x00,0x08,0xff,0xe9,0xac,0x92,0x00,0x10,0x08,0x08,0xff,0xe9,0xbe, + 0x9c,0x00,0x08,0xff,0xf0,0xa2,0xa1,0x8a,0x00,0xd3,0x45,0xd2,0x22,0xd1,0x12,0x10, + 0x09,0x08,0xff,0xf0,0xa2,0xa1,0x84,0x00,0x08,0xff,0xf0,0xa3,0x8f,0x95,0x00,0x10, + 0x08,0x08,0xff,0xe3,0xae,0x9d,0x00,0x08,0xff,0xe4,0x80,0x98,0x00,0xd1,0x11,0x10, + 0x08,0x08,0xff,0xe4,0x80,0xb9,0x00,0x08,0xff,0xf0,0xa5,0x89,0x89,0x00,0x10,0x09, + 0x08,0xff,0xf0,0xa5,0xb3,0x90,0x00,0x08,0xff,0xf0,0xa7,0xbb,0x93,0x00,0x92,0x14, + 0x91,0x10,0x10,0x08,0x08,0xff,0xe9,0xbd,0x83,0x00,0x08,0xff,0xe9,0xbe,0x8e,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xe1,0x94,0x01,0xe0,0x08,0x01,0xcf,0x86,0xd5,0x42, + 0xd4,0x14,0x93,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x00,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x00,0x00,0x04,0xff, + 0xd7,0x99,0xd6,0xb4,0x00,0x10,0x04,0x01,0x1a,0x01,0xff,0xd7,0xb2,0xd6,0xb7,0x00, + 0xd4,0x42,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, + 0xd7,0xa9,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd7,0x82,0x00,0xd1,0x16,0x10,0x0b, + 0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7, + 0x82,0x00,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xb7,0x00,0x01,0xff,0xd7,0x90,0xd6, + 0xb8,0x00,0xd3,0x43,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xbc, + 0x00,0x01,0xff,0xd7,0x91,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x92,0xd6,0xbc, + 0x00,0x01,0xff,0xd7,0x93,0xd6,0xbc,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x94, + 0xd6,0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x96, + 0xd6,0xbc,0x00,0x00,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x98,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x99,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x9a,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7, + 0x9c,0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0x9e,0xd6,0xbc,0x00,0x00, + 0x00,0xcf,0x86,0x95,0x85,0x94,0x81,0xd3,0x3e,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xd7,0xa0,0xd6,0xbc,0x00,0x01,0xff,0xd7,0xa1,0xd6,0xbc,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0xd7,0xa3,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7,0xa4, + 0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0xa6,0xd6,0xbc,0x00,0x01,0xff, + 0xd7,0xa7,0xd6,0xbc,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0xa8,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0xaa,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xb9,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7, + 0x91,0xd6,0xbf,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbf,0x00,0x10,0x09,0x01,0xff,0xd7, + 0xa4,0xd6,0xbf,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0xd4,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, + 0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd3,0x5a,0xd2,0x06, + 0xcf,0x06,0x01,0x00,0xd1,0x14,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x95,0x08, + 0x14,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04, + 0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x01,0x00,0x00,0x00,0x05,0x00, + 0x54,0x04,0x05,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04, + 0x06,0x00,0x07,0x00,0x00,0x00,0xd2,0xce,0xd1,0xa5,0xd0,0x37,0xcf,0x86,0xd5,0x15, + 0x54,0x05,0x06,0xff,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00,0x00, + 0x00,0x00,0x00,0x94,0x1c,0xd3,0x10,0x52,0x04,0x01,0xe6,0x51,0x04,0x0a,0xe6,0x10, + 0x04,0x0a,0xe6,0x10,0xdc,0x52,0x04,0x10,0xdc,0x11,0x04,0x10,0xdc,0x11,0xe6,0x01, + 0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10, + 0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00,0x07,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0xd3,0x10,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x12,0x04,0x01, + 0x00,0x00,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06, + 0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x06,0xcf, + 0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01, + 0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0x00,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00, + 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x94,0x14, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xd0,0x2f,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x15,0x93,0x11, + 0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x18,0xd3,0x0c,0x92,0x08,0x11,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, + 0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x00, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd4,0x20,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x53,0x05,0x00, + 0xff,0x00,0xd2,0x0d,0x91,0x09,0x10,0x05,0x00,0xff,0x00,0x04,0x00,0x04,0x00,0x91, + 0x08,0x10,0x04,0x03,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x46,0x3e,0xe1,0x1f,0x3b, + 0xe0,0x9c,0x39,0xcf,0x86,0xe5,0x40,0x26,0xc4,0xe3,0x16,0x14,0xe2,0xef,0x11,0xe1, + 0xd0,0x10,0xe0,0x60,0x07,0xcf,0x86,0xe5,0x53,0x03,0xe4,0x4c,0x02,0xe3,0x3d,0x01, + 0xd2,0x94,0xd1,0x70,0xd0,0x4a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x07,0x00, + 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00, + 0xd4,0x14,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, + 0x00,0x00,0x07,0x00,0x53,0x04,0x07,0x00,0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04, + 0x07,0x00,0x00,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86, + 0x95,0x20,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00, + 0x00,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00, + 0x00,0x00,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x54,0x04, + 0x07,0x00,0x53,0x04,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, + 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x94,0x1c,0x93,0x18, + 0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00,0x00,0x51,0x04,0x00,0x00, + 0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0x93,0x10, + 0x52,0x04,0x07,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00, + 0xcf,0x06,0x08,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x20,0x53,0x04,0x08,0x00, + 0xd2,0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x10,0x00,0xd1,0x08,0x10,0x04, + 0x10,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x12,0x04, + 0x0a,0x00,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04, + 0x00,0x00,0x0a,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00, + 0x91,0x08,0x10,0x04,0x0a,0x00,0x0a,0xdc,0x00,0x00,0xd2,0x5e,0xd1,0x06,0xcf,0x06, + 0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00, + 0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x0a,0x00, + 0xcf,0x86,0xd5,0x18,0x54,0x04,0x0a,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x10,0xdc,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04, + 0x10,0x00,0x12,0x04,0x10,0x00,0x00,0x00,0xd1,0x70,0xd0,0x36,0xcf,0x86,0xd5,0x18, + 0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00, + 0x10,0x04,0x05,0x00,0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x05,0x00,0x00,0x00, + 0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x05,0x00, + 0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x05,0x00,0x92,0x0c,0x51,0x04,0x05,0x00, + 0x10,0x04,0x05,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c, + 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x10,0xe6,0x92,0x0c,0x51,0x04,0x10,0xe6, + 0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, + 0x00,0x00,0x07,0x00,0x08,0x00,0xcf,0x86,0x95,0x1c,0xd4,0x0c,0x93,0x08,0x12,0x04, + 0x08,0x00,0x00,0x00,0x08,0x00,0x93,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0xba,0xd2,0x80,0xd1,0x34,0xd0,0x1a,0xcf,0x86, + 0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00, + 0x07,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x05,0x00, + 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd0,0x2a, + 0xcf,0x86,0xd5,0x14,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, + 0x11,0x04,0x07,0x00,0x00,0x00,0x94,0x10,0x53,0x04,0x07,0x00,0x92,0x08,0x11,0x04, + 0x07,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xcf,0x86,0xd5,0x10,0x54,0x04,0x12,0x00, + 0x93,0x08,0x12,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x54,0x04,0x12,0x00,0x53,0x04, + 0x12,0x00,0x12,0x04,0x12,0x00,0x00,0x00,0xd1,0x34,0xd0,0x12,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x10,0x00,0x00,0x00,0x52,0x04,0x00,0x00, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, + 0xd2,0x06,0xcf,0x06,0x10,0x00,0xd1,0x40,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00, + 0x54,0x04,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c, + 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x08,0x13,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe4,0xce,0x02,0xe3,0x45,0x01, + 0xd2,0xd0,0xd1,0x70,0xd0,0x52,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x52,0x04, + 0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, + 0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0xd3,0x10,0x52,0x04, + 0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xd2,0x0c,0x91,0x08, + 0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x00,0x00, + 0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x0b,0x00,0x93,0x10, + 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, + 0x10,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, + 0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14, + 0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00, + 0x10,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x11,0x00,0xd3,0x14, + 0xd2,0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x04,0x11,0x00, + 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00, + 0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x1c,0x54,0x04,0x09,0x00,0x53,0x04,0x09,0x00, + 0xd2,0x08,0x11,0x04,0x09,0x00,0x0b,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, + 0x09,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0xcf,0x06,0x00,0x00, + 0xd0,0x1a,0xcf,0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d,0x00, + 0x52,0x04,0x00,0x00,0x11,0x04,0x11,0x00,0x0d,0x00,0xcf,0x86,0x95,0x14,0x54,0x04, + 0x11,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00, + 0x11,0x00,0xd2,0xec,0xd1,0xa4,0xd0,0x76,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x14, + 0x52,0x04,0x08,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x10,0x04,0x08,0x00, + 0x00,0x00,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x08,0x00,0x08,0xdc,0x10,0x04, + 0x08,0x00,0x08,0xe6,0xd3,0x10,0x52,0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x08,0x00,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x08,0x00, + 0x08,0x00,0x54,0x04,0x08,0x00,0xd3,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x14,0x00, + 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe6,0x08,0x01,0x10,0x04,0x08,0xdc, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x08,0x09,0xcf,0x86,0x95,0x28, + 0xd4,0x14,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x08,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x10,0x00, + 0x00,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x24,0xd3,0x14,0x52,0x04,0x10,0x00, + 0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0xe6,0x10,0x04,0x10,0xdc,0x00,0x00,0x92,0x0c, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04, + 0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0x54, + 0xd0,0x26,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04, + 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, + 0x0b,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x0b,0x00,0x93,0x0c, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00, + 0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00, + 0x0b,0x00,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x54,0x04,0x10,0x00,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, + 0x10,0x00,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14, + 0x53,0x04,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00, + 0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x96,0xd2,0x68,0xd1,0x24,0xd0,0x06, + 0xcf,0x06,0x0b,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0x93,0x10,0x92,0x0c, + 0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, + 0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x11,0x00, + 0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00, + 0x11,0x00,0xd1,0x28,0xd0,0x22,0xcf,0x86,0x55,0x04,0x14,0x00,0xd4,0x0c,0x93,0x08, + 0x12,0x04,0x14,0x00,0x14,0xe6,0x00,0x00,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04, + 0x14,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a, + 0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, + 0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x58,0xd0,0x12,0xcf,0x86,0x55,0x04, + 0x14,0x00,0x94,0x08,0x13,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0xcf,0x86,0x95,0x40, + 0xd4,0x24,0xd3,0x0c,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x14,0xdc,0xd2,0x0c, + 0x51,0x04,0x14,0xe6,0x10,0x04,0x14,0xe6,0x14,0xdc,0x91,0x08,0x10,0x04,0x14,0xe6, + 0x14,0xdc,0x14,0xdc,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0xdc,0x14,0x00, + 0x14,0x00,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x15,0x00, + 0x93,0x10,0x52,0x04,0x15,0x00,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x86,0xe5,0x0f,0x06,0xe4,0xf8,0x03,0xe3,0x02,0x02,0xd2,0xfb,0xd1, + 0x4c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52, + 0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x09,0x0c,0x00,0x52,0x04,0x0c, + 0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x0c, + 0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00, + 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x09,0xd0,0x69,0xcf,0x86,0xd5, + 0x32,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0xd2,0x15,0x51,0x04,0x0b,0x00,0x10, + 0x0d,0x0b,0xff,0xf0,0x91,0x82,0x99,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x91,0x11, + 0x10,0x0d,0x0b,0xff,0xf0,0x91,0x82,0x9b,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x0b, + 0x00,0xd4,0x1d,0x53,0x04,0x0b,0x00,0x92,0x15,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, + 0x00,0x0b,0xff,0xf0,0x91,0x82,0xa5,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x53,0x04, + 0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x10,0x04,0x0b,0x07, + 0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x0b,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x14,0x00,0x00,0x00,0x0d,0x00,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x08, + 0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0xd1,0x96,0xd0,0x5c,0xcf,0x86,0xd5,0x18, + 0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x0d,0x00, + 0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x26,0x53,0x04,0x0d,0x00,0x52,0x04,0x0d,0x00, + 0x51,0x04,0x0d,0x00,0x10,0x0d,0x0d,0xff,0xf0,0x91,0x84,0xb1,0xf0,0x91,0x84,0xa7, + 0x00,0x0d,0xff,0xf0,0x91,0x84,0xb2,0xf0,0x91,0x84,0xa7,0x00,0x93,0x18,0xd2,0x0c, + 0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x0d,0x09,0x91,0x08,0x10,0x04,0x0d,0x09, + 0x00,0x00,0x0d,0x00,0x0d,0x00,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04, + 0x0d,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x10,0x00, + 0x54,0x04,0x10,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00, + 0x10,0x07,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x09,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x0d,0x00,0x11,0x00,0x10,0x04,0x11,0x07,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00, + 0x10,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04, + 0x10,0x00,0x11,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xc8,0xd1,0x48, + 0xd0,0x42,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, + 0x10,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x54,0x04,0x10,0x00, + 0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x09,0x10,0x04, + 0x10,0x07,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xd0,0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10, + 0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04, + 0x00,0x00,0x11,0x00,0x53,0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00, + 0x10,0x04,0x00,0x00,0x11,0x00,0x94,0x10,0x53,0x04,0x11,0x00,0x92,0x08,0x11,0x04, + 0x11,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x18, + 0x53,0x04,0x10,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x07,0x10,0x04, + 0x10,0x09,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00, + 0x00,0x00,0x00,0x00,0xe1,0x27,0x01,0xd0,0x8a,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3, + 0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x10,0x00,0x10,0x00,0x91,0x08,0x10, + 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10, + 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xd4, + 0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10, + 0x00,0x10,0x04,0x00,0x00,0x14,0x07,0x91,0x08,0x10,0x04,0x10,0x07,0x10,0x00,0x10, + 0x00,0xcf,0x86,0xd5,0x6a,0xd4,0x42,0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0xd2,0x19,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0, + 0x91,0x8c,0xbe,0x00,0x91,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0,0x91, + 0x8d,0x97,0x00,0x10,0x09,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11, + 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x52, + 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0xd3, + 0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0xe6,0x52,0x04,0x10,0xe6,0x91, + 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x10,0xe6,0x91, + 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3, + 0x30,0x01,0xd2,0xb7,0xd1,0x48,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95,0x3c, + 0xd4,0x1c,0x93,0x18,0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x09,0x12,0x00, + 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x07,0x12,0x00,0x12,0x00,0x53,0x04,0x12,0x00, + 0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x00,0x00,0x12,0x00,0xd1,0x08,0x10,0x04, + 0x00,0x00,0x12,0x00,0x10,0x04,0x14,0xe6,0x15,0x00,0x00,0x00,0xd0,0x45,0xcf,0x86, + 0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0xd2,0x15,0x51,0x04, + 0x10,0x00,0x10,0x04,0x10,0x00,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xba, + 0x00,0xd1,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xb0,0x00, + 0x10,0x00,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xbd,0x00,0x10, + 0x00,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, + 0x04,0x10,0x09,0x10,0x07,0x10,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0, + 0x40,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c,0x52,0x04,0x10, + 0x00,0x11,0x04,0x10,0x00,0x00,0x00,0xd2,0x1e,0x51,0x04,0x10,0x00,0x10,0x0d,0x10, + 0xff,0xf0,0x91,0x96,0xb8,0xf0,0x91,0x96,0xaf,0x00,0x10,0xff,0xf0,0x91,0x96,0xb9, + 0xf0,0x91,0x96,0xaf,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10,0x09,0xcf, + 0x86,0x95,0x2c,0xd4,0x1c,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x07,0x10, + 0x00,0x10,0x00,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x53, + 0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0xd2, + 0xa0,0xd1,0x5c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53, + 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10, + 0x09,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, + 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x2a,0xcf, + 0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0xd3,0x10,0x52,0x04,0x0d,0x00,0x51, + 0x04,0x0d,0x00,0x10,0x04,0x0d,0x09,0x0d,0x07,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x0d, + 0x00,0x92,0x08,0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x54,0x04,0x11,0x00,0x53,0x04,0x11,0x00,0xd2, + 0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00, + 0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00,0x92,0x0c,0x51,0x04,0x11, + 0x00,0x10,0x04,0x11,0x00,0x11,0x09,0x00,0x00,0x11,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xe4,0x59,0x01,0xd3,0xb2,0xd2,0x5c,0xd1,0x28,0xd0,0x22,0xcf,0x86, + 0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92,0x10,0xd1,0x08, + 0x10,0x04,0x14,0x00,0x14,0x09,0x10,0x04,0x14,0x07,0x14,0x00,0x00,0x00,0xcf,0x06, + 0x00,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x10,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x00,0x00,0x94,0x10,0x53,0x04,0x15,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00, + 0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x15,0x00,0x53,0x04,0x15,0x00, + 0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x94,0x1c,0x93,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x15,0x09,0x15,0x00,0x15,0x00,0x91,0x08,0x10,0x04,0x15,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xa0,0xd1,0x3c,0xd0,0x1e,0xcf,0x86, + 0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x93,0x10,0x52,0x04,0x13,0x00,0x91,0x08, + 0x10,0x04,0x13,0x09,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x86,0x95,0x18,0x94,0x14, + 0x93,0x10,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x13,0x09, + 0x00,0x00,0x13,0x00,0x13,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x10,0x93,0x0c, + 0x52,0x04,0x13,0x00,0x11,0x04,0x15,0x00,0x13,0x00,0x13,0x00,0x53,0x04,0x13,0x00, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x13,0x00,0x13,0x09,0x13,0x00,0x91,0x08,0x10,0x04, + 0x13,0x00,0x14,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x13,0x00, + 0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0xa9,0x01,0xd2, + 0xb0,0xd1,0x6c,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x54, + 0x04,0x12,0x00,0xd3,0x10,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12, + 0x00,0x00,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x12, + 0x09,0xcf,0x86,0xd5,0x14,0x94,0x10,0x93,0x0c,0x52,0x04,0x12,0x00,0x11,0x04,0x12, + 0x00,0x00,0x00,0x00,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, + 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xd0,0x3e,0xcf, + 0x86,0xd5,0x14,0x54,0x04,0x12,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x12, + 0x00,0x12,0x00,0x12,0x00,0xd4,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x93,0x10,0x52,0x04,0x12,0x00,0x51, + 0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, + 0xa0,0xd0,0x52,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3,0x10,0x52,0x04,0x13,0x00,0x51, + 0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10, + 0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x54,0x04,0x13,0x00,0xd3,0x10,0x52, + 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0xd2,0x0c,0x51, + 0x04,0x00,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x00, + 0x00,0x13,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x51,0x04,0x13, + 0x00,0x10,0x04,0x13,0x07,0x13,0x00,0x11,0x04,0x13,0x09,0x13,0x00,0x00,0x00,0x53, + 0x04,0x13,0x00,0x92,0x08,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3, + 0x10,0x52,0x04,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0, + 0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x14,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14, + 0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x14, + 0x09,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94, + 0x10,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15, + 0x00,0x54,0x04,0x15,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x00, + 0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xd0, + 0xca,0xcf,0x86,0xd5,0xc2,0xd4,0x54,0xd3,0x06,0xcf,0x06,0x09,0x00,0xd2,0x06,0xcf, + 0x06,0x09,0x00,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09, + 0x00,0x94,0x14,0x53,0x04,0x09,0x00,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10, + 0x04,0x09,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x10, + 0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x68,0xd2,0x46,0xd1,0x40,0xd0, + 0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x20,0xd3,0x10,0x92, + 0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10, + 0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x09, + 0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x11, + 0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x10,0x94,0x0c,0x93, + 0x08,0x12,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x06,0xcf, + 0x06,0x0b,0x00,0xd3,0x40,0xd2,0x3a,0xd1,0x34,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0b, + 0x00,0xd4,0x14,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10, + 0x04,0x0b,0x00,0x00,0x00,0x53,0x04,0x15,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x4c,0xd0,0x44,0xcf, + 0x86,0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2, + 0x2a,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x93, + 0x10,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0xd2,0x01,0xcf,0x86,0xd5,0x06,0xcf,0x06, + 0x00,0x00,0xe4,0x0b,0x01,0xd3,0x06,0xcf,0x06,0x0c,0x00,0xd2,0x84,0xd1,0x50,0xd0, + 0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5, + 0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10, + 0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x10,0x00,0xd2,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x00,0x00,0x10,0x00,0xd4,0x10,0x53, + 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52, + 0x04,0x10,0x01,0x91,0x08,0x10,0x04,0x10,0x01,0x10,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x6c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x93,0x10,0x52, + 0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x10,0xe6,0x10,0x00,0x10,0x00,0xcf, + 0x86,0xd5,0x24,0xd4,0x10,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, + 0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x00, + 0x00,0x10,0x00,0x10,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, + 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x00, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd0,0x0e,0xcf,0x86,0x95, + 0x08,0x14,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf, + 0x06,0x00,0x00,0xd2,0x30,0xd1,0x0c,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x06,0x14, + 0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92, + 0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xd1,0x4c,0xd0,0x06,0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x2c,0x94, + 0x28,0xd3,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x15,0x00,0x15, + 0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d, + 0x00,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x15,0x00,0xd0, + 0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0x55, + 0x04,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x13, + 0x00,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xcf,0x06,0x12,0x00,0xe2, + 0xc6,0x01,0xd1,0x8e,0xd0,0x86,0xcf,0x86,0xd5,0x48,0xd4,0x06,0xcf,0x06,0x12,0x00, + 0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x06,0xcf,0x06,0x12,0x00,0xd1,0x06,0xcf,0x06, + 0x12,0x00,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0xd4,0x14, + 0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x14,0x00, + 0x14,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0x00,0x00, + 0xd4,0x36,0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x12,0x00, + 0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0x54,0x04,0x12,0x00, + 0x93,0x10,0x92,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0xa2,0xd4,0x9c,0xd3,0x74, + 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x06, + 0x13,0x00,0xcf,0x06,0x13,0x00,0xd1,0x48,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x13,0x00,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04, + 0x13,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x00,0x00,0x93,0x10, + 0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x94,0x0c,0x93,0x08,0x12,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x13,0x00,0xcf,0x06, + 0x13,0x00,0xd2,0x22,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00, + 0xcf,0x86,0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x53,0x04,0x13,0x00,0x12,0x04, + 0x13,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06, + 0x00,0x00,0xd3,0x7f,0xd2,0x79,0xd1,0x34,0xd0,0x06,0xcf,0x06,0x10,0x00,0xcf,0x86, + 0x55,0x04,0x10,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00, + 0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x3f,0xcf,0x86,0xd5,0x2c, + 0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0xd2,0x08,0x11,0x04,0x10,0x00,0x00,0x00, + 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x01,0x10,0x00,0x94,0x0d,0x93,0x09,0x12,0x05, + 0x10,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xe1,0x96,0x04,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xe5,0x33,0x04,0xe4,0x83,0x02,0xe3,0xf8,0x01,0xd2,0x26,0xd1,0x06,0xcf, + 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x55,0x04,0x05,0x00,0x54, + 0x04,0x05,0x00,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x00,0x00,0x00, + 0x00,0xd1,0xef,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x20,0xd3,0x10,0x52, + 0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0xd5, + 0x2a,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05, + 0x00,0x10,0x0d,0x05,0xff,0xf0,0x9d,0x85,0x97,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff, + 0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0x00,0xd4,0x75,0xd3,0x61,0xd2,0x44,0xd1, + 0x22,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, + 0xae,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xaf, + 0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, + 0xb0,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xb1, + 0x00,0xd1,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0, + 0x9d,0x85,0xb2,0x00,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0x01,0xd2,0x08,0x11,0x04, + 0x05,0x01,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe2,0x05,0xd8,0xd3,0x12, + 0x92,0x0d,0x51,0x04,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0xff,0x00,0x05,0xff,0x00, + 0x92,0x0e,0x51,0x05,0x05,0xff,0x00,0x10,0x05,0x05,0xff,0x00,0x05,0xdc,0x05,0xdc, + 0xd0,0x97,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x05,0xdc, + 0x10,0x04,0x05,0xdc,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe6,0x05,0xe6, + 0x92,0x08,0x11,0x04,0x05,0xe6,0x05,0xdc,0x05,0x00,0x05,0x00,0xd4,0x14,0x53,0x04, + 0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x05,0xe6,0x11,0x04,0x05,0xe6,0x05,0x00, + 0x53,0x04,0x05,0x00,0xd2,0x15,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x05,0xff, + 0xf0,0x9d,0x86,0xb9,0xf0,0x9d,0x85,0xa5,0x00,0xd1,0x1e,0x10,0x0d,0x05,0xff,0xf0, + 0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, + 0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0, + 0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, + 0x85,0xa5,0xf0,0x9d,0x85,0xaf,0x00,0xcf,0x86,0xd5,0x31,0xd4,0x21,0x93,0x1d,0x92, + 0x19,0x91,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0xf0, + 0x9d,0x85,0xaf,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x53,0x04,0x05,0x00, + 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd2,0x44,0xd1,0x28,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x95,0x1c,0x94,0x18, + 0x93,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x08,0xe6,0x91,0x08,0x10,0x04,0x08,0xe6, + 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x93,0x08,0x12,0x04,0x14,0x00, + 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0xd5,0x18, + 0x54,0x04,0x07,0x00,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, + 0x07,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x09,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x09,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0x5f,0x01,0xd2,0xb4,0xd1,0x24,0xd0, + 0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x05,0x00,0x93,0x10,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, + 0x00,0xd0,0x6a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0xd4,0x34,0xd3, + 0x1c,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd1,0x08,0x10, + 0x04,0x00,0x00,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05, + 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x20,0x94, + 0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x07,0x00,0x05,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xd1, + 0xa4,0xd0,0x6a,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x10,0x52,0x04,0x05,0x00,0x51, + 0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10, + 0x04,0x05,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xd3, + 0x10,0x52,0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x54,0x04,0x05, + 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, + 0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4, + 0x20,0xd3,0x14,0x52,0x04,0x05,0x00,0xd1,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x10, + 0x04,0x05,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x93, + 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, + 0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06,0x05,0x00,0xd0, + 0x1a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11, + 0x04,0x08,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd1,0x06,0xcf, + 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53, + 0x04,0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x09,0x00,0x11,0x04,0x00,0x00,0x05, + 0x00,0x05,0x00,0x05,0x00,0xd4,0x52,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x46,0xd1, + 0x06,0xcf,0x06,0x11,0x00,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x53,0x04,0x11, + 0x00,0x12,0x04,0x11,0x00,0x00,0x00,0x53,0x04,0x00,0x00,0x92,0x0c,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe0,0xc2,0x03,0xcf,0x86, + 0xe5,0x03,0x01,0xd4,0xfc,0xd3,0xc0,0xd2,0x66,0xd1,0x60,0xd0,0x5a,0xcf,0x86,0xd5, + 0x2c,0xd4,0x14,0x93,0x10,0x52,0x04,0x12,0xe6,0x51,0x04,0x12,0xe6,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x12,0xe6,0x53,0x04,0x12,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x10,0x04,0x00,0x00,0x12,0xe6,0x12,0xe6,0x94,0x28,0xd3,0x18,0xd2, + 0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x00,0x00,0x12,0xe6,0x91,0x08,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x12,0xe6,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x54,0xd0, + 0x36,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x15, + 0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x15, + 0xe6,0x51,0x04,0x15,0xe6,0x10,0x04,0x15,0xe6,0x15,0x00,0x52,0x04,0x15,0x00,0x11, + 0x04,0x15,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0xd2, + 0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd2,0x36,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x0c,0x53,0x04,0x15,0x00,0x12, + 0x04,0x15,0x00,0x15,0xe6,0x53,0x04,0x15,0x00,0xd2,0x08,0x11,0x04,0x15,0x00,0x00, + 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xd4,0x82,0xd3,0x7c,0xd2,0x3e,0xd1,0x06,0xcf,0x06,0x10,0x00,0xd0, + 0x06,0xcf,0x06,0x10,0x00,0xcf,0x86,0x95,0x2c,0xd4,0x18,0x93,0x14,0x52,0x04,0x10, + 0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10, + 0x00,0x93,0x10,0x52,0x04,0x10,0xdc,0x51,0x04,0x10,0xdc,0x10,0x04,0x10,0xdc,0x00, + 0x00,0x00,0x00,0x00,0x00,0xd1,0x38,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95, + 0x2c,0xd4,0x18,0xd3,0x08,0x12,0x04,0x12,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12, + 0xe6,0x10,0x04,0x12,0x07,0x15,0x00,0x00,0x00,0x53,0x04,0x12,0x00,0xd2,0x08,0x11, + 0x04,0x12,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd3,0x82,0xd2,0x48,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0,0x1e,0xcf, + 0x86,0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x93,0x10,0x52,0x04,0x14,0x00,0x91, + 0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, + 0x34,0xd0,0x2e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x54,0x04,0x15, + 0x00,0x53,0x04,0x15,0x00,0x52,0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe2,0xb2,0x01,0xe1,0x41,0x01,0xd0,0x6e,0xcf, + 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1, + 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10, + 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d, + 0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0xd5,0x74,0xd4, + 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, + 0x00,0x0d,0x00,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, + 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d, + 0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, + 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, + 0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, + 0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x08,0x10, + 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd0,0x56,0xcf,0x86,0xd5, + 0x20,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00,0x00,0xd4, + 0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x0c,0x92,0x08,0x11, + 0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xe5, + 0x96,0x05,0xe4,0x28,0x03,0xe3,0xed,0x01,0xd2,0xa0,0xd1,0x1c,0xd0,0x16,0xcf,0x86, + 0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00,0x00,0x00, + 0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x10,0x54,0x04,0x0a,0x00, + 0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x0c,0x00, + 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0xd3,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04, + 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x28, + 0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, + 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0xe4,0xd0,0x5a,0xcf,0x86,0xd5,0x20, + 0x94,0x1c,0x53,0x04,0x0b,0x00,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd4,0x14, + 0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x14,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00, + 0x0c,0x00,0x52,0x04,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04, + 0x0c,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04, + 0x0c,0x00,0x10,0x04,0x0b,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0b,0x00, + 0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x0c,0x00,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, + 0x0c,0x00,0x0b,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x0b,0x00,0xd4,0x18,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0d,0x00, + 0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0c,0x00,0xd2,0x10, + 0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd1,0x08, + 0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd0,0x4e,0xcf,0x86, + 0xd5,0x34,0xd4,0x14,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00, + 0x11,0x04,0x0b,0x00,0x0c,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x12,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08, + 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00, + 0x94,0x10,0x93,0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, + 0x0c,0x00,0xd2,0x7e,0xd1,0x78,0xd0,0x3e,0xcf,0x86,0xd5,0x1c,0x94,0x18,0x93,0x14, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x0b,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x10,0x93,0x0c,0x52,0x04, + 0x13,0x00,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, + 0xd1,0x58,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x93,0x10,0x52,0x04,0x0c,0x00, + 0x51,0x04,0x0c,0x00,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04, + 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x91,0x08, + 0x10,0x04,0x0c,0x00,0x10,0x00,0x11,0x00,0xd0,0x16,0xcf,0x86,0x95,0x10,0x54,0x04, + 0x0c,0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x0c,0x00,0xcf,0x86, + 0xd5,0x34,0xd4,0x28,0xd3,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10,0x04,0x0c,0x00, + 0x10,0x00,0x0c,0x00,0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00, + 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x93,0x08,0x12,0x04,0x11,0x00, + 0x10,0x00,0x10,0x00,0x54,0x04,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x11,0x00,0xd3,0xfc,0xd2,0x6c,0xd1,0x3c, + 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00, + 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86, + 0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00, + 0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00, + 0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0xd1,0x54,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04, + 0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x1c,0x94,0x18,0xd3,0x08,0x12,0x04,0x0d,0x00, + 0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00, + 0x0c,0x00,0xd4,0x08,0x13,0x04,0x0c,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c, + 0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86, + 0x55,0x04,0x10,0x00,0x94,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10,0x04, + 0x12,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00, + 0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x0c,0x00,0x0c,0x00,0xe2,0x19,0x01,0xd1,0xa8,0xd0,0x7e,0xcf,0x86,0xd5, + 0x4c,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0d,0x00,0x0c,0x00,0x0c, + 0x00,0x0c,0x00,0x0c,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0d,0x00,0xd4,0x1c,0xd3,0x0c,0x52, + 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x0d,0x00,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, + 0x04,0x0d,0x00,0x0c,0x00,0x0d,0x00,0x93,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, + 0x04,0x0d,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0x94,0x20,0x93, + 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x11, + 0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x10,0x00,0x10, + 0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x30,0xd4,0x10,0x93,0x0c,0x52, + 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x93,0x1c,0xd2,0x10,0xd1, + 0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x91,0x08,0x10, + 0x04,0x13,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x52, + 0x04,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52, + 0x04,0x10,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x92,0x10,0xd1, + 0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x1c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c, + 0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x10, + 0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x10,0x00,0xd3,0x10,0x52,0x04,0x10,0x00,0x91, + 0x08,0x10,0x04,0x10,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x0c,0x53,0x04,0x15,0x00,0x12,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0xe4,0x40,0x02,0xe3,0xc9,0x01,0xd2,0x5c,0xd1,0x34,0xd0, + 0x16,0xcf,0x86,0x95,0x10,0x94,0x0c,0x53,0x04,0x10,0x00,0x12,0x04,0x10,0x00,0x00, + 0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x95,0x18,0xd4,0x08,0x13,0x04,0x10,0x00,0x00, + 0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x10, + 0x00,0xd0,0x22,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x94,0x10,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, + 0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0xc0,0xd0,0x5e,0xcf,0x86,0xd5,0x30,0xd4, + 0x14,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x15, + 0x00,0x15,0x00,0x53,0x04,0x11,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x12, + 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd4,0x08,0x13, + 0x04,0x12,0x00,0x13,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x12,0x00,0x13, + 0x00,0x10,0x04,0x13,0x00,0x12,0x00,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12, + 0x00,0x10,0x04,0x12,0x00,0x15,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x12, + 0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x53, + 0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13, + 0x00,0xd4,0x0c,0x53,0x04,0x13,0x00,0x12,0x04,0x13,0x00,0x14,0x00,0xd3,0x1c,0xd2, + 0x10,0xd1,0x08,0x10,0x04,0x14,0x00,0x15,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10, + 0x04,0x14,0x00,0x15,0x00,0x14,0x00,0xd0,0x62,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93, + 0x10,0x52,0x04,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x12,0x00,0x12, + 0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x12,0x00,0x13,0x00,0x13,0x00,0x14,0x00,0xd4, + 0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10, + 0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x53,0x04,0x14,0x00,0x92, + 0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x30,0x94,0x2c,0xd3, + 0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x11,0x00,0x14,0x00,0x10,0x04,0x14,0x00,0x15, + 0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x52, + 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14, + 0x00,0xd2,0x70,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x15,0x00,0xcf,0x86,0xd5,0x10,0x54, + 0x04,0x15,0x00,0x93,0x08,0x12,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53, + 0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x00,0x00,0xd3,0x08,0x12, + 0x04,0x15,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00, + 0x00,0x00,0x00,0xd0,0x2a,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51, + 0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x52, + 0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00, + 0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55, + 0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11, + 0x04,0x00,0x00,0x02,0x00,0xe4,0xf9,0x12,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, + 0xd2,0xc2,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x3c, + 0xd4,0x06,0xcf,0x06,0x05,0x00,0xd3,0x06,0xcf,0x06,0x05,0x00,0xd2,0x2a,0xd1,0x06, + 0xcf,0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x05,0x00,0x93,0x10,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x86, + 0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x0b,0x00,0xd3,0x06,0xcf,0x06,0x0b,0x00,0xd2,0x06, + 0xcf,0x06,0x0b,0x00,0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, + 0x0b,0x00,0x93,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x06,0x0c,0x00,0xcf,0x06,0x0c,0x00,0xd4,0x32,0xd3,0x2c, + 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04,0x0c,0x00,0x53,0x04, + 0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x11,0x00,0xcf,0x06, + 0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06, + 0x11,0x00,0xd1,0x48,0xd0,0x40,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x11,0x00,0xd4,0x06, + 0xcf,0x06,0x11,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06, + 0x11,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04,0x11,0x00,0x94,0x10,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13,0x00,0xcf,0x06,0x13,0x00, + 0xcf,0x06,0x13,0x00,0xcf,0x86,0xcf,0x06,0x13,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x13,0x00,0xd4,0x36,0xd3,0x06,0xcf,0x06,0x13,0x00,0xd2,0x06,0xcf,0x06, + 0x13,0x00,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00,0xcf,0x86, + 0x55,0x04,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x13,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x68,0x11,0xe3,0x51,0x10,0xe2,0x17,0x08,0xe1, + 0x06,0x04,0xe0,0x03,0x02,0xcf,0x86,0xe5,0x06,0x01,0xd4,0x82,0xd3,0x41,0xd2,0x21, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00, + 0x10,0x08,0x05,0xff,0xe4,0xb9,0x81,0x00,0x05,0xff,0xf0,0xa0,0x84,0xa2,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe4,0xbd,0xa0,0x00,0x05,0xff,0xe4,0xbe,0xae,0x00,0x10, + 0x08,0x05,0xff,0xe4,0xbe,0xbb,0x00,0x05,0xff,0xe5,0x80,0x82,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x81,0xba,0x00,0x05,0xff,0xe5,0x82,0x99,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x83,0xa7,0x00,0x05,0xff,0xe5,0x83,0x8f,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe3,0x92,0x9e,0x00,0x05,0xff,0xf0,0xa0,0x98,0xba,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x85,0x8d,0x00,0x05,0xff,0xe5,0x85,0x94,0x00,0xd3,0x42,0xd2,0x21, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa4,0x00,0x05,0xff,0xe5,0x85,0xb7,0x00, + 0x10,0x09,0x05,0xff,0xf0,0xa0,0x94,0x9c,0x00,0x05,0xff,0xe3,0x92,0xb9,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa7,0x00,0x05,0xff,0xe5,0x86,0x8d,0x00,0x10, + 0x09,0x05,0xff,0xf0,0xa0,0x95,0x8b,0x00,0x05,0xff,0xe5,0x86,0x97,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x86,0xa4,0x00,0x05,0xff,0xe4,0xbb,0x8c,0x00, + 0x10,0x08,0x05,0xff,0xe5,0x86,0xac,0x00,0x05,0xff,0xe5,0x86,0xb5,0x00,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa9,0x87,0x9f,0x00,0x05,0xff,0xe5,0x87,0xb5,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x88,0x83,0x00,0x05,0xff,0xe3,0x93,0x9f,0x00,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x88,0xbb,0x00,0x05,0xff,0xe5, + 0x89,0x86,0x00,0x10,0x08,0x05,0xff,0xe5,0x89,0xb2,0x00,0x05,0xff,0xe5,0x89,0xb7, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0x94,0x95,0x00,0x05,0xff,0xe5,0x8b,0x87, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8b,0x89,0x00,0x05,0xff,0xe5,0x8b,0xa4,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x8b,0xba,0x00,0x05,0xff,0xe5,0x8c,0x85, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8c,0x86,0x00,0x05,0xff,0xe5,0x8c,0x97,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0x89,0x00,0x05,0xff,0xe5,0x8d,0x91,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x8d,0x9a,0x00,0x05,0xff,0xe5,0x8d,0xb3,0x00,0xd3,0x39,0xd2, + 0x18,0x91,0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0xbd,0x00,0x05,0xff,0xe5,0x8d,0xbf, + 0x00,0x05,0xff,0xe5,0x8d,0xbf,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0,0xa8, + 0xac,0x00,0x05,0xff,0xe7,0x81,0xb0,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0x8a,0x00, + 0x05,0xff,0xe5,0x8f,0x9f,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0, + 0xad,0xa3,0x00,0x05,0xff,0xe5,0x8f,0xab,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0xb1, + 0x00,0x05,0xff,0xe5,0x90,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0x9e, + 0x00,0x05,0xff,0xe5,0x90,0xb8,0x00,0x10,0x08,0x05,0xff,0xe5,0x91,0x88,0x00,0x05, + 0xff,0xe5,0x91,0xa8,0x00,0xcf,0x86,0xe5,0x02,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6,0x00, + 0x10,0x08,0x05,0xff,0xe5,0x94,0x90,0x00,0x05,0xff,0xe5,0x95,0x93,0x00,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x95,0xa3,0x00,0x05,0xff,0xe5,0x96,0x84,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x96,0x84,0x00,0x05,0xff,0xe5,0x96,0x99,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x96,0xab,0x00,0x05,0xff,0xe5,0x96,0xb3,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x97,0x82,0x00,0x05,0xff,0xe5,0x9c,0x96,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe5,0x98,0x86,0x00,0x05,0xff,0xe5,0x9c,0x97,0x00,0x10,0x08,0x05,0xff, + 0xe5,0x99,0x91,0x00,0x05,0xff,0xe5,0x99,0xb4,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x88,0x87,0x00,0x05,0xff,0xe5,0xa3,0xae,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x9f,0x8e,0x00,0x05,0xff,0xe5,0x9f,0xb4,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe5,0xa0,0x8d,0x00,0x05,0xff,0xe5,0x9e,0x8b,0x00,0x10,0x08,0x05,0xff, + 0xe5,0xa0,0xb2,0x00,0x05,0xff,0xe5,0xa0,0xb1,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe5,0xa2,0xac,0x00,0x05,0xff,0xf0,0xa1,0x93,0xa4,0x00,0x10,0x08,0x05, + 0xff,0xe5,0xa3,0xb2,0x00,0x05,0xff,0xe5,0xa3,0xb7,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe5,0xa4,0x86,0x00,0x05,0xff,0xe5,0xa4,0x9a,0x00,0x10,0x08,0x05,0xff,0xe5, + 0xa4,0xa2,0x00,0x05,0xff,0xe5,0xa5,0xa2,0x00,0xd4,0x7b,0xd3,0x42,0xd2,0x22,0xd1, + 0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa, + 0x00,0x10,0x08,0x05,0xff,0xe5,0xa7,0xac,0x00,0x05,0xff,0xe5,0xa8,0x9b,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0xa8,0xa7,0x00,0x05,0xff,0xe5,0xa7,0x98,0x00,0x10, + 0x08,0x05,0xff,0xe5,0xa9,0xa6,0x00,0x05,0xff,0xe3,0x9b,0xae,0x00,0xd2,0x18,0x91, + 0x10,0x10,0x08,0x05,0xff,0xe3,0x9b,0xbc,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, + 0xff,0xe5,0xac,0xbe,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0xa7,0x88,0x00, + 0x05,0xff,0xe5,0xaf,0x83,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0x98,0x00,0x05,0xff, + 0xe5,0xaf,0xa7,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xaf, + 0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0xbf, + 0x00,0x05,0xff,0xe5,0xb0,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xbd,0x93, + 0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0x9e,0x81,0x00,0x05, + 0xff,0xe5,0xb1,0xa0,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb1,0xae, + 0x00,0x05,0xff,0xe5,0xb3,0x80,0x00,0x10,0x08,0x05,0xff,0xe5,0xb2,0x8d,0x00,0x05, + 0xff,0xf0,0xa1,0xb7,0xa4,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xb5,0x83,0x00, + 0x05,0xff,0xf0,0xa1,0xb7,0xa6,0x00,0x10,0x08,0x05,0xff,0xe5,0xb5,0xae,0x00,0x05, + 0xff,0xe5,0xb5,0xab,0x00,0xe0,0x04,0x02,0xcf,0x86,0xd5,0xfe,0xd4,0x82,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb5,0xbc,0x00,0x05,0xff,0xe5,0xb7, + 0xa1,0x00,0x10,0x08,0x05,0xff,0xe5,0xb7,0xa2,0x00,0x05,0xff,0xe3,0xa0,0xaf,0x00, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb7,0xbd,0x00,0x05,0xff,0xe5,0xb8,0xa8,0x00, + 0x10,0x08,0x05,0xff,0xe5,0xb8,0xbd,0x00,0x05,0xff,0xe5,0xb9,0xa9,0x00,0xd2,0x21, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe3,0xa1,0xa2,0x00,0x05,0xff,0xf0,0xa2,0x86,0x83, + 0x00,0x10,0x08,0x05,0xff,0xe3,0xa1,0xbc,0x00,0x05,0xff,0xe5,0xba,0xb0,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0xba,0xb3,0x00,0x05,0xff,0xe5,0xba,0xb6,0x00,0x10, + 0x08,0x05,0xff,0xe5,0xbb,0x8a,0x00,0x05,0xff,0xf0,0xaa,0x8e,0x92,0x00,0xd3,0x3b, + 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xbb,0xbe,0x00,0x05,0xff,0xf0,0xa2, + 0x8c,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2,0x8c,0xb1,0x00,0x05,0xff,0xe8,0x88, + 0x81,0x00,0x51,0x08,0x05,0xff,0xe5,0xbc,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0xa3, + 0x87,0x00,0x05,0xff,0xf0,0xa3,0x8a,0xb8,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, + 0xff,0xf0,0xa6,0x87,0x9a,0x00,0x05,0xff,0xe5,0xbd,0xa2,0x00,0x10,0x08,0x05,0xff, + 0xe5,0xbd,0xab,0x00,0x05,0xff,0xe3,0xa3,0xa3,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe5,0xbe,0x9a,0x00,0x05,0xff,0xe5,0xbf,0x8d,0x00,0x10,0x08,0x05,0xff,0xe5,0xbf, + 0x97,0x00,0x05,0xff,0xe5,0xbf,0xb9,0x00,0xd4,0x81,0xd3,0x41,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe6,0x82,0x81,0x00,0x05,0xff,0xe3,0xa4,0xba,0x00,0x10,0x08, + 0x05,0xff,0xe3,0xa4,0x9c,0x00,0x05,0xff,0xe6,0x82,0x94,0x00,0xd1,0x11,0x10,0x09, + 0x05,0xff,0xf0,0xa2,0x9b,0x94,0x00,0x05,0xff,0xe6,0x83,0x87,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x85,0x88,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe6,0x85,0x8e,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x85,0xba,0x00,0x05,0xff,0xe6,0x86,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe6,0x86,0xb2,0x00,0x05,0xff,0xe6,0x86,0xa4,0x00,0x10,0x08,0x05,0xff,0xe6, + 0x86,0xaf,0x00,0x05,0xff,0xe6,0x87,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe6,0x87,0xb2,0x00,0x05,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x88,0x90,0x00,0x05,0xff,0xe6,0x88,0x9b,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe6,0x89,0x9d,0x00,0x05,0xff,0xe6,0x8a,0xb1,0x00,0x10,0x08,0x05,0xff,0xe6, + 0x8b,0x94,0x00,0x05,0xff,0xe6,0x8d,0x90,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, + 0xff,0xf0,0xa2,0xac,0x8c,0x00,0x05,0xff,0xe6,0x8c,0xbd,0x00,0x10,0x08,0x05,0xff, + 0xe6,0x8b,0xbc,0x00,0x05,0xff,0xe6,0x8d,0xa8,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe6,0x8e,0x83,0x00,0x05,0xff,0xe6,0x8f,0xa4,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2, + 0xaf,0xb1,0x00,0x05,0xff,0xe6,0x90,0xa2,0x00,0xcf,0x86,0xe5,0x03,0x01,0xd4,0x81, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x8f,0x85,0x00,0x05,0xff, + 0xe6,0x8e,0xa9,0x00,0x10,0x08,0x05,0xff,0xe3,0xa8,0xae,0x00,0x05,0xff,0xe6,0x91, + 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x91,0xbe,0x00,0x05,0xff,0xe6,0x92, + 0x9d,0x00,0x10,0x08,0x05,0xff,0xe6,0x91,0xb7,0x00,0x05,0xff,0xe3,0xa9,0xac,0x00, + 0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x95,0x8f,0x00,0x05,0xff,0xe6,0x95, + 0xac,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0x80,0x8a,0x00,0x05,0xff,0xe6,0x97,0xa3, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9b,0xb8,0x00,0x05,0xff,0xe6,0x99,0x89, + 0x00,0x10,0x08,0x05,0xff,0xe3,0xac,0x99,0x00,0x05,0xff,0xe6,0x9a,0x91,0x00,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0xac,0x88,0x00,0x05,0xff,0xe3, + 0xab,0xa4,0x00,0x10,0x08,0x05,0xff,0xe5,0x86,0x92,0x00,0x05,0xff,0xe5,0x86,0x95, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x80,0x00,0x05,0xff,0xe6,0x9a,0x9c, + 0x00,0x10,0x08,0x05,0xff,0xe8,0x82,0xad,0x00,0x05,0xff,0xe4,0x8f,0x99,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x97,0x00,0x05,0xff,0xe6,0x9c,0x9b, + 0x00,0x10,0x08,0x05,0xff,0xe6,0x9c,0xa1,0x00,0x05,0xff,0xe6,0x9d,0x9e,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe6,0x9d,0x93,0x00,0x05,0xff,0xf0,0xa3,0x8f,0x83,0x00, + 0x10,0x08,0x05,0xff,0xe3,0xad,0x89,0x00,0x05,0xff,0xe6,0x9f,0xba,0x00,0xd4,0x82, + 0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9e,0x85,0x00,0x05,0xff, + 0xe6,0xa1,0x92,0x00,0x10,0x08,0x05,0xff,0xe6,0xa2,0x85,0x00,0x05,0xff,0xf0,0xa3, + 0x91,0xad,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa2,0x8e,0x00,0x05,0xff,0xe6, + 0xa0,0x9f,0x00,0x10,0x08,0x05,0xff,0xe6,0xa4,0x94,0x00,0x05,0xff,0xe3,0xae,0x9d, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa5,0x82,0x00,0x05,0xff,0xe6, + 0xa6,0xa3,0x00,0x10,0x08,0x05,0xff,0xe6,0xa7,0xaa,0x00,0x05,0xff,0xe6,0xaa,0xa8, + 0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0x9a,0xa3,0x00,0x05,0xff,0xe6,0xab, + 0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb0,0x98,0x00,0x05,0xff,0xe6,0xac,0xa1,0x00, + 0xd3,0x42,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0xa2,0xa7,0x00,0x05, + 0xff,0xe6,0xad,0x94,0x00,0x10,0x08,0x05,0xff,0xe3,0xb1,0x8e,0x00,0x05,0xff,0xe6, + 0xad,0xb2,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xae,0x9f,0x00,0x05,0xff,0xe6, + 0xae,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xae,0xbb,0x00,0x05,0xff,0xf0,0xa3,0xaa, + 0x8d,0x00,0xd2,0x23,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0xb4,0x8b,0x00,0x05, + 0xff,0xf0,0xa3,0xab,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0x8e,0x00,0x05,0xff, + 0xf0,0xa3,0xb2,0xbc,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb2,0xbf,0x00,0x05, + 0xff,0xe6,0xb3,0x8d,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0xa7,0x00,0x05,0xff,0xe6, + 0xb4,0x96,0x00,0xe1,0x1d,0x04,0xe0,0x0c,0x02,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x82, + 0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, + 0xe6,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe6,0xb5,0x81,0x00,0x05,0xff,0xe6,0xb5, + 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb5,0xb8,0x00,0x05,0xff,0xe6,0xb6, + 0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0xb4,0x9e,0x00,0x05,0xff,0xe6,0xb4,0xb4, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb8,0xaf,0x00,0x05,0xff,0xe6, + 0xb9,0xae,0x00,0x10,0x08,0x05,0xff,0xe3,0xb4,0xb3,0x00,0x05,0xff,0xe6,0xbb,0x8b, + 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe6,0xbb,0x87,0x00,0x05,0xff,0xf0,0xa3,0xbb, + 0x91,0x00,0x10,0x08,0x05,0xff,0xe6,0xb7,0xb9,0x00,0x05,0xff,0xe6,0xbd,0xae,0x00, + 0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00,0x05, + 0xff,0xf0,0xa3,0xbe,0x8e,0x00,0x10,0x08,0x05,0xff,0xe6,0xbf,0x86,0x00,0x05,0xff, + 0xe7,0x80,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x80,0x9e,0x00,0x05,0xff, + 0xe7,0x80,0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb6,0x96,0x00,0x05,0xff,0xe7,0x81, + 0x8a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff, + 0xe7,0x81,0xb7,0x00,0x10,0x08,0x05,0xff,0xe7,0x82,0xad,0x00,0x05,0xff,0xf0,0xa0, + 0x94,0xa5,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0, + 0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xf0,0xa4, + 0x8e,0xab,0x00,0xd4,0x7b,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7, + 0x88,0xa8,0x00,0x05,0xff,0xe7,0x88,0xb5,0x00,0x10,0x08,0x05,0xff,0xe7,0x89,0x90, + 0x00,0x05,0xff,0xf0,0xa4,0x98,0x88,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x8a, + 0x80,0x00,0x05,0xff,0xe7,0x8a,0x95,0x00,0x10,0x09,0x05,0xff,0xf0,0xa4,0x9c,0xb5, + 0x00,0x05,0xff,0xf0,0xa4,0xa0,0x94,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe7,0x8d,0xba,0x00,0x05,0xff,0xe7,0x8e,0x8b,0x00,0x10,0x08,0x05,0xff,0xe3,0xba, + 0xac,0x00,0x05,0xff,0xe7,0x8e,0xa5,0x00,0x51,0x08,0x05,0xff,0xe3,0xba,0xb8,0x00, + 0x10,0x08,0x05,0xff,0xe7,0x91,0x87,0x00,0x05,0xff,0xe7,0x91,0x9c,0x00,0xd3,0x42, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x91,0xb1,0x00,0x05,0xff,0xe7,0x92, + 0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0x93,0x8a,0x00,0x05,0xff,0xe3,0xbc,0x9b,0x00, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x94,0xa4,0x00,0x05,0xff,0xf0,0xa4,0xb0,0xb6, + 0x00,0x10,0x08,0x05,0xff,0xe7,0x94,0xbe,0x00,0x05,0xff,0xf0,0xa4,0xb2,0x92,0x00, + 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x95,0xb0,0x00,0x05,0xff,0xf0,0xa2, + 0x86,0x9f,0x00,0x10,0x08,0x05,0xff,0xe7,0x98,0x90,0x00,0x05,0xff,0xf0,0xa4,0xbe, + 0xa1,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa4,0xbe,0xb8,0x00,0x05,0xff,0xf0, + 0xa5,0x81,0x84,0x00,0x10,0x08,0x05,0xff,0xe3,0xbf,0xbc,0x00,0x05,0xff,0xe4,0x80, + 0x88,0x00,0xcf,0x86,0xe5,0x04,0x01,0xd4,0x7d,0xd3,0x3c,0xd2,0x23,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0x10,0x09, + 0x05,0xff,0xf0,0xa5,0x83,0xb2,0x00,0x05,0xff,0xf0,0xa5,0x84,0x99,0x00,0x91,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa5,0x84,0xb3,0x00,0x05,0xff,0xe7,0x9c,0x9e,0x00,0x05, + 0xff,0xe7,0x9c,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x9d,0x8a, + 0x00,0x05,0xff,0xe4,0x80,0xb9,0x00,0x10,0x08,0x05,0xff,0xe7,0x9e,0x8b,0x00,0x05, + 0xff,0xe4,0x81,0x86,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe4,0x82,0x96,0x00,0x05, + 0xff,0xf0,0xa5,0x90,0x9d,0x00,0x10,0x08,0x05,0xff,0xe7,0xa1,0x8e,0x00,0x05,0xff, + 0xe7,0xa2,0x8c,0x00,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xa3, + 0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0x98,0xa6, + 0x00,0x05,0xff,0xe7,0xa5,0x96,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0x9a, + 0x9a,0x00,0x05,0xff,0xf0,0xa5,0x9b,0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0xa6,0x8f, + 0x00,0x05,0xff,0xe7,0xa7,0xab,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4, + 0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80,0x00,0x10,0x08,0x05,0xff,0xe7,0xa9,0x8a, + 0x00,0x05,0xff,0xe7,0xa9,0x8f,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5, + 0xbc,0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa, + 0xa7,0x00,0x05,0xff,0xe7,0xab,0xae,0x00,0xd4,0x83,0xd3,0x42,0xd2,0x21,0xd1,0x11, + 0x10,0x08,0x05,0xff,0xe4,0x88,0x82,0x00,0x05,0xff,0xf0,0xa5,0xae,0xab,0x00,0x10, + 0x08,0x05,0xff,0xe7,0xaf,0x86,0x00,0x05,0xff,0xe7,0xaf,0x89,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe4,0x88,0xa7,0x00,0x05,0xff,0xf0,0xa5,0xb2,0x80,0x00,0x10,0x08, + 0x05,0xff,0xe7,0xb3,0x92,0x00,0x05,0xff,0xe4,0x8a,0xa0,0x00,0xd2,0x21,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe7,0xb3,0xa8,0x00,0x05,0xff,0xe7,0xb3,0xa3,0x00,0x10,0x08, + 0x05,0xff,0xe7,0xb4,0x80,0x00,0x05,0xff,0xf0,0xa5,0xbe,0x86,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe7,0xb5,0xa3,0x00,0x05,0xff,0xe4,0x8c,0x81,0x00,0x10,0x08,0x05, + 0xff,0xe7,0xb7,0x87,0x00,0x05,0xff,0xe7,0xb8,0x82,0x00,0xd3,0x44,0xd2,0x22,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe7,0xb9,0x85,0x00,0x05,0xff,0xe4,0x8c,0xb4,0x00,0x10, + 0x09,0x05,0xff,0xf0,0xa6,0x88,0xa8,0x00,0x05,0xff,0xf0,0xa6,0x89,0x87,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe4,0x8d,0x99,0x00,0x05,0xff,0xf0,0xa6,0x8b,0x99,0x00, + 0x10,0x08,0x05,0xff,0xe7,0xbd,0xba,0x00,0x05,0xff,0xf0,0xa6,0x8c,0xbe,0x00,0xd2, + 0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xbe,0x95,0x00,0x05,0xff,0xe7,0xbf,0xba, + 0x00,0x10,0x08,0x05,0xff,0xe8,0x80,0x85,0x00,0x05,0xff,0xf0,0xa6,0x93,0x9a,0x00, + 0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa6,0x94,0xa3,0x00,0x05,0xff,0xe8,0x81,0xa0, + 0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0x96,0xa8,0x00,0x05,0xff,0xe8,0x81,0xb0,0x00, + 0xe0,0x11,0x02,0xcf,0x86,0xe5,0x07,0x01,0xd4,0x85,0xd3,0x42,0xd2,0x21,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0x10, + 0x08,0x05,0xff,0xe8,0x82,0xb2,0x00,0x05,0xff,0xe8,0x84,0x83,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe4,0x90,0x8b,0x00,0x05,0xff,0xe8,0x84,0xbe,0x00,0x10,0x08,0x05, + 0xff,0xe5,0xaa,0xb5,0x00,0x05,0xff,0xf0,0xa6,0x9e,0xa7,0x00,0xd2,0x23,0xd1,0x12, + 0x10,0x09,0x05,0xff,0xf0,0xa6,0x9e,0xb5,0x00,0x05,0xff,0xf0,0xa3,0x8e,0x93,0x00, + 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8e,0x9c,0x00,0x05,0xff,0xe8,0x88,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe8,0x88,0x84,0x00,0x05,0xff,0xe8,0xbe,0x9e,0x00,0x10, + 0x08,0x05,0xff,0xe4,0x91,0xab,0x00,0x05,0xff,0xe8,0x8a,0x91,0x00,0xd3,0x41,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0x8b,0x00,0x05,0xff,0xe8,0x8a,0x9d, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xb1,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xbd,0x00,0x10, + 0x08,0x05,0xff,0xe8,0x8b,0xa6,0x00,0x05,0xff,0xf0,0xa6,0xac,0xbc,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8b,0xa5,0x00,0x05,0xff,0xe8,0x8c,0x9d,0x00, + 0x10,0x08,0x05,0xff,0xe8,0x8d,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xad,0x00,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe8,0x8c,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xbd,0x00,0x10,0x08, + 0x05,0xff,0xe8,0x8f,0xa7,0x00,0x05,0xff,0xe8,0x91,0x97,0x00,0xd4,0x85,0xd3,0x43, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, + 0x8a,0x00,0x10,0x08,0x05,0xff,0xe8,0x8f,0x8c,0x00,0x05,0xff,0xe8,0x8f,0x9c,0x00, + 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb0,0xb6,0x00,0x05,0xff,0xf0,0xa6,0xb5, + 0xab,0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb3,0x95,0x00,0x05,0xff,0xe4,0x94,0xab, + 0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x93,0xb1,0x00,0x05,0xff,0xe8, + 0x93,0xb3,0x00,0x10,0x08,0x05,0xff,0xe8,0x94,0x96,0x00,0x05,0xff,0xf0,0xa7,0x8f, + 0x8a,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe8,0x95,0xa4,0x00,0x05,0xff,0xf0,0xa6, + 0xbc,0xac,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0x9d,0x00,0x05,0xff,0xe4,0x95,0xa1, + 0x00,0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, + 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0xab,0x00,0x05, + 0xff,0xe8,0x99,0x90,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x99,0x9c,0x00,0x05, + 0xff,0xe8,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8,0x99,0xa9,0x00,0x05,0xff,0xe8, + 0x9a,0xa9,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05, + 0xff,0xe8,0x9c,0x8e,0x00,0x10,0x08,0x05,0xff,0xe8,0x9b,0xa2,0x00,0x05,0xff,0xe8, + 0x9d,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8, + 0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0x97,0x97, + 0x00,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x83,0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe8,0x9f,0xa1,0x00,0x05,0xff,0xe8,0xa0,0x81,0x00,0x10,0x08,0x05,0xff, + 0xe4,0x97,0xb9,0x00,0x05,0xff,0xe8,0xa1,0xa0,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, + 0xe8,0xa1,0xa3,0x00,0x05,0xff,0xf0,0xa7,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8, + 0xa3,0x97,0x00,0x05,0xff,0xe8,0xa3,0x9e,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe4,0x98,0xb5,0x00,0x05,0xff,0xe8,0xa3,0xba,0x00,0x10,0x08,0x05,0xff,0xe3, + 0x92,0xbb,0x00,0x05,0xff,0xf0,0xa7,0xa2,0xae,0x00,0xd1,0x11,0x10,0x09,0x05,0xff, + 0xf0,0xa7,0xa5,0xa6,0x00,0x05,0xff,0xe4,0x9a,0xbe,0x00,0x10,0x08,0x05,0xff,0xe4, + 0x9b,0x87,0x00,0x05,0xff,0xe8,0xaa,0xa0,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe8,0xab,0xad,0x00,0x05,0xff,0xe8,0xae,0x8a,0x00,0x10,0x08,0x05, + 0xff,0xe8,0xb1,0x95,0x00,0x05,0xff,0xf0,0xa7,0xb2,0xa8,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe8,0xb2,0xab,0x00,0x05,0xff,0xe8,0xb3,0x81,0x00,0x10,0x08,0x05,0xff, + 0xe8,0xb4,0x9b,0x00,0x05,0xff,0xe8,0xb5,0xb7,0x00,0xd2,0x22,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0xa7,0xbc,0xaf,0x00,0x05,0xff,0xf0,0xa0,0xa0,0x84,0x00,0x10,0x08, + 0x05,0xff,0xe8,0xb7,0x8b,0x00,0x05,0xff,0xe8,0xb6,0xbc,0x00,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe8,0xb7,0xb0,0x00,0x05,0xff,0xf0,0xa0,0xa3,0x9e,0x00,0x10,0x08,0x05, + 0xff,0xe8,0xbb,0x94,0x00,0x05,0xff,0xe8,0xbc,0xb8,0x00,0xd4,0x84,0xd3,0x43,0xd2, + 0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa8,0x97,0x92,0x00,0x05,0xff,0xf0,0xa8, + 0x97,0xad,0x00,0x10,0x08,0x05,0xff,0xe9,0x82,0x94,0x00,0x05,0xff,0xe9,0x83,0xb1, + 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x84,0x91,0x00,0x05,0xff,0xf0,0xa8,0x9c, + 0xae,0x00,0x10,0x08,0x05,0xff,0xe9,0x84,0x9b,0x00,0x05,0xff,0xe9,0x88,0xb8,0x00, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe9,0x8b,0x97,0x00,0x05,0xff,0xe9,0x8b, + 0x98,0x00,0x10,0x08,0x05,0xff,0xe9,0x89,0xbc,0x00,0x05,0xff,0xe9,0x8f,0xb9,0x00, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x90,0x95,0x00,0x05,0xff,0xf0,0xa8,0xaf,0xba, + 0x00,0x10,0x08,0x05,0xff,0xe9,0x96,0x8b,0x00,0x05,0xff,0xe4,0xa6,0x95,0x00,0xd3, + 0x43,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x96,0xb7,0x00,0x05,0xff,0xf0, + 0xa8,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe4,0xa7,0xa6,0x00,0x05,0xff,0xe9,0x9b, + 0x83,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb6,0xb2,0x00,0x05,0xff,0xe9,0x9c, + 0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa9,0x85,0x85,0x00,0x05,0xff,0xf0,0xa9,0x88, + 0x9a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xa9,0xae,0x00,0x05,0xff, + 0xe4,0xa9,0xb6,0x00,0x10,0x08,0x05,0xff,0xe9,0x9f,0xa0,0x00,0x05,0xff,0xf0,0xa9, + 0x90,0x8a,0x00,0x91,0x11,0x10,0x08,0x05,0xff,0xe4,0xaa,0xb2,0x00,0x05,0xff,0xf0, + 0xa9,0x92,0x96,0x00,0x05,0xff,0xe9,0xa0,0x8b,0x00,0xe2,0x10,0x01,0xe1,0x09,0x01, + 0xe0,0x02,0x01,0xcf,0x86,0x95,0xfb,0xd4,0x82,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe9,0xa0,0xa9,0x00,0x05,0xff,0xf0,0xa9,0x96,0xb6,0x00,0x10,0x08, + 0x05,0xff,0xe9,0xa3,0xa2,0x00,0x05,0xff,0xe4,0xac,0xb3,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe9,0xa4,0xa9,0x00,0x05,0xff,0xe9,0xa6,0xa7,0x00,0x10,0x08,0x05,0xff, + 0xe9,0xa7,0x82,0x00,0x05,0xff,0xe9,0xa7,0xbe,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe4,0xaf,0x8e,0x00,0x05,0xff,0xf0,0xa9,0xac,0xb0,0x00,0x10,0x08,0x05, + 0xff,0xe9,0xac,0x92,0x00,0x05,0xff,0xe9,0xb1,0x80,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe9,0xb3,0xbd,0x00,0x05,0xff,0xe4,0xb3,0x8e,0x00,0x10,0x08,0x05,0xff,0xe4, + 0xb3,0xad,0x00,0x05,0xff,0xe9,0xb5,0xa7,0x00,0xd3,0x44,0xd2,0x23,0xd1,0x11,0x10, + 0x09,0x05,0xff,0xf0,0xaa,0x83,0x8e,0x00,0x05,0xff,0xe4,0xb3,0xb8,0x00,0x10,0x09, + 0x05,0xff,0xf0,0xaa,0x84,0x85,0x00,0x05,0xff,0xf0,0xaa,0x88,0x8e,0x00,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xaa,0x8a,0x91,0x00,0x05,0xff,0xe9,0xba,0xbb,0x00,0x10, + 0x08,0x05,0xff,0xe4,0xb5,0x96,0x00,0x05,0xff,0xe9,0xbb,0xb9,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe9,0xbb,0xbe,0x00,0x05,0xff,0xe9,0xbc,0x85,0x00,0x10, + 0x08,0x05,0xff,0xe9,0xbc,0x8f,0x00,0x05,0xff,0xe9,0xbc,0x96,0x00,0x91,0x11,0x10, + 0x08,0x05,0xff,0xe9,0xbc,0xbb,0x00,0x05,0xff,0xf0,0xaa,0x98,0x80,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, + 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, + 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xd5,0xc0, + 0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06, + 0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06, + 0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00, + 0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06, + 0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, + 0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00, + 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd4,0x60, + 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, + 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, + 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xe0,0x83,0x01,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, + 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, + 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00, + 0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3, + 0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00, + 0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00, + 0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, + 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, + 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd4,0xd9,0xd3,0x81,0xd2,0x79,0xd1,0x71,0xd0,0x69,0xcf,0x86,0xd5,0x60,0xd4, + 0x59,0xd3,0x52,0xd2,0x33,0xd1,0x2c,0xd0,0x25,0xcf,0x86,0x95,0x1e,0x94,0x19,0x93, + 0x14,0x92,0x0f,0x91,0x0a,0x10,0x05,0x00,0xff,0x00,0x05,0xff,0x00,0x00,0xff,0x00, + 0x00,0xff,0x00,0x00,0xff,0x00,0x00,0xff,0x00,0x05,0xff,0x00,0xcf,0x06,0x05,0xff, + 0x00,0xcf,0x06,0x00,0xff,0x00,0xd1,0x07,0xcf,0x06,0x07,0xff,0x00,0xd0,0x07,0xcf, + 0x06,0x07,0xff,0x00,0xcf,0x86,0x55,0x05,0x07,0xff,0x00,0x14,0x05,0x07,0xff,0x00, + 0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00, + 0xff,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86, + 0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00, + 0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06, + 0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00, + 0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xcf,0x06,0x02,0x00, + 0x81,0x80,0xcf,0x86,0x85,0x84,0xcf,0x86,0xcf,0x06,0x02,0x00,0x00,0x00,0x00,0x00 +}; + +struct utf8data_table utf8_data_table = { + .utf8agetab = utf8agetab, + .utf8agetab_size = ARRAY_SIZE(utf8agetab), + + .utf8nfdicfdata = utf8nfdicfdata, + .utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata), + + .utf8nfdidata = utf8nfdidata, + .utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata), + + .utf8data = utf8data, +}; +EXPORT_SYMBOL_GPL(utf8_data_table); +MODULE_LICENSE("GPL v2"); diff --git a/fs/unicode/utf8data.h_shipped b/fs/unicode/utf8data.h_shipped deleted file mode 100644 index 76e4f0e1b089..000000000000 --- a/fs/unicode/utf8data.h_shipped +++ /dev/null @@ -1,4109 +0,0 @@ -/* This file is generated code, do not edit. */ -#ifndef __INCLUDED_FROM_UTF8NORM_C__ -#error Only nls_utf8-norm.c should include this file. -#endif - -static const unsigned int utf8vers = 0xc0100; - -static const unsigned int utf8agetab[] = { - 0, - 0x10100, - 0x20000, - 0x20100, - 0x30000, - 0x30100, - 0x30200, - 0x40000, - 0x40100, - 0x50000, - 0x50100, - 0x50200, - 0x60000, - 0x60100, - 0x60200, - 0x60300, - 0x70000, - 0x80000, - 0x90000, - 0xa0000, - 0xb0000, - 0xc0000, - 0xc0100 -}; - -static const struct utf8data utf8nfdicfdata[] = { - { 0, 0 }, - { 0x10100, 0 }, - { 0x20000, 0 }, - { 0x20100, 0 }, - { 0x30000, 0 }, - { 0x30100, 0 }, - { 0x30200, 1792 }, - { 0x40000, 3200 }, - { 0x40100, 3200 }, - { 0x50000, 3200 }, - { 0x50100, 3200 }, - { 0x50200, 3200 }, - { 0x60000, 3200 }, - { 0x60100, 3200 }, - { 0x60200, 3200 }, - { 0x60300, 3200 }, - { 0x70000, 3200 }, - { 0x80000, 3200 }, - { 0x90000, 3200 }, - { 0xa0000, 3200 }, - { 0xb0000, 3200 }, - { 0xc0000, 3200 }, - { 0xc0100, 3200 } -}; - -static const struct utf8data utf8nfdidata[] = { - { 0, 896 }, - { 0x10100, 896 }, - { 0x20000, 896 }, - { 0x20100, 896 }, - { 0x30000, 896 }, - { 0x30100, 896 }, - { 0x30200, 2496 }, - { 0x40000, 20736 }, - { 0x40100, 20736 }, - { 0x50000, 20736 }, - { 0x50100, 20736 }, - { 0x50200, 20736 }, - { 0x60000, 20736 }, - { 0x60100, 20736 }, - { 0x60200, 20736 }, - { 0x60300, 20736 }, - { 0x70000, 20736 }, - { 0x80000, 20736 }, - { 0x90000, 20736 }, - { 0xa0000, 20736 }, - { 0xb0000, 20736 }, - { 0xc0000, 20736 }, - { 0xc0100, 20736 } -}; - -static const unsigned char utf8data[64256] = { - /* nfdicf_30100 */ - 0xd7,0x07,0x66,0x84,0x0c,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x1a,0xe3,0x63,0x15, - 0xe2,0x4c,0x0e,0xc1,0xe0,0x4e,0x0d,0xcf,0x86,0x65,0x2d,0x0d,0x01,0x00,0xd4,0xb8, - 0xd3,0x27,0xe2,0x89,0xa3,0xe1,0xce,0x35,0xe0,0x2c,0x22,0xcf,0x86,0xc5,0xe4,0x15, - 0x6d,0xe3,0x60,0x68,0xe2,0xf6,0x65,0xe1,0x29,0x65,0xe0,0xee,0x64,0xcf,0x86,0xe5, - 0xb3,0x64,0x64,0x96,0x64,0x0b,0x00,0xd2,0x0e,0xe1,0xb5,0x3c,0xe0,0xba,0xa3,0xcf, - 0x86,0xcf,0x06,0x01,0x00,0xd1,0x0c,0xe0,0x1e,0xa9,0xcf,0x86,0xcf,0x06,0x02,0xff, - 0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01, - 0x00,0xe4,0xe1,0x45,0xe3,0x3b,0x45,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x87,0xad, - 0xd0,0x21,0xcf,0x86,0xe5,0x81,0xaa,0xe4,0x00,0xaa,0xe3,0xbf,0xa9,0xe2,0x9e,0xa9, - 0xe1,0x8d,0xa9,0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4, - 0x00,0xcf,0x86,0xe5,0x63,0xac,0xd4,0x19,0xe3,0xa2,0xab,0xe2,0x81,0xab,0xe1,0x70, - 0xab,0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3, - 0x09,0xac,0xe2,0xe8,0xab,0xe1,0xd7,0xab,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00, - 0x01,0xff,0xe9,0x9b,0xbb,0x00,0x83,0xe2,0x19,0xfa,0xe1,0xf2,0xf6,0xe0,0x6f,0xf5, - 0xcf,0x86,0xd5,0x31,0xc4,0xe3,0x54,0x4e,0xe2,0xf5,0x4c,0xe1,0xa4,0xcc,0xe0,0x9c, - 0x4b,0xcf,0x86,0xe5,0x8e,0x49,0xe4,0xaf,0x46,0xe3,0x11,0xbd,0xe2,0x68,0xbc,0xe1, - 0x43,0xbc,0xe0,0x1c,0xbc,0xcf,0x86,0xe5,0xe9,0xbb,0x94,0x07,0x63,0xd4,0xbb,0x07, - 0x00,0x07,0x00,0xe4,0xdb,0xf4,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b, - 0xe1,0xea,0xe1,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xd9,0xe2,0xcf,0x86, - 0xe5,0x9e,0xe2,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xd9,0xe2,0xcf,0x06, - 0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x74,0xf4,0xe3,0x5d,0xf3, - 0xd2,0xa0,0xe1,0x13,0xe7,0xd0,0x21,0xcf,0x86,0xe5,0x14,0xe4,0xe4,0x90,0xe3,0xe3, - 0x4e,0xe3,0xe2,0x2d,0xe3,0xe1,0x1b,0xe3,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00, - 0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x70,0xe5,0xe3,0x2f,0xe5, - 0xe2,0x0e,0xe5,0xe1,0xfd,0xe4,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff, - 0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xf7,0xe5,0xe1,0xe6,0xe5,0x10,0x09, - 0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x17, - 0xe6,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac, - 0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x5d,0xe6,0xd2,0x14,0xe1,0x2c,0xe6, - 0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1, - 0x38,0xe6,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00, - 0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x8d,0xeb,0xd4,0x19,0xe3,0xc6,0xea,0xe2,0xa4, - 0xea,0xe1,0x93,0xea,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5, - 0xb7,0x00,0xd3,0x18,0xe2,0x10,0xeb,0xe1,0xff,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3, - 0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0x28,0xeb,0x10, - 0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10, - 0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08, - 0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0x2a, - 0xed,0xd4,0x1a,0xe3,0x62,0xec,0xe2,0x48,0xec,0xe1,0x35,0xec,0x10,0x08,0x05,0xff, - 0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0xaa,0xec, - 0xe1,0x98,0xec,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3, - 0x00,0xd2,0x13,0xe1,0xc6,0xec,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff, - 0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05, - 0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05, - 0xff,0xe7,0xaa,0xae,0x00,0xe0,0xdc,0xef,0xcf,0x86,0xd5,0x1d,0xe4,0x51,0xee,0xe3, - 0x0d,0xee,0xe2,0xeb,0xed,0xe1,0xda,0xed,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f, - 0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xf8,0xee,0xe2,0xd4,0xee,0xe1, - 0xc3,0xee,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00, - 0xd3,0x18,0xe2,0x43,0xef,0xe1,0x32,0xef,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1, - 0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x5b,0xef,0x10,0x08,0x05, - 0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8, - 0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - /* nfdi_30100 */ - 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0xc2,0x59,0xe3,0xfb,0x54,0xe2,0x74,0x4f, - 0xc1,0xe0,0xa0,0x4d,0xcf,0x86,0x65,0x84,0x4d,0x01,0x00,0xd4,0xb8,0xd3,0x27,0xe2, - 0x0c,0xa0,0xe1,0xdf,0x8d,0xe0,0x39,0x71,0xcf,0x86,0xc5,0xe4,0x98,0x69,0xe3,0xe3, - 0x64,0xe2,0x79,0x62,0xe1,0xac,0x61,0xe0,0x71,0x61,0xcf,0x86,0xe5,0x36,0x61,0x64, - 0x19,0x61,0x0b,0x00,0xd2,0x0e,0xe1,0xc2,0xa0,0xe0,0x3d,0xa0,0xcf,0x86,0xcf,0x06, - 0x01,0x00,0xd1,0x0c,0xe0,0xa1,0xa5,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd0,0x08, - 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xe4,0x9e, - 0xb6,0xe3,0x18,0xae,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x0a,0xaa,0xd0,0x21,0xcf, - 0x86,0xe5,0x04,0xa7,0xe4,0x83,0xa6,0xe3,0x42,0xa6,0xe2,0x21,0xa6,0xe1,0x10,0xa6, - 0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0xcf,0x86, - 0xe5,0xe6,0xa8,0xd4,0x19,0xe3,0x25,0xa8,0xe2,0x04,0xa8,0xe1,0xf3,0xa7,0x10,0x08, - 0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3,0x8c,0xa8,0xe2, - 0x6b,0xa8,0xe1,0x5a,0xa8,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9, - 0x9b,0xbb,0x00,0x83,0xe2,0x9c,0xf6,0xe1,0x75,0xf3,0xe0,0xf2,0xf1,0xcf,0x86,0xd5, - 0x31,0xc4,0xe3,0x6d,0xcc,0xe2,0x46,0xca,0xe1,0x27,0xc9,0xe0,0xb7,0xbf,0xcf,0x86, - 0xe5,0xaa,0xbb,0xe4,0xa3,0xba,0xe3,0x94,0xb9,0xe2,0xeb,0xb8,0xe1,0xc6,0xb8,0xe0, - 0x9f,0xb8,0xcf,0x86,0xe5,0x6c,0xb8,0x94,0x07,0x63,0x57,0xb8,0x07,0x00,0x07,0x00, - 0xe4,0x5e,0xf1,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1,0x6d,0xde, - 0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x5c,0xdf,0xcf,0x86,0xe5,0x21,0xdf, - 0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x5c,0xdf,0xcf,0x06,0x13,0x00,0xcf, - 0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0xf7,0xf0,0xe3,0xe0,0xef,0xd2,0xa0,0xe1, - 0x96,0xe3,0xd0,0x21,0xcf,0x86,0xe5,0x97,0xe0,0xe4,0x13,0xe0,0xe3,0xd1,0xdf,0xe2, - 0xb0,0xdf,0xe1,0x9e,0xdf,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4, - 0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xf3,0xe1,0xe3,0xb2,0xe1,0xe2,0x91,0xe1, - 0xe1,0x80,0xe1,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6, - 0x00,0xd4,0x34,0xd3,0x18,0xe2,0x7a,0xe2,0xe1,0x69,0xe2,0x10,0x09,0x05,0xff,0xf0, - 0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x9a,0xe2,0x91,0x11, - 0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, - 0xff,0xe5,0xac,0xbe,0x00,0xe3,0xe0,0xe2,0xd2,0x14,0xe1,0xaf,0xe2,0x10,0x08,0x05, - 0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0xbb,0xe2,0x10, - 0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1,0xd5,0xd0, - 0x6a,0xcf,0x86,0xe5,0x10,0xe8,0xd4,0x19,0xe3,0x49,0xe7,0xe2,0x27,0xe7,0xe1,0x16, - 0xe7,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7,0x00,0xd3, - 0x18,0xe2,0x93,0xe7,0xe1,0x82,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00, - 0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xab,0xe7,0x10,0x08,0x05,0xff, - 0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, - 0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7, - 0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xad,0xe9,0xd4,0x1a, - 0xe3,0xe5,0xe8,0xe2,0xcb,0xe8,0xe1,0xb8,0xe8,0x10,0x08,0x05,0xff,0xe7,0x9b,0xb4, - 0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x2d,0xe9,0xe1,0x1b,0xe9, - 0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0xd2,0x13, - 0xe1,0x49,0xe9,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80, - 0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff,0xf0,0xa5, - 0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff,0xe7,0xaa, - 0xae,0x00,0xe0,0x5f,0xec,0xcf,0x86,0xd5,0x1d,0xe4,0xd4,0xea,0xe3,0x90,0xea,0xe2, - 0x6e,0xea,0xe1,0x5d,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff, - 0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x7b,0xeb,0xe2,0x57,0xeb,0xe1,0x46,0xeb,0x10, - 0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3,0x18,0xe2, - 0xc6,0xeb,0xe1,0xb5,0xeb,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00,0x05,0xff, - 0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xde,0xeb,0x10,0x08,0x05,0xff,0xe8,0x9a, - 0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c, - 0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00, - 0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - /* nfdicf_30200 */ - 0xd7,0x07,0x66,0x84,0x05,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x13,0xe3,0x63,0x0e, - 0xe2,0x4c,0x07,0xc1,0xe0,0x4e,0x06,0xcf,0x86,0x65,0x2d,0x06,0x01,0x00,0xd4,0x2a, - 0xe3,0xd0,0x35,0xe2,0x88,0x9c,0xe1,0xcd,0x2e,0xe0,0x2b,0x1b,0xcf,0x86,0xc5,0xe4, - 0x14,0x66,0xe3,0x5f,0x61,0xe2,0xf5,0x5e,0xe1,0x28,0x5e,0xe0,0xed,0x5d,0xcf,0x86, - 0xe5,0xb2,0x5d,0x64,0x95,0x5d,0x0b,0x00,0x83,0xe2,0xa7,0xf3,0xe1,0x80,0xf0,0xe0, - 0xfd,0xee,0xcf,0x86,0xd5,0x31,0xc4,0xe3,0xe2,0x47,0xe2,0x83,0x46,0xe1,0x32,0xc6, - 0xe0,0x2a,0x45,0xcf,0x86,0xe5,0x1c,0x43,0xe4,0x3d,0x40,0xe3,0x9f,0xb6,0xe2,0xf6, - 0xb5,0xe1,0xd1,0xb5,0xe0,0xaa,0xb5,0xcf,0x86,0xe5,0x77,0xb5,0x94,0x07,0x63,0x62, - 0xb5,0x07,0x00,0x07,0x00,0xe4,0x69,0xee,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, - 0xd2,0x0b,0xe1,0x78,0xdb,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x67,0xdc, - 0xcf,0x86,0xe5,0x2c,0xdc,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x67,0xdc, - 0xcf,0x06,0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x02,0xee,0xe3, - 0xeb,0xec,0xd2,0xa0,0xe1,0xa1,0xe0,0xd0,0x21,0xcf,0x86,0xe5,0xa2,0xdd,0xe4,0x1e, - 0xdd,0xe3,0xdc,0xdc,0xe2,0xbb,0xdc,0xe1,0xa9,0xdc,0x10,0x08,0x05,0xff,0xe4,0xb8, - 0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xfe,0xde,0xe3, - 0xbd,0xde,0xe2,0x9c,0xde,0xe1,0x8b,0xde,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00, - 0x05,0xff,0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0x85,0xdf,0xe1,0x74,0xdf, - 0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00, - 0xe2,0xa5,0xdf,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff, - 0xe5,0xac,0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0xeb,0xdf,0xd2,0x14,0xe1, - 0xba,0xdf,0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98, - 0x00,0xe1,0xc6,0xdf,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0, - 0xa2,0x00,0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x1b,0xe5,0xd4,0x19,0xe3,0x54,0xe4, - 0xe2,0x32,0xe4,0xe1,0x21,0xe4,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, - 0xe6,0xb5,0xb7,0x00,0xd3,0x18,0xe2,0x9e,0xe4,0xe1,0x8d,0xe4,0x10,0x09,0x05,0xff, - 0xf0,0xa3,0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xb6, - 0xe4,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1, - 0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00, - 0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86, - 0xe5,0xb8,0xe6,0xd4,0x1a,0xe3,0xf0,0xe5,0xe2,0xd6,0xe5,0xe1,0xc3,0xe5,0x10,0x08, - 0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2, - 0x38,0xe6,0xe1,0x26,0xe6,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4, - 0x83,0xa3,0x00,0xd2,0x13,0xe1,0x54,0xe6,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00, - 0x05,0xff,0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc, - 0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7, - 0x00,0x05,0xff,0xe7,0xaa,0xae,0x00,0xe0,0x6a,0xe9,0xcf,0x86,0xd5,0x1d,0xe4,0xdf, - 0xe7,0xe3,0x9b,0xe7,0xe2,0x79,0xe7,0xe1,0x68,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3, - 0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x86,0xe8,0xe2,0x62, - 0xe8,0xe1,0x51,0xe8,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, - 0x8a,0x00,0xd3,0x18,0xe2,0xd1,0xe8,0xe1,0xc0,0xe8,0x10,0x09,0x05,0xff,0xf0,0xa6, - 0xbe,0xb1,0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xe9,0xe8,0x10, - 0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05, - 0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00, - /* nfdi_30200 */ - 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x82,0x53,0xe3,0xbb,0x4e,0xe2,0x34,0x49, - 0xc1,0xe0,0x60,0x47,0xcf,0x86,0x65,0x44,0x47,0x01,0x00,0xd4,0x2a,0xe3,0x1c,0x9a, - 0xe2,0xcb,0x99,0xe1,0x9e,0x87,0xe0,0xf8,0x6a,0xcf,0x86,0xc5,0xe4,0x57,0x63,0xe3, - 0xa2,0x5e,0xe2,0x38,0x5c,0xe1,0x6b,0x5b,0xe0,0x30,0x5b,0xcf,0x86,0xe5,0xf5,0x5a, - 0x64,0xd8,0x5a,0x0b,0x00,0x83,0xe2,0xea,0xf0,0xe1,0xc3,0xed,0xe0,0x40,0xec,0xcf, - 0x86,0xd5,0x31,0xc4,0xe3,0xbb,0xc6,0xe2,0x94,0xc4,0xe1,0x75,0xc3,0xe0,0x05,0xba, - 0xcf,0x86,0xe5,0xf8,0xb5,0xe4,0xf1,0xb4,0xe3,0xe2,0xb3,0xe2,0x39,0xb3,0xe1,0x14, - 0xb3,0xe0,0xed,0xb2,0xcf,0x86,0xe5,0xba,0xb2,0x94,0x07,0x63,0xa5,0xb2,0x07,0x00, - 0x07,0x00,0xe4,0xac,0xeb,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1, - 0xbb,0xd8,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xaa,0xd9,0xcf,0x86,0xe5, - 0x6f,0xd9,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xaa,0xd9,0xcf,0x06,0x13, - 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x45,0xeb,0xe3,0x2e,0xea,0xd2, - 0xa0,0xe1,0xe4,0xdd,0xd0,0x21,0xcf,0x86,0xe5,0xe5,0xda,0xe4,0x61,0xda,0xe3,0x1f, - 0xda,0xe2,0xfe,0xd9,0xe1,0xec,0xd9,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05, - 0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x41,0xdc,0xe3,0x00,0xdc,0xe2, - 0xdf,0xdb,0xe1,0xce,0xdb,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5, - 0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xc8,0xdc,0xe1,0xb7,0xdc,0x10,0x09,0x05, - 0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0xe8,0xdc, - 0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88, - 0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x2e,0xdd,0xd2,0x14,0xe1,0xfd,0xdc,0x10, - 0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0x09, - 0xdd,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1, - 0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x5e,0xe2,0xd4,0x19,0xe3,0x97,0xe1,0xe2,0x75,0xe1, - 0xe1,0x64,0xe1,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7, - 0x00,0xd3,0x18,0xe2,0xe1,0xe1,0xe1,0xd0,0xe1,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd, - 0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xf9,0xe1,0x10,0x08, - 0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08, - 0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05, - 0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xfb,0xe3, - 0xd4,0x1a,0xe3,0x33,0xe3,0xe2,0x19,0xe3,0xe1,0x06,0xe3,0x10,0x08,0x05,0xff,0xe7, - 0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x7b,0xe3,0xe1, - 0x69,0xe3,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00, - 0xd2,0x13,0xe1,0x97,0xe3,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7, - 0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff, - 0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff, - 0xe7,0xaa,0xae,0x00,0xe0,0xad,0xe6,0xcf,0x86,0xd5,0x1d,0xe4,0x22,0xe5,0xe3,0xde, - 0xe4,0xe2,0xbc,0xe4,0xe1,0xab,0xe4,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00, - 0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xc9,0xe5,0xe2,0xa5,0xe5,0xe1,0x94, - 0xe5,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3, - 0x18,0xe2,0x14,0xe6,0xe1,0x03,0xe6,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, - 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x2c,0xe6,0x10,0x08,0x05,0xff, - 0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, - 0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e, - 0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - /* nfdicf_c0100 */ - 0xd7,0xb0,0x56,0x04,0x01,0x00,0x95,0xa8,0xd4,0x5e,0xd3,0x2e,0xd2,0x16,0xd1,0x0a, - 0x10,0x04,0x01,0x00,0x01,0xff,0x61,0x00,0x10,0x06,0x01,0xff,0x62,0x00,0x01,0xff, - 0x63,0x00,0xd1,0x0c,0x10,0x06,0x01,0xff,0x64,0x00,0x01,0xff,0x65,0x00,0x10,0x06, - 0x01,0xff,0x66,0x00,0x01,0xff,0x67,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, - 0x68,0x00,0x01,0xff,0x69,0x00,0x10,0x06,0x01,0xff,0x6a,0x00,0x01,0xff,0x6b,0x00, - 0xd1,0x0c,0x10,0x06,0x01,0xff,0x6c,0x00,0x01,0xff,0x6d,0x00,0x10,0x06,0x01,0xff, - 0x6e,0x00,0x01,0xff,0x6f,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, - 0x70,0x00,0x01,0xff,0x71,0x00,0x10,0x06,0x01,0xff,0x72,0x00,0x01,0xff,0x73,0x00, - 0xd1,0x0c,0x10,0x06,0x01,0xff,0x74,0x00,0x01,0xff,0x75,0x00,0x10,0x06,0x01,0xff, - 0x76,0x00,0x01,0xff,0x77,0x00,0x92,0x16,0xd1,0x0c,0x10,0x06,0x01,0xff,0x78,0x00, - 0x01,0xff,0x79,0x00,0x10,0x06,0x01,0xff,0x7a,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0xc6,0xe5,0xf9,0x14,0xe4,0x6f,0x0d,0xe3,0x39,0x08,0xe2,0x22,0x01,0xc1,0xd0,0x24, - 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x07,0x63,0xd8,0x43,0x01,0x00,0x93,0x13,0x52, - 0x04,0x01,0x00,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xce,0xbc,0x00,0x01,0x00, - 0x01,0x00,0xcf,0x86,0xe5,0xb3,0x44,0xd4,0x7f,0xd3,0x3f,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, - 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x07,0x01,0xff,0xc3, - 0xa6,0x00,0x01,0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x65,0xcc,0x80,0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, - 0x82,0x00,0x01,0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc, - 0x80,0x00,0x01,0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00, - 0x01,0xff,0x69,0xcc,0x88,0x00,0xd3,0x3b,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff, - 0xc3,0xb0,0x00,0x01,0xff,0x6e,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80, - 0x00,0x01,0xff,0x6f,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82, - 0x00,0x01,0xff,0x6f,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01, - 0x00,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff,0xc3,0xb8,0x00,0x01,0xff,0x75,0xcc, - 0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x82,0x00, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88,0x00,0x01,0xff,0x79,0xcc,0x81,0x00, - 0x10,0x07,0x01,0xff,0xc3,0xbe,0x00,0x01,0xff,0x73,0x73,0x00,0xe1,0xd4,0x03,0xe0, - 0xeb,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0x61,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84,0x00,0x10,0x08,0x01,0xff, - 0x61,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x61,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, - 0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x63,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, - 0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x63,0xcc, - 0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x8c,0x00, - 0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x3b,0xd2,0x1b,0xd1,0x0b,0x10,0x07,0x01,0xff, - 0xc4,0x91,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x84,0x00,0x01,0xff,0x65, - 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x86,0x00,0x01,0xff,0x65, - 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0xa8,0x00,0x01,0xff,0x65, - 0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82, - 0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4, - 0x7b,0xd3,0x3b,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x87,0x00,0x01, - 0xff,0x67,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0xa7,0x00,0x01,0xff,0x67, - 0xcc,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x68,0xcc,0x82,0x00,0x01,0xff,0x68, - 0xcc,0x82,0x00,0x10,0x07,0x01,0xff,0xc4,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0x69,0xcc,0x83,0x00,0x01,0xff,0x69,0xcc,0x83,0x00,0x10,0x08, - 0x01,0xff,0x69,0xcc,0x84,0x00,0x01,0xff,0x69,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0x69,0xcc,0x86,0x00,0x01,0xff,0x69,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, - 0x69,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8,0x00,0xd3,0x37,0xd2,0x17,0xd1,0x0c, - 0x10,0x08,0x01,0xff,0x69,0xcc,0x87,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc4,0xb3, - 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6a,0xcc,0x82,0x00,0x01,0xff,0x6a, - 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, - 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6c,0xcc,0x81,0x00,0x10, - 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x6c,0xcc,0xa7,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x6c,0xcc,0x8c,0x00,0x10,0x08,0x01, - 0xff,0x6c,0xcc,0x8c,0x00,0x01,0xff,0xc5,0x80,0x00,0xcf,0x86,0xd5,0xed,0xd4,0x72, - 0xd3,0x37,0xd2,0x17,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc5,0x82,0x00,0x10, - 0x04,0x01,0x00,0x01,0xff,0x6e,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, - 0xcc,0x81,0x00,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa7, - 0x00,0x01,0xff,0x6e,0xcc,0x8c,0x00,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, - 0xcc,0x8c,0x00,0x01,0xff,0xca,0xbc,0x6e,0x00,0x10,0x07,0x01,0xff,0xc5,0x8b,0x00, - 0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc, - 0x84,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00, - 0xd3,0x3b,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01,0xff, - 0x6f,0xcc,0x8b,0x00,0x10,0x07,0x01,0xff,0xc5,0x93,0x00,0x01,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x72,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81,0x00,0x10,0x08,0x01, - 0xff,0x72,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x72,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c,0x00,0x10,0x08,0x01, - 0xff,0x73,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x73,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x73, - 0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x7b,0xd3,0x3b,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x73,0xcc,0x8c,0x00,0x01,0xff,0x73,0xcc,0x8c,0x00,0x10, - 0x08,0x01,0xff,0x74,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x74,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c,0x00,0x10,0x07,0x01, - 0xff,0xc5,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, - 0x83,0x00,0x01,0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x84,0x00, - 0x01,0xff,0x75,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x86,0x00, - 0x01,0xff,0x75,0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x8a,0x00,0x01,0xff, - 0x75,0xcc,0x8a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, - 0x8b,0x00,0x01,0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa8,0x00, - 0x01,0xff,0x75,0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x82,0x00, - 0x01,0xff,0x77,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x82,0x00,0x01,0xff, - 0x79,0xcc,0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x79,0xcc,0x88,0x00, - 0x01,0xff,0x7a,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff, - 0x7a,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff, - 0x7a,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0xff,0x73,0x00, - 0xe0,0x65,0x01,0xcf,0x86,0xd5,0xb4,0xd4,0x5a,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10, - 0x04,0x01,0x00,0x01,0xff,0xc9,0x93,0x00,0x10,0x07,0x01,0xff,0xc6,0x83,0x00,0x01, - 0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff, - 0xc9,0x94,0x00,0x01,0xff,0xc6,0x88,0x00,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x01,0x00, - 0x01,0xff,0xc9,0x96,0x00,0x10,0x07,0x01,0xff,0xc9,0x97,0x00,0x01,0xff,0xc6,0x8c, - 0x00,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xc7,0x9d,0x00,0x01,0xff,0xc9,0x99, - 0x00,0xd3,0x32,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0x9b,0x00,0x01,0xff, - 0xc6,0x92,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xa0,0x00,0xd1,0x0b,0x10,0x07, - 0x01,0xff,0xc9,0xa3,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc9,0xa9,0x00,0x01,0xff, - 0xc9,0xa8,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0x99,0x00,0x01,0x00, - 0x01,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0xaf,0x00,0x01,0xff,0xc9,0xb2,0x00, - 0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xb5,0x00,0xd4,0x5d,0xd3,0x34,0xd2,0x1b,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x10, - 0x07,0x01,0xff,0xc6,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0xa5, - 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x80,0x00,0x01,0xff,0xc6,0xa8,0x00,0xd2, - 0x0f,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x83,0x00,0x01,0x00,0xd1,0x0b, - 0x10,0x07,0x01,0xff,0xc6,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x88,0x00, - 0x01,0xff,0x75,0xcc,0x9b,0x00,0xd3,0x33,0xd2,0x1d,0xd1,0x0f,0x10,0x08,0x01,0xff, - 0x75,0xcc,0x9b,0x00,0x01,0xff,0xca,0x8a,0x00,0x10,0x07,0x01,0xff,0xca,0x8b,0x00, - 0x01,0xff,0xc6,0xb4,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc6,0xb6,0x00, - 0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x92,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01, - 0xff,0xc6,0xb9,0x00,0x01,0x00,0x01,0x00,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0xbd, - 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x44,0xd3,0x16,0x52,0x04,0x01, - 0x00,0x51,0x07,0x01,0xff,0xc7,0x86,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc7,0x89, - 0x00,0xd2,0x12,0x91,0x0b,0x10,0x07,0x01,0xff,0xc7,0x89,0x00,0x01,0x00,0x01,0xff, - 0xc7,0x8c,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x61,0xcc,0x8c,0x00,0x10, - 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x69,0xcc,0x8c,0x00,0xd3,0x46,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x6f,0xcc,0x8c, - 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x8c,0x00,0xd1, - 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84, - 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x75,0xcc,0x88, - 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, - 0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, - 0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, - 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x88, - 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x87,0xd3,0x41,0xd2, - 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, - 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x84,0x00,0x01,0xff, - 0xc3,0xa6,0xcc,0x84,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc7,0xa5,0x00,0x01,0x00, - 0x10,0x08,0x01,0xff,0x67,0xcc,0x8c,0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00, - 0x10,0x08,0x01,0xff,0x6f,0xcc,0xa8,0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14, - 0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc, - 0x84,0x00,0x10,0x09,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc, - 0x8c,0x00,0xd3,0x38,0xd2,0x1a,0xd1,0x0f,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00, - 0x01,0xff,0xc7,0xb3,0x00,0x10,0x07,0x01,0xff,0xc7,0xb3,0x00,0x01,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0x67,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x10,0x07, - 0x04,0xff,0xc6,0x95,0x00,0x04,0xff,0xc6,0xbf,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08, - 0x04,0xff,0x6e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, - 0x61,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a,0xcc,0x81,0x00,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00, - 0x10,0x09,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00, - 0xe2,0x31,0x02,0xe1,0xc3,0x44,0xe0,0xc8,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0x8f,0x00,0x01,0xff,0x61, - 0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x91,0x00,0x01,0xff,0x61,0xcc,0x91, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x8f,0x00,0x01,0xff,0x65,0xcc,0x8f, - 0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x91,0x00,0x01,0xff,0x65,0xcc,0x91,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8f,0x00,0x01,0xff,0x69,0xcc,0x8f, - 0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x91,0x00,0x01,0xff,0x69,0xcc,0x91,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x10, - 0x08,0x01,0xff,0x6f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc,0x91,0x00,0xd3,0x40,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x8f,0x00,0x01,0xff,0x72,0xcc,0x8f, - 0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0x91,0x00,0x01,0xff,0x72,0xcc,0x91,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x8f,0x00,0x01,0xff,0x75,0xcc,0x8f,0x00,0x10, - 0x08,0x01,0xff,0x75,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc,0x91,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x04,0xff,0x73,0xcc,0xa6,0x00,0x04,0xff,0x73,0xcc,0xa6,0x00,0x10, - 0x08,0x04,0xff,0x74,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc,0xa6,0x00,0xd1,0x0b,0x10, - 0x07,0x04,0xff,0xc8,0x9d,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x68,0xcc,0x8c,0x00, - 0x04,0xff,0x68,0xcc,0x8c,0x00,0xd4,0x79,0xd3,0x31,0xd2,0x16,0xd1,0x0b,0x10,0x07, - 0x06,0xff,0xc6,0x9e,0x00,0x07,0x00,0x10,0x07,0x04,0xff,0xc8,0xa3,0x00,0x04,0x00, - 0xd1,0x0b,0x10,0x07,0x04,0xff,0xc8,0xa5,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x61, - 0xcc,0x87,0x00,0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04, - 0xff,0x65,0xcc,0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x6f, - 0xcc,0x88,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10, - 0x0a,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84, - 0x00,0x10,0x08,0x04,0xff,0x6f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0xd3, - 0x27,0xe2,0x21,0x43,0xd1,0x14,0x10,0x0a,0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00, - 0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x79,0xcc,0x84,0x00, - 0x04,0xff,0x79,0xcc,0x84,0x00,0xd2,0x13,0x51,0x04,0x08,0x00,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0xa5,0x00,0x08,0xff,0xc8,0xbc,0x00,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08, - 0xff,0xc6,0x9a,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0xa6,0x00,0x08,0x00,0xcf,0x86, - 0x95,0x5f,0x94,0x5b,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08,0xff, - 0xc9,0x82,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xc6,0x80,0x00,0xd1,0x0e,0x10,0x07, - 0x09,0xff,0xca,0x89,0x00,0x09,0xff,0xca,0x8c,0x00,0x10,0x07,0x09,0xff,0xc9,0x87, - 0x00,0x09,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9,0x89,0x00,0x09,0x00, - 0x10,0x07,0x09,0xff,0xc9,0x8b,0x00,0x09,0x00,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9, - 0x8d,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xc9,0x8f,0x00,0x09,0x00,0x01,0x00,0x01, - 0x00,0xd1,0x8b,0xd0,0x0c,0xcf,0x86,0xe5,0x10,0x43,0x64,0xef,0x42,0x01,0xe6,0xcf, - 0x86,0xd5,0x2a,0xe4,0x99,0x43,0xe3,0x7f,0x43,0xd2,0x11,0xe1,0x5e,0x43,0x10,0x07, - 0x01,0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0xe1,0x65,0x43,0x10,0x09,0x01, - 0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0x00,0xd4,0x0f,0x93,0x0b,0x92, - 0x07,0x61,0xab,0x43,0x01,0xea,0x06,0xe6,0x06,0xe6,0xd3,0x2c,0xd2,0x16,0xd1,0x0b, - 0x10,0x07,0x0a,0xff,0xcd,0xb1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xcd,0xb3,0x00, - 0x0a,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x10,0x07,0x0a, - 0xff,0xcd,0xb7,0x00,0x0a,0x00,0xd2,0x07,0x61,0x97,0x43,0x00,0x00,0x51,0x04,0x09, - 0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10,0xff,0xcf,0xb3,0x00,0xe0,0x31,0x01,0xcf, - 0x86,0xd5,0xd3,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x01, - 0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x81, - 0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5, - 0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9, - 0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00, - 0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x01,0xff,0xcf,0x89,0xcc, - 0x81,0x00,0xd3,0x3c,0xd2,0x20,0xd1,0x12,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x88, - 0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0x00,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01, - 0xff,0xce,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb4,0x00,0x01,0xff,0xce, - 0xb5,0x00,0x10,0x07,0x01,0xff,0xce,0xb6,0x00,0x01,0xff,0xce,0xb7,0x00,0xd2,0x1c, - 0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb8,0x00,0x01,0xff,0xce,0xb9,0x00,0x10,0x07, - 0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xce,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff, - 0xce,0xbc,0x00,0x01,0xff,0xce,0xbd,0x00,0x10,0x07,0x01,0xff,0xce,0xbe,0x00,0x01, - 0xff,0xce,0xbf,0x00,0xe4,0x85,0x43,0xd3,0x35,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, - 0xff,0xcf,0x80,0x00,0x01,0xff,0xcf,0x81,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, - 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xcf,0x84,0x00,0x01,0xff,0xcf,0x85,0x00, - 0x10,0x07,0x01,0xff,0xcf,0x86,0x00,0x01,0xff,0xcf,0x87,0x00,0xe2,0x2b,0x43,0xd1, - 0x0e,0x10,0x07,0x01,0xff,0xcf,0x88,0x00,0x01,0xff,0xcf,0x89,0x00,0x10,0x09,0x01, - 0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc,0x88,0x00,0xcf,0x86,0xd5, - 0x94,0xd4,0x3c,0xd3,0x13,0x92,0x0f,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xcf, - 0x83,0x00,0x01,0x00,0x01,0x00,0xd2,0x07,0x61,0x3a,0x43,0x01,0x00,0xd1,0x12,0x10, - 0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x10, - 0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0xff,0xcf,0x97,0x00,0xd3,0x2c,0xd2, - 0x11,0xe1,0x46,0x43,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01,0xff,0xce,0xb8,0x00, - 0xd1,0x10,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0xff,0xcf,0x86,0x00, - 0x10,0x07,0x01,0xff,0xcf,0x80,0x00,0x04,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06, - 0xff,0xcf,0x99,0x00,0x06,0x00,0x10,0x07,0x01,0xff,0xcf,0x9b,0x00,0x04,0x00,0xd1, - 0x0b,0x10,0x07,0x01,0xff,0xcf,0x9d,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0x9f, - 0x00,0x04,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf, - 0xa1,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, - 0x07,0x01,0xff,0xcf,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xa7,0x00,0x01, - 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xa9,0x00,0x01,0x00,0x10,0x07, - 0x01,0xff,0xcf,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xad,0x00, - 0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xaf,0x00,0x01,0x00,0xd3,0x2b,0xd2,0x12,0x91, - 0x0e,0x10,0x07,0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xcf,0x81,0x00,0x01,0x00,0xd1, - 0x0e,0x10,0x07,0x05,0xff,0xce,0xb8,0x00,0x05,0xff,0xce,0xb5,0x00,0x10,0x04,0x06, - 0x00,0x07,0xff,0xcf,0xb8,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x07,0x00,0x07,0xff, - 0xcf,0xb2,0x00,0x10,0x07,0x07,0xff,0xcf,0xbb,0x00,0x07,0x00,0xd1,0x0b,0x10,0x04, - 0x08,0x00,0x08,0xff,0xcd,0xbb,0x00,0x10,0x07,0x08,0xff,0xcd,0xbc,0x00,0x08,0xff, - 0xcd,0xbd,0x00,0xe3,0xed,0x46,0xe2,0x3d,0x05,0xe1,0x27,0x02,0xe0,0x66,0x01,0xcf, - 0x86,0xd5,0xf0,0xd4,0x7e,0xd3,0x40,0xd2,0x22,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, - 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x07,0x01,0xff,0xd1, - 0x92,0x00,0x01,0xff,0xd0,0xb3,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1, - 0x94,0x00,0x01,0xff,0xd1,0x95,0x00,0x10,0x07,0x01,0xff,0xd1,0x96,0x00,0x01,0xff, - 0xd1,0x96,0xcc,0x88,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x98,0x00, - 0x01,0xff,0xd1,0x99,0x00,0x10,0x07,0x01,0xff,0xd1,0x9a,0x00,0x01,0xff,0xd1,0x9b, - 0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8, - 0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x86,0x00,0x01,0xff,0xd1,0x9f, - 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xb0,0x00,0x01,0xff, - 0xd0,0xb1,0x00,0x10,0x07,0x01,0xff,0xd0,0xb2,0x00,0x01,0xff,0xd0,0xb3,0x00,0xd1, - 0x0e,0x10,0x07,0x01,0xff,0xd0,0xb4,0x00,0x01,0xff,0xd0,0xb5,0x00,0x10,0x07,0x01, - 0xff,0xd0,0xb6,0x00,0x01,0xff,0xd0,0xb7,0x00,0xd2,0x1e,0xd1,0x10,0x10,0x07,0x01, - 0xff,0xd0,0xb8,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x10,0x07,0x01,0xff,0xd0, - 0xba,0x00,0x01,0xff,0xd0,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xbc,0x00, - 0x01,0xff,0xd0,0xbd,0x00,0x10,0x07,0x01,0xff,0xd0,0xbe,0x00,0x01,0xff,0xd0,0xbf, - 0x00,0xe4,0x25,0x42,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x80, - 0x00,0x01,0xff,0xd1,0x81,0x00,0x10,0x07,0x01,0xff,0xd1,0x82,0x00,0x01,0xff,0xd1, - 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x84,0x00,0x01,0xff,0xd1,0x85,0x00, - 0x10,0x07,0x01,0xff,0xd1,0x86,0x00,0x01,0xff,0xd1,0x87,0x00,0xd2,0x1c,0xd1,0x0e, - 0x10,0x07,0x01,0xff,0xd1,0x88,0x00,0x01,0xff,0xd1,0x89,0x00,0x10,0x07,0x01,0xff, - 0xd1,0x8a,0x00,0x01,0xff,0xd1,0x8b,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x8c, - 0x00,0x01,0xff,0xd1,0x8d,0x00,0x10,0x07,0x01,0xff,0xd1,0x8e,0x00,0x01,0xff,0xd1, - 0x8f,0x00,0xcf,0x86,0xd5,0x07,0x64,0xcf,0x41,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2, - 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff, - 0xd1,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa5,0x00,0x01,0x00, - 0x10,0x07,0x01,0xff,0xd1,0xa7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, - 0xff,0xd1,0xa9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xab,0x00,0x01,0x00,0xd1, - 0x0b,0x10,0x07,0x01,0xff,0xd1,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xaf, - 0x00,0x01,0x00,0xd3,0x33,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb1,0x00, - 0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xb3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01, - 0xff,0xd1,0xb5,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb5,0xcc,0x8f,0x00,0x01, - 0xff,0xd1,0xb5,0xcc,0x8f,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb9, - 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07, - 0x01,0xff,0xd1,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbf,0x00,0x01,0x00, - 0xe0,0x41,0x01,0xcf,0x86,0xd5,0x8e,0xd4,0x36,0xd3,0x11,0xe2,0x91,0x41,0xe1,0x88, - 0x41,0x10,0x07,0x01,0xff,0xd2,0x81,0x00,0x01,0x00,0xd2,0x0f,0x51,0x04,0x04,0x00, - 0x10,0x07,0x06,0xff,0xd2,0x8b,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x04,0xff,0xd2, - 0x8d,0x00,0x04,0x00,0x10,0x07,0x04,0xff,0xd2,0x8f,0x00,0x04,0x00,0xd3,0x2c,0xd2, - 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x91,0x00,0x01,0x00,0x10,0x07,0x01,0xff, - 0xd2,0x93,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x95,0x00,0x01,0x00, - 0x10,0x07,0x01,0xff,0xd2,0x97,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, - 0xff,0xd2,0x99,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9b,0x00,0x01,0x00,0xd1, - 0x0b,0x10,0x07,0x01,0xff,0xd2,0x9d,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9f, - 0x00,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, - 0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, - 0x07,0x01,0xff,0xd2,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa7,0x00,0x01, - 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xa9,0x00,0x01,0x00,0x10,0x07, - 0x01,0xff,0xd2,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xad,0x00, - 0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xaf,0x00,0x01,0x00,0xd3,0x2c,0xd2,0x16,0xd1, - 0x0b,0x10,0x07,0x01,0xff,0xd2,0xb1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xb3, - 0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xb5,0x00,0x01,0x00,0x10,0x07, - 0x01,0xff,0xd2,0xb7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, - 0xb9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10, - 0x07,0x01,0xff,0xd2,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbf,0x00,0x01, - 0x00,0xcf,0x86,0xd5,0xdc,0xd4,0x5a,0xd3,0x36,0xd2,0x20,0xd1,0x10,0x10,0x07,0x01, - 0xff,0xd3,0x8f,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, - 0xb6,0xcc,0x86,0x00,0x01,0xff,0xd3,0x84,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x06, - 0xff,0xd3,0x86,0x00,0x10,0x04,0x06,0x00,0x01,0xff,0xd3,0x88,0x00,0xd2,0x16,0xd1, - 0x0b,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8a,0x00,0x10,0x04,0x06,0x00,0x01,0xff, - 0xd3,0x8c,0x00,0xe1,0x69,0x40,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8e,0x00,0xd3, - 0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x01,0xff, - 0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x01,0xff, - 0xd0,0xb0,0xcc,0x88,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x95,0x00,0x01,0x00, - 0x10,0x09,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00, - 0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x99,0x00,0x01,0x00,0x10,0x09,0x01, - 0xff,0xd3,0x99,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc,0x88,0x00,0xd1,0x12,0x10, - 0x09,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x10, - 0x09,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0xd4, - 0x82,0xd3,0x41,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa1,0x00,0x01,0x00, - 0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc, - 0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc, - 0x88,0x00,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa9,0x00,0x01,0x00,0x10, - 0x09,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1, - 0x12,0x10,0x09,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88, - 0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0x01,0xff,0xd1,0x83,0xcc,0x84, - 0x00,0xd3,0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00, - 0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00, - 0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x87,0xcc, - 0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00,0x10,0x07,0x08,0xff,0xd3,0xb7,0x00, - 0x08,0x00,0xd2,0x1d,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x8b,0xcc,0x88,0x00,0x01, - 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x10,0x07,0x09,0xff,0xd3,0xbb,0x00,0x09,0x00,0xd1, - 0x0b,0x10,0x07,0x09,0xff,0xd3,0xbd,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd3,0xbf, - 0x00,0x09,0x00,0xe1,0x26,0x02,0xe0,0x78,0x01,0xcf,0x86,0xd5,0xb0,0xd4,0x58,0xd3, - 0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x81,0x00,0x06,0x00,0x10,0x07, - 0x06,0xff,0xd4,0x83,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x85,0x00, - 0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x87,0x00,0x06,0x00,0xd2,0x16,0xd1,0x0b,0x10, - 0x07,0x06,0xff,0xd4,0x89,0x00,0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x8b,0x00,0x06, - 0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x8d,0x00,0x06,0x00,0x10,0x07,0x06,0xff, - 0xd4,0x8f,0x00,0x06,0x00,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xd4, - 0x91,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd4,0x93,0x00,0x09,0x00,0xd1,0x0b,0x10, - 0x07,0x0a,0xff,0xd4,0x95,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x97,0x00,0x0a, - 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x99,0x00,0x0a,0x00,0x10,0x07, - 0x0a,0xff,0xd4,0x9b,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x9d,0x00, - 0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x9f,0x00,0x0a,0x00,0xd4,0x58,0xd3,0x2c,0xd2, - 0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0xa1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff, - 0xd4,0xa3,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xd4,0xa5,0x00,0x0b,0x00, - 0x10,0x07,0x0c,0xff,0xd4,0xa7,0x00,0x0c,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x10, - 0xff,0xd4,0xa9,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xab,0x00,0x10,0x00,0xd1, - 0x0b,0x10,0x07,0x10,0xff,0xd4,0xad,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xaf, - 0x00,0x10,0x00,0xd3,0x35,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x00,0x00,0x01,0xff,0xd5, - 0xa1,0x00,0x10,0x07,0x01,0xff,0xd5,0xa2,0x00,0x01,0xff,0xd5,0xa3,0x00,0xd1,0x0e, - 0x10,0x07,0x01,0xff,0xd5,0xa4,0x00,0x01,0xff,0xd5,0xa5,0x00,0x10,0x07,0x01,0xff, - 0xd5,0xa6,0x00,0x01,0xff,0xd5,0xa7,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff, - 0xd5,0xa8,0x00,0x01,0xff,0xd5,0xa9,0x00,0x10,0x07,0x01,0xff,0xd5,0xaa,0x00,0x01, - 0xff,0xd5,0xab,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xac,0x00,0x01,0xff,0xd5, - 0xad,0x00,0x10,0x07,0x01,0xff,0xd5,0xae,0x00,0x01,0xff,0xd5,0xaf,0x00,0xcf,0x86, - 0xe5,0x08,0x3f,0xd4,0x70,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, - 0xb0,0x00,0x01,0xff,0xd5,0xb1,0x00,0x10,0x07,0x01,0xff,0xd5,0xb2,0x00,0x01,0xff, - 0xd5,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xb4,0x00,0x01,0xff,0xd5,0xb5, - 0x00,0x10,0x07,0x01,0xff,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb7,0x00,0xd2,0x1c,0xd1, - 0x0e,0x10,0x07,0x01,0xff,0xd5,0xb8,0x00,0x01,0xff,0xd5,0xb9,0x00,0x10,0x07,0x01, - 0xff,0xd5,0xba,0x00,0x01,0xff,0xd5,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, - 0xbc,0x00,0x01,0xff,0xd5,0xbd,0x00,0x10,0x07,0x01,0xff,0xd5,0xbe,0x00,0x01,0xff, - 0xd5,0xbf,0x00,0xe3,0x87,0x3e,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x80, - 0x00,0x01,0xff,0xd6,0x81,0x00,0x10,0x07,0x01,0xff,0xd6,0x82,0x00,0x01,0xff,0xd6, - 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x84,0x00,0x01,0xff,0xd6,0x85,0x00, - 0x10,0x07,0x01,0xff,0xd6,0x86,0x00,0x00,0x00,0xe0,0x2f,0x3f,0xcf,0x86,0xe5,0xc0, - 0x3e,0xe4,0x97,0x3e,0xe3,0x76,0x3e,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x01,0xff,0xd5,0xa5,0xd6,0x82,0x00,0xe4,0x3e,0x25,0xe3,0xc3,0x1a, - 0xe2,0x7b,0x81,0xe1,0xc0,0x13,0xd0,0x1e,0xcf,0x86,0xc5,0xe4,0x08,0x4b,0xe3,0x53, - 0x46,0xe2,0xe9,0x43,0xe1,0x1c,0x43,0xe0,0xe1,0x42,0xcf,0x86,0xe5,0xa6,0x42,0x64, - 0x89,0x42,0x0b,0x00,0xcf,0x86,0xe5,0xfa,0x01,0xe4,0x03,0x56,0xe3,0x76,0x01,0xe2, - 0x8e,0x53,0xd1,0x0c,0xe0,0xef,0x52,0xcf,0x86,0x65,0x8d,0x52,0x04,0x00,0xe0,0x0d, - 0x01,0xcf,0x86,0xd5,0x0a,0xe4,0x10,0x53,0x63,0xff,0x52,0x0a,0x00,0xd4,0x80,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x80,0x00,0x01,0xff,0xe2, - 0xb4,0x81,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x82,0x00,0x01,0xff,0xe2,0xb4,0x83, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x84,0x00,0x01,0xff,0xe2,0xb4,0x85, - 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x86,0x00,0x01,0xff,0xe2,0xb4,0x87,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x88,0x00,0x01,0xff,0xe2,0xb4,0x89, - 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8a,0x00,0x01,0xff,0xe2,0xb4,0x8b,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8c,0x00,0x01,0xff,0xe2,0xb4,0x8d,0x00,0x10, - 0x08,0x01,0xff,0xe2,0xb4,0x8e,0x00,0x01,0xff,0xe2,0xb4,0x8f,0x00,0xd3,0x40,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x90,0x00,0x01,0xff,0xe2,0xb4,0x91, - 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x92,0x00,0x01,0xff,0xe2,0xb4,0x93,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x94,0x00,0x01,0xff,0xe2,0xb4,0x95,0x00,0x10, - 0x08,0x01,0xff,0xe2,0xb4,0x96,0x00,0x01,0xff,0xe2,0xb4,0x97,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x98,0x00,0x01,0xff,0xe2,0xb4,0x99,0x00,0x10, - 0x08,0x01,0xff,0xe2,0xb4,0x9a,0x00,0x01,0xff,0xe2,0xb4,0x9b,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe2,0xb4,0x9c,0x00,0x01,0xff,0xe2,0xb4,0x9d,0x00,0x10,0x08,0x01, - 0xff,0xe2,0xb4,0x9e,0x00,0x01,0xff,0xe2,0xb4,0x9f,0x00,0xcf,0x86,0xe5,0x42,0x52, - 0x94,0x50,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa0,0x00, - 0x01,0xff,0xe2,0xb4,0xa1,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa2,0x00,0x01,0xff, - 0xe2,0xb4,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa4,0x00,0x01,0xff, - 0xe2,0xb4,0xa5,0x00,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xa7,0x00,0x52,0x04, - 0x00,0x00,0x91,0x0c,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xad,0x00,0x00,0x00, - 0x01,0x00,0xd2,0x1b,0xe1,0xfc,0x52,0xe0,0xad,0x52,0xcf,0x86,0x95,0x0f,0x94,0x0b, - 0x93,0x07,0x62,0x92,0x52,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xd1,0x13,0xe0, - 0xd3,0x53,0xcf,0x86,0x95,0x0a,0xe4,0xa8,0x53,0x63,0x97,0x53,0x04,0x00,0x04,0x00, - 0xd0,0x0d,0xcf,0x86,0x95,0x07,0x64,0x22,0x54,0x08,0x00,0x04,0x00,0xcf,0x86,0x55, - 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x07,0x62,0x2f,0x54,0x04,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0xb0,0x00,0x11,0xff,0xe1,0x8f,0xb1,0x00, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb2,0x00,0x11,0xff,0xe1,0x8f,0xb3,0x00,0x91,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb4,0x00,0x11,0xff,0xe1,0x8f,0xb5,0x00,0x00,0x00, - 0xd4,0x1c,0xe3,0xe0,0x56,0xe2,0x17,0x56,0xe1,0xda,0x55,0xe0,0xbb,0x55,0xcf,0x86, - 0x95,0x0a,0xe4,0xa4,0x55,0x63,0x88,0x55,0x04,0x00,0x04,0x00,0xe3,0xd2,0x01,0xe2, - 0x2b,0x5a,0xd1,0x0c,0xe0,0x4c,0x59,0xcf,0x86,0x65,0x25,0x59,0x0a,0x00,0xe0,0x9c, - 0x59,0xcf,0x86,0xd5,0xc5,0xd4,0x45,0xd3,0x31,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x12, - 0xff,0xd0,0xb2,0x00,0x12,0xff,0xd0,0xb4,0x00,0x10,0x07,0x12,0xff,0xd0,0xbe,0x00, - 0x12,0xff,0xd1,0x81,0x00,0x51,0x07,0x12,0xff,0xd1,0x82,0x00,0x10,0x07,0x12,0xff, - 0xd1,0x8a,0x00,0x12,0xff,0xd1,0xa3,0x00,0x92,0x10,0x91,0x0c,0x10,0x08,0x12,0xff, - 0xea,0x99,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x14,0xff,0xe1,0x83,0x90,0x00,0x14,0xff,0xe1,0x83,0x91,0x00,0x10,0x08, - 0x14,0xff,0xe1,0x83,0x92,0x00,0x14,0xff,0xe1,0x83,0x93,0x00,0xd1,0x10,0x10,0x08, - 0x14,0xff,0xe1,0x83,0x94,0x00,0x14,0xff,0xe1,0x83,0x95,0x00,0x10,0x08,0x14,0xff, - 0xe1,0x83,0x96,0x00,0x14,0xff,0xe1,0x83,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x14,0xff,0xe1,0x83,0x98,0x00,0x14,0xff,0xe1,0x83,0x99,0x00,0x10,0x08,0x14,0xff, - 0xe1,0x83,0x9a,0x00,0x14,0xff,0xe1,0x83,0x9b,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, - 0xe1,0x83,0x9c,0x00,0x14,0xff,0xe1,0x83,0x9d,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, - 0x9e,0x00,0x14,0xff,0xe1,0x83,0x9f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x14,0xff,0xe1,0x83,0xa0,0x00,0x14,0xff,0xe1,0x83,0xa1,0x00,0x10,0x08, - 0x14,0xff,0xe1,0x83,0xa2,0x00,0x14,0xff,0xe1,0x83,0xa3,0x00,0xd1,0x10,0x10,0x08, - 0x14,0xff,0xe1,0x83,0xa4,0x00,0x14,0xff,0xe1,0x83,0xa5,0x00,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xa6,0x00,0x14,0xff,0xe1,0x83,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x14,0xff,0xe1,0x83,0xa8,0x00,0x14,0xff,0xe1,0x83,0xa9,0x00,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xaa,0x00,0x14,0xff,0xe1,0x83,0xab,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xac,0x00,0x14,0xff,0xe1,0x83,0xad,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, - 0xae,0x00,0x14,0xff,0xe1,0x83,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x14,0xff,0xe1,0x83,0xb0,0x00,0x14,0xff,0xe1,0x83,0xb1,0x00,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xb2,0x00,0x14,0xff,0xe1,0x83,0xb3,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xb4,0x00,0x14,0xff,0xe1,0x83,0xb5,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, - 0xb6,0x00,0x14,0xff,0xe1,0x83,0xb7,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x14,0xff, - 0xe1,0x83,0xb8,0x00,0x14,0xff,0xe1,0x83,0xb9,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, - 0xba,0x00,0x00,0x00,0xd1,0x0c,0x10,0x04,0x00,0x00,0x14,0xff,0xe1,0x83,0xbd,0x00, - 0x10,0x08,0x14,0xff,0xe1,0x83,0xbe,0x00,0x14,0xff,0xe1,0x83,0xbf,0x00,0xe2,0x9d, - 0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa5,0x00,0x01,0xff,0x61,0xcc, - 0xa5,0x00,0x10,0x08,0x01,0xff,0x62,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc,0x87,0x00, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x62,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc,0xa3,0x00, - 0x10,0x08,0x01,0xff,0x62,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00,0xd2,0x24, - 0xd1,0x14,0x10,0x0a,0x01,0xff,0x63,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc, - 0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x87,0x00,0x01,0xff,0x64,0xcc, - 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa3,0x00,0x01,0xff,0x64,0xcc, - 0xa3,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc,0xb1,0x00, - 0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa7,0x00,0x01,0xff, - 0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xad,0x00,0x01,0xff,0x64,0xcc, - 0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff, - 0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00, - 0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x65,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, - 0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc, - 0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, - 0x66,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc,0x84,0x00, - 0x10,0x08,0x01,0xff,0x68,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0x68,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00,0x10,0x08, - 0x01,0xff,0x68,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0x68,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00,0x10,0x08, - 0x01,0xff,0x68,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0x69,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a,0x01,0xff, - 0x69,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x81,0x00,0x01,0xff,0x6b,0xcc, - 0x81,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc,0xa3,0x00, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc,0xb1,0x00, - 0x10,0x08,0x01,0xff,0x6c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00,0xd2,0x24, - 0xd1,0x14,0x10,0x0a,0x01,0xff,0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff,0x6c,0xcc, - 0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x6c,0xcc,0xb1,0x00,0x01,0xff,0x6c,0xcc, - 0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6c,0xcc,0xad,0x00,0x01,0xff,0x6c,0xcc, - 0xad,0x00,0x10,0x08,0x01,0xff,0x6d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc,0x81,0x00, - 0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x6d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6d, - 0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, - 0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa3, - 0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, - 0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xad, - 0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x83, - 0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f, - 0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3,0x48,0xd2, - 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff,0x6f, - 0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0x01, - 0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x70,0xcc,0x81, - 0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x70,0xcc,0x87,0x00,0x01, - 0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x87, - 0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xa3,0x00,0x01, - 0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84, - 0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xb1, - 0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x73,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10,0x08,0x01, - 0xff,0x73,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01, - 0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x10, - 0x0a,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87, - 0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x01, - 0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0x87,0x00,0x01, - 0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xa3,0x00,0x01, - 0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0xb1,0x00,0x01,0xff,0x74, - 0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xad, - 0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa4,0x00,0x01, - 0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0xb0,0x00,0x01, - 0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xad,0x00,0x01,0xff,0x75, - 0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81, - 0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x84, - 0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x76,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x76, - 0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x11,0x02,0xcf,0x86,0xd5,0xe2, - 0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x80,0x00, - 0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x81,0x00,0x01,0xff, - 0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x88,0x00,0x01,0xff, - 0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x87,0x00,0x01,0xff,0x77,0xcc, - 0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0xa3,0x00,0x01,0xff, - 0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x78,0xcc,0x87,0x00,0x01,0xff,0x78,0xcc, - 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x78,0xcc,0x88,0x00,0x01,0xff,0x78,0xcc, - 0x88,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc,0x87,0x00, - 0xd3,0x33,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x82,0x00,0x01,0xff, - 0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0xa3,0x00,0x01,0xff,0x7a,0xcc, - 0xa3,0x00,0xe1,0x12,0x59,0x10,0x08,0x01,0xff,0x7a,0xcc,0xb1,0x00,0x01,0xff,0x7a, - 0xcc,0xb1,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01, - 0xff,0x79,0xcc,0x8a,0x00,0x10,0x08,0x01,0xff,0x61,0xca,0xbe,0x00,0x02,0xff,0x73, - 0xcc,0x87,0x00,0x51,0x04,0x0a,0x00,0x10,0x07,0x0a,0xff,0x73,0x73,0x00,0x0a,0x00, - 0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa3,0x00, - 0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x89,0x00,0x01,0xff, - 0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00, - 0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc, - 0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a, - 0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00, - 0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc, - 0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, - 0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00, - 0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a, - 0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00, - 0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc, - 0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x01,0xff, - 0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00, - 0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x65,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, - 0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc, - 0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc, - 0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01,0xd4, - 0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80, - 0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82, - 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01, - 0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x10, - 0x0a,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x89,0x00,0x01,0xff,0x69, - 0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc,0xa3, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc,0xa3, - 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00,0xd3, - 0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x01, - 0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80, - 0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f, - 0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a,0x01, - 0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0xd2, - 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x6f, - 0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0x01, - 0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b, - 0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f, - 0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98,0xd3, - 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x01, - 0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3, - 0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75, - 0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x89, - 0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, - 0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a,0x01, - 0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0xd1, - 0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc,0x9b, - 0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff,0x75, - 0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, - 0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08,0x01, - 0xff,0x79,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x79,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x79, - 0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x79,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x10,0x08,0x0a,0xff,0xe1, - 0xbb,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbd,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbf,0x00,0x0a,0x00,0xe1,0xbf,0x02,0xe0,0xa1, - 0x01,0xcf,0x86,0xd5,0xc6,0xd4,0x6c,0xd3,0x18,0xe2,0x0e,0x59,0xe1,0xf7,0x58,0x10, - 0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0x00,0xd2, - 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, - 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, - 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, - 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, - 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, - 0x00,0xd3,0x18,0xe2,0x4a,0x59,0xe1,0x33,0x59,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, - 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, - 0xff,0xce,0xb5,0xcc,0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01, - 0xff,0xce,0xb5,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80, - 0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, - 0xce,0xb5,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd4,0x6c,0xd3,0x18,0xe2,0x74,0x59, - 0xe1,0x5d,0x59,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7, - 0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00, - 0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc, - 0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01, - 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81, - 0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7, - 0xcc,0x94,0xcd,0x82,0x00,0xd3,0x18,0xe2,0xb0,0x59,0xe1,0x99,0x59,0x10,0x09,0x01, - 0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0x00,0xd2,0x28,0xd1, - 0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94, - 0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9, - 0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc, - 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce, - 0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82,0x00,0xcf, - 0x86,0xd5,0xac,0xd4,0x5a,0xd3,0x18,0xe2,0xed,0x59,0xe1,0xd6,0x59,0x10,0x09,0x01, - 0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00,0xd2,0x28,0xd1, - 0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94, - 0x00,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf, - 0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc, - 0x81,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x18,0xe2, - 0x17,0x5a,0xe1,0x00,0x5a,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff, - 0xcf,0x85,0xcc,0x94,0x00,0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, - 0x85,0xcc,0x94,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x80, - 0x00,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00, - 0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00,0xe4,0xd3,0x5a, - 0xd3,0x18,0xe2,0x52,0x5a,0xe1,0x3b,0x5a,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93, - 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff, - 0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff, - 0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0x00, - 0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xcf, - 0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82, - 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xe0,0xd9,0x02,0xcf,0x86,0xe5, - 0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1, - 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, - 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc, - 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93, - 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, - 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce, - 0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff, - 0xce,0xb1,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00, - 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce, - 0xb1,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1, - 0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce, - 0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01, - 0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x64,0xd2,0x30,0xd1,0x16, - 0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, - 0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0xce,0xb9, - 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d, - 0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, - 0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82, - 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30, - 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce, - 0xb7,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80, - 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a, - 0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce, - 0xb7,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93, - 0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00, - 0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93, - 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff, - 0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, - 0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, - 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d, - 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, - 0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89, - 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, - 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, - 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93, - 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, - 0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf, - 0x89,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x49,0xd2,0x26,0xd1,0x12,0x10,0x09, - 0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce,0xb1,0xcc,0x84,0x00,0x10,0x0b, - 0x01,0xff,0xce,0xb1,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xce,0xb9,0x00, - 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10, - 0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcd,0x82,0xce,0xb9, - 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff, - 0xce,0xb1,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01,0xff, - 0xce,0xb1,0xcc,0x81,0x00,0xe1,0xf3,0x5a,0x10,0x09,0x01,0xff,0xce,0xb1,0xce,0xb9, - 0x00,0x01,0x00,0xcf,0x86,0xd5,0xbd,0xd4,0x7e,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10, - 0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7, - 0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0xd1,0x0f,0x10,0x0b, - 0x01,0xff,0xce,0xb7,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce, - 0xb7,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24,0xd1, - 0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81, - 0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81, - 0x00,0xe1,0x02,0x5b,0x10,0x09,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0x01,0xff,0xe1, - 0xbe,0xbf,0xcc,0x80,0x00,0xd3,0x18,0xe2,0x28,0x5b,0xe1,0x11,0x5b,0x10,0x09,0x01, - 0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0xe2,0x4c,0x5b, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc, - 0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc, - 0x81,0x00,0xd4,0x51,0xd3,0x18,0xe2,0x6f,0x5b,0xe1,0x58,0x5b,0x10,0x09,0x01,0xff, - 0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00,0xd2,0x24,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00, - 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00, - 0xe1,0x8f,0x5b,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8, - 0xcc,0x80,0x00,0xd3,0x3b,0xd2,0x18,0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf, - 0x89,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0xd1,0x0f,0x10, - 0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff, - 0xcf,0x89,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, - 0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, - 0x81,0x00,0xe1,0x99,0x5b,0x10,0x09,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0x01,0xff, - 0xc2,0xb4,0x00,0xe0,0x0c,0x68,0xcf,0x86,0xe5,0x23,0x02,0xe4,0x25,0x01,0xe3,0x85, - 0x5e,0xd2,0x2a,0xe1,0x5f,0x5c,0xe0,0xdd,0x5b,0xcf,0x86,0xe5,0xbb,0x5b,0x94,0x1b, - 0xe3,0xa4,0x5b,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01, - 0xff,0xe2,0x80,0x83,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd1,0xd6,0xd0,0x46,0xcf, - 0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04,0x01, - 0x00,0x10,0x07,0x01,0xff,0xcf,0x89,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01,0x00, - 0x10,0x06,0x01,0xff,0x6b,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x01,0x00,0xe3,0x25, - 0x5d,0x92,0x10,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0x8e,0x00,0x01, - 0x00,0x01,0x00,0xcf,0x86,0xd5,0x0a,0xe4,0x42,0x5d,0x63,0x2d,0x5d,0x06,0x00,0x94, - 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb0,0x00,0x01, - 0xff,0xe2,0x85,0xb1,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb2,0x00,0x01,0xff,0xe2, - 0x85,0xb3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb4,0x00,0x01,0xff,0xe2, - 0x85,0xb5,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb6,0x00,0x01,0xff,0xe2,0x85,0xb7, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb8,0x00,0x01,0xff,0xe2, - 0x85,0xb9,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xba,0x00,0x01,0xff,0xe2,0x85,0xbb, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xbc,0x00,0x01,0xff,0xe2,0x85,0xbd, - 0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xbe,0x00,0x01,0xff,0xe2,0x85,0xbf,0x00,0x01, - 0x00,0xe0,0x34,0x5d,0xcf,0x86,0xe5,0x13,0x5d,0xe4,0xf2,0x5c,0xe3,0xe1,0x5c,0xe2, - 0xd4,0x5c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0xff,0xe2,0x86,0x84,0x00, - 0xe3,0x23,0x61,0xe2,0xf0,0x60,0xd1,0x0c,0xe0,0x9d,0x60,0xcf,0x86,0x65,0x7e,0x60, - 0x01,0x00,0xd0,0x62,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x18, - 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x90,0x00, - 0x01,0xff,0xe2,0x93,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93, - 0x92,0x00,0x01,0xff,0xe2,0x93,0x93,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x94,0x00, - 0x01,0xff,0xe2,0x93,0x95,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x96,0x00, - 0x01,0xff,0xe2,0x93,0x97,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x98,0x00,0x01,0xff, - 0xe2,0x93,0x99,0x00,0xcf,0x86,0xe5,0x57,0x60,0x94,0x80,0xd3,0x40,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x9a,0x00,0x01,0xff,0xe2,0x93,0x9b,0x00,0x10, - 0x08,0x01,0xff,0xe2,0x93,0x9c,0x00,0x01,0xff,0xe2,0x93,0x9d,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe2,0x93,0x9e,0x00,0x01,0xff,0xe2,0x93,0x9f,0x00,0x10,0x08,0x01, - 0xff,0xe2,0x93,0xa0,0x00,0x01,0xff,0xe2,0x93,0xa1,0x00,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe2,0x93,0xa2,0x00,0x01,0xff,0xe2,0x93,0xa3,0x00,0x10,0x08,0x01, - 0xff,0xe2,0x93,0xa4,0x00,0x01,0xff,0xe2,0x93,0xa5,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe2,0x93,0xa6,0x00,0x01,0xff,0xe2,0x93,0xa7,0x00,0x10,0x08,0x01,0xff,0xe2, - 0x93,0xa8,0x00,0x01,0xff,0xe2,0x93,0xa9,0x00,0x01,0x00,0xd4,0x0c,0xe3,0x33,0x62, - 0xe2,0x2c,0x62,0xcf,0x06,0x04,0x00,0xe3,0x0c,0x65,0xe2,0xff,0x63,0xe1,0x2e,0x02, - 0xe0,0x84,0x01,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x08,0xff,0xe2,0xb0,0xb0,0x00,0x08,0xff,0xe2,0xb0,0xb1,0x00,0x10,0x08, - 0x08,0xff,0xe2,0xb0,0xb2,0x00,0x08,0xff,0xe2,0xb0,0xb3,0x00,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe2,0xb0,0xb4,0x00,0x08,0xff,0xe2,0xb0,0xb5,0x00,0x10,0x08,0x08,0xff, - 0xe2,0xb0,0xb6,0x00,0x08,0xff,0xe2,0xb0,0xb7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe2,0xb0,0xb8,0x00,0x08,0xff,0xe2,0xb0,0xb9,0x00,0x10,0x08,0x08,0xff, - 0xe2,0xb0,0xba,0x00,0x08,0xff,0xe2,0xb0,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe2,0xb0,0xbc,0x00,0x08,0xff,0xe2,0xb0,0xbd,0x00,0x10,0x08,0x08,0xff,0xe2,0xb0, - 0xbe,0x00,0x08,0xff,0xe2,0xb0,0xbf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe2,0xb1,0x80,0x00,0x08,0xff,0xe2,0xb1,0x81,0x00,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x82,0x00,0x08,0xff,0xe2,0xb1,0x83,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x84,0x00,0x08,0xff,0xe2,0xb1,0x85,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x86,0x00,0x08,0xff,0xe2,0xb1,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x88,0x00,0x08,0xff,0xe2,0xb1,0x89,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x8a,0x00,0x08,0xff,0xe2,0xb1,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x8c,0x00,0x08,0xff,0xe2,0xb1,0x8d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x8e,0x00, - 0x08,0xff,0xe2,0xb1,0x8f,0x00,0x94,0x7c,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe2,0xb1,0x90,0x00,0x08,0xff,0xe2,0xb1,0x91,0x00,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x92,0x00,0x08,0xff,0xe2,0xb1,0x93,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x94,0x00,0x08,0xff,0xe2,0xb1,0x95,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x96,0x00,0x08,0xff,0xe2,0xb1,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe2,0xb1,0x98,0x00,0x08,0xff,0xe2,0xb1,0x99,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x9a,0x00,0x08,0xff,0xe2,0xb1,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, - 0x9c,0x00,0x08,0xff,0xe2,0xb1,0x9d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x9e,0x00, - 0x00,0x00,0x08,0x00,0xcf,0x86,0xd5,0x07,0x64,0xef,0x61,0x08,0x00,0xd4,0x63,0xd3, - 0x32,0xd2,0x1b,0xd1,0x0c,0x10,0x08,0x09,0xff,0xe2,0xb1,0xa1,0x00,0x09,0x00,0x10, - 0x07,0x09,0xff,0xc9,0xab,0x00,0x09,0xff,0xe1,0xb5,0xbd,0x00,0xd1,0x0b,0x10,0x07, - 0x09,0xff,0xc9,0xbd,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xa8, - 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xaa,0x00,0x10, - 0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xac,0x00,0xd1,0x0b,0x10,0x04,0x09,0x00,0x0a, - 0xff,0xc9,0x91,0x00,0x10,0x07,0x0a,0xff,0xc9,0xb1,0x00,0x0a,0xff,0xc9,0x90,0x00, - 0xd3,0x27,0xd2,0x17,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xc9,0x92,0x00,0x0a,0x00,0x10, - 0x08,0x0a,0xff,0xe2,0xb1,0xb3,0x00,0x0a,0x00,0x91,0x0c,0x10,0x04,0x09,0x00,0x09, - 0xff,0xe2,0xb1,0xb6,0x00,0x09,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10, - 0x07,0x0b,0xff,0xc8,0xbf,0x00,0x0b,0xff,0xc9,0x80,0x00,0xe0,0x83,0x01,0xcf,0x86, - 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, - 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x83,0x00,0x08,0x00,0xd1,0x0c, - 0x10,0x08,0x08,0xff,0xe2,0xb2,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2, - 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x89,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, - 0x08,0xff,0xe2,0xb2,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8f,0x00, - 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x91,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, - 0x08,0xff,0xe2,0xb2,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x97,0x00, - 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x99,0x00,0x08,0x00, - 0x10,0x08,0x08,0xff,0xe2,0xb2,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, - 0xe2,0xb2,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x9f,0x00,0x08,0x00, - 0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa1,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, - 0x08,0xff,0xe2,0xb2,0xa5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa7,0x00, - 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa9,0x00,0x08,0x00, - 0x10,0x08,0x08,0xff,0xe2,0xb2,0xab,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, - 0xe2,0xb2,0xad,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xaf,0x00,0x08,0x00, - 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb1,0x00,0x08,0x00, - 0x10,0x08,0x08,0xff,0xe2,0xb2,0xb3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, - 0xe2,0xb2,0xb5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb7,0x00,0x08,0x00, - 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb9,0x00,0x08,0x00,0x10,0x08, - 0x08,0xff,0xe2,0xb2,0xbb,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, - 0xbd,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xbf,0x00,0x08,0x00,0xcf,0x86, - 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3, - 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x83,0x00,0x08,0x00,0xd1,0x0c, - 0x10,0x08,0x08,0xff,0xe2,0xb3,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3, - 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x89,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, - 0x08,0xff,0xe2,0xb3,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8f,0x00, - 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x91,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, - 0x08,0xff,0xe2,0xb3,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x97,0x00, - 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x99,0x00,0x08,0x00, - 0x10,0x08,0x08,0xff,0xe2,0xb3,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, - 0xe2,0xb3,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x9f,0x00,0x08,0x00, - 0xd4,0x3b,0xd3,0x1c,0x92,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa1,0x00, - 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa3,0x00,0x08,0x00,0x08,0x00,0xd2,0x10, - 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x0b,0xff,0xe2,0xb3,0xac,0x00,0xe1,0x3b, - 0x5f,0x10,0x04,0x0b,0x00,0x0b,0xff,0xe2,0xb3,0xae,0x00,0xe3,0x40,0x5f,0x92,0x10, - 0x51,0x04,0x0b,0xe6,0x10,0x08,0x0d,0xff,0xe2,0xb3,0xb3,0x00,0x0d,0x00,0x00,0x00, - 0xe2,0x98,0x08,0xd1,0x0b,0xe0,0x11,0x67,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe0,0x65, - 0x6c,0xcf,0x86,0xe5,0xa7,0x05,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x0c,0xe2,0xf8, - 0x67,0xe1,0x8f,0x67,0xcf,0x06,0x04,0x00,0xe2,0xdb,0x01,0xe1,0x26,0x01,0xd0,0x09, - 0xcf,0x86,0x65,0xf4,0x67,0x0a,0x00,0xcf,0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2, - 0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a, - 0xff,0xea,0x99,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x85, - 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1, - 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, - 0x99,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x8d,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1, - 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, - 0x99,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x95,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x97,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, - 0x08,0x0a,0xff,0xea,0x99,0x99,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x9b, - 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x9d,0x00,0x0a,0x00,0x10, - 0x08,0x0a,0xff,0xea,0x99,0x9f,0x00,0x0a,0x00,0xe4,0x5d,0x67,0xd3,0x30,0xd2,0x18, - 0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x99,0xa1,0x00,0x0c,0x00,0x10,0x08,0x0a,0xff, - 0xea,0x99,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0xa5,0x00, - 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c, - 0x10,0x08,0x0a,0xff,0xea,0x99,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99, - 0xab,0x00,0x0a,0x00,0xe1,0x0c,0x67,0x10,0x08,0x0a,0xff,0xea,0x99,0xad,0x00,0x0a, - 0x00,0xe0,0x35,0x67,0xcf,0x86,0x95,0xab,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c, - 0x10,0x08,0x0a,0xff,0xea,0x9a,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a, - 0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x85,0x00,0x0a,0x00, - 0x10,0x08,0x0a,0xff,0xea,0x9a,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08, - 0x0a,0xff,0xea,0x9a,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8b,0x00, - 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8d,0x00,0x0a,0x00,0x10,0x08, - 0x0a,0xff,0xea,0x9a,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08, - 0x0a,0xff,0xea,0x9a,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x93,0x00, - 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x95,0x00,0x0a,0x00,0x10,0x08, - 0x0a,0xff,0xea,0x9a,0x97,0x00,0x0a,0x00,0xe2,0x92,0x66,0xd1,0x0c,0x10,0x08,0x10, - 0xff,0xea,0x9a,0x99,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9a,0x9b,0x00,0x10, - 0x00,0x0b,0x00,0xe1,0x10,0x02,0xd0,0xb9,0xcf,0x86,0xd5,0x07,0x64,0x9e,0x66,0x08, - 0x00,0xd4,0x58,0xd3,0x28,0xd2,0x10,0x51,0x04,0x09,0x00,0x10,0x08,0x0a,0xff,0xea, - 0x9c,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa5,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, - 0x08,0x0a,0xff,0xea,0x9c,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xab, - 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xad,0x00,0x0a,0x00,0x10, - 0x08,0x0a,0xff,0xea,0x9c,0xaf,0x00,0x0a,0x00,0xd3,0x28,0xd2,0x10,0x51,0x04,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, - 0xff,0xea,0x9c,0xb5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb7,0x00,0x0a, - 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb9,0x00,0x0a,0x00,0x10, - 0x08,0x0a,0xff,0xea,0x9c,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, - 0x9c,0xbd,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xbf,0x00,0x0a,0x00,0xcf, - 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, - 0x9d,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x83,0x00,0x0a,0x00,0xd1, - 0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x85,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, - 0x9d,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x89, - 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10, - 0x08,0x0a,0xff,0xea,0x9d,0x8d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8f, - 0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x91, - 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10, - 0x08,0x0a,0xff,0xea,0x9d,0x95,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x97, - 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x99,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, - 0xff,0xea,0x9d,0x9d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9f,0x00,0x0a, - 0x00,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa1, - 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10, - 0x08,0x0a,0xff,0xea,0x9d,0xa5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa7, - 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa9,0x00,0x0a, - 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xab,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, - 0xff,0xea,0x9d,0xad,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xaf,0x00,0x0a, - 0x00,0x53,0x04,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea, - 0x9d,0xba,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9d,0xbc,0x00,0xd1,0x0c,0x10, - 0x04,0x0a,0x00,0x0a,0xff,0xe1,0xb5,0xb9,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xbf, - 0x00,0x0a,0x00,0xe0,0x71,0x01,0xcf,0x86,0xd5,0xa6,0xd4,0x4e,0xd3,0x30,0xd2,0x18, - 0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff, - 0xea,0x9e,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x85,0x00, - 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9e,0x87,0x00,0x0a,0x00,0xd2,0x10,0x51,0x04, - 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9e,0x8c,0x00,0xe1,0x9a,0x64,0x10, - 0x04,0x0a,0x00,0x0c,0xff,0xc9,0xa5,0x00,0xd3,0x28,0xd2,0x18,0xd1,0x0c,0x10,0x08, - 0x0c,0xff,0xea,0x9e,0x91,0x00,0x0c,0x00,0x10,0x08,0x0d,0xff,0xea,0x9e,0x93,0x00, - 0x0d,0x00,0x51,0x04,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x97,0x00,0x10,0x00, - 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e,0x99,0x00,0x10,0x00,0x10,0x08, - 0x10,0xff,0xea,0x9e,0x9b,0x00,0x10,0x00,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e, - 0x9d,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x9f,0x00,0x10,0x00,0xd4,0x63, - 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa1,0x00,0x0c,0x00, - 0x10,0x08,0x0c,0xff,0xea,0x9e,0xa3,0x00,0x0c,0x00,0xd1,0x0c,0x10,0x08,0x0c,0xff, - 0xea,0x9e,0xa5,0x00,0x0c,0x00,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa7,0x00,0x0c,0x00, - 0xd2,0x1a,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa9,0x00,0x0c,0x00,0x10,0x07, - 0x0d,0xff,0xc9,0xa6,0x00,0x10,0xff,0xc9,0x9c,0x00,0xd1,0x0e,0x10,0x07,0x10,0xff, - 0xc9,0xa1,0x00,0x10,0xff,0xc9,0xac,0x00,0x10,0x07,0x12,0xff,0xc9,0xaa,0x00,0x14, - 0x00,0xd3,0x35,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x10,0xff,0xca,0x9e,0x00,0x10,0xff, - 0xca,0x87,0x00,0x10,0x07,0x11,0xff,0xca,0x9d,0x00,0x11,0xff,0xea,0xad,0x93,0x00, - 0xd1,0x0c,0x10,0x08,0x11,0xff,0xea,0x9e,0xb5,0x00,0x11,0x00,0x10,0x08,0x11,0xff, - 0xea,0x9e,0xb7,0x00,0x11,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x14,0xff,0xea,0x9e, - 0xb9,0x00,0x14,0x00,0x10,0x08,0x15,0xff,0xea,0x9e,0xbb,0x00,0x15,0x00,0xd1,0x0c, - 0x10,0x08,0x15,0xff,0xea,0x9e,0xbd,0x00,0x15,0x00,0x10,0x08,0x15,0xff,0xea,0x9e, - 0xbf,0x00,0x15,0x00,0xcf,0x86,0xe5,0xd4,0x63,0x94,0x2f,0x93,0x2b,0xd2,0x10,0x51, - 0x04,0x00,0x00,0x10,0x08,0x15,0xff,0xea,0x9f,0x83,0x00,0x15,0x00,0xd1,0x0f,0x10, - 0x08,0x15,0xff,0xea,0x9e,0x94,0x00,0x15,0xff,0xca,0x82,0x00,0x10,0x08,0x15,0xff, - 0xe1,0xb6,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe4,0xb4,0x66,0xd3,0x1d,0xe2, - 0x5b,0x64,0xe1,0x0a,0x64,0xe0,0xf7,0x63,0xcf,0x86,0xe5,0xd8,0x63,0x94,0x0b,0x93, - 0x07,0x62,0xc3,0x63,0x08,0x00,0x08,0x00,0x08,0x00,0xd2,0x0f,0xe1,0x5a,0x65,0xe0, - 0x27,0x65,0xcf,0x86,0x65,0x0c,0x65,0x0a,0x00,0xd1,0xab,0xd0,0x1a,0xcf,0x86,0xe5, - 0x17,0x66,0xe4,0xfa,0x65,0xe3,0xe1,0x65,0xe2,0xd4,0x65,0x91,0x08,0x10,0x04,0x00, - 0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x0b,0x93,0x07,0x62, - 0x27,0x66,0x11,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff, - 0xe1,0x8e,0xa0,0x00,0x11,0xff,0xe1,0x8e,0xa1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e, - 0xa2,0x00,0x11,0xff,0xe1,0x8e,0xa3,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, - 0xa4,0x00,0x11,0xff,0xe1,0x8e,0xa5,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xa6,0x00, - 0x11,0xff,0xe1,0x8e,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, - 0xa8,0x00,0x11,0xff,0xe1,0x8e,0xa9,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xaa,0x00, - 0x11,0xff,0xe1,0x8e,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xac,0x00, - 0x11,0xff,0xe1,0x8e,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xae,0x00,0x11,0xff, - 0xe1,0x8e,0xaf,0x00,0xe0,0xb2,0x65,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb0,0x00,0x11,0xff,0xe1,0x8e, - 0xb1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb2,0x00,0x11,0xff,0xe1,0x8e,0xb3,0x00, - 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb4,0x00,0x11,0xff,0xe1,0x8e,0xb5,0x00, - 0x10,0x08,0x11,0xff,0xe1,0x8e,0xb6,0x00,0x11,0xff,0xe1,0x8e,0xb7,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb8,0x00,0x11,0xff,0xe1,0x8e,0xb9,0x00, - 0x10,0x08,0x11,0xff,0xe1,0x8e,0xba,0x00,0x11,0xff,0xe1,0x8e,0xbb,0x00,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8e,0xbc,0x00,0x11,0xff,0xe1,0x8e,0xbd,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8e,0xbe,0x00,0x11,0xff,0xe1,0x8e,0xbf,0x00,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x80,0x00,0x11,0xff,0xe1,0x8f,0x81,0x00, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x82,0x00,0x11,0xff,0xe1,0x8f,0x83,0x00,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x84,0x00,0x11,0xff,0xe1,0x8f,0x85,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x86,0x00,0x11,0xff,0xe1,0x8f,0x87,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x88,0x00,0x11,0xff,0xe1,0x8f,0x89,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x8a,0x00,0x11,0xff,0xe1,0x8f,0x8b,0x00,0xd1,0x10,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x8c,0x00,0x11,0xff,0xe1,0x8f,0x8d,0x00,0x10,0x08,0x11,0xff, - 0xe1,0x8f,0x8e,0x00,0x11,0xff,0xe1,0x8f,0x8f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x90,0x00,0x11,0xff,0xe1,0x8f,0x91,0x00, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x92,0x00,0x11,0xff,0xe1,0x8f,0x93,0x00,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x94,0x00,0x11,0xff,0xe1,0x8f,0x95,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x96,0x00,0x11,0xff,0xe1,0x8f,0x97,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0x98,0x00,0x11,0xff,0xe1,0x8f,0x99,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x9a,0x00,0x11,0xff,0xe1,0x8f,0x9b,0x00,0xd1,0x10,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0x9c,0x00,0x11,0xff,0xe1,0x8f,0x9d,0x00,0x10,0x08,0x11,0xff, - 0xe1,0x8f,0x9e,0x00,0x11,0xff,0xe1,0x8f,0x9f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x11,0xff,0xe1,0x8f,0xa0,0x00,0x11,0xff,0xe1,0x8f,0xa1,0x00,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0xa2,0x00,0x11,0xff,0xe1,0x8f,0xa3,0x00,0xd1,0x10,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0xa4,0x00,0x11,0xff,0xe1,0x8f,0xa5,0x00,0x10,0x08,0x11,0xff, - 0xe1,0x8f,0xa6,0x00,0x11,0xff,0xe1,0x8f,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x11,0xff,0xe1,0x8f,0xa8,0x00,0x11,0xff,0xe1,0x8f,0xa9,0x00,0x10,0x08,0x11,0xff, - 0xe1,0x8f,0xaa,0x00,0x11,0xff,0xe1,0x8f,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff, - 0xe1,0x8f,0xac,0x00,0x11,0xff,0xe1,0x8f,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8f, - 0xae,0x00,0x11,0xff,0xe1,0x8f,0xaf,0x00,0xd1,0x0c,0xe0,0xeb,0x63,0xcf,0x86,0xcf, - 0x06,0x02,0xff,0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06, - 0xcf,0x06,0x01,0x00,0xd4,0xae,0xd3,0x09,0xe2,0x54,0x64,0xcf,0x06,0x01,0x00,0xd2, - 0x27,0xe1,0x1f,0x70,0xe0,0x26,0x6e,0xcf,0x86,0xe5,0x3f,0x6d,0xe4,0xce,0x6c,0xe3, - 0x99,0x6c,0xe2,0x78,0x6c,0xe1,0x67,0x6c,0x10,0x08,0x01,0xff,0xe5,0x88,0x87,0x00, - 0x01,0xff,0xe5,0xba,0xa6,0x00,0xe1,0x74,0x74,0xe0,0xe8,0x73,0xcf,0x86,0xe5,0x22, - 0x73,0xd4,0x3b,0x93,0x37,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x01,0xff,0x66,0x66,0x00, - 0x01,0xff,0x66,0x69,0x00,0x10,0x07,0x01,0xff,0x66,0x6c,0x00,0x01,0xff,0x66,0x66, - 0x69,0x00,0xd1,0x0f,0x10,0x08,0x01,0xff,0x66,0x66,0x6c,0x00,0x01,0xff,0x73,0x74, - 0x00,0x10,0x07,0x01,0xff,0x73,0x74,0x00,0x00,0x00,0x00,0x00,0xe3,0xc8,0x72,0xd2, - 0x11,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xd5,0xb4,0xd5,0xb6,0x00, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xd5,0xb4,0xd5,0xa5,0x00,0x01,0xff,0xd5,0xb4,0xd5, - 0xab,0x00,0x10,0x09,0x01,0xff,0xd5,0xbe,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb4,0xd5, - 0xad,0x00,0xd3,0x09,0xe2,0x40,0x74,0xcf,0x06,0x01,0x00,0xd2,0x13,0xe1,0x30,0x75, - 0xe0,0xc1,0x74,0xcf,0x86,0xe5,0x9e,0x74,0x64,0x8d,0x74,0x06,0xff,0x00,0xe1,0x96, - 0x75,0xe0,0x63,0x75,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x7c, - 0xd3,0x3c,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xef,0xbd,0x81,0x00, - 0x10,0x08,0x01,0xff,0xef,0xbd,0x82,0x00,0x01,0xff,0xef,0xbd,0x83,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xef,0xbd,0x84,0x00,0x01,0xff,0xef,0xbd,0x85,0x00,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x86,0x00,0x01,0xff,0xef,0xbd,0x87,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xef,0xbd,0x88,0x00,0x01,0xff,0xef,0xbd,0x89,0x00,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x8a,0x00,0x01,0xff,0xef,0xbd,0x8b,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x8c,0x00,0x01,0xff,0xef,0xbd,0x8d,0x00,0x10,0x08,0x01,0xff, - 0xef,0xbd,0x8e,0x00,0x01,0xff,0xef,0xbd,0x8f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xef,0xbd,0x90,0x00,0x01,0xff,0xef,0xbd,0x91,0x00,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x92,0x00,0x01,0xff,0xef,0xbd,0x93,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x94,0x00,0x01,0xff,0xef,0xbd,0x95,0x00,0x10,0x08,0x01,0xff, - 0xef,0xbd,0x96,0x00,0x01,0xff,0xef,0xbd,0x97,0x00,0x92,0x1c,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xef,0xbd,0x98,0x00,0x01,0xff,0xef,0xbd,0x99,0x00,0x10,0x08,0x01,0xff, - 0xef,0xbd,0x9a,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x87,0xb3,0xe1,0x60,0xb0,0xe0, - 0xdd,0xae,0xcf,0x86,0xe5,0x81,0x9b,0xc4,0xe3,0xc1,0x07,0xe2,0x62,0x06,0xe1,0x11, - 0x86,0xe0,0x09,0x05,0xcf,0x86,0xe5,0xfb,0x02,0xd4,0x1c,0xe3,0x7f,0x76,0xe2,0xd6, - 0x75,0xe1,0xb1,0x75,0xe0,0x8a,0x75,0xcf,0x86,0xe5,0x57,0x75,0x94,0x07,0x63,0x42, - 0x75,0x07,0x00,0x07,0x00,0xe3,0x2b,0x78,0xe2,0xf0,0x77,0xe1,0x77,0x01,0xe0,0x88, - 0x77,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09, - 0x05,0xff,0xf0,0x90,0x90,0xa8,0x00,0x05,0xff,0xf0,0x90,0x90,0xa9,0x00,0x10,0x09, - 0x05,0xff,0xf0,0x90,0x90,0xaa,0x00,0x05,0xff,0xf0,0x90,0x90,0xab,0x00,0xd1,0x12, - 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xac,0x00,0x05,0xff,0xf0,0x90,0x90,0xad,0x00, - 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xae,0x00,0x05,0xff,0xf0,0x90,0x90,0xaf,0x00, - 0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb0,0x00,0x05,0xff,0xf0, - 0x90,0x90,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb2,0x00,0x05,0xff,0xf0, - 0x90,0x90,0xb3,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb4,0x00,0x05, - 0xff,0xf0,0x90,0x90,0xb5,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb6,0x00,0x05, - 0xff,0xf0,0x90,0x90,0xb7,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, - 0xf0,0x90,0x90,0xb8,0x00,0x05,0xff,0xf0,0x90,0x90,0xb9,0x00,0x10,0x09,0x05,0xff, - 0xf0,0x90,0x90,0xba,0x00,0x05,0xff,0xf0,0x90,0x90,0xbb,0x00,0xd1,0x12,0x10,0x09, - 0x05,0xff,0xf0,0x90,0x90,0xbc,0x00,0x05,0xff,0xf0,0x90,0x90,0xbd,0x00,0x10,0x09, - 0x05,0xff,0xf0,0x90,0x90,0xbe,0x00,0x05,0xff,0xf0,0x90,0x90,0xbf,0x00,0xd2,0x24, - 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x80,0x00,0x05,0xff,0xf0,0x90,0x91, - 0x81,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x82,0x00,0x05,0xff,0xf0,0x90,0x91, - 0x83,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x84,0x00,0x05,0xff,0xf0, - 0x90,0x91,0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x86,0x00,0x05,0xff,0xf0, - 0x90,0x91,0x87,0x00,0x94,0x4c,0x93,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, - 0xf0,0x90,0x91,0x88,0x00,0x05,0xff,0xf0,0x90,0x91,0x89,0x00,0x10,0x09,0x05,0xff, - 0xf0,0x90,0x91,0x8a,0x00,0x05,0xff,0xf0,0x90,0x91,0x8b,0x00,0xd1,0x12,0x10,0x09, - 0x05,0xff,0xf0,0x90,0x91,0x8c,0x00,0x05,0xff,0xf0,0x90,0x91,0x8d,0x00,0x10,0x09, - 0x07,0xff,0xf0,0x90,0x91,0x8e,0x00,0x07,0xff,0xf0,0x90,0x91,0x8f,0x00,0x05,0x00, - 0x05,0x00,0xd0,0xa0,0xcf,0x86,0xd5,0x07,0x64,0x30,0x76,0x07,0x00,0xd4,0x07,0x63, - 0x3d,0x76,0x07,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90, - 0x93,0x98,0x00,0x12,0xff,0xf0,0x90,0x93,0x99,0x00,0x10,0x09,0x12,0xff,0xf0,0x90, - 0x93,0x9a,0x00,0x12,0xff,0xf0,0x90,0x93,0x9b,0x00,0xd1,0x12,0x10,0x09,0x12,0xff, - 0xf0,0x90,0x93,0x9c,0x00,0x12,0xff,0xf0,0x90,0x93,0x9d,0x00,0x10,0x09,0x12,0xff, - 0xf0,0x90,0x93,0x9e,0x00,0x12,0xff,0xf0,0x90,0x93,0x9f,0x00,0xd2,0x24,0xd1,0x12, - 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa0,0x00,0x12,0xff,0xf0,0x90,0x93,0xa1,0x00, - 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa2,0x00,0x12,0xff,0xf0,0x90,0x93,0xa3,0x00, - 0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa4,0x00,0x12,0xff,0xf0,0x90,0x93, - 0xa5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa6,0x00,0x12,0xff,0xf0,0x90,0x93, - 0xa7,0x00,0xcf,0x86,0xe5,0xc6,0x75,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, - 0x09,0x12,0xff,0xf0,0x90,0x93,0xa8,0x00,0x12,0xff,0xf0,0x90,0x93,0xa9,0x00,0x10, - 0x09,0x12,0xff,0xf0,0x90,0x93,0xaa,0x00,0x12,0xff,0xf0,0x90,0x93,0xab,0x00,0xd1, - 0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xac,0x00,0x12,0xff,0xf0,0x90,0x93,0xad, - 0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xae,0x00,0x12,0xff,0xf0,0x90,0x93,0xaf, - 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb0,0x00,0x12,0xff, - 0xf0,0x90,0x93,0xb1,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb2,0x00,0x12,0xff, - 0xf0,0x90,0x93,0xb3,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb4,0x00, - 0x12,0xff,0xf0,0x90,0x93,0xb5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb6,0x00, - 0x12,0xff,0xf0,0x90,0x93,0xb7,0x00,0x93,0x28,0x92,0x24,0xd1,0x12,0x10,0x09,0x12, - 0xff,0xf0,0x90,0x93,0xb8,0x00,0x12,0xff,0xf0,0x90,0x93,0xb9,0x00,0x10,0x09,0x12, - 0xff,0xf0,0x90,0x93,0xba,0x00,0x12,0xff,0xf0,0x90,0x93,0xbb,0x00,0x00,0x00,0x12, - 0x00,0xd4,0x1f,0xe3,0xdf,0x76,0xe2,0x6a,0x76,0xe1,0x09,0x76,0xe0,0xea,0x75,0xcf, - 0x86,0xe5,0xb7,0x75,0x94,0x0a,0xe3,0xa2,0x75,0x62,0x99,0x75,0x07,0x00,0x07,0x00, - 0xe3,0xde,0x78,0xe2,0xaf,0x78,0xd1,0x09,0xe0,0x4c,0x78,0xcf,0x06,0x0b,0x00,0xe0, - 0x7f,0x78,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0x80,0x00,0x11,0xff,0xf0,0x90,0xb3,0x81,0x00,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0x82,0x00,0x11,0xff,0xf0,0x90,0xb3,0x83,0x00,0xd1, - 0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x84,0x00,0x11,0xff,0xf0,0x90,0xb3,0x85, - 0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x86,0x00,0x11,0xff,0xf0,0x90,0xb3,0x87, - 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x88,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0x89,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8a,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0x8b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8c,0x00, - 0x11,0xff,0xf0,0x90,0xb3,0x8d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8e,0x00, - 0x11,0xff,0xf0,0x90,0xb3,0x8f,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, - 0xff,0xf0,0x90,0xb3,0x90,0x00,0x11,0xff,0xf0,0x90,0xb3,0x91,0x00,0x10,0x09,0x11, - 0xff,0xf0,0x90,0xb3,0x92,0x00,0x11,0xff,0xf0,0x90,0xb3,0x93,0x00,0xd1,0x12,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0x94,0x00,0x11,0xff,0xf0,0x90,0xb3,0x95,0x00,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0x96,0x00,0x11,0xff,0xf0,0x90,0xb3,0x97,0x00,0xd2, - 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x98,0x00,0x11,0xff,0xf0,0x90, - 0xb3,0x99,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9a,0x00,0x11,0xff,0xf0,0x90, - 0xb3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9c,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0x9d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9e,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0x9f,0x00,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, - 0xff,0xf0,0x90,0xb3,0xa0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa1,0x00,0x10,0x09,0x11, - 0xff,0xf0,0x90,0xb3,0xa2,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa3,0x00,0xd1,0x12,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa4,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa5,0x00,0x10, - 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa6,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa7,0x00,0xd2, - 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xa8,0x00,0x11,0xff,0xf0,0x90, - 0xb3,0xa9,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xaa,0x00,0x11,0xff,0xf0,0x90, - 0xb3,0xab,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xac,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0xad,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xae,0x00,0x11,0xff, - 0xf0,0x90,0xb3,0xaf,0x00,0x93,0x23,0x92,0x1f,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0, - 0x90,0xb3,0xb0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xb1,0x00,0x10,0x09,0x11,0xff,0xf0, - 0x90,0xb3,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x15,0xe4,0x91, - 0x7b,0xe3,0x9b,0x79,0xe2,0x94,0x78,0xe1,0xe4,0x77,0xe0,0x9d,0x77,0xcf,0x06,0x0c, - 0x00,0xe4,0xeb,0x7e,0xe3,0x44,0x7e,0xe2,0xed,0x7d,0xd1,0x0c,0xe0,0xb2,0x7d,0xcf, - 0x86,0x65,0x93,0x7d,0x14,0x00,0xe0,0xb6,0x7d,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4, - 0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x80,0x00, - 0x10,0xff,0xf0,0x91,0xa3,0x81,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x82,0x00, - 0x10,0xff,0xf0,0x91,0xa3,0x83,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, - 0x84,0x00,0x10,0xff,0xf0,0x91,0xa3,0x85,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, - 0x86,0x00,0x10,0xff,0xf0,0x91,0xa3,0x87,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10, - 0xff,0xf0,0x91,0xa3,0x88,0x00,0x10,0xff,0xf0,0x91,0xa3,0x89,0x00,0x10,0x09,0x10, - 0xff,0xf0,0x91,0xa3,0x8a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8b,0x00,0xd1,0x12,0x10, - 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8d,0x00,0x10, - 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8f,0x00,0xd3, - 0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x90,0x00,0x10,0xff, - 0xf0,0x91,0xa3,0x91,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x92,0x00,0x10,0xff, - 0xf0,0x91,0xa3,0x93,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x94,0x00, - 0x10,0xff,0xf0,0x91,0xa3,0x95,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x96,0x00, - 0x10,0xff,0xf0,0x91,0xa3,0x97,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0, - 0x91,0xa3,0x98,0x00,0x10,0xff,0xf0,0x91,0xa3,0x99,0x00,0x10,0x09,0x10,0xff,0xf0, - 0x91,0xa3,0x9a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x10, - 0xff,0xf0,0x91,0xa3,0x9c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9d,0x00,0x10,0x09,0x10, - 0xff,0xf0,0x91,0xa3,0x9e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9f,0x00,0xd1,0x11,0xe0, - 0x12,0x81,0xcf,0x86,0xe5,0x09,0x81,0xe4,0xd2,0x80,0xcf,0x06,0x00,0x00,0xe0,0xdb, - 0x82,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x09,0xe3,0x10,0x81,0xcf,0x06, - 0x0c,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xe2,0x3b,0x82,0xe1,0x16,0x82,0xd0,0x06, - 0xcf,0x06,0x00,0x00,0xcf,0x86,0xa5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, - 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa1, - 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa3, - 0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa4,0x00,0x14,0xff,0xf0,0x96, - 0xb9,0xa5,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa6,0x00,0x14,0xff,0xf0,0x96, - 0xb9,0xa7,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa8,0x00, - 0x14,0xff,0xf0,0x96,0xb9,0xa9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xaa,0x00, - 0x14,0xff,0xf0,0x96,0xb9,0xab,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, - 0xac,0x00,0x14,0xff,0xf0,0x96,0xb9,0xad,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, - 0xae,0x00,0x14,0xff,0xf0,0x96,0xb9,0xaf,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, - 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb1,0x00,0x10, - 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb3,0x00,0xd1, - 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb4,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb5, - 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb6,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb7, - 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb8,0x00,0x14,0xff, - 0xf0,0x96,0xb9,0xb9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xba,0x00,0x14,0xff, - 0xf0,0x96,0xb9,0xbb,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbc,0x00, - 0x14,0xff,0xf0,0x96,0xb9,0xbd,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbe,0x00, - 0x14,0xff,0xf0,0x96,0xb9,0xbf,0x00,0x14,0x00,0xd2,0x14,0xe1,0x25,0x82,0xe0,0x1c, - 0x82,0xcf,0x86,0xe5,0xdd,0x81,0xe4,0x9a,0x81,0xcf,0x06,0x12,0x00,0xd1,0x0b,0xe0, - 0x51,0x83,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0x95,0x8b,0xcf,0x86,0xd5,0x22,0xe4, - 0xd0,0x88,0xe3,0x93,0x88,0xe2,0x38,0x88,0xe1,0x31,0x88,0xe0,0x2a,0x88,0xcf,0x86, - 0xe5,0xfb,0x87,0xe4,0xe2,0x87,0x93,0x07,0x62,0xd1,0x87,0x12,0xe6,0x12,0xe6,0xe4, - 0x36,0x89,0xe3,0x2f,0x89,0xd2,0x09,0xe1,0xb8,0x88,0xcf,0x06,0x10,0x00,0xe1,0x1f, - 0x89,0xe0,0xec,0x88,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, - 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa3, - 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa5, - 0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa6,0x00,0x12,0xff,0xf0,0x9e, - 0xa4,0xa7,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa8,0x00,0x12,0xff,0xf0,0x9e, - 0xa4,0xa9,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xaa,0x00, - 0x12,0xff,0xf0,0x9e,0xa4,0xab,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xac,0x00, - 0x12,0xff,0xf0,0x9e,0xa4,0xad,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, - 0xae,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xaf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, - 0xb0,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb1,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, - 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb3,0x00,0x10, - 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb5,0x00,0xd1, - 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb6,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb7, - 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb8,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb9, - 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xba,0x00,0x12,0xff, - 0xf0,0x9e,0xa4,0xbb,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbc,0x00,0x12,0xff, - 0xf0,0x9e,0xa4,0xbd,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbe,0x00, - 0x12,0xff,0xf0,0x9e,0xa4,0xbf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa5,0x80,0x00, - 0x12,0xff,0xf0,0x9e,0xa5,0x81,0x00,0x94,0x1e,0x93,0x1a,0x92,0x16,0x91,0x12,0x10, - 0x09,0x12,0xff,0xf0,0x9e,0xa5,0x82,0x00,0x12,0xff,0xf0,0x9e,0xa5,0x83,0x00,0x12, - 0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - /* nfdi_c0100 */ - 0x57,0x04,0x01,0x00,0xc6,0xe5,0xac,0x13,0xe4,0x41,0x0c,0xe3,0x7a,0x07,0xe2,0xf3, - 0x01,0xc1,0xd0,0x1f,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x53,0x04,0x01,0x00, - 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x04,0x01,0x00,0x01,0xff,0x00,0x01,0x00,0x01, - 0x00,0xcf,0x86,0xd5,0xe4,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x41,0xcc,0x80,0x00,0x01,0xff,0x41,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x41, - 0xcc,0x82,0x00,0x01,0xff,0x41,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x41, - 0xcc,0x88,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x43, - 0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x80,0x00,0x01, - 0xff,0x45,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x82,0x00,0x01,0xff,0x45, - 0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x80,0x00,0x01,0xff,0x49, - 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x82,0x00,0x01,0xff,0x49,0xcc,0x88, - 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x83, - 0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x80,0x00,0x01,0xff,0x4f,0xcc,0x81,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x82,0x00,0x01,0xff,0x4f,0xcc,0x83,0x00,0x10, - 0x08,0x01,0xff,0x4f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01, - 0x00,0x01,0xff,0x55,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x81,0x00,0x01, - 0xff,0x55,0xcc,0x82,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x88,0x00,0x01, - 0xff,0x59,0xcc,0x81,0x00,0x01,0x00,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, - 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01, - 0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x80, - 0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x82,0x00,0x01, - 0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x80,0x00,0x01, - 0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00,0x01,0xff,0x69, - 0xcc,0x88,0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6e, - 0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x81, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82,0x00,0x01,0xff,0x6f,0xcc,0x83, - 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10, - 0x04,0x01,0x00,0x01,0xff,0x75,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81, - 0x00,0x01,0xff,0x75,0xcc,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88, - 0x00,0x01,0xff,0x79,0xcc,0x81,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x79,0xcc,0x88, - 0x00,0xe1,0x9a,0x03,0xe0,0xd3,0x01,0xcf,0x86,0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84, - 0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10, - 0x08,0x01,0xff,0x43,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x43,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10, - 0x08,0x01,0xff,0x43,0xcc,0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x43,0xcc,0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01, - 0xff,0x44,0xcc,0x8c,0x00,0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x34,0xd2,0x14,0x51, - 0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x84,0x00,0x01,0xff,0x65,0xcc,0x84, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0x86, - 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0xa8,0x00,0x01,0xff,0x65,0xcc,0xa8, - 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82,0x00,0x10, - 0x08,0x01,0xff,0x47,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4,0x74,0xd3, - 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x87,0x00,0x01,0xff,0x67, - 0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0xa7,0x00,0x01,0xff,0x67,0xcc,0xa7, - 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0x82,0x00,0x01,0xff,0x68,0xcc,0x82, - 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x83,0x00,0x01, - 0xff,0x69,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x84,0x00,0x01,0xff,0x69, - 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x86,0x00,0x01,0xff,0x69, - 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8, - 0x00,0xd3,0x30,0xd2,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x49,0xcc,0x87,0x00,0x01, - 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4a,0xcc,0x82,0x00,0x01,0xff,0x6a, - 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, - 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4c,0xcc,0x81,0x00,0x10, - 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x4c,0xcc,0xa7,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x4c,0xcc,0x8c,0x00,0x10,0x08,0x01, - 0xff,0x6c,0xcc,0x8c,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x60,0xd3,0x30,0xd2, - 0x10,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x81,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x6e,0xcc,0x81,0x00,0x01,0xff,0x4e,0xcc,0xa7,0x00,0x10, - 0x08,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x01,0xff,0x4e,0xcc,0x8c,0x00,0xd2,0x10,0x91, - 0x0c,0x10,0x08,0x01,0xff,0x6e,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x4f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0x84,0x00,0x10,0x08,0x01, - 0xff,0x4f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00,0xd3,0x34,0xd2,0x14,0x91, - 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8b,0x00,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81, - 0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c, - 0x00,0x10,0x08,0x01,0xff,0x53,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10, - 0x08,0x01,0xff,0x53,0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x74,0xd3, - 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x8c,0x00,0x01,0xff,0x73, - 0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7, - 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c, - 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x83,0x00,0x01, - 0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x84,0x00,0x01,0xff,0x75, - 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x86,0x00,0x01,0xff,0x75, - 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x8a,0x00,0x01,0xff,0x75,0xcc,0x8a, - 0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8b,0x00,0x01, - 0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa8,0x00,0x01,0xff,0x75, - 0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x82,0x00,0x01,0xff,0x77, - 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x82,0x00,0x01,0xff,0x79,0xcc,0x82, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x59,0xcc,0x88,0x00,0x01,0xff,0x5a, - 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff,0x5a,0xcc,0x87, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff,0x5a,0xcc,0x8c, - 0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0x00,0xd0,0x4a,0xcf,0x86,0x55, - 0x04,0x01,0x00,0xd4,0x2c,0xd3,0x18,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0x4f, - 0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01, - 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x55,0xcc,0x9b,0x00,0x93, - 0x14,0x92,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x75,0xcc,0x9b,0x00,0x01,0x00,0x01, - 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xb4,0xd4,0x24,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x41,0xcc,0x8c,0x00,0x10, - 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x49,0xcc,0x8c,0x00,0xd3,0x46,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x4f,0xcc,0x8c, - 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x8c,0x00,0xd1, - 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x84, - 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x55,0xcc,0x88, - 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, - 0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, - 0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, - 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x88, - 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x80,0xd3,0x3a,0xd2, - 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, - 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x84,0x00,0x01,0xff, - 0xc3,0xa6,0xcc,0x84,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0x8c, - 0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b, - 0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa8, - 0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa8, - 0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc6, - 0xb7,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0xd3,0x24,0xd2,0x10,0x91, - 0x0c,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0x91,0x10,0x10, - 0x08,0x01,0xff,0x47,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x04,0x00,0xd2, - 0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x4e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80, - 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a, - 0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x81,0x00,0x01,0xff, - 0xc3,0xa6,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xc3,0x98,0xcc,0x81,0x00,0x01,0xff, - 0xc3,0xb8,0xcc,0x81,0x00,0xe2,0x07,0x02,0xe1,0xae,0x01,0xe0,0x93,0x01,0xcf,0x86, - 0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc, - 0x8f,0x00,0x01,0xff,0x61,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x91,0x00, - 0x01,0xff,0x61,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x8f,0x00, - 0x01,0xff,0x65,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x91,0x00,0x01,0xff, - 0x65,0xcc,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x8f,0x00, - 0x01,0xff,0x69,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x91,0x00,0x01,0xff, - 0x69,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8f,0x00,0x01,0xff, - 0x6f,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc, - 0x91,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8f,0x00, - 0x01,0xff,0x72,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0x91,0x00,0x01,0xff, - 0x72,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8f,0x00,0x01,0xff, - 0x75,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc, - 0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x04,0xff,0x53,0xcc,0xa6,0x00,0x04,0xff, - 0x73,0xcc,0xa6,0x00,0x10,0x08,0x04,0xff,0x54,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc, - 0xa6,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x48,0xcc,0x8c,0x00,0x04,0xff, - 0x68,0xcc,0x8c,0x00,0xd4,0x68,0xd3,0x20,0xd2,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, - 0x07,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x41,0xcc,0x87,0x00, - 0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x45,0xcc, - 0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x88,0xcc, - 0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10,0x0a,0x04,0xff, - 0x4f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x10,0x08, - 0x04,0xff,0x4f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0x93,0x30,0xd2,0x24, - 0xd1,0x14,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x87,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc, - 0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x59,0xcc,0x84,0x00,0x04,0xff,0x79,0xcc, - 0x84,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0xcf,0x86, - 0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04,0x08,0x00,0x09,0x00,0x09,0x00, - 0x09,0x00,0x01,0x00,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18, - 0x53,0x04,0x01,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00, - 0x11,0x04,0x04,0x00,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00, - 0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, - 0x04,0x00,0x94,0x18,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04,0x00, - 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xe1,0x35,0x01,0xd0, - 0x72,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0xe6,0xd3,0x10,0x52,0x04,0x01,0xe6,0x91, - 0x08,0x10,0x04,0x01,0xe6,0x01,0xe8,0x01,0xdc,0x92,0x0c,0x51,0x04,0x01,0xdc,0x10, - 0x04,0x01,0xe8,0x01,0xd8,0x01,0xdc,0xd4,0x2c,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10, - 0x04,0x01,0xdc,0x01,0xca,0x10,0x04,0x01,0xca,0x01,0xdc,0x51,0x04,0x01,0xdc,0x10, - 0x04,0x01,0xdc,0x01,0xca,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0xca,0x01,0xdc,0x01, - 0xdc,0x01,0xdc,0xd3,0x08,0x12,0x04,0x01,0xdc,0x01,0x01,0xd2,0x0c,0x91,0x08,0x10, - 0x04,0x01,0x01,0x01,0xdc,0x01,0xdc,0x91,0x08,0x10,0x04,0x01,0xdc,0x01,0xe6,0x01, - 0xe6,0xcf,0x86,0xd5,0x7f,0xd4,0x47,0xd3,0x2e,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, - 0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0x10,0x04,0x01,0xe6,0x01,0xff,0xcc, - 0x93,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xf0,0x10, - 0x04,0x04,0xe6,0x04,0xdc,0xd2,0x08,0x11,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10, - 0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x04,0xdc,0x06,0xff,0x00,0xd3,0x18,0xd2,0x0c, - 0x51,0x04,0x07,0xe6,0x10,0x04,0x07,0xe6,0x07,0xdc,0x51,0x04,0x07,0xdc,0x10,0x04, - 0x07,0xdc,0x07,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe8,0x08,0xdc,0x10,0x04, - 0x08,0xdc,0x08,0xe6,0xd1,0x08,0x10,0x04,0x08,0xe9,0x07,0xea,0x10,0x04,0x07,0xea, - 0x07,0xe9,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0xea,0x10,0x04,0x04,0xe9, - 0x06,0xe6,0x06,0xe6,0x06,0xe6,0xd3,0x13,0x52,0x04,0x0a,0x00,0x91,0x0b,0x10,0x07, - 0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10, - 0x04,0x01,0x00,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10, - 0x00,0xd0,0xe1,0xcf,0x86,0xd5,0x7a,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1, - 0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, - 0xce,0x91,0xcc,0x81,0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09, - 0x01,0xff,0xce,0x95,0xcc,0x81,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00,0x10,0x09, - 0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce, - 0x9f,0xcc,0x81,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0x01, - 0xff,0xce,0xa9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff, - 0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4, - 0x4a,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01, - 0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x88,0x00, - 0x01,0xff,0xce,0xa5,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc, - 0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc, - 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10, - 0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x39,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04, - 0x01,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc, - 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf, - 0x85,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0x00,0xd3, - 0x26,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xcf,0x92,0xcc, - 0x81,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0x00,0x10, - 0x04,0x01,0x00,0x04,0x00,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x01,0x00,0x04, - 0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x01,0x00,0x04,0x00,0xd4, - 0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x01,0x00,0x01, - 0x00,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x06, - 0x00,0x07,0x00,0x12,0x04,0x07,0x00,0x08,0x00,0xe3,0x47,0x04,0xe2,0xbe,0x02,0xe1, - 0x07,0x01,0xd0,0x8b,0xcf,0x86,0xd5,0x6c,0xd4,0x53,0xd3,0x30,0xd2,0x1f,0xd1,0x12, - 0x10,0x09,0x04,0xff,0xd0,0x95,0xcc,0x80,0x00,0x01,0xff,0xd0,0x95,0xcc,0x88,0x00, - 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x93,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x01,0xff,0xd0,0x86,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xd0,0x9a,0xcc,0x81,0x00,0x04,0xff,0xd0,0x98,0xcc,0x80,0x00, - 0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x86,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0x92, - 0x11,0x91,0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x98,0xcc,0x86,0x00,0x01,0x00, - 0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x11,0x91,0x0d,0x10,0x04, - 0x01,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5, - 0x57,0x54,0x04,0x01,0x00,0xd3,0x30,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, - 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x04,0x01,0x00,0x01, - 0xff,0xd0,0xb3,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, - 0xd1,0x96,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0, - 0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8,0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1, - 0x83,0xcc,0x86,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x1a,0x52,0x04,0x01,0x00, - 0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb4,0xcc,0x8f,0x00,0x01,0xff,0xd1, - 0xb5,0xcc,0x8f,0x00,0x01,0x00,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x94,0x24,0xd3,0x18, - 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0x51,0x04,0x01,0xe6, - 0x10,0x04,0x01,0xe6,0x0a,0xe6,0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00, - 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xbe,0xd4,0x4a,0xd3,0x2a,0xd2,0x1a,0xd1,0x0d, - 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x96,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, - 0xb6,0xcc,0x86,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, - 0x06,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, - 0x06,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00, - 0x09,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x86, - 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x88, - 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, - 0xd0,0x95,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0xd2,0x16,0x51,0x04, - 0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0x98,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc, - 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x96,0xcc,0x88,0x00,0x01,0xff,0xd0, - 0xb6,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0x97,0xcc,0x88,0x00,0x01,0xff,0xd0, - 0xb7,0xcc,0x88,0x00,0xd4,0x74,0xd3,0x3a,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09, - 0x01,0xff,0xd0,0x98,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xd0,0x98,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00, - 0x10,0x09,0x01,0xff,0xd0,0x9e,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00, - 0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0xa8,0xcc,0x88,0x00,0x01, - 0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0,0xad,0xcc,0x88, - 0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x84, - 0x00,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09, - 0x01,0xff,0xd0,0xa3,0xcc,0x88,0x00,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09, - 0x01,0xff,0xd0,0xa3,0xcc,0x8b,0x00,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0x91,0x12, - 0x10,0x09,0x01,0xff,0xd0,0xa7,0xcc,0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00, - 0x08,0x00,0x92,0x16,0x91,0x12,0x10,0x09,0x01,0xff,0xd0,0xab,0xcc,0x88,0x00,0x01, - 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x09,0x00,0x09,0x00,0xd1,0x74,0xd0,0x36,0xcf,0x86, - 0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00, - 0xd4,0x10,0x93,0x0c,0x52,0x04,0x0a,0x00,0x11,0x04,0x0b,0x00,0x0c,0x00,0x10,0x00, - 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00, - 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0xba, - 0xcf,0x86,0xd5,0x4c,0xd4,0x24,0x53,0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, - 0x14,0x00,0x01,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, - 0x10,0x00,0x10,0x04,0x10,0x00,0x0d,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, - 0x00,0x00,0x02,0xdc,0x02,0xe6,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6, - 0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xde,0x02,0xdc,0x02,0xe6,0xd4,0x2c, - 0xd3,0x10,0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x08,0xdc,0x02,0xdc,0x02,0xdc, - 0xd2,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6,0xd1,0x08,0x10,0x04, - 0x02,0xe6,0x02,0xde,0x10,0x04,0x02,0xe4,0x02,0xe6,0xd3,0x20,0xd2,0x10,0xd1,0x08, - 0x10,0x04,0x01,0x0a,0x01,0x0b,0x10,0x04,0x01,0x0c,0x01,0x0d,0xd1,0x08,0x10,0x04, - 0x01,0x0e,0x01,0x0f,0x10,0x04,0x01,0x10,0x01,0x11,0xd2,0x10,0xd1,0x08,0x10,0x04, - 0x01,0x12,0x01,0x13,0x10,0x04,0x09,0x13,0x01,0x14,0xd1,0x08,0x10,0x04,0x01,0x15, - 0x01,0x16,0x10,0x04,0x01,0x00,0x01,0x17,0xcf,0x86,0xd5,0x28,0x94,0x24,0x93,0x20, - 0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x18,0x10,0x04,0x01,0x19,0x01,0x00, - 0xd1,0x08,0x10,0x04,0x02,0xe6,0x08,0xdc,0x10,0x04,0x08,0x00,0x08,0x12,0x00,0x00, - 0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, - 0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x93,0x10, - 0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xe2,0xfb,0x01,0xe1,0x2b,0x01,0xd0,0xa8,0xcf,0x86,0xd5,0x55,0xd4,0x28,0xd3,0x10, - 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x10,0x00,0x0a,0x00,0xd2,0x0c, - 0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00,0x08,0x00,0x91,0x08,0x10,0x04,0x01,0x00, - 0x07,0x00,0x07,0x00,0xd3,0x0c,0x52,0x04,0x07,0xe6,0x11,0x04,0x07,0xe6,0x0a,0xe6, - 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0a,0x1e,0x0a,0x1f,0x10,0x04,0x0a,0x20,0x01,0x00, - 0xd1,0x09,0x10,0x05,0x0f,0xff,0x00,0x00,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd4, - 0x3d,0x93,0x39,0xd2,0x1a,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x09,0x01, - 0xff,0xd8,0xa7,0xd9,0x93,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x94,0x00,0xd1,0x12,0x10, - 0x09,0x01,0xff,0xd9,0x88,0xd9,0x94,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x95,0x00,0x10, - 0x09,0x01,0xff,0xd9,0x8a,0xd9,0x94,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00, - 0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86, - 0xd5,0x5c,0xd4,0x20,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, - 0x01,0x00,0x01,0x1b,0xd1,0x08,0x10,0x04,0x01,0x1c,0x01,0x1d,0x10,0x04,0x01,0x1e, - 0x01,0x1f,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x20,0x01,0x21,0x10,0x04, - 0x01,0x22,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x07,0xdc, - 0x07,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x07,0xe6,0x08,0xe6,0x08,0xe6,0xd1,0x08, - 0x10,0x04,0x08,0xdc,0x08,0xe6,0x10,0x04,0x08,0xe6,0x0c,0xdc,0xd4,0x10,0x53,0x04, - 0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x93,0x10,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x01,0x23,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x22, - 0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x08, - 0x11,0x04,0x04,0x00,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0x00, - 0xcf,0x86,0xd5,0x5b,0xd4,0x2e,0xd3,0x1e,0x92,0x1a,0xd1,0x0d,0x10,0x09,0x01,0xff, - 0xdb,0x95,0xd9,0x94,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xdb,0x81,0xd9,0x94,0x00, - 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, - 0x04,0x00,0xd3,0x19,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, - 0xdb,0x92,0xd9,0x94,0x00,0x11,0x04,0x01,0x00,0x01,0xe6,0x52,0x04,0x01,0xe6,0xd1, - 0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd4,0x38,0xd3, - 0x1c,0xd2,0x0c,0x51,0x04,0x01,0xe6,0x10,0x04,0x01,0xe6,0x01,0xdc,0xd1,0x08,0x10, - 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd2,0x10,0xd1,0x08,0x10, - 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0xdc,0x01,0xe6,0x91,0x08,0x10,0x04,0x01, - 0xe6,0x01,0xdc,0x07,0x00,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04, - 0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0xd1,0xc8,0xd0,0x76,0xcf, - 0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04, - 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04, - 0x00,0x04,0x24,0x04,0x00,0x04,0x00,0x04,0x00,0xd4,0x14,0x53,0x04,0x04,0x00,0x52, - 0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xd3,0x1c,0xd2, - 0x0c,0x91,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04, - 0xdc,0x04,0xe6,0x10,0x04,0x04,0xe6,0x04,0xdc,0xd2,0x0c,0x51,0x04,0x04,0xdc,0x10, - 0x04,0x04,0xe6,0x04,0xdc,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10,0x04,0x04, - 0xdc,0x04,0xe6,0xcf,0x86,0xd5,0x3c,0x94,0x38,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x04, - 0xe6,0x10,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, - 0x04,0x04,0xdc,0x04,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, - 0x04,0x04,0xe6,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x08, - 0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0a, - 0x00,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93, - 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x92,0x0c,0x51, - 0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xe6,0x09,0xe6,0xd3,0x10,0x92,0x0c,0x51, - 0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, - 0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x14,0xdc,0x14, - 0x00,0xe4,0xf8,0x57,0xe3,0x45,0x3f,0xe2,0xf4,0x3e,0xe1,0xc7,0x2c,0xe0,0x21,0x10, - 0xcf,0x86,0xc5,0xe4,0x80,0x08,0xe3,0xcb,0x03,0xe2,0x61,0x01,0xd1,0x94,0xd0,0x5a, - 0xcf,0x86,0xd5,0x20,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04,0x0b,0x00,0x11,0x04, - 0x0b,0x00,0x0b,0xe6,0x92,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04,0x0b,0x00,0x0b,0xe6, - 0x0b,0xe6,0xd4,0x24,0xd3,0x10,0x52,0x04,0x0b,0xe6,0x91,0x08,0x10,0x04,0x0b,0x00, - 0x0b,0xe6,0x0b,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6,0x0b,0xe6, - 0x11,0x04,0x0b,0xe6,0x00,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04, - 0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0x54,0x04,0x0c,0x00, - 0x53,0x04,0x0c,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0c,0xdc,0x0c,0xdc, - 0x51,0x04,0x00,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x13,0x00, - 0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xd0,0x4a,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0d,0x00,0x10,0x00,0x0d,0x00,0x0d,0x00,0x52,0x04,0x0d,0x00,0x91,0x08, - 0x10,0x04,0x0d,0x00,0x10,0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00, - 0x10,0x04,0x10,0x00,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x12,0x00, - 0x52,0x04,0x12,0x00,0x11,0x04,0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04, - 0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0xdc, - 0x12,0xe6,0x12,0xe6,0xd4,0x30,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04, - 0x12,0x00,0x11,0xdc,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x0d,0xe6,0x91,0x08,0x10,0x04,0x0d,0xe6, - 0x0d,0xdc,0x0d,0xdc,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0x1b,0x0d,0x1c, - 0x10,0x04,0x0d,0x1d,0x0d,0xe6,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6, - 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x10,0x04,0x0d,0xdc,0x0d,0xe6, - 0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x10,0xe6,0xe1,0x3a,0x01,0xd0,0x77,0xcf, - 0x86,0xd5,0x20,0x94,0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x01, - 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, - 0x00,0xd4,0x1b,0x53,0x04,0x01,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00,0x01, - 0xff,0xe0,0xa4,0xa8,0xe0,0xa4,0xbc,0x00,0x01,0x00,0x01,0x00,0xd3,0x26,0xd2,0x13, - 0x91,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa4,0xb0,0xe0,0xa4,0xbc,0x00,0x01, - 0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xa4,0xb3,0xe0,0xa4,0xbc,0x00,0x01,0x00, - 0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x91,0x08,0x10,0x04,0x01,0x07, - 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x8c,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04, - 0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x10,0x04,0x0b,0x00,0x0c,0x00, - 0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x10,0x04,0x01,0xdc, - 0x01,0xe6,0x91,0x08,0x10,0x04,0x01,0xe6,0x0b,0x00,0x0c,0x00,0xd2,0x2c,0xd1,0x16, - 0x10,0x0b,0x01,0xff,0xe0,0xa4,0x95,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0x96, - 0xe0,0xa4,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa4,0x97,0xe0,0xa4,0xbc,0x00,0x01, - 0xff,0xe0,0xa4,0x9c,0xe0,0xa4,0xbc,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa4, - 0xa1,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xa2,0xe0,0xa4,0xbc,0x00,0x10,0x0b, - 0x01,0xff,0xe0,0xa4,0xab,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xaf,0xe0,0xa4, - 0xbc,0x00,0x54,0x04,0x01,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, - 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0c,0x00,0x0c,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, - 0x10,0x00,0x0b,0x00,0x10,0x04,0x0b,0x00,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, - 0x08,0x00,0x09,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00, - 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, - 0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, - 0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, - 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, - 0xd3,0x18,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, - 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00, - 0x91,0x08,0x10,0x04,0x01,0x07,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x42, - 0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, - 0x00,0x00,0x01,0x00,0xd2,0x17,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, - 0x00,0x00,0x01,0xff,0xe0,0xa7,0x87,0xe0,0xa6,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01, - 0xff,0xe0,0xa7,0x87,0xe0,0xa7,0x97,0x00,0x01,0x09,0x10,0x04,0x08,0x00,0x00,0x00, - 0xd3,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, - 0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa6,0xa1,0xe0,0xa6,0xbc, - 0x00,0x01,0xff,0xe0,0xa6,0xa2,0xe0,0xa6,0xbc,0x00,0x10,0x04,0x00,0x00,0x01,0xff, - 0xe0,0xa6,0xaf,0xe0,0xa6,0xbc,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x01,0x00,0x11, - 0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x14,0xe6,0x00, - 0x00,0xe2,0x48,0x02,0xe1,0x4f,0x01,0xd0,0xa4,0xcf,0x86,0xd5,0x4c,0xd4,0x34,0xd3, - 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x10,0x04,0x01,0x00,0x07, - 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01, - 0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x2e,0xd2,0x17,0xd1, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa8,0xb2, - 0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x0b,0x01,0xff, - 0xe0,0xa8,0xb8,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00, - 0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x00,0x00,0x01,0x00,0xcf,0x86,0xd5,0x80,0xd4, - 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51, - 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01, - 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01, - 0x09,0x00,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x00, - 0x00,0x00,0x00,0xd2,0x25,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xe0,0xa8,0x96, - 0xe0,0xa8,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa8,0x97,0xe0,0xa8,0xbc,0x00,0x01, - 0xff,0xe0,0xa8,0x9c,0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, - 0x10,0x0b,0x01,0xff,0xe0,0xa8,0xab,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd4,0x10,0x93, - 0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x52, - 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04,0x14,0x00,0x00, - 0x00,0x00,0x00,0xd0,0x82,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91, - 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, - 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x10, - 0x04,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, - 0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01, - 0x07,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10,0x52,0x04,0x01, - 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x00, - 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x07, - 0x00,0x07,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x0d,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x11,0x00,0x13,0x00,0x13,0x00,0xe1,0x24,0x01,0xd0,0x86,0xcf,0x86, - 0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, - 0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, - 0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14, - 0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, - 0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, - 0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x01,0x00, - 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x01,0x00, - 0x01,0x00,0xcf,0x86,0xd5,0x73,0xd4,0x45,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08, - 0x10,0x04,0x0a,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x0f, - 0x10,0x0b,0x01,0xff,0xe0,0xad,0x87,0xe0,0xad,0x96,0x00,0x00,0x00,0x10,0x04,0x00, - 0x00,0x01,0xff,0xe0,0xad,0x87,0xe0,0xac,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff, - 0xe0,0xad,0x87,0xe0,0xad,0x97,0x00,0x01,0x09,0x00,0x00,0xd3,0x0c,0x52,0x04,0x00, - 0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01, - 0xff,0xe0,0xac,0xa1,0xe0,0xac,0xbc,0x00,0x01,0xff,0xe0,0xac,0xa2,0xe0,0xac,0xbc, - 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01, - 0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x0c,0x00,0x0c,0x00,0x00,0x00,0xd0,0xb1,0xcf, - 0x86,0xd5,0x63,0xd4,0x28,0xd3,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x91, - 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd3,0x1f,0xd2,0x0c,0x91, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, - 0xae,0x92,0xe0,0xaf,0x97,0x00,0x01,0x00,0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, - 0x00,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x00, - 0x00,0x00,0x01,0x00,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04, - 0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c, - 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00, - 0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x08,0x00,0x01,0x00, - 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, - 0xd5,0x61,0xd4,0x45,0xd3,0x14,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, - 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00, - 0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0,0xae,0xbe,0x00,0x01,0xff,0xe0, - 0xaf,0x87,0xe0,0xae,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0, - 0xaf,0x97,0x00,0x01,0x09,0x00,0x00,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0a, - 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x00, - 0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x08, - 0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x07,0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00, - 0x00,0x00,0x00,0xe3,0x1c,0x04,0xe2,0x1a,0x02,0xd1,0xf3,0xd0,0x76,0xcf,0x86,0xd5, - 0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01, - 0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91, - 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01, - 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, - 0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0xd2, - 0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x01, - 0x00,0xcf,0x86,0xd5,0x53,0xd4,0x2f,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, - 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, - 0xb1,0x86,0xe0,0xb1,0x96,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00, - 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, - 0x01,0x54,0x10,0x04,0x01,0x5b,0x00,0x00,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, - 0x11,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, - 0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x52,0x04,0x00,0x00, - 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0a,0x00,0xd0,0x76,0xcf,0x86, - 0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x10,0x00, - 0x01,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, - 0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, - 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, - 0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00, - 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x07,0x07,0x07,0x00, - 0x01,0x00,0xcf,0x86,0xd5,0x82,0xd4,0x5e,0xd3,0x2a,0xd2,0x13,0x91,0x0f,0x10,0x0b, - 0x01,0xff,0xe0,0xb2,0xbf,0xe0,0xb3,0x95,0x00,0x01,0x00,0x01,0x00,0xd1,0x08,0x10, - 0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3, - 0x95,0x00,0xd2,0x28,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x96, - 0x00,0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x82,0x00,0x01,0xff, - 0xe0,0xb3,0x86,0xe0,0xb3,0x82,0xe0,0xb3,0x95,0x00,0x91,0x08,0x10,0x04,0x01,0x00, - 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, - 0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00, - 0x10,0x04,0x01,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, - 0x09,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08, - 0x10,0x04,0x00,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xe1,0x06,0x01,0xd0,0x6e,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91, - 0x08,0x10,0x04,0x13,0x00,0x10,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, - 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01, - 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01, - 0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, - 0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x0c,0x00,0x13,0x09,0x91,0x08,0x10,0x04,0x13,0x09,0x0a,0x00,0x01, - 0x00,0xcf,0x86,0xd5,0x65,0xd4,0x45,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, - 0x04,0x0a,0x00,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00,0x00, - 0x00,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb4,0xbe,0x00,0x01,0xff,0xe0,0xb5, - 0x87,0xe0,0xb4,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb5, - 0x97,0x00,0x01,0x09,0x10,0x04,0x0c,0x00,0x12,0x00,0xd3,0x10,0x52,0x04,0x00,0x00, - 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x01,0x00,0x52,0x04,0x12,0x00,0x51,0x04, - 0x12,0x00,0x10,0x04,0x12,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04, - 0x01,0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x0c,0x52,0x04, - 0x0a,0x00,0x11,0x04,0x0a,0x00,0x12,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00, - 0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x5a,0xcf,0x86,0xd5,0x34,0xd4,0x18,0x93,0x14, - 0xd2,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00, - 0x04,0x00,0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, - 0x04,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x54,0x04, - 0x04,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x04,0x00,0x10,0x04,0x00,0x00,0x04,0x00, - 0x04,0x00,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00,0x00,0x00, - 0xcf,0x86,0xd5,0x77,0xd4,0x28,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00, - 0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x04,0x09, - 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x04,0x00,0xd3,0x14,0x52,0x04, - 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x10,0x04,0x04,0x00,0x00,0x00, - 0xd2,0x13,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8a, - 0x00,0x04,0x00,0xd1,0x19,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0x00, - 0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0xe0,0xb7,0x8a,0x00,0x10,0x0b,0x04,0xff, - 0xe0,0xb7,0x99,0xe0,0xb7,0x9f,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x00, - 0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x14,0xd2,0x08,0x11,0x04,0x00, - 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe2, - 0x31,0x01,0xd1,0x58,0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x67,0x10,0x04, - 0x01,0x09,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, - 0x95,0x18,0xd4,0x0c,0x53,0x04,0x01,0x00,0x12,0x04,0x01,0x6b,0x01,0x00,0x53,0x04, - 0x01,0x00,0x12,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd0,0x9e,0xcf,0x86,0xd5,0x54, - 0xd4,0x3c,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04, - 0x01,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x15,0x00, - 0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x15,0x00,0x10,0x04,0x01,0x00, - 0x00,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x15,0x00,0xd3,0x08,0x12,0x04, - 0x15,0x00,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, - 0x01,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00, - 0xd2,0x08,0x11,0x04,0x15,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, - 0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x76,0x10,0x04,0x15,0x09, - 0x01,0x00,0x11,0x04,0x01,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4,0x20,0xd3,0x14, - 0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, - 0x00,0x00,0x52,0x04,0x01,0x7a,0x11,0x04,0x01,0x00,0x00,0x00,0x53,0x04,0x01,0x00, - 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x01,0x00,0x0d,0x00,0x00,0x00, - 0xe1,0x2b,0x01,0xd0,0x3e,0xcf,0x86,0xd5,0x14,0x54,0x04,0x02,0x00,0x53,0x04,0x02, - 0x00,0x92,0x08,0x11,0x04,0x02,0xdc,0x02,0x00,0x02,0x00,0x54,0x04,0x02,0x00,0xd3, - 0x14,0x52,0x04,0x02,0x00,0xd1,0x08,0x10,0x04,0x02,0x00,0x02,0xdc,0x10,0x04,0x02, - 0x00,0x02,0xdc,0x92,0x0c,0x91,0x08,0x10,0x04,0x02,0x00,0x02,0xd8,0x02,0x00,0x02, - 0x00,0xcf,0x86,0xd5,0x73,0xd4,0x36,0xd3,0x17,0x92,0x13,0x51,0x04,0x02,0x00,0x10, - 0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x82,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x00,0x00,0x02,0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00, - 0x02,0xff,0xe0,0xbd,0x8c,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd3,0x26,0xd2,0x13,0x51, - 0x04,0x02,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x91,0xe0,0xbe,0xb7,0x00,0x02,0x00, - 0x51,0x04,0x02,0x00,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x96,0xe0,0xbe,0xb7, - 0x00,0x52,0x04,0x02,0x00,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x9b,0xe0,0xbe, - 0xb7,0x00,0x02,0x00,0x02,0x00,0xd4,0x27,0x53,0x04,0x02,0x00,0xd2,0x17,0xd1,0x0f, - 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x80,0xe0,0xbe,0xb5,0x00,0x10,0x04,0x04, - 0x00,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd3,0x35,0xd2, - 0x17,0xd1,0x08,0x10,0x04,0x00,0x00,0x02,0x81,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, - 0xbd,0xb1,0xe0,0xbd,0xb2,0x00,0xd1,0x0f,0x10,0x04,0x02,0x84,0x02,0xff,0xe0,0xbd, - 0xb1,0xe0,0xbd,0xb4,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb2,0xe0,0xbe,0x80,0x00, - 0x02,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb3,0xe0,0xbe,0x80, - 0x00,0x02,0x00,0x02,0x82,0x11,0x04,0x02,0x82,0x02,0x00,0xd0,0xd3,0xcf,0x86,0xd5, - 0x65,0xd4,0x27,0xd3,0x1f,0xd2,0x13,0x91,0x0f,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, - 0xbd,0xb1,0xe0,0xbe,0x80,0x00,0x02,0xe6,0x91,0x08,0x10,0x04,0x02,0x09,0x02,0x00, - 0x02,0xe6,0x12,0x04,0x02,0x00,0x0c,0x00,0xd3,0x1f,0xd2,0x13,0x51,0x04,0x02,0x00, - 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x92,0xe0,0xbe,0xb7,0x00,0x51,0x04,0x02, - 0x00,0x10,0x04,0x04,0x00,0x02,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x02, - 0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x9c,0xe0,0xbe, - 0xb7,0x00,0x02,0x00,0xd4,0x3d,0xd3,0x26,0xd2,0x13,0x51,0x04,0x02,0x00,0x10,0x0b, - 0x02,0xff,0xe0,0xbe,0xa1,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x51,0x04,0x02,0x00,0x10, - 0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0xa6,0xe0,0xbe,0xb7,0x00,0x52,0x04,0x02,0x00, - 0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xab,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x04, - 0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x02,0x00,0x02,0x00,0x02, - 0x00,0xd2,0x13,0x91,0x0f,0x10,0x04,0x04,0x00,0x02,0xff,0xe0,0xbe,0x90,0xe0,0xbe, - 0xb5,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xcf,0x86, - 0x95,0x4c,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, - 0x04,0xdc,0x04,0x00,0x52,0x04,0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00, - 0x10,0x04,0x0a,0x00,0x04,0x00,0xd3,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x0a,0x00, - 0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04, - 0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, - 0xe5,0xf7,0x04,0xe4,0x79,0x03,0xe3,0x7b,0x01,0xe2,0x04,0x01,0xd1,0x7f,0xd0,0x65, - 0xcf,0x86,0x55,0x04,0x04,0x00,0xd4,0x33,0xd3,0x1f,0xd2,0x0c,0x51,0x04,0x04,0x00, - 0x10,0x04,0x0a,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe1,0x80, - 0xa5,0xe1,0x80,0xae,0x00,0x04,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0a,0x00,0x04, - 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x04,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x04, - 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x04,0x00,0x04, - 0x07,0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0x09,0x10,0x04,0x0a,0x09,0x0a, - 0x00,0x0a,0x00,0xcf,0x86,0x95,0x14,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92, - 0x08,0x11,0x04,0x04,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x2e,0xcf,0x86,0x95, - 0x28,0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a, - 0x00,0x0a,0xdc,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00,0x0b, - 0x00,0x11,0x04,0x0b,0x00,0x0a,0x00,0x01,0x00,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3, - 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x52, - 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x54, - 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x06,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x06,0x00,0x08,0x00,0x10,0x04,0x08, - 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x3e,0xd0, - 0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x1d,0x54,0x04,0x01,0x00,0x53,0x04,0x01, - 0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, - 0x00,0x01,0xff,0x00,0x94,0x15,0x93,0x11,0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, - 0x04,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x0b,0x00,0x0b,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, - 0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x0b, - 0x00,0xe2,0x21,0x01,0xd1,0x6c,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, - 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00, - 0x04,0x00,0x04,0x00,0xcf,0x86,0x95,0x48,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00, - 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04, - 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, - 0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08, - 0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00, - 0xd0,0x62,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04, - 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00, - 0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xd4,0x14,0x53,0x04, - 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, - 0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04, - 0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00, - 0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04, - 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00, - 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x93,0x10,0x52,0x04,0x04,0x00, - 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x94,0x14,0x53,0x04, - 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, - 0x04,0x00,0xd1,0x9c,0xd0,0x3e,0xcf,0x86,0x95,0x38,0xd4,0x14,0x53,0x04,0x04,0x00, - 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd3,0x14, - 0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00, - 0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, - 0x04,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04, - 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00,0x53,0x04,0x04,0x00,0xd2,0x0c, - 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, - 0x0c,0xe6,0x10,0x04,0x0c,0xe6,0x08,0xe6,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x08,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x53,0x04,0x04,0x00, - 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a, - 0xcf,0x86,0x95,0x14,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04, - 0x08,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04, - 0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x11,0x00, - 0x00,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0xd3,0x30,0xd2,0x2a, - 0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0b,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00, - 0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd2,0x6c,0xd1,0x24, - 0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, - 0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x0b,0x00, - 0x0b,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00, - 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00, - 0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x04,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x46,0xcf,0x86,0xd5,0x28, - 0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x00, - 0x00,0x00,0x06,0x00,0x93,0x10,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x09, - 0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x06,0x00,0x93,0x14,0x52,0x04,0x06,0x00, - 0xd1,0x08,0x10,0x04,0x06,0x09,0x06,0x00,0x10,0x04,0x06,0x00,0x00,0x00,0x00,0x00, - 0xcf,0x86,0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x06,0x00,0x00,0x00, - 0x00,0x00,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04, - 0x06,0x00,0x00,0x00,0x06,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, - 0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0xd0,0x1b,0xcf,0x86,0x55,0x04,0x04,0x00, - 0x54,0x04,0x04,0x00,0x93,0x0d,0x52,0x04,0x04,0x00,0x11,0x05,0x04,0xff,0x00,0x04, - 0x00,0x04,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x51, - 0x04,0x04,0x00,0x10,0x04,0x04,0x09,0x04,0x00,0x04,0x00,0x52,0x04,0x04,0x00,0x91, - 0x08,0x10,0x04,0x04,0x00,0x07,0xe6,0x00,0x00,0xd4,0x10,0x53,0x04,0x04,0x00,0x92, - 0x08,0x11,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x08,0x11, - 0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xe4,0xb7,0x03,0xe3,0x58,0x01,0xd2,0x8f,0xd1, - 0x53,0xd0,0x35,0xcf,0x86,0x95,0x2f,0xd4,0x1f,0x53,0x04,0x04,0x00,0xd2,0x0d,0x51, - 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x04,0xff,0x00,0x51,0x05,0x04,0xff,0x00,0x10, - 0x05,0x04,0xff,0x00,0x00,0x00,0x53,0x04,0x04,0x00,0x92,0x08,0x11,0x04,0x04,0x00, - 0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, - 0x53,0x04,0x04,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x04,0x00,0x94,0x18,0x53,0x04,0x04,0x00, - 0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0xe4,0x10,0x04,0x0a,0x00,0x00,0x00, - 0x00,0x00,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x0c, - 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x42, - 0xcf,0x86,0xd5,0x1c,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, - 0xd1,0x08,0x10,0x04,0x07,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0xd4,0x0c, - 0x53,0x04,0x07,0x00,0x12,0x04,0x07,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x10, - 0xd1,0x08,0x10,0x04,0x07,0x00,0x07,0xde,0x10,0x04,0x07,0xe6,0x07,0xdc,0x00,0x00, - 0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, - 0x00,0x00,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd4,0x10,0x53,0x04,0x07,0x00, - 0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x07,0x00, - 0x91,0x08,0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a,0xcf,0x86, - 0x55,0x04,0x08,0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00, - 0x0b,0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x95,0x28,0xd4,0x10,0x53,0x04,0x08,0x00, - 0x92,0x08,0x11,0x04,0x08,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0xd2,0x0c, - 0x51,0x04,0x08,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x08,0x00, - 0x07,0x00,0xd2,0xe4,0xd1,0x80,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x54,0x04,0x08,0x00, - 0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x08,0xe6, - 0xd2,0x0c,0x91,0x08,0x10,0x04,0x08,0xdc,0x08,0x00,0x08,0x00,0x11,0x04,0x00,0x00, - 0x08,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, - 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xd4,0x14, - 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00, - 0x0b,0x00,0xd3,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6, - 0x0b,0xe6,0x52,0x04,0x0b,0xe6,0xd1,0x08,0x10,0x04,0x0b,0xe6,0x00,0x00,0x10,0x04, - 0x00,0x00,0x0b,0xdc,0xd0,0x5e,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0b,0x00, - 0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0b,0x00,0x92,0x08, - 0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53,0x04,0x0b,0x00,0x52,0x04, - 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x10,0xe6,0x91,0x08, - 0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0xdc,0xd2,0x0c,0x51,0x04,0x10,0xdc,0x10,0x04, - 0x10,0xdc,0x10,0xe6,0xd1,0x08,0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0x04,0x10,0x00, - 0x00,0x00,0xcf,0x06,0x00,0x00,0xe1,0x1e,0x01,0xd0,0xaa,0xcf,0x86,0xd5,0x6e,0xd4, - 0x53,0xd3,0x17,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1, - 0xac,0x85,0xe1,0xac,0xb5,0x00,0x09,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x09,0xff, - 0xe1,0xac,0x87,0xe1,0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x89, - 0xe1,0xac,0xb5,0x00,0x09,0x00,0xd1,0x0f,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8b,0xe1, - 0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8d,0xe1,0xac,0xb5,0x00, - 0x09,0x00,0x93,0x17,0x92,0x13,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac, - 0x91,0xe1,0xac,0xb5,0x00,0x09,0x00,0x09,0x00,0x09,0x00,0x54,0x04,0x09,0x00,0xd3, - 0x10,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x07,0x09,0x00,0x09,0x00,0xd2, - 0x13,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xba,0xe1,0xac, - 0xb5,0x00,0x91,0x0f,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xbc,0xe1,0xac,0xb5, - 0x00,0x09,0x00,0xcf,0x86,0xd5,0x3d,0x94,0x39,0xd3,0x31,0xd2,0x25,0xd1,0x16,0x10, - 0x0b,0x09,0xff,0xe1,0xac,0xbe,0xe1,0xac,0xb5,0x00,0x09,0xff,0xe1,0xac,0xbf,0xe1, - 0xac,0xb5,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xad,0x82,0xe1,0xac,0xb5,0x00, - 0x91,0x08,0x10,0x04,0x09,0x09,0x09,0x00,0x09,0x00,0x12,0x04,0x09,0x00,0x00,0x00, - 0x09,0x00,0xd4,0x1c,0x53,0x04,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09,0x00,0x10,0x04, - 0x09,0x00,0x09,0xe6,0x91,0x08,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0xe6,0xd3,0x08, - 0x12,0x04,0x09,0xe6,0x09,0x00,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, - 0x00,0x00,0x00,0x00,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x18,0x53,0x04, - 0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x09,0x0d,0x09,0x11,0x04, - 0x0d,0x00,0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x0d,0x00, - 0x0d,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x0c,0x00, - 0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x07,0x0c,0x00,0x0c,0x00,0xd3,0x0c,0x92,0x08, - 0x11,0x04,0x0c,0x00,0x0c,0x09,0x00,0x00,0x12,0x04,0x00,0x00,0x0c,0x00,0xe3,0xb2, - 0x01,0xe2,0x09,0x01,0xd1,0x4c,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x0a,0x00,0x54,0x04, - 0x0a,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00, - 0x0a,0x07,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00, - 0xcf,0x86,0x95,0x1c,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, - 0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00, - 0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x54,0x04,0x14,0x00, - 0x53,0x04,0x14,0x00,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00, - 0x91,0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x08, - 0x13,0x04,0x0d,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04, - 0x0b,0xe6,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x01,0x0b,0xdc,0x0b,0xdc,0x92,0x08, - 0x11,0x04,0x0b,0xdc,0x0b,0xe6,0x0b,0xdc,0xd4,0x28,0xd3,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x01,0x0b,0x01,0xd2,0x0c,0x91,0x08,0x10,0x04, - 0x0b,0x01,0x0b,0x00,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xdc,0x0b,0x00, - 0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0d,0x00,0xd1,0x08, - 0x10,0x04,0x0d,0xe6,0x0d,0x00,0x10,0x04,0x0d,0x00,0x13,0x00,0x92,0x0c,0x51,0x04, - 0x10,0xe6,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06, - 0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x94,0x0c,0x53,0x04,0x07,0x00,0x12,0x04, - 0x07,0x00,0x08,0x00,0x08,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0xd5,0x40, - 0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x51,0x04,0x08,0xe6,0x10,0x04,0x08,0xdc,0x08,0xe6, - 0x09,0xe6,0xd2,0x0c,0x51,0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x0a,0xe6,0xd1,0x08, - 0x10,0x04,0x0a,0xe6,0x0a,0xea,0x10,0x04,0x0a,0xd6,0x0a,0xdc,0x93,0x10,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x0a,0xca,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0xd4,0x14, - 0x93,0x10,0x52,0x04,0x0a,0xe6,0x51,0x04,0x0a,0xe6,0x10,0x04,0x0a,0xe6,0x10,0xe6, - 0x10,0xe6,0xd3,0x10,0x52,0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x13,0xe8, - 0x13,0xe4,0xd2,0x10,0xd1,0x08,0x10,0x04,0x13,0xe4,0x13,0xdc,0x10,0x04,0x00,0x00, - 0x12,0xe6,0xd1,0x08,0x10,0x04,0x0c,0xe9,0x0b,0xdc,0x10,0x04,0x09,0xe6,0x09,0xdc, - 0xe2,0x80,0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84, - 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa5,0x00,0x01,0xff, - 0x61,0xcc,0xa5,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc, - 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x42,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc, - 0xa3,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00, - 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x43,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff, - 0x63,0xcc,0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0x87,0x00,0x01,0xff, - 0x64,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa3,0x00,0x01,0xff, - 0x64,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc, - 0xb1,0x00,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa7,0x00, - 0x01,0xff,0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xad,0x00,0x01,0xff, - 0x64,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc,0x80,0x00, - 0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc, - 0x81,0x00,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0x45,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff, - 0x45,0xcc,0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, - 0x45,0xcc,0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08, - 0x01,0xff,0x46,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc, - 0x84,0x00,0x10,0x08,0x01,0xff,0x48,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00, - 0x10,0x08,0x01,0xff,0x48,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00, - 0x10,0x08,0x01,0xff,0x48,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0x49,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a, - 0x01,0xff,0x49,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00, - 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0x81,0x00,0x01,0xff, - 0x6b,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc, - 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc, - 0xb1,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00, - 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff, - 0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xb1,0x00,0x01,0xff, - 0x6c,0xcc,0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4c,0xcc,0xad,0x00,0x01,0xff, - 0x6c,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x4d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc, - 0x81,0x00,0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x4d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01, - 0xff,0x4d,0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x4e,0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x4e, - 0xcc,0xa3,0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x4e,0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x4e, - 0xcc,0xad,0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f, - 0xcc,0x83,0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01, - 0xff,0x4f,0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3, - 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x80,0x00,0x01, - 0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x81, - 0x00,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x50, - 0xcc,0x81,0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x50,0xcc,0x87, - 0x00,0x01,0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52, - 0xcc,0x87,0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa3, - 0x00,0x01,0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x52,0xcc,0xa3, - 0xcc,0x84,0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x52, - 0xcc,0xb1,0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10, - 0x08,0x01,0xff,0x53,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10, - 0x0a,0x01,0xff,0x53,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87, - 0x00,0x10,0x0a,0x01,0xff,0x53,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c, - 0xcc,0x87,0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x53,0xcc,0xa3,0xcc,0x87, - 0x00,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0x87, - 0x00,0x01,0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0xa3, - 0x00,0x01,0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xb1,0x00,0x01, - 0xff,0x74,0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x54, - 0xcc,0xad,0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa4, - 0x00,0x01,0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0xb0, - 0x00,0x01,0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xad,0x00,0x01, - 0xff,0x75,0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x83, - 0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x55, - 0xcc,0x84,0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0x56,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01, - 0xff,0x56,0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x10,0x02,0xcf,0x86, - 0xd5,0xe1,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc, - 0x80,0x00,0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x81,0x00, - 0x01,0xff,0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x88,0x00, - 0x01,0xff,0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x87,0x00,0x01,0xff, - 0x77,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0xa3,0x00, - 0x01,0xff,0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x58,0xcc,0x87,0x00,0x01,0xff, - 0x78,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x58,0xcc,0x88,0x00,0x01,0xff, - 0x78,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc, - 0x87,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0x82,0x00, - 0x01,0xff,0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x5a,0xcc,0xa3,0x00,0x01,0xff, - 0x7a,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0xb1,0x00,0x01,0xff, - 0x7a,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x68,0xcc,0xb1,0x00,0x01,0xff,0x74,0xcc, - 0x88,0x00,0x92,0x1d,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01,0xff, - 0x79,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x02,0xff,0xc5,0xbf,0xcc,0x87,0x00,0x0a, - 0x00,0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa3, - 0x00,0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x89,0x00,0x01, - 0xff,0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x81, - 0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82, - 0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10, - 0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89, - 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82, - 0xcc,0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x82,0x00,0x01, - 0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x81, - 0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10, - 0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80, - 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86, - 0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x83,0x00,0x01, - 0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x86, - 0x00,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0x45,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x45, - 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45, - 0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82, - 0xcc,0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01, - 0xd4,0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82,0xcc, - 0x80,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc, - 0x82,0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a, - 0x01,0xff,0x45,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00, - 0x10,0x0a,0x01,0xff,0x45,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc, - 0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x89,0x00,0x01,0xff, - 0x69,0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc, - 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc, - 0xa3,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00, - 0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc,0x81,0x00, - 0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc, - 0x80,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, - 0x4f,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a, - 0x01,0xff,0x4f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00, - 0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, - 0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x81,0x00, - 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc, - 0x9b,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, - 0x4f,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98, - 0xd3,0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x83,0x00, - 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc, - 0xa3,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0x55,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x55,0xcc, - 0x89,0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff, - 0x55,0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a, - 0x01,0xff,0x55,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00, - 0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc, - 0x9b,0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff, - 0x75,0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff, - 0x55,0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08, - 0x01,0xff,0x59,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0x59,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff, - 0x59,0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0x92,0x14,0x91,0x10,0x10,0x08, - 0x01,0xff,0x59,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x0a,0x00,0x0a,0x00, - 0xe1,0xc0,0x04,0xe0,0x80,0x02,0xcf,0x86,0xe5,0x2d,0x01,0xd4,0xa8,0xd3,0x54,0xd2, - 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, - 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, - 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, - 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, - 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, - 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x91,0xcc,0x93,0x00,0x01,0xff, - 0xce,0x91,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80,0x00, - 0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, - 0x91,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x81,0x00,0x10, - 0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x91,0xcc,0x94, - 0xcd,0x82,0x00,0xd3,0x42,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, - 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc, - 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, - 0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94, - 0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x95,0xcc, - 0x93,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x95,0xcc, - 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, - 0x0b,0x01,0xff,0xce,0x95,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x95,0xcc,0x94, - 0xcc,0x81,0x00,0x00,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, - 0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01, - 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80, - 0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, - 0xce,0xb7,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd, - 0x82,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28,0xd1,0x12,0x10, - 0x09,0x01,0xff,0xce,0x97,0xcc,0x93,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0x00,0x10, - 0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x94, - 0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81,0x00, - 0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc, - 0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x82,0x00,0xd3,0x54,0xd2, - 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9, - 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, - 0xce,0xb9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc, - 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, - 0xff,0xce,0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82, - 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x93,0x00,0x01,0xff, - 0xce,0x99,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcc,0x80,0x00, - 0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, - 0x99,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x81,0x00,0x10, - 0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x99,0xcc,0x94, - 0xcd,0x82,0x00,0xcf,0x86,0xe5,0x13,0x01,0xd4,0x84,0xd3,0x42,0xd2,0x28,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00, - 0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, - 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x81, - 0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xce,0x9f,0xcc,0x93,0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0x00, - 0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc, - 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x81, - 0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x54,0xd2,0x28, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff,0xcf,0x85,0xcc, - 0x94,0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf, - 0x85,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93, - 0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, - 0xcf,0x85,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00, - 0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0x00,0x10, - 0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x0f,0x10,0x04, - 0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x81,0x00,0x10,0x04,0x00,0x00,0x01, - 0xff,0xce,0xa5,0xcc,0x94,0xcd,0x82,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00, - 0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, - 0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, - 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89, - 0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28, - 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xa9,0xcc,0x93,0x00,0x01,0xff,0xce,0xa9,0xcc, - 0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce, - 0xa9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93, - 0xcc,0x81,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, - 0xce,0xa9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0x00, - 0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01, - 0xff,0xce,0xb1,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01, - 0xff,0xce,0xb5,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80, - 0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80, - 0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff, - 0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, - 0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x91,0x12,0x10,0x09, - 0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x00,0x00, - 0xe0,0xe1,0x02,0xcf,0x86,0xe5,0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16, - 0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, - 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xcd,0x85, - 0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, - 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, - 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82, - 0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30, - 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce, - 0x91,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80, - 0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a, - 0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce, - 0x91,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93, - 0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00, - 0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x85, - 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7, - 0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xcd, - 0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xcd,0x85, - 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, - 0xce,0xb7,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd, - 0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93, - 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, - 0xce,0x97,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc, - 0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81, - 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d, - 0x01,0xff,0xce,0x97,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc, - 0x94,0xcd,0x82,0xcd,0x85,0x00,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b, - 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd, - 0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01, - 0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff, - 0xcf,0x89,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, - 0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xcd,0x85, - 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16, - 0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, - 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0xcd,0x85, - 0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, - 0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, - 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x82, - 0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd3,0x49, - 0xd2,0x26,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce, - 0xb1,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x80,0xcd,0x85,0x00,0x01, - 0xff,0xce,0xb1,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81, - 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff, - 0xce,0xb1,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, - 0x91,0xcc,0x86,0x00,0x01,0xff,0xce,0x91,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce, - 0x91,0xcc,0x80,0x00,0x01,0xff,0xce,0x91,0xcc,0x81,0x00,0xd1,0x0d,0x10,0x09,0x01, - 0xff,0xce,0x91,0xcd,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xb9,0x00,0x01, - 0x00,0xcf,0x86,0xe5,0x16,0x01,0xd4,0x8f,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10,0x04, - 0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc, - 0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01, - 0xff,0xce,0xb7,0xcc,0x81,0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb7, - 0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12, - 0x10,0x09,0x01,0xff,0xce,0x95,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x81,0x00, - 0x10,0x09,0x01,0xff,0xce,0x97,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00, - 0xd1,0x13,0x10,0x09,0x01,0xff,0xce,0x97,0xcd,0x85,0x00,0x01,0xff,0xe1,0xbe,0xbf, - 0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbe,0xbf,0xcc,0x81,0x00,0x01,0xff,0xe1, - 0xbe,0xbf,0xcd,0x82,0x00,0xd3,0x40,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, - 0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce, - 0xb9,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x51, - 0x04,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9, - 0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, - 0x86,0x00,0x01,0xff,0xce,0x99,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, - 0x80,0x00,0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x04,0x00,0x00,0x01, - 0xff,0xe1,0xbf,0xbe,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbf,0xbe,0xcc,0x81, - 0x00,0x01,0xff,0xe1,0xbf,0xbe,0xcd,0x82,0x00,0xd4,0x93,0xd3,0x4e,0xd2,0x28,0xd1, - 0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84, - 0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85, - 0xcc,0x88,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x93,0x00, - 0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcd,0x82,0x00, - 0x01,0xff,0xcf,0x85,0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01, - 0xff,0xce,0xa5,0xcc,0x86,0x00,0x01,0xff,0xce,0xa5,0xcc,0x84,0x00,0x10,0x09,0x01, - 0xff,0xce,0xa5,0xcc,0x80,0x00,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0xd1,0x12,0x10, - 0x09,0x01,0xff,0xce,0xa1,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x80,0x00,0x10, - 0x09,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x01,0xff,0x60,0x00,0xd3,0x3b,0xd2,0x18, - 0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x80,0xcd,0x85,0x00,0x01, - 0xff,0xcf,0x89,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81, - 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcd,0x82,0x00,0x01,0xff, - 0xcf,0x89,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, - 0x9f,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce, - 0xa9,0xcc,0x80,0x00,0x01,0xff,0xce,0xa9,0xcc,0x81,0x00,0xd1,0x10,0x10,0x09,0x01, - 0xff,0xce,0xa9,0xcd,0x85,0x00,0x01,0xff,0xc2,0xb4,0x00,0x10,0x04,0x01,0x00,0x00, - 0x00,0xe0,0x7e,0x0c,0xcf,0x86,0xe5,0xbb,0x08,0xe4,0x14,0x06,0xe3,0xf7,0x02,0xe2, - 0xbd,0x01,0xd1,0xd0,0xd0,0x4f,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0xd3,0x18,0x92,0x14, - 0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01,0xff,0xe2,0x80,0x83,0x00, - 0x01,0x00,0x01,0x00,0x92,0x0d,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, - 0x00,0x01,0xff,0x00,0x01,0x00,0x94,0x1b,0x53,0x04,0x01,0x00,0xd2,0x09,0x11,0x04, - 0x01,0x00,0x01,0xff,0x00,0x51,0x05,0x01,0xff,0x00,0x10,0x05,0x01,0xff,0x00,0x04, - 0x00,0x01,0x00,0xcf,0x86,0xd5,0x48,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x01,0x00,0x51, - 0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x52,0x04,0x04,0x00,0x11,0x04,0x04, - 0x00,0x06,0x00,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x06,0x00,0x07, - 0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0x52, - 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0xd4,0x23,0xd3, - 0x14,0x52,0x05,0x06,0xff,0x00,0x91,0x0a,0x10,0x05,0x0a,0xff,0x00,0x00,0xff,0x00, - 0x0f,0xff,0x00,0x92,0x0a,0x11,0x05,0x0f,0xff,0x00,0x01,0xff,0x00,0x01,0xff,0x00, - 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x00,0x00,0x01,0x00, - 0x01,0x00,0xd0,0x7e,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x53,0x04,0x01,0x00,0x52,0x04, - 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, - 0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04,0x0c,0x00, - 0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00, - 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x02,0x00,0x91,0x08,0x10,0x04, - 0x03,0x00,0x04,0x00,0x04,0x00,0xd3,0x10,0xd2,0x08,0x11,0x04,0x06,0x00,0x08,0x00, - 0x11,0x04,0x08,0x00,0x0b,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00, - 0x10,0x04,0x0e,0x00,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x11,0x00,0x13,0x00, - 0xcf,0x86,0xd5,0x28,0x54,0x04,0x00,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x01,0xe6, - 0x01,0x01,0x01,0xe6,0xd2,0x0c,0x51,0x04,0x01,0x01,0x10,0x04,0x01,0x01,0x01,0xe6, - 0x91,0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x04,0x00,0xd1,0x08,0x10,0x04,0x06,0x00, - 0x06,0x01,0x10,0x04,0x06,0x01,0x06,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x06,0xdc, - 0x06,0xe6,0x10,0x04,0x06,0x01,0x08,0x01,0x09,0xdc,0x93,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0a,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x81,0xd0,0x4f, - 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04, - 0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xa9,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01, - 0x00,0x10,0x06,0x01,0xff,0x4b,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x01,0x00,0x53, - 0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04, - 0x00,0x07,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0x95, - 0x2c,0xd4,0x18,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0xd1,0x08,0x10,0x04,0x08, - 0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b, - 0x00,0x10,0x04,0x0b,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x68,0xcf, - 0x86,0xd5,0x48,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x11,0x00,0x00,0x00,0x53,0x04,0x01,0x00,0x92, - 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x90,0xcc,0xb8,0x00,0x01, - 0xff,0xe2,0x86,0x92,0xcc,0xb8,0x00,0x01,0x00,0x94,0x1a,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x94,0xcc,0xb8, - 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0xd1,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x87,0x90,0xcc,0xb8, - 0x00,0x10,0x0a,0x01,0xff,0xe2,0x87,0x94,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x87,0x92, - 0xcc,0xb8,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x06, - 0x00,0x06,0x00,0xe2,0x38,0x02,0xe1,0x3f,0x01,0xd0,0x68,0xcf,0x86,0xd5,0x3e,0x94, - 0x3a,0xd3,0x16,0x52,0x04,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0x83, - 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01, - 0xff,0xe2,0x88,0x88,0xcc,0xb8,0x00,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2, - 0x88,0x8b,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x24,0x93,0x20,0x52, - 0x04,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa3,0xcc,0xb8,0x00,0x01, - 0x00,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa5,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01, - 0x00,0xcf,0x86,0xd5,0x48,0x94,0x44,0xd3,0x2e,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01, - 0x00,0x01,0xff,0xe2,0x88,0xbc,0xcc,0xb8,0x00,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01, - 0xff,0xe2,0x89,0x83,0xcc,0xb8,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, - 0x89,0x85,0xcc,0xb8,0x00,0x92,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, - 0x89,0x88,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x40,0xd3,0x1e,0x92, - 0x1a,0xd1,0x0c,0x10,0x08,0x01,0xff,0x3d,0xcc,0xb8,0x00,0x01,0x00,0x10,0x0a,0x01, - 0xff,0xe2,0x89,0xa1,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1, - 0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x89,0x8d,0xcc,0xb8,0x00,0x10,0x08,0x01, - 0xff,0x3c,0xcc,0xb8,0x00,0x01,0xff,0x3e,0xcc,0xb8,0x00,0xd3,0x30,0xd2,0x18,0x91, - 0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xa4,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xa5, - 0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xb2,0xcc,0xb8, - 0x00,0x01,0xff,0xe2,0x89,0xb3,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91,0x14,0x10, - 0x0a,0x01,0xff,0xe2,0x89,0xb6,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xb7,0xcc,0xb8, - 0x00,0x01,0x00,0x01,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x50,0x94,0x4c,0xd3,0x30,0xd2, - 0x18,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xba,0xcc,0xb8,0x00,0x01,0xff,0xe2, - 0x89,0xbb,0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x82, - 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x83,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91, - 0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x86,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x87, - 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x30,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa2,0xcc,0xb8,0x00,0x01, - 0xff,0xe2,0x8a,0xa8,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa9,0xcc,0xb8, - 0x00,0x01,0xff,0xe2,0x8a,0xab,0xcc,0xb8,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, - 0x00,0xd4,0x5c,0xd3,0x2c,0x92,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xbc, - 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xbd,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2, - 0x8a,0x91,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x92,0xcc,0xb8,0x00,0x01,0x00,0xd2, - 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb2,0xcc,0xb8,0x00,0x01, - 0xff,0xe2,0x8a,0xb3,0xcc,0xb8,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb4, - 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0xb5,0xcc,0xb8,0x00,0x01,0x00,0x93,0x0c,0x92, - 0x08,0x11,0x04,0x01,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xd1,0x64,0xd0,0x3e,0xcf, - 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x20,0x53,0x04,0x01,0x00,0x92, - 0x18,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x80,0x88,0x00,0x10,0x08,0x01, - 0xff,0xe3,0x80,0x89,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, - 0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0xd0, - 0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51, - 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5, - 0x2c,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10, - 0x04,0x06,0x00,0x07,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x08, - 0x00,0x08,0x00,0x08,0x00,0x12,0x04,0x08,0x00,0x09,0x00,0xd4,0x14,0x53,0x04,0x09, - 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd3, - 0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10, - 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd3,0xa6,0xd2, - 0x74,0xd1,0x40,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18,0x93,0x14,0x52, - 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x92, - 0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, - 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x14,0x53, - 0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x06, - 0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x06, - 0x00,0x07,0x00,0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54, - 0x04,0x01,0x00,0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x06, - 0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x13,0x04,0x04, - 0x00,0x06,0x00,0xd2,0xdc,0xd1,0x48,0xd0,0x26,0xcf,0x86,0x95,0x20,0x54,0x04,0x01, - 0x00,0xd3,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x07,0x00,0x06,0x00,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x08,0x00,0x04,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55, - 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x04,0x00,0x06, - 0x00,0x06,0x00,0x52,0x04,0x06,0x00,0x11,0x04,0x06,0x00,0x08,0x00,0xd0,0x5e,0xcf, - 0x86,0xd5,0x2c,0xd4,0x10,0x53,0x04,0x06,0x00,0x92,0x08,0x11,0x04,0x06,0x00,0x07, - 0x00,0x07,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x52, - 0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0a,0x00,0x0b,0x00,0xd4,0x10,0x93, - 0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd3,0x10,0x92, - 0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a, - 0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x1c,0x94, - 0x18,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b, - 0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0b,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51, - 0x04,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00,0x0b,0x00,0x0b,0x00,0xd1, - 0xa8,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10, - 0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x01, - 0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x53, - 0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x01,0x00,0x01, - 0x00,0x01,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04,0x01, - 0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x04,0x0c,0x00,0x01,0x00,0xd3, - 0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0c,0x00,0x51,0x04,0x0c, - 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x0c,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x06,0x00,0x93,0x0c,0x52,0x04,0x06,0x00,0x11, - 0x04,0x06,0x00,0x01,0x00,0x01,0x00,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01, - 0x00,0x93,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x0c, - 0x00,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, - 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, - 0x04,0x01,0x00,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0x94,0x28,0xd3,0x10,0x52,0x04,0x08, - 0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, - 0x00,0x10,0x04,0x09,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0d,0x00,0x0c, - 0x00,0x06,0x00,0x94,0x0c,0x53,0x04,0x06,0x00,0x12,0x04,0x06,0x00,0x0a,0x00,0x06, - 0x00,0xe4,0x39,0x01,0xd3,0x0c,0xd2,0x06,0xcf,0x06,0x04,0x00,0xcf,0x06,0x06,0x00, - 0xd2,0x30,0xd1,0x06,0xcf,0x06,0x06,0x00,0xd0,0x06,0xcf,0x06,0x06,0x00,0xcf,0x86, - 0x95,0x1e,0x54,0x04,0x06,0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x0e, - 0x10,0x0a,0x06,0xff,0xe2,0xab,0x9d,0xcc,0xb8,0x00,0x06,0x00,0x06,0x00,0x06,0x00, - 0xd1,0x80,0xd0,0x3a,0xcf,0x86,0xd5,0x28,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04, - 0x07,0x00,0x11,0x04,0x07,0x00,0x08,0x00,0xd3,0x08,0x12,0x04,0x08,0x00,0x09,0x00, - 0x92,0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x0c, - 0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x30, - 0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, - 0x10,0x00,0x10,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, - 0x0b,0x00,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x10,0x00,0x10,0x00,0x54,0x04, - 0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00, - 0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, - 0x11,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, - 0xd2,0x08,0x11,0x04,0x10,0x00,0x14,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x10,0x00, - 0x10,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x10,0x00,0x15,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, - 0x10,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd4,0x0c,0x53,0x04, - 0x14,0x00,0x12,0x04,0x14,0x00,0x11,0x00,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00, - 0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0xe3,0xb9,0x01,0xd2,0xac,0xd1, - 0x68,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x08,0x00,0x94,0x14,0x53,0x04,0x08,0x00,0x52, - 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x08,0x00,0xcf, - 0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x51, - 0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x52, - 0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0xd3,0x10,0x92, - 0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0a,0x00,0x0a,0x00,0x09,0x00,0x52,0x04,0x0a, - 0x00,0x11,0x04,0x0a,0x00,0x0b,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x55, - 0x04,0x08,0x00,0xd4,0x1c,0x53,0x04,0x08,0x00,0xd2,0x0c,0x51,0x04,0x08,0x00,0x10, - 0x04,0x08,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd3, - 0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0d,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd1,0x6c,0xd0,0x2a,0xcf,0x86,0x55, - 0x04,0x08,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, - 0x04,0x00,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, - 0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x55,0x04,0x08,0x00,0xd4,0x1c,0xd3,0x0c,0x52, - 0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00, - 0x00,0x10,0x04,0x00,0x00,0x08,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10, - 0x04,0x00,0x00,0x0c,0x09,0xd0,0x5a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x93, - 0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x00, - 0x00,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, - 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, - 0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, - 0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xcf, - 0x86,0x95,0x40,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, - 0x04,0x08,0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, - 0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, - 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, - 0x00,0x0a,0xe6,0xd2,0x9c,0xd1,0x68,0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x08, - 0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x0a,0x00,0x11,0x04,0x08,0x00,0x0a,0x00,0x54, - 0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0d, - 0x00,0x0d,0x00,0x12,0x04,0x0d,0x00,0x10,0x00,0xcf,0x86,0x95,0x30,0x94,0x2c,0xd3, - 0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x12,0x00,0x91,0x08,0x10, - 0x04,0x12,0x00,0x13,0x00,0x13,0x00,0xd2,0x08,0x11,0x04,0x13,0x00,0x14,0x00,0x51, - 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf, - 0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x04, - 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04, - 0x00,0x54,0x04,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd1, - 0x06,0xcf,0x06,0x04,0x00,0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0xd5,0x14,0x54, - 0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x00, - 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x04,0x00,0x12,0x04,0x04,0x00,0x00,0x00,0xcf, - 0x86,0xe5,0xa6,0x05,0xe4,0x9f,0x05,0xe3,0x96,0x04,0xe2,0xe4,0x03,0xe1,0xc0,0x01, - 0xd0,0x3e,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c, - 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0xda,0x01,0xe4,0x91,0x08,0x10,0x04,0x01,0xe8, - 0x01,0xde,0x01,0xe0,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x04,0x00,0x10,0x04, - 0x04,0x00,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x04,0x00,0x01,0x00,0xcf,0x86, - 0xd5,0xaa,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, - 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, - 0x8b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x8d,0xe3,0x82, - 0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, - 0x8f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x91,0xe3,0x82, - 0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x93,0xe3,0x82,0x99, - 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x95,0xe3,0x82,0x99,0x00,0x01,0x00, - 0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x97,0xe3,0x82,0x99,0x00,0x01, - 0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x99,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f, - 0x10,0x0b,0x01,0xff,0xe3,0x81,0x9b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01, - 0xff,0xe3,0x81,0x9d,0xe3,0x82,0x99,0x00,0x01,0x00,0xd4,0x53,0xd3,0x3c,0xd2,0x1e, - 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x9f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10, - 0x0b,0x01,0xff,0xe3,0x81,0xa1,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x04, - 0x01,0x00,0x01,0xff,0xe3,0x81,0xa4,0xe3,0x82,0x99,0x00,0x10,0x04,0x01,0x00,0x01, - 0xff,0xe3,0x81,0xa6,0xe3,0x82,0x99,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00, - 0x01,0xff,0xe3,0x81,0xa8,0xe3,0x82,0x99,0x00,0x01,0x00,0x01,0x00,0xd3,0x4a,0xd2, - 0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xaf,0xe3,0x82,0x99,0x00,0x01,0xff, - 0xe3,0x81,0xaf,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x81,0xb2, - 0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb2,0xe3,0x82,0x9a, - 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb5,0xe3,0x82,0x99,0x00,0x01,0xff, - 0xe3,0x81,0xb5,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01, - 0xff,0xe3,0x81,0xb8,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb8,0xe3, - 0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82, - 0x99,0x00,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82,0x9a,0x00,0x01,0x00,0xd0,0xee,0xcf, - 0x86,0xd5,0x42,0x54,0x04,0x01,0x00,0xd3,0x1b,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10, - 0x0b,0x01,0xff,0xe3,0x81,0x86,0xe3,0x82,0x99,0x00,0x06,0x00,0x10,0x04,0x06,0x00, - 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x08,0x10,0x04,0x01,0x08, - 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0x9d,0xe3,0x82,0x99, - 0x00,0x06,0x00,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01, - 0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, - 0x82,0xab,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xad,0xe3, - 0x82,0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, - 0x82,0xaf,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb1,0xe3, - 0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb3,0xe3,0x82, - 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb5,0xe3,0x82,0x99,0x00,0x01, - 0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb7,0xe3,0x82,0x99,0x00, - 0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb9,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1, - 0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbb,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b, - 0x01,0xff,0xe3,0x82,0xbd,0xe3,0x82,0x99,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd5,0xd4, - 0x53,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbf,0xe3,0x82, - 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x81,0xe3,0x82,0x99,0x00,0x01, - 0x00,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x84,0xe3,0x82,0x99,0x00, - 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x86,0xe3,0x82,0x99,0x00,0x92,0x13,0x91, - 0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x88,0xe3,0x82,0x99,0x00,0x01,0x00, - 0x01,0x00,0xd3,0x4a,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x83,0x8f,0xe3, - 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x8f,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00, - 0x01,0xff,0xe3,0x83,0x92,0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, - 0x83,0x92,0xe3,0x82,0x9a,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x95,0xe3, - 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x95,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f, - 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x98,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01, - 0xff,0xe3,0x83,0x98,0xe3,0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff, - 0xe3,0x83,0x9b,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x9b,0xe3,0x82,0x9a,0x00, - 0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x22,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b, - 0x01,0xff,0xe3,0x82,0xa6,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01, - 0xff,0xe3,0x83,0xaf,0xe3,0x82,0x99,0x00,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff, - 0xe3,0x83,0xb0,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0xb1,0xe3,0x82,0x99,0x00, - 0x10,0x0b,0x01,0xff,0xe3,0x83,0xb2,0xe3,0x82,0x99,0x00,0x01,0x00,0x51,0x04,0x01, - 0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0xbd,0xe3,0x82,0x99,0x00,0x06,0x00,0xd1,0x65, - 0xd0,0x46,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x91,0x08, - 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0x53,0x04, - 0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04, - 0x13,0x00,0x14,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, - 0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x93,0x11, - 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01, - 0x00,0x01,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x54, - 0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x08,0x00,0x0a,0x00,0x94, - 0x0c,0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0xd2,0xa4,0xd1, - 0x5c,0xd0,0x22,0xcf,0x86,0x95,0x1c,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52, - 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x10,0x04,0x07,0x00,0x00, - 0x00,0x01,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x93,0x08,0x12,0x04,0x01,0x00,0x0b, - 0x00,0x0b,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x06,0x00,0x06, - 0x00,0x06,0x00,0x06,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01, - 0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, - 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, - 0x00,0x06,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5,0x10,0x94,0x0c,0x53, - 0x04,0x01,0x00,0x12,0x04,0x01,0x00,0x07,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x53, - 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x16, - 0x00,0xd1,0x30,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, - 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x01,0x00,0x01, - 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x01,0x00,0x53, - 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x07,0x00,0x54,0x04,0x01, - 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x07,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd1,0x48,0xd0,0x40,0xcf, - 0x86,0xd5,0x06,0xcf,0x06,0x04,0x00,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x2c,0xd2, - 0x06,0xcf,0x06,0x04,0x00,0xd1,0x06,0xcf,0x06,0x04,0x00,0xd0,0x1a,0xcf,0x86,0x55, - 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x07,0x00,0xcf,0x06,0x01,0x00,0xcf,0x86,0xcf, - 0x06,0x01,0x00,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe2,0x71,0x05,0xd1,0x8c,0xd0,0x08, - 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xd4,0x06, - 0xcf,0x06,0x01,0x00,0xd3,0x06,0xcf,0x06,0x01,0x00,0xd2,0x06,0xcf,0x06,0x01,0x00, - 0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x10, - 0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x08,0x00,0x08,0x00,0x53,0x04, - 0x08,0x00,0x12,0x04,0x08,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0xd3,0x08, - 0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0d,0x00, - 0x11,0x00,0x11,0x00,0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x13,0x00, - 0x13,0x00,0x94,0x14,0x53,0x04,0x13,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04, - 0x13,0x00,0x14,0x00,0x14,0x00,0x00,0x00,0xe0,0xdb,0x04,0xcf,0x86,0xe5,0xdf,0x01, - 0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x74,0xd2,0x6e,0xd1,0x06,0xcf,0x06,0x04,0x00, - 0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00, - 0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c, - 0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00,0x04,0x00,0x93,0x10,0x52,0x04, - 0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86, - 0x95,0x24,0x94,0x20,0x93,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00, - 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x10,0x04,0x04,0x00,0x00,0x00, - 0x00,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x06,0x0a,0x00,0xd2,0x84,0xd1,0x4c,0xd0,0x16, - 0xcf,0x86,0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00, - 0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x1c,0xd3,0x0c,0x92,0x08, - 0x11,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00, - 0x10,0x04,0x0a,0x00,0x0a,0xe6,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0d,0xe6,0x52,0x04, - 0x0d,0xe6,0x11,0x04,0x0a,0xe6,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, - 0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, - 0x11,0xe6,0x0d,0xe6,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00, - 0x93,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x00,0x00,0xd1,0x40, - 0xd0,0x3a,0xcf,0x86,0xd5,0x24,0x54,0x04,0x08,0x00,0xd3,0x10,0x52,0x04,0x08,0x00, - 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0x92,0x0c,0x51,0x04,0x09,0x00, - 0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, - 0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x5e, - 0xcf,0x86,0xd5,0x28,0xd4,0x18,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0xd1,0x08, - 0x10,0x04,0x0a,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x11,0x00,0x93,0x0c,0x92,0x08, - 0x11,0x04,0x0c,0x00,0x0d,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0x53,0x04,0x0c,0x00, - 0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0d,0x00,0x10,0x00,0x51,0x04,0x10,0x00, - 0x10,0x04,0x12,0x00,0x14,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00, - 0x11,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x1c, - 0x94,0x18,0x93,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x51,0x04,0x15,0x00, - 0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x00,0x00,0xd3,0x10, - 0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x92,0x0c, - 0x51,0x04,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0xe4,0xf2,0x02,0xe3, - 0x65,0x01,0xd2,0x98,0xd1,0x48,0xd0,0x36,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10, - 0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x09,0x08,0x00,0x08,0x00, - 0x08,0x00,0xd4,0x0c,0x53,0x04,0x08,0x00,0x12,0x04,0x08,0x00,0x00,0x00,0x53,0x04, - 0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, - 0x09,0x00,0x54,0x04,0x09,0x00,0x13,0x04,0x09,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06, - 0x0a,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08, - 0x10,0x04,0x0a,0x09,0x12,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00, - 0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x00,0x00,0x00,0x00, - 0x54,0x04,0x0b,0xe6,0xd3,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00, - 0x52,0x04,0x0b,0x00,0x11,0x04,0x11,0x00,0x14,0x00,0xd1,0x60,0xd0,0x22,0xcf,0x86, - 0x55,0x04,0x0a,0x00,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00, - 0x10,0x04,0x0a,0x00,0x0a,0xdc,0x11,0x04,0x0a,0xdc,0x0a,0x00,0x0a,0x00,0xcf,0x86, - 0xd5,0x24,0x54,0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, - 0x0a,0x00,0x0a,0x09,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, - 0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00, - 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04, - 0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04, - 0x0b,0x00,0x0b,0x07,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x20,0xd3,0x10, - 0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00,0x52,0x04, - 0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x53,0x04,0x0b,0x00, - 0xd2,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x0b,0x00,0x54,0x04, - 0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, - 0x10,0x00,0x00,0x00,0xd2,0xd0,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0a,0x00, - 0x54,0x04,0x0a,0x00,0x93,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04, - 0x0a,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0a,0x00, - 0x52,0x04,0x0a,0x00,0x11,0x04,0x0a,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x92,0x08, - 0x11,0x04,0x0a,0x00,0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, - 0x12,0x04,0x0b,0x00,0x10,0x00,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, - 0x0b,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0xe6, - 0xd1,0x08,0x10,0x04,0x0b,0xdc,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x11,0x04,0x0b,0x00,0x0b,0xe6, - 0xcf,0x86,0xd5,0x2c,0xd4,0x18,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00, - 0x0b,0xe6,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x00,0x00, - 0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00,0x54,0x04, - 0x0d,0x00,0x93,0x10,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x09, - 0x00,0x00,0x00,0x00,0xd1,0x8c,0xd0,0x72,0xcf,0x86,0xd5,0x4c,0xd4,0x30,0xd3,0x18, - 0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00, - 0x10,0x04,0x0c,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, - 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x93,0x18,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, - 0x0c,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04, - 0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00, - 0x10,0x04,0x0c,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0x94,0x10, - 0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x10,0x00,0x15,0x00,0x00,0x00,0x11,0x00, - 0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0xd4,0x14,0x53,0x04, - 0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x00,0x00, - 0x53,0x04,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x06, - 0x02,0xff,0xff,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd1,0x76,0xd0,0x09,0xcf,0x86, - 0xcf,0x06,0x02,0xff,0xff,0xcf,0x86,0x85,0xd4,0x07,0xcf,0x06,0x02,0xff,0xff,0xd3, - 0x07,0xcf,0x06,0x02,0xff,0xff,0xd2,0x07,0xcf,0x06,0x02,0xff,0xff,0xd1,0x07,0xcf, - 0x06,0x02,0xff,0xff,0xd0,0x18,0xcf,0x86,0x55,0x05,0x02,0xff,0xff,0x94,0x0d,0x93, - 0x09,0x12,0x05,0x02,0xff,0xff,0x00,0x00,0x00,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x24, - 0x94,0x20,0xd3,0x10,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, - 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, - 0x0b,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x12,0x04,0x0b,0x00,0x00,0x00, - 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00, - 0xe4,0x9c,0x10,0xe3,0x16,0x08,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x08,0x04,0xe0, - 0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0x10,0x08,0x01, - 0xff,0xe8,0xbb,0x8a,0x00,0x01,0xff,0xe8,0xb3,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe6,0xbb,0x91,0x00,0x01,0xff,0xe4,0xb8,0xb2,0x00,0x10,0x08,0x01,0xff,0xe5, - 0x8f,0xa5,0x00,0x01,0xff,0xe9,0xbe,0x9c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe9,0xbe,0x9c,0x00,0x01,0xff,0xe5,0xa5,0x91,0x00,0x10,0x08,0x01,0xff,0xe9, - 0x87,0x91,0x00,0x01,0xff,0xe5,0x96,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, - 0xa5,0x88,0x00,0x01,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x01,0xff,0xe7,0x99,0xa9, - 0x00,0x01,0xff,0xe7,0xbe,0x85,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe8,0x98,0xbf,0x00,0x01,0xff,0xe8,0x9e,0xba,0x00,0x10,0x08,0x01,0xff,0xe8, - 0xa3,0xb8,0x00,0x01,0xff,0xe9,0x82,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, - 0xa8,0x82,0x00,0x01,0xff,0xe6,0xb4,0x9b,0x00,0x10,0x08,0x01,0xff,0xe7,0x83,0x99, - 0x00,0x01,0xff,0xe7,0x8f,0x9e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, - 0x90,0xbd,0x00,0x01,0xff,0xe9,0x85,0xaa,0x00,0x10,0x08,0x01,0xff,0xe9,0xa7,0xb1, - 0x00,0x01,0xff,0xe4,0xba,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x8d,0xb5, - 0x00,0x01,0xff,0xe6,0xac,0x84,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x9b,0x00,0x01, - 0xff,0xe8,0x98,0xad,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe9,0xb8,0x9e,0x00,0x01,0xff,0xe5,0xb5,0x90,0x00,0x10,0x08,0x01,0xff,0xe6, - 0xbf,0xab,0x00,0x01,0xff,0xe8,0x97,0x8d,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, - 0xa5,0xa4,0x00,0x01,0xff,0xe6,0x8b,0x89,0x00,0x10,0x08,0x01,0xff,0xe8,0x87,0x98, - 0x00,0x01,0xff,0xe8,0xa0,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, - 0xbb,0x8a,0x00,0x01,0xff,0xe6,0x9c,0x97,0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0xaa, - 0x00,0x01,0xff,0xe7,0x8b,0xbc,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x83,0x8e, - 0x00,0x01,0xff,0xe4,0xbe,0x86,0x00,0x10,0x08,0x01,0xff,0xe5,0x86,0xb7,0x00,0x01, - 0xff,0xe5,0x8b,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, - 0x93,0x84,0x00,0x01,0xff,0xe6,0xab,0x93,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x90, - 0x00,0x01,0xff,0xe7,0x9b,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x80,0x81, - 0x00,0x01,0xff,0xe8,0x98,0x86,0x00,0x10,0x08,0x01,0xff,0xe8,0x99,0x9c,0x00,0x01, - 0xff,0xe8,0xb7,0xaf,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x9c,0xb2, - 0x00,0x01,0xff,0xe9,0xad,0xaf,0x00,0x10,0x08,0x01,0xff,0xe9,0xb7,0xba,0x00,0x01, - 0xff,0xe7,0xa2,0x8c,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa5,0xbf,0x00,0x01, - 0xff,0xe7,0xb6,0xa0,0x00,0x10,0x08,0x01,0xff,0xe8,0x8f,0x89,0x00,0x01,0xff,0xe9, - 0x8c,0x84,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0x10,0x08, - 0x01,0xff,0xe5,0xa3,0x9f,0x00,0x01,0xff,0xe5,0xbc,0x84,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe7,0xb1,0xa0,0x00,0x01,0xff,0xe8,0x81,0xbe,0x00,0x10,0x08,0x01,0xff, - 0xe7,0x89,0xa2,0x00,0x01,0xff,0xe7,0xa3,0x8a,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe8,0xb3,0x82,0x00,0x01,0xff,0xe9,0x9b,0xb7,0x00,0x10,0x08,0x01,0xff, - 0xe5,0xa3,0x98,0x00,0x01,0xff,0xe5,0xb1,0xa2,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe6,0xa8,0x93,0x00,0x01,0xff,0xe6,0xb7,0x9a,0x00,0x10,0x08,0x01,0xff,0xe6,0xbc, - 0x8f,0x00,0x01,0xff,0xe7,0xb4,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9,0x99,0x8b,0x00,0x10,0x08,0x01,0xff, - 0xe5,0x8b,0x92,0x00,0x01,0xff,0xe8,0x82,0x8b,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe5,0x87,0x9c,0x00,0x01,0xff,0xe5,0x87,0x8c,0x00,0x10,0x08,0x01,0xff,0xe7,0xa8, - 0x9c,0x00,0x01,0xff,0xe7,0xb6,0xbe,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe8,0x8f,0xb1,0x00,0x01,0xff,0xe9,0x99,0xb5,0x00,0x10,0x08,0x01,0xff,0xe8,0xae, - 0x80,0x00,0x01,0xff,0xe6,0x8b,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xa8, - 0x82,0x00,0x01,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x01,0xff,0xe4,0xb8,0xb9,0x00, - 0x01,0xff,0xe5,0xaf,0xa7,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe6,0x80,0x92,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0x10,0x08,0x01,0xff, - 0xe7,0x95,0xb0,0x00,0x01,0xff,0xe5,0x8c,0x97,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe7,0xa3,0xbb,0x00,0x01,0xff,0xe4,0xbe,0xbf,0x00,0x10,0x08,0x01,0xff,0xe5,0xbe, - 0xa9,0x00,0x01,0xff,0xe4,0xb8,0x8d,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe6,0xb3,0x8c,0x00,0x01,0xff,0xe6,0x95,0xb8,0x00,0x10,0x08,0x01,0xff,0xe7,0xb4, - 0xa2,0x00,0x01,0xff,0xe5,0x8f,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0xa1, - 0x9e,0x00,0x01,0xff,0xe7,0x9c,0x81,0x00,0x10,0x08,0x01,0xff,0xe8,0x91,0x89,0x00, - 0x01,0xff,0xe8,0xaa,0xaa,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe6,0xae,0xba,0x00,0x01,0xff,0xe8,0xbe,0xb0,0x00,0x10,0x08,0x01,0xff,0xe6,0xb2, - 0x88,0x00,0x01,0xff,0xe6,0x8b,0xbe,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x8b, - 0xa5,0x00,0x01,0xff,0xe6,0x8e,0xa0,0x00,0x10,0x08,0x01,0xff,0xe7,0x95,0xa5,0x00, - 0x01,0xff,0xe4,0xba,0xae,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, - 0xa9,0x00,0x01,0xff,0xe5,0x87,0x89,0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0x81,0x00, - 0x01,0xff,0xe7,0xb3,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x89,0xaf,0x00, - 0x01,0xff,0xe8,0xab,0x92,0x00,0x10,0x08,0x01,0xff,0xe9,0x87,0x8f,0x00,0x01,0xff, - 0xe5,0x8b,0xb5,0x00,0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x91,0x82,0x00,0x01,0xff,0xe5,0xa5, - 0xb3,0x00,0x10,0x08,0x01,0xff,0xe5,0xbb,0xac,0x00,0x01,0xff,0xe6,0x97,0x85,0x00, - 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xbf,0xbe,0x00,0x01,0xff,0xe7,0xa4,0xaa,0x00, - 0x10,0x08,0x01,0xff,0xe9,0x96,0xad,0x00,0x01,0xff,0xe9,0xa9,0xaa,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xba,0x97,0x00,0x01,0xff,0xe9,0xbb,0x8e,0x00, - 0x10,0x08,0x01,0xff,0xe5,0x8a,0x9b,0x00,0x01,0xff,0xe6,0x9b,0x86,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe6,0xad,0xb7,0x00,0x01,0xff,0xe8,0xbd,0xa2,0x00,0x10,0x08, - 0x01,0xff,0xe5,0xb9,0xb4,0x00,0x01,0xff,0xe6,0x86,0x90,0x00,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x88,0x80,0x00,0x01,0xff,0xe6,0x92,0x9a,0x00, - 0x10,0x08,0x01,0xff,0xe6,0xbc,0xa3,0x00,0x01,0xff,0xe7,0x85,0x89,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe7,0x92,0x89,0x00,0x01,0xff,0xe7,0xa7,0x8a,0x00,0x10,0x08, - 0x01,0xff,0xe7,0xb7,0xb4,0x00,0x01,0xff,0xe8,0x81,0xaf,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe8,0xbc,0xa6,0x00,0x01,0xff,0xe8,0x93,0xae,0x00,0x10,0x08, - 0x01,0xff,0xe9,0x80,0xa3,0x00,0x01,0xff,0xe9,0x8d,0x8a,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe5,0x88,0x97,0x00,0x01,0xff,0xe5,0x8a,0xa3,0x00,0x10,0x08,0x01,0xff, - 0xe5,0x92,0xbd,0x00,0x01,0xff,0xe7,0x83,0x88,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0x82,0x00,0x01,0xff,0xe8,0xaa,0xaa,0x00, - 0x10,0x08,0x01,0xff,0xe5,0xbb,0x89,0x00,0x01,0xff,0xe5,0xbf,0xb5,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe6,0x8d,0xbb,0x00,0x01,0xff,0xe6,0xae,0xae,0x00,0x10,0x08, - 0x01,0xff,0xe7,0xb0,0xbe,0x00,0x01,0xff,0xe7,0x8d,0xb5,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe4,0xbb,0xa4,0x00,0x01,0xff,0xe5,0x9b,0xb9,0x00,0x10,0x08, - 0x01,0xff,0xe5,0xaf,0xa7,0x00,0x01,0xff,0xe5,0xb6,0xba,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe6,0x80,0x9c,0x00,0x01,0xff,0xe7,0x8e,0xb2,0x00,0x10,0x08,0x01,0xff, - 0xe7,0x91,0xa9,0x00,0x01,0xff,0xe7,0xbe,0x9a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe8,0x81,0x86,0x00,0x01,0xff,0xe9,0x88,0xb4,0x00,0x10,0x08, - 0x01,0xff,0xe9,0x9b,0xb6,0x00,0x01,0xff,0xe9,0x9d,0x88,0x00,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe9,0xa0,0x98,0x00,0x01,0xff,0xe4,0xbe,0x8b,0x00,0x10,0x08,0x01,0xff, - 0xe7,0xa6,0xae,0x00,0x01,0xff,0xe9,0x86,0xb4,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe9,0x9a,0xb8,0x00,0x01,0xff,0xe6,0x83,0xa1,0x00,0x10,0x08,0x01,0xff, - 0xe4,0xba,0x86,0x00,0x01,0xff,0xe5,0x83,0x9a,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe5,0xaf,0xae,0x00,0x01,0xff,0xe5,0xb0,0xbf,0x00,0x10,0x08,0x01,0xff,0xe6,0x96, - 0x99,0x00,0x01,0xff,0xe6,0xa8,0x82,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0x87,0x8e,0x00,0x01,0xff,0xe7, - 0x99,0x82,0x00,0x10,0x08,0x01,0xff,0xe8,0x93,0xbc,0x00,0x01,0xff,0xe9,0x81,0xbc, - 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xbe,0x8d,0x00,0x01,0xff,0xe6,0x9a,0x88, - 0x00,0x10,0x08,0x01,0xff,0xe9,0x98,0xae,0x00,0x01,0xff,0xe5,0x8a,0x89,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x9d,0xbb,0x00,0x01,0xff,0xe6,0x9f,0xb3, - 0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0x81,0x00,0x01,0xff,0xe6,0xba,0x9c,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x89,0x00,0x01,0xff,0xe7,0x95,0x99,0x00,0x10, - 0x08,0x01,0xff,0xe7,0xa1,0xab,0x00,0x01,0xff,0xe7,0xb4,0x90,0x00,0xd3,0x40,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xa1,0x9e,0x00,0x01,0xff,0xe5,0x85,0xad, - 0x00,0x10,0x08,0x01,0xff,0xe6,0x88,0xae,0x00,0x01,0xff,0xe9,0x99,0xb8,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe5,0x80,0xab,0x00,0x01,0xff,0xe5,0xb4,0x99,0x00,0x10, - 0x08,0x01,0xff,0xe6,0xb7,0xaa,0x00,0x01,0xff,0xe8,0xbc,0xaa,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe5,0xbe,0x8b,0x00,0x01,0xff,0xe6,0x85,0x84,0x00,0x10, - 0x08,0x01,0xff,0xe6,0xa0,0x97,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe9,0x9a,0x86,0x00,0x01,0xff,0xe5,0x88,0xa9,0x00,0x10,0x08,0x01, - 0xff,0xe5,0x90,0x8f,0x00,0x01,0xff,0xe5,0xb1,0xa5,0x00,0xd4,0x80,0xd3,0x40,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x98,0x93,0x00,0x01,0xff,0xe6,0x9d,0x8e, - 0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0xa8,0x00,0x01,0xff,0xe6,0xb3,0xa5,0x00,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x86,0x00,0x01,0xff,0xe7,0x97,0xa2,0x00,0x10, - 0x08,0x01,0xff,0xe7,0xbd,0xb9,0x00,0x01,0xff,0xe8,0xa3,0x8f,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0xa1,0x00,0x01,0xff,0xe9,0x87,0x8c,0x00,0x10, - 0x08,0x01,0xff,0xe9,0x9b,0xa2,0x00,0x01,0xff,0xe5,0x8c,0xbf,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe6,0xba,0xba,0x00,0x01,0xff,0xe5,0x90,0x9d,0x00,0x10,0x08,0x01, - 0xff,0xe7,0x87,0x90,0x00,0x01,0xff,0xe7,0x92,0x98,0x00,0xd3,0x40,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x01,0xff,0xe8,0x97,0xba,0x00,0x01,0xff,0xe9,0x9a,0xa3,0x00,0x10, - 0x08,0x01,0xff,0xe9,0xb1,0x97,0x00,0x01,0xff,0xe9,0xba,0x9f,0x00,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe6,0x9e,0x97,0x00,0x01,0xff,0xe6,0xb7,0x8b,0x00,0x10,0x08,0x01, - 0xff,0xe8,0x87,0xa8,0x00,0x01,0xff,0xe7,0xab,0x8b,0x00,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x01,0xff,0xe7,0xac,0xa0,0x00,0x01,0xff,0xe7,0xb2,0x92,0x00,0x10,0x08,0x01, - 0xff,0xe7,0x8b,0x80,0x00,0x01,0xff,0xe7,0x82,0x99,0x00,0xd1,0x10,0x10,0x08,0x01, - 0xff,0xe8,0xad,0x98,0x00,0x01,0xff,0xe4,0xbb,0x80,0x00,0x10,0x08,0x01,0xff,0xe8, - 0x8c,0xb6,0x00,0x01,0xff,0xe5,0x88,0xba,0x00,0xe2,0xad,0x06,0xe1,0xc4,0x03,0xe0, - 0xcb,0x01,0xcf,0x86,0xd5,0xe4,0xd4,0x74,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x01,0xff,0xe5,0x88,0x87,0x00,0x01,0xff,0xe5,0xba,0xa6,0x00,0x10,0x08,0x01,0xff, - 0xe6,0x8b,0x93,0x00,0x01,0xff,0xe7,0xb3,0x96,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe5,0xae,0x85,0x00,0x01,0xff,0xe6,0xb4,0x9e,0x00,0x10,0x08,0x01,0xff,0xe6,0x9a, - 0xb4,0x00,0x01,0xff,0xe8,0xbc,0xbb,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, - 0xe8,0xa1,0x8c,0x00,0x01,0xff,0xe9,0x99,0x8d,0x00,0x10,0x08,0x01,0xff,0xe8,0xa6, - 0x8b,0x00,0x01,0xff,0xe5,0xbb,0x93,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, - 0x80,0x00,0x01,0xff,0xe5,0x97,0x80,0x00,0x01,0x00,0xd3,0x34,0xd2,0x18,0xd1,0x0c, - 0x10,0x08,0x01,0xff,0xe5,0xa1,0x9a,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe6,0x99, - 0xb4,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe5,0x87,0x9e,0x00, - 0x10,0x08,0x01,0xff,0xe7,0x8c,0xaa,0x00,0x01,0xff,0xe7,0x9b,0x8a,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa4,0xbc,0x00,0x01,0xff,0xe7,0xa5,0x9e,0x00, - 0x10,0x08,0x01,0xff,0xe7,0xa5,0xa5,0x00,0x01,0xff,0xe7,0xa6,0x8f,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe9,0x9d,0x96,0x00,0x01,0xff,0xe7,0xb2,0xbe,0x00,0x10,0x08, - 0x01,0xff,0xe7,0xbe,0xbd,0x00,0x01,0x00,0xd4,0x64,0xd3,0x30,0xd2,0x18,0xd1,0x0c, - 0x10,0x08,0x01,0xff,0xe8,0x98,0x92,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe8,0xab, - 0xb8,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe9,0x80,0xb8,0x00, - 0x10,0x08,0x01,0xff,0xe9,0x83,0xbd,0x00,0x01,0x00,0xd2,0x14,0x51,0x04,0x01,0x00, - 0x10,0x08,0x01,0xff,0xe9,0xa3,0xaf,0x00,0x01,0xff,0xe9,0xa3,0xbc,0x00,0xd1,0x10, - 0x10,0x08,0x01,0xff,0xe9,0xa4,0xa8,0x00,0x01,0xff,0xe9,0xb6,0xb4,0x00,0x10,0x08, - 0x0d,0xff,0xe9,0x83,0x9e,0x00,0x0d,0xff,0xe9,0x9a,0xb7,0x00,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x06,0xff,0xe4,0xbe,0xae,0x00,0x06,0xff,0xe5,0x83,0xa7,0x00, - 0x10,0x08,0x06,0xff,0xe5,0x85,0x8d,0x00,0x06,0xff,0xe5,0x8b,0x89,0x00,0xd1,0x10, - 0x10,0x08,0x06,0xff,0xe5,0x8b,0xa4,0x00,0x06,0xff,0xe5,0x8d,0x91,0x00,0x10,0x08, - 0x06,0xff,0xe5,0x96,0x9d,0x00,0x06,0xff,0xe5,0x98,0x86,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x06,0xff,0xe5,0x99,0xa8,0x00,0x06,0xff,0xe5,0xa1,0x80,0x00,0x10,0x08, - 0x06,0xff,0xe5,0xa2,0xa8,0x00,0x06,0xff,0xe5,0xb1,0xa4,0x00,0xd1,0x10,0x10,0x08, - 0x06,0xff,0xe5,0xb1,0xae,0x00,0x06,0xff,0xe6,0x82,0x94,0x00,0x10,0x08,0x06,0xff, - 0xe6,0x85,0xa8,0x00,0x06,0xff,0xe6,0x86,0x8e,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4, - 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0x87,0xb2,0x00,0x06, - 0xff,0xe6,0x95,0x8f,0x00,0x10,0x08,0x06,0xff,0xe6,0x97,0xa2,0x00,0x06,0xff,0xe6, - 0x9a,0x91,0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0xa2,0x85,0x00,0x06,0xff,0xe6, - 0xb5,0xb7,0x00,0x10,0x08,0x06,0xff,0xe6,0xb8,0x9a,0x00,0x06,0xff,0xe6,0xbc,0xa2, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0x85,0xae,0x00,0x06,0xff,0xe7, - 0x88,0xab,0x00,0x10,0x08,0x06,0xff,0xe7,0x90,0xa2,0x00,0x06,0xff,0xe7,0xa2,0x91, - 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa4,0xbe,0x00,0x06,0xff,0xe7,0xa5,0x89, - 0x00,0x10,0x08,0x06,0xff,0xe7,0xa5,0x88,0x00,0x06,0xff,0xe7,0xa5,0x90,0x00,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa5,0x96,0x00,0x06,0xff,0xe7, - 0xa5,0x9d,0x00,0x10,0x08,0x06,0xff,0xe7,0xa6,0x8d,0x00,0x06,0xff,0xe7,0xa6,0x8e, - 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa9,0x80,0x00,0x06,0xff,0xe7,0xaa,0x81, - 0x00,0x10,0x08,0x06,0xff,0xe7,0xaf,0x80,0x00,0x06,0xff,0xe7,0xb7,0xb4,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xb8,0x89,0x00,0x06,0xff,0xe7,0xb9,0x81, - 0x00,0x10,0x08,0x06,0xff,0xe7,0xbd,0xb2,0x00,0x06,0xff,0xe8,0x80,0x85,0x00,0xd1, - 0x10,0x10,0x08,0x06,0xff,0xe8,0x87,0xad,0x00,0x06,0xff,0xe8,0x89,0xb9,0x00,0x10, - 0x08,0x06,0xff,0xe8,0x89,0xb9,0x00,0x06,0xff,0xe8,0x91,0x97,0x00,0xd4,0x75,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xa4,0x90,0x00,0x06,0xff,0xe8, - 0xa6,0x96,0x00,0x10,0x08,0x06,0xff,0xe8,0xac,0x81,0x00,0x06,0xff,0xe8,0xac,0xb9, - 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xb3,0x93,0x00,0x06,0xff,0xe8,0xb4,0x88, - 0x00,0x10,0x08,0x06,0xff,0xe8,0xbe,0xb6,0x00,0x06,0xff,0xe9,0x80,0xb8,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe9,0x9b,0xa3,0x00,0x06,0xff,0xe9,0x9f,0xbf, - 0x00,0x10,0x08,0x06,0xff,0xe9,0xa0,0xbb,0x00,0x0b,0xff,0xe6,0x81,0xb5,0x00,0x91, - 0x11,0x10,0x09,0x0b,0xff,0xf0,0xa4,0x8b,0xae,0x00,0x0b,0xff,0xe8,0x88,0x98,0x00, - 0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe4,0xb8,0xa6,0x00, - 0x08,0xff,0xe5,0x86,0xb5,0x00,0x10,0x08,0x08,0xff,0xe5,0x85,0xa8,0x00,0x08,0xff, - 0xe4,0xbe,0x80,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x85,0x85,0x00,0x08,0xff, - 0xe5,0x86,0x80,0x00,0x10,0x08,0x08,0xff,0xe5,0x8b,0x87,0x00,0x08,0xff,0xe5,0x8b, - 0xba,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x96,0x9d,0x00,0x08,0xff, - 0xe5,0x95,0x95,0x00,0x10,0x08,0x08,0xff,0xe5,0x96,0x99,0x00,0x08,0xff,0xe5,0x97, - 0xa2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0xa1,0x9a,0x00,0x08,0xff,0xe5,0xa2, - 0xb3,0x00,0x10,0x08,0x08,0xff,0xe5,0xa5,0x84,0x00,0x08,0xff,0xe5,0xa5,0x94,0x00, - 0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x08,0xff,0xe5,0xa9,0xa2,0x00,0x08,0xff,0xe5,0xac,0xa8,0x00,0x10,0x08, - 0x08,0xff,0xe5,0xbb,0x92,0x00,0x08,0xff,0xe5,0xbb,0x99,0x00,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe5,0xbd,0xa9,0x00,0x08,0xff,0xe5,0xbe,0xad,0x00,0x10,0x08,0x08,0xff, - 0xe6,0x83,0x98,0x00,0x08,0xff,0xe6,0x85,0x8e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe6,0x84,0x88,0x00,0x08,0xff,0xe6,0x86,0x8e,0x00,0x10,0x08,0x08,0xff, - 0xe6,0x85,0xa0,0x00,0x08,0xff,0xe6,0x87,0xb2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe6,0x88,0xb4,0x00,0x08,0xff,0xe6,0x8f,0x84,0x00,0x10,0x08,0x08,0xff,0xe6,0x90, - 0x9c,0x00,0x08,0xff,0xe6,0x91,0x92,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe6,0x95,0x96,0x00,0x08,0xff,0xe6,0x99,0xb4,0x00,0x10,0x08,0x08,0xff, - 0xe6,0x9c,0x97,0x00,0x08,0xff,0xe6,0x9c,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe6,0x9d,0x96,0x00,0x08,0xff,0xe6,0xad,0xb9,0x00,0x10,0x08,0x08,0xff,0xe6,0xae, - 0xba,0x00,0x08,0xff,0xe6,0xb5,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe6,0xbb,0x9b,0x00,0x08,0xff,0xe6,0xbb,0x8b,0x00,0x10,0x08,0x08,0xff,0xe6,0xbc, - 0xa2,0x00,0x08,0xff,0xe7,0x80,0x9e,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0x85, - 0xae,0x00,0x08,0xff,0xe7,0x9e,0xa7,0x00,0x10,0x08,0x08,0xff,0xe7,0x88,0xb5,0x00, - 0x08,0xff,0xe7,0x8a,0xaf,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe7,0x8c,0xaa,0x00,0x08,0xff,0xe7,0x91,0xb1,0x00,0x10,0x08,0x08,0xff, - 0xe7,0x94,0x86,0x00,0x08,0xff,0xe7,0x94,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe7,0x98,0x9d,0x00,0x08,0xff,0xe7,0x98,0x9f,0x00,0x10,0x08,0x08,0xff,0xe7,0x9b, - 0x8a,0x00,0x08,0xff,0xe7,0x9b,0x9b,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe7,0x9b,0xb4,0x00,0x08,0xff,0xe7,0x9d,0x8a,0x00,0x10,0x08,0x08,0xff,0xe7,0x9d, - 0x80,0x00,0x08,0xff,0xe7,0xa3,0x8c,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0xaa, - 0xb1,0x00,0x08,0xff,0xe7,0xaf,0x80,0x00,0x10,0x08,0x08,0xff,0xe7,0xb1,0xbb,0x00, - 0x08,0xff,0xe7,0xb5,0x9b,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe7,0xb7,0xb4,0x00,0x08,0xff,0xe7,0xbc,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0x80, - 0x85,0x00,0x08,0xff,0xe8,0x8d,0x92,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0x8f, - 0xaf,0x00,0x08,0xff,0xe8,0x9d,0xb9,0x00,0x10,0x08,0x08,0xff,0xe8,0xa5,0x81,0x00, - 0x08,0xff,0xe8,0xa6,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xa6, - 0x96,0x00,0x08,0xff,0xe8,0xaa,0xbf,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xb8,0x00, - 0x08,0xff,0xe8,0xab,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xac,0x81,0x00, - 0x08,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xad,0x00,0x08,0xff, - 0xe8,0xac,0xb9,0x00,0xcf,0x86,0x95,0xde,0xd4,0x81,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x08,0xff,0xe8,0xae,0x8a,0x00,0x08,0xff,0xe8,0xb4,0x88,0x00,0x10,0x08, - 0x08,0xff,0xe8,0xbc,0xb8,0x00,0x08,0xff,0xe9,0x81,0xb2,0x00,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe9,0x86,0x99,0x00,0x08,0xff,0xe9,0x89,0xb6,0x00,0x10,0x08,0x08,0xff, - 0xe9,0x99,0xbc,0x00,0x08,0xff,0xe9,0x9b,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x08,0xff,0xe9,0x9d,0x96,0x00,0x08,0xff,0xe9,0x9f,0x9b,0x00,0x10,0x08,0x08,0xff, - 0xe9,0x9f,0xbf,0x00,0x08,0xff,0xe9,0xa0,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, - 0xe9,0xa0,0xbb,0x00,0x08,0xff,0xe9,0xac,0x92,0x00,0x10,0x08,0x08,0xff,0xe9,0xbe, - 0x9c,0x00,0x08,0xff,0xf0,0xa2,0xa1,0x8a,0x00,0xd3,0x45,0xd2,0x22,0xd1,0x12,0x10, - 0x09,0x08,0xff,0xf0,0xa2,0xa1,0x84,0x00,0x08,0xff,0xf0,0xa3,0x8f,0x95,0x00,0x10, - 0x08,0x08,0xff,0xe3,0xae,0x9d,0x00,0x08,0xff,0xe4,0x80,0x98,0x00,0xd1,0x11,0x10, - 0x08,0x08,0xff,0xe4,0x80,0xb9,0x00,0x08,0xff,0xf0,0xa5,0x89,0x89,0x00,0x10,0x09, - 0x08,0xff,0xf0,0xa5,0xb3,0x90,0x00,0x08,0xff,0xf0,0xa7,0xbb,0x93,0x00,0x92,0x14, - 0x91,0x10,0x10,0x08,0x08,0xff,0xe9,0xbd,0x83,0x00,0x08,0xff,0xe9,0xbe,0x8e,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xe1,0x94,0x01,0xe0,0x08,0x01,0xcf,0x86,0xd5,0x42, - 0xd4,0x14,0x93,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, - 0x00,0x00,0x00,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, - 0x01,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x00,0x00,0x04,0xff, - 0xd7,0x99,0xd6,0xb4,0x00,0x10,0x04,0x01,0x1a,0x01,0xff,0xd7,0xb2,0xd6,0xb7,0x00, - 0xd4,0x42,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, - 0xd7,0xa9,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd7,0x82,0x00,0xd1,0x16,0x10,0x0b, - 0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7, - 0x82,0x00,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xb7,0x00,0x01,0xff,0xd7,0x90,0xd6, - 0xb8,0x00,0xd3,0x43,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xbc, - 0x00,0x01,0xff,0xd7,0x91,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x92,0xd6,0xbc, - 0x00,0x01,0xff,0xd7,0x93,0xd6,0xbc,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x94, - 0xd6,0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x96, - 0xd6,0xbc,0x00,0x00,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x98,0xd6, - 0xbc,0x00,0x01,0xff,0xd7,0x99,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x9a,0xd6, - 0xbc,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7, - 0x9c,0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0x9e,0xd6,0xbc,0x00,0x00, - 0x00,0xcf,0x86,0x95,0x85,0x94,0x81,0xd3,0x3e,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01, - 0xff,0xd7,0xa0,0xd6,0xbc,0x00,0x01,0xff,0xd7,0xa1,0xd6,0xbc,0x00,0x10,0x04,0x00, - 0x00,0x01,0xff,0xd7,0xa3,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7,0xa4, - 0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0xa6,0xd6,0xbc,0x00,0x01,0xff, - 0xd7,0xa7,0xd6,0xbc,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0xa8,0xd6, - 0xbc,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0xaa,0xd6, - 0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xb9,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7, - 0x91,0xd6,0xbf,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbf,0x00,0x10,0x09,0x01,0xff,0xd7, - 0xa4,0xd6,0xbf,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, - 0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00, - 0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0xd4,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, - 0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00, - 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd3,0x5a,0xd2,0x06, - 0xcf,0x06,0x01,0x00,0xd1,0x14,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x95,0x08, - 0x14,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04, - 0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x01,0x00,0x00,0x00,0x05,0x00, - 0x54,0x04,0x05,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04, - 0x06,0x00,0x07,0x00,0x00,0x00,0xd2,0xce,0xd1,0xa5,0xd0,0x37,0xcf,0x86,0xd5,0x15, - 0x54,0x05,0x06,0xff,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00,0x00, - 0x00,0x00,0x00,0x94,0x1c,0xd3,0x10,0x52,0x04,0x01,0xe6,0x51,0x04,0x0a,0xe6,0x10, - 0x04,0x0a,0xe6,0x10,0xdc,0x52,0x04,0x10,0xdc,0x11,0x04,0x10,0xdc,0x11,0xe6,0x01, - 0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10, - 0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00,0x07,0x00,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01, - 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0xd3,0x10,0x52, - 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x12,0x04,0x01, - 0x00,0x00,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06, - 0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x06,0xcf, - 0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01, - 0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, - 0x00,0x01,0xff,0x00,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, - 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00, - 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x94,0x14, - 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01,0x00,0x01,0x00,0x01,0x00, - 0x01,0x00,0x01,0x00,0xd0,0x2f,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x15,0x93,0x11, - 0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, - 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, - 0x00,0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x18,0xd3,0x0c,0x92,0x08,0x11,0x04,0x00, - 0x00,0x01,0x00,0x01,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, - 0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x00, - 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd4,0x20,0xd3, - 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52, - 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x53,0x05,0x00, - 0xff,0x00,0xd2,0x0d,0x91,0x09,0x10,0x05,0x00,0xff,0x00,0x04,0x00,0x04,0x00,0x91, - 0x08,0x10,0x04,0x03,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x46,0x3e,0xe1,0x1f,0x3b, - 0xe0,0x9c,0x39,0xcf,0x86,0xe5,0x40,0x26,0xc4,0xe3,0x16,0x14,0xe2,0xef,0x11,0xe1, - 0xd0,0x10,0xe0,0x60,0x07,0xcf,0x86,0xe5,0x53,0x03,0xe4,0x4c,0x02,0xe3,0x3d,0x01, - 0xd2,0x94,0xd1,0x70,0xd0,0x4a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x07,0x00, - 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00, - 0xd4,0x14,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, - 0x00,0x00,0x07,0x00,0x53,0x04,0x07,0x00,0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04, - 0x07,0x00,0x00,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86, - 0x95,0x20,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00, - 0x00,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00, - 0x00,0x00,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x54,0x04, - 0x07,0x00,0x53,0x04,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, - 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x94,0x1c,0x93,0x18, - 0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00,0x00,0x51,0x04,0x00,0x00, - 0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0x93,0x10, - 0x52,0x04,0x07,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00, - 0xcf,0x06,0x08,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x20,0x53,0x04,0x08,0x00, - 0xd2,0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x10,0x00,0xd1,0x08,0x10,0x04, - 0x10,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x12,0x04, - 0x0a,0x00,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04, - 0x00,0x00,0x0a,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00, - 0x91,0x08,0x10,0x04,0x0a,0x00,0x0a,0xdc,0x00,0x00,0xd2,0x5e,0xd1,0x06,0xcf,0x06, - 0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00, - 0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x0a,0x00, - 0xcf,0x86,0xd5,0x18,0x54,0x04,0x0a,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x10,0xdc,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04, - 0x10,0x00,0x12,0x04,0x10,0x00,0x00,0x00,0xd1,0x70,0xd0,0x36,0xcf,0x86,0xd5,0x18, - 0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00, - 0x10,0x04,0x05,0x00,0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x05,0x00,0x00,0x00, - 0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x05,0x00, - 0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x05,0x00,0x92,0x0c,0x51,0x04,0x05,0x00, - 0x10,0x04,0x05,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c, - 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x10,0xe6,0x92,0x0c,0x51,0x04,0x10,0xe6, - 0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, - 0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, - 0x00,0x00,0x07,0x00,0x08,0x00,0xcf,0x86,0x95,0x1c,0xd4,0x0c,0x93,0x08,0x12,0x04, - 0x08,0x00,0x00,0x00,0x08,0x00,0x93,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0xba,0xd2,0x80,0xd1,0x34,0xd0,0x1a,0xcf,0x86, - 0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00, - 0x07,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x05,0x00, - 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd0,0x2a, - 0xcf,0x86,0xd5,0x14,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, - 0x11,0x04,0x07,0x00,0x00,0x00,0x94,0x10,0x53,0x04,0x07,0x00,0x92,0x08,0x11,0x04, - 0x07,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xcf,0x86,0xd5,0x10,0x54,0x04,0x12,0x00, - 0x93,0x08,0x12,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x54,0x04,0x12,0x00,0x53,0x04, - 0x12,0x00,0x12,0x04,0x12,0x00,0x00,0x00,0xd1,0x34,0xd0,0x12,0xcf,0x86,0x55,0x04, - 0x10,0x00,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, - 0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x10,0x00,0x00,0x00,0x52,0x04,0x00,0x00, - 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, - 0xd2,0x06,0xcf,0x06,0x10,0x00,0xd1,0x40,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00, - 0x54,0x04,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, - 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c, - 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x08,0x13,0x04, - 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe4,0xce,0x02,0xe3,0x45,0x01, - 0xd2,0xd0,0xd1,0x70,0xd0,0x52,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x52,0x04, - 0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, - 0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0xd3,0x10,0x52,0x04, - 0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xd2,0x0c,0x91,0x08, - 0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x00,0x00, - 0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x0b,0x00,0x93,0x10, - 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, - 0x10,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, - 0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14, - 0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00, - 0x10,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x11,0x00,0xd3,0x14, - 0xd2,0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x04,0x11,0x00, - 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00, - 0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x1c,0x54,0x04,0x09,0x00,0x53,0x04,0x09,0x00, - 0xd2,0x08,0x11,0x04,0x09,0x00,0x0b,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, - 0x09,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, - 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0xcf,0x06,0x00,0x00, - 0xd0,0x1a,0xcf,0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d,0x00, - 0x52,0x04,0x00,0x00,0x11,0x04,0x11,0x00,0x0d,0x00,0xcf,0x86,0x95,0x14,0x54,0x04, - 0x11,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00, - 0x11,0x00,0xd2,0xec,0xd1,0xa4,0xd0,0x76,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x14, - 0x52,0x04,0x08,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x10,0x04,0x08,0x00, - 0x00,0x00,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x08,0x00,0x08,0xdc,0x10,0x04, - 0x08,0x00,0x08,0xe6,0xd3,0x10,0x52,0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x00,0x00, - 0x08,0x00,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x08,0x00, - 0x08,0x00,0x54,0x04,0x08,0x00,0xd3,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x14,0x00, - 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe6,0x08,0x01,0x10,0x04,0x08,0xdc, - 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x08,0x09,0xcf,0x86,0x95,0x28, - 0xd4,0x14,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x08,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x10,0x00, - 0x00,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x24,0xd3,0x14,0x52,0x04,0x10,0x00, - 0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0xe6,0x10,0x04,0x10,0xdc,0x00,0x00,0x92,0x0c, - 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04, - 0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0x54, - 0xd0,0x26,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04, - 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, - 0x0b,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x0b,0x00,0x93,0x0c, - 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00, - 0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00, - 0x0b,0x00,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x54,0x04,0x10,0x00,0xd3,0x0c,0x92,0x08, - 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, - 0x10,0x00,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14, - 0x53,0x04,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00, - 0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x96,0xd2,0x68,0xd1,0x24,0xd0,0x06, - 0xcf,0x06,0x0b,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0x93,0x10,0x92,0x0c, - 0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, - 0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x11,0x00, - 0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00, - 0x11,0x00,0xd1,0x28,0xd0,0x22,0xcf,0x86,0x55,0x04,0x14,0x00,0xd4,0x0c,0x93,0x08, - 0x12,0x04,0x14,0x00,0x14,0xe6,0x00,0x00,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04, - 0x14,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a, - 0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, - 0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, - 0x0b,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x58,0xd0,0x12,0xcf,0x86,0x55,0x04, - 0x14,0x00,0x94,0x08,0x13,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0xcf,0x86,0x95,0x40, - 0xd4,0x24,0xd3,0x0c,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x14,0xdc,0xd2,0x0c, - 0x51,0x04,0x14,0xe6,0x10,0x04,0x14,0xe6,0x14,0xdc,0x91,0x08,0x10,0x04,0x14,0xe6, - 0x14,0xdc,0x14,0xdc,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0xdc,0x14,0x00, - 0x14,0x00,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x15,0x00, - 0x93,0x10,0x52,0x04,0x15,0x00,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00, - 0x00,0x00,0xcf,0x86,0xe5,0x0f,0x06,0xe4,0xf8,0x03,0xe3,0x02,0x02,0xd2,0xfb,0xd1, - 0x4c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52, - 0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x09,0x0c,0x00,0x52,0x04,0x0c, - 0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x0c, - 0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00, - 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x09,0xd0,0x69,0xcf,0x86,0xd5, - 0x32,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0xd2,0x15,0x51,0x04,0x0b,0x00,0x10, - 0x0d,0x0b,0xff,0xf0,0x91,0x82,0x99,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x91,0x11, - 0x10,0x0d,0x0b,0xff,0xf0,0x91,0x82,0x9b,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x0b, - 0x00,0xd4,0x1d,0x53,0x04,0x0b,0x00,0x92,0x15,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, - 0x00,0x0b,0xff,0xf0,0x91,0x82,0xa5,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x53,0x04, - 0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x10,0x04,0x0b,0x07, - 0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x92,0x08,0x11,0x04, - 0x0b,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00, - 0x14,0x00,0x00,0x00,0x0d,0x00,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x08, - 0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0xd1,0x96,0xd0,0x5c,0xcf,0x86,0xd5,0x18, - 0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x0d,0x00, - 0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x26,0x53,0x04,0x0d,0x00,0x52,0x04,0x0d,0x00, - 0x51,0x04,0x0d,0x00,0x10,0x0d,0x0d,0xff,0xf0,0x91,0x84,0xb1,0xf0,0x91,0x84,0xa7, - 0x00,0x0d,0xff,0xf0,0x91,0x84,0xb2,0xf0,0x91,0x84,0xa7,0x00,0x93,0x18,0xd2,0x0c, - 0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x0d,0x09,0x91,0x08,0x10,0x04,0x0d,0x09, - 0x00,0x00,0x0d,0x00,0x0d,0x00,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04, - 0x0d,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x10,0x00, - 0x54,0x04,0x10,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00, - 0x10,0x07,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x06, - 0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x91,0x08, - 0x10,0x04,0x0d,0x09,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, - 0x0d,0x00,0x11,0x00,0x10,0x04,0x11,0x07,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00, - 0x10,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04, - 0x10,0x00,0x11,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00, - 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xc8,0xd1,0x48, - 0xd0,0x42,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, - 0x10,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x54,0x04,0x10,0x00, - 0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x09,0x10,0x04, - 0x10,0x07,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00, - 0x00,0x00,0xcf,0x06,0x00,0x00,0xd0,0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10, - 0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04, - 0x00,0x00,0x11,0x00,0x53,0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00, - 0x10,0x04,0x00,0x00,0x11,0x00,0x94,0x10,0x53,0x04,0x11,0x00,0x92,0x08,0x11,0x04, - 0x11,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x18, - 0x53,0x04,0x10,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x07,0x10,0x04, - 0x10,0x09,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00, - 0x00,0x00,0x00,0x00,0xe1,0x27,0x01,0xd0,0x8a,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3, - 0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x10,0x00,0x10,0x00,0x91,0x08,0x10, - 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10, - 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10, - 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xd4, - 0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, - 0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, - 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10, - 0x00,0x10,0x04,0x00,0x00,0x14,0x07,0x91,0x08,0x10,0x04,0x10,0x07,0x10,0x00,0x10, - 0x00,0xcf,0x86,0xd5,0x6a,0xd4,0x42,0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10, - 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0xd2,0x19,0xd1,0x08,0x10, - 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0, - 0x91,0x8c,0xbe,0x00,0x91,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0,0x91, - 0x8d,0x97,0x00,0x10,0x09,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11, - 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x52, - 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0xd3, - 0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0xe6,0x52,0x04,0x10,0xe6,0x91, - 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x10,0xe6,0x91, - 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3, - 0x30,0x01,0xd2,0xb7,0xd1,0x48,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95,0x3c, - 0xd4,0x1c,0x93,0x18,0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x09,0x12,0x00, - 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x07,0x12,0x00,0x12,0x00,0x53,0x04,0x12,0x00, - 0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x00,0x00,0x12,0x00,0xd1,0x08,0x10,0x04, - 0x00,0x00,0x12,0x00,0x10,0x04,0x14,0xe6,0x15,0x00,0x00,0x00,0xd0,0x45,0xcf,0x86, - 0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0xd2,0x15,0x51,0x04, - 0x10,0x00,0x10,0x04,0x10,0x00,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xba, - 0x00,0xd1,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xb0,0x00, - 0x10,0x00,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xbd,0x00,0x10, - 0x00,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, - 0x04,0x10,0x09,0x10,0x07,0x10,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, - 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0, - 0x40,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c,0x52,0x04,0x10, - 0x00,0x11,0x04,0x10,0x00,0x00,0x00,0xd2,0x1e,0x51,0x04,0x10,0x00,0x10,0x0d,0x10, - 0xff,0xf0,0x91,0x96,0xb8,0xf0,0x91,0x96,0xaf,0x00,0x10,0xff,0xf0,0x91,0x96,0xb9, - 0xf0,0x91,0x96,0xaf,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10,0x09,0xcf, - 0x86,0x95,0x2c,0xd4,0x1c,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x07,0x10, - 0x00,0x10,0x00,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x53, - 0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0xd2, - 0xa0,0xd1,0x5c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53, - 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10, - 0x09,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10, - 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, - 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, - 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x2a,0xcf, - 0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0xd3,0x10,0x52,0x04,0x0d,0x00,0x51, - 0x04,0x0d,0x00,0x10,0x04,0x0d,0x09,0x0d,0x07,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x0d, - 0x00,0x92,0x08,0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, - 0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x54,0x04,0x11,0x00,0x53,0x04,0x11,0x00,0xd2, - 0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00, - 0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00,0x92,0x0c,0x51,0x04,0x11, - 0x00,0x10,0x04,0x11,0x00,0x11,0x09,0x00,0x00,0x11,0x00,0xcf,0x06,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xe4,0x59,0x01,0xd3,0xb2,0xd2,0x5c,0xd1,0x28,0xd0,0x22,0xcf,0x86, - 0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92,0x10,0xd1,0x08, - 0x10,0x04,0x14,0x00,0x14,0x09,0x10,0x04,0x14,0x07,0x14,0x00,0x00,0x00,0xcf,0x06, - 0x00,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, - 0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, - 0x10,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, - 0x00,0x00,0x10,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, - 0x00,0x00,0x94,0x10,0x53,0x04,0x15,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00, - 0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x15,0x00,0x53,0x04,0x15,0x00, - 0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x94,0x1c,0x93,0x18,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x15,0x09,0x15,0x00,0x15,0x00,0x91,0x08,0x10,0x04,0x15,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xa0,0xd1,0x3c,0xd0,0x1e,0xcf,0x86, - 0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x93,0x10,0x52,0x04,0x13,0x00,0x91,0x08, - 0x10,0x04,0x13,0x09,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x86,0x95,0x18,0x94,0x14, - 0x93,0x10,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x13,0x09, - 0x00,0x00,0x13,0x00,0x13,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x10,0x93,0x0c, - 0x52,0x04,0x13,0x00,0x11,0x04,0x15,0x00,0x13,0x00,0x13,0x00,0x53,0x04,0x13,0x00, - 0xd2,0x0c,0x91,0x08,0x10,0x04,0x13,0x00,0x13,0x09,0x13,0x00,0x91,0x08,0x10,0x04, - 0x13,0x00,0x14,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x13,0x00, - 0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, - 0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0xa9,0x01,0xd2, - 0xb0,0xd1,0x6c,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92, - 0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x54, - 0x04,0x12,0x00,0xd3,0x10,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12, - 0x00,0x00,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x12, - 0x09,0xcf,0x86,0xd5,0x14,0x94,0x10,0x93,0x0c,0x52,0x04,0x12,0x00,0x11,0x04,0x12, - 0x00,0x00,0x00,0x00,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, - 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xd0,0x3e,0xcf, - 0x86,0xd5,0x14,0x54,0x04,0x12,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x12, - 0x00,0x12,0x00,0x12,0x00,0xd4,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x93,0x10,0x52,0x04,0x12,0x00,0x51, - 0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, - 0xa0,0xd0,0x52,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3,0x10,0x52,0x04,0x13,0x00,0x51, - 0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10, - 0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x54,0x04,0x13,0x00,0xd3,0x10,0x52, - 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0xd2,0x0c,0x51, - 0x04,0x00,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x00, - 0x00,0x13,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x51,0x04,0x13, - 0x00,0x10,0x04,0x13,0x07,0x13,0x00,0x11,0x04,0x13,0x09,0x13,0x00,0x00,0x00,0x53, - 0x04,0x13,0x00,0x92,0x08,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3, - 0x10,0x52,0x04,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x92, - 0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0, - 0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x14,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x51, - 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14, - 0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x14, - 0x09,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94, - 0x10,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14, - 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, - 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15, - 0x00,0x54,0x04,0x15,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x00, - 0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xd0, - 0xca,0xcf,0x86,0xd5,0xc2,0xd4,0x54,0xd3,0x06,0xcf,0x06,0x09,0x00,0xd2,0x06,0xcf, - 0x06,0x09,0x00,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09, - 0x00,0x94,0x14,0x53,0x04,0x09,0x00,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10, - 0x04,0x09,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x10, - 0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x68,0xd2,0x46,0xd1,0x40,0xd0, - 0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x20,0xd3,0x10,0x92, - 0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10, - 0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x09, - 0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x11, - 0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x10,0x94,0x0c,0x93, - 0x08,0x12,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x06,0xcf, - 0x06,0x0b,0x00,0xd3,0x40,0xd2,0x3a,0xd1,0x34,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0b, - 0x00,0xd4,0x14,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10, - 0x04,0x0b,0x00,0x00,0x00,0x53,0x04,0x15,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x4c,0xd0,0x44,0xcf, - 0x86,0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2, - 0x2a,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x93, - 0x10,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, - 0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0xd2,0x01,0xcf,0x86,0xd5,0x06,0xcf,0x06, - 0x00,0x00,0xe4,0x0b,0x01,0xd3,0x06,0xcf,0x06,0x0c,0x00,0xd2,0x84,0xd1,0x50,0xd0, - 0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x92, - 0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5, - 0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10, - 0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x10,0x00,0xd2,0x08,0x11, - 0x04,0x10,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xd0,0x06,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x00,0x00,0x10,0x00,0xd4,0x10,0x53, - 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52, - 0x04,0x10,0x01,0x91,0x08,0x10,0x04,0x10,0x01,0x10,0x00,0x00,0x00,0x00,0x00,0xd1, - 0x6c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x93,0x10,0x52, - 0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x10,0xe6,0x10,0x00,0x10,0x00,0xcf, - 0x86,0xd5,0x24,0xd4,0x10,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, - 0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x00, - 0x00,0x10,0x00,0x10,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, - 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x00, - 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd0,0x0e,0xcf,0x86,0x95, - 0x08,0x14,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf, - 0x06,0x00,0x00,0xd2,0x30,0xd1,0x0c,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x06,0x14, - 0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92, - 0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xd1,0x4c,0xd0,0x06,0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x2c,0x94, - 0x28,0xd3,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x15,0x00,0x15, - 0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x51,0x04,0x00, - 0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d, - 0x00,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x15,0x00,0xd0, - 0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x00,0x00,0x51, - 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0x55, - 0x04,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x13, - 0x00,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xcf,0x06,0x12,0x00,0xe2, - 0xc6,0x01,0xd1,0x8e,0xd0,0x86,0xcf,0x86,0xd5,0x48,0xd4,0x06,0xcf,0x06,0x12,0x00, - 0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x06,0xcf,0x06,0x12,0x00,0xd1,0x06,0xcf,0x06, - 0x12,0x00,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0xd4,0x14, - 0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x14,0x00, - 0x14,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0x00,0x00, - 0xd4,0x36,0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x12,0x00, - 0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0x54,0x04,0x12,0x00, - 0x93,0x10,0x92,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0xa2,0xd4,0x9c,0xd3,0x74, - 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08, - 0x11,0x04,0x0c,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x06, - 0x13,0x00,0xcf,0x06,0x13,0x00,0xd1,0x48,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, - 0x13,0x00,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04, - 0x13,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x00,0x00,0x93,0x10, - 0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x94,0x0c,0x93,0x08,0x12,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x13,0x00,0xcf,0x06, - 0x13,0x00,0xd2,0x22,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00, - 0xcf,0x86,0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x53,0x04,0x13,0x00,0x12,0x04, - 0x13,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06, - 0x00,0x00,0xd3,0x7f,0xd2,0x79,0xd1,0x34,0xd0,0x06,0xcf,0x06,0x10,0x00,0xcf,0x86, - 0x55,0x04,0x10,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00, - 0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00, - 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x3f,0xcf,0x86,0xd5,0x2c, - 0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0xd2,0x08,0x11,0x04,0x10,0x00,0x00,0x00, - 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x01,0x10,0x00,0x94,0x0d,0x93,0x09,0x12,0x05, - 0x10,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xe1,0x96,0x04,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xcf,0x86,0xe5,0x33,0x04,0xe4,0x83,0x02,0xe3,0xf8,0x01,0xd2,0x26,0xd1,0x06,0xcf, - 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x55,0x04,0x05,0x00,0x54, - 0x04,0x05,0x00,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x00,0x00,0x00, - 0x00,0xd1,0xef,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x20,0xd3,0x10,0x52, - 0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0xd5, - 0x2a,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05, - 0x00,0x10,0x0d,0x05,0xff,0xf0,0x9d,0x85,0x97,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff, - 0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0x00,0xd4,0x75,0xd3,0x61,0xd2,0x44,0xd1, - 0x22,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, - 0xae,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xaf, - 0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, - 0xb0,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xb1, - 0x00,0xd1,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0, - 0x9d,0x85,0xb2,0x00,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0x01,0xd2,0x08,0x11,0x04, - 0x05,0x01,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe2,0x05,0xd8,0xd3,0x12, - 0x92,0x0d,0x51,0x04,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0xff,0x00,0x05,0xff,0x00, - 0x92,0x0e,0x51,0x05,0x05,0xff,0x00,0x10,0x05,0x05,0xff,0x00,0x05,0xdc,0x05,0xdc, - 0xd0,0x97,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x05,0xdc, - 0x10,0x04,0x05,0xdc,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe6,0x05,0xe6, - 0x92,0x08,0x11,0x04,0x05,0xe6,0x05,0xdc,0x05,0x00,0x05,0x00,0xd4,0x14,0x53,0x04, - 0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x05,0xe6,0x11,0x04,0x05,0xe6,0x05,0x00, - 0x53,0x04,0x05,0x00,0xd2,0x15,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x05,0xff, - 0xf0,0x9d,0x86,0xb9,0xf0,0x9d,0x85,0xa5,0x00,0xd1,0x1e,0x10,0x0d,0x05,0xff,0xf0, - 0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, - 0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0, - 0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, - 0x85,0xa5,0xf0,0x9d,0x85,0xaf,0x00,0xcf,0x86,0xd5,0x31,0xd4,0x21,0x93,0x1d,0x92, - 0x19,0x91,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0xf0, - 0x9d,0x85,0xaf,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x53,0x04,0x05,0x00, - 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00, - 0x92,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xd2,0x44,0xd1,0x28,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x95,0x1c,0x94,0x18, - 0x93,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x08,0xe6,0x91,0x08,0x10,0x04,0x08,0xe6, - 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00, - 0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x93,0x08,0x12,0x04,0x14,0x00, - 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0xd5,0x18, - 0x54,0x04,0x07,0x00,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, - 0x07,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x09,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, - 0x09,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0x5f,0x01,0xd2,0xb4,0xd1,0x24,0xd0, - 0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x05,0x00,0x93,0x10,0x52, - 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, - 0x00,0xd0,0x6a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52, - 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0xd4,0x34,0xd3, - 0x1c,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd1,0x08,0x10, - 0x04,0x00,0x00,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05, - 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, - 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x20,0x94, - 0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x07,0x00,0x05,0x00,0x91, - 0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xd1, - 0xa4,0xd0,0x6a,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x10,0x52,0x04,0x05,0x00,0x51, - 0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10, - 0x04,0x05,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xd3, - 0x10,0x52,0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x52, - 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x54,0x04,0x05, - 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, - 0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4, - 0x20,0xd3,0x14,0x52,0x04,0x05,0x00,0xd1,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x10, - 0x04,0x05,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x93, - 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, - 0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06,0x05,0x00,0xd0, - 0x1a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11, - 0x04,0x08,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd1,0x06,0xcf, - 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53, - 0x04,0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x09,0x00,0x11,0x04,0x00,0x00,0x05, - 0x00,0x05,0x00,0x05,0x00,0xd4,0x52,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x46,0xd1, - 0x06,0xcf,0x06,0x11,0x00,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x53,0x04,0x11, - 0x00,0x12,0x04,0x11,0x00,0x00,0x00,0x53,0x04,0x00,0x00,0x92,0x0c,0x51,0x04,0x00, - 0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe0,0xc2,0x03,0xcf,0x86, - 0xe5,0x03,0x01,0xd4,0xfc,0xd3,0xc0,0xd2,0x66,0xd1,0x60,0xd0,0x5a,0xcf,0x86,0xd5, - 0x2c,0xd4,0x14,0x93,0x10,0x52,0x04,0x12,0xe6,0x51,0x04,0x12,0xe6,0x10,0x04,0x12, - 0xe6,0x00,0x00,0x12,0xe6,0x53,0x04,0x12,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x12, - 0xe6,0x00,0x00,0x10,0x04,0x00,0x00,0x12,0xe6,0x12,0xe6,0x94,0x28,0xd3,0x18,0xd2, - 0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x00,0x00,0x12,0xe6,0x91,0x08,0x10,0x04,0x12, - 0xe6,0x00,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x12,0xe6,0x00, - 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x54,0xd0, - 0x36,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x15, - 0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x15, - 0xe6,0x51,0x04,0x15,0xe6,0x10,0x04,0x15,0xe6,0x15,0x00,0x52,0x04,0x15,0x00,0x11, - 0x04,0x15,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0xd2, - 0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xd2,0x36,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x0c,0x53,0x04,0x15,0x00,0x12, - 0x04,0x15,0x00,0x15,0xe6,0x53,0x04,0x15,0x00,0xd2,0x08,0x11,0x04,0x15,0x00,0x00, - 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xcf,0x06,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xd4,0x82,0xd3,0x7c,0xd2,0x3e,0xd1,0x06,0xcf,0x06,0x10,0x00,0xd0, - 0x06,0xcf,0x06,0x10,0x00,0xcf,0x86,0x95,0x2c,0xd4,0x18,0x93,0x14,0x52,0x04,0x10, - 0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10, - 0x00,0x93,0x10,0x52,0x04,0x10,0xdc,0x51,0x04,0x10,0xdc,0x10,0x04,0x10,0xdc,0x00, - 0x00,0x00,0x00,0x00,0x00,0xd1,0x38,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95, - 0x2c,0xd4,0x18,0xd3,0x08,0x12,0x04,0x12,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12, - 0xe6,0x10,0x04,0x12,0x07,0x15,0x00,0x00,0x00,0x53,0x04,0x12,0x00,0xd2,0x08,0x11, - 0x04,0x12,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x06,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xd3,0x82,0xd2,0x48,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00, - 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x10,0x92,0x0c,0x91, - 0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0,0x1e,0xcf, - 0x86,0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x93,0x10,0x52,0x04,0x14,0x00,0x91, - 0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, - 0x34,0xd0,0x2e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x54,0x04,0x15, - 0x00,0x53,0x04,0x15,0x00,0x52,0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0xcf, - 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe2,0xb2,0x01,0xe1,0x41,0x01,0xd0,0x6e,0xcf, - 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x00, - 0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1, - 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10, - 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x92,0x0c,0x91,0x08,0x10, - 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d, - 0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x00, - 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0xd5,0x74,0xd4, - 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x51, - 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00, - 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, - 0x00,0x0d,0x00,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, - 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, - 0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00, - 0x00,0x0d,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d, - 0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, - 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, - 0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, - 0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, - 0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x08,0x10, - 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd0,0x56,0xcf,0x86,0xd5, - 0x20,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x00, - 0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00,0x00,0xd4, - 0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x91, - 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10, - 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00, - 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x0c,0x92,0x08,0x11, - 0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xe5, - 0x96,0x05,0xe4,0x28,0x03,0xe3,0xed,0x01,0xd2,0xa0,0xd1,0x1c,0xd0,0x16,0xcf,0x86, - 0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00,0x00,0x00, - 0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x10,0x54,0x04,0x0a,0x00, - 0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x0c,0x00, - 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0xd3,0x10, - 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04, - 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x28, - 0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, - 0x0c,0x00,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, - 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, - 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0xe4,0xd0,0x5a,0xcf,0x86,0xd5,0x20, - 0x94,0x1c,0x53,0x04,0x0b,0x00,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, - 0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd4,0x14, - 0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, - 0x14,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00, - 0x0c,0x00,0x52,0x04,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04, - 0x0c,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04, - 0x0c,0x00,0x10,0x04,0x0b,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0b,0x00, - 0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, - 0x0b,0x00,0x0c,0x00,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, - 0x0c,0x00,0x0b,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, - 0x0b,0x00,0xd4,0x18,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0d,0x00, - 0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0c,0x00,0xd2,0x10, - 0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd1,0x08, - 0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd0,0x4e,0xcf,0x86, - 0xd5,0x34,0xd4,0x14,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00, - 0x11,0x04,0x0b,0x00,0x0c,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00, - 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, - 0x12,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08, - 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00, - 0x94,0x10,0x93,0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, - 0x0c,0x00,0xd2,0x7e,0xd1,0x78,0xd0,0x3e,0xcf,0x86,0xd5,0x1c,0x94,0x18,0x93,0x14, - 0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, - 0x0b,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, - 0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x92,0x08, - 0x11,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x10,0x93,0x0c,0x52,0x04, - 0x13,0x00,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, - 0xd1,0x58,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c, - 0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00, - 0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x93,0x10,0x52,0x04,0x0c,0x00, - 0x51,0x04,0x0c,0x00,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04, - 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x91,0x08, - 0x10,0x04,0x0c,0x00,0x10,0x00,0x11,0x00,0xd0,0x16,0xcf,0x86,0x95,0x10,0x54,0x04, - 0x0c,0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x0c,0x00,0xcf,0x86, - 0xd5,0x34,0xd4,0x28,0xd3,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10,0x04,0x0c,0x00, - 0x10,0x00,0x0c,0x00,0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00, - 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x93,0x08,0x12,0x04,0x11,0x00, - 0x10,0x00,0x10,0x00,0x54,0x04,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, - 0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x11,0x00,0xd3,0xfc,0xd2,0x6c,0xd1,0x3c, - 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00, - 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86, - 0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00, - 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00, - 0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0xd2,0x0c, - 0x91,0x08,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00, - 0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0xd1,0x54,0xd0,0x1a,0xcf,0x86,0x55,0x04, - 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04, - 0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x1c,0x94,0x18,0xd3,0x08,0x12,0x04,0x0d,0x00, - 0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00, - 0x0c,0x00,0xd4,0x08,0x13,0x04,0x0c,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c, - 0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86, - 0x55,0x04,0x10,0x00,0x94,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10,0x04, - 0x12,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00, - 0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, - 0x10,0x00,0x0c,0x00,0x0c,0x00,0xe2,0x19,0x01,0xd1,0xa8,0xd0,0x7e,0xcf,0x86,0xd5, - 0x4c,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0d,0x00,0x0c,0x00,0x0c, - 0x00,0x0c,0x00,0x0c,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0d, - 0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, - 0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, - 0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0d,0x00,0xd4,0x1c,0xd3,0x0c,0x52, - 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x0d,0x00,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, - 0x04,0x0d,0x00,0x0c,0x00,0x0d,0x00,0x93,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, - 0x04,0x0d,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0x94,0x20,0x93, - 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x11, - 0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x10,0x00,0x10, - 0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x30,0xd4,0x10,0x93,0x0c,0x52, - 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x93,0x1c,0xd2,0x10,0xd1, - 0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x91,0x08,0x10, - 0x04,0x13,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x52, - 0x04,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52, - 0x04,0x10,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x92,0x10,0xd1, - 0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1, - 0x1c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c, - 0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x10, - 0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x10,0x00,0xd3,0x10,0x52,0x04,0x10,0x00,0x91, - 0x08,0x10,0x04,0x10,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x0c,0x53,0x04,0x15,0x00,0x12,0x04,0x15, - 0x00,0x00,0x00,0x00,0x00,0xe4,0x40,0x02,0xe3,0xc9,0x01,0xd2,0x5c,0xd1,0x34,0xd0, - 0x16,0xcf,0x86,0x95,0x10,0x94,0x0c,0x53,0x04,0x10,0x00,0x12,0x04,0x10,0x00,0x00, - 0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x95,0x18,0xd4,0x08,0x13,0x04,0x10,0x00,0x00, - 0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x10, - 0x00,0xd0,0x22,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10, - 0x00,0x94,0x10,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, - 0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0xc0,0xd0,0x5e,0xcf,0x86,0xd5,0x30,0xd4, - 0x14,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x15, - 0x00,0x15,0x00,0x53,0x04,0x11,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x12, - 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd4,0x08,0x13, - 0x04,0x12,0x00,0x13,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x12,0x00,0x13, - 0x00,0x10,0x04,0x13,0x00,0x12,0x00,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12, - 0x00,0x10,0x04,0x12,0x00,0x15,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x12, - 0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x53, - 0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13, - 0x00,0xd4,0x0c,0x53,0x04,0x13,0x00,0x12,0x04,0x13,0x00,0x14,0x00,0xd3,0x1c,0xd2, - 0x10,0xd1,0x08,0x10,0x04,0x14,0x00,0x15,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51, - 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10, - 0x04,0x14,0x00,0x15,0x00,0x14,0x00,0xd0,0x62,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93, - 0x10,0x52,0x04,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x12,0x00,0x12, - 0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x12,0x00,0x13,0x00,0x13,0x00,0x14,0x00,0xd4, - 0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91, - 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10, - 0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x53,0x04,0x14,0x00,0x92, - 0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x30,0x94,0x2c,0xd3, - 0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x11,0x00,0x14,0x00,0x10,0x04,0x14,0x00,0x15, - 0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x91, - 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x52, - 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14, - 0x00,0xd2,0x70,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x15,0x00,0xcf,0x86,0xd5,0x10,0x54, - 0x04,0x15,0x00,0x93,0x08,0x12,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53, - 0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x00,0x00,0xd3,0x08,0x12, - 0x04,0x15,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00, - 0x00,0x00,0x00,0xd0,0x2a,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51, - 0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x52, - 0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, - 0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00, - 0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55, - 0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11, - 0x04,0x00,0x00,0x02,0x00,0xe4,0xf9,0x12,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, - 0xd2,0xc2,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x3c, - 0xd4,0x06,0xcf,0x06,0x05,0x00,0xd3,0x06,0xcf,0x06,0x05,0x00,0xd2,0x2a,0xd1,0x06, - 0xcf,0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04, - 0x05,0x00,0x93,0x10,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x86, - 0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x0b,0x00,0xd3,0x06,0xcf,0x06,0x0b,0x00,0xd2,0x06, - 0xcf,0x06,0x0b,0x00,0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, - 0x0b,0x00,0x93,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xcf,0x06,0x0c,0x00,0xcf,0x06,0x0c,0x00,0xd4,0x32,0xd3,0x2c, - 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04,0x0c,0x00,0x53,0x04, - 0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x11,0x00,0xcf,0x06, - 0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06, - 0x11,0x00,0xd1,0x48,0xd0,0x40,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x11,0x00,0xd4,0x06, - 0xcf,0x06,0x11,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06, - 0x11,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04,0x11,0x00,0x94,0x10,0x93,0x0c,0x92,0x08, - 0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13,0x00,0xcf,0x06,0x13,0x00, - 0xcf,0x06,0x13,0x00,0xcf,0x86,0xcf,0x06,0x13,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x06, - 0xcf,0x06,0x13,0x00,0xd4,0x36,0xd3,0x06,0xcf,0x06,0x13,0x00,0xd2,0x06,0xcf,0x06, - 0x13,0x00,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00,0xcf,0x86, - 0x55,0x04,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x13,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86, - 0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x68,0x11,0xe3,0x51,0x10,0xe2,0x17,0x08,0xe1, - 0x06,0x04,0xe0,0x03,0x02,0xcf,0x86,0xe5,0x06,0x01,0xd4,0x82,0xd3,0x41,0xd2,0x21, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00, - 0x10,0x08,0x05,0xff,0xe4,0xb9,0x81,0x00,0x05,0xff,0xf0,0xa0,0x84,0xa2,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe4,0xbd,0xa0,0x00,0x05,0xff,0xe4,0xbe,0xae,0x00,0x10, - 0x08,0x05,0xff,0xe4,0xbe,0xbb,0x00,0x05,0xff,0xe5,0x80,0x82,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe5,0x81,0xba,0x00,0x05,0xff,0xe5,0x82,0x99,0x00,0x10, - 0x08,0x05,0xff,0xe5,0x83,0xa7,0x00,0x05,0xff,0xe5,0x83,0x8f,0x00,0xd1,0x11,0x10, - 0x08,0x05,0xff,0xe3,0x92,0x9e,0x00,0x05,0xff,0xf0,0xa0,0x98,0xba,0x00,0x10,0x08, - 0x05,0xff,0xe5,0x85,0x8d,0x00,0x05,0xff,0xe5,0x85,0x94,0x00,0xd3,0x42,0xd2,0x21, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa4,0x00,0x05,0xff,0xe5,0x85,0xb7,0x00, - 0x10,0x09,0x05,0xff,0xf0,0xa0,0x94,0x9c,0x00,0x05,0xff,0xe3,0x92,0xb9,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa7,0x00,0x05,0xff,0xe5,0x86,0x8d,0x00,0x10, - 0x09,0x05,0xff,0xf0,0xa0,0x95,0x8b,0x00,0x05,0xff,0xe5,0x86,0x97,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x86,0xa4,0x00,0x05,0xff,0xe4,0xbb,0x8c,0x00, - 0x10,0x08,0x05,0xff,0xe5,0x86,0xac,0x00,0x05,0xff,0xe5,0x86,0xb5,0x00,0xd1,0x11, - 0x10,0x09,0x05,0xff,0xf0,0xa9,0x87,0x9f,0x00,0x05,0xff,0xe5,0x87,0xb5,0x00,0x10, - 0x08,0x05,0xff,0xe5,0x88,0x83,0x00,0x05,0xff,0xe3,0x93,0x9f,0x00,0xd4,0x80,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x88,0xbb,0x00,0x05,0xff,0xe5, - 0x89,0x86,0x00,0x10,0x08,0x05,0xff,0xe5,0x89,0xb2,0x00,0x05,0xff,0xe5,0x89,0xb7, - 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0x94,0x95,0x00,0x05,0xff,0xe5,0x8b,0x87, - 0x00,0x10,0x08,0x05,0xff,0xe5,0x8b,0x89,0x00,0x05,0xff,0xe5,0x8b,0xa4,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x8b,0xba,0x00,0x05,0xff,0xe5,0x8c,0x85, - 0x00,0x10,0x08,0x05,0xff,0xe5,0x8c,0x86,0x00,0x05,0xff,0xe5,0x8c,0x97,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0x89,0x00,0x05,0xff,0xe5,0x8d,0x91,0x00,0x10, - 0x08,0x05,0xff,0xe5,0x8d,0x9a,0x00,0x05,0xff,0xe5,0x8d,0xb3,0x00,0xd3,0x39,0xd2, - 0x18,0x91,0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0xbd,0x00,0x05,0xff,0xe5,0x8d,0xbf, - 0x00,0x05,0xff,0xe5,0x8d,0xbf,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0,0xa8, - 0xac,0x00,0x05,0xff,0xe7,0x81,0xb0,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0x8a,0x00, - 0x05,0xff,0xe5,0x8f,0x9f,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0, - 0xad,0xa3,0x00,0x05,0xff,0xe5,0x8f,0xab,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0xb1, - 0x00,0x05,0xff,0xe5,0x90,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0x9e, - 0x00,0x05,0xff,0xe5,0x90,0xb8,0x00,0x10,0x08,0x05,0xff,0xe5,0x91,0x88,0x00,0x05, - 0xff,0xe5,0x91,0xa8,0x00,0xcf,0x86,0xe5,0x02,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6,0x00, - 0x10,0x08,0x05,0xff,0xe5,0x94,0x90,0x00,0x05,0xff,0xe5,0x95,0x93,0x00,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe5,0x95,0xa3,0x00,0x05,0xff,0xe5,0x96,0x84,0x00,0x10,0x08, - 0x05,0xff,0xe5,0x96,0x84,0x00,0x05,0xff,0xe5,0x96,0x99,0x00,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe5,0x96,0xab,0x00,0x05,0xff,0xe5,0x96,0xb3,0x00,0x10,0x08, - 0x05,0xff,0xe5,0x97,0x82,0x00,0x05,0xff,0xe5,0x9c,0x96,0x00,0xd1,0x10,0x10,0x08, - 0x05,0xff,0xe5,0x98,0x86,0x00,0x05,0xff,0xe5,0x9c,0x97,0x00,0x10,0x08,0x05,0xff, - 0xe5,0x99,0x91,0x00,0x05,0xff,0xe5,0x99,0xb4,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe5,0x88,0x87,0x00,0x05,0xff,0xe5,0xa3,0xae,0x00,0x10,0x08, - 0x05,0xff,0xe5,0x9f,0x8e,0x00,0x05,0xff,0xe5,0x9f,0xb4,0x00,0xd1,0x10,0x10,0x08, - 0x05,0xff,0xe5,0xa0,0x8d,0x00,0x05,0xff,0xe5,0x9e,0x8b,0x00,0x10,0x08,0x05,0xff, - 0xe5,0xa0,0xb2,0x00,0x05,0xff,0xe5,0xa0,0xb1,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, - 0x05,0xff,0xe5,0xa2,0xac,0x00,0x05,0xff,0xf0,0xa1,0x93,0xa4,0x00,0x10,0x08,0x05, - 0xff,0xe5,0xa3,0xb2,0x00,0x05,0xff,0xe5,0xa3,0xb7,0x00,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe5,0xa4,0x86,0x00,0x05,0xff,0xe5,0xa4,0x9a,0x00,0x10,0x08,0x05,0xff,0xe5, - 0xa4,0xa2,0x00,0x05,0xff,0xe5,0xa5,0xa2,0x00,0xd4,0x7b,0xd3,0x42,0xd2,0x22,0xd1, - 0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa, - 0x00,0x10,0x08,0x05,0xff,0xe5,0xa7,0xac,0x00,0x05,0xff,0xe5,0xa8,0x9b,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe5,0xa8,0xa7,0x00,0x05,0xff,0xe5,0xa7,0x98,0x00,0x10, - 0x08,0x05,0xff,0xe5,0xa9,0xa6,0x00,0x05,0xff,0xe3,0x9b,0xae,0x00,0xd2,0x18,0x91, - 0x10,0x10,0x08,0x05,0xff,0xe3,0x9b,0xbc,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, - 0xff,0xe5,0xac,0xbe,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0xa7,0x88,0x00, - 0x05,0xff,0xe5,0xaf,0x83,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0x98,0x00,0x05,0xff, - 0xe5,0xaf,0xa7,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xaf, - 0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0xbf, - 0x00,0x05,0xff,0xe5,0xb0,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xbd,0x93, - 0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0x9e,0x81,0x00,0x05, - 0xff,0xe5,0xb1,0xa0,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb1,0xae, - 0x00,0x05,0xff,0xe5,0xb3,0x80,0x00,0x10,0x08,0x05,0xff,0xe5,0xb2,0x8d,0x00,0x05, - 0xff,0xf0,0xa1,0xb7,0xa4,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xb5,0x83,0x00, - 0x05,0xff,0xf0,0xa1,0xb7,0xa6,0x00,0x10,0x08,0x05,0xff,0xe5,0xb5,0xae,0x00,0x05, - 0xff,0xe5,0xb5,0xab,0x00,0xe0,0x04,0x02,0xcf,0x86,0xd5,0xfe,0xd4,0x82,0xd3,0x40, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb5,0xbc,0x00,0x05,0xff,0xe5,0xb7, - 0xa1,0x00,0x10,0x08,0x05,0xff,0xe5,0xb7,0xa2,0x00,0x05,0xff,0xe3,0xa0,0xaf,0x00, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb7,0xbd,0x00,0x05,0xff,0xe5,0xb8,0xa8,0x00, - 0x10,0x08,0x05,0xff,0xe5,0xb8,0xbd,0x00,0x05,0xff,0xe5,0xb9,0xa9,0x00,0xd2,0x21, - 0xd1,0x11,0x10,0x08,0x05,0xff,0xe3,0xa1,0xa2,0x00,0x05,0xff,0xf0,0xa2,0x86,0x83, - 0x00,0x10,0x08,0x05,0xff,0xe3,0xa1,0xbc,0x00,0x05,0xff,0xe5,0xba,0xb0,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe5,0xba,0xb3,0x00,0x05,0xff,0xe5,0xba,0xb6,0x00,0x10, - 0x08,0x05,0xff,0xe5,0xbb,0x8a,0x00,0x05,0xff,0xf0,0xaa,0x8e,0x92,0x00,0xd3,0x3b, - 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xbb,0xbe,0x00,0x05,0xff,0xf0,0xa2, - 0x8c,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2,0x8c,0xb1,0x00,0x05,0xff,0xe8,0x88, - 0x81,0x00,0x51,0x08,0x05,0xff,0xe5,0xbc,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0xa3, - 0x87,0x00,0x05,0xff,0xf0,0xa3,0x8a,0xb8,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, - 0xff,0xf0,0xa6,0x87,0x9a,0x00,0x05,0xff,0xe5,0xbd,0xa2,0x00,0x10,0x08,0x05,0xff, - 0xe5,0xbd,0xab,0x00,0x05,0xff,0xe3,0xa3,0xa3,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, - 0xe5,0xbe,0x9a,0x00,0x05,0xff,0xe5,0xbf,0x8d,0x00,0x10,0x08,0x05,0xff,0xe5,0xbf, - 0x97,0x00,0x05,0xff,0xe5,0xbf,0xb9,0x00,0xd4,0x81,0xd3,0x41,0xd2,0x20,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe6,0x82,0x81,0x00,0x05,0xff,0xe3,0xa4,0xba,0x00,0x10,0x08, - 0x05,0xff,0xe3,0xa4,0x9c,0x00,0x05,0xff,0xe6,0x82,0x94,0x00,0xd1,0x11,0x10,0x09, - 0x05,0xff,0xf0,0xa2,0x9b,0x94,0x00,0x05,0xff,0xe6,0x83,0x87,0x00,0x10,0x08,0x05, - 0xff,0xe6,0x85,0x88,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe6,0x85,0x8e,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0x10,0x08,0x05, - 0xff,0xe6,0x85,0xba,0x00,0x05,0xff,0xe6,0x86,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe6,0x86,0xb2,0x00,0x05,0xff,0xe6,0x86,0xa4,0x00,0x10,0x08,0x05,0xff,0xe6, - 0x86,0xaf,0x00,0x05,0xff,0xe6,0x87,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe6,0x87,0xb2,0x00,0x05,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x05, - 0xff,0xe6,0x88,0x90,0x00,0x05,0xff,0xe6,0x88,0x9b,0x00,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe6,0x89,0x9d,0x00,0x05,0xff,0xe6,0x8a,0xb1,0x00,0x10,0x08,0x05,0xff,0xe6, - 0x8b,0x94,0x00,0x05,0xff,0xe6,0x8d,0x90,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, - 0xff,0xf0,0xa2,0xac,0x8c,0x00,0x05,0xff,0xe6,0x8c,0xbd,0x00,0x10,0x08,0x05,0xff, - 0xe6,0x8b,0xbc,0x00,0x05,0xff,0xe6,0x8d,0xa8,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, - 0xe6,0x8e,0x83,0x00,0x05,0xff,0xe6,0x8f,0xa4,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2, - 0xaf,0xb1,0x00,0x05,0xff,0xe6,0x90,0xa2,0x00,0xcf,0x86,0xe5,0x03,0x01,0xd4,0x81, - 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x8f,0x85,0x00,0x05,0xff, - 0xe6,0x8e,0xa9,0x00,0x10,0x08,0x05,0xff,0xe3,0xa8,0xae,0x00,0x05,0xff,0xe6,0x91, - 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x91,0xbe,0x00,0x05,0xff,0xe6,0x92, - 0x9d,0x00,0x10,0x08,0x05,0xff,0xe6,0x91,0xb7,0x00,0x05,0xff,0xe3,0xa9,0xac,0x00, - 0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x95,0x8f,0x00,0x05,0xff,0xe6,0x95, - 0xac,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0x80,0x8a,0x00,0x05,0xff,0xe6,0x97,0xa3, - 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9b,0xb8,0x00,0x05,0xff,0xe6,0x99,0x89, - 0x00,0x10,0x08,0x05,0xff,0xe3,0xac,0x99,0x00,0x05,0xff,0xe6,0x9a,0x91,0x00,0xd3, - 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0xac,0x88,0x00,0x05,0xff,0xe3, - 0xab,0xa4,0x00,0x10,0x08,0x05,0xff,0xe5,0x86,0x92,0x00,0x05,0xff,0xe5,0x86,0x95, - 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x80,0x00,0x05,0xff,0xe6,0x9a,0x9c, - 0x00,0x10,0x08,0x05,0xff,0xe8,0x82,0xad,0x00,0x05,0xff,0xe4,0x8f,0x99,0x00,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x97,0x00,0x05,0xff,0xe6,0x9c,0x9b, - 0x00,0x10,0x08,0x05,0xff,0xe6,0x9c,0xa1,0x00,0x05,0xff,0xe6,0x9d,0x9e,0x00,0xd1, - 0x11,0x10,0x08,0x05,0xff,0xe6,0x9d,0x93,0x00,0x05,0xff,0xf0,0xa3,0x8f,0x83,0x00, - 0x10,0x08,0x05,0xff,0xe3,0xad,0x89,0x00,0x05,0xff,0xe6,0x9f,0xba,0x00,0xd4,0x82, - 0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9e,0x85,0x00,0x05,0xff, - 0xe6,0xa1,0x92,0x00,0x10,0x08,0x05,0xff,0xe6,0xa2,0x85,0x00,0x05,0xff,0xf0,0xa3, - 0x91,0xad,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa2,0x8e,0x00,0x05,0xff,0xe6, - 0xa0,0x9f,0x00,0x10,0x08,0x05,0xff,0xe6,0xa4,0x94,0x00,0x05,0xff,0xe3,0xae,0x9d, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa5,0x82,0x00,0x05,0xff,0xe6, - 0xa6,0xa3,0x00,0x10,0x08,0x05,0xff,0xe6,0xa7,0xaa,0x00,0x05,0xff,0xe6,0xaa,0xa8, - 0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0x9a,0xa3,0x00,0x05,0xff,0xe6,0xab, - 0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb0,0x98,0x00,0x05,0xff,0xe6,0xac,0xa1,0x00, - 0xd3,0x42,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0xa2,0xa7,0x00,0x05, - 0xff,0xe6,0xad,0x94,0x00,0x10,0x08,0x05,0xff,0xe3,0xb1,0x8e,0x00,0x05,0xff,0xe6, - 0xad,0xb2,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xae,0x9f,0x00,0x05,0xff,0xe6, - 0xae,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xae,0xbb,0x00,0x05,0xff,0xf0,0xa3,0xaa, - 0x8d,0x00,0xd2,0x23,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0xb4,0x8b,0x00,0x05, - 0xff,0xf0,0xa3,0xab,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0x8e,0x00,0x05,0xff, - 0xf0,0xa3,0xb2,0xbc,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb2,0xbf,0x00,0x05, - 0xff,0xe6,0xb3,0x8d,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0xa7,0x00,0x05,0xff,0xe6, - 0xb4,0x96,0x00,0xe1,0x1d,0x04,0xe0,0x0c,0x02,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x82, - 0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, - 0xe6,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe6,0xb5,0x81,0x00,0x05,0xff,0xe6,0xb5, - 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb5,0xb8,0x00,0x05,0xff,0xe6,0xb6, - 0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0xb4,0x9e,0x00,0x05,0xff,0xe6,0xb4,0xb4, - 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb8,0xaf,0x00,0x05,0xff,0xe6, - 0xb9,0xae,0x00,0x10,0x08,0x05,0xff,0xe3,0xb4,0xb3,0x00,0x05,0xff,0xe6,0xbb,0x8b, - 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe6,0xbb,0x87,0x00,0x05,0xff,0xf0,0xa3,0xbb, - 0x91,0x00,0x10,0x08,0x05,0xff,0xe6,0xb7,0xb9,0x00,0x05,0xff,0xe6,0xbd,0xae,0x00, - 0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00,0x05, - 0xff,0xf0,0xa3,0xbe,0x8e,0x00,0x10,0x08,0x05,0xff,0xe6,0xbf,0x86,0x00,0x05,0xff, - 0xe7,0x80,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x80,0x9e,0x00,0x05,0xff, - 0xe7,0x80,0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb6,0x96,0x00,0x05,0xff,0xe7,0x81, - 0x8a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff, - 0xe7,0x81,0xb7,0x00,0x10,0x08,0x05,0xff,0xe7,0x82,0xad,0x00,0x05,0xff,0xf0,0xa0, - 0x94,0xa5,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0, - 0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xf0,0xa4, - 0x8e,0xab,0x00,0xd4,0x7b,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7, - 0x88,0xa8,0x00,0x05,0xff,0xe7,0x88,0xb5,0x00,0x10,0x08,0x05,0xff,0xe7,0x89,0x90, - 0x00,0x05,0xff,0xf0,0xa4,0x98,0x88,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x8a, - 0x80,0x00,0x05,0xff,0xe7,0x8a,0x95,0x00,0x10,0x09,0x05,0xff,0xf0,0xa4,0x9c,0xb5, - 0x00,0x05,0xff,0xf0,0xa4,0xa0,0x94,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff, - 0xe7,0x8d,0xba,0x00,0x05,0xff,0xe7,0x8e,0x8b,0x00,0x10,0x08,0x05,0xff,0xe3,0xba, - 0xac,0x00,0x05,0xff,0xe7,0x8e,0xa5,0x00,0x51,0x08,0x05,0xff,0xe3,0xba,0xb8,0x00, - 0x10,0x08,0x05,0xff,0xe7,0x91,0x87,0x00,0x05,0xff,0xe7,0x91,0x9c,0x00,0xd3,0x42, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x91,0xb1,0x00,0x05,0xff,0xe7,0x92, - 0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0x93,0x8a,0x00,0x05,0xff,0xe3,0xbc,0x9b,0x00, - 0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x94,0xa4,0x00,0x05,0xff,0xf0,0xa4,0xb0,0xb6, - 0x00,0x10,0x08,0x05,0xff,0xe7,0x94,0xbe,0x00,0x05,0xff,0xf0,0xa4,0xb2,0x92,0x00, - 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x95,0xb0,0x00,0x05,0xff,0xf0,0xa2, - 0x86,0x9f,0x00,0x10,0x08,0x05,0xff,0xe7,0x98,0x90,0x00,0x05,0xff,0xf0,0xa4,0xbe, - 0xa1,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa4,0xbe,0xb8,0x00,0x05,0xff,0xf0, - 0xa5,0x81,0x84,0x00,0x10,0x08,0x05,0xff,0xe3,0xbf,0xbc,0x00,0x05,0xff,0xe4,0x80, - 0x88,0x00,0xcf,0x86,0xe5,0x04,0x01,0xd4,0x7d,0xd3,0x3c,0xd2,0x23,0xd1,0x11,0x10, - 0x08,0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0x10,0x09, - 0x05,0xff,0xf0,0xa5,0x83,0xb2,0x00,0x05,0xff,0xf0,0xa5,0x84,0x99,0x00,0x91,0x11, - 0x10,0x09,0x05,0xff,0xf0,0xa5,0x84,0xb3,0x00,0x05,0xff,0xe7,0x9c,0x9e,0x00,0x05, - 0xff,0xe7,0x9c,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x9d,0x8a, - 0x00,0x05,0xff,0xe4,0x80,0xb9,0x00,0x10,0x08,0x05,0xff,0xe7,0x9e,0x8b,0x00,0x05, - 0xff,0xe4,0x81,0x86,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe4,0x82,0x96,0x00,0x05, - 0xff,0xf0,0xa5,0x90,0x9d,0x00,0x10,0x08,0x05,0xff,0xe7,0xa1,0x8e,0x00,0x05,0xff, - 0xe7,0xa2,0x8c,0x00,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xa3, - 0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0x98,0xa6, - 0x00,0x05,0xff,0xe7,0xa5,0x96,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0x9a, - 0x9a,0x00,0x05,0xff,0xf0,0xa5,0x9b,0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0xa6,0x8f, - 0x00,0x05,0xff,0xe7,0xa7,0xab,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4, - 0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80,0x00,0x10,0x08,0x05,0xff,0xe7,0xa9,0x8a, - 0x00,0x05,0xff,0xe7,0xa9,0x8f,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5, - 0xbc,0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa, - 0xa7,0x00,0x05,0xff,0xe7,0xab,0xae,0x00,0xd4,0x83,0xd3,0x42,0xd2,0x21,0xd1,0x11, - 0x10,0x08,0x05,0xff,0xe4,0x88,0x82,0x00,0x05,0xff,0xf0,0xa5,0xae,0xab,0x00,0x10, - 0x08,0x05,0xff,0xe7,0xaf,0x86,0x00,0x05,0xff,0xe7,0xaf,0x89,0x00,0xd1,0x11,0x10, - 0x08,0x05,0xff,0xe4,0x88,0xa7,0x00,0x05,0xff,0xf0,0xa5,0xb2,0x80,0x00,0x10,0x08, - 0x05,0xff,0xe7,0xb3,0x92,0x00,0x05,0xff,0xe4,0x8a,0xa0,0x00,0xd2,0x21,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe7,0xb3,0xa8,0x00,0x05,0xff,0xe7,0xb3,0xa3,0x00,0x10,0x08, - 0x05,0xff,0xe7,0xb4,0x80,0x00,0x05,0xff,0xf0,0xa5,0xbe,0x86,0x00,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe7,0xb5,0xa3,0x00,0x05,0xff,0xe4,0x8c,0x81,0x00,0x10,0x08,0x05, - 0xff,0xe7,0xb7,0x87,0x00,0x05,0xff,0xe7,0xb8,0x82,0x00,0xd3,0x44,0xd2,0x22,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe7,0xb9,0x85,0x00,0x05,0xff,0xe4,0x8c,0xb4,0x00,0x10, - 0x09,0x05,0xff,0xf0,0xa6,0x88,0xa8,0x00,0x05,0xff,0xf0,0xa6,0x89,0x87,0x00,0xd1, - 0x11,0x10,0x08,0x05,0xff,0xe4,0x8d,0x99,0x00,0x05,0xff,0xf0,0xa6,0x8b,0x99,0x00, - 0x10,0x08,0x05,0xff,0xe7,0xbd,0xba,0x00,0x05,0xff,0xf0,0xa6,0x8c,0xbe,0x00,0xd2, - 0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xbe,0x95,0x00,0x05,0xff,0xe7,0xbf,0xba, - 0x00,0x10,0x08,0x05,0xff,0xe8,0x80,0x85,0x00,0x05,0xff,0xf0,0xa6,0x93,0x9a,0x00, - 0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa6,0x94,0xa3,0x00,0x05,0xff,0xe8,0x81,0xa0, - 0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0x96,0xa8,0x00,0x05,0xff,0xe8,0x81,0xb0,0x00, - 0xe0,0x11,0x02,0xcf,0x86,0xe5,0x07,0x01,0xd4,0x85,0xd3,0x42,0xd2,0x21,0xd1,0x11, - 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0x10, - 0x08,0x05,0xff,0xe8,0x82,0xb2,0x00,0x05,0xff,0xe8,0x84,0x83,0x00,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe4,0x90,0x8b,0x00,0x05,0xff,0xe8,0x84,0xbe,0x00,0x10,0x08,0x05, - 0xff,0xe5,0xaa,0xb5,0x00,0x05,0xff,0xf0,0xa6,0x9e,0xa7,0x00,0xd2,0x23,0xd1,0x12, - 0x10,0x09,0x05,0xff,0xf0,0xa6,0x9e,0xb5,0x00,0x05,0xff,0xf0,0xa3,0x8e,0x93,0x00, - 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8e,0x9c,0x00,0x05,0xff,0xe8,0x88,0x81,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe8,0x88,0x84,0x00,0x05,0xff,0xe8,0xbe,0x9e,0x00,0x10, - 0x08,0x05,0xff,0xe4,0x91,0xab,0x00,0x05,0xff,0xe8,0x8a,0x91,0x00,0xd3,0x41,0xd2, - 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0x8b,0x00,0x05,0xff,0xe8,0x8a,0x9d, - 0x00,0x10,0x08,0x05,0xff,0xe5,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xb1,0x00,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xbd,0x00,0x10, - 0x08,0x05,0xff,0xe8,0x8b,0xa6,0x00,0x05,0xff,0xf0,0xa6,0xac,0xbc,0x00,0xd2,0x20, - 0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8b,0xa5,0x00,0x05,0xff,0xe8,0x8c,0x9d,0x00, - 0x10,0x08,0x05,0xff,0xe8,0x8d,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xad,0x00,0xd1,0x10, - 0x10,0x08,0x05,0xff,0xe8,0x8c,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xbd,0x00,0x10,0x08, - 0x05,0xff,0xe8,0x8f,0xa7,0x00,0x05,0xff,0xe8,0x91,0x97,0x00,0xd4,0x85,0xd3,0x43, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, - 0x8a,0x00,0x10,0x08,0x05,0xff,0xe8,0x8f,0x8c,0x00,0x05,0xff,0xe8,0x8f,0x9c,0x00, - 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb0,0xb6,0x00,0x05,0xff,0xf0,0xa6,0xb5, - 0xab,0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb3,0x95,0x00,0x05,0xff,0xe4,0x94,0xab, - 0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x93,0xb1,0x00,0x05,0xff,0xe8, - 0x93,0xb3,0x00,0x10,0x08,0x05,0xff,0xe8,0x94,0x96,0x00,0x05,0xff,0xf0,0xa7,0x8f, - 0x8a,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe8,0x95,0xa4,0x00,0x05,0xff,0xf0,0xa6, - 0xbc,0xac,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0x9d,0x00,0x05,0xff,0xe4,0x95,0xa1, - 0x00,0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, - 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0xab,0x00,0x05, - 0xff,0xe8,0x99,0x90,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x99,0x9c,0x00,0x05, - 0xff,0xe8,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8,0x99,0xa9,0x00,0x05,0xff,0xe8, - 0x9a,0xa9,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05, - 0xff,0xe8,0x9c,0x8e,0x00,0x10,0x08,0x05,0xff,0xe8,0x9b,0xa2,0x00,0x05,0xff,0xe8, - 0x9d,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8, - 0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0x97,0x97, - 0x00,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x83,0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08, - 0x05,0xff,0xe8,0x9f,0xa1,0x00,0x05,0xff,0xe8,0xa0,0x81,0x00,0x10,0x08,0x05,0xff, - 0xe4,0x97,0xb9,0x00,0x05,0xff,0xe8,0xa1,0xa0,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, - 0xe8,0xa1,0xa3,0x00,0x05,0xff,0xf0,0xa7,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8, - 0xa3,0x97,0x00,0x05,0xff,0xe8,0xa3,0x9e,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe4,0x98,0xb5,0x00,0x05,0xff,0xe8,0xa3,0xba,0x00,0x10,0x08,0x05,0xff,0xe3, - 0x92,0xbb,0x00,0x05,0xff,0xf0,0xa7,0xa2,0xae,0x00,0xd1,0x11,0x10,0x09,0x05,0xff, - 0xf0,0xa7,0xa5,0xa6,0x00,0x05,0xff,0xe4,0x9a,0xbe,0x00,0x10,0x08,0x05,0xff,0xe4, - 0x9b,0x87,0x00,0x05,0xff,0xe8,0xaa,0xa0,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10, - 0x08,0x05,0xff,0xe8,0xab,0xad,0x00,0x05,0xff,0xe8,0xae,0x8a,0x00,0x10,0x08,0x05, - 0xff,0xe8,0xb1,0x95,0x00,0x05,0xff,0xf0,0xa7,0xb2,0xa8,0x00,0xd1,0x10,0x10,0x08, - 0x05,0xff,0xe8,0xb2,0xab,0x00,0x05,0xff,0xe8,0xb3,0x81,0x00,0x10,0x08,0x05,0xff, - 0xe8,0xb4,0x9b,0x00,0x05,0xff,0xe8,0xb5,0xb7,0x00,0xd2,0x22,0xd1,0x12,0x10,0x09, - 0x05,0xff,0xf0,0xa7,0xbc,0xaf,0x00,0x05,0xff,0xf0,0xa0,0xa0,0x84,0x00,0x10,0x08, - 0x05,0xff,0xe8,0xb7,0x8b,0x00,0x05,0xff,0xe8,0xb6,0xbc,0x00,0xd1,0x11,0x10,0x08, - 0x05,0xff,0xe8,0xb7,0xb0,0x00,0x05,0xff,0xf0,0xa0,0xa3,0x9e,0x00,0x10,0x08,0x05, - 0xff,0xe8,0xbb,0x94,0x00,0x05,0xff,0xe8,0xbc,0xb8,0x00,0xd4,0x84,0xd3,0x43,0xd2, - 0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa8,0x97,0x92,0x00,0x05,0xff,0xf0,0xa8, - 0x97,0xad,0x00,0x10,0x08,0x05,0xff,0xe9,0x82,0x94,0x00,0x05,0xff,0xe9,0x83,0xb1, - 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x84,0x91,0x00,0x05,0xff,0xf0,0xa8,0x9c, - 0xae,0x00,0x10,0x08,0x05,0xff,0xe9,0x84,0x9b,0x00,0x05,0xff,0xe9,0x88,0xb8,0x00, - 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe9,0x8b,0x97,0x00,0x05,0xff,0xe9,0x8b, - 0x98,0x00,0x10,0x08,0x05,0xff,0xe9,0x89,0xbc,0x00,0x05,0xff,0xe9,0x8f,0xb9,0x00, - 0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x90,0x95,0x00,0x05,0xff,0xf0,0xa8,0xaf,0xba, - 0x00,0x10,0x08,0x05,0xff,0xe9,0x96,0x8b,0x00,0x05,0xff,0xe4,0xa6,0x95,0x00,0xd3, - 0x43,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x96,0xb7,0x00,0x05,0xff,0xf0, - 0xa8,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe4,0xa7,0xa6,0x00,0x05,0xff,0xe9,0x9b, - 0x83,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb6,0xb2,0x00,0x05,0xff,0xe9,0x9c, - 0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa9,0x85,0x85,0x00,0x05,0xff,0xf0,0xa9,0x88, - 0x9a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xa9,0xae,0x00,0x05,0xff, - 0xe4,0xa9,0xb6,0x00,0x10,0x08,0x05,0xff,0xe9,0x9f,0xa0,0x00,0x05,0xff,0xf0,0xa9, - 0x90,0x8a,0x00,0x91,0x11,0x10,0x08,0x05,0xff,0xe4,0xaa,0xb2,0x00,0x05,0xff,0xf0, - 0xa9,0x92,0x96,0x00,0x05,0xff,0xe9,0xa0,0x8b,0x00,0xe2,0x10,0x01,0xe1,0x09,0x01, - 0xe0,0x02,0x01,0xcf,0x86,0x95,0xfb,0xd4,0x82,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10, - 0x08,0x05,0xff,0xe9,0xa0,0xa9,0x00,0x05,0xff,0xf0,0xa9,0x96,0xb6,0x00,0x10,0x08, - 0x05,0xff,0xe9,0xa3,0xa2,0x00,0x05,0xff,0xe4,0xac,0xb3,0x00,0xd1,0x10,0x10,0x08, - 0x05,0xff,0xe9,0xa4,0xa9,0x00,0x05,0xff,0xe9,0xa6,0xa7,0x00,0x10,0x08,0x05,0xff, - 0xe9,0xa7,0x82,0x00,0x05,0xff,0xe9,0xa7,0xbe,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, - 0x05,0xff,0xe4,0xaf,0x8e,0x00,0x05,0xff,0xf0,0xa9,0xac,0xb0,0x00,0x10,0x08,0x05, - 0xff,0xe9,0xac,0x92,0x00,0x05,0xff,0xe9,0xb1,0x80,0x00,0xd1,0x10,0x10,0x08,0x05, - 0xff,0xe9,0xb3,0xbd,0x00,0x05,0xff,0xe4,0xb3,0x8e,0x00,0x10,0x08,0x05,0xff,0xe4, - 0xb3,0xad,0x00,0x05,0xff,0xe9,0xb5,0xa7,0x00,0xd3,0x44,0xd2,0x23,0xd1,0x11,0x10, - 0x09,0x05,0xff,0xf0,0xaa,0x83,0x8e,0x00,0x05,0xff,0xe4,0xb3,0xb8,0x00,0x10,0x09, - 0x05,0xff,0xf0,0xaa,0x84,0x85,0x00,0x05,0xff,0xf0,0xaa,0x88,0x8e,0x00,0xd1,0x11, - 0x10,0x09,0x05,0xff,0xf0,0xaa,0x8a,0x91,0x00,0x05,0xff,0xe9,0xba,0xbb,0x00,0x10, - 0x08,0x05,0xff,0xe4,0xb5,0x96,0x00,0x05,0xff,0xe9,0xbb,0xb9,0x00,0xd2,0x20,0xd1, - 0x10,0x10,0x08,0x05,0xff,0xe9,0xbb,0xbe,0x00,0x05,0xff,0xe9,0xbc,0x85,0x00,0x10, - 0x08,0x05,0xff,0xe9,0xbc,0x8f,0x00,0x05,0xff,0xe9,0xbc,0x96,0x00,0x91,0x11,0x10, - 0x08,0x05,0xff,0xe9,0xbc,0xbb,0x00,0x05,0xff,0xf0,0xaa,0x98,0x80,0x00,0x00,0x00, - 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06, - 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, - 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, - 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, - 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, - 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, - 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, - 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, - 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, - 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xd5,0xc0, - 0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06, - 0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06, - 0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00, - 0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06, - 0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, - 0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00, - 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, - 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, - 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, - 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd4,0x60, - 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, - 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, - 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, - 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, - 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, - 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, - 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, - 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, - 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, - 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, - 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xe0,0x83,0x01,0xcf, - 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, - 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, - 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, - 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, - 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, - 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, - 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, - 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, - 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, - 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, - 0x00,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, - 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, - 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, - 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, - 0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00, - 0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00, - 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3, - 0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00, - 0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00, - 0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf, - 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, - 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, - 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, - 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, - 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, - 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, - 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, - 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, - 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, - 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, - 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, - 0x00,0xd4,0xd9,0xd3,0x81,0xd2,0x79,0xd1,0x71,0xd0,0x69,0xcf,0x86,0xd5,0x60,0xd4, - 0x59,0xd3,0x52,0xd2,0x33,0xd1,0x2c,0xd0,0x25,0xcf,0x86,0x95,0x1e,0x94,0x19,0x93, - 0x14,0x92,0x0f,0x91,0x0a,0x10,0x05,0x00,0xff,0x00,0x05,0xff,0x00,0x00,0xff,0x00, - 0x00,0xff,0x00,0x00,0xff,0x00,0x00,0xff,0x00,0x05,0xff,0x00,0xcf,0x06,0x05,0xff, - 0x00,0xcf,0x06,0x00,0xff,0x00,0xd1,0x07,0xcf,0x06,0x07,0xff,0x00,0xd0,0x07,0xcf, - 0x06,0x07,0xff,0x00,0xcf,0x86,0x55,0x05,0x07,0xff,0x00,0x14,0x05,0x07,0xff,0x00, - 0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00, - 0xff,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, - 0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86, - 0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06, - 0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00, - 0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06, - 0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00, - 0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xcf,0x06,0x02,0x00, - 0x81,0x80,0xcf,0x86,0x85,0x84,0xcf,0x86,0xcf,0x06,0x02,0x00,0x00,0x00,0x00,0x00 -}; diff --git a/fs/unicode/utf8n.h b/fs/unicode/utf8n.h index 206c89f0dbf7..bd00d587747a 100644 --- a/fs/unicode/utf8n.h +++ b/fs/unicode/utf8n.h @@ -13,25 +13,7 @@ #include #include -int utf8version_is_supported(unsigned int version); - -/* - * Look for the correct const struct utf8data for a unicode version. - * Returns NULL if the version requested is too new. - * - * Two normalization forms are supported: nfdi and nfdicf. - * - * nfdi: - * - Apply unicode normalization form NFD. - * - Remove any Default_Ignorable_Code_Point. - * - * nfdicf: - * - Apply unicode normalization form NFD. - * - Remove any Default_Ignorable_Code_Point. - * - Apply a full casefold (C + F). - */ -extern const struct utf8data *utf8nfdi(unsigned int maxage); -extern const struct utf8data *utf8nfdicf(unsigned int maxage); +int utf8version_is_supported(const struct unicode_map *um, unsigned int version); /* * Determine the length of the normalized from of the string, @@ -78,4 +60,24 @@ int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um, */ extern int utf8byte(struct utf8cursor *u8c); +struct utf8data { + unsigned int maxage; + unsigned int offset; +}; + +struct utf8data_table { + const unsigned int *utf8agetab; + int utf8agetab_size; + + const struct utf8data *utf8nfdicfdata; + int utf8nfdicfdata_size; + + const struct utf8data *utf8nfdidata; + int utf8nfdidata_size; + + const unsigned char *utf8data; +}; + +extern struct utf8data_table utf8_data_table; + #endif /* UTF8NORM_H */ diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 526ca8b8391a..4d39e6e11a95 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -6,6 +6,7 @@ #include struct utf8data; +struct utf8data_table; #define UNICODE_MAJ_SHIFT 16 #define UNICODE_MIN_SHIFT 8 @@ -49,6 +50,7 @@ enum utf8_normalization { struct unicode_map { unsigned int version; const struct utf8data *ntab[UTF8_NMAX]; + const struct utf8data_table *tables; }; int utf8_validate(const struct unicode_map *um, const struct qstr *str); -- cgit v1.2.3-59-g8ed1b From dd66f56caea6bb1a3703fb3bfc3106444d05a930 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 21 Oct 2021 08:55:24 +0200 Subject: dma-buf: fix kerneldoc for renamed members MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Those members where renamed, update the kerneldoc as well. Signed-off-by: Christian König Reviewed-by: Alex Deucher Acked-by: Sumit Semwal Link: https://patchwork.freedesktop.org/patch/msgid/20211021141945.84023-1-christian.koenig@amd.com --- include/linux/dma-buf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 02c2eb874da6..9807aef33685 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -433,8 +433,8 @@ struct dma_buf { /** @poll: for userspace poll support */ wait_queue_head_t poll; - /** @cb_excl: for userspace poll support */ - /** @cb_shared: for userspace poll support */ + /** @cb_in: for userspace poll support */ + /** @cb_out: for userspace poll support */ struct dma_buf_poll_cb_t { struct dma_fence_cb cb; wait_queue_head_t *poll; -- cgit v1.2.3-59-g8ed1b From 1705643faecde95bdeb11bea5ab5baed084e9f91 Mon Sep 17 00:00:00 2001 From: Sean Wang Date: Tue, 19 Oct 2021 05:30:20 +0800 Subject: mmc: add MT7921 SDIO identifiers for MediaTek Bluetooth devices The MT7961 SDIO identifier for MediaTek Bluetooth devices were being referred in the MediaTek Bluetooth driver. Co-developed-by: Mark-yw Chen Signed-off-by: Mark-yw Chen Signed-off-by: Sean Wang Signed-off-by: Marcel Holtmann --- include/linux/mmc/sdio_ids.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index a85c9f0bd470..53f0efa0bccf 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -105,6 +105,7 @@ #define SDIO_VENDOR_ID_MEDIATEK 0x037a #define SDIO_DEVICE_ID_MEDIATEK_MT7663 0x7663 #define SDIO_DEVICE_ID_MEDIATEK_MT7668 0x7668 +#define SDIO_DEVICE_ID_MEDIATEK_MT7961 0x7961 #define SDIO_VENDOR_ID_MICROCHIP_WILC 0x0296 #define SDIO_DEVICE_ID_MICROCHIP_WILC1000 0x5347 -- cgit v1.2.3-59-g8ed1b From 3ab7b6ac5d829e60c3b89d415811ff1c9f358c8e Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Mon, 25 Oct 2021 10:09:23 -0700 Subject: pwm: Introduce single-PWM of_xlate function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing pxa driver and the upcoming addition of PWM support in the TI sn565dsi86 DSI/eDP bridge driver both has a single PWM channel and thereby a need for a of_xlate function with the period as its single argument. Introduce a common helper function in the core that can be used as of_xlate by such drivers and migrate the pxa driver to use this. Signed-off-by: Bjorn Andersson Acked-by: Uwe Kleine-König Tested-by: Steev Klimaszewski Tested-By: Steev Klimaszewski Signed-off-by: Robert Foss Link: https://patchwork.freedesktop.org/patch/msgid/20211025170925.3096444-1-bjorn.andersson@linaro.org --- drivers/pwm/core.c | 26 ++++++++++++++++++++++++++ drivers/pwm/pwm-pxa.c | 16 +--------------- include/linux/pwm.h | 2 ++ 3 files changed, 29 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 4527f09a5c50..2c6b155002a2 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -152,6 +152,32 @@ of_pwm_xlate_with_flags(struct pwm_chip *pc, const struct of_phandle_args *args) } EXPORT_SYMBOL_GPL(of_pwm_xlate_with_flags); +struct pwm_device * +of_pwm_single_xlate(struct pwm_chip *pc, const struct of_phandle_args *args) +{ + struct pwm_device *pwm; + + if (pc->of_pwm_n_cells < 1) + return ERR_PTR(-EINVAL); + + /* validate that one cell is specified, optionally with flags */ + if (args->args_count != 1 && args->args_count != 2) + return ERR_PTR(-EINVAL); + + pwm = pwm_request_from_chip(pc, 0, NULL); + if (IS_ERR(pwm)) + return pwm; + + pwm->args.period = args->args[0]; + pwm->args.polarity = PWM_POLARITY_NORMAL; + + if (args->args_count == 2 && args->args[2] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; + + return pwm; +} +EXPORT_SYMBOL_GPL(of_pwm_single_xlate); + static void of_pwmchip_add(struct pwm_chip *chip) { if (!chip->dev || !chip->dev->of_node) diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c index a9efdcf839ae..238ec88c130b 100644 --- a/drivers/pwm/pwm-pxa.c +++ b/drivers/pwm/pwm-pxa.c @@ -148,20 +148,6 @@ static const struct platform_device_id *pxa_pwm_get_id_dt(struct device *dev) return id ? id->data : NULL; } -static struct pwm_device * -pxa_pwm_of_xlate(struct pwm_chip *pc, const struct of_phandle_args *args) -{ - struct pwm_device *pwm; - - pwm = pwm_request_from_chip(pc, 0, NULL); - if (IS_ERR(pwm)) - return pwm; - - pwm->args.period = args->args[0]; - - return pwm; -} - static int pwm_probe(struct platform_device *pdev) { const struct platform_device_id *id = platform_get_device_id(pdev); @@ -187,7 +173,7 @@ static int pwm_probe(struct platform_device *pdev) pc->chip.npwm = (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1; if (IS_ENABLED(CONFIG_OF)) { - pc->chip.of_xlate = pxa_pwm_of_xlate; + pc->chip.of_xlate = of_pwm_single_xlate; pc->chip.of_pwm_n_cells = 1; } diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 725c9b784e60..dd51d4931fdc 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -414,6 +414,8 @@ struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, struct pwm_device *of_pwm_xlate_with_flags(struct pwm_chip *pc, const struct of_phandle_args *args); +struct pwm_device *of_pwm_single_xlate(struct pwm_chip *pc, + const struct of_phandle_args *args); struct pwm_device *pwm_get(struct device *dev, const char *con_id); struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np, -- cgit v1.2.3-59-g8ed1b From 7c7e3d31e7856a8260a254f8c71db416f7f9f5a1 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 5 Nov 2021 16:23:29 -0700 Subject: bpf: Introduce helper bpf_find_vma In some profiler use cases, it is necessary to map an address to the backing file, e.g., a shared library. bpf_find_vma helper provides a flexible way to achieve this. bpf_find_vma maps an address of a task to the vma (vm_area_struct) for this address, and feed the vma to an callback BPF function. The callback function is necessary here, as we need to ensure mmap_sem is unlocked. It is necessary to lock mmap_sem for find_vma. To lock and unlock mmap_sem safely when irqs are disable, we use the same mechanism as stackmap with build_id. Specifically, when irqs are disabled, the unlocked is postponed in an irq_work. Refactor stackmap.c so that the irq_work is shared among bpf_find_vma and stackmap helpers. Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Tested-by: Hengqi Chen Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211105232330.1936330-2-songliubraving@fb.com --- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 20 +++++++++++ kernel/bpf/btf.c | 5 ++- kernel/bpf/mmap_unlock_work.h | 65 ++++++++++++++++++++++++++++++++++ kernel/bpf/stackmap.c | 80 ++++-------------------------------------- kernel/bpf/task_iter.c | 76 +++++++++++++++++++++++++++++++++++---- kernel/bpf/verifier.c | 34 ++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 20 +++++++++++ 9 files changed, 222 insertions(+), 81 deletions(-) create mode 100644 kernel/bpf/mmap_unlock_work.h (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2be6dfd68df9..df3410bff4b0 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2157,6 +2157,7 @@ extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto; +extern const struct bpf_func_proto bpf_find_vma_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index ba5af15e25f5..509eee5f0393 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4938,6 +4938,25 @@ union bpf_attr { * **-ENOENT** if symbol is not found. * * **-EPERM** if caller does not have permission to obtain kernel address. + * + * long bpf_find_vma(struct task_struct *task, u64 addr, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * Find vma of *task* that contains *addr*, call *callback_fn* + * function with *task*, *vma*, and *callback_ctx*. + * The *callback_fn* should be a static function and + * the *callback_ctx* should be a pointer to the stack. + * The *flags* is used to control certain aspects of the helper. + * Currently, the *flags* must be 0. + * + * The expected callback signature is + * + * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); + * + * Return + * 0 on success. + * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. + * **-EBUSY** if failed to try lock mmap_lock. + * **-EINVAL** for invalid **flags**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5120,6 +5139,7 @@ union bpf_attr { FN(trace_vprintk), \ FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ + FN(find_vma), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index dbc3ad07e21b..cdb0fba65600 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6342,7 +6342,10 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .arg4_type = ARG_ANYTHING, }; -BTF_ID_LIST_GLOBAL_SINGLE(btf_task_struct_ids, struct, task_struct) +BTF_ID_LIST_GLOBAL(btf_task_struct_ids) +BTF_ID(struct, task_struct) +BTF_ID(struct, file) +BTF_ID(struct, vm_area_struct) /* BTF ID set registration API for modules */ diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h new file mode 100644 index 000000000000..5d18d7d85bef --- /dev/null +++ b/kernel/bpf/mmap_unlock_work.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2021 Facebook + */ + +#ifndef __MMAP_UNLOCK_WORK_H__ +#define __MMAP_UNLOCK_WORK_H__ +#include + +/* irq_work to run mmap_read_unlock() in irq_work */ +struct mmap_unlock_irq_work { + struct irq_work irq_work; + struct mm_struct *mm; +}; + +DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); + +/* + * We cannot do mmap_read_unlock() when the irq is disabled, because of + * risk to deadlock with rq_lock. To look up vma when the irqs are + * disabled, we need to run mmap_read_unlock() in irq_work. We use a + * percpu variable to do the irq_work. If the irq_work is already used + * by another lookup, we fall over. + */ +static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +{ + struct mmap_unlock_irq_work *work = NULL; + bool irq_work_busy = false; + + if (irqs_disabled()) { + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work)) { + /* cannot queue more up_read, fallback */ + irq_work_busy = true; + } + } else { + /* + * PREEMPT_RT does not allow to trylock mmap sem in + * interrupt disabled context. Force the fallback code. + */ + irq_work_busy = true; + } + } + + *work_ptr = work; + return irq_work_busy; +} + +static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) +{ + if (!work) { + mmap_read_unlock(mm); + } else { + work->mm = mm; + + /* The lock will be released once we're out of interrupt + * context. Tell lockdep that we've released it now so + * it doesn't complain that we forgot to release it. + */ + rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_); + irq_work_queue(&work->irq_work); + } +} + +#endif /* __MMAP_UNLOCK_WORK_H__ */ diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 6e75bbee39f0..1de0a1b03636 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -7,10 +7,10 @@ #include #include #include -#include #include #include #include "percpu_freelist.h" +#include "mmap_unlock_work.h" #define STACK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \ @@ -31,25 +31,6 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[]; }; -/* irq_work to run up_read() for build_id lookup in nmi context */ -struct stack_map_irq_work { - struct irq_work irq_work; - struct mm_struct *mm; -}; - -static void do_up_read(struct irq_work *entry) -{ - struct stack_map_irq_work *work; - - if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) - return; - - work = container_of(entry, struct stack_map_irq_work, irq_work); - mmap_read_unlock_non_owner(work->mm); -} - -static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work); - static inline bool stack_map_use_build_id(struct bpf_map *map) { return (map->map_flags & BPF_F_STACK_BUILD_ID); @@ -149,35 +130,13 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u64 *ips, u32 trace_nr, bool user) { int i; + struct mmap_unlock_irq_work *work = NULL; + bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); struct vm_area_struct *vma; - bool irq_work_busy = false; - struct stack_map_irq_work *work = NULL; - - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&up_read_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } - /* - * We cannot do up_read() when the irq is disabled, because of - * risk to deadlock with rq_lock. To do build_id lookup when the - * irqs are disabled, we need to run up_read() in irq_work. We use - * a percpu variable to do the irq_work. If the irq_work is - * already used by another lookup, we fall back to report ips. - * - * Same fallback is used for kernel stack (!user) on a stackmap - * with build_id. + /* If the irq_work is in use, fall back to report ips. Same + * fallback is used for kernel stack (!user) on a stackmap with + * build_id. */ if (!user || !current || !current->mm || irq_work_busy || !mmap_read_trylock(current->mm)) { @@ -203,19 +162,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, - vma->vm_start; id_offs[i].status = BPF_STACK_BUILD_ID_VALID; } - - if (!work) { - mmap_read_unlock(current->mm); - } else { - work->mm = current->mm; - - /* The lock will be released once we're out of interrupt - * context. Tell lockdep that we've released it now so - * it doesn't complain that we forgot to release it. - */ - rwsem_release(¤t->mm->mmap_lock.dep_map, _RET_IP_); - irq_work_queue(&work->irq_work); - } + bpf_mmap_unlock_mm(work, current->mm); } static struct perf_callchain_entry * @@ -719,16 +666,3 @@ const struct bpf_map_ops stack_trace_map_ops = { .map_btf_name = "bpf_stack_map", .map_btf_id = &stack_trace_map_btf_id, }; - -static int __init stack_map_init(void) -{ - int cpu; - struct stack_map_irq_work *work; - - for_each_possible_cpu(cpu) { - work = per_cpu_ptr(&up_read_work, cpu); - init_irq_work(&work->irq_work, do_up_read); - } - return 0; -} -subsys_initcall(stack_map_init); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index b48750bfba5a..f171479f7dd6 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -8,6 +8,7 @@ #include #include #include +#include "mmap_unlock_work.h" struct bpf_iter_seq_task_common { struct pid_namespace *ns; @@ -524,10 +525,6 @@ static const struct seq_operations task_vma_seq_ops = { .show = task_vma_seq_show, }; -BTF_ID_LIST(btf_task_file_ids) -BTF_ID(struct, file) -BTF_ID(struct, vm_area_struct) - static const struct bpf_iter_seq_info task_seq_info = { .seq_ops = &task_seq_ops, .init_seq_private = init_seq_pidns, @@ -586,9 +583,74 @@ static struct bpf_iter_reg task_vma_reg_info = { .seq_info = &task_vma_seq_info, }; +BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, + bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) +{ + struct mmap_unlock_irq_work *work = NULL; + struct vm_area_struct *vma; + bool irq_work_busy = false; + struct mm_struct *mm; + int ret = -ENOENT; + + if (flags) + return -EINVAL; + + if (!task) + return -ENOENT; + + mm = task->mm; + if (!mm) + return -ENOENT; + + irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + + if (irq_work_busy || !mmap_read_trylock(mm)) + return -EBUSY; + + vma = find_vma(mm, start); + + if (vma && vma->vm_start <= start && vma->vm_end > start) { + callback_fn((u64)(long)task, (u64)(long)vma, + (u64)(long)callback_ctx, 0, 0); + ret = 0; + } + bpf_mmap_unlock_mm(work, mm); + return ret; +} + +const struct bpf_func_proto bpf_find_vma_proto = { + .func = bpf_find_vma, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID, + .arg1_btf_id = &btf_task_struct_ids[0], + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_FUNC, + .arg4_type = ARG_PTR_TO_STACK_OR_NULL, + .arg5_type = ARG_ANYTHING, +}; + +DEFINE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); + +static void do_mmap_read_unlock(struct irq_work *entry) +{ + struct mmap_unlock_irq_work *work; + + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return; + + work = container_of(entry, struct mmap_unlock_irq_work, irq_work); + mmap_read_unlock_non_owner(work->mm); +} + static int __init task_iter_init(void) { - int ret; + struct mmap_unlock_irq_work *work; + int ret, cpu; + + for_each_possible_cpu(cpu) { + work = per_cpu_ptr(&mmap_unlock_work, cpu); + init_irq_work(&work->irq_work, do_mmap_read_unlock); + } task_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; ret = bpf_iter_reg_target(&task_reg_info); @@ -596,13 +658,13 @@ static int __init task_iter_init(void) return ret; task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[0]; + task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_struct_ids[1]; ret = bpf_iter_reg_target(&task_file_reg_info); if (ret) return ret; task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1]; + task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_struct_ids[2]; return bpf_iter_reg_target(&task_vma_reg_info); } late_initcall(task_iter_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f0dca726ebfd..1aafb43f61d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6132,6 +6132,33 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_find_vma_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* bpf_find_vma(struct task_struct *task, u64 addr, + * void *callback_fn, void *callback_ctx, u64 flags) + * (callback_fn)(struct task_struct *task, + * struct vm_area_struct *vma, void *callback_ctx); + */ + callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; + + callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].btf = btf_vmlinux; + callee->regs[BPF_REG_2].btf_id = btf_task_struct_ids[2]; + + /* pointer to stack or null */ + callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_callback_fn = true; + return 0; +} + static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; @@ -6489,6 +6516,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } + if (func_id == BPF_FUNC_find_vma) { + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_find_vma_callback_state); + if (err < 0) + return -EINVAL; + } + if (func_id == BPF_FUNC_snprintf) { err = check_bpf_snprintf_call(env, regs); if (err < 0) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 7396488793ff..390176a3031a 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1208,6 +1208,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_func_ip_proto_tracing; case BPF_FUNC_get_branch_snapshot: return &bpf_get_branch_snapshot_proto; + case BPF_FUNC_find_vma: + return &bpf_find_vma_proto; case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index ba5af15e25f5..509eee5f0393 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4938,6 +4938,25 @@ union bpf_attr { * **-ENOENT** if symbol is not found. * * **-EPERM** if caller does not have permission to obtain kernel address. + * + * long bpf_find_vma(struct task_struct *task, u64 addr, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * Find vma of *task* that contains *addr*, call *callback_fn* + * function with *task*, *vma*, and *callback_ctx*. + * The *callback_fn* should be a static function and + * the *callback_ctx* should be a pointer to the stack. + * The *flags* is used to control certain aspects of the helper. + * Currently, the *flags* must be 0. + * + * The expected callback signature is + * + * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); + * + * Return + * 0 on success. + * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. + * **-EBUSY** if failed to try lock mmap_lock. + * **-EINVAL** for invalid **flags**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5120,6 +5139,7 @@ union bpf_attr { FN(trace_vprintk), \ FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ + FN(find_vma), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3-59-g8ed1b From f89315650ba34ec6c91a8bded72796980bee2a4d Mon Sep 17 00:00:00 2001 From: Mark Pashmfouroush Date: Wed, 10 Nov 2021 11:10:15 +0000 Subject: bpf: Add ingress_ifindex to bpf_sk_lookup It may be helpful to have access to the ifindex during bpf socket lookup. An example may be to scope certain socket lookup logic to specific interfaces, i.e. an interface may be made exempt from custom lookup code. Add the ifindex of the arriving connection to the bpf_sk_lookup API. Signed-off-by: Mark Pashmfouroush Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211110111016.5670-2-markpash@cloudflare.com --- include/linux/filter.h | 7 +++++-- include/uapi/linux/bpf.h | 1 + net/core/filter.c | 7 +++++++ net/ipv4/inet_hashtables.c | 8 ++++---- net/ipv4/udp.c | 8 ++++---- net/ipv6/inet6_hashtables.c | 8 ++++---- net/ipv6/udp.c | 8 ++++---- tools/include/uapi/linux/bpf.h | 1 + 8 files changed, 30 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 24b7ed2677af..b6a216eb217a 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1374,6 +1374,7 @@ struct bpf_sk_lookup_kern { const struct in6_addr *daddr; } v6; struct sock *selected_sk; + u32 ingress_ifindex; bool no_reuseport; }; @@ -1436,7 +1437,7 @@ extern struct static_key_false bpf_sk_lookup_enabled; static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol, const __be32 saddr, const __be16 sport, const __be32 daddr, const u16 dport, - struct sock **psk) + const int ifindex, struct sock **psk) { struct bpf_prog_array *run_array; struct sock *selected_sk = NULL; @@ -1452,6 +1453,7 @@ static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol, .v4.daddr = daddr, .sport = sport, .dport = dport, + .ingress_ifindex = ifindex, }; u32 act; @@ -1474,7 +1476,7 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol, const __be16 sport, const struct in6_addr *daddr, const u16 dport, - struct sock **psk) + const int ifindex, struct sock **psk) { struct bpf_prog_array *run_array; struct sock *selected_sk = NULL; @@ -1490,6 +1492,7 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol, .v6.daddr = daddr, .sport = sport, .dport = dport, + .ingress_ifindex = ifindex, }; u32 act; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 509eee5f0393..6297eafdc40f 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6316,6 +6316,7 @@ struct bpf_sk_lookup { __u32 local_ip4; /* Network byte order */ __u32 local_ip6[4]; /* Network byte order */ __u32 local_port; /* Host byte order */ + __u32 ingress_ifindex; /* The arriving interface. Determined by inet_iif. */ }; /* diff --git a/net/core/filter.c b/net/core/filter.c index 8e8d3b49c297..315a58466fc9 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -10491,6 +10491,7 @@ static bool sk_lookup_is_valid_access(int off, int size, case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]): case bpf_ctx_range(struct bpf_sk_lookup, remote_port): case bpf_ctx_range(struct bpf_sk_lookup, local_port): + case bpf_ctx_range(struct bpf_sk_lookup, ingress_ifindex): bpf_ctx_record_field_size(info, sizeof(__u32)); return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32)); @@ -10580,6 +10581,12 @@ static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type, bpf_target_off(struct bpf_sk_lookup_kern, dport, 2, target_size)); break; + + case offsetof(struct bpf_sk_lookup, ingress_ifindex): + *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, + bpf_target_off(struct bpf_sk_lookup_kern, + ingress_ifindex, 4, target_size)); + break; } return insn - insn_buf; diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 75737267746f..30ab717ff1b8 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -307,7 +307,7 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net, struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, __be32 saddr, __be16 sport, - __be32 daddr, u16 hnum) + __be32 daddr, u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -315,8 +315,8 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net, if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ - no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -340,7 +340,7 @@ struct sock *__inet_lookup_listener(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet_lookup_run_bpf(net, hashinfo, skb, doff, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (result) goto done; } diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 2fffcf2b54f3..5fceee3de65d 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -460,7 +460,7 @@ static struct sock *udp4_lookup_run_bpf(struct net *net, struct udp_table *udptable, struct sk_buff *skb, __be32 saddr, __be16 sport, - __be32 daddr, u16 hnum) + __be32 daddr, u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -468,8 +468,8 @@ static struct sock *udp4_lookup_run_bpf(struct net *net, if (udptable != &udp_table) return NULL; /* only UDP is supported */ - no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -505,7 +505,7 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { sk = udp4_lookup_run_bpf(net, udptable, skb, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (sk) { result = sk; goto done; diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 67c9114835c8..4514444e96c8 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -165,7 +165,7 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, - const u16 hnum) + const u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -173,8 +173,8 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net, if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ - no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -198,7 +198,7 @@ struct sock *inet6_lookup_listener(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet6_lookup_run_bpf(net, hashinfo, skb, doff, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (result) goto done; } diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 12c12619ee35..ea4ea525f94a 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -195,7 +195,7 @@ static inline struct sock *udp6_lookup_run_bpf(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, - u16 hnum) + u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -203,8 +203,8 @@ static inline struct sock *udp6_lookup_run_bpf(struct net *net, if (udptable != &udp_table) return NULL; /* only UDP is supported */ - no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -240,7 +240,7 @@ struct sock *__udp6_lib_lookup(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { sk = udp6_lookup_run_bpf(net, udptable, skb, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (sk) { result = sk; goto done; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 509eee5f0393..6297eafdc40f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -6316,6 +6316,7 @@ struct bpf_sk_lookup { __u32 local_ip4; /* Network byte order */ __u32 local_ip6[4]; /* Network byte order */ __u32 local_port; /* Host byte order */ + __u32 ingress_ifindex; /* The arriving interface. Determined by inet_iif. */ }; /* -- cgit v1.2.3-59-g8ed1b From a25efb3863d068929f0bbeb87a995df11507e691 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 23 Sep 2021 13:57:42 +0200 Subject: dma-buf: add dma_fence_describe and dma_resv_describe v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add functions to dump dma_fence and dma_resv objects into a seq_file and use them for printing the debugfs information. v2: fix missing include reported by test robot. Signed-off-by: Christian König Reviewed-by: Rob Clark Link: https://patchwork.freedesktop.org/patch/msgid/20211103081231.18578-2-christian.koenig@amd.com --- drivers/dma-buf/dma-buf.c | 11 +---------- drivers/dma-buf/dma-fence.c | 17 +++++++++++++++++ drivers/dma-buf/dma-resv.c | 23 +++++++++++++++++++++++ include/linux/dma-fence.h | 1 + include/linux/dma-resv.h | 1 + 5 files changed, 43 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 3f63d58bf68a..385cd037325e 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -1321,8 +1321,6 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) { struct dma_buf *buf_obj; struct dma_buf_attachment *attach_obj; - struct dma_resv_iter cursor; - struct dma_fence *fence; int count = 0, attach_count; size_t size = 0; int ret; @@ -1350,14 +1348,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) file_inode(buf_obj->file)->i_ino, buf_obj->name ?: ""); - dma_resv_for_each_fence(&cursor, buf_obj->resv, true, fence) { - seq_printf(s, "\t%s fence: %s %s %ssignalled\n", - dma_resv_iter_is_exclusive(&cursor) ? - "Exclusive" : "Shared", - fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), - dma_fence_is_signaled(fence) ? "" : "un"); - } + dma_resv_describe(buf_obj->resv, s); seq_puts(s, "\tAttached Devices:\n"); attach_count = 0; diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 1e82ecd443fa..066400ed8841 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -15,6 +15,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -907,6 +908,22 @@ err_free_cb: } EXPORT_SYMBOL(dma_fence_wait_any_timeout); +/** + * dma_fence_describe - Dump fence describtion into seq_file + * @fence: the 6fence to describe + * @seq: the seq_file to put the textual description into + * + * Dump a textual description of the fence and it's state into the seq_file. + */ +void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq) +{ + seq_printf(seq, "%s %s seq %llu %ssignalled\n", + fence->ops->get_driver_name(fence), + fence->ops->get_timeline_name(fence), fence->seqno, + dma_fence_is_signaled(fence) ? "" : "un"); +} +EXPORT_SYMBOL(dma_fence_describe); + /** * dma_fence_init - Initialize a custom fence. * @fence: the fence to initialize diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c index 9eb2baa387d4..ff3c0558b3b8 100644 --- a/drivers/dma-buf/dma-resv.c +++ b/drivers/dma-buf/dma-resv.c @@ -38,6 +38,7 @@ #include #include #include +#include /** * DOC: Reservation Object Overview @@ -666,6 +667,28 @@ bool dma_resv_test_signaled(struct dma_resv *obj, bool test_all) } EXPORT_SYMBOL_GPL(dma_resv_test_signaled); +/** + * dma_resv_describe - Dump description of the resv object into seq_file + * @obj: the reservation object + * @seq: the seq_file to dump the description into + * + * Dump a textual description of the fences inside an dma_resv object into the + * seq_file. + */ +void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_for_each_fence(&cursor, obj, true, fence) { + seq_printf(seq, "\t%s fence:", + dma_resv_iter_is_exclusive(&cursor) ? + "Exclusive" : "Shared"); + dma_fence_describe(fence, seq); + } +} +EXPORT_SYMBOL_GPL(dma_resv_describe); + #if IS_ENABLED(CONFIG_LOCKDEP) static int __init dma_resv_lockdep(void) { diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index a706b7bf51d7..1ea691753bd3 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -264,6 +264,7 @@ void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, void dma_fence_release(struct kref *kref); void dma_fence_free(struct dma_fence *fence); +void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq); /** * dma_fence_put - decreases refcount of the fence diff --git a/include/linux/dma-resv.h b/include/linux/dma-resv.h index dbd235ab447f..09c6063b199a 100644 --- a/include/linux/dma-resv.h +++ b/include/linux/dma-resv.h @@ -490,5 +490,6 @@ int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src); long dma_resv_wait_timeout(struct dma_resv *obj, bool wait_all, bool intr, unsigned long timeout); bool dma_resv_test_signaled(struct dma_resv *obj, bool test_all); +void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq); #endif /* _LINUX_RESERVATION_H */ -- cgit v1.2.3-59-g8ed1b From 9e2ad638ae3632ef916ceb39f70e3104bf8fdc97 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 12 Nov 2021 07:02:42 -0800 Subject: bpf: Extend BTF_ID_LIST_GLOBAL with parameter for number of IDs syzbot reported the following BUG w/o CONFIG_DEBUG_INFO_BTF BUG: KASAN: global-out-of-bounds in task_iter_init+0x212/0x2e7 kernel/bpf/task_iter.c:661 Read of size 4 at addr ffffffff90297404 by task swapper/0/1 CPU: 1 PID: 1 Comm: swapper/0 Not tainted 5.15.0-syzkaller #0 Hardware name: ... Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106 print_address_description.constprop.0.cold+0xf/0x309 mm/kasan/report.c:256 __kasan_report mm/kasan/report.c:442 [inline] kasan_report.cold+0x83/0xdf mm/kasan/report.c:459 task_iter_init+0x212/0x2e7 kernel/bpf/task_iter.c:661 do_one_initcall+0x103/0x650 init/main.c:1295 do_initcall_level init/main.c:1368 [inline] do_initcalls init/main.c:1384 [inline] do_basic_setup init/main.c:1403 [inline] kernel_init_freeable+0x6b1/0x73a init/main.c:1606 kernel_init+0x1a/0x1d0 init/main.c:1497 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 This is caused by hard-coded name[1] in BTF_ID_LIST_GLOBAL (w/o CONFIG_DEBUG_INFO_BTF). Fix this by adding a parameter n to BTF_ID_LIST_GLOBAL. This avoids ifdef CONFIG_DEBUG_INFO_BTF in btf.c and filter.c. Fixes: 7c7e3d31e785 ("bpf: Introduce helper bpf_find_vma") Reported-by: syzbot+e0d81ec552a21d9071aa@syzkaller.appspotmail.com Reported-by: Eric Dumazet Suggested-by: Eric Dumazet Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211112150243.1270987-2-songliubraving@fb.com --- include/linux/btf_ids.h | 6 +++--- kernel/bpf/btf.c | 2 +- net/core/filter.c | 6 +----- 3 files changed, 5 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 47d9abfbdb55..6bb42b785293 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -73,7 +73,7 @@ asm( \ __BTF_ID_LIST(name, local) \ extern u32 name[]; -#define BTF_ID_LIST_GLOBAL(name) \ +#define BTF_ID_LIST_GLOBAL(name, n) \ __BTF_ID_LIST(name, globl) /* The BTF_ID_LIST_SINGLE macro defines a BTF_ID_LIST with @@ -83,7 +83,7 @@ __BTF_ID_LIST(name, globl) BTF_ID_LIST(name) \ BTF_ID(prefix, typename) #define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) \ - BTF_ID_LIST_GLOBAL(name) \ + BTF_ID_LIST_GLOBAL(name, 1) \ BTF_ID(prefix, typename) /* @@ -149,7 +149,7 @@ extern struct btf_id_set name; #define BTF_ID_LIST(name) static u32 name[5]; #define BTF_ID(prefix, name) #define BTF_ID_UNUSED -#define BTF_ID_LIST_GLOBAL(name) u32 name[1]; +#define BTF_ID_LIST_GLOBAL(name, n) u32 name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 name[1]; #define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) u32 name[1]; #define BTF_SET_START(name) static struct btf_id_set name = { 0 }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 1dd9ba82da1e..2a9d8a1fee1d 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6354,7 +6354,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .arg4_type = ARG_ANYTHING, }; -BTF_ID_LIST_GLOBAL(btf_task_struct_ids) +BTF_ID_LIST_GLOBAL(btf_task_struct_ids, 3) BTF_ID(struct, task_struct) BTF_ID(struct, file) BTF_ID(struct, vm_area_struct) diff --git a/net/core/filter.c b/net/core/filter.c index 315a58466fc9..46f09a8fba20 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -10611,14 +10611,10 @@ void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog) bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), prev_prog, prog); } -#ifdef CONFIG_DEBUG_INFO_BTF -BTF_ID_LIST_GLOBAL(btf_sock_ids) +BTF_ID_LIST_GLOBAL(btf_sock_ids, MAX_BTF_SOCK_TYPE) #define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type) BTF_SOCK_TYPE_xxx #undef BTF_SOCK_TYPE -#else -u32 btf_sock_ids[MAX_BTF_SOCK_TYPE]; -#endif BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk) { -- cgit v1.2.3-59-g8ed1b From d19ddb476a539fd78ad1028ae13bb38506286931 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Fri, 12 Nov 2021 07:02:43 -0800 Subject: bpf: Introduce btf_tracing_ids Similar to btf_sock_ids, btf_tracing_ids provides btf ID for task_struct, file, and vm_area_struct via easy to understand format like btf_tracing_ids[BTF_TRACING_TYPE_[TASK|file|VMA]]. Suggested-by: Alexei Starovoitov Signed-off-by: Song Liu Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20211112150243.1270987-3-songliubraving@fb.com --- include/linux/btf_ids.h | 14 +++++++++++++- kernel/bpf/bpf_task_storage.c | 4 ++-- kernel/bpf/btf.c | 8 ++++---- kernel/bpf/stackmap.c | 2 +- kernel/bpf/task_iter.c | 12 ++++++------ kernel/bpf/verifier.c | 2 +- kernel/trace/bpf_trace.c | 4 ++-- 7 files changed, 29 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 6bb42b785293..919c0fde1c51 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -189,6 +189,18 @@ MAX_BTF_SOCK_TYPE, extern u32 btf_sock_ids[]; #endif -extern u32 btf_task_struct_ids[]; +#define BTF_TRACING_TYPE_xxx \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_TASK, task_struct) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_FILE, file) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_VMA, vm_area_struct) + +enum { +#define BTF_TRACING_TYPE(name, type) name, +BTF_TRACING_TYPE_xxx +#undef BTF_TRACING_TYPE +MAX_BTF_TRACING_TYPE, +}; + +extern u32 btf_tracing_ids[]; #endif diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c index ebfa8bc90892..bb69aea1a777 100644 --- a/kernel/bpf/bpf_task_storage.c +++ b/kernel/bpf/bpf_task_storage.c @@ -323,7 +323,7 @@ const struct bpf_func_proto bpf_task_storage_get_proto = { .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_task_struct_ids[0], + .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, .arg4_type = ARG_ANYTHING, }; @@ -334,5 +334,5 @@ const struct bpf_func_proto bpf_task_storage_delete_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_task_struct_ids[0], + .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 2a9d8a1fee1d..6b9d23be1e99 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6354,10 +6354,10 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .arg4_type = ARG_ANYTHING, }; -BTF_ID_LIST_GLOBAL(btf_task_struct_ids, 3) -BTF_ID(struct, task_struct) -BTF_ID(struct, file) -BTF_ID(struct, vm_area_struct) +BTF_ID_LIST_GLOBAL(btf_tracing_ids, MAX_BTF_TRACING_TYPE) +#define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type) +BTF_TRACING_TYPE_xxx +#undef BTF_TRACING_TYPE /* BTF ID set registration API for modules */ diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 1de0a1b03636..49e567209c6b 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -489,7 +489,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_task_struct_ids[0], + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index f171479f7dd6..d94696198ef8 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -622,7 +622,7 @@ const struct bpf_func_proto bpf_find_vma_proto = { .func = bpf_find_vma, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_task_struct_ids[0], + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_FUNC, .arg4_type = ARG_PTR_TO_STACK_OR_NULL, @@ -652,19 +652,19 @@ static int __init task_iter_init(void) init_irq_work(&work->irq_work, do_mmap_read_unlock); } - task_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; + task_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; ret = bpf_iter_reg_target(&task_reg_info); if (ret) return ret; - task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_struct_ids[1]; + task_file_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; + task_file_reg_info.ctx_arg_info[1].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_FILE]; ret = bpf_iter_reg_target(&task_file_reg_info); if (ret) return ret; - task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_struct_ids[2]; + task_vma_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; + task_vma_reg_info.ctx_arg_info[1].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA]; return bpf_iter_reg_target(&task_vma_reg_info); } late_initcall(task_iter_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1aafb43f61d1..d31a031ab377 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6147,7 +6147,7 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].btf = btf_vmlinux; - callee->regs[BPF_REG_2].btf_id = btf_task_struct_ids[2]; + callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], /* pointer to stack or null */ callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 390176a3031a..25ea521fb8f1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -764,7 +764,7 @@ const struct bpf_func_proto bpf_get_current_task_btf_proto = { .func = bpf_get_current_task_btf, .gpl_only = true, .ret_type = RET_PTR_TO_BTF_ID, - .ret_btf_id = &btf_task_struct_ids[0], + .ret_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], }; BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task) @@ -779,7 +779,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { .func = bpf_task_pt_regs, .gpl_only = true, .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_task_struct_ids[0], + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .ret_type = RET_PTR_TO_BTF_ID, .ret_btf_id = &bpf_task_pt_regs_ids[0], }; -- cgit v1.2.3-59-g8ed1b From 4c7924fb905b02323ff6d9d20f370892615dccfa Mon Sep 17 00:00:00 2001 From: Julien Massot Date: Fri, 22 Oct 2021 14:21:01 +0200 Subject: soc: renesas: rcar-rst: Add support to set rproc boot address R-Car Gen3 SoC series has a realtime processor, the boot address of this processor can be set thanks to CR7BAR register of the reset module. Export this function so that it's possible to set the boot address from a remoteproc driver. Also drop the __initdata qualifier on rcar_rst_base, since we will use this address later than init time. Signed-off-by: Julien Massot Link: https://lore.kernel.org/r/20211022122101.66998-1-julien.massot@iot.bzh Signed-off-by: Geert Uytterhoeven --- drivers/soc/renesas/rcar-rst.c | 43 +++++++++++++++++++++++++++++++++--- include/linux/soc/renesas/rcar-rst.h | 2 ++ 2 files changed, 42 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c index 8a1e402ea799..1ad54e0c32e1 100644 --- a/drivers/soc/renesas/rcar-rst.c +++ b/drivers/soc/renesas/rcar-rst.c @@ -13,15 +13,43 @@ #define WDTRSTCR_RESET 0xA55A0002 #define WDTRSTCR 0x0054 +#define CR7BAR 0x0070 +#define CR7BAREN BIT(4) +#define CR7BAR_MASK 0xFFFC0000 + +static void __iomem *rcar_rst_base; +static u32 saved_mode __initdata; +static int (*rcar_rst_set_rproc_boot_addr_func)(u64 boot_addr); + static int rcar_rst_enable_wdt_reset(void __iomem *base) { iowrite32(WDTRSTCR_RESET, base + WDTRSTCR); return 0; } +/* + * Most of the R-Car Gen3 SoCs have an ARM Realtime Core. + * Firmware boot address has to be set in CR7BAR before + * starting the realtime core. + * Boot address must be aligned on a 256k boundary. + */ +static int rcar_rst_set_gen3_rproc_boot_addr(u64 boot_addr) +{ + if (boot_addr & ~(u64)CR7BAR_MASK) { + pr_err("Invalid boot address got %llx\n", boot_addr); + return -EINVAL; + } + + iowrite32(boot_addr, rcar_rst_base + CR7BAR); + iowrite32(boot_addr | CR7BAREN, rcar_rst_base + CR7BAR); + + return 0; +} + struct rst_config { unsigned int modemr; /* Mode Monitoring Register Offset */ int (*configure)(void __iomem *base); /* Platform specific config */ + int (*set_rproc_boot_addr)(u64 boot_addr); }; static const struct rst_config rcar_rst_gen1 __initconst = { @@ -35,6 +63,7 @@ static const struct rst_config rcar_rst_gen2 __initconst = { static const struct rst_config rcar_rst_gen3 __initconst = { .modemr = 0x60, + .set_rproc_boot_addr = rcar_rst_set_gen3_rproc_boot_addr, }; static const struct rst_config rcar_rst_r8a779a0 __initconst = { @@ -76,9 +105,6 @@ static const struct of_device_id rcar_rst_matches[] __initconst = { { /* sentinel */ } }; -static void __iomem *rcar_rst_base __initdata; -static u32 saved_mode __initdata; - static int __init rcar_rst_init(void) { const struct of_device_id *match; @@ -100,6 +126,8 @@ static int __init rcar_rst_init(void) rcar_rst_base = base; cfg = match->data; + rcar_rst_set_rproc_boot_addr_func = cfg->set_rproc_boot_addr; + saved_mode = ioread32(base + cfg->modemr); if (cfg->configure) { error = cfg->configure(base); @@ -130,3 +158,12 @@ int __init rcar_rst_read_mode_pins(u32 *mode) *mode = saved_mode; return 0; } + +int rcar_rst_set_rproc_boot_addr(u64 boot_addr) +{ + if (!rcar_rst_set_rproc_boot_addr_func) + return -EIO; + + return rcar_rst_set_rproc_boot_addr_func(boot_addr); +} +EXPORT_SYMBOL_GPL(rcar_rst_set_rproc_boot_addr); diff --git a/include/linux/soc/renesas/rcar-rst.h b/include/linux/soc/renesas/rcar-rst.h index 7899a5b8c247..1f1fe8bfaa76 100644 --- a/include/linux/soc/renesas/rcar-rst.h +++ b/include/linux/soc/renesas/rcar-rst.h @@ -4,8 +4,10 @@ #ifdef CONFIG_RST_RCAR int rcar_rst_read_mode_pins(u32 *mode); +int rcar_rst_set_rproc_boot_addr(u64 boot_addr); #else static inline int rcar_rst_read_mode_pins(u32 *mode) { return -ENODEV; } +static inline int rcar_rst_set_rproc_boot_addr(u64 boot_addr) { return -ENODEV; } #endif #endif /* __LINUX_SOC_RENESAS_RCAR_RST_H__ */ -- cgit v1.2.3-59-g8ed1b From 032816fbbfafe3198bb5c71fbbe4e8e5be33b352 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Wed, 27 Oct 2021 14:45:07 +0100 Subject: pinctrl: pinconf-generic: Add support for "output-impedance-ohms" to be extracted from DT files Add "output-impedance-ohms" property to generic options used for DT parsing files. This enables drivers, which use generic pin configurations, to get the value passed to this property. Signed-off-by: Lad Prabhakar Reviewed-by: Biju Das Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20211027134509.5036-3-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Geert Uytterhoeven --- drivers/pinctrl/pinconf-generic.c | 2 ++ include/linux/pinctrl/pinconf-generic.h | 3 +++ 2 files changed, 5 insertions(+) (limited to 'include/linux') diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 22e8d4c4040e..f8edcc88ac01 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -46,6 +46,7 @@ static const struct pin_config_item conf_items[] = { PCONFDUMP(PIN_CONFIG_MODE_LOW_POWER, "pin low power", "mode", true), PCONFDUMP(PIN_CONFIG_OUTPUT_ENABLE, "output enabled", NULL, false), PCONFDUMP(PIN_CONFIG_OUTPUT, "pin output", "level", true), + PCONFDUMP(PIN_CONFIG_OUTPUT_IMPEDANCE_OHMS, "output impedance", "ohms", true), PCONFDUMP(PIN_CONFIG_POWER_SOURCE, "pin power source", "selector", true), PCONFDUMP(PIN_CONFIG_SLEEP_HARDWARE_STATE, "sleep hardware state", NULL, false), PCONFDUMP(PIN_CONFIG_SLEW_RATE, "slew rate", NULL, true), @@ -179,6 +180,7 @@ static const struct pinconf_generic_params dt_params[] = { { "output-disable", PIN_CONFIG_OUTPUT_ENABLE, 0 }, { "output-enable", PIN_CONFIG_OUTPUT_ENABLE, 1 }, { "output-high", PIN_CONFIG_OUTPUT, 1, }, + { "output-impedance-ohms", PIN_CONFIG_OUTPUT_IMPEDANCE_OHMS, 0 }, { "output-low", PIN_CONFIG_OUTPUT, 0, }, { "power-source", PIN_CONFIG_POWER_SOURCE, 0 }, { "sleep-hardware-state", PIN_CONFIG_SLEEP_HARDWARE_STATE, 0 }, diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index eee0e3948537..2422211d6a5a 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -91,6 +91,8 @@ struct pinctrl_map; * configuration (eg. the currently selected mux function) drive values on * the line. Use argument 1 to enable output mode, argument 0 to disable * it. + * @PIN_CONFIG_OUTPUT_IMPEDANCE_OHMS: this will configure the output impedance + * of the pin with the value passed as argument. The argument is in ohms. * @PIN_CONFIG_PERSIST_STATE: retain pin state across sleep or controller reset * @PIN_CONFIG_POWER_SOURCE: if the pin can select between different power * supplies, the argument to this parameter (on a custom format) tells @@ -129,6 +131,7 @@ enum pin_config_param { PIN_CONFIG_MODE_PWM, PIN_CONFIG_OUTPUT, PIN_CONFIG_OUTPUT_ENABLE, + PIN_CONFIG_OUTPUT_IMPEDANCE_OHMS, PIN_CONFIG_PERSIST_STATE, PIN_CONFIG_POWER_SOURCE, PIN_CONFIG_SKEW_DELAY, -- cgit v1.2.3-59-g8ed1b From 507805b83ff108473dba9d4909e41abd50cf07f5 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 10 Nov 2021 15:47:42 +0200 Subject: gpiolib: acpi: Remove never used devm_acpi_dev_remove_driver_gpios() Remove never used devm_acpi_dev_remove_driver_gpios(). Signed-off-by: Andy Shevchenko Reviewed-by: Mika Westerberg --- drivers/gpio/gpiolib-acpi.c | 6 ------ include/linux/gpio/consumer.h | 2 -- 2 files changed, 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 985e8589c58b..25ecc0a37054 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -604,12 +604,6 @@ int devm_acpi_dev_add_driver_gpios(struct device *dev, } EXPORT_SYMBOL_GPL(devm_acpi_dev_add_driver_gpios); -void devm_acpi_dev_remove_driver_gpios(struct device *dev) -{ - WARN_ON(devres_release(dev, devm_acpi_dev_release_driver_gpios, NULL, NULL)); -} -EXPORT_SYMBOL_GPL(devm_acpi_dev_remove_driver_gpios); - static bool acpi_get_driver_gpio_data(struct acpi_device *adev, const char *name, int index, struct fwnode_reference_args *args, diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h index 97a28ad3393b..3ad67b4a72be 100644 --- a/include/linux/gpio/consumer.h +++ b/include/linux/gpio/consumer.h @@ -690,7 +690,6 @@ void acpi_dev_remove_driver_gpios(struct acpi_device *adev); int devm_acpi_dev_add_driver_gpios(struct device *dev, const struct acpi_gpio_mapping *gpios); -void devm_acpi_dev_remove_driver_gpios(struct device *dev); struct gpio_desc *acpi_get_and_request_gpiod(char *path, int pin, char *label); @@ -708,7 +707,6 @@ static inline int devm_acpi_dev_add_driver_gpios(struct device *dev, { return -ENXIO; } -static inline void devm_acpi_dev_remove_driver_gpios(struct device *dev) {} #endif /* CONFIG_GPIOLIB && CONFIG_ACPI */ -- cgit v1.2.3-59-g8ed1b From 02d6fdecb9c38de19065f6bed8d5214556fd061d Mon Sep 17 00:00:00 2001 From: Ansuel Smith Date: Thu, 4 Nov 2021 16:00:40 +0100 Subject: regmap: allow to define reg_update_bits for no bus configuration Some device requires a special handling for reg_update_bits and can't use the normal regmap read write logic. An example is when locking is handled by the device and rmw operations requires to do atomic operations. Allow to declare a dedicated function in regmap_config for reg_update_bits in no bus configuration. Signed-off-by: Ansuel Smith Link: https://lore.kernel.org/r/20211104150040.1260-1-ansuelsmth@gmail.com Signed-off-by: Mark Brown --- drivers/base/regmap/regmap.c | 1 + include/linux/regmap.h | 7 +++++++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 21a0c2562ec0..2d74f9f82aa9 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -876,6 +876,7 @@ struct regmap *__regmap_init(struct device *dev, if (!bus) { map->reg_read = config->reg_read; map->reg_write = config->reg_write; + map->reg_update_bits = config->reg_update_bits; map->defer_caching = false; goto skip_format_initialization; diff --git a/include/linux/regmap.h b/include/linux/regmap.h index e3c9a25a853a..22652e5fbc38 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -290,6 +290,11 @@ typedef void (*regmap_unlock)(void *); * read operation on a bus such as SPI, I2C, etc. Most of the * devices do not need this. * @reg_write: Same as above for writing. + * @reg_update_bits: Optional callback that if filled will be used to perform + * all the update_bits(rmw) operation. Should only be provided + * if the function require special handling with lock and reg + * handling and the operation cannot be represented as a simple + * update_bits operation on a bus such as SPI, I2C, etc. * @fast_io: Register IO is fast. Use a spinlock instead of a mutex * to perform locking. This field is ignored if custom lock/unlock * functions are used (see fields lock/unlock of struct regmap_config). @@ -372,6 +377,8 @@ struct regmap_config { int (*reg_read)(void *context, unsigned int reg, unsigned int *val); int (*reg_write)(void *context, unsigned int reg, unsigned int val); + int (*reg_update_bits)(void *context, unsigned int reg, + unsigned int mask, unsigned int val); bool fast_io; -- cgit v1.2.3-59-g8ed1b From 45971bdd8ca8b5a99a49f4db86737401c45e246f Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 2 Nov 2021 16:02:02 -0600 Subject: spi: remove unused header file Commit 6acaadc852f1 ("spi: clps711x: Driver refactor") removed the only use of , but left the header file behind. This file is unused, delete it. Cc: Signed-off-by: Arnd Bergmann Cc: Signed-off-by: Mark Brown Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: Jonathan Corbet Acked-by: Arnd Bergmann Link: https://lore.kernel.org/r/20211102220203.940290-9-corbet@lwn.net Signed-off-by: Mark Brown --- include/linux/platform_data/spi-clps711x.h | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 include/linux/platform_data/spi-clps711x.h (limited to 'include/linux') diff --git a/include/linux/platform_data/spi-clps711x.h b/include/linux/platform_data/spi-clps711x.h deleted file mode 100644 index efaa596848c9..000000000000 --- a/include/linux/platform_data/spi-clps711x.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * CLPS711X SPI bus driver definitions - * - * Copyright (C) 2012 Alexander Shiyan - */ - -#ifndef ____LINUX_PLATFORM_DATA_SPI_CLPS711X_H -#define ____LINUX_PLATFORM_DATA_SPI_CLPS711X_H - -/* Board specific platform_data */ -struct spi_clps711x_pdata { - int *chipselect; /* Array of GPIO-numbers */ - int num_chipselect; /* Total count of GPIOs */ -}; - -#endif -- cgit v1.2.3-59-g8ed1b From 13cae4a104d2b7205696229ba85d34cc035f8c84 Mon Sep 17 00:00:00 2001 From: Matt Johnston Date: Mon, 15 Nov 2021 10:49:21 +0800 Subject: i2c: core: Allow 255 byte transfers for SMBus 3.x SMBus 3.0 increased the maximum block transfer size from 32 bytes to 255 bytes. We increase the size of struct i2c_smbus_data's block[] member. i2c_smbus_xfer() and i2c_smbus_xfer_emulated() now support 255 byte block operations, other block functions remain limited to 32 bytes for compatibility with existing callers. We allow adapters to indicate support for the larger size with I2C_FUNC_SMBUS_V3_BLOCK. Most emulated drivers should be able to use 255 byte blocks by replacing I2C_SMBUS_BLOCK_MAX with I2C_SMBUS_V3_BLOCK_MAX though some will have hardware limitations that need testing. Signed-off-by: Matt Johnston Signed-off-by: David S. Miller --- drivers/i2c/i2c-core-smbus.c | 20 +++++++++++++------- include/linux/i2c.h | 13 +++++++++++++ include/uapi/linux/i2c.h | 5 ++++- 3 files changed, 30 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c index e5b2d1465e7e..743415584aba 100644 --- a/drivers/i2c/i2c-core-smbus.c +++ b/drivers/i2c/i2c-core-smbus.c @@ -303,7 +303,8 @@ static void i2c_smbus_try_get_dmabuf(struct i2c_msg *msg, u8 init_val) bool is_read = msg->flags & I2C_M_RD; unsigned char *dma_buf; - dma_buf = kzalloc(I2C_SMBUS_BLOCK_MAX + (is_read ? 2 : 3), GFP_KERNEL); + dma_buf = kzalloc(I2C_SMBUS_V3_BLOCK_MAX + (is_read ? 2 : 3), + GFP_KERNEL); if (!dma_buf) return; @@ -329,9 +330,10 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, * initialize most things with sane defaults, to keep the code below * somewhat simpler. */ - unsigned char msgbuf0[I2C_SMBUS_BLOCK_MAX+3]; - unsigned char msgbuf1[I2C_SMBUS_BLOCK_MAX+2]; + unsigned char msgbuf0[I2C_SMBUS_V3_BLOCK_MAX+3]; + unsigned char msgbuf1[I2C_SMBUS_V3_BLOCK_MAX+2]; int nmsgs = read_write == I2C_SMBUS_READ ? 2 : 1; + u16 block_max; u8 partial_pec = 0; int status; struct i2c_msg msg[2] = { @@ -350,6 +352,10 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, bool wants_pec = ((flags & I2C_CLIENT_PEC) && size != I2C_SMBUS_QUICK && size != I2C_SMBUS_I2C_BLOCK_DATA); + /* Drivers must opt in to 255 byte max block size */ + block_max = i2c_check_functionality(adapter, I2C_FUNC_SMBUS_V3_BLOCK) + ? I2C_SMBUS_V3_BLOCK_MAX : I2C_SMBUS_BLOCK_MAX; + msgbuf0[0] = command; switch (size) { case I2C_SMBUS_QUICK: @@ -399,7 +405,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, i2c_smbus_try_get_dmabuf(&msg[1], 0); } else { msg[0].len = data->block[0] + 2; - if (msg[0].len > I2C_SMBUS_BLOCK_MAX + 2) { + if (msg[0].len > block_max + 2) { dev_err(&adapter->dev, "Invalid block write size %d\n", data->block[0]); @@ -413,7 +419,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, case I2C_SMBUS_BLOCK_PROC_CALL: nmsgs = 2; /* Another special case */ read_write = I2C_SMBUS_READ; - if (data->block[0] > I2C_SMBUS_BLOCK_MAX) { + if (data->block[0] > block_max) { dev_err(&adapter->dev, "Invalid block write size %d\n", data->block[0]); @@ -430,7 +436,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, i2c_smbus_try_get_dmabuf(&msg[1], 0); break; case I2C_SMBUS_I2C_BLOCK_DATA: - if (data->block[0] > I2C_SMBUS_BLOCK_MAX) { + if (data->block[0] > block_max) { dev_err(&adapter->dev, "Invalid block %s size %d\n", read_write == I2C_SMBUS_READ ? "read" : "write", data->block[0]); @@ -498,7 +504,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, break; case I2C_SMBUS_BLOCK_DATA: case I2C_SMBUS_BLOCK_PROC_CALL: - if (msg[1].buf[0] > I2C_SMBUS_BLOCK_MAX) { + if (msg[1].buf[0] > block_max) { dev_err(&adapter->dev, "Invalid block size returned: %d\n", msg[1].buf[0]); diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 16119ac1aa97..353d6b4e7a53 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -52,6 +52,19 @@ typedef int (*i2c_slave_cb_t)(struct i2c_client *client, struct module; struct property_entry; +/* SMBus 3.0 extends the maximum block read/write size to 255 (from 32). + * The larger size is only supported by some drivers, indicated by + * the I2C_FUNC_SMBUS_V3_BLOCK functionality bit. + */ +#define I2C_SMBUS_V3_BLOCK_MAX 255 /* As specified in SMBus 3.0 standard */ + +/* Note compatibility definition in uapi header with 32 byte block */ +union i2c_smbus_data { + __u8 byte; + __u16 word; + __u8 block[I2C_SMBUS_V3_BLOCK_MAX + 1]; /* block[0] is used for length */ +}; + #if IS_ENABLED(CONFIG_I2C) /* Return the Frequency mode string based on the bus frequency */ const char *i2c_freq_mode_string(u32 bus_freq_hz); diff --git a/include/uapi/linux/i2c.h b/include/uapi/linux/i2c.h index 92326ebde350..7b7d90b50cf0 100644 --- a/include/uapi/linux/i2c.h +++ b/include/uapi/linux/i2c.h @@ -108,6 +108,7 @@ struct i2c_msg { #define I2C_FUNC_SMBUS_READ_I2C_BLOCK 0x04000000 /* I2C-like block xfer */ #define I2C_FUNC_SMBUS_WRITE_I2C_BLOCK 0x08000000 /* w/ 1-byte reg. addr. */ #define I2C_FUNC_SMBUS_HOST_NOTIFY 0x10000000 /* SMBus 2.0 or later */ +#define I2C_FUNC_SMBUS_V3_BLOCK 0x20000000 /* Device supports 255 byte block */ #define I2C_FUNC_SMBUS_BYTE (I2C_FUNC_SMBUS_READ_BYTE | \ I2C_FUNC_SMBUS_WRITE_BYTE) @@ -137,13 +138,15 @@ struct i2c_msg { /* * Data for SMBus Messages */ -#define I2C_SMBUS_BLOCK_MAX 32 /* As specified in SMBus standard */ +#define I2C_SMBUS_BLOCK_MAX 32 /* As specified in SMBus 2.0 standard */ +#ifndef __KERNEL__ union i2c_smbus_data { __u8 byte; __u16 word; __u8 block[I2C_SMBUS_BLOCK_MAX + 2]; /* block[0] is used for length */ /* and one more for user-space compatibility */ }; +#endif /* i2c_smbus_xfer read or write markers */ #define I2C_SMBUS_READ 1 -- cgit v1.2.3-59-g8ed1b From 34ae2c09d46a2d0abd907e139b466f798e4095a8 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Mon, 15 Nov 2021 10:00:27 +0000 Subject: net: phylink: add generic validate implementation Add a generic validate() implementation using the supported_interfaces and a bitmask of MAC pause/speed/duplex capabilities. This allows us to entirely eliminate many driver private validate() implementations. We expose the underlying phylink_get_linkmodes() function so that drivers which have special needs can still benefit from conversion. Signed-off-by: Russell King (Oracle) Signed-off-by: David S. Miller --- drivers/net/phy/phylink.c | 252 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/phylink.h | 31 ++++++ 2 files changed, 283 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 3ad7397b8119..33462fdc7add 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -166,6 +166,258 @@ static const char *phylink_an_mode_str(unsigned int mode) return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown"; } +static void phylink_caps_to_linkmodes(unsigned long *linkmodes, + unsigned long caps) +{ + if (caps & MAC_SYM_PAUSE) + __set_bit(ETHTOOL_LINK_MODE_Pause_BIT, linkmodes); + + if (caps & MAC_ASYM_PAUSE) + __set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, linkmodes); + + if (caps & MAC_10HD) + __set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, linkmodes); + + if (caps & MAC_10FD) + __set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, linkmodes); + + if (caps & MAC_100HD) { + __set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseFX_Half_BIT, linkmodes); + } + + if (caps & MAC_100FD) { + __set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseT1_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseFX_Full_BIT, linkmodes); + } + + if (caps & MAC_1000HD) + __set_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT, linkmodes); + + if (caps & MAC_1000FD) { + __set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_1000baseT1_Full_BIT, linkmodes); + } + + if (caps & MAC_2500FD) { + __set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, linkmodes); + } + + if (caps & MAC_5000FD) + __set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, linkmodes); + + if (caps & MAC_10000FD) { + __set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseLR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseER_Full_BIT, linkmodes); + } + + if (caps & MAC_25000FD) { + __set_bit(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, linkmodes); + } + + if (caps & MAC_40000FD) { + __set_bit(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, linkmodes); + } + + if (caps & MAC_50000FD) { + __set_bit(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseLR_ER_FR_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseDR_Full_BIT, linkmodes); + } + + if (caps & MAC_56000FD) { + __set_bit(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT, linkmodes); + } + + if (caps & MAC_100000FD) { + __set_bit(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR2_ER2_FR2_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseDR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR_ER_FR_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseDR_Full_BIT, linkmodes); + } + + if (caps & MAC_200000FD) { + __set_bit(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseLR2_ER2_FR2_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseDR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseCR2_Full_BIT, linkmodes); + } + + if (caps & MAC_400000FD) { + __set_bit(ETHTOOL_LINK_MODE_400000baseKR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseSR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseLR8_ER8_FR8_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseDR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseCR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseLR4_ER4_FR4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseDR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseCR4_Full_BIT, linkmodes); + } +} + +/** + * phylink_get_linkmodes() - get acceptable link modes + * @linkmodes: ethtool linkmode mask (must be already initialised) + * @interface: phy interface mode defined by &typedef phy_interface_t + * @mac_capabilities: bitmask of MAC capabilities + * + * Set all possible pause, speed and duplex linkmodes in @linkmodes that + * are supported by the @interface mode and @mac_capabilities. @linkmodes + * must have been initialised previously. + */ +void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, + unsigned long mac_capabilities) +{ + unsigned long caps = MAC_SYM_PAUSE | MAC_ASYM_PAUSE; + + switch (interface) { + case PHY_INTERFACE_MODE_USXGMII: + caps |= MAC_10000FD | MAC_5000FD | MAC_2500FD; + fallthrough; + + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_GMII: + caps |= MAC_1000HD | MAC_1000FD; + fallthrough; + + case PHY_INTERFACE_MODE_REVRMII: + case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_REVMII: + case PHY_INTERFACE_MODE_MII: + caps |= MAC_10HD | MAC_10FD; + fallthrough; + + case PHY_INTERFACE_MODE_100BASEX: + caps |= MAC_100HD | MAC_100FD; + break; + + case PHY_INTERFACE_MODE_TBI: + case PHY_INTERFACE_MODE_MOCA: + case PHY_INTERFACE_MODE_RTBI: + case PHY_INTERFACE_MODE_1000BASEX: + caps |= MAC_1000HD; + fallthrough; + case PHY_INTERFACE_MODE_TRGMII: + caps |= MAC_1000FD; + break; + + case PHY_INTERFACE_MODE_2500BASEX: + caps |= MAC_2500FD; + break; + + case PHY_INTERFACE_MODE_5GBASER: + caps |= MAC_5000FD; + break; + + case PHY_INTERFACE_MODE_XGMII: + case PHY_INTERFACE_MODE_RXAUI: + case PHY_INTERFACE_MODE_XAUI: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_10GKR: + caps |= MAC_10000FD; + break; + + case PHY_INTERFACE_MODE_25GBASER: + caps |= MAC_25000FD; + break; + + case PHY_INTERFACE_MODE_XLGMII: + caps |= MAC_40000FD; + break; + + case PHY_INTERFACE_MODE_INTERNAL: + caps |= ~0; + break; + + case PHY_INTERFACE_MODE_NA: + case PHY_INTERFACE_MODE_MAX: + case PHY_INTERFACE_MODE_SMII: + break; + } + + phylink_caps_to_linkmodes(linkmodes, caps & mac_capabilities); +} +EXPORT_SYMBOL_GPL(phylink_get_linkmodes); + +/** + * phylink_generic_validate() - generic validate() callback implementation + * @config: a pointer to a &struct phylink_config. + * @supported: ethtool bitmask for supported link modes. + * @state: a pointer to a &struct phylink_link_state. + * + * Generic implementation of the validate() callback that MAC drivers can + * use when they pass the range of supported interfaces and MAC capabilities. + * This makes use of phylink_get_linkmodes(). + */ +void phylink_generic_validate(struct phylink_config *config, + unsigned long *supported, + struct phylink_link_state *state) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; + + phylink_set_port_modes(mask); + phylink_set(mask, Autoneg); + phylink_get_linkmodes(mask, state->interface, config->mac_capabilities); + + linkmode_and(supported, supported, mask); + linkmode_and(state->advertising, state->advertising, mask); +} +EXPORT_SYMBOL_GPL(phylink_generic_validate); + static int phylink_validate_any(struct phylink *pl, unsigned long *supported, struct phylink_link_state *state) { diff --git a/include/linux/phylink.h b/include/linux/phylink.h index f037470b6fb3..3563820a1765 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -20,6 +20,29 @@ enum { MLO_AN_PHY = 0, /* Conventional PHY */ MLO_AN_FIXED, /* Fixed-link mode */ MLO_AN_INBAND, /* In-band protocol */ + + MAC_SYM_PAUSE = BIT(0), + MAC_ASYM_PAUSE = BIT(1), + MAC_10HD = BIT(2), + MAC_10FD = BIT(3), + MAC_10 = MAC_10HD | MAC_10FD, + MAC_100HD = BIT(4), + MAC_100FD = BIT(5), + MAC_100 = MAC_100HD | MAC_100FD, + MAC_1000HD = BIT(6), + MAC_1000FD = BIT(7), + MAC_1000 = MAC_1000HD | MAC_1000FD, + MAC_2500FD = BIT(8), + MAC_5000FD = BIT(9), + MAC_10000FD = BIT(10), + MAC_20000FD = BIT(11), + MAC_25000FD = BIT(12), + MAC_40000FD = BIT(13), + MAC_50000FD = BIT(14), + MAC_56000FD = BIT(15), + MAC_100000FD = BIT(16), + MAC_200000FD = BIT(17), + MAC_400000FD = BIT(18), }; static inline bool phylink_autoneg_inband(unsigned int mode) @@ -69,6 +92,7 @@ enum phylink_op_type { * if MAC link is at %MLO_AN_FIXED mode. * @supported_interfaces: bitmap describing which PHY_INTERFACE_MODE_xxx * are supported by the MAC/PCS. + * @mac_capabilities: MAC pause/speed/duplex capabilities. */ struct phylink_config { struct device *dev; @@ -79,6 +103,7 @@ struct phylink_config { void (*get_fixed_state)(struct phylink_config *config, struct phylink_link_state *state); DECLARE_PHY_INTERFACE_MASK(supported_interfaces); + unsigned long mac_capabilities; }; /** @@ -442,6 +467,12 @@ void pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex); #endif +void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, + unsigned long mac_capabilities); +void phylink_generic_validate(struct phylink_config *config, + unsigned long *supported, + struct phylink_link_state *state); + struct phylink *phylink_create(struct phylink_config *, struct fwnode_handle *, phy_interface_t iface, const struct phylink_mac_ops *mac_ops); -- cgit v1.2.3-59-g8ed1b From 2f6a470d6545841cf1891b87e360d3998ef024c8 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 15 Nov 2021 07:49:46 -0800 Subject: Revert "Merge branch 'mctp-i2c-driver'" This reverts commit 71812af7234f30362b43ccff33f93890ae4c0655, reversing changes made to cc0be1ad686fb29a4d127948486f40b17fb34b50. Wolfram Sang says: Please revert. Besides the driver in net, it modifies the I2C core code. This has not been acked by the I2C maintainer (in this case me). So, please don't pull this in via the net tree. The question raised here (extending SMBus calls to 255 byte) is complicated because we need ABI backwards compatibility. Link: https://lore.kernel.org/all/YZJ9H4eM%2FM7OXVN0@shikoro/ Signed-off-by: Jakub Kicinski --- Documentation/devicetree/bindings/i2c/i2c.txt | 4 - .../bindings/net/mctp-i2c-controller.yaml | 92 -- drivers/i2c/busses/i2c-aspeed.c | 5 +- drivers/i2c/busses/i2c-npcm7xx.c | 3 +- drivers/i2c/i2c-core-smbus.c | 20 +- drivers/i2c/i2c-dev.c | 93 +- drivers/net/mctp/Kconfig | 12 - drivers/net/mctp/Makefile | 1 - drivers/net/mctp/mctp-i2c.c | 982 --------------------- include/linux/i2c.h | 13 - include/uapi/linux/i2c-dev.h | 2 - include/uapi/linux/i2c.h | 7 +- 12 files changed, 25 insertions(+), 1209 deletions(-) delete mode 100644 Documentation/devicetree/bindings/net/mctp-i2c-controller.yaml delete mode 100644 drivers/net/mctp/mctp-i2c.c (limited to 'include/linux') diff --git a/Documentation/devicetree/bindings/i2c/i2c.txt b/Documentation/devicetree/bindings/i2c/i2c.txt index fc3dd7ec0445..b864916e087f 100644 --- a/Documentation/devicetree/bindings/i2c/i2c.txt +++ b/Documentation/devicetree/bindings/i2c/i2c.txt @@ -95,10 +95,6 @@ wants to support one of the below features, it should adapt these bindings. - smbus-alert states that the optional SMBus-Alert feature apply to this bus. -- mctp-controller - indicates that the system is accessible via this bus as an endpoint for - MCTP over I2C transport. - Required properties (per child device) -------------------------------------- diff --git a/Documentation/devicetree/bindings/net/mctp-i2c-controller.yaml b/Documentation/devicetree/bindings/net/mctp-i2c-controller.yaml deleted file mode 100644 index afd11c9422fa..000000000000 --- a/Documentation/devicetree/bindings/net/mctp-i2c-controller.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/net/mctp-i2c-controller.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: MCTP I2C transport binding - -maintainers: - - Matt Johnston - -description: | - An mctp-i2c-controller defines a local MCTP endpoint on an I2C controller. - MCTP I2C is specified by DMTF DSP0237. - - An mctp-i2c-controller must be attached to an I2C adapter which supports - slave functionality. I2C busses (either directly or as subordinate mux - busses) are attached to the mctp-i2c-controller with a 'mctp-controller' - property on each used bus. Each mctp-controller I2C bus will be presented - to the host system as a separate MCTP I2C instance. - -properties: - compatible: - const: mctp-i2c-controller - - reg: - minimum: 0x40000000 - maximum: 0x4000007f - description: | - 7 bit I2C address of the local endpoint. - I2C_OWN_SLAVE_ADDRESS (1<<30) flag must be set. - -additionalProperties: false - -required: - - compatible - - reg - -examples: - - | - // Basic case of a single I2C bus - #include - - i2c { - #address-cells = <1>; - #size-cells = <0>; - mctp-controller; - - mctp@30 { - compatible = "mctp-i2c-controller"; - reg = <(0x30 | I2C_OWN_SLAVE_ADDRESS)>; - }; - }; - - - | - // Mux topology with multiple MCTP-handling busses under - // a single mctp-i2c-controller. - // i2c1 and i2c6 can have MCTP devices, i2c5 does not. - #include - - i2c1: i2c { - #address-cells = <1>; - #size-cells = <0>; - mctp-controller; - - mctp@50 { - compatible = "mctp-i2c-controller"; - reg = <(0x50 | I2C_OWN_SLAVE_ADDRESS)>; - }; - }; - - i2c-mux { - #address-cells = <1>; - #size-cells = <0>; - i2c-parent = <&i2c1>; - - i2c5: i2c@0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0>; - eeprom@33 { - reg = <0x33>; - }; - }; - - i2c6: i2c@1 { - #address-cells = <1>; - #size-cells = <0>; - reg = <1>; - mctp-controller; - }; - }; diff --git a/drivers/i2c/busses/i2c-aspeed.c b/drivers/i2c/busses/i2c-aspeed.c index 7395f3702fae..67e8b97c0c95 100644 --- a/drivers/i2c/busses/i2c-aspeed.c +++ b/drivers/i2c/busses/i2c-aspeed.c @@ -533,7 +533,7 @@ static u32 aspeed_i2c_master_irq(struct aspeed_i2c_bus *bus, u32 irq_status) msg->buf[bus->buf_index++] = recv_byte; if (msg->flags & I2C_M_RECV_LEN) { - if (unlikely(recv_byte > I2C_SMBUS_V3_BLOCK_MAX)) { + if (unlikely(recv_byte > I2C_SMBUS_BLOCK_MAX)) { bus->cmd_err = -EPROTO; aspeed_i2c_do_stop(bus); goto out_no_complete; @@ -718,8 +718,7 @@ static int aspeed_i2c_master_xfer(struct i2c_adapter *adap, static u32 aspeed_i2c_functionality(struct i2c_adapter *adap) { - return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | - I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_V3_BLOCK; + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_SMBUS_BLOCK_DATA; } #if IS_ENABLED(CONFIG_I2C_SLAVE) diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c index 6d60f65add85..2ad166355ec9 100644 --- a/drivers/i2c/busses/i2c-npcm7xx.c +++ b/drivers/i2c/busses/i2c-npcm7xx.c @@ -1399,7 +1399,7 @@ static void npcm_i2c_irq_master_handler_read(struct npcm_i2c *bus) if (bus->read_block_use) { /* first byte in block protocol is the size: */ data = npcm_i2c_rd_byte(bus); - data = clamp_val(data, 1, I2C_SMBUS_V3_BLOCK_MAX); + data = clamp_val(data, 1, I2C_SMBUS_BLOCK_MAX); bus->rd_size = data + block_extra_bytes_size; bus->rd_buf[bus->rd_ind++] = data; @@ -2187,7 +2187,6 @@ static u32 npcm_i2c_functionality(struct i2c_adapter *adap) I2C_FUNC_SMBUS_EMUL | I2C_FUNC_SMBUS_BLOCK_DATA | I2C_FUNC_SMBUS_PEC | - I2C_FUNC_SMBUS_V3_BLOCK | I2C_FUNC_SLAVE; } diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c index 743415584aba..e5b2d1465e7e 100644 --- a/drivers/i2c/i2c-core-smbus.c +++ b/drivers/i2c/i2c-core-smbus.c @@ -303,8 +303,7 @@ static void i2c_smbus_try_get_dmabuf(struct i2c_msg *msg, u8 init_val) bool is_read = msg->flags & I2C_M_RD; unsigned char *dma_buf; - dma_buf = kzalloc(I2C_SMBUS_V3_BLOCK_MAX + (is_read ? 2 : 3), - GFP_KERNEL); + dma_buf = kzalloc(I2C_SMBUS_BLOCK_MAX + (is_read ? 2 : 3), GFP_KERNEL); if (!dma_buf) return; @@ -330,10 +329,9 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, * initialize most things with sane defaults, to keep the code below * somewhat simpler. */ - unsigned char msgbuf0[I2C_SMBUS_V3_BLOCK_MAX+3]; - unsigned char msgbuf1[I2C_SMBUS_V3_BLOCK_MAX+2]; + unsigned char msgbuf0[I2C_SMBUS_BLOCK_MAX+3]; + unsigned char msgbuf1[I2C_SMBUS_BLOCK_MAX+2]; int nmsgs = read_write == I2C_SMBUS_READ ? 2 : 1; - u16 block_max; u8 partial_pec = 0; int status; struct i2c_msg msg[2] = { @@ -352,10 +350,6 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, bool wants_pec = ((flags & I2C_CLIENT_PEC) && size != I2C_SMBUS_QUICK && size != I2C_SMBUS_I2C_BLOCK_DATA); - /* Drivers must opt in to 255 byte max block size */ - block_max = i2c_check_functionality(adapter, I2C_FUNC_SMBUS_V3_BLOCK) - ? I2C_SMBUS_V3_BLOCK_MAX : I2C_SMBUS_BLOCK_MAX; - msgbuf0[0] = command; switch (size) { case I2C_SMBUS_QUICK: @@ -405,7 +399,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, i2c_smbus_try_get_dmabuf(&msg[1], 0); } else { msg[0].len = data->block[0] + 2; - if (msg[0].len > block_max + 2) { + if (msg[0].len > I2C_SMBUS_BLOCK_MAX + 2) { dev_err(&adapter->dev, "Invalid block write size %d\n", data->block[0]); @@ -419,7 +413,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, case I2C_SMBUS_BLOCK_PROC_CALL: nmsgs = 2; /* Another special case */ read_write = I2C_SMBUS_READ; - if (data->block[0] > block_max) { + if (data->block[0] > I2C_SMBUS_BLOCK_MAX) { dev_err(&adapter->dev, "Invalid block write size %d\n", data->block[0]); @@ -436,7 +430,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, i2c_smbus_try_get_dmabuf(&msg[1], 0); break; case I2C_SMBUS_I2C_BLOCK_DATA: - if (data->block[0] > block_max) { + if (data->block[0] > I2C_SMBUS_BLOCK_MAX) { dev_err(&adapter->dev, "Invalid block %s size %d\n", read_write == I2C_SMBUS_READ ? "read" : "write", data->block[0]); @@ -504,7 +498,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr, break; case I2C_SMBUS_BLOCK_DATA: case I2C_SMBUS_BLOCK_PROC_CALL: - if (msg[1].buf[0] > block_max) { + if (msg[1].buf[0] > I2C_SMBUS_BLOCK_MAX) { dev_err(&adapter->dev, "Invalid block size returned: %d\n", msg[1].buf[0]); diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c index 5ee9118c0407..bce0e8bb7852 100644 --- a/drivers/i2c/i2c-dev.c +++ b/drivers/i2c/i2c-dev.c @@ -46,24 +46,6 @@ struct i2c_dev { struct cdev cdev; }; -/* The userspace union i2c_smbus_data for I2C_SMBUS ioctl is limited - * to 32 bytes (I2C_SMBUS_BLOCK_MAX) for compatibility. - */ -union compat_i2c_smbus_data { - __u8 byte; - __u16 word; - __u8 block[I2C_SMBUS_BLOCK_MAX + 2]; /* block[0] is used for length */ - /* and one more for user-space compatibility */ -}; - -/* Must match i2c-dev.h definition with compat .data member */ -struct i2c_smbus_ioctl_data { - __u8 read_write; - __u8 command; - __u32 size; - union compat_i2c_smbus_data __user *data; -}; - #define I2C_MINORS (MINORMASK + 1) static LIST_HEAD(i2c_dev_list); static DEFINE_SPINLOCK(i2c_dev_list_lock); @@ -253,17 +235,14 @@ static int i2cdev_check_addr(struct i2c_adapter *adapter, unsigned int addr) static noinline int i2cdev_ioctl_rdwr(struct i2c_client *client, unsigned nmsgs, struct i2c_msg *msgs) { - u8 __user **data_ptrs = NULL; - u16 *orig_lens = NULL; + u8 __user **data_ptrs; int i, res; - res = -ENOMEM; data_ptrs = kmalloc_array(nmsgs, sizeof(u8 __user *), GFP_KERNEL); - if (data_ptrs == NULL) - goto out; - orig_lens = kmalloc_array(nmsgs, sizeof(u16), GFP_KERNEL); - if (orig_lens == NULL) - goto out; + if (data_ptrs == NULL) { + kfree(msgs); + return -ENOMEM; + } res = 0; for (i = 0; i < nmsgs; i++) { @@ -274,30 +253,12 @@ static noinline int i2cdev_ioctl_rdwr(struct i2c_client *client, } data_ptrs[i] = (u8 __user *)msgs[i].buf; - msgs[i].buf = NULL; - if (msgs[i].len < 1) { - /* Sanity check */ - res = -EINVAL; - break; - - } - /* Allocate a larger buffer to accommodate possible 255 byte - * blocks. Read results will be dropped later - * if they are too large for the original length. - */ - orig_lens[i] = msgs[i].len; - msgs[i].buf = kmalloc(msgs[i].len + I2C_SMBUS_V3_BLOCK_MAX, - GFP_USER | __GFP_NOWARN); + msgs[i].buf = memdup_user(data_ptrs[i], msgs[i].len); if (IS_ERR(msgs[i].buf)) { res = PTR_ERR(msgs[i].buf); break; } - if (copy_from_user(msgs[i].buf, data_ptrs[i], msgs[i].len)) { - kfree(msgs[i].buf); - res = -EFAULT; - break; - } - /* Buffer from kmalloc, so DMA is ok */ + /* memdup_user allocates with GFP_KERNEL, so DMA is ok */ msgs[i].flags |= I2C_M_DMA_SAFE; /* @@ -313,7 +274,7 @@ static noinline int i2cdev_ioctl_rdwr(struct i2c_client *client, */ if (msgs[i].flags & I2C_M_RECV_LEN) { if (!(msgs[i].flags & I2C_M_RD) || - msgs[i].buf[0] < 1 || + msgs[i].len < 1 || msgs[i].buf[0] < 1 || msgs[i].len < msgs[i].buf[0] + I2C_SMBUS_BLOCK_MAX) { i++; @@ -336,16 +297,12 @@ static noinline int i2cdev_ioctl_rdwr(struct i2c_client *client, res = i2c_transfer(client->adapter, msgs, nmsgs); while (i-- > 0) { if (res >= 0 && (msgs[i].flags & I2C_M_RD)) { - if (orig_lens[i] < msgs[i].len) - res = -EINVAL; - else if (copy_to_user(data_ptrs[i], msgs[i].buf, - msgs[i].len)) + if (copy_to_user(data_ptrs[i], msgs[i].buf, + msgs[i].len)) res = -EFAULT; } kfree(msgs[i].buf); } -out: - kfree(orig_lens); kfree(data_ptrs); kfree(msgs); return res; @@ -353,7 +310,7 @@ out: static noinline int i2cdev_ioctl_smbus(struct i2c_client *client, u8 read_write, u8 command, u32 size, - union compat_i2c_smbus_data __user *data) + union i2c_smbus_data __user *data) { union i2c_smbus_data temp = {}; int datasize, res; @@ -414,16 +371,6 @@ static noinline int i2cdev_ioctl_smbus(struct i2c_client *client, if (copy_from_user(&temp, data, datasize)) return -EFAULT; } - if ((size == I2C_SMBUS_BLOCK_PROC_CALL || - size == I2C_SMBUS_I2C_BLOCK_DATA || - size == I2C_SMBUS_BLOCK_DATA) && - read_write == I2C_SMBUS_WRITE && - temp.block[0] > I2C_SMBUS_BLOCK_MAX) { - /* Don't accept writes larger than the buffer size */ - dev_dbg(&client->adapter->dev, "block write is too large"); - return -EINVAL; - - } if (size == I2C_SMBUS_I2C_BLOCK_BROKEN) { /* Convert old I2C block commands to the new convention. This preserves binary compatibility. */ @@ -433,21 +380,9 @@ static noinline int i2cdev_ioctl_smbus(struct i2c_client *client, } res = i2c_smbus_xfer(client->adapter, client->addr, client->flags, read_write, command, size, &temp); - if (res) - return res; - if ((size == I2C_SMBUS_BLOCK_PROC_CALL || - size == I2C_SMBUS_I2C_BLOCK_DATA || - size == I2C_SMBUS_BLOCK_DATA) && - read_write == I2C_SMBUS_READ && - temp.block[0] > I2C_SMBUS_BLOCK_MAX) { - /* Don't accept reads larger than the buffer size */ - dev_dbg(&client->adapter->dev, "block read is too large"); - return -EINVAL; - - } - if ((size == I2C_SMBUS_PROC_CALL) || - (size == I2C_SMBUS_BLOCK_PROC_CALL) || - (read_write == I2C_SMBUS_READ)) { + if (!res && ((size == I2C_SMBUS_PROC_CALL) || + (size == I2C_SMBUS_BLOCK_PROC_CALL) || + (read_write == I2C_SMBUS_READ))) { if (copy_to_user(data, &temp, datasize)) return -EFAULT; } diff --git a/drivers/net/mctp/Kconfig b/drivers/net/mctp/Kconfig index b758b29c2ddf..d8f966cedc89 100644 --- a/drivers/net/mctp/Kconfig +++ b/drivers/net/mctp/Kconfig @@ -3,18 +3,6 @@ if MCTP menu "MCTP Device Drivers" -config MCTP_TRANSPORT_I2C - tristate "MCTP SMBus/I2C transport" - # i2c-mux is optional, but we must build as a module if i2c-mux is a module - depends on I2C_MUX || !I2C_MUX - depends on I2C - depends on I2C_SLAVE - select MCTP_FLOWS - help - Provides a driver to access MCTP devices over SMBus/I2C transport, - from DMTF specification DSP0237. A MCTP protocol network device is - created for each I2C bus that has been assigned a mctp-i2c device. - endmenu endif diff --git a/drivers/net/mctp/Makefile b/drivers/net/mctp/Makefile index 73dc411986a6..e69de29bb2d1 100644 --- a/drivers/net/mctp/Makefile +++ b/drivers/net/mctp/Makefile @@ -1 +0,0 @@ -obj-$(CONFIG_MCTP_TRANSPORT_I2C) += mctp-i2c.o diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c deleted file mode 100644 index ed213b4765a1..000000000000 --- a/drivers/net/mctp/mctp-i2c.c +++ /dev/null @@ -1,982 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Management Controller Transport Protocol (MCTP) - * - * Copyright (c) 2021 Code Construct - * Copyright (c) 2021 Google - */ - -#include -#include -#include -#include -#include -#include -#include - -/* SMBus 3.0 allows 255 data bytes (plus PEC), but the - * first byte is taken for source slave address. - */ -#define MCTP_I2C_MAXBLOCK 255 -#define MCTP_I2C_MAXMTU (MCTP_I2C_MAXBLOCK - 1) -#define MCTP_I2C_MINMTU (64 + 4) -/* Allow space for address, command, byte_count, databytes, PEC */ -#define MCTP_I2C_RXBUFSZ (3 + MCTP_I2C_MAXBLOCK + 1) -#define MCTP_I2C_MINLEN 8 -#define MCTP_I2C_COMMANDCODE 0x0f -#define MCTP_I2C_TX_WORK_LEN 100 -// sufficient for 64kB at min mtu -#define MCTP_I2C_TX_QUEUE_LEN 1100 - -#define MCTP_I2C_OF_PROP "mctp-controller" - -enum { - MCTP_I2C_FLOW_STATE_NEW = 0, - MCTP_I2C_FLOW_STATE_ACTIVE, -}; - -static struct { - /* lock protects clients and also prevents adding/removing adapters - * during mctp_i2c_client probe/remove. - */ - struct mutex lock; - // list of struct mctp_i2c_client - struct list_head clients; -} mi_driver_state; - -struct mctp_i2c_client; - -// The netdev structure. One of these per I2C adapter. -struct mctp_i2c_dev { - struct net_device *ndev; - struct i2c_adapter *adapter; - struct mctp_i2c_client *client; - struct list_head list; // for mctp_i2c_client.devs - - size_t pos; - u8 buffer[MCTP_I2C_RXBUFSZ]; - - struct task_struct *tx_thread; - wait_queue_head_t tx_wq; - struct sk_buff_head tx_queue; - - // a fake entry in our tx queue to perform an unlock operation - struct sk_buff unlock_marker; - - spinlock_t flow_lock; // protects i2c_lock_count and release_count - int i2c_lock_count; - int release_count; -}; - -/* The i2c client structure. One per hardware i2c bus at the top of the - * mux tree, shared by multiple netdevs - */ -struct mctp_i2c_client { - struct i2c_client *client; - u8 lladdr; - - struct mctp_i2c_dev *sel; - struct list_head devs; - spinlock_t curr_lock; // protects sel - - struct list_head list; // for mi_driver_state.clients -}; - -// Header on the wire -struct mctp_i2c_hdr { - u8 dest_slave; - u8 command; - u8 byte_count; - u8 source_slave; -}; - -static int mctp_i2c_recv(struct mctp_i2c_dev *midev); -static int mctp_i2c_slave_cb(struct i2c_client *client, - enum i2c_slave_event event, u8 *val); - -static struct i2c_adapter *mux_root_adapter(struct i2c_adapter *adap) -{ -#if IS_ENABLED(CONFIG_I2C_MUX) - return i2c_root_adapter(&adap->dev); -#else - /* In non-mux config all i2c adapters are root adapters */ - return adap; -#endif -} - -static ssize_t mctp_current_mux_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct mctp_i2c_client *mcli = i2c_get_clientdata(to_i2c_client(dev)); - struct net_device *ndev = NULL; - unsigned long flags; - ssize_t l; - - spin_lock_irqsave(&mcli->curr_lock, flags); - if (mcli->sel) { - ndev = mcli->sel->ndev; - dev_hold(ndev); - } - spin_unlock_irqrestore(&mcli->curr_lock, flags); - l = scnprintf(buf, PAGE_SIZE, "%s\n", ndev ? ndev->name : "(none)"); - if (ndev) - dev_put(ndev); - return l; -} -static DEVICE_ATTR_RO(mctp_current_mux); - -/* Creates a new i2c slave device attached to the root adapter. - * Sets up the slave callback. - * Must be called with a client on a root adapter. - */ -static struct mctp_i2c_client *mctp_i2c_new_client(struct i2c_client *client) -{ - struct mctp_i2c_client *mcli = NULL; - struct i2c_adapter *root = NULL; - int rc; - - if (client->flags & I2C_CLIENT_TEN) { - dev_err(&client->dev, "%s failed, MCTP requires a 7-bit I2C address, addr=0x%x", - __func__, client->addr); - rc = -EINVAL; - goto err; - } - - root = mux_root_adapter(client->adapter); - if (!root) { - dev_err(&client->dev, "%s failed to find root adapter\n", __func__); - rc = -ENOENT; - goto err; - } - if (root != client->adapter) { - dev_err(&client->dev, - "A mctp-i2c-controller client cannot be placed on an I2C mux adapter.\n" - " It should be placed on the mux tree root adapter\n" - " then set mctp-controller property on adapters to attach\n"); - rc = -EINVAL; - goto err; - } - - mcli = kzalloc(sizeof(*mcli), GFP_KERNEL); - if (!mcli) { - rc = -ENOMEM; - goto err; - } - spin_lock_init(&mcli->curr_lock); - INIT_LIST_HEAD(&mcli->devs); - INIT_LIST_HEAD(&mcli->list); - mcli->lladdr = client->addr & 0xff; - mcli->client = client; - i2c_set_clientdata(client, mcli); - - rc = i2c_slave_register(mcli->client, mctp_i2c_slave_cb); - if (rc) { - dev_err(&client->dev, "%s i2c register failed %d\n", __func__, rc); - mcli->client = NULL; - i2c_set_clientdata(client, NULL); - goto err; - } - - rc = device_create_file(&client->dev, &dev_attr_mctp_current_mux); - if (rc) { - dev_err(&client->dev, "%s adding sysfs \"%s\" failed %d\n", __func__, - dev_attr_mctp_current_mux.attr.name, rc); - // continue anyway - } - - return mcli; -err: - if (mcli) { - if (mcli->client) { - device_remove_file(&mcli->client->dev, &dev_attr_mctp_current_mux); - i2c_unregister_device(mcli->client); - } - kfree(mcli); - } - return ERR_PTR(rc); -} - -static void mctp_i2c_free_client(struct mctp_i2c_client *mcli) -{ - int rc; - - WARN_ON(!mutex_is_locked(&mi_driver_state.lock)); - WARN_ON(!list_empty(&mcli->devs)); - WARN_ON(mcli->sel); // sanity check, no locking - - device_remove_file(&mcli->client->dev, &dev_attr_mctp_current_mux); - rc = i2c_slave_unregister(mcli->client); - // leak if it fails, we can't propagate errors upwards - if (rc) - dev_err(&mcli->client->dev, "%s i2c unregister failed %d\n", __func__, rc); - else - kfree(mcli); -} - -/* Switch the mctp i2c device to receive responses. - * Call with curr_lock held - */ -static void __mctp_i2c_device_select(struct mctp_i2c_client *mcli, - struct mctp_i2c_dev *midev) -{ - assert_spin_locked(&mcli->curr_lock); - if (midev) - dev_hold(midev->ndev); - if (mcli->sel) - dev_put(mcli->sel->ndev); - mcli->sel = midev; -} - -// Switch the mctp i2c device to receive responses -static void mctp_i2c_device_select(struct mctp_i2c_client *mcli, - struct mctp_i2c_dev *midev) -{ - unsigned long flags; - - spin_lock_irqsave(&mcli->curr_lock, flags); - __mctp_i2c_device_select(mcli, midev); - spin_unlock_irqrestore(&mcli->curr_lock, flags); -} - -static int mctp_i2c_slave_cb(struct i2c_client *client, - enum i2c_slave_event event, u8 *val) -{ - struct mctp_i2c_client *mcli = i2c_get_clientdata(client); - struct mctp_i2c_dev *midev = NULL; - unsigned long flags; - int rc = 0; - - spin_lock_irqsave(&mcli->curr_lock, flags); - midev = mcli->sel; - if (midev) - dev_hold(midev->ndev); - spin_unlock_irqrestore(&mcli->curr_lock, flags); - - if (!midev) - return 0; - - switch (event) { - case I2C_SLAVE_WRITE_RECEIVED: - if (midev->pos < MCTP_I2C_RXBUFSZ) { - midev->buffer[midev->pos] = *val; - midev->pos++; - } else { - midev->ndev->stats.rx_over_errors++; - } - - break; - case I2C_SLAVE_WRITE_REQUESTED: - /* dest_slave as first byte */ - midev->buffer[0] = mcli->lladdr << 1; - midev->pos = 1; - break; - case I2C_SLAVE_STOP: - rc = mctp_i2c_recv(midev); - break; - default: - break; - } - - dev_put(midev->ndev); - return rc; -} - -// Processes incoming data that has been accumulated by the slave cb -static int mctp_i2c_recv(struct mctp_i2c_dev *midev) -{ - struct net_device *ndev = midev->ndev; - struct mctp_i2c_hdr *hdr; - struct mctp_skb_cb *cb; - struct sk_buff *skb; - u8 pec, calc_pec; - size_t recvlen; - - /* + 1 for the PEC */ - if (midev->pos < MCTP_I2C_MINLEN + 1) { - ndev->stats.rx_length_errors++; - return -EINVAL; - } - recvlen = midev->pos - 1; - - hdr = (void *)midev->buffer; - if (hdr->command != MCTP_I2C_COMMANDCODE) { - ndev->stats.rx_dropped++; - return -EINVAL; - } - - pec = midev->buffer[midev->pos - 1]; - calc_pec = i2c_smbus_pec(0, midev->buffer, recvlen); - if (pec != calc_pec) { - ndev->stats.rx_crc_errors++; - return -EINVAL; - } - - skb = netdev_alloc_skb(ndev, recvlen); - if (!skb) { - ndev->stats.rx_dropped++; - return -ENOMEM; - } - - skb->protocol = htons(ETH_P_MCTP); - skb_put_data(skb, midev->buffer, recvlen); - skb_reset_mac_header(skb); - skb_pull(skb, sizeof(struct mctp_i2c_hdr)); - skb_reset_network_header(skb); - - cb = __mctp_cb(skb); - cb->halen = 1; - cb->haddr[0] = hdr->source_slave; - - if (netif_rx(skb) == NET_RX_SUCCESS) { - ndev->stats.rx_packets++; - ndev->stats.rx_bytes += skb->len; - } else { - ndev->stats.rx_dropped++; - } - return 0; -} - -enum mctp_i2c_flow_state { - MCTP_I2C_TX_FLOW_INVALID, - MCTP_I2C_TX_FLOW_NONE, - MCTP_I2C_TX_FLOW_NEW, - MCTP_I2C_TX_FLOW_EXISTING, -}; - -static enum mctp_i2c_flow_state -mctp_i2c_get_tx_flow_state(struct mctp_i2c_dev *midev, struct sk_buff *skb) -{ - enum mctp_i2c_flow_state state; - struct mctp_sk_key *key; - struct mctp_flow *flow; - unsigned long flags; - - flow = skb_ext_find(skb, SKB_EXT_MCTP); - if (!flow) - return MCTP_I2C_TX_FLOW_NONE; - - key = flow->key; - if (!key) - return MCTP_I2C_TX_FLOW_NONE; - - spin_lock_irqsave(&key->lock, flags); - /* if the key is present but invalid, we're unlikely to be able - * to handle the flow at all; just drop now - */ - if (!key->valid) { - state = MCTP_I2C_TX_FLOW_INVALID; - - } else if (key->dev_flow_state == MCTP_I2C_FLOW_STATE_NEW) { - key->dev_flow_state = MCTP_I2C_FLOW_STATE_ACTIVE; - state = MCTP_I2C_TX_FLOW_NEW; - } else { - state = MCTP_I2C_TX_FLOW_EXISTING; - } - - spin_unlock_irqrestore(&key->lock, flags); - - return state; -} - -/* We're not contending with ourselves here; we only need to exclude other - * i2c clients from using the bus. refcounts are simply to prevent - * recursive locking. - */ -static void mctp_i2c_lock_nest(struct mctp_i2c_dev *midev) -{ - unsigned long flags; - bool lock; - - spin_lock_irqsave(&midev->flow_lock, flags); - lock = midev->i2c_lock_count == 0; - midev->i2c_lock_count++; - spin_unlock_irqrestore(&midev->flow_lock, flags); - - if (lock) - i2c_lock_bus(midev->adapter, I2C_LOCK_SEGMENT); -} - -static void mctp_i2c_unlock_nest(struct mctp_i2c_dev *midev) -{ - unsigned long flags; - bool unlock; - - spin_lock_irqsave(&midev->flow_lock, flags); - if (!WARN_ONCE(midev->i2c_lock_count == 0, "lock count underflow!")) - midev->i2c_lock_count--; - unlock = midev->i2c_lock_count == 0; - spin_unlock_irqrestore(&midev->flow_lock, flags); - - if (unlock) - i2c_unlock_bus(midev->adapter, I2C_LOCK_SEGMENT); -} - -static void mctp_i2c_xmit(struct mctp_i2c_dev *midev, struct sk_buff *skb) -{ - struct net_device_stats *stats = &midev->ndev->stats; - enum mctp_i2c_flow_state fs; - union i2c_smbus_data *data; - struct mctp_i2c_hdr *hdr; - unsigned int len; - u16 daddr; - int rc; - - fs = mctp_i2c_get_tx_flow_state(midev, skb); - - len = skb->len; - hdr = (void *)skb_mac_header(skb); - data = (void *)&hdr->byte_count; - daddr = hdr->dest_slave >> 1; - - switch (fs) { - case MCTP_I2C_TX_FLOW_NONE: - /* no flow: full lock & unlock */ - mctp_i2c_lock_nest(midev); - mctp_i2c_device_select(midev->client, midev); - rc = __i2c_smbus_xfer(midev->adapter, daddr, I2C_CLIENT_PEC, - I2C_SMBUS_WRITE, hdr->command, - I2C_SMBUS_BLOCK_DATA, data); - mctp_i2c_unlock_nest(midev); - break; - - case MCTP_I2C_TX_FLOW_NEW: - /* new flow: lock, tx, but don't unlock; that will happen - * on flow release - */ - mctp_i2c_lock_nest(midev); - mctp_i2c_device_select(midev->client, midev); - fallthrough; - - case MCTP_I2C_TX_FLOW_EXISTING: - /* existing flow: we already have the lock; just tx */ - rc = __i2c_smbus_xfer(midev->adapter, daddr, I2C_CLIENT_PEC, - I2C_SMBUS_WRITE, hdr->command, - I2C_SMBUS_BLOCK_DATA, data); - break; - - case MCTP_I2C_TX_FLOW_INVALID: - return; - } - - if (rc) { - dev_warn_ratelimited(&midev->adapter->dev, - "%s i2c_smbus_xfer failed %d", __func__, rc); - stats->tx_errors++; - } else { - stats->tx_bytes += len; - stats->tx_packets++; - } -} - -static void mctp_i2c_flow_release(struct mctp_i2c_dev *midev) -{ - unsigned long flags; - bool unlock; - - spin_lock_irqsave(&midev->flow_lock, flags); - if (midev->release_count > midev->i2c_lock_count) { - WARN_ONCE(1, "release count overflow"); - midev->release_count = midev->i2c_lock_count; - } - - midev->i2c_lock_count -= midev->release_count; - unlock = midev->i2c_lock_count == 0 && midev->release_count > 0; - midev->release_count = 0; - spin_unlock_irqrestore(&midev->flow_lock, flags); - - if (unlock) - i2c_unlock_bus(midev->adapter, I2C_LOCK_SEGMENT); -} - -static int mctp_i2c_header_create(struct sk_buff *skb, struct net_device *dev, - unsigned short type, const void *daddr, - const void *saddr, unsigned int len) -{ - struct mctp_i2c_hdr *hdr; - struct mctp_hdr *mhdr; - u8 lldst, llsrc; - - lldst = *((u8 *)daddr); - llsrc = *((u8 *)saddr); - - skb_push(skb, sizeof(struct mctp_i2c_hdr)); - skb_reset_mac_header(skb); - hdr = (void *)skb_mac_header(skb); - mhdr = mctp_hdr(skb); - hdr->dest_slave = (lldst << 1) & 0xff; - hdr->command = MCTP_I2C_COMMANDCODE; - hdr->byte_count = len + 1; - if (hdr->byte_count > MCTP_I2C_MAXBLOCK) - return -EMSGSIZE; - hdr->source_slave = ((llsrc << 1) & 0xff) | 0x01; - mhdr->ver = 0x01; - - return 0; -} - -static int mctp_i2c_tx_thread(void *data) -{ - struct mctp_i2c_dev *midev = data; - struct sk_buff *skb; - unsigned long flags; - - for (;;) { - if (kthread_should_stop()) - break; - - spin_lock_irqsave(&midev->tx_queue.lock, flags); - skb = __skb_dequeue(&midev->tx_queue); - if (netif_queue_stopped(midev->ndev)) - netif_wake_queue(midev->ndev); - spin_unlock_irqrestore(&midev->tx_queue.lock, flags); - - if (skb == &midev->unlock_marker) { - mctp_i2c_flow_release(midev); - - } else if (skb) { - mctp_i2c_xmit(midev, skb); - kfree_skb(skb); - - } else { - wait_event(midev->tx_wq, - !skb_queue_empty(&midev->tx_queue) || - kthread_should_stop()); - } - } - - return 0; -} - -static netdev_tx_t mctp_i2c_start_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - struct mctp_i2c_dev *midev = netdev_priv(dev); - unsigned long flags; - - spin_lock_irqsave(&midev->tx_queue.lock, flags); - if (skb_queue_len(&midev->tx_queue) >= MCTP_I2C_TX_WORK_LEN) { - netif_stop_queue(dev); - spin_unlock_irqrestore(&midev->tx_queue.lock, flags); - netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); - return NETDEV_TX_BUSY; - } - - __skb_queue_tail(&midev->tx_queue, skb); - if (skb_queue_len(&midev->tx_queue) == MCTP_I2C_TX_WORK_LEN) - netif_stop_queue(dev); - spin_unlock_irqrestore(&midev->tx_queue.lock, flags); - - wake_up(&midev->tx_wq); - return NETDEV_TX_OK; -} - -static void mctp_i2c_release_flow(struct mctp_dev *mdev, - struct mctp_sk_key *key) - -{ - struct mctp_i2c_dev *midev = netdev_priv(mdev->dev); - unsigned long flags; - - spin_lock_irqsave(&midev->flow_lock, flags); - midev->release_count++; - spin_unlock_irqrestore(&midev->flow_lock, flags); - - /* Ensure we have a release operation queued, through the fake - * marker skb - */ - spin_lock(&midev->tx_queue.lock); - if (!midev->unlock_marker.next) - __skb_queue_tail(&midev->tx_queue, &midev->unlock_marker); - spin_unlock(&midev->tx_queue.lock); - - wake_up(&midev->tx_wq); -} - -static const struct net_device_ops mctp_i2c_ops = { - .ndo_start_xmit = mctp_i2c_start_xmit, -}; - -static const struct header_ops mctp_i2c_headops = { - .create = mctp_i2c_header_create, -}; - -static const struct mctp_netdev_ops mctp_i2c_mctp_ops = { - .release_flow = mctp_i2c_release_flow, -}; - -static void mctp_i2c_net_setup(struct net_device *dev) -{ - dev->type = ARPHRD_MCTP; - - dev->mtu = MCTP_I2C_MAXMTU; - dev->min_mtu = MCTP_I2C_MINMTU; - dev->max_mtu = MCTP_I2C_MAXMTU; - dev->tx_queue_len = MCTP_I2C_TX_QUEUE_LEN; - - dev->hard_header_len = sizeof(struct mctp_i2c_hdr); - dev->addr_len = 1; - - dev->netdev_ops = &mctp_i2c_ops; - dev->header_ops = &mctp_i2c_headops; - dev->needs_free_netdev = true; -} - -static int mctp_i2c_add_netdev(struct mctp_i2c_client *mcli, - struct i2c_adapter *adap) -{ - unsigned long flags; - struct mctp_i2c_dev *midev = NULL; - struct net_device *ndev = NULL; - struct i2c_adapter *root; - char namebuf[30]; - int rc; - - root = mux_root_adapter(adap); - if (root != mcli->client->adapter) { - dev_err(&mcli->client->dev, - "I2C adapter %s is not a child bus of %s", - mcli->client->adapter->name, root->name); - return -EINVAL; - } - - WARN_ON(!mutex_is_locked(&mi_driver_state.lock)); - snprintf(namebuf, sizeof(namebuf), "mctpi2c%d", adap->nr); - ndev = alloc_netdev(sizeof(*midev), namebuf, NET_NAME_ENUM, mctp_i2c_net_setup); - if (!ndev) { - dev_err(&mcli->client->dev, "%s alloc netdev failed\n", __func__); - rc = -ENOMEM; - goto err; - } - dev_net_set(ndev, current->nsproxy->net_ns); - SET_NETDEV_DEV(ndev, &adap->dev); - ndev->dev_addr = &mcli->lladdr; - - midev = netdev_priv(ndev); - skb_queue_head_init(&midev->tx_queue); - INIT_LIST_HEAD(&midev->list); - midev->adapter = adap; - midev->client = mcli; - spin_lock_init(&midev->flow_lock); - midev->i2c_lock_count = 0; - midev->release_count = 0; - /* Hold references */ - get_device(&midev->adapter->dev); - get_device(&midev->client->client->dev); - midev->ndev = ndev; - init_waitqueue_head(&midev->tx_wq); - midev->tx_thread = kthread_create(mctp_i2c_tx_thread, midev, - "%s/tx", namebuf); - if (IS_ERR_OR_NULL(midev->tx_thread)) { - rc = -ENOMEM; - goto err_free; - } - - rc = mctp_register_netdev(ndev, &mctp_i2c_mctp_ops); - if (rc) { - dev_err(&mcli->client->dev, - "%s register netdev \"%s\" failed %d\n", __func__, - ndev->name, rc); - goto err_stop_kthread; - } - spin_lock_irqsave(&mcli->curr_lock, flags); - list_add(&midev->list, &mcli->devs); - // Select a device by default - if (!mcli->sel) - __mctp_i2c_device_select(mcli, midev); - spin_unlock_irqrestore(&mcli->curr_lock, flags); - - wake_up_process(midev->tx_thread); - - return 0; - -err_stop_kthread: - kthread_stop(midev->tx_thread); - -err_free: - free_netdev(ndev); - -err: - return rc; -} - -// Removes and unregisters a mctp-i2c netdev -static void mctp_i2c_free_netdev(struct mctp_i2c_dev *midev) -{ - struct mctp_i2c_client *mcli = midev->client; - unsigned long flags; - - netif_stop_queue(midev->ndev); - kthread_stop(midev->tx_thread); - skb_queue_purge(&midev->tx_queue); - - /* Release references, used only for TX which has stopped */ - put_device(&midev->adapter->dev); - put_device(&mcli->client->dev); - - /* Remove it from the parent mcli */ - spin_lock_irqsave(&mcli->curr_lock, flags); - list_del(&midev->list); - if (mcli->sel == midev) { - struct mctp_i2c_dev *first; - - first = list_first_entry_or_null(&mcli->devs, struct mctp_i2c_dev, list); - __mctp_i2c_device_select(mcli, first); - } - spin_unlock_irqrestore(&mcli->curr_lock, flags); - - /* Remove netdev. mctp_i2c_slave_cb() takes a dev_hold() so removing - * it now is safe. unregister_netdev() frees ndev and midev. - */ - mctp_unregister_netdev(midev->ndev); -} - -// Removes any netdev for adap. mcli is the parent root i2c client -static void mctp_i2c_remove_netdev(struct mctp_i2c_client *mcli, - struct i2c_adapter *adap) -{ - unsigned long flags; - struct mctp_i2c_dev *midev = NULL, *m = NULL; - - WARN_ON(!mutex_is_locked(&mi_driver_state.lock)); - spin_lock_irqsave(&mcli->curr_lock, flags); - // list size is limited by number of MCTP netdevs on a single hardware bus - list_for_each_entry(m, &mcli->devs, list) - if (m->adapter == adap) { - midev = m; - break; - } - spin_unlock_irqrestore(&mcli->curr_lock, flags); - - if (midev) - mctp_i2c_free_netdev(midev); -} - -/* Determines whether a device is an i2c adapter. - * Optionally returns the root i2c_adapter - */ -static struct i2c_adapter *mctp_i2c_get_adapter(struct device *dev, - struct i2c_adapter **ret_root) -{ - struct i2c_adapter *root, *adap; - - if (dev->type != &i2c_adapter_type) - return NULL; - adap = to_i2c_adapter(dev); - root = mux_root_adapter(adap); - WARN_ONCE(!root, "%s failed to find root adapter for %s\n", - __func__, dev_name(dev)); - if (!root) - return NULL; - if (ret_root) - *ret_root = root; - return adap; -} - -/* Determines whether a device is an i2c adapter with the "mctp-controller" - * devicetree property set. If adap is not an OF node, returns match_no_of - */ -static bool mctp_i2c_adapter_match(struct i2c_adapter *adap, bool match_no_of) -{ - if (!adap->dev.of_node) - return match_no_of; - return of_property_read_bool(adap->dev.of_node, MCTP_I2C_OF_PROP); -} - -/* Called for each existing i2c device (adapter or client) when a - * new mctp-i2c client is probed. - */ -static int mctp_i2c_client_try_attach(struct device *dev, void *data) -{ - struct i2c_adapter *adap = NULL, *root = NULL; - struct mctp_i2c_client *mcli = data; - - adap = mctp_i2c_get_adapter(dev, &root); - if (!adap) - return 0; - if (mcli->client->adapter != root) - return 0; - // Must either have mctp-controller property on the adapter, or - // be a root adapter if it's non-devicetree - if (!mctp_i2c_adapter_match(adap, adap == root)) - return 0; - - return mctp_i2c_add_netdev(mcli, adap); -} - -static void mctp_i2c_notify_add(struct device *dev) -{ - struct mctp_i2c_client *mcli = NULL, *m = NULL; - struct i2c_adapter *root = NULL, *adap = NULL; - int rc; - - adap = mctp_i2c_get_adapter(dev, &root); - if (!adap) - return; - // Check for mctp-controller property on the adapter - if (!mctp_i2c_adapter_match(adap, false)) - return; - - /* Find an existing mcli for adap's root */ - mutex_lock(&mi_driver_state.lock); - list_for_each_entry(m, &mi_driver_state.clients, list) { - if (m->client->adapter == root) { - mcli = m; - break; - } - } - - if (mcli) { - rc = mctp_i2c_add_netdev(mcli, adap); - if (rc) - dev_warn(dev, "%s Failed adding mctp-i2c device", - __func__); - } - mutex_unlock(&mi_driver_state.lock); -} - -static void mctp_i2c_notify_del(struct device *dev) -{ - struct i2c_adapter *root = NULL, *adap = NULL; - struct mctp_i2c_client *mcli = NULL; - - adap = mctp_i2c_get_adapter(dev, &root); - if (!adap) - return; - - mutex_lock(&mi_driver_state.lock); - list_for_each_entry(mcli, &mi_driver_state.clients, list) { - if (mcli->client->adapter == root) { - mctp_i2c_remove_netdev(mcli, adap); - break; - } - } - mutex_unlock(&mi_driver_state.lock); -} - -static int mctp_i2c_probe(struct i2c_client *client) -{ - struct mctp_i2c_client *mcli = NULL; - int rc; - - /* Check for >32 byte block support required for MCTP */ - if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_V3_BLOCK)) { - dev_err(&client->dev, - "%s failed, I2C bus driver does not support 255 byte block transfer\n", - __func__); - return -EOPNOTSUPP; - } - - mutex_lock(&mi_driver_state.lock); - mcli = mctp_i2c_new_client(client); - if (IS_ERR(mcli)) { - rc = PTR_ERR(mcli); - mcli = NULL; - goto out; - } else { - list_add(&mcli->list, &mi_driver_state.clients); - } - - // Add a netdev for adapters that have a 'mctp-controller' property - i2c_for_each_dev(mcli, mctp_i2c_client_try_attach); - rc = 0; -out: - mutex_unlock(&mi_driver_state.lock); - return rc; -} - -static int mctp_i2c_remove(struct i2c_client *client) -{ - struct mctp_i2c_client *mcli = i2c_get_clientdata(client); - struct mctp_i2c_dev *midev = NULL, *tmp = NULL; - - mutex_lock(&mi_driver_state.lock); - list_del(&mcli->list); - // Remove all child adapter netdevs - list_for_each_entry_safe(midev, tmp, &mcli->devs, list) - mctp_i2c_free_netdev(midev); - - mctp_i2c_free_client(mcli); - mutex_unlock(&mi_driver_state.lock); - // Callers ignore return code - return 0; -} - -/* We look for a 'mctp-controller' property on I2C busses as they are - * added/deleted, creating/removing netdevs as required. - */ -static int mctp_i2c_notifier_call(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct device *dev = data; - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - mctp_i2c_notify_add(dev); - break; - case BUS_NOTIFY_DEL_DEVICE: - mctp_i2c_notify_del(dev); - break; - } - return NOTIFY_DONE; -} - -static struct notifier_block mctp_i2c_notifier = { - .notifier_call = mctp_i2c_notifier_call, -}; - -static const struct i2c_device_id mctp_i2c_id[] = { - { "mctp-i2c", 0 }, - {}, -}; -MODULE_DEVICE_TABLE(i2c, mctp_i2c_id); - -static const struct of_device_id mctp_i2c_of_match[] = { - { .compatible = "mctp-i2c-controller" }, - {}, -}; -MODULE_DEVICE_TABLE(of, mctp_i2c_of_match); - -static struct i2c_driver mctp_i2c_driver = { - .driver = { - .name = "mctp-i2c", - .of_match_table = mctp_i2c_of_match, - }, - .probe_new = mctp_i2c_probe, - .remove = mctp_i2c_remove, - .id_table = mctp_i2c_id, -}; - -static __init int mctp_i2c_init(void) -{ - int rc; - - INIT_LIST_HEAD(&mi_driver_state.clients); - mutex_init(&mi_driver_state.lock); - pr_info("MCTP SMBus/I2C transport driver\n"); - rc = i2c_add_driver(&mctp_i2c_driver); - if (rc) - return rc; - rc = bus_register_notifier(&i2c_bus_type, &mctp_i2c_notifier); - if (rc) { - i2c_del_driver(&mctp_i2c_driver); - return rc; - } - return 0; -} - -static __exit void mctp_i2c_exit(void) -{ - int rc; - - rc = bus_unregister_notifier(&i2c_bus_type, &mctp_i2c_notifier); - if (rc) - pr_warn("%s Could not unregister notifier, %d", __func__, rc); - i2c_del_driver(&mctp_i2c_driver); -} - -module_init(mctp_i2c_init); -module_exit(mctp_i2c_exit); - -MODULE_DESCRIPTION("MCTP SMBus/I2C device"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Matt Johnston "); diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 353d6b4e7a53..16119ac1aa97 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -52,19 +52,6 @@ typedef int (*i2c_slave_cb_t)(struct i2c_client *client, struct module; struct property_entry; -/* SMBus 3.0 extends the maximum block read/write size to 255 (from 32). - * The larger size is only supported by some drivers, indicated by - * the I2C_FUNC_SMBUS_V3_BLOCK functionality bit. - */ -#define I2C_SMBUS_V3_BLOCK_MAX 255 /* As specified in SMBus 3.0 standard */ - -/* Note compatibility definition in uapi header with 32 byte block */ -union i2c_smbus_data { - __u8 byte; - __u16 word; - __u8 block[I2C_SMBUS_V3_BLOCK_MAX + 1]; /* block[0] is used for length */ -}; - #if IS_ENABLED(CONFIG_I2C) /* Return the Frequency mode string based on the bus frequency */ const char *i2c_freq_mode_string(u32 bus_freq_hz); diff --git a/include/uapi/linux/i2c-dev.h b/include/uapi/linux/i2c-dev.h index 46ce31d42f7d..1c4cec4ddd84 100644 --- a/include/uapi/linux/i2c-dev.h +++ b/include/uapi/linux/i2c-dev.h @@ -39,14 +39,12 @@ /* This is the structure as used in the I2C_SMBUS ioctl call */ -#ifndef __KERNEL__ struct i2c_smbus_ioctl_data { __u8 read_write; __u8 command; __u32 size; union i2c_smbus_data __user *data; }; -#endif /* This is the structure as used in the I2C_RDWR ioctl call */ struct i2c_rdwr_ioctl_data { diff --git a/include/uapi/linux/i2c.h b/include/uapi/linux/i2c.h index c3534ab1ae53..92326ebde350 100644 --- a/include/uapi/linux/i2c.h +++ b/include/uapi/linux/i2c.h @@ -108,9 +108,6 @@ struct i2c_msg { #define I2C_FUNC_SMBUS_READ_I2C_BLOCK 0x04000000 /* I2C-like block xfer */ #define I2C_FUNC_SMBUS_WRITE_I2C_BLOCK 0x08000000 /* w/ 1-byte reg. addr. */ #define I2C_FUNC_SMBUS_HOST_NOTIFY 0x10000000 /* SMBus 2.0 or later */ -#define I2C_FUNC_SMBUS_V3_BLOCK 0x20000000 /* Device supports 255 byte block */ - /* Note that I2C_SMBUS ioctl only */ - /* supports a 32 byte block */ #define I2C_FUNC_SMBUS_BYTE (I2C_FUNC_SMBUS_READ_BYTE | \ I2C_FUNC_SMBUS_WRITE_BYTE) @@ -140,15 +137,13 @@ struct i2c_msg { /* * Data for SMBus Messages */ -#define I2C_SMBUS_BLOCK_MAX 32 /* As specified in SMBus 2.0 standard */ -#ifndef __KERNEL__ +#define I2C_SMBUS_BLOCK_MAX 32 /* As specified in SMBus standard */ union i2c_smbus_data { __u8 byte; __u16 word; __u8 block[I2C_SMBUS_BLOCK_MAX + 2]; /* block[0] is used for length */ /* and one more for user-space compatibility */ }; -#endif /* i2c_smbus_xfer read or write markers */ #define I2C_SMBUS_READ 1 -- cgit v1.2.3-59-g8ed1b From f64bd790b750dd281406964af40d16adfc88a074 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 29 Oct 2021 12:51:32 -0700 Subject: ACPI: Keep sub-table parsing infrastructure available for modules The NFIT driver and now the CXL ACPI driver have both open-coded ACPI table parsing. Before another instance is added arrange for the core ACPI sub-table parsing to be optionally available to drivers via the CONFIG_ACPI_TABLE_LIB symbol. If no drivers select the symbol then the infrastructure reverts back to being tagged __init via the __init_or_acpilib annotation. For now, only tag the core sub-table routines and data that the CEDT parsing in the cxl_acpi driver would want to reuse, a CEDT parsing helper is added in a later change. Cc: "Rafael J. Wysocki" Cc: Len Brown Cc: Alison Schofield Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/163553709227.2509508.8215196520233473814.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/acpi/Kconfig | 3 +++ drivers/acpi/tables.c | 27 +++++++++++++-------------- include/linux/acpi.h | 22 +++++++++++++++------- 3 files changed, 31 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index cdbdf68bd98f..c97ee0cfe26e 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -59,6 +59,9 @@ config ACPI_SYSTEM_POWER_STATES_SUPPORT config ACPI_CCA_REQUIRED bool +config ACPI_TABLE_LIB + bool + config ACPI_DEBUGGER bool "AML debugger interface" select ACPI_DEBUG diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 71419eb16e09..b80a3de655d7 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -35,7 +35,7 @@ static char *mps_inti_flags_trigger[] = { "dfl", "edge", "res", "level" }; static struct acpi_table_desc initial_tables[ACPI_MAX_TABLES] __initdata; -static int acpi_apic_instance __initdata; +static int acpi_apic_instance __initdata_or_acpilib; enum acpi_subtable_type { ACPI_SUBTABLE_COMMON, @@ -52,7 +52,7 @@ struct acpi_subtable_entry { * Disable table checksum verification for the early stage due to the size * limitation of the current x86 early mapping implementation. */ -static bool acpi_verify_table_checksum __initdata = false; +static bool acpi_verify_table_checksum __initdata_or_acpilib = false; void acpi_table_print_madt_entry(struct acpi_subtable_header *header) { @@ -216,7 +216,7 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) } } -static unsigned long __init +static unsigned long __init_or_acpilib acpi_get_entry_type(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -230,7 +230,7 @@ acpi_get_entry_type(struct acpi_subtable_entry *entry) return 0; } -static unsigned long __init +static unsigned long __init_or_acpilib acpi_get_entry_length(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -244,7 +244,7 @@ acpi_get_entry_length(struct acpi_subtable_entry *entry) return 0; } -static unsigned long __init +static unsigned long __init_or_acpilib acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) { switch (entry->type) { @@ -258,7 +258,7 @@ acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) return 0; } -static enum acpi_subtable_type __init +static enum acpi_subtable_type __init_or_acpilib acpi_get_subtable_type(char *id) { if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) @@ -291,10 +291,10 @@ acpi_get_subtable_type(char *id) * On success returns sum of all matching entries for all proc handlers. * Otherwise, -ENODEV or -EINVAL is returned. */ -static int __init acpi_parse_entries_array(char *id, unsigned long table_size, - struct acpi_table_header *table_header, - struct acpi_subtable_proc *proc, int proc_num, - unsigned int max_entries) +static int __init_or_acpilib acpi_parse_entries_array( + char *id, unsigned long table_size, + struct acpi_table_header *table_header, struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries) { struct acpi_subtable_entry entry; unsigned long table_end, subtable_len, entry_len; @@ -352,10 +352,9 @@ static int __init acpi_parse_entries_array(char *id, unsigned long table_size, return errs ? -EINVAL : count; } -int __init acpi_table_parse_entries_array(char *id, - unsigned long table_size, - struct acpi_subtable_proc *proc, int proc_num, - unsigned int max_entries) +int __init_or_acpilib acpi_table_parse_entries_array( + char *id, unsigned long table_size, struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries) { struct acpi_table_header *table_header = NULL; int count; diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 143ce7e0bee1..edfa3c8f3562 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -232,14 +232,22 @@ int acpi_locate_initial_tables (void); void acpi_reserve_initial_tables (void); void acpi_table_init_complete (void); int acpi_table_init (void); + +#ifdef CONFIG_ACPI_TABLE_LIB +#define __init_or_acpilib +#define __initdata_or_acpilib +#else +#define __init_or_acpilib __init +#define __initdata_or_acpilib __initdata +#endif + int acpi_table_parse(char *id, acpi_tbl_table_handler handler); -int __init acpi_table_parse_entries(char *id, unsigned long table_size, - int entry_id, - acpi_tbl_entry_handler handler, - unsigned int max_entries); -int __init acpi_table_parse_entries_array(char *id, unsigned long table_size, - struct acpi_subtable_proc *proc, int proc_num, - unsigned int max_entries); +int __init_or_acpilib acpi_table_parse_entries(char *id, + unsigned long table_size, int entry_id, + acpi_tbl_entry_handler handler, unsigned int max_entries); +int __init_or_acpilib acpi_table_parse_entries_array(char *id, + unsigned long table_size, struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries); int acpi_table_parse_madt(enum acpi_madt_type id, acpi_tbl_entry_handler handler, unsigned int max_entries); -- cgit v1.2.3-59-g8ed1b From ad2f63971e9655e3987db32dac85aa50658790eb Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 29 Oct 2021 12:51:37 -0700 Subject: ACPI: Teach ACPI table parsing about the CEDT header format The CEDT adds yet one more unique subtable header type where the length is a 16-bit value. Extend the subtable helpers to detect this scenario. Cc: "Rafael J. Wysocki" Cc: Len Brown Tested-by: Alison Schofield Reviewed-by: Alison Schofield Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/163553709742.2509508.5177761945441327574.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/acpi/tables.c | 9 +++++++++ include/linux/acpi.h | 1 + 2 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index b80a3de655d7..8d052b65f6bc 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -41,6 +41,7 @@ enum acpi_subtable_type { ACPI_SUBTABLE_COMMON, ACPI_SUBTABLE_HMAT, ACPI_SUBTABLE_PRMT, + ACPI_SUBTABLE_CEDT, }; struct acpi_subtable_entry { @@ -226,6 +227,8 @@ acpi_get_entry_type(struct acpi_subtable_entry *entry) return entry->hdr->hmat.type; case ACPI_SUBTABLE_PRMT: return 0; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.type; } return 0; } @@ -240,6 +243,8 @@ acpi_get_entry_length(struct acpi_subtable_entry *entry) return entry->hdr->hmat.length; case ACPI_SUBTABLE_PRMT: return entry->hdr->prmt.length; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.length; } return 0; } @@ -254,6 +259,8 @@ acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) return sizeof(entry->hdr->hmat); case ACPI_SUBTABLE_PRMT: return sizeof(entry->hdr->prmt); + case ACPI_SUBTABLE_CEDT: + return sizeof(entry->hdr->cedt); } return 0; } @@ -265,6 +272,8 @@ acpi_get_subtable_type(char *id) return ACPI_SUBTABLE_HMAT; if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) return ACPI_SUBTABLE_PRMT; + if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) + return ACPI_SUBTABLE_CEDT; return ACPI_SUBTABLE_COMMON; } diff --git a/include/linux/acpi.h b/include/linux/acpi.h index edfa3c8f3562..6b7f181d51e2 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -133,6 +133,7 @@ union acpi_subtable_headers { struct acpi_subtable_header common; struct acpi_hmat_structure hmat; struct acpi_prmt_module_header prmt; + struct acpi_cedt_header cedt; }; typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table); -- cgit v1.2.3-59-g8ed1b From 2d03e46a4bad20191d07b83ec1242d5f002577be Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Fri, 29 Oct 2021 12:51:42 -0700 Subject: ACPI: Add a context argument for table parsing handlers In preparation for drivers reusing the core table parsing infrastructure, arrange for handlers to take a context argument. This allows driver table parsing to wrap ACPI table entries in driver-specific data. The first consumer of this infrastructure is the CEDT parsing that happens in the cxl_acpi driver, add a conditional (CONFIG_ACPI_TABLE_LIB=y) export of a acpi_table_parse_cedt() helper for this case. Cc: "Rafael J. Wysocki" Cc: Len Brown Tested-by: Alison Schofield Reviewed-by: Alison Schofield Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/163553710257.2509508.14310494417463866020.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Dan Williams --- drivers/acpi/tables.c | 51 ++++++++++++++++++++++++++++++++++++++++++++------- include/linux/acpi.h | 11 +++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 8d052b65f6bc..e9bdbb6fbc36 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -277,6 +277,22 @@ acpi_get_subtable_type(char *id) return ACPI_SUBTABLE_COMMON; } +static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) +{ + return proc->handler || proc->handler_arg; +} + +static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, + union acpi_subtable_headers *hdr, + unsigned long end) +{ + if (proc->handler) + return proc->handler(hdr, end); + if (proc->handler_arg) + return proc->handler_arg(hdr, proc->arg, end); + return -EINVAL; +} + /** * acpi_parse_entries_array - for each proc_num find a suitable subtable * @@ -327,8 +343,9 @@ static int __init_or_acpilib acpi_parse_entries_array( for (i = 0; i < proc_num; i++) { if (acpi_get_entry_type(&entry) != proc[i].id) continue; - if (!proc[i].handler || - (!errs && proc[i].handler(entry.hdr, table_end))) { + if (!has_handler(&proc[i]) || + (!errs && + call_handler(&proc[i], entry.hdr, table_end))) { errs++; continue; } @@ -394,21 +411,41 @@ int __init_or_acpilib acpi_table_parse_entries_array( return count; } -int __init acpi_table_parse_entries(char *id, - unsigned long table_size, - int entry_id, - acpi_tbl_entry_handler handler, - unsigned int max_entries) +static int __init_or_acpilib __acpi_table_parse_entries( + char *id, unsigned long table_size, int entry_id, + acpi_tbl_entry_handler handler, acpi_tbl_entry_handler_arg handler_arg, + void *arg, unsigned int max_entries) { struct acpi_subtable_proc proc = { .id = entry_id, .handler = handler, + .handler_arg = handler_arg, + .arg = arg, }; return acpi_table_parse_entries_array(id, table_size, &proc, 1, max_entries); } +int __init_or_acpilib +acpi_table_parse_cedt(enum acpi_cedt_type id, + acpi_tbl_entry_handler_arg handler_arg, void *arg) +{ + return __acpi_table_parse_entries(ACPI_SIG_CEDT, + sizeof(struct acpi_table_cedt), id, + NULL, handler_arg, arg, 0); +} +EXPORT_SYMBOL_ACPI_LIB(acpi_table_parse_cedt); + +int __init acpi_table_parse_entries(char *id, unsigned long table_size, + int entry_id, + acpi_tbl_entry_handler handler, + unsigned int max_entries) +{ + return __acpi_table_parse_entries(id, table_size, entry_id, handler, + NULL, NULL, max_entries); +} + int __init acpi_table_parse_madt(enum acpi_madt_type id, acpi_tbl_entry_handler handler, unsigned int max_entries) { diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 6b7f181d51e2..95f88108f664 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -141,6 +141,9 @@ typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table); typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header, const unsigned long end); +typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header, + void *arg, const unsigned long end); + /* Debugger support */ struct acpi_debugger_ops { @@ -217,6 +220,8 @@ static inline int acpi_debugger_notify_command_complete(void) struct acpi_subtable_proc { int id; acpi_tbl_entry_handler handler; + acpi_tbl_entry_handler_arg handler_arg; + void *arg; int count; }; @@ -235,9 +240,11 @@ void acpi_table_init_complete (void); int acpi_table_init (void); #ifdef CONFIG_ACPI_TABLE_LIB +#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) #define __init_or_acpilib #define __initdata_or_acpilib #else +#define EXPORT_SYMBOL_ACPI_LIB(x) #define __init_or_acpilib __init #define __initdata_or_acpilib __initdata #endif @@ -252,6 +259,10 @@ int __init_or_acpilib acpi_table_parse_entries_array(char *id, int acpi_table_parse_madt(enum acpi_madt_type id, acpi_tbl_entry_handler handler, unsigned int max_entries); +int __init_or_acpilib +acpi_table_parse_cedt(enum acpi_cedt_type id, + acpi_tbl_entry_handler_arg handler_arg, void *arg); + int acpi_parse_mcfg (struct acpi_table_header *header); void acpi_table_print_madt_entry (struct acpi_subtable_header *madt); -- cgit v1.2.3-59-g8ed1b From 03b122da74b22fbe7cd98184fa5657a9ce13970c Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Tue, 26 Oct 2021 15:00:48 -0700 Subject: x86/sgx: Hook arch_memory_failure() into mainline code Add a call inside memory_failure() to call the arch specific code to check if the address is an SGX EPC page and handle it. Note the SGX EPC pages do not have a "struct page" entry, so the hook goes in at the same point as the device mapping hook. Pull the call to acquire the mutex earlier so the SGX errors are also protected. Make set_mce_nospec() skip SGX pages when trying to adjust the 1:1 map. Signed-off-by: Tony Luck Signed-off-by: Dave Hansen Reviewed-by: Jarkko Sakkinen Reviewed-by: Naoya Horiguchi Tested-by: Reinette Chatre Link: https://lkml.kernel.org/r/20211026220050.697075-6-tony.luck@intel.com --- arch/x86/include/asm/processor.h | 8 ++++++++ arch/x86/include/asm/set_memory.h | 4 ++++ include/linux/mm.h | 13 +++++++++++++ mm/memory-failure.c | 19 +++++++++++++------ 4 files changed, 38 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 355d38c0cf60..2c5f12ae7d04 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -855,4 +855,12 @@ enum mds_mitigations { MDS_MITIGATION_VMWERV, }; +#ifdef CONFIG_X86_SGX +int arch_memory_failure(unsigned long pfn, int flags); +#define arch_memory_failure arch_memory_failure + +bool arch_is_platform_page(u64 paddr); +#define arch_is_platform_page arch_is_platform_page +#endif + #endif /* _ASM_X86_PROCESSOR_H */ diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 872617542bbc..ff0f2d90338a 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -2,6 +2,7 @@ #ifndef _ASM_X86_SET_MEMORY_H #define _ASM_X86_SET_MEMORY_H +#include #include #include @@ -99,6 +100,9 @@ static inline int set_mce_nospec(unsigned long pfn, bool unmap) unsigned long decoy_addr; int rc; + /* SGX pages are not in the 1:1 map */ + if (arch_is_platform_page(pfn << PAGE_SHIFT)) + return 0; /* * We would like to just call: * set_memory_XX((unsigned long)pfn_to_kaddr(pfn), 1); diff --git a/include/linux/mm.h b/include/linux/mm.h index a7e4a9e7d807..57f1aa2a33b6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3231,6 +3231,19 @@ extern void shake_page(struct page *p); extern atomic_long_t num_poisoned_pages __read_mostly; extern int soft_offline_page(unsigned long pfn, int flags); +#ifndef arch_memory_failure +static inline int arch_memory_failure(unsigned long pfn, int flags) +{ + return -ENXIO; +} +#endif + +#ifndef arch_is_platform_page +static inline bool arch_is_platform_page(u64 paddr) +{ + return false; +} +#endif /* * Error handlers for various types of pages. diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 07c875fdeaf0..fddee33bc6cb 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1651,21 +1651,28 @@ int memory_failure(unsigned long pfn, int flags) if (!sysctl_memory_failure_recovery) panic("Memory failure on page %lx", pfn); + mutex_lock(&mf_mutex); + p = pfn_to_online_page(pfn); if (!p) { + res = arch_memory_failure(pfn, flags); + if (res == 0) + goto unlock_mutex; + if (pfn_valid(pfn)) { pgmap = get_dev_pagemap(pfn, NULL); - if (pgmap) - return memory_failure_dev_pagemap(pfn, flags, - pgmap); + if (pgmap) { + res = memory_failure_dev_pagemap(pfn, flags, + pgmap); + goto unlock_mutex; + } } pr_err("Memory failure: %#lx: memory outside kernel control\n", pfn); - return -ENXIO; + res = -ENXIO; + goto unlock_mutex; } - mutex_lock(&mf_mutex); - try_again: if (PageHuge(p)) { res = memory_failure_hugetlb(pfn, flags); -- cgit v1.2.3-59-g8ed1b From 749303055b78bc38ec0790ccc596cae235446367 Mon Sep 17 00:00:00 2001 From: Simon Trimmer Date: Mon, 15 Nov 2021 12:02:15 +0000 Subject: firmware: cs_dsp: tidy includes in cs_dsp.c and cs_dsp.h This patch removes unused included header files and moves others into cs_dsp.h to ensure that types referenced in the header file are properly described to prevent compiler warnings. Signed-off-by: Simon Trimmer Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20211115120215.56824-1-simont@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 6 ------ include/linux/firmware/cirrus/cs_dsp.h | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 948dd8382686..1a0c6c793f6a 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -12,16 +12,10 @@ #include #include #include -#include -#include -#include -#include #include #include -#include #include #include -#include #include #include diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 9ad9eaaaa552..3a54b1afc48f 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -11,6 +11,11 @@ #ifndef __CS_DSP_H #define __CS_DSP_H +#include +#include +#include +#include + #define CS_ADSP2_REGION_0 BIT(0) #define CS_ADSP2_REGION_1 BIT(1) #define CS_ADSP2_REGION_2 BIT(2) -- cgit v1.2.3-59-g8ed1b From 0f0ac158d28ff78e75c334e869b1cb8e69372a1f Mon Sep 17 00:00:00 2001 From: "Luke D. Jones" Date: Sun, 24 Oct 2021 16:37:05 +1300 Subject: platform/x86: asus-wmi: Add support for custom fan curves Add support for custom fan curves found on some ASUS ROG laptops. These laptops have the ability to set a custom curve for the CPU and GPU fans via two ACPI methods. This patch adds two pwm attributes to the hwmon sysfs, pwm1 for CPU fan, pwm2 for GPU fan. Both are under the hwmon of the name `asus_custom_fan_curve`. There is no safety check of the set fan curves - this must be done in userspace. The fans have settings [1,2,3] under pwm_enable: 1. Enable and write settings out 2. Disable and use factory fan mode 3. Same as 2, additionally restoring default factory curve. Use of 2 means that the curve the user has set is still stored and won't be erased, but the laptop will be using its default auto-fan mode. Re-enabling the manual mode then activates the curves again. Notes: - pwm_enable = 0 is an invalid setting. - pwm is actually a percentage and is scaled on writing to device. Signed-off-by: Luke D. Jones Link: https://lore.kernel.org/r/20211024033705.5595-2-luke@ljones.dev Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/asus-wmi.c | 567 ++++++++++++++++++++++++++++- include/linux/platform_data/x86/asus-wmi.h | 2 + 2 files changed, 564 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 8f067ac4e952..ab0c087d40cf 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -106,8 +106,17 @@ module_param(fnlock_default, bool, 0444); #define WMI_EVENT_MASK 0xFFFF +#define FAN_CURVE_POINTS 8 +#define FAN_CURVE_BUF_LEN (FAN_CURVE_POINTS * 2) +#define FAN_CURVE_DEV_CPU 0x00 +#define FAN_CURVE_DEV_GPU 0x01 +/* Mask to determine if setting temperature or percentage */ +#define FAN_CURVE_PWM_MASK 0x04 + static const char * const ashs_ids[] = { "ATK4001", "ATK4002", NULL }; +static int throttle_thermal_policy_write(struct asus_wmi *); + static bool ashs_present(void) { int i = 0; @@ -122,7 +131,8 @@ struct bios_args { u32 arg0; u32 arg1; u32 arg2; /* At least TUF Gaming series uses 3 dword input buffer. */ - u32 arg4; + u32 arg3; + u32 arg4; /* Some ROG laptops require a full 5 input args */ u32 arg5; } __packed; @@ -173,6 +183,13 @@ enum fan_type { FAN_TYPE_SPEC83, /* starting in Spec 8.3, use CPU_FAN_CTRL */ }; +struct fan_curve_data { + bool enabled; + u32 device_id; + u8 temps[FAN_CURVE_POINTS]; + u8 percents[FAN_CURVE_POINTS]; +}; + struct asus_wmi { int dsts_id; int spec; @@ -220,6 +237,10 @@ struct asus_wmi { bool throttle_thermal_policy_available; u8 throttle_thermal_policy_mode; + bool cpu_fan_curve_available; + bool gpu_fan_curve_available; + struct fan_curve_data custom_fan_curves[2]; + struct platform_profile_handler platform_profile_handler; bool platform_profile_support; @@ -285,6 +306,103 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval) } EXPORT_SYMBOL_GPL(asus_wmi_evaluate_method); +static int asus_wmi_evaluate_method5(u32 method_id, + u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4, u32 *retval) +{ + struct bios_args args = { + .arg0 = arg0, + .arg1 = arg1, + .arg2 = arg2, + .arg3 = arg3, + .arg4 = arg4, + }; + struct acpi_buffer input = { (acpi_size) sizeof(args), &args }; + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_status status; + union acpi_object *obj; + u32 tmp = 0; + + status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID, 0, method_id, + &input, &output); + + if (ACPI_FAILURE(status)) + return -EIO; + + obj = (union acpi_object *)output.pointer; + if (obj && obj->type == ACPI_TYPE_INTEGER) + tmp = (u32) obj->integer.value; + + if (retval) + *retval = tmp; + + kfree(obj); + + if (tmp == ASUS_WMI_UNSUPPORTED_METHOD) + return -ENODEV; + + return 0; +} + +/* + * Returns as an error if the method output is not a buffer. Typically this + * means that the method called is unsupported. + */ +static int asus_wmi_evaluate_method_buf(u32 method_id, + u32 arg0, u32 arg1, u8 *ret_buffer, size_t size) +{ + struct bios_args args = { + .arg0 = arg0, + .arg1 = arg1, + .arg2 = 0, + }; + struct acpi_buffer input = { (acpi_size) sizeof(args), &args }; + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_status status; + union acpi_object *obj; + int err = 0; + + status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID, 0, method_id, + &input, &output); + + if (ACPI_FAILURE(status)) + return -EIO; + + obj = (union acpi_object *)output.pointer; + + switch (obj->type) { + case ACPI_TYPE_BUFFER: + if (obj->buffer.length > size) + err = -ENOSPC; + if (obj->buffer.length == 0) + err = -ENODATA; + + memcpy(ret_buffer, obj->buffer.pointer, obj->buffer.length); + break; + case ACPI_TYPE_INTEGER: + err = (u32)obj->integer.value; + + if (err == ASUS_WMI_UNSUPPORTED_METHOD) + err = -ENODEV; + /* + * At least one method returns a 0 with no buffer if no arg + * is provided, such as ASUS_WMI_DEVID_CPU_FAN_CURVE + */ + if (err == 0) + err = -ENODATA; + break; + default: + err = -ENODATA; + break; + } + + kfree(obj); + + if (err) + return err; + + return 0; +} + static int asus_wmi_evaluate_method_agfn(const struct acpi_buffer args) { struct acpi_buffer input; @@ -1806,6 +1924,13 @@ static ssize_t pwm1_enable_store(struct device *dev, } asus->fan_pwm_mode = state; + + /* Must set to disabled if mode is toggled */ + if (asus->cpu_fan_curve_available) + asus->custom_fan_curves[FAN_CURVE_DEV_CPU].enabled = false; + if (asus->gpu_fan_curve_available) + asus->custom_fan_curves[FAN_CURVE_DEV_GPU].enabled = false; + return count; } @@ -1953,9 +2078,9 @@ static int fan_boost_mode_check_present(struct asus_wmi *asus) static int fan_boost_mode_write(struct asus_wmi *asus) { - int err; - u8 value; u32 retval; + u8 value; + int err; value = asus->fan_boost_mode; @@ -2013,10 +2138,10 @@ static ssize_t fan_boost_mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - int result; - u8 new_mode; struct asus_wmi *asus = dev_get_drvdata(dev); u8 mask = asus->fan_boost_mode_mask; + u8 new_mode; + int result; result = kstrtou8(buf, 10, &new_mode); if (result < 0) { @@ -2043,6 +2168,426 @@ static ssize_t fan_boost_mode_store(struct device *dev, // Fan boost mode: 0 - normal, 1 - overboost, 2 - silent static DEVICE_ATTR_RW(fan_boost_mode); +/* Custom fan curves **********************************************************/ + +static void fan_curve_copy_from_buf(struct fan_curve_data *data, u8 *buf) +{ + int i; + + for (i = 0; i < FAN_CURVE_POINTS; i++) { + data->temps[i] = buf[i]; + } + + for (i = 0; i < FAN_CURVE_POINTS; i++) { + data->percents[i] = + 255 * buf[i + FAN_CURVE_POINTS] / 100; + } +} + +static int fan_curve_get_factory_default(struct asus_wmi *asus, u32 fan_dev) +{ + struct fan_curve_data *curves; + u8 buf[FAN_CURVE_BUF_LEN]; + int fan_idx = 0; + u8 mode = 0; + int err; + + if (asus->throttle_thermal_policy_available) + mode = asus->throttle_thermal_policy_mode; + /* DEVID_PU_FAN_CURVE is switched for OVERBOOST vs SILENT */ + if (mode == 2) + mode = 1; + else if (mode == 1) + mode = 2; + + if (fan_dev == ASUS_WMI_DEVID_GPU_FAN_CURVE) + fan_idx = FAN_CURVE_DEV_GPU; + + curves = &asus->custom_fan_curves[fan_idx]; + err = asus_wmi_evaluate_method_buf(asus->dsts_id, fan_dev, mode, buf, + FAN_CURVE_BUF_LEN); + if (err) + return err; + + fan_curve_copy_from_buf(curves, buf); + curves->device_id = fan_dev; + + return 0; +} + +/* Check if capability exists, and populate defaults */ +static int fan_curve_check_present(struct asus_wmi *asus, bool *available, + u32 fan_dev) +{ + int err; + + *available = false; + + err = fan_curve_get_factory_default(asus, fan_dev); + if (err) { + if (err == -ENODEV) + return 0; + return err; + } + + *available = true; + return 0; +} + +/* Determine which fan the attribute is for if SENSOR_ATTR */ +static struct fan_curve_data *fan_curve_attr_select(struct asus_wmi *asus, + struct device_attribute *attr) +{ + int index = to_sensor_dev_attr(attr)->index; + + return &asus->custom_fan_curves[index & FAN_CURVE_DEV_GPU]; +} + +/* Determine which fan the attribute is for if SENSOR_ATTR_2 */ +static struct fan_curve_data *fan_curve_attr_2_select(struct asus_wmi *asus, + struct device_attribute *attr) +{ + int nr = to_sensor_dev_attr_2(attr)->nr; + + return &asus->custom_fan_curves[nr & FAN_CURVE_DEV_GPU]; +} + +static ssize_t fan_curve_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sensor_device_attribute_2 *dev_attr = to_sensor_dev_attr_2(attr); + struct asus_wmi *asus = dev_get_drvdata(dev); + struct fan_curve_data *data; + int value, index, nr; + + data = fan_curve_attr_2_select(asus, attr); + index = dev_attr->index; + nr = dev_attr->nr; + + if (nr & FAN_CURVE_PWM_MASK) + value = data->percents[index]; + else + value = data->temps[index]; + + return sysfs_emit(buf, "%d\n", value); +} + +/* + * "fan_dev" is the related WMI method such as ASUS_WMI_DEVID_CPU_FAN_CURVE. + */ +static int fan_curve_write(struct asus_wmi *asus, + struct fan_curve_data *data) +{ + u32 arg1 = 0, arg2 = 0, arg3 = 0, arg4 = 0; + u8 *percents = data->percents; + u8 *temps = data->temps; + int ret, i, shift = 0; + + if (!data->enabled) + return 0; + + for (i = 0; i < FAN_CURVE_POINTS / 2; i++) { + arg1 += (temps[i]) << shift; + arg2 += (temps[i + 4]) << shift; + /* Scale to percentage for device */ + arg3 += (100 * percents[i] / 255) << shift; + arg4 += (100 * percents[i + 4] / 255) << shift; + shift += 8; + } + + return asus_wmi_evaluate_method5(ASUS_WMI_METHODID_DEVS, + data->device_id, + arg1, arg2, arg3, arg4, &ret); +} + +static ssize_t fan_curve_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + struct sensor_device_attribute_2 *dev_attr = to_sensor_dev_attr_2(attr); + struct asus_wmi *asus = dev_get_drvdata(dev); + struct fan_curve_data *data; + u8 value; + int err; + + int pwm = dev_attr->nr & FAN_CURVE_PWM_MASK; + int index = dev_attr->index; + + data = fan_curve_attr_2_select(asus, attr); + + err = kstrtou8(buf, 10, &value); + if (err < 0) + return err; + + if (pwm) { + data->percents[index] = value; + } else { + data->temps[index] = value; + } + + /* + * Mark as disabled so the user has to explicitly enable to apply a + * changed fan curve. This prevents potential lockups from writing out + * many changes as one-write-per-change. + */ + data->enabled = false; + + return count; +} + +static ssize_t fan_curve_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + struct fan_curve_data *data; + int out = 2; + + data = fan_curve_attr_select(asus, attr); + + if (data->enabled) + out = 1; + + return sysfs_emit(buf, "%d\n", out); +} + +static ssize_t fan_curve_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + struct fan_curve_data *data; + int value, err; + + data = fan_curve_attr_select(asus, attr); + + err = kstrtoint(buf, 10, &value); + if (err < 0) + return err; + + switch (value) { + case 1: + data->enabled = true; + break; + case 2: + data->enabled = false; + break; + /* + * Auto + reset the fan curve data to defaults. Make it an explicit + * option so that users don't accidentally overwrite a set fan curve. + */ + case 3: + err = fan_curve_get_factory_default(asus, data->device_id); + if (err) + return err; + data->enabled = false; + break; + default: + return -EINVAL; + }; + + if (data->enabled) { + err = fan_curve_write(asus, data); + if (err) + return err; + } else { + /* + * For machines with throttle this is the only way to reset fans + * to default mode of operation (does not erase curve data). + */ + if (asus->throttle_thermal_policy_available) { + err = throttle_thermal_policy_write(asus); + if (err) + return err; + /* Similar is true for laptops with this fan */ + } else if (asus->fan_type == FAN_TYPE_SPEC83) { + err = asus_fan_set_auto(asus); + if (err) + return err; + } else { + /* Safeguard against fautly ACPI tables */ + err = fan_curve_get_factory_default(asus, data->device_id); + if (err) + return err; + err = fan_curve_write(asus, data); + if (err) + return err; + } + } + return count; +} + +/* CPU */ +static SENSOR_DEVICE_ATTR_RW(pwm1_enable, fan_curve_enable, FAN_CURVE_DEV_CPU); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point1_temp, fan_curve, + FAN_CURVE_DEV_CPU, 0); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point2_temp, fan_curve, + FAN_CURVE_DEV_CPU, 1); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point3_temp, fan_curve, + FAN_CURVE_DEV_CPU, 2); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point4_temp, fan_curve, + FAN_CURVE_DEV_CPU, 3); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point5_temp, fan_curve, + FAN_CURVE_DEV_CPU, 4); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point6_temp, fan_curve, + FAN_CURVE_DEV_CPU, 5); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point7_temp, fan_curve, + FAN_CURVE_DEV_CPU, 6); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point8_temp, fan_curve, + FAN_CURVE_DEV_CPU, 7); + +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point1_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 0); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point2_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 1); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point3_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 2); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point4_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 3); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point5_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 4); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point6_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 5); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point7_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 6); +static SENSOR_DEVICE_ATTR_2_RW(pwm1_auto_point8_pwm, fan_curve, + FAN_CURVE_DEV_CPU | FAN_CURVE_PWM_MASK, 7); + +/* GPU */ +static SENSOR_DEVICE_ATTR_RW(pwm2_enable, fan_curve_enable, FAN_CURVE_DEV_GPU); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point1_temp, fan_curve, + FAN_CURVE_DEV_GPU, 0); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point2_temp, fan_curve, + FAN_CURVE_DEV_GPU, 1); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point3_temp, fan_curve, + FAN_CURVE_DEV_GPU, 2); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point4_temp, fan_curve, + FAN_CURVE_DEV_GPU, 3); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point5_temp, fan_curve, + FAN_CURVE_DEV_GPU, 4); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point6_temp, fan_curve, + FAN_CURVE_DEV_GPU, 5); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point7_temp, fan_curve, + FAN_CURVE_DEV_GPU, 6); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point8_temp, fan_curve, + FAN_CURVE_DEV_GPU, 7); + +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point1_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 0); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point2_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 1); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point3_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 2); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point4_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 3); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point5_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 4); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point6_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 5); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point7_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 6); +static SENSOR_DEVICE_ATTR_2_RW(pwm2_auto_point8_pwm, fan_curve, + FAN_CURVE_DEV_GPU | FAN_CURVE_PWM_MASK, 7); + +static struct attribute *asus_fan_curve_attr[] = { + /* CPU */ + &sensor_dev_attr_pwm1_enable.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point1_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point6_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point7_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point8_temp.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_pwm1_auto_point8_pwm.dev_attr.attr, + /* GPU */ + &sensor_dev_attr_pwm2_enable.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point1_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point2_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point3_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point4_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point5_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point6_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point7_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point8_temp.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_pwm2_auto_point8_pwm.dev_attr.attr, + NULL +}; + +static umode_t asus_fan_curve_is_visible(struct kobject *kobj, + struct attribute *attr, int idx) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct asus_wmi *asus = dev_get_drvdata(dev->parent); + + /* + * Check the char instead of casting attr as there are two attr types + * involved here (attr1 and attr2) + */ + if (asus->cpu_fan_curve_available && attr->name[3] == '1') + return 0644; + + if (asus->gpu_fan_curve_available && attr->name[3] == '2') + return 0644; + + return 0; +} + +static const struct attribute_group asus_fan_curve_attr_group = { + .is_visible = asus_fan_curve_is_visible, + .attrs = asus_fan_curve_attr, +}; +__ATTRIBUTE_GROUPS(asus_fan_curve_attr); + +/* + * Must be initialised after throttle_thermal_policy_check_present() as + * we check the status of throttle_thermal_policy_available during init. + */ +static int asus_wmi_custom_fan_curve_init(struct asus_wmi *asus) +{ + struct device *dev = &asus->platform_device->dev; + struct device *hwmon; + int err; + + err = fan_curve_check_present(asus, &asus->cpu_fan_curve_available, + ASUS_WMI_DEVID_CPU_FAN_CURVE); + if (err) + return err; + + err = fan_curve_check_present(asus, &asus->gpu_fan_curve_available, + ASUS_WMI_DEVID_GPU_FAN_CURVE); + if (err) + return err; + + if (!asus->cpu_fan_curve_available && !asus->gpu_fan_curve_available) + return 0; + + hwmon = devm_hwmon_device_register_with_groups( + dev, "asus_custom_fan_curve", asus, asus_fan_curve_attr_groups); + + if (IS_ERR(hwmon)) { + dev_err(dev, + "Could not register asus_custom_fan_curve device\n"); + return PTR_ERR(hwmon); + } + + return 0; +} + /* Throttle thermal policy ****************************************************/ static int throttle_thermal_policy_check_present(struct asus_wmi *asus) @@ -2092,6 +2637,12 @@ static int throttle_thermal_policy_write(struct asus_wmi *asus) return -EIO; } + /* Must set to disabled if mode is toggled */ + if (asus->cpu_fan_curve_available) + asus->custom_fan_curves[FAN_CURVE_DEV_CPU].enabled = false; + if (asus->gpu_fan_curve_available) + asus->custom_fan_curves[FAN_CURVE_DEV_GPU].enabled = false; + return 0; } @@ -3035,6 +3586,10 @@ static int asus_wmi_add(struct platform_device *pdev) if (err) goto fail_hwmon; + err = asus_wmi_custom_fan_curve_init(asus); + if (err) + goto fail_custom_fan_curve; + err = asus_wmi_led_init(asus); if (err) goto fail_leds; @@ -3106,6 +3661,7 @@ fail_input: asus_wmi_sysfs_exit(asus->platform_device); fail_sysfs: fail_throttle_thermal_policy: +fail_custom_fan_curve: fail_platform_profile_setup: if (asus->platform_profile_support) platform_profile_remove(); @@ -3131,6 +3687,7 @@ static int asus_wmi_remove(struct platform_device *device) asus_wmi_debugfs_exit(asus); asus_wmi_sysfs_exit(asus->platform_device); asus_fan_set_auto(asus); + throttle_thermal_policy_set_default(asus); asus_wmi_battery_exit(asus); if (asus->platform_profile_support) diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 17dc5cb6f3f2..a571b47ff362 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -77,6 +77,8 @@ #define ASUS_WMI_DEVID_THERMAL_CTRL 0x00110011 #define ASUS_WMI_DEVID_FAN_CTRL 0x00110012 /* deprecated */ #define ASUS_WMI_DEVID_CPU_FAN_CTRL 0x00110013 +#define ASUS_WMI_DEVID_CPU_FAN_CURVE 0x00110024 +#define ASUS_WMI_DEVID_GPU_FAN_CURVE 0x00110025 /* Power */ #define ASUS_WMI_DEVID_PROCESSOR_STATE 0x00120012 -- cgit v1.2.3-59-g8ed1b From 38543b72fbe52b7eec0dedd420d80a06c652d8e4 Mon Sep 17 00:00:00 2001 From: Maximilian Luz Date: Thu, 28 Oct 2021 02:22:41 +0200 Subject: platform/surface: aggregator: Make client device removal more generic Currently, there are similar functions defined in the Aggregator Registry and the controller core. Make client device removal more generic and export it. We can then use this function later on to remove client devices from device hubs as well as the controller and avoid re-defining similar things. Signed-off-by: Maximilian Luz Link: https://lore.kernel.org/r/20211028002243.1586083-2-luzmaximilian@gmail.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/surface/aggregator/bus.c | 24 ++++++++---------------- drivers/platform/surface/aggregator/bus.h | 3 --- drivers/platform/surface/aggregator/core.c | 3 ++- include/linux/surface_aggregator/device.h | 9 +++++++++ 4 files changed, 19 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/surface/aggregator/bus.c b/drivers/platform/surface/aggregator/bus.c index 0a40dd9c94ed..abbbb5b08b07 100644 --- a/drivers/platform/surface/aggregator/bus.c +++ b/drivers/platform/surface/aggregator/bus.c @@ -374,27 +374,19 @@ static int ssam_remove_device(struct device *dev, void *_data) } /** - * ssam_controller_remove_clients() - Remove SSAM client devices registered as - * direct children under the given controller. - * @ctrl: The controller to remove all direct clients for. + * ssam_remove_clients() - Remove SSAM client devices registered as direct + * children under the given parent device. + * @dev: The (parent) device to remove all direct clients for. * - * Remove all SSAM client devices registered as direct children under the - * given controller. Note that this only accounts for direct children of the - * controller device. This does not take care of any client devices where the - * parent device has been manually set before calling ssam_device_add. Refer - * to ssam_device_add()/ssam_device_remove() for more details on those cases. - * - * To avoid new devices being added in parallel to this call, the main - * controller lock (not statelock) must be held during this (and if - * necessary, any subsequent deinitialization) call. + * Remove all SSAM client devices registered as direct children under the given + * device. Note that this only accounts for direct children of the device. + * Refer to ssam_device_add()/ssam_device_remove() for more details. */ -void ssam_controller_remove_clients(struct ssam_controller *ctrl) +void ssam_remove_clients(struct device *dev) { - struct device *dev; - - dev = ssam_controller_device(ctrl); device_for_each_child_reverse(dev, NULL, ssam_remove_device); } +EXPORT_SYMBOL_GPL(ssam_remove_clients); /** * ssam_bus_register() - Register and set-up the SSAM client device bus. diff --git a/drivers/platform/surface/aggregator/bus.h b/drivers/platform/surface/aggregator/bus.h index ed032c2cbdb2..6964ee84e79c 100644 --- a/drivers/platform/surface/aggregator/bus.h +++ b/drivers/platform/surface/aggregator/bus.h @@ -12,14 +12,11 @@ #ifdef CONFIG_SURFACE_AGGREGATOR_BUS -void ssam_controller_remove_clients(struct ssam_controller *ctrl); - int ssam_bus_register(void); void ssam_bus_unregister(void); #else /* CONFIG_SURFACE_AGGREGATOR_BUS */ -static inline void ssam_controller_remove_clients(struct ssam_controller *ctrl) {} static inline int ssam_bus_register(void) { return 0; } static inline void ssam_bus_unregister(void) {} diff --git a/drivers/platform/surface/aggregator/core.c b/drivers/platform/surface/aggregator/core.c index c61bbeeec2df..d384d36098c2 100644 --- a/drivers/platform/surface/aggregator/core.c +++ b/drivers/platform/surface/aggregator/core.c @@ -22,6 +22,7 @@ #include #include +#include #include "bus.h" #include "controller.h" @@ -735,7 +736,7 @@ static void ssam_serial_hub_remove(struct serdev_device *serdev) ssam_controller_lock(ctrl); /* Remove all client devices. */ - ssam_controller_remove_clients(ctrl); + ssam_remove_clients(&serdev->dev); /* Act as if suspending to silence events. */ status = ssam_ctrl_notif_display_off(ctrl); diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index f636c5310321..cc257097eb05 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -319,6 +319,15 @@ void ssam_device_driver_unregister(struct ssam_device_driver *d); ssam_device_driver_unregister) +/* -- Helpers for controller and hub devices. ------------------------------- */ + +#ifdef CONFIG_SURFACE_AGGREGATOR_BUS +void ssam_remove_clients(struct device *dev); +#else /* CONFIG_SURFACE_AGGREGATOR_BUS */ +static inline void ssam_remove_clients(struct device *dev) {} +#endif /* CONFIG_SURFACE_AGGREGATOR_BUS */ + + /* -- Helpers for client-device requests. ----------------------------------- */ /** -- cgit v1.2.3-59-g8ed1b From ebf7f6f0a6cdcc17a3da52b81e4b3a98c4005028 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Fri, 5 Nov 2021 09:30:00 +0800 Subject: bpf: Change value of MAX_TAIL_CALL_CNT from 32 to 33 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the current code, the actual max tail call count is 33 which is greater than MAX_TAIL_CALL_CNT (defined as 32). The actual limit is not consistent with the meaning of MAX_TAIL_CALL_CNT and thus confusing at first glance. We can see the historical evolution from commit 04fd61ab36ec ("bpf: allow bpf programs to tail-call other bpf programs") and commit f9dabe016b63 ("bpf: Undo off-by-one in interpreter tail call count limit"). In order to avoid changing existing behavior, the actual limit is 33 now, this is reasonable. After commit 874be05f525e ("bpf, tests: Add tail call test suite"), we can see there exists failed testcase. On all archs when CONFIG_BPF_JIT_ALWAYS_ON is not set: # echo 0 > /proc/sys/net/core/bpf_jit_enable # modprobe test_bpf # dmesg | grep -w FAIL Tail call error path, max count reached jited:0 ret 34 != 33 FAIL On some archs: # echo 1 > /proc/sys/net/core/bpf_jit_enable # modprobe test_bpf # dmesg | grep -w FAIL Tail call error path, max count reached jited:1 ret 34 != 33 FAIL Although the above failed testcase has been fixed in commit 18935a72eb25 ("bpf/tests: Fix error in tail call limit tests"), it would still be good to change the value of MAX_TAIL_CALL_CNT from 32 to 33 to make the code more readable. The 32-bit x86 JIT was using a limit of 32, just fix the wrong comments and limit to 33 tail calls as the constant MAX_TAIL_CALL_CNT updated. For the mips64 JIT, use "ori" instead of "addiu" as suggested by Johan Almbladh. For the riscv JIT, use RV_REG_TCC directly to save one register move as suggested by Björn Töpel. For the other implementations, no function changes, it does not change the current limit 33, the new value of MAX_TAIL_CALL_CNT can reflect the actual max tail call count, the related tail call testcases in test_bpf module and selftests can work well for the interpreter and the JIT. Here are the test results on x86_64: # uname -m x86_64 # echo 0 > /proc/sys/net/core/bpf_jit_enable # modprobe test_bpf test_suite=test_tail_calls # dmesg | tail -1 test_bpf: test_tail_calls: Summary: 8 PASSED, 0 FAILED, [0/8 JIT'ed] # rmmod test_bpf # echo 1 > /proc/sys/net/core/bpf_jit_enable # modprobe test_bpf test_suite=test_tail_calls # dmesg | tail -1 test_bpf: test_tail_calls: Summary: 8 PASSED, 0 FAILED, [8/8 JIT'ed] # rmmod test_bpf # ./test_progs -t tailcalls #142 tailcalls:OK Summary: 1/11 PASSED, 0 SKIPPED, 0 FAILED Signed-off-by: Tiezhu Yang Signed-off-by: Daniel Borkmann Tested-by: Johan Almbladh Tested-by: Ilya Leoshkevich Acked-by: Björn Töpel Acked-by: Johan Almbladh Acked-by: Ilya Leoshkevich Link: https://lore.kernel.org/bpf/1636075800-3264-1-git-send-email-yangtiezhu@loongson.cn --- arch/arm/net/bpf_jit_32.c | 5 +++-- arch/arm64/net/bpf_jit_comp.c | 5 +++-- arch/mips/net/bpf_jit_comp32.c | 3 +-- arch/mips/net/bpf_jit_comp64.c | 2 +- arch/powerpc/net/bpf_jit_comp32.c | 4 ++-- arch/powerpc/net/bpf_jit_comp64.c | 4 ++-- arch/riscv/net/bpf_jit_comp32.c | 6 ++---- arch/riscv/net/bpf_jit_comp64.c | 7 +++---- arch/s390/net/bpf_jit_comp.c | 6 +++--- arch/sparc/net/bpf_jit_comp_64.c | 2 +- arch/x86/net/bpf_jit_comp.c | 10 +++++----- arch/x86/net/bpf_jit_comp32.c | 4 ++-- include/linux/bpf.h | 2 +- include/uapi/linux/bpf.h | 2 +- kernel/bpf/core.c | 3 ++- lib/test_bpf.c | 4 ++-- tools/include/uapi/linux/bpf.h | 2 +- 17 files changed, 35 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index eeb6dc0ecf46..e59b41e9ab0c 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1199,7 +1199,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) /* tmp2[0] = array, tmp2[1] = index */ - /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) + /* + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; * tail_call_cnt++; */ @@ -1208,7 +1209,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) tc = arm_bpf_get_reg64(tcc, tmp, ctx); emit(ARM_CMP_I(tc[0], hi), ctx); _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx); - _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); + _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); arm_bpf_put_reg64(tcc, tmp, ctx); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 86c9dc0681cc..07c12c42b751 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -287,13 +287,14 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) emit(A64_CMP(0, r3, tmp), ctx); emit(A64_B_(A64_COND_CS, jmp_offset), ctx); - /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) + /* + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; * tail_call_cnt++; */ emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx); emit(A64_CMP(1, tcc, tmp), ctx); - emit(A64_B_(A64_COND_HI, jmp_offset), ctx); + emit(A64_B_(A64_COND_CS, jmp_offset), ctx); emit(A64_ADD_I(1, tcc, tcc, 1), ctx); /* prog = array->ptrs[index]; diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index bd996ede12f8..044b11b65bca 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1381,8 +1381,7 @@ void build_prologue(struct jit_context *ctx) * 16-byte area in the parent's stack frame. On a tail call, the * calling function jumps into the prologue after these instructions. */ - emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, - min(MAX_TAIL_CALL_CNT + 1, 0xffff)); + emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff)); emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); /* diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 815ade724227..6475828ffb36 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -552,7 +552,7 @@ void build_prologue(struct jit_context *ctx) * On a tail call, the calling function jumps into the prologue * after this instruction. */ - emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); + emit(ctx, ori, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff)); /* === Entry-point for tail calls === */ diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 0da31d41d413..8a4faa05f9e4 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -221,13 +221,13 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o PPC_BCC(COND_GE, out); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT)); /* tail_call_cnt++; */ EMIT(PPC_RAW_ADDIC(_R0, _R0, 1)); - PPC_BCC(COND_GT, out); + PPC_BCC(COND_GE, out); /* prog = array->ptrs[index]; */ EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29)); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 8b5157ccfeba..8571aafcc9e1 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -228,12 +228,12 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o PPC_BCC(COND_GE, out); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) * goto out; */ PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT)); - PPC_BCC(COND_GT, out); + PPC_BCC(COND_GE, out); /* * tail_call_cnt++; diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index e6497424cbf6..529a83b85c1c 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -799,11 +799,10 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); /* - * temp_tcc = tcc - 1; - * if (tcc < 0) + * if (--tcc < 0) * goto out; */ - emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx); + emit(rv_addi(RV_REG_TCC, RV_REG_TCC, -1), ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -829,7 +828,6 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); - emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); /* Epilogue jumps to *(t0 + 4). */ __build_epilogue(true, ctx); return 0; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f2a779c7e225..603630b6f3c5 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -327,12 +327,12 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); - /* if (TCC-- < 0) + /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_T1, tcc, -1, ctx); + emit_addi(RV_REG_TCC, tcc, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); - emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); + emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) @@ -352,7 +352,6 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) if (is_12b_check(off, insn)) return -1; emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); - emit_mv(RV_REG_TCC, RV_REG_T1, ctx); __build_epilogue(true, ctx); return 0; } diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 233cc9bcd652..9ff2bd83aad7 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1369,7 +1369,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, jit->prg); /* - * if (tail_call_cnt++ > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ @@ -1381,9 +1381,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, EMIT4_IMM(0xa7080000, REG_W0, 1); /* laal %w1,%w0,off(%r15) */ EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off); - /* clij %w1,MAX_TAIL_CALL_CNT,0x2,out */ + /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ patch_2_clij = jit->prg; - EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT, + EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1, 2, jit->prg); /* diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index 9a2f20cbd48b..0bfe1c72a0c9 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -867,7 +867,7 @@ static void emit_tail_call(struct jit_ctx *ctx) emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); #define OFFSET2 13 - emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx); + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx); emit_nop(ctx); emit_alu_K(ADD, tmp, 1, ctx); diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 726700fabca6..631847907786 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -412,7 +412,7 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... * if (index >= array->map.max_entries) * goto out; - * if (++tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; * prog = array->ptrs[index]; * if (prog == NULL) @@ -446,14 +446,14 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, EMIT2(X86_JBE, offset); /* jbe out */ /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_indirect_label - (prog + 2 - start); - EMIT2(X86_JA, offset); /* ja out */ + EMIT2(X86_JAE, offset); /* jae out */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ @@ -504,14 +504,14 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, int offset; /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_direct_label - (prog + 2 - start); - EMIT2(X86_JA, offset); /* ja out */ + EMIT2(X86_JAE, offset); /* jae out */ EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index da9b7cfa4632..429a89c5468b 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -1323,7 +1323,7 @@ static void emit_bpf_tail_call(u8 **pprog, u8 *ip) EMIT2(IA32_JBE, jmp_label(jmp_label1, 2)); /* - * if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; */ lo = (u32)MAX_TAIL_CALL_CNT; @@ -1337,7 +1337,7 @@ static void emit_bpf_tail_call(u8 **pprog, u8 *ip) /* cmp ecx,lo */ EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo); - /* ja out */ + /* jae out */ EMIT2(IA32_JAE, jmp_label(jmp_label1, 2)); /* add eax,0x1 */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 56098c866704..cc7a0c36e7df 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1081,7 +1081,7 @@ struct bpf_array { }; #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ -#define MAX_TAIL_CALL_CNT 32 +#define MAX_TAIL_CALL_CNT 33 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ BPF_F_RDONLY_PROG | \ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 6297eafdc40f..a69e4b04ffeb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1744,7 +1744,7 @@ union bpf_attr { * if the maximum number of tail calls has been reached for this * chain of programs. This limit is defined in the kernel by the * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), - * which is currently set to 32. + * which is currently set to 33. * Return * 0 on success, or a negative error in case of failure. * diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 2405e39d800f..b52dc845ecea 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1574,7 +1574,8 @@ select_insn: if (unlikely(index >= array->map.max_entries)) goto out; - if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT)) + + if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT)) goto out; tail_call_cnt++; diff --git a/lib/test_bpf.c b/lib/test_bpf.c index adae39567264..0c5cb2d6436a 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -14683,7 +14683,7 @@ static struct tail_call_test tail_call_tests[] = { BPF_EXIT_INSN(), }, .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, }, { "Tail call count preserved across function calls", @@ -14705,7 +14705,7 @@ static struct tail_call_test tail_call_tests[] = { }, .stack_depth = 8, .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, }, { "Tail call error path, NULL target", diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6297eafdc40f..a69e4b04ffeb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1744,7 +1744,7 @@ union bpf_attr { * if the maximum number of tail calls has been reached for this * chain of programs. This limit is defined in the kernel by the * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), - * which is currently set to 32. + * which is currently set to 33. * Return * 0 on success, or a negative error in case of failure. * -- cgit v1.2.3-59-g8ed1b From 42f67eea3ba36cef2dce2e853de6ddcb2e89eb39 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 11:02:33 -0800 Subject: net: use sk_is_tcp() in more places Move sk_is_tcp() to include/net/sock.h and use it where we can. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/skmsg.h | 6 ------ include/net/sock.h | 5 +++++ net/core/skbuff.c | 6 ++---- net/core/sock.c | 6 ++---- 4 files changed, 9 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 584d94be9c8b..18a717fe62eb 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -507,12 +507,6 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } -static inline bool sk_is_tcp(const struct sock *sk) -{ - return sk->sk_type == SOCK_STREAM && - sk->sk_protocol == IPPROTO_TCP; -} - static inline bool sk_is_udp(const struct sock *sk) { return sk->sk_type == SOCK_DGRAM && diff --git a/include/net/sock.h b/include/net/sock.h index b32906e1ab55..5bdeffdea5ec 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2638,6 +2638,11 @@ static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags) &skb_shinfo(skb)->tskey); } +static inline bool sk_is_tcp(const struct sock *sk) +{ + return sk->sk_type == SOCK_STREAM && sk->sk_protocol == IPPROTO_TCP; +} + /** * sk_eat_skb - Release a skb if it is no longer needed * @sk: socket to eat this skb from diff --git a/net/core/skbuff.c b/net/core/skbuff.c index ba2f38246f07..d57796f38a0b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4849,8 +4849,7 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb, serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { serr->ee.ee_data = skb_shinfo(skb)->tskey; - if (sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) + if (sk_is_tcp(sk)) serr->ee.ee_data -= sk->sk_tskey; } @@ -4919,8 +4918,7 @@ void __skb_tstamp_tx(struct sk_buff *orig_skb, if (tsonly) { #ifdef CONFIG_INET if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && - sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) { + sk_is_tcp(sk)) { skb = tcp_get_timestamping_opt_stats(sk, orig_skb, ack_skb); opt_stats = true; diff --git a/net/core/sock.c b/net/core/sock.c index 8f2b2f2c0e7b..0be8e43f44b9 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -872,8 +872,7 @@ int sock_set_timestamping(struct sock *sk, int optname, if (val & SOF_TIMESTAMPING_OPT_ID && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { - if (sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) { + if (sk_is_tcp(sk)) { if ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) return -EINVAL; @@ -1370,8 +1369,7 @@ set_sndbuf: case SO_ZEROCOPY: if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { - if (!((sk->sk_type == SOCK_STREAM && - sk->sk_protocol == IPPROTO_TCP) || + if (!(sk_is_tcp(sk) || (sk->sk_type == SOCK_DGRAM && sk->sk_protocol == IPPROTO_UDP))) ret = -ENOTSUPP; -- cgit v1.2.3-59-g8ed1b From f35f821935d8df76f9c92e2431a225bdff938169 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 11:02:46 -0800 Subject: tcp: defer skb freeing after socket lock is released tcp recvmsg() (or rx zerocopy) spends a fair amount of time freeing skbs after their payload has been consumed. A typical ~64KB GRO packet has to release ~45 page references, eventually going to page allocator for each of them. Currently, this freeing is performed while socket lock is held, meaning that there is a high chance that BH handler has to queue incoming packets to tcp socket backlog. This can cause additional latencies, because the user thread has to process the backlog at release_sock() time, and while doing so, additional frames can be added by BH handler. This patch adds logic to defer these frees after socket lock is released, or directly from BH handler if possible. Being able to free these skbs from BH handler helps a lot, because this avoids the usual alloc/free assymetry, when BH handler and user thread do not run on same cpu or NUMA node. One cpu can now be fully utilized for the kernel->user copy, and another cpu is handling BH processing and skb/page allocs/frees (assuming RFS is not forcing use of a single CPU) Tested: 100Gbit NIC Max throughput for one TCP_STREAM flow, over 10 runs MTU : 1500 Before: 55 Gbit After: 66 Gbit MTU : 4096+(headers) Before: 82 Gbit After: 95 Gbit Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 ++ include/net/sock.h | 3 +++ include/net/tcp.h | 10 ++++++++++ net/ipv4/tcp.c | 27 +++++++++++++++++++++++++-- net/ipv4/tcp_ipv4.c | 1 + net/ipv6/tcp_ipv6.c | 1 + 6 files changed, 42 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 686a666d073d..b8b806512e16 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #if IS_ENABLED(CONFIG_NF_CONNTRACK) @@ -743,6 +744,7 @@ struct sk_buff { }; struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */ struct list_head list; + struct llist_node ll_node; }; union { diff --git a/include/net/sock.h b/include/net/sock.h index 2d40fe4c7718..2578d1f455a7 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -63,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -408,6 +409,8 @@ struct sock { struct sk_buff *head; struct sk_buff *tail; } sk_backlog; + struct llist_head defer_list; + #define sk_rmem_alloc sk_backlog.rmem_alloc int sk_forward_alloc; diff --git a/include/net/tcp.h b/include/net/tcp.h index 05c81677aaf7..44e442bf23f9 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1368,6 +1368,16 @@ static inline bool tcp_checksum_complete(struct sk_buff *skb) } bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb); + +void __sk_defer_free_flush(struct sock *sk); + +static inline void sk_defer_free_flush(struct sock *sk) +{ + if (llist_empty(&sk->defer_list)) + return; + __sk_defer_free_flush(sk); +} + int tcp_filter(struct sock *sk, struct sk_buff *skb); void tcp_set_state(struct sock *sk, int state); void tcp_done(struct sock *sk); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 4e7011672aa9..33cd9a1c199c 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1580,14 +1580,34 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) tcp_send_ack(sk); } +void __sk_defer_free_flush(struct sock *sk) +{ + struct llist_node *head; + struct sk_buff *skb, *n; + + head = llist_del_all(&sk->defer_list); + llist_for_each_entry_safe(skb, n, head, ll_node) { + prefetch(n); + skb_mark_not_on_list(skb); + __kfree_skb(skb); + } +} +EXPORT_SYMBOL(__sk_defer_free_flush); + static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) { + __skb_unlink(skb, &sk->sk_receive_queue); if (likely(skb->destructor == sock_rfree)) { sock_rfree(skb); skb->destructor = NULL; skb->sk = NULL; + if (!skb_queue_empty(&sk->sk_receive_queue) || + !llist_empty(&sk->defer_list)) { + llist_add(&skb->ll_node, &sk->defer_list); + return; + } } - sk_eat_skb(sk, skb); + __kfree_skb(skb); } static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) @@ -2422,6 +2442,7 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, /* Do not sleep, just process backlog. */ __sk_flush_backlog(sk); } else { + sk_defer_free_flush(sk); sk_wait_data(sk, &timeo, last); } @@ -2540,6 +2561,7 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, ret = tcp_recvmsg_locked(sk, msg, len, nonblock, flags, &tss, &cmsg_flags); release_sock(sk); + sk_defer_free_flush(sk); if (cmsg_flags && ret >= 0) { if (cmsg_flags & TCP_CMSG_TS) @@ -3065,7 +3087,7 @@ int tcp_disconnect(struct sock *sk, int flags) sk->sk_frag.page = NULL; sk->sk_frag.offset = 0; } - + sk_defer_free_flush(sk); sk_error_report(sk); return 0; } @@ -4194,6 +4216,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level, err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); + sk_defer_free_flush(sk); if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) goto zerocopy_rcv_cmsg; switch (len) { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 5ad81bfb27b2..3dd19a2bf06c 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2102,6 +2102,7 @@ process: sk_incoming_cpu_update(sk); + sk_defer_free_flush(sk); bh_lock_sock_nested(sk); tcp_segs_in(tcp_sk(sk), skb); ret = 0; diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index f41f14b70123..3b7d6ede1364 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1758,6 +1758,7 @@ process: sk_incoming_cpu_update(sk); + sk_defer_free_flush(sk); bh_lock_sock_nested(sk); tcp_segs_in(tcp_sk(sk), skb); ret = 0; -- cgit v1.2.3-59-g8ed1b From 4721031c3559db8eae61df305f10c00099a7c1d0 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:05:51 -0800 Subject: net: move gro definitions to include/net/gro.h include/linux/netdevice.h became too big, move gro stuff into include/net/gro.h Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt.c | 1 + drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 1 + drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 1 + drivers/net/ethernet/qlogic/qede/qede_fp.c | 1 + drivers/net/geneve.c | 1 + drivers/net/vxlan.c | 1 + include/linux/netdevice.h | 348 --------------------- include/net/gro.h | 396 +++++++++++++++++++++++- include/net/ip.h | 8 - include/net/ip6_checksum.h | 8 - include/net/udp.h | 24 -- net/core/skbuff.c | 1 + net/ipv4/af_inet.c | 1 + net/ipv4/esp4_offload.c | 1 + net/ipv4/fou.c | 1 + net/ipv4/gre_offload.c | 1 + net/ipv4/tcp_offload.c | 1 + net/ipv4/udp_offload.c | 1 + net/ipv6/esp6_offload.c | 1 + net/ipv6/tcpv6_offload.c | 1 + net/ipv6/udp_offload.c | 1 + 22 files changed, 411 insertions(+), 390 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index e8e8c2d593c5..54a2334dee56 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index c04ea83188e2..c057b1df86a9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 9ccebbaa0d69..13835a37b3a2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 96967b0a2441..e384f6458c06 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include "en.h" diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index 065e9004598e..e113fbd56e86 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 1ab94b5f9bbf..9d26d1b965d2 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #define GENEVE_NETDEV_VER "0.6" diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 141635a35c28..563f86de0e0d 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3ec42495a43a..d95c9839ce90 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2520,109 +2520,6 @@ static inline void netif_napi_del(struct napi_struct *napi) synchronize_net(); } -struct napi_gro_cb { - /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ - void *frag0; - - /* Length of frag0. */ - unsigned int frag0_len; - - /* This indicates where we are processing relative to skb->data. */ - int data_offset; - - /* This is non-zero if the packet cannot be merged with the new skb. */ - u16 flush; - - /* Save the IP ID here and check when we get to the transport layer */ - u16 flush_id; - - /* Number of segments aggregated. */ - u16 count; - - /* Start offset for remote checksum offload */ - u16 gro_remcsum_start; - - /* jiffies when first packet was created/queued */ - unsigned long age; - - /* Used in ipv6_gro_receive() and foo-over-udp */ - u16 proto; - - /* This is non-zero if the packet may be of the same flow. */ - u8 same_flow:1; - - /* Used in tunnel GRO receive */ - u8 encap_mark:1; - - /* GRO checksum is valid */ - u8 csum_valid:1; - - /* Number of checksums via CHECKSUM_UNNECESSARY */ - u8 csum_cnt:3; - - /* Free the skb? */ - u8 free:2; -#define NAPI_GRO_FREE 1 -#define NAPI_GRO_FREE_STOLEN_HEAD 2 - - /* Used in foo-over-udp, set in udp[46]_gro_receive */ - u8 is_ipv6:1; - - /* Used in GRE, set in fou/gue_gro_receive */ - u8 is_fou:1; - - /* Used to determine if flush_id can be ignored */ - u8 is_atomic:1; - - /* Number of gro_receive callbacks this packet already went through */ - u8 recursion_counter:4; - - /* GRO is done by frag_list pointer chaining. */ - u8 is_flist:1; - - /* used to support CHECKSUM_COMPLETE for tunneling protocols */ - __wsum csum; - - /* used in skb_gro_receive() slow path */ - struct sk_buff *last; -}; - -#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) - -#define GRO_RECURSION_LIMIT 15 -static inline int gro_recursion_inc_test(struct sk_buff *skb) -{ - return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; -} - -typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); -static inline struct sk_buff *call_gro_receive(gro_receive_t cb, - struct list_head *head, - struct sk_buff *skb) -{ - if (unlikely(gro_recursion_inc_test(skb))) { - NAPI_GRO_CB(skb)->flush |= 1; - return NULL; - } - - return cb(head, skb); -} - -typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, - struct sk_buff *); -static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, - struct sock *sk, - struct list_head *head, - struct sk_buff *skb) -{ - if (unlikely(gro_recursion_inc_test(skb))) { - NAPI_GRO_CB(skb)->flush |= 1; - return NULL; - } - - return cb(sk, head, skb); -} - struct packet_type { __be16 type; /* This is really htons(ether_type). */ bool ignore_outgoing; @@ -3008,251 +2905,6 @@ int dev_restart(struct net_device *dev); int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb); -static inline unsigned int skb_gro_offset(const struct sk_buff *skb) -{ - return NAPI_GRO_CB(skb)->data_offset; -} - -static inline unsigned int skb_gro_len(const struct sk_buff *skb) -{ - return skb->len - NAPI_GRO_CB(skb)->data_offset; -} - -static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) -{ - NAPI_GRO_CB(skb)->data_offset += len; -} - -static inline void *skb_gro_header_fast(struct sk_buff *skb, - unsigned int offset) -{ - return NAPI_GRO_CB(skb)->frag0 + offset; -} - -static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) -{ - return NAPI_GRO_CB(skb)->frag0_len < hlen; -} - -static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) -{ - NAPI_GRO_CB(skb)->frag0 = NULL; - NAPI_GRO_CB(skb)->frag0_len = 0; -} - -static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, - unsigned int offset) -{ - if (!pskb_may_pull(skb, hlen)) - return NULL; - - skb_gro_frag0_invalidate(skb); - return skb->data + offset; -} - -static inline void *skb_gro_network_header(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + - skb_network_offset(skb); -} - -static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, - const void *start, unsigned int len) -{ - if (NAPI_GRO_CB(skb)->csum_valid) - NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum, - csum_partial(start, len, 0)); -} - -/* GRO checksum functions. These are logical equivalents of the normal - * checksum functions (in skbuff.h) except that they operate on the GRO - * offsets and fields in sk_buff. - */ - -__sum16 __skb_gro_checksum_complete(struct sk_buff *skb); - -static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); -} - -static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, - bool zero_okay, - __sum16 check) -{ - return ((skb->ip_summed != CHECKSUM_PARTIAL || - skb_checksum_start_offset(skb) < - skb_gro_offset(skb)) && - !skb_at_gro_remcsum_start(skb) && - NAPI_GRO_CB(skb)->csum_cnt == 0 && - (!zero_okay || check)); -} - -static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, - __wsum psum) -{ - if (NAPI_GRO_CB(skb)->csum_valid && - !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) - return 0; - - NAPI_GRO_CB(skb)->csum = psum; - - return __skb_gro_checksum_complete(skb); -} - -static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) -{ - if (NAPI_GRO_CB(skb)->csum_cnt > 0) { - /* Consume a checksum from CHECKSUM_UNNECESSARY */ - NAPI_GRO_CB(skb)->csum_cnt--; - } else { - /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we - * verified a new top level checksum or an encapsulated one - * during GRO. This saves work if we fallback to normal path. - */ - __skb_incr_checksum_unnecessary(skb); - } -} - -#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ - compute_pseudo) \ -({ \ - __sum16 __ret = 0; \ - if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ - __ret = __skb_gro_checksum_validate_complete(skb, \ - compute_pseudo(skb, proto)); \ - if (!__ret) \ - skb_gro_incr_csum_unnecessary(skb); \ - __ret; \ -}) - -#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ - __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) - -#define skb_gro_checksum_validate_zero_check(skb, proto, check, \ - compute_pseudo) \ - __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) - -#define skb_gro_checksum_simple_validate(skb) \ - __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) - -static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->csum_cnt == 0 && - !NAPI_GRO_CB(skb)->csum_valid); -} - -static inline void __skb_gro_checksum_convert(struct sk_buff *skb, - __wsum pseudo) -{ - NAPI_GRO_CB(skb)->csum = ~pseudo; - NAPI_GRO_CB(skb)->csum_valid = 1; -} - -#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ -do { \ - if (__skb_gro_checksum_convert_check(skb)) \ - __skb_gro_checksum_convert(skb, \ - compute_pseudo(skb, proto)); \ -} while (0) - -struct gro_remcsum { - int offset; - __wsum delta; -}; - -static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) -{ - grc->offset = 0; - grc->delta = 0; -} - -static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, - unsigned int off, size_t hdrlen, - int start, int offset, - struct gro_remcsum *grc, - bool nopartial) -{ - __wsum delta; - size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); - - BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); - - if (!nopartial) { - NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; - return ptr; - } - - ptr = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, off + plen)) { - ptr = skb_gro_header_slow(skb, off + plen, off); - if (!ptr) - return NULL; - } - - delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, - start, offset); - - /* Adjust skb->csum since we changed the packet */ - NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); - - grc->offset = off + hdrlen + offset; - grc->delta = delta; - - return ptr; -} - -static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, - struct gro_remcsum *grc) -{ - void *ptr; - size_t plen = grc->offset + sizeof(u16); - - if (!grc->delta) - return; - - ptr = skb_gro_header_fast(skb, grc->offset); - if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { - ptr = skb_gro_header_slow(skb, plen, grc->offset); - if (!ptr) - return; - } - - remcsum_unadjust((__sum16 *)ptr, grc->delta); -} - -#ifdef CONFIG_XFRM_OFFLOAD -static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) -{ - if (PTR_ERR(pp) != -EINPROGRESS) - NAPI_GRO_CB(skb)->flush |= flush; -} -static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, - struct sk_buff *pp, - int flush, - struct gro_remcsum *grc) -{ - if (PTR_ERR(pp) != -EINPROGRESS) { - NAPI_GRO_CB(skb)->flush |= flush; - skb_gro_remcsum_cleanup(skb, grc); - skb->remcsum_offload = 0; - } -} -#else -static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) -{ - NAPI_GRO_CB(skb)->flush |= flush; -} -static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, - struct sk_buff *pp, - int flush, - struct gro_remcsum *grc) -{ - NAPI_GRO_CB(skb)->flush |= flush; - skb_gro_remcsum_cleanup(skb, grc); - skb->remcsum_offload = 0; -} -#endif static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, diff --git a/include/net/gro.h b/include/net/gro.h index 01edaf3fdda0..1ffbe74b2e35 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -4,9 +4,367 @@ #define _NET_IPV6_GRO_H #include +#include +#include +#include +#include -struct list_head; -struct sk_buff; +struct napi_gro_cb { + /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ + void *frag0; + + /* Length of frag0. */ + unsigned int frag0_len; + + /* This indicates where we are processing relative to skb->data. */ + int data_offset; + + /* This is non-zero if the packet cannot be merged with the new skb. */ + u16 flush; + + /* Save the IP ID here and check when we get to the transport layer */ + u16 flush_id; + + /* Number of segments aggregated. */ + u16 count; + + /* Start offset for remote checksum offload */ + u16 gro_remcsum_start; + + /* jiffies when first packet was created/queued */ + unsigned long age; + + /* Used in ipv6_gro_receive() and foo-over-udp */ + u16 proto; + + /* This is non-zero if the packet may be of the same flow. */ + u8 same_flow:1; + + /* Used in tunnel GRO receive */ + u8 encap_mark:1; + + /* GRO checksum is valid */ + u8 csum_valid:1; + + /* Number of checksums via CHECKSUM_UNNECESSARY */ + u8 csum_cnt:3; + + /* Free the skb? */ + u8 free:2; +#define NAPI_GRO_FREE 1 +#define NAPI_GRO_FREE_STOLEN_HEAD 2 + + /* Used in foo-over-udp, set in udp[46]_gro_receive */ + u8 is_ipv6:1; + + /* Used in GRE, set in fou/gue_gro_receive */ + u8 is_fou:1; + + /* Used to determine if flush_id can be ignored */ + u8 is_atomic:1; + + /* Number of gro_receive callbacks this packet already went through */ + u8 recursion_counter:4; + + /* GRO is done by frag_list pointer chaining. */ + u8 is_flist:1; + + /* used to support CHECKSUM_COMPLETE for tunneling protocols */ + __wsum csum; + + /* used in skb_gro_receive() slow path */ + struct sk_buff *last; +}; + +#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) + +#define GRO_RECURSION_LIMIT 15 +static inline int gro_recursion_inc_test(struct sk_buff *skb) +{ + return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; +} + +typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); +static inline struct sk_buff *call_gro_receive(gro_receive_t cb, + struct list_head *head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(head, skb); +} + +typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, + struct sk_buff *); +static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, + struct sock *sk, + struct list_head *head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(sk, head, skb); +} + +static inline unsigned int skb_gro_offset(const struct sk_buff *skb) +{ + return NAPI_GRO_CB(skb)->data_offset; +} + +static inline unsigned int skb_gro_len(const struct sk_buff *skb) +{ + return skb->len - NAPI_GRO_CB(skb)->data_offset; +} + +static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) +{ + NAPI_GRO_CB(skb)->data_offset += len; +} + +static inline void *skb_gro_header_fast(struct sk_buff *skb, + unsigned int offset) +{ + return NAPI_GRO_CB(skb)->frag0 + offset; +} + +static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) +{ + return NAPI_GRO_CB(skb)->frag0_len < hlen; +} + +static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) +{ + NAPI_GRO_CB(skb)->frag0 = NULL; + NAPI_GRO_CB(skb)->frag0_len = 0; +} + +static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, + unsigned int offset) +{ + if (!pskb_may_pull(skb, hlen)) + return NULL; + + skb_gro_frag0_invalidate(skb); + return skb->data + offset; +} + +static inline void *skb_gro_network_header(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + + skb_network_offset(skb); +} + +static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) +{ + const struct iphdr *iph = skb_gro_network_header(skb); + + return csum_tcpudp_nofold(iph->saddr, iph->daddr, + skb_gro_len(skb), proto, 0); +} + +static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, + const void *start, unsigned int len) +{ + if (NAPI_GRO_CB(skb)->csum_valid) + NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum, + csum_partial(start, len, 0)); +} + +/* GRO checksum functions. These are logical equivalents of the normal + * checksum functions (in skbuff.h) except that they operate on the GRO + * offsets and fields in sk_buff. + */ + +__sum16 __skb_gro_checksum_complete(struct sk_buff *skb); + +static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); +} + +static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, + bool zero_okay, + __sum16 check) +{ + return ((skb->ip_summed != CHECKSUM_PARTIAL || + skb_checksum_start_offset(skb) < + skb_gro_offset(skb)) && + !skb_at_gro_remcsum_start(skb) && + NAPI_GRO_CB(skb)->csum_cnt == 0 && + (!zero_okay || check)); +} + +static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, + __wsum psum) +{ + if (NAPI_GRO_CB(skb)->csum_valid && + !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) + return 0; + + NAPI_GRO_CB(skb)->csum = psum; + + return __skb_gro_checksum_complete(skb); +} + +static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) +{ + if (NAPI_GRO_CB(skb)->csum_cnt > 0) { + /* Consume a checksum from CHECKSUM_UNNECESSARY */ + NAPI_GRO_CB(skb)->csum_cnt--; + } else { + /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we + * verified a new top level checksum or an encapsulated one + * during GRO. This saves work if we fallback to normal path. + */ + __skb_incr_checksum_unnecessary(skb); + } +} + +#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ + compute_pseudo) \ +({ \ + __sum16 __ret = 0; \ + if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ + __ret = __skb_gro_checksum_validate_complete(skb, \ + compute_pseudo(skb, proto)); \ + if (!__ret) \ + skb_gro_incr_csum_unnecessary(skb); \ + __ret; \ +}) + +#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ + __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) + +#define skb_gro_checksum_validate_zero_check(skb, proto, check, \ + compute_pseudo) \ + __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) + +#define skb_gro_checksum_simple_validate(skb) \ + __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) + +static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->csum_cnt == 0 && + !NAPI_GRO_CB(skb)->csum_valid); +} + +static inline void __skb_gro_checksum_convert(struct sk_buff *skb, + __wsum pseudo) +{ + NAPI_GRO_CB(skb)->csum = ~pseudo; + NAPI_GRO_CB(skb)->csum_valid = 1; +} + +#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ +do { \ + if (__skb_gro_checksum_convert_check(skb)) \ + __skb_gro_checksum_convert(skb, \ + compute_pseudo(skb, proto)); \ +} while (0) + +struct gro_remcsum { + int offset; + __wsum delta; +}; + +static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) +{ + grc->offset = 0; + grc->delta = 0; +} + +static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, + unsigned int off, size_t hdrlen, + int start, int offset, + struct gro_remcsum *grc, + bool nopartial) +{ + __wsum delta; + size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); + + BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); + + if (!nopartial) { + NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; + return ptr; + } + + ptr = skb_gro_header_fast(skb, off); + if (skb_gro_header_hard(skb, off + plen)) { + ptr = skb_gro_header_slow(skb, off + plen, off); + if (!ptr) + return NULL; + } + + delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, + start, offset); + + /* Adjust skb->csum since we changed the packet */ + NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); + + grc->offset = off + hdrlen + offset; + grc->delta = delta; + + return ptr; +} + +static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, + struct gro_remcsum *grc) +{ + void *ptr; + size_t plen = grc->offset + sizeof(u16); + + if (!grc->delta) + return; + + ptr = skb_gro_header_fast(skb, grc->offset); + if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { + ptr = skb_gro_header_slow(skb, plen, grc->offset); + if (!ptr) + return; + } + + remcsum_unadjust((__sum16 *)ptr, grc->delta); +} + +#ifdef CONFIG_XFRM_OFFLOAD +static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) +{ + if (PTR_ERR(pp) != -EINPROGRESS) + NAPI_GRO_CB(skb)->flush |= flush; +} +static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, + struct sk_buff *pp, + int flush, + struct gro_remcsum *grc) +{ + if (PTR_ERR(pp) != -EINPROGRESS) { + NAPI_GRO_CB(skb)->flush |= flush; + skb_gro_remcsum_cleanup(skb, grc); + skb->remcsum_offload = 0; + } +} +#else +static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) +{ + NAPI_GRO_CB(skb)->flush |= flush; +} +static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, + struct sk_buff *pp, + int flush, + struct gro_remcsum *grc) +{ + NAPI_GRO_CB(skb)->flush |= flush; + skb_gro_remcsum_cleanup(skb, grc); + skb->remcsum_offload = 0; +} +#endif INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, struct sk_buff *)); @@ -15,6 +373,14 @@ INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); +INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, + struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); + +INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, + struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); + #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ ({ \ unlikely(gro_recursion_inc_test(skb)) ? \ @@ -22,4 +388,30 @@ INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ }) +struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, + struct udphdr *uh, struct sock *sk); +int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); + +static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) +{ + struct udphdr *uh; + unsigned int hlen, off; + + off = skb_gro_offset(skb); + hlen = off + sizeof(*uh); + uh = skb_gro_header_fast(skb, off); + if (skb_gro_header_hard(skb, hlen)) + uh = skb_gro_header_slow(skb, hlen, off); + + return uh; +} + +static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) +{ + const struct ipv6hdr *iph = skb_gro_network_header(skb); + + return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, + skb_gro_len(skb), proto, 0)); +} + #endif /* _NET_IPV6_GRO_H */ diff --git a/include/net/ip.h b/include/net/ip.h index b71e88507c4a..7d1088888c10 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -568,14 +568,6 @@ static inline void iph_to_flow_copy_v4addrs(struct flow_keys *flow, flow->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; } -static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) -{ - const struct iphdr *iph = skb_gro_network_header(skb); - - return csum_tcpudp_nofold(iph->saddr, iph->daddr, - skb_gro_len(skb), proto, 0); -} - /* * Map a multicast IP onto multicast MAC for type ethernet. */ diff --git a/include/net/ip6_checksum.h b/include/net/ip6_checksum.h index ea681910b7a3..c8a96b888277 100644 --- a/include/net/ip6_checksum.h +++ b/include/net/ip6_checksum.h @@ -43,14 +43,6 @@ static inline __wsum ip6_compute_pseudo(struct sk_buff *skb, int proto) skb->len, proto, 0)); } -static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) -{ - const struct ipv6hdr *iph = skb_gro_network_header(skb); - - return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, - skb_gro_len(skb), proto, 0)); -} - static __inline__ __sum16 tcp_v6_check(int len, const struct in6_addr *saddr, const struct in6_addr *daddr, diff --git a/include/net/udp.h b/include/net/udp.h index 909ecf447e0f..f1c2a88c9005 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -167,36 +167,12 @@ static inline void udp_csum_pull_header(struct sk_buff *skb) typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport, __be16 dport); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, - struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, - struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); -struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, - struct udphdr *uh, struct sock *sk); -int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); - struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, netdev_features_t features, bool is_ipv6); -static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) -{ - struct udphdr *uh; - unsigned int hlen, off; - - off = skb_gro_offset(skb); - hlen = off + sizeof(*uh); - uh = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) - uh = skb_gro_header_slow(skb, hlen, off); - - return uh; -} - /* hash routines shared between UDPv4/6 and UDP-Litev4/6 */ static inline int udp_lib_hash(struct sock *sk) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index d57796f38a0b..5b9472bdceff 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -64,6 +64,7 @@ #include #include +#include #include #include #include diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 0189e3cd4a7d..6d52b6491255 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -99,6 +99,7 @@ #include #include #include +#include #include #include #include diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 8e4e9aa12130..d87f02a6e934 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c index 8fcbc6258ec5..b56d6b40c0a2 100644 --- a/net/ipv4/fou.c +++ b/net/ipv4/fou.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c index 1121a9d5fed9..740298dac7d3 100644 --- a/net/ipv4/gre_offload.c +++ b/net/ipv4/gre_offload.c @@ -10,6 +10,7 @@ #include #include #include +#include static struct sk_buff *gre_gso_segment(struct sk_buff *skb, netdev_features_t features) diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index fc61cd3fea65..30abde86db45 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -8,6 +8,7 @@ #include #include +#include #include #include diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 86d32a1e62ac..7fbf9975e8c0 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index a349d4798077..ba5e81cd569c 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c index 1796856bc24f..39db5a226855 100644 --- a/net/ipv6/tcpv6_offload.c +++ b/net/ipv6/tcpv6_offload.c @@ -7,6 +7,7 @@ */ #include #include +#include #include #include #include diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index b3d9ed96e5ea..50a8a65fad23 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -13,6 +13,7 @@ #include #include #include "ip6_offload.h" +#include static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, netdev_features_t features) -- cgit v1.2.3-59-g8ed1b From 0b935d7f8c07bf0a192712bdbf76dbf45ef8b115 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:05:52 -0800 Subject: net: gro: move skb_gro_receive_list to udp_offload.c This helper is used once, no need to keep it in fat net/core/skbuff.c Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 - net/core/skbuff.c | 26 -------------------------- net/ipv4/udp_offload.c | 27 +++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d95c9839ce90..ce6ee1453dbc 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2903,7 +2903,6 @@ struct net_device *dev_get_by_napi_id(unsigned int napi_id); int netdev_get_name(struct net *net, char *name, int ifindex); int dev_restart(struct net_device *dev); int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); -int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb); static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 5b9472bdceff..8560d50c960b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3920,32 +3920,6 @@ err_linearize: } EXPORT_SYMBOL_GPL(skb_segment_list); -int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) -{ - if (unlikely(p->len + skb->len >= 65536)) - return -E2BIG; - - if (NAPI_GRO_CB(p)->last == p) - skb_shinfo(p)->frag_list = skb; - else - NAPI_GRO_CB(p)->last->next = skb; - - skb_pull(skb, skb_gro_offset(skb)); - - NAPI_GRO_CB(p)->last = skb; - NAPI_GRO_CB(p)->count++; - p->data_len += skb->len; - - /* sk owenrship - if any - completely transferred to the aggregated packet */ - skb->destructor = NULL; - p->truesize += skb->truesize; - p->len += skb->len; - - NAPI_GRO_CB(skb)->same_flow = 1; - - return 0; -} - /** * skb_segment - Perform protocol segmentation on skb. * @head_skb: buffer to segment diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 7fbf9975e8c0..cbeb8965d1b7 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -425,6 +425,33 @@ out: return segs; } +static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) +{ + if (unlikely(p->len + skb->len >= 65536)) + return -E2BIG; + + if (NAPI_GRO_CB(p)->last == p) + skb_shinfo(p)->frag_list = skb; + else + NAPI_GRO_CB(p)->last->next = skb; + + skb_pull(skb, skb_gro_offset(skb)); + + NAPI_GRO_CB(p)->last = skb; + NAPI_GRO_CB(p)->count++; + p->data_len += skb->len; + + /* sk owenrship - if any - completely transferred to the aggregated packet */ + skb->destructor = NULL; + p->truesize += skb->truesize; + p->len += skb->len; + + NAPI_GRO_CB(skb)->same_flow = 1; + + return 0; +} + + #define UDP_GRO_CNT_MAX 64 static struct sk_buff *udp_gro_receive_segment(struct list_head *head, struct sk_buff *skb) -- cgit v1.2.3-59-g8ed1b From e456a18a390b96f22b0de2acd4d0f49c72ed2280 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:05:53 -0800 Subject: net: gro: move skb_gro_receive into net/core/gro.c net/core/gro.c will contain all core gro functions, to shrink net/core/skbuff.c and net/core/dev.c Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 - include/net/gro.h | 2 + net/core/Makefile | 2 +- net/core/gro.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++ net/core/skbuff.c | 117 --------------------------------------------- 5 files changed, 121 insertions(+), 119 deletions(-) create mode 100644 net/core/gro.c (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ce6ee1453dbc..93d397db9ec4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2902,7 +2902,6 @@ struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); struct net_device *dev_get_by_napi_id(unsigned int napi_id); int netdev_get_name(struct net *net, char *name, int ifindex); int dev_restart(struct net_device *dev); -int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, diff --git a/include/net/gro.h b/include/net/gro.h index 1ffbe74b2e35..f988bf3440f8 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -414,4 +414,6 @@ static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) skb_gro_len(skb), proto, 0)); } +int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); + #endif /* _NET_IPV6_GRO_H */ diff --git a/net/core/Makefile b/net/core/Makefile index 4268846f2f47..6bdcb2cafed8 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_SYSCTL) += sysctl_net_core.o obj-y += dev.o dev_addr_lists.o dst.o netevent.o \ neighbour.o rtnetlink.o utils.o link_watch.o filter.o \ sock_diag.o dev_ioctl.o tso.o sock_reuseport.o \ - fib_notifier.o xdp.o flow_offload.o + fib_notifier.o xdp.o flow_offload.o gro.o obj-y += net-sysfs.o obj-$(CONFIG_PAGE_POOL) += page_pool.o diff --git a/net/core/gro.c b/net/core/gro.c new file mode 100644 index 000000000000..91a74c4da9ff --- /dev/null +++ b/net/core/gro.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include + +int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) +{ + struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); + unsigned int offset = skb_gro_offset(skb); + unsigned int headlen = skb_headlen(skb); + unsigned int len = skb_gro_len(skb); + unsigned int delta_truesize; + unsigned int new_truesize; + struct sk_buff *lp; + + if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) + return -E2BIG; + + lp = NAPI_GRO_CB(p)->last; + pinfo = skb_shinfo(lp); + + if (headlen <= offset) { + skb_frag_t *frag; + skb_frag_t *frag2; + int i = skbinfo->nr_frags; + int nr_frags = pinfo->nr_frags + i; + + if (nr_frags > MAX_SKB_FRAGS) + goto merge; + + offset -= headlen; + pinfo->nr_frags = nr_frags; + skbinfo->nr_frags = 0; + + frag = pinfo->frags + nr_frags; + frag2 = skbinfo->frags + i; + do { + *--frag = *--frag2; + } while (--i); + + skb_frag_off_add(frag, offset); + skb_frag_size_sub(frag, offset); + + /* all fragments truesize : remove (head size + sk_buff) */ + new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); + delta_truesize = skb->truesize - new_truesize; + + skb->truesize = new_truesize; + skb->len -= skb->data_len; + skb->data_len = 0; + + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; + goto done; + } else if (skb->head_frag) { + int nr_frags = pinfo->nr_frags; + skb_frag_t *frag = pinfo->frags + nr_frags; + struct page *page = virt_to_head_page(skb->head); + unsigned int first_size = headlen - offset; + unsigned int first_offset; + + if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) + goto merge; + + first_offset = skb->data - + (unsigned char *)page_address(page) + + offset; + + pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; + + __skb_frag_set_page(frag, page); + skb_frag_off_set(frag, first_offset); + skb_frag_size_set(frag, first_size); + + memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); + /* We dont need to clear skbinfo->nr_frags here */ + + new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); + delta_truesize = skb->truesize - new_truesize; + skb->truesize = new_truesize; + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; + goto done; + } + +merge: + /* sk owenrship - if any - completely transferred to the aggregated packet */ + skb->destructor = NULL; + delta_truesize = skb->truesize; + if (offset > headlen) { + unsigned int eat = offset - headlen; + + skb_frag_off_add(&skbinfo->frags[0], eat); + skb_frag_size_sub(&skbinfo->frags[0], eat); + skb->data_len -= eat; + skb->len -= eat; + offset = headlen; + } + + __skb_pull(skb, offset); + + if (NAPI_GRO_CB(p)->last == p) + skb_shinfo(p)->frag_list = skb; + else + NAPI_GRO_CB(p)->last->next = skb; + NAPI_GRO_CB(p)->last = skb; + __skb_header_release(skb); + lp = p; + +done: + NAPI_GRO_CB(p)->count++; + p->data_len += len; + p->truesize += delta_truesize; + p->len += len; + if (lp != p) { + lp->data_len += len; + lp->truesize += delta_truesize; + lp->len += len; + } + NAPI_GRO_CB(skb)->same_flow = 1; + return 0; +} diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 8560d50c960b..73720bc1daa3 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -64,7 +64,6 @@ #include #include -#include #include #include #include @@ -4272,122 +4271,6 @@ err: } EXPORT_SYMBOL_GPL(skb_segment); -int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) -{ - struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); - unsigned int offset = skb_gro_offset(skb); - unsigned int headlen = skb_headlen(skb); - unsigned int len = skb_gro_len(skb); - unsigned int delta_truesize; - unsigned int new_truesize; - struct sk_buff *lp; - - if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) - return -E2BIG; - - lp = NAPI_GRO_CB(p)->last; - pinfo = skb_shinfo(lp); - - if (headlen <= offset) { - skb_frag_t *frag; - skb_frag_t *frag2; - int i = skbinfo->nr_frags; - int nr_frags = pinfo->nr_frags + i; - - if (nr_frags > MAX_SKB_FRAGS) - goto merge; - - offset -= headlen; - pinfo->nr_frags = nr_frags; - skbinfo->nr_frags = 0; - - frag = pinfo->frags + nr_frags; - frag2 = skbinfo->frags + i; - do { - *--frag = *--frag2; - } while (--i); - - skb_frag_off_add(frag, offset); - skb_frag_size_sub(frag, offset); - - /* all fragments truesize : remove (head size + sk_buff) */ - new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); - delta_truesize = skb->truesize - new_truesize; - - skb->truesize = new_truesize; - skb->len -= skb->data_len; - skb->data_len = 0; - - NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; - goto done; - } else if (skb->head_frag) { - int nr_frags = pinfo->nr_frags; - skb_frag_t *frag = pinfo->frags + nr_frags; - struct page *page = virt_to_head_page(skb->head); - unsigned int first_size = headlen - offset; - unsigned int first_offset; - - if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) - goto merge; - - first_offset = skb->data - - (unsigned char *)page_address(page) + - offset; - - pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; - - __skb_frag_set_page(frag, page); - skb_frag_off_set(frag, first_offset); - skb_frag_size_set(frag, first_size); - - memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); - /* We dont need to clear skbinfo->nr_frags here */ - - new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); - delta_truesize = skb->truesize - new_truesize; - skb->truesize = new_truesize; - NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; - goto done; - } - -merge: - /* sk owenrship - if any - completely transferred to the aggregated packet */ - skb->destructor = NULL; - delta_truesize = skb->truesize; - if (offset > headlen) { - unsigned int eat = offset - headlen; - - skb_frag_off_add(&skbinfo->frags[0], eat); - skb_frag_size_sub(&skbinfo->frags[0], eat); - skb->data_len -= eat; - skb->len -= eat; - offset = headlen; - } - - __skb_pull(skb, offset); - - if (NAPI_GRO_CB(p)->last == p) - skb_shinfo(p)->frag_list = skb; - else - NAPI_GRO_CB(p)->last->next = skb; - NAPI_GRO_CB(p)->last = skb; - __skb_header_release(skb); - lp = p; - -done: - NAPI_GRO_CB(p)->count++; - p->data_len += len; - p->truesize += delta_truesize; - p->len += len; - if (lp != p) { - lp->data_len += len; - lp->truesize += delta_truesize; - lp->len += len; - } - NAPI_GRO_CB(skb)->same_flow = 1; - return 0; -} - #ifdef CONFIG_SKB_EXTENSIONS #define SKB_EXT_ALIGN_VALUE 8 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) -- cgit v1.2.3-59-g8ed1b From 587652bbdd06ab38a4c1b85e40f933d2cf4a1147 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:05:54 -0800 Subject: net: gro: populate net/core/gro.c Move gro code and data from net/core/dev.c to net/core/gro.c to ease maintenance. gro_normal_list() and gro_normal_one() are inlined because they are called from both files. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + include/net/gro.h | 22 ++ net/core/dev.c | 668 +--------------------------------------------- net/core/gro.c | 648 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 672 insertions(+), 667 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 93d397db9ec4..31a7e6b27681 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3657,6 +3657,7 @@ int netif_rx_ni(struct sk_buff *skb); int netif_rx_any_context(struct sk_buff *skb); int netif_receive_skb(struct sk_buff *skb); int netif_receive_skb_core(struct sk_buff *skb); +void netif_receive_skb_list_internal(struct list_head *head); void netif_receive_skb_list(struct list_head *head); gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb); void napi_gro_flush(struct napi_struct *napi, bool flush_old); diff --git a/include/net/gro.h b/include/net/gro.h index f988bf3440f8..d0e7df691a80 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -416,4 +416,26 @@ static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); +/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ +static inline void gro_normal_list(struct napi_struct *napi) +{ + if (!napi->rx_count) + return; + netif_receive_skb_list_internal(&napi->rx_list); + INIT_LIST_HEAD(&napi->rx_list); + napi->rx_count = 0; +} + +/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, + * pass the whole batch up to the stack. + */ +static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) +{ + list_add_tail(&skb->list, &napi->rx_list); + napi->rx_count += segs; + if (napi->rx_count >= gro_normal_batch) + gro_normal_list(napi); +} + + #endif /* _NET_IPV6_GRO_H */ diff --git a/net/core/dev.c b/net/core/dev.c index 15ac064b5562..92c9258cbf28 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -153,16 +153,10 @@ #include "net-sysfs.h" -#define MAX_GRO_SKBS 8 - -/* This should be increased if a protocol with a bigger head is added. */ -#define GRO_MAX_HEAD (MAX_HEADER + 128) static DEFINE_SPINLOCK(ptype_lock); -static DEFINE_SPINLOCK(offload_lock); struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; struct list_head ptype_all __read_mostly; /* Taps */ -static struct list_head offload_base __read_mostly; static int netif_rx_internal(struct sk_buff *skb); static int call_netdevice_notifiers_info(unsigned long val, @@ -604,84 +598,6 @@ void dev_remove_pack(struct packet_type *pt) EXPORT_SYMBOL(dev_remove_pack); -/** - * dev_add_offload - register offload handlers - * @po: protocol offload declaration - * - * Add protocol offload handlers to the networking stack. The passed - * &proto_offload is linked into kernel lists and may not be freed until - * it has been removed from the kernel lists. - * - * This call does not sleep therefore it can not - * guarantee all CPU's that are in middle of receiving packets - * will see the new offload handlers (until the next received packet). - */ -void dev_add_offload(struct packet_offload *po) -{ - struct packet_offload *elem; - - spin_lock(&offload_lock); - list_for_each_entry(elem, &offload_base, list) { - if (po->priority < elem->priority) - break; - } - list_add_rcu(&po->list, elem->list.prev); - spin_unlock(&offload_lock); -} -EXPORT_SYMBOL(dev_add_offload); - -/** - * __dev_remove_offload - remove offload handler - * @po: packet offload declaration - * - * Remove a protocol offload handler that was previously added to the - * kernel offload handlers by dev_add_offload(). The passed &offload_type - * is removed from the kernel lists and can be freed or reused once this - * function returns. - * - * The packet type might still be in use by receivers - * and must not be freed until after all the CPU's have gone - * through a quiescent state. - */ -static void __dev_remove_offload(struct packet_offload *po) -{ - struct list_head *head = &offload_base; - struct packet_offload *po1; - - spin_lock(&offload_lock); - - list_for_each_entry(po1, head, list) { - if (po == po1) { - list_del_rcu(&po->list); - goto out; - } - } - - pr_warn("dev_remove_offload: %p not found\n", po); -out: - spin_unlock(&offload_lock); -} - -/** - * dev_remove_offload - remove packet offload handler - * @po: packet offload declaration - * - * Remove a packet offload handler that was previously added to the kernel - * offload handlers by dev_add_offload(). The passed &offload_type is - * removed from the kernel lists and can be freed or reused once this - * function returns. - * - * This call sleeps to guarantee that no CPU is looking at the packet - * type after return. - */ -void dev_remove_offload(struct packet_offload *po) -{ - __dev_remove_offload(po); - - synchronize_net(); -} -EXPORT_SYMBOL(dev_remove_offload); - /******************************************************************************* * * Device Interface Subroutines @@ -3315,40 +3231,6 @@ __be16 skb_network_protocol(struct sk_buff *skb, int *depth) return __vlan_get_protocol(skb, type, depth); } -/** - * skb_mac_gso_segment - mac layer segmentation handler. - * @skb: buffer to segment - * @features: features for the output path (see dev->features) - */ -struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, - netdev_features_t features) -{ - struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); - struct packet_offload *ptype; - int vlan_depth = skb->mac_len; - __be16 type = skb_network_protocol(skb, &vlan_depth); - - if (unlikely(!type)) - return ERR_PTR(-EINVAL); - - __skb_pull(skb, vlan_depth); - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, &offload_base, list) { - if (ptype->type == type && ptype->callbacks.gso_segment) { - segs = ptype->callbacks.gso_segment(skb, features); - break; - } - } - rcu_read_unlock(); - - __skb_push(skb, skb->data - skb_mac_header(skb)); - - return segs; -} -EXPORT_SYMBOL(skb_mac_gso_segment); - - /* openvswitch calls this on rx path, so we need a different check. */ static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) @@ -4320,8 +4202,6 @@ int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ int dev_rx_weight __read_mostly = 64; int dev_tx_weight __read_mostly = 64; -/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ -int gro_normal_batch __read_mostly = 8; /* Called with irq disabled */ static inline void ____napi_schedule(struct softnet_data *sd, @@ -5664,7 +5544,7 @@ static int netif_receive_skb_internal(struct sk_buff *skb) return ret; } -static void netif_receive_skb_list_internal(struct list_head *head) +void netif_receive_skb_list_internal(struct list_head *head) { struct sk_buff *skb, *next; struct list_head sublist; @@ -5842,550 +5722,6 @@ static void flush_all_backlogs(void) cpus_read_unlock(); } -/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ -static void gro_normal_list(struct napi_struct *napi) -{ - if (!napi->rx_count) - return; - netif_receive_skb_list_internal(&napi->rx_list); - INIT_LIST_HEAD(&napi->rx_list); - napi->rx_count = 0; -} - -/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, - * pass the whole batch up to the stack. - */ -static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) -{ - list_add_tail(&skb->list, &napi->rx_list); - napi->rx_count += segs; - if (napi->rx_count >= gro_normal_batch) - gro_normal_list(napi); -} - -static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) -{ - struct packet_offload *ptype; - __be16 type = skb->protocol; - struct list_head *head = &offload_base; - int err = -ENOENT; - - BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); - - if (NAPI_GRO_CB(skb)->count == 1) { - skb_shinfo(skb)->gso_size = 0; - goto out; - } - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, head, list) { - if (ptype->type != type || !ptype->callbacks.gro_complete) - continue; - - err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, - ipv6_gro_complete, inet_gro_complete, - skb, 0); - break; - } - rcu_read_unlock(); - - if (err) { - WARN_ON(&ptype->list == head); - kfree_skb(skb); - return; - } - -out: - gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); -} - -static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, - bool flush_old) -{ - struct list_head *head = &napi->gro_hash[index].list; - struct sk_buff *skb, *p; - - list_for_each_entry_safe_reverse(skb, p, head, list) { - if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) - return; - skb_list_del_init(skb); - napi_gro_complete(napi, skb); - napi->gro_hash[index].count--; - } - - if (!napi->gro_hash[index].count) - __clear_bit(index, &napi->gro_bitmask); -} - -/* napi->gro_hash[].list contains packets ordered by age. - * youngest packets at the head of it. - * Complete skbs in reverse order to reduce latencies. - */ -void napi_gro_flush(struct napi_struct *napi, bool flush_old) -{ - unsigned long bitmask = napi->gro_bitmask; - unsigned int i, base = ~0U; - - while ((i = ffs(bitmask)) != 0) { - bitmask >>= i; - base += i; - __napi_gro_flush_chain(napi, base, flush_old); - } -} -EXPORT_SYMBOL(napi_gro_flush); - -static void gro_list_prepare(const struct list_head *head, - const struct sk_buff *skb) -{ - unsigned int maclen = skb->dev->hard_header_len; - u32 hash = skb_get_hash_raw(skb); - struct sk_buff *p; - - list_for_each_entry(p, head, list) { - unsigned long diffs; - - NAPI_GRO_CB(p)->flush = 0; - - if (hash != skb_get_hash_raw(p)) { - NAPI_GRO_CB(p)->same_flow = 0; - continue; - } - - diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; - diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); - if (skb_vlan_tag_present(p)) - diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); - diffs |= skb_metadata_differs(p, skb); - if (maclen == ETH_HLEN) - diffs |= compare_ether_header(skb_mac_header(p), - skb_mac_header(skb)); - else if (!diffs) - diffs = memcmp(skb_mac_header(p), - skb_mac_header(skb), - maclen); - - /* in most common scenarions 'slow_gro' is 0 - * otherwise we are already on some slower paths - * either skip all the infrequent tests altogether or - * avoid trying too hard to skip each of them individually - */ - if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { -#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) - struct tc_skb_ext *skb_ext; - struct tc_skb_ext *p_ext; -#endif - - diffs |= p->sk != skb->sk; - diffs |= skb_metadata_dst_cmp(p, skb); - diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); - -#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) - skb_ext = skb_ext_find(skb, TC_SKB_EXT); - p_ext = skb_ext_find(p, TC_SKB_EXT); - - diffs |= (!!p_ext) ^ (!!skb_ext); - if (!diffs && unlikely(skb_ext)) - diffs |= p_ext->chain ^ skb_ext->chain; -#endif - } - - NAPI_GRO_CB(p)->same_flow = !diffs; - } -} - -static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) -{ - const struct skb_shared_info *pinfo = skb_shinfo(skb); - const skb_frag_t *frag0 = &pinfo->frags[0]; - - NAPI_GRO_CB(skb)->data_offset = 0; - NAPI_GRO_CB(skb)->frag0 = NULL; - NAPI_GRO_CB(skb)->frag0_len = 0; - - if (!skb_headlen(skb) && pinfo->nr_frags && - !PageHighMem(skb_frag_page(frag0)) && - (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { - NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); - NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, - skb_frag_size(frag0), - skb->end - skb->tail); - } -} - -static void gro_pull_from_frag0(struct sk_buff *skb, int grow) -{ - struct skb_shared_info *pinfo = skb_shinfo(skb); - - BUG_ON(skb->end - skb->tail < grow); - - memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); - - skb->data_len -= grow; - skb->tail += grow; - - skb_frag_off_add(&pinfo->frags[0], grow); - skb_frag_size_sub(&pinfo->frags[0], grow); - - if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { - skb_frag_unref(skb, 0); - memmove(pinfo->frags, pinfo->frags + 1, - --pinfo->nr_frags * sizeof(pinfo->frags[0])); - } -} - -static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) -{ - struct sk_buff *oldest; - - oldest = list_last_entry(head, struct sk_buff, list); - - /* We are called with head length >= MAX_GRO_SKBS, so this is - * impossible. - */ - if (WARN_ON_ONCE(!oldest)) - return; - - /* Do not adjust napi->gro_hash[].count, caller is adding a new - * SKB to the chain. - */ - skb_list_del_init(oldest); - napi_gro_complete(napi, oldest); -} - -static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -{ - u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); - struct gro_list *gro_list = &napi->gro_hash[bucket]; - struct list_head *head = &offload_base; - struct packet_offload *ptype; - __be16 type = skb->protocol; - struct sk_buff *pp = NULL; - enum gro_result ret; - int same_flow; - int grow; - - if (netif_elide_gro(skb->dev)) - goto normal; - - gro_list_prepare(&gro_list->list, skb); - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, head, list) { - if (ptype->type != type || !ptype->callbacks.gro_receive) - continue; - - skb_set_network_header(skb, skb_gro_offset(skb)); - skb_reset_mac_len(skb); - NAPI_GRO_CB(skb)->same_flow = 0; - NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); - NAPI_GRO_CB(skb)->free = 0; - NAPI_GRO_CB(skb)->encap_mark = 0; - NAPI_GRO_CB(skb)->recursion_counter = 0; - NAPI_GRO_CB(skb)->is_fou = 0; - NAPI_GRO_CB(skb)->is_atomic = 1; - NAPI_GRO_CB(skb)->gro_remcsum_start = 0; - - /* Setup for GRO checksum validation */ - switch (skb->ip_summed) { - case CHECKSUM_COMPLETE: - NAPI_GRO_CB(skb)->csum = skb->csum; - NAPI_GRO_CB(skb)->csum_valid = 1; - NAPI_GRO_CB(skb)->csum_cnt = 0; - break; - case CHECKSUM_UNNECESSARY: - NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; - NAPI_GRO_CB(skb)->csum_valid = 0; - break; - default: - NAPI_GRO_CB(skb)->csum_cnt = 0; - NAPI_GRO_CB(skb)->csum_valid = 0; - } - - pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, - ipv6_gro_receive, inet_gro_receive, - &gro_list->list, skb); - break; - } - rcu_read_unlock(); - - if (&ptype->list == head) - goto normal; - - if (PTR_ERR(pp) == -EINPROGRESS) { - ret = GRO_CONSUMED; - goto ok; - } - - same_flow = NAPI_GRO_CB(skb)->same_flow; - ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; - - if (pp) { - skb_list_del_init(pp); - napi_gro_complete(napi, pp); - gro_list->count--; - } - - if (same_flow) - goto ok; - - if (NAPI_GRO_CB(skb)->flush) - goto normal; - - if (unlikely(gro_list->count >= MAX_GRO_SKBS)) - gro_flush_oldest(napi, &gro_list->list); - else - gro_list->count++; - - NAPI_GRO_CB(skb)->count = 1; - NAPI_GRO_CB(skb)->age = jiffies; - NAPI_GRO_CB(skb)->last = skb; - skb_shinfo(skb)->gso_size = skb_gro_len(skb); - list_add(&skb->list, &gro_list->list); - ret = GRO_HELD; - -pull: - grow = skb_gro_offset(skb) - skb_headlen(skb); - if (grow > 0) - gro_pull_from_frag0(skb, grow); -ok: - if (gro_list->count) { - if (!test_bit(bucket, &napi->gro_bitmask)) - __set_bit(bucket, &napi->gro_bitmask); - } else if (test_bit(bucket, &napi->gro_bitmask)) { - __clear_bit(bucket, &napi->gro_bitmask); - } - - return ret; - -normal: - ret = GRO_NORMAL; - goto pull; -} - -struct packet_offload *gro_find_receive_by_type(__be16 type) -{ - struct list_head *offload_head = &offload_base; - struct packet_offload *ptype; - - list_for_each_entry_rcu(ptype, offload_head, list) { - if (ptype->type != type || !ptype->callbacks.gro_receive) - continue; - return ptype; - } - return NULL; -} -EXPORT_SYMBOL(gro_find_receive_by_type); - -struct packet_offload *gro_find_complete_by_type(__be16 type) -{ - struct list_head *offload_head = &offload_base; - struct packet_offload *ptype; - - list_for_each_entry_rcu(ptype, offload_head, list) { - if (ptype->type != type || !ptype->callbacks.gro_complete) - continue; - return ptype; - } - return NULL; -} -EXPORT_SYMBOL(gro_find_complete_by_type); - -static gro_result_t napi_skb_finish(struct napi_struct *napi, - struct sk_buff *skb, - gro_result_t ret) -{ - switch (ret) { - case GRO_NORMAL: - gro_normal_one(napi, skb, 1); - break; - - case GRO_MERGED_FREE: - if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) - napi_skb_free_stolen_head(skb); - else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) - __kfree_skb(skb); - else - __kfree_skb_defer(skb); - break; - - case GRO_HELD: - case GRO_MERGED: - case GRO_CONSUMED: - break; - } - - return ret; -} - -gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -{ - gro_result_t ret; - - skb_mark_napi_id(skb, napi); - trace_napi_gro_receive_entry(skb); - - skb_gro_reset_offset(skb, 0); - - ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); - trace_napi_gro_receive_exit(ret); - - return ret; -} -EXPORT_SYMBOL(napi_gro_receive); - -static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) -{ - if (unlikely(skb->pfmemalloc)) { - consume_skb(skb); - return; - } - __skb_pull(skb, skb_headlen(skb)); - /* restore the reserve we had after netdev_alloc_skb_ip_align() */ - skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); - __vlan_hwaccel_clear_tag(skb); - skb->dev = napi->dev; - skb->skb_iif = 0; - - /* eth_type_trans() assumes pkt_type is PACKET_HOST */ - skb->pkt_type = PACKET_HOST; - - skb->encapsulation = 0; - skb_shinfo(skb)->gso_type = 0; - skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); - if (unlikely(skb->slow_gro)) { - skb_orphan(skb); - skb_ext_reset(skb); - nf_reset_ct(skb); - skb->slow_gro = 0; - } - - napi->skb = skb; -} - -struct sk_buff *napi_get_frags(struct napi_struct *napi) -{ - struct sk_buff *skb = napi->skb; - - if (!skb) { - skb = napi_alloc_skb(napi, GRO_MAX_HEAD); - if (skb) { - napi->skb = skb; - skb_mark_napi_id(skb, napi); - } - } - return skb; -} -EXPORT_SYMBOL(napi_get_frags); - -static gro_result_t napi_frags_finish(struct napi_struct *napi, - struct sk_buff *skb, - gro_result_t ret) -{ - switch (ret) { - case GRO_NORMAL: - case GRO_HELD: - __skb_push(skb, ETH_HLEN); - skb->protocol = eth_type_trans(skb, skb->dev); - if (ret == GRO_NORMAL) - gro_normal_one(napi, skb, 1); - break; - - case GRO_MERGED_FREE: - if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) - napi_skb_free_stolen_head(skb); - else - napi_reuse_skb(napi, skb); - break; - - case GRO_MERGED: - case GRO_CONSUMED: - break; - } - - return ret; -} - -/* Upper GRO stack assumes network header starts at gro_offset=0 - * Drivers could call both napi_gro_frags() and napi_gro_receive() - * We copy ethernet header into skb->data to have a common layout. - */ -static struct sk_buff *napi_frags_skb(struct napi_struct *napi) -{ - struct sk_buff *skb = napi->skb; - const struct ethhdr *eth; - unsigned int hlen = sizeof(*eth); - - napi->skb = NULL; - - skb_reset_mac_header(skb); - skb_gro_reset_offset(skb, hlen); - - if (unlikely(skb_gro_header_hard(skb, hlen))) { - eth = skb_gro_header_slow(skb, hlen, 0); - if (unlikely(!eth)) { - net_warn_ratelimited("%s: dropping impossible skb from %s\n", - __func__, napi->dev->name); - napi_reuse_skb(napi, skb); - return NULL; - } - } else { - eth = (const struct ethhdr *)skb->data; - gro_pull_from_frag0(skb, hlen); - NAPI_GRO_CB(skb)->frag0 += hlen; - NAPI_GRO_CB(skb)->frag0_len -= hlen; - } - __skb_pull(skb, hlen); - - /* - * This works because the only protocols we care about don't require - * special handling. - * We'll fix it up properly in napi_frags_finish() - */ - skb->protocol = eth->h_proto; - - return skb; -} - -gro_result_t napi_gro_frags(struct napi_struct *napi) -{ - gro_result_t ret; - struct sk_buff *skb = napi_frags_skb(napi); - - trace_napi_gro_frags_entry(skb); - - ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); - trace_napi_gro_frags_exit(ret); - - return ret; -} -EXPORT_SYMBOL(napi_gro_frags); - -/* Compute the checksum from gro_offset and return the folded value - * after adding in any pseudo checksum. - */ -__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) -{ - __wsum wsum; - __sum16 sum; - - wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); - - /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ - sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); - /* See comments in __skb_checksum_complete(). */ - if (likely(!sum)) { - if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && - !skb->csum_complete_sw) - netdev_rx_csum_fault(skb->dev, skb); - } - - NAPI_GRO_CB(skb)->csum = wsum; - NAPI_GRO_CB(skb)->csum_valid = 1; - - return sum; -} -EXPORT_SYMBOL(__skb_gro_checksum_complete); - static void net_rps_send_ipi(struct softnet_data *remsd) { #ifdef CONFIG_RPS @@ -11640,8 +10976,6 @@ static int __init net_dev_init(void) for (i = 0; i < PTYPE_HASH_SIZE; i++) INIT_LIST_HEAD(&ptype_base[i]); - INIT_LIST_HEAD(&offload_base); - if (register_pernet_subsys(&netdev_net_ops)) goto out; diff --git a/net/core/gro.c b/net/core/gro.c index 91a74c4da9ff..8ec8b44596da 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -1,5 +1,129 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include +#include + +#define MAX_GRO_SKBS 8 + +/* This should be increased if a protocol with a bigger head is added. */ +#define GRO_MAX_HEAD (MAX_HEADER + 128) + +static DEFINE_SPINLOCK(offload_lock); +static struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base); +/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ +int gro_normal_batch __read_mostly = 8; + +/** + * dev_add_offload - register offload handlers + * @po: protocol offload declaration + * + * Add protocol offload handlers to the networking stack. The passed + * &proto_offload is linked into kernel lists and may not be freed until + * it has been removed from the kernel lists. + * + * This call does not sleep therefore it can not + * guarantee all CPU's that are in middle of receiving packets + * will see the new offload handlers (until the next received packet). + */ +void dev_add_offload(struct packet_offload *po) +{ + struct packet_offload *elem; + + spin_lock(&offload_lock); + list_for_each_entry(elem, &offload_base, list) { + if (po->priority < elem->priority) + break; + } + list_add_rcu(&po->list, elem->list.prev); + spin_unlock(&offload_lock); +} +EXPORT_SYMBOL(dev_add_offload); + +/** + * __dev_remove_offload - remove offload handler + * @po: packet offload declaration + * + * Remove a protocol offload handler that was previously added to the + * kernel offload handlers by dev_add_offload(). The passed &offload_type + * is removed from the kernel lists and can be freed or reused once this + * function returns. + * + * The packet type might still be in use by receivers + * and must not be freed until after all the CPU's have gone + * through a quiescent state. + */ +static void __dev_remove_offload(struct packet_offload *po) +{ + struct list_head *head = &offload_base; + struct packet_offload *po1; + + spin_lock(&offload_lock); + + list_for_each_entry(po1, head, list) { + if (po == po1) { + list_del_rcu(&po->list); + goto out; + } + } + + pr_warn("dev_remove_offload: %p not found\n", po); +out: + spin_unlock(&offload_lock); +} + +/** + * dev_remove_offload - remove packet offload handler + * @po: packet offload declaration + * + * Remove a packet offload handler that was previously added to the kernel + * offload handlers by dev_add_offload(). The passed &offload_type is + * removed from the kernel lists and can be freed or reused once this + * function returns. + * + * This call sleeps to guarantee that no CPU is looking at the packet + * type after return. + */ +void dev_remove_offload(struct packet_offload *po) +{ + __dev_remove_offload(po); + + synchronize_net(); +} +EXPORT_SYMBOL(dev_remove_offload); + +/** + * skb_mac_gso_segment - mac layer segmentation handler. + * @skb: buffer to segment + * @features: features for the output path (see dev->features) + */ +struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, + netdev_features_t features) +{ + struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); + struct packet_offload *ptype; + int vlan_depth = skb->mac_len; + __be16 type = skb_network_protocol(skb, &vlan_depth); + + if (unlikely(!type)) + return ERR_PTR(-EINVAL); + + __skb_pull(skb, vlan_depth); + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, &offload_base, list) { + if (ptype->type == type && ptype->callbacks.gso_segment) { + segs = ptype->callbacks.gso_segment(skb, features); + break; + } + } + rcu_read_unlock(); + + __skb_push(skb, skb->data - skb_mac_header(skb)); + + return segs; +} +EXPORT_SYMBOL(skb_mac_gso_segment); int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) { @@ -116,3 +240,527 @@ done: NAPI_GRO_CB(skb)->same_flow = 1; return 0; } + + +static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) +{ + struct packet_offload *ptype; + __be16 type = skb->protocol; + struct list_head *head = &offload_base; + int err = -ENOENT; + + BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); + + if (NAPI_GRO_CB(skb)->count == 1) { + skb_shinfo(skb)->gso_size = 0; + goto out; + } + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, head, list) { + if (ptype->type != type || !ptype->callbacks.gro_complete) + continue; + + err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, + ipv6_gro_complete, inet_gro_complete, + skb, 0); + break; + } + rcu_read_unlock(); + + if (err) { + WARN_ON(&ptype->list == head); + kfree_skb(skb); + return; + } + +out: + gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); +} + +static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, + bool flush_old) +{ + struct list_head *head = &napi->gro_hash[index].list; + struct sk_buff *skb, *p; + + list_for_each_entry_safe_reverse(skb, p, head, list) { + if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) + return; + skb_list_del_init(skb); + napi_gro_complete(napi, skb); + napi->gro_hash[index].count--; + } + + if (!napi->gro_hash[index].count) + __clear_bit(index, &napi->gro_bitmask); +} + +/* napi->gro_hash[].list contains packets ordered by age. + * youngest packets at the head of it. + * Complete skbs in reverse order to reduce latencies. + */ +void napi_gro_flush(struct napi_struct *napi, bool flush_old) +{ + unsigned long bitmask = napi->gro_bitmask; + unsigned int i, base = ~0U; + + while ((i = ffs(bitmask)) != 0) { + bitmask >>= i; + base += i; + __napi_gro_flush_chain(napi, base, flush_old); + } +} +EXPORT_SYMBOL(napi_gro_flush); + +static void gro_list_prepare(const struct list_head *head, + const struct sk_buff *skb) +{ + unsigned int maclen = skb->dev->hard_header_len; + u32 hash = skb_get_hash_raw(skb); + struct sk_buff *p; + + list_for_each_entry(p, head, list) { + unsigned long diffs; + + NAPI_GRO_CB(p)->flush = 0; + + if (hash != skb_get_hash_raw(p)) { + NAPI_GRO_CB(p)->same_flow = 0; + continue; + } + + diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; + diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); + if (skb_vlan_tag_present(p)) + diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); + diffs |= skb_metadata_differs(p, skb); + if (maclen == ETH_HLEN) + diffs |= compare_ether_header(skb_mac_header(p), + skb_mac_header(skb)); + else if (!diffs) + diffs = memcmp(skb_mac_header(p), + skb_mac_header(skb), + maclen); + + /* in most common scenarions 'slow_gro' is 0 + * otherwise we are already on some slower paths + * either skip all the infrequent tests altogether or + * avoid trying too hard to skip each of them individually + */ + if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { +#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) + struct tc_skb_ext *skb_ext; + struct tc_skb_ext *p_ext; +#endif + + diffs |= p->sk != skb->sk; + diffs |= skb_metadata_dst_cmp(p, skb); + diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); + +#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) + skb_ext = skb_ext_find(skb, TC_SKB_EXT); + p_ext = skb_ext_find(p, TC_SKB_EXT); + + diffs |= (!!p_ext) ^ (!!skb_ext); + if (!diffs && unlikely(skb_ext)) + diffs |= p_ext->chain ^ skb_ext->chain; +#endif + } + + NAPI_GRO_CB(p)->same_flow = !diffs; + } +} + +static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) +{ + const struct skb_shared_info *pinfo = skb_shinfo(skb); + const skb_frag_t *frag0 = &pinfo->frags[0]; + + NAPI_GRO_CB(skb)->data_offset = 0; + NAPI_GRO_CB(skb)->frag0 = NULL; + NAPI_GRO_CB(skb)->frag0_len = 0; + + if (!skb_headlen(skb) && pinfo->nr_frags && + !PageHighMem(skb_frag_page(frag0)) && + (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { + NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); + NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, + skb_frag_size(frag0), + skb->end - skb->tail); + } +} + +static void gro_pull_from_frag0(struct sk_buff *skb, int grow) +{ + struct skb_shared_info *pinfo = skb_shinfo(skb); + + BUG_ON(skb->end - skb->tail < grow); + + memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); + + skb->data_len -= grow; + skb->tail += grow; + + skb_frag_off_add(&pinfo->frags[0], grow); + skb_frag_size_sub(&pinfo->frags[0], grow); + + if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { + skb_frag_unref(skb, 0); + memmove(pinfo->frags, pinfo->frags + 1, + --pinfo->nr_frags * sizeof(pinfo->frags[0])); + } +} + +static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) +{ + struct sk_buff *oldest; + + oldest = list_last_entry(head, struct sk_buff, list); + + /* We are called with head length >= MAX_GRO_SKBS, so this is + * impossible. + */ + if (WARN_ON_ONCE(!oldest)) + return; + + /* Do not adjust napi->gro_hash[].count, caller is adding a new + * SKB to the chain. + */ + skb_list_del_init(oldest); + napi_gro_complete(napi, oldest); +} + +static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +{ + u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); + struct gro_list *gro_list = &napi->gro_hash[bucket]; + struct list_head *head = &offload_base; + struct packet_offload *ptype; + __be16 type = skb->protocol; + struct sk_buff *pp = NULL; + enum gro_result ret; + int same_flow; + int grow; + + if (netif_elide_gro(skb->dev)) + goto normal; + + gro_list_prepare(&gro_list->list, skb); + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, head, list) { + if (ptype->type != type || !ptype->callbacks.gro_receive) + continue; + + skb_set_network_header(skb, skb_gro_offset(skb)); + skb_reset_mac_len(skb); + NAPI_GRO_CB(skb)->same_flow = 0; + NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); + NAPI_GRO_CB(skb)->free = 0; + NAPI_GRO_CB(skb)->encap_mark = 0; + NAPI_GRO_CB(skb)->recursion_counter = 0; + NAPI_GRO_CB(skb)->is_fou = 0; + NAPI_GRO_CB(skb)->is_atomic = 1; + NAPI_GRO_CB(skb)->gro_remcsum_start = 0; + + /* Setup for GRO checksum validation */ + switch (skb->ip_summed) { + case CHECKSUM_COMPLETE: + NAPI_GRO_CB(skb)->csum = skb->csum; + NAPI_GRO_CB(skb)->csum_valid = 1; + NAPI_GRO_CB(skb)->csum_cnt = 0; + break; + case CHECKSUM_UNNECESSARY: + NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; + NAPI_GRO_CB(skb)->csum_valid = 0; + break; + default: + NAPI_GRO_CB(skb)->csum_cnt = 0; + NAPI_GRO_CB(skb)->csum_valid = 0; + } + + pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, + ipv6_gro_receive, inet_gro_receive, + &gro_list->list, skb); + break; + } + rcu_read_unlock(); + + if (&ptype->list == head) + goto normal; + + if (PTR_ERR(pp) == -EINPROGRESS) { + ret = GRO_CONSUMED; + goto ok; + } + + same_flow = NAPI_GRO_CB(skb)->same_flow; + ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; + + if (pp) { + skb_list_del_init(pp); + napi_gro_complete(napi, pp); + gro_list->count--; + } + + if (same_flow) + goto ok; + + if (NAPI_GRO_CB(skb)->flush) + goto normal; + + if (unlikely(gro_list->count >= MAX_GRO_SKBS)) + gro_flush_oldest(napi, &gro_list->list); + else + gro_list->count++; + + NAPI_GRO_CB(skb)->count = 1; + NAPI_GRO_CB(skb)->age = jiffies; + NAPI_GRO_CB(skb)->last = skb; + skb_shinfo(skb)->gso_size = skb_gro_len(skb); + list_add(&skb->list, &gro_list->list); + ret = GRO_HELD; + +pull: + grow = skb_gro_offset(skb) - skb_headlen(skb); + if (grow > 0) + gro_pull_from_frag0(skb, grow); +ok: + if (gro_list->count) { + if (!test_bit(bucket, &napi->gro_bitmask)) + __set_bit(bucket, &napi->gro_bitmask); + } else if (test_bit(bucket, &napi->gro_bitmask)) { + __clear_bit(bucket, &napi->gro_bitmask); + } + + return ret; + +normal: + ret = GRO_NORMAL; + goto pull; +} + +struct packet_offload *gro_find_receive_by_type(__be16 type) +{ + struct list_head *offload_head = &offload_base; + struct packet_offload *ptype; + + list_for_each_entry_rcu(ptype, offload_head, list) { + if (ptype->type != type || !ptype->callbacks.gro_receive) + continue; + return ptype; + } + return NULL; +} +EXPORT_SYMBOL(gro_find_receive_by_type); + +struct packet_offload *gro_find_complete_by_type(__be16 type) +{ + struct list_head *offload_head = &offload_base; + struct packet_offload *ptype; + + list_for_each_entry_rcu(ptype, offload_head, list) { + if (ptype->type != type || !ptype->callbacks.gro_complete) + continue; + return ptype; + } + return NULL; +} +EXPORT_SYMBOL(gro_find_complete_by_type); + +static gro_result_t napi_skb_finish(struct napi_struct *napi, + struct sk_buff *skb, + gro_result_t ret) +{ + switch (ret) { + case GRO_NORMAL: + gro_normal_one(napi, skb, 1); + break; + + case GRO_MERGED_FREE: + if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) + napi_skb_free_stolen_head(skb); + else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) + __kfree_skb(skb); + else + __kfree_skb_defer(skb); + break; + + case GRO_HELD: + case GRO_MERGED: + case GRO_CONSUMED: + break; + } + + return ret; +} + +gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +{ + gro_result_t ret; + + skb_mark_napi_id(skb, napi); + trace_napi_gro_receive_entry(skb); + + skb_gro_reset_offset(skb, 0); + + ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); + trace_napi_gro_receive_exit(ret); + + return ret; +} +EXPORT_SYMBOL(napi_gro_receive); + +static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) +{ + if (unlikely(skb->pfmemalloc)) { + consume_skb(skb); + return; + } + __skb_pull(skb, skb_headlen(skb)); + /* restore the reserve we had after netdev_alloc_skb_ip_align() */ + skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); + __vlan_hwaccel_clear_tag(skb); + skb->dev = napi->dev; + skb->skb_iif = 0; + + /* eth_type_trans() assumes pkt_type is PACKET_HOST */ + skb->pkt_type = PACKET_HOST; + + skb->encapsulation = 0; + skb_shinfo(skb)->gso_type = 0; + skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); + if (unlikely(skb->slow_gro)) { + skb_orphan(skb); + skb_ext_reset(skb); + nf_reset_ct(skb); + skb->slow_gro = 0; + } + + napi->skb = skb; +} + +struct sk_buff *napi_get_frags(struct napi_struct *napi) +{ + struct sk_buff *skb = napi->skb; + + if (!skb) { + skb = napi_alloc_skb(napi, GRO_MAX_HEAD); + if (skb) { + napi->skb = skb; + skb_mark_napi_id(skb, napi); + } + } + return skb; +} +EXPORT_SYMBOL(napi_get_frags); + +static gro_result_t napi_frags_finish(struct napi_struct *napi, + struct sk_buff *skb, + gro_result_t ret) +{ + switch (ret) { + case GRO_NORMAL: + case GRO_HELD: + __skb_push(skb, ETH_HLEN); + skb->protocol = eth_type_trans(skb, skb->dev); + if (ret == GRO_NORMAL) + gro_normal_one(napi, skb, 1); + break; + + case GRO_MERGED_FREE: + if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) + napi_skb_free_stolen_head(skb); + else + napi_reuse_skb(napi, skb); + break; + + case GRO_MERGED: + case GRO_CONSUMED: + break; + } + + return ret; +} + +/* Upper GRO stack assumes network header starts at gro_offset=0 + * Drivers could call both napi_gro_frags() and napi_gro_receive() + * We copy ethernet header into skb->data to have a common layout. + */ +static struct sk_buff *napi_frags_skb(struct napi_struct *napi) +{ + struct sk_buff *skb = napi->skb; + const struct ethhdr *eth; + unsigned int hlen = sizeof(*eth); + + napi->skb = NULL; + + skb_reset_mac_header(skb); + skb_gro_reset_offset(skb, hlen); + + if (unlikely(skb_gro_header_hard(skb, hlen))) { + eth = skb_gro_header_slow(skb, hlen, 0); + if (unlikely(!eth)) { + net_warn_ratelimited("%s: dropping impossible skb from %s\n", + __func__, napi->dev->name); + napi_reuse_skb(napi, skb); + return NULL; + } + } else { + eth = (const struct ethhdr *)skb->data; + gro_pull_from_frag0(skb, hlen); + NAPI_GRO_CB(skb)->frag0 += hlen; + NAPI_GRO_CB(skb)->frag0_len -= hlen; + } + __skb_pull(skb, hlen); + + /* + * This works because the only protocols we care about don't require + * special handling. + * We'll fix it up properly in napi_frags_finish() + */ + skb->protocol = eth->h_proto; + + return skb; +} + +gro_result_t napi_gro_frags(struct napi_struct *napi) +{ + gro_result_t ret; + struct sk_buff *skb = napi_frags_skb(napi); + + trace_napi_gro_frags_entry(skb); + + ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); + trace_napi_gro_frags_exit(ret); + + return ret; +} +EXPORT_SYMBOL(napi_gro_frags); + +/* Compute the checksum from gro_offset and return the folded value + * after adding in any pseudo checksum. + */ +__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) +{ + __wsum wsum; + __sum16 sum; + + wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); + + /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ + sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); + /* See comments in __skb_checksum_complete(). */ + if (likely(!sum)) { + if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && + !skb->csum_complete_sw) + netdev_rx_csum_fault(skb->dev, skb); + } + + NAPI_GRO_CB(skb)->csum = wsum; + NAPI_GRO_CB(skb)->csum_valid = 1; + + return sum; +} +EXPORT_SYMBOL(__skb_gro_checksum_complete); -- cgit v1.2.3-59-g8ed1b From 77d641baa3c8e18a1056bec6c64c6103c1a17b1e Mon Sep 17 00:00:00 2001 From: Luca Ceresoli Date: Mon, 8 Nov 2021 17:27:05 +0100 Subject: power: supply: core: add POWER_SUPPLY_HEALTH_NO_BATTERY Some chargers can keep the system powered from the mains even when no battery is present. It this case none of the currently defined health statuses applies. Add a new status to report that no battery is present. Suggested-by: Sebastian Reichel Signed-off-by: Luca Ceresoli Signed-off-by: Sebastian Reichel --- Documentation/ABI/testing/sysfs-class-power | 2 +- drivers/power/supply/power_supply_sysfs.c | 1 + include/linux/power_supply.h | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power index f7904efc4cfa..a0b2a4280e38 100644 --- a/Documentation/ABI/testing/sysfs-class-power +++ b/Documentation/ABI/testing/sysfs-class-power @@ -413,7 +413,7 @@ Description: "Over voltage", "Unspecified failure", "Cold", "Watchdog timer expire", "Safety timer expire", "Over current", "Calibration required", "Warm", - "Cool", "Hot" + "Cool", "Hot", "No battery" What: /sys/class/power_supply//precharge_current Date: June 2017 diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index c3d7cbcd4fad..6ac88fbee3cb 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -106,6 +106,7 @@ static const char * const POWER_SUPPLY_HEALTH_TEXT[] = { [POWER_SUPPLY_HEALTH_WARM] = "Warm", [POWER_SUPPLY_HEALTH_COOL] = "Cool", [POWER_SUPPLY_HEALTH_HOT] = "Hot", + [POWER_SUPPLY_HEALTH_NO_BATTERY] = "No battery", }; static const char * const POWER_SUPPLY_TECHNOLOGY_TEXT[] = { diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 9ca1f120a211..2d1318fe2455 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -66,6 +66,7 @@ enum { POWER_SUPPLY_HEALTH_WARM, POWER_SUPPLY_HEALTH_COOL, POWER_SUPPLY_HEALTH_HOT, + POWER_SUPPLY_HEALTH_NO_BATTERY, }; enum { -- cgit v1.2.3-59-g8ed1b From c2c60ea37e5b6be58c9dd7aff0b2e86ba0f18e0b Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:23:01 -0800 Subject: once: use __section(".data.once") .data.once contains nicely packed bool variables. It is used already by DO_ONCE_LITE(). Using it also in DO_ONCE() removes holes in .data section. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/once.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/once.h b/include/linux/once.h index d361fb14ac3a..f54523052bbc 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -38,7 +38,7 @@ void __do_once_done(bool *done, struct static_key_true *once_key, #define DO_ONCE(func, ...) \ ({ \ bool ___ret = false; \ - static bool ___done = false; \ + static bool __section(".data.once") ___done = false; \ static DEFINE_STATIC_KEY_TRUE(___once_key); \ if (static_branch_unlikely(&___once_key)) { \ unsigned long ___flags; \ -- cgit v1.2.3-59-g8ed1b From 7071732c26fe2cf141185ed16a8a85d02495ae8c Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:23:02 -0800 Subject: net: use .data.once section in netdev_level_once() Same rationale than prior patch : using the dedicated section avoid holes and pack all these bool values. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 31a7e6b27681..dd328364dfe9 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4942,7 +4942,7 @@ void netdev_info(const struct net_device *dev, const char *format, ...); #define netdev_level_once(level, dev, fmt, ...) \ do { \ - static bool __print_once __read_mostly; \ + static bool __section(".data.once") __print_once; \ \ if (!__print_once) { \ __print_once = true; \ -- cgit v1.2.3-59-g8ed1b From 49ecc2e9c3abd269951972fa8b23a4d081111b80 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 15 Nov 2021 09:23:03 -0800 Subject: net: align static siphash keys siphash keys use 16 bytes. Define siphash_aligned_key_t macro so that we can make sure they are not crossing a cache line boundary. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/siphash.h | 2 ++ net/core/flow_dissector.c | 2 +- net/core/secure_seq.c | 4 ++-- net/ipv4/route.c | 2 +- net/ipv4/syncookies.c | 2 +- net/ipv6/route.c | 2 +- net/ipv6/syncookies.c | 2 +- net/netfilter/nf_conntrack_core.c | 4 ++-- net/netfilter/nf_conntrack_expect.c | 2 +- net/netfilter/nf_conntrack_netlink.c | 2 +- net/netfilter/nf_nat_core.c | 2 +- 11 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/siphash.h b/include/linux/siphash.h index bf21591a9e5e..3f7427b9e935 100644 --- a/include/linux/siphash.h +++ b/include/linux/siphash.h @@ -21,6 +21,8 @@ typedef struct { u64 key[2]; } siphash_key_t; +#define siphash_aligned_key_t siphash_key_t __aligned(16) + static inline bool siphash_key_is_zero(const siphash_key_t *key) { return !(key->key[0] | key->key[1]); diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 3255f57f5131..257976cb55ce 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1460,7 +1460,7 @@ out_bad: } EXPORT_SYMBOL(__skb_flow_dissect); -static siphash_key_t hashrnd __read_mostly; +static siphash_aligned_key_t hashrnd; static __always_inline void __flow_hash_secret_init(void) { net_get_random_once(&hashrnd, sizeof(hashrnd)); diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index b5bc680d4755..9b8443774449 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -19,8 +19,8 @@ #include #include -static siphash_key_t net_secret __read_mostly; -static siphash_key_t ts_secret __read_mostly; +static siphash_aligned_key_t net_secret; +static siphash_aligned_key_t ts_secret; static __always_inline void net_secret_init(void) { diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 0b4103b1e622..243a0c52be42 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -602,7 +602,7 @@ static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash) static u32 fnhe_hashfun(__be32 daddr) { - static siphash_key_t fnhe_hash_key __read_mostly; + static siphash_aligned_key_t fnhe_hash_key; u64 hval; net_get_random_once(&fnhe_hash_key, sizeof(fnhe_hash_key)); diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 8696dc343ad2..2cb3b852d148 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -14,7 +14,7 @@ #include #include -static siphash_key_t syncookie_secret[2] __read_mostly; +static siphash_aligned_key_t syncookie_secret[2]; #define COOKIEBITS 24 /* Upper bits store count */ #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 3ae25b8ffbd6..5e8f2f15607d 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -1485,7 +1485,7 @@ static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) static u32 rt6_exception_hash(const struct in6_addr *dst, const struct in6_addr *src) { - static siphash_key_t rt6_exception_key __read_mostly; + static siphash_aligned_key_t rt6_exception_key; struct { struct in6_addr dst; struct in6_addr src; diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index e8cfb9e997bf..d1b61d00368e 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -20,7 +20,7 @@ #define COOKIEBITS 24 /* Upper bits store count */ #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1) -static siphash_key_t syncookie6_secret[2] __read_mostly; +static siphash_aligned_key_t syncookie6_secret[2]; /* RFC 2460, Section 8.3: * [ipv6 tcp] MSS must be computed as the maximum packet size minus 60 [..] diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 770a63103c7a..054ee9d25efe 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -189,7 +189,7 @@ EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); unsigned int nf_conntrack_max __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_max); seqcount_spinlock_t nf_conntrack_generation __read_mostly; -static siphash_key_t nf_conntrack_hash_rnd __read_mostly; +static siphash_aligned_key_t nf_conntrack_hash_rnd; static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, unsigned int zoneid, @@ -482,7 +482,7 @@ EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); */ u32 nf_ct_get_id(const struct nf_conn *ct) { - static __read_mostly siphash_key_t ct_id_seed; + static siphash_aligned_key_t ct_id_seed; unsigned long a, b, c, d; net_get_random_once(&ct_id_seed, sizeof(ct_id_seed)); diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index f562eeef4234..1e89b595ecd0 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL_GPL(nf_ct_expect_hash); unsigned int nf_ct_expect_max __read_mostly; static struct kmem_cache *nf_ct_expect_cachep __read_mostly; -static siphash_key_t nf_ct_expect_hashrnd __read_mostly; +static siphash_aligned_key_t nf_ct_expect_hashrnd; /* nf_conntrack_expect helper functions */ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index f1e5443fe7c7..3d6c8da3de1f 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -2997,7 +2997,7 @@ static const union nf_inet_addr any_addr; static __be32 nf_expect_get_id(const struct nf_conntrack_expect *exp) { - static __read_mostly siphash_key_t exp_id_seed; + static siphash_aligned_key_t exp_id_seed; unsigned long a, b, c, d; net_get_random_once(&exp_id_seed, sizeof(exp_id_seed)); diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 4d50d51db796..ab9f6c75524d 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -34,7 +34,7 @@ static unsigned int nat_net_id __read_mostly; static struct hlist_head *nf_nat_bysource __read_mostly; static unsigned int nf_nat_htable_size __read_mostly; -static siphash_key_t nf_nat_hash_rnd __read_mostly; +static siphash_aligned_key_t nf_nat_hash_rnd; struct nf_nat_lookup_hook_priv { struct nf_hook_entries __rcu *entries; -- cgit v1.2.3-59-g8ed1b From b9241f54138ca5af4d3c5ca6db56be83d7491508 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Mon, 15 Nov 2021 17:11:17 +0000 Subject: net: document SMII and correct phylink's new validation mechanism SMII has not been documented in the kernel, but information on this PHY interface mode has been recently found. Document it, and correct the recently introduced phylink handling for this interface mode. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/E1mmfVl-0075nP-14@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- Documentation/networking/phy.rst | 5 +++++ drivers/net/phy/phylink.c | 2 +- include/linux/phy.h | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index 571ba08386e7..d43da709bf40 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -237,6 +237,11 @@ negotiation results. Some of the interface modes are described below: +``PHY_INTERFACE_MODE_SMII`` + This is serial MII, clocked at 125MHz, supporting 100M and 10M speeds. + Some details can be found in + https://opencores.org/ocsvn/smii/smii/trunk/doc/SMII.pdf + ``PHY_INTERFACE_MODE_1000BASEX`` This defines the 1000BASE-X single-lane serdes link as defined by the 802.3 standard section 36. The link operates at a fixed bit rate of diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 33462fdc7add..f7156b6868e7 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -336,6 +336,7 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, case PHY_INTERFACE_MODE_REVRMII: case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_SMII: case PHY_INTERFACE_MODE_REVMII: case PHY_INTERFACE_MODE_MII: caps |= MAC_10HD | MAC_10FD; @@ -385,7 +386,6 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, case PHY_INTERFACE_MODE_NA: case PHY_INTERFACE_MODE_MAX: - case PHY_INTERFACE_MODE_SMII: break; } diff --git a/include/linux/phy.h b/include/linux/phy.h index 96e43fbb2dd8..1e57cdd95da3 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -99,7 +99,7 @@ extern const int phy_10gbit_features_array[1]; * @PHY_INTERFACE_MODE_RGMII_RXID: RGMII with Internal RX delay * @PHY_INTERFACE_MODE_RGMII_TXID: RGMII with Internal RX delay * @PHY_INTERFACE_MODE_RTBI: Reduced TBI - * @PHY_INTERFACE_MODE_SMII: ??? MII + * @PHY_INTERFACE_MODE_SMII: Serial MII * @PHY_INTERFACE_MODE_XGMII: 10 gigabit media-independent interface * @PHY_INTERFACE_MODE_XLGMII:40 gigabit media-independent interface * @PHY_INTERFACE_MODE_MOCA: Multimedia over Coax -- cgit v1.2.3-59-g8ed1b From 5854d4a6cc356ba3e16d8593ac1c089a32d1759c Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 29 Oct 2021 20:26:12 +0300 Subject: mtd: spi-nor: Get rid of nor->page_size nor->page_size duplicated what nor->params->page_size indicates for no good reason. page_size is a flash parameter of fixed value and it is better suited to be found in nor->params->page_size. Signed-off-by: Tudor Ambarus Reviewed-by: Pratyush Yadav Reviewed-by: Michael Walle Link: https://lore.kernel.org/r/20211029172633.886453-5-tudor.ambarus@microchip.com --- drivers/mtd/spi-nor/core.c | 19 +++++++++---------- drivers/mtd/spi-nor/xilinx.c | 17 ++++++++++------- include/linux/mtd/spi-nor.h | 2 -- 3 files changed, 19 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index 277d1fde84c8..3ec0959ffc20 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -1952,6 +1952,7 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len, struct spi_nor *nor = mtd_to_spi_nor(mtd); size_t page_offset, page_remain, i; ssize_t ret; + u32 page_size = nor->params->page_size; dev_dbg(nor->dev, "to 0x%08x, len %zd\n", (u32)to, len); @@ -1968,16 +1969,15 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len, * calculated with an AND operation. On the other cases we * need to do a modulus operation (more expensive). */ - if (is_power_of_2(nor->page_size)) { - page_offset = addr & (nor->page_size - 1); + if (is_power_of_2(page_size)) { + page_offset = addr & (page_size - 1); } else { uint64_t aux = addr; - page_offset = do_div(aux, nor->page_size); + page_offset = do_div(aux, page_size); } /* the size of data remaining on the first page */ - page_remain = min_t(size_t, - nor->page_size - page_offset, len - i); + page_remain = min_t(size_t, page_size - page_offset, len - i); addr = spi_nor_convert_addr(nor, addr); @@ -3094,7 +3094,7 @@ int spi_nor_scan(struct spi_nor *nor, const char *name, * We need the bounce buffer early to read/write registers when going * through the spi-mem layer (buffers have to be DMA-able). * For spi-mem drivers, we'll reallocate a new buffer if - * nor->page_size turns out to be greater than PAGE_SIZE (which + * nor->params->page_size turns out to be greater than PAGE_SIZE (which * shouldn't happen before long since NOR pages are usually less * than 1KB) after spi_nor_scan() returns. */ @@ -3170,8 +3170,7 @@ int spi_nor_scan(struct spi_nor *nor, const char *name, mtd->flags |= MTD_NO_ERASE; mtd->dev.parent = dev; - nor->page_size = nor->params->page_size; - mtd->writebufsize = nor->page_size; + mtd->writebufsize = nor->params->page_size; if (of_property_read_bool(np, "broken-flash-reset")) nor->flags |= SNOR_F_BROKEN_RESET; @@ -3340,8 +3339,8 @@ static int spi_nor_probe(struct spi_mem *spimem) * and add this logic so that if anyone ever adds support for such * a NOR we don't end up with buffer overflows. */ - if (nor->page_size > PAGE_SIZE) { - nor->bouncebuf_size = nor->page_size; + if (nor->params->page_size > PAGE_SIZE) { + nor->bouncebuf_size = nor->params->page_size; devm_kfree(nor->dev, nor->bouncebuf); nor->bouncebuf = devm_kmalloc(nor->dev, nor->bouncebuf_size, diff --git a/drivers/mtd/spi-nor/xilinx.c b/drivers/mtd/spi-nor/xilinx.c index 1138bdbf4199..0658e47564ba 100644 --- a/drivers/mtd/spi-nor/xilinx.c +++ b/drivers/mtd/spi-nor/xilinx.c @@ -28,11 +28,12 @@ static const struct flash_info xilinx_parts[] = { */ static u32 s3an_convert_addr(struct spi_nor *nor, u32 addr) { + u32 page_size = nor->params->page_size; u32 offset, page; - offset = addr % nor->page_size; - page = addr / nor->page_size; - page <<= (nor->page_size > 512) ? 10 : 9; + offset = addr % page_size; + page = addr / page_size; + page <<= (page_size > 512) ? 10 : 9; return page | offset; } @@ -40,6 +41,7 @@ static u32 s3an_convert_addr(struct spi_nor *nor, u32 addr) static int xilinx_nor_setup(struct spi_nor *nor, const struct spi_nor_hwcaps *hwcaps) { + u32 page_size; int ret; ret = spi_nor_xread_sr(nor, nor->bouncebuf); @@ -64,10 +66,11 @@ static int xilinx_nor_setup(struct spi_nor *nor, */ if (nor->bouncebuf[0] & XSR_PAGESIZE) { /* Flash in Power of 2 mode */ - nor->page_size = (nor->page_size == 264) ? 256 : 512; - nor->mtd.writebufsize = nor->page_size; - nor->mtd.size = 8 * nor->page_size * nor->info->n_sectors; - nor->mtd.erasesize = 8 * nor->page_size; + page_size = (nor->params->page_size == 264) ? 256 : 512; + nor->params->page_size = page_size; + nor->mtd.writebufsize = page_size; + nor->mtd.size = 8 * page_size * nor->info->n_sectors; + nor->mtd.erasesize = 8 * page_size; } else { /* Flash in Default addressing mode */ nor->params->convert_addr = s3an_convert_addr; diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index f67457748ed8..fc90fce26e33 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -371,7 +371,6 @@ struct spi_nor_flash_parameter; * @bouncebuf_size: size of the bounce buffer * @info: SPI NOR part JEDEC MFR ID and other info * @manufacturer: SPI NOR manufacturer - * @page_size: the page size of the SPI NOR * @addr_width: number of address bytes * @erase_opcode: the opcode for erasing a sector * @read_opcode: the read opcode @@ -401,7 +400,6 @@ struct spi_nor { size_t bouncebuf_size; const struct flash_info *info; const struct spi_nor_manufacturer *manufacturer; - u32 page_size; u8 addr_width; u8 erase_opcode; u8 read_opcode; -- cgit v1.2.3-59-g8ed1b From 1e2c3ef0496e72ba9001da5fd1b7ed56ccb30597 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Mon, 4 Oct 2021 16:11:52 +0200 Subject: tee: export teedev_open() and teedev_close_context() Exports the two functions teedev_open() and teedev_close_context() in order to make it easier to create a driver internal struct tee_context. Reviewed-by: Sumit Garg Signed-off-by: Jens Wiklander --- drivers/tee/tee_core.c | 6 ++++-- include/linux/tee_drv.h | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 85102d12d716..3fc426dad2df 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -43,7 +43,7 @@ static DEFINE_SPINLOCK(driver_lock); static struct class *tee_class; static dev_t tee_devt; -static struct tee_context *teedev_open(struct tee_device *teedev) +struct tee_context *teedev_open(struct tee_device *teedev) { int rc; struct tee_context *ctx; @@ -70,6 +70,7 @@ err: return ERR_PTR(rc); } +EXPORT_SYMBOL_GPL(teedev_open); void teedev_ctx_get(struct tee_context *ctx) { @@ -96,13 +97,14 @@ void teedev_ctx_put(struct tee_context *ctx) kref_put(&ctx->refcount, teedev_ctx_release); } -static void teedev_close_context(struct tee_context *ctx) +void teedev_close_context(struct tee_context *ctx) { struct tee_device *teedev = ctx->teedev; teedev_ctx_put(ctx); tee_device_put(teedev); } +EXPORT_SYMBOL_GPL(teedev_close_context); static int tee_open(struct inode *inode, struct file *filp) { diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h index a1f03461369b..468a7d83dc6c 100644 --- a/include/linux/tee_drv.h +++ b/include/linux/tee_drv.h @@ -587,4 +587,18 @@ struct tee_client_driver { #define to_tee_client_driver(d) \ container_of(d, struct tee_client_driver, driver) +/** + * teedev_open() - Open a struct tee_device + * @teedev: Device to open + * + * @return a pointer to struct tee_context on success or an ERR_PTR on failure. + */ +struct tee_context *teedev_open(struct tee_device *teedev); + +/** + * teedev_close_context() - closes a struct tee_context + * @ctx: The struct tee_context to close + */ +void teedev_close_context(struct tee_context *ctx); + #endif /*__TEE_DRV_H*/ -- cgit v1.2.3-59-g8ed1b From 49c39ec4670a8f045729e3717af2e1a74caf89a5 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 13 Sep 2021 14:21:25 +0200 Subject: dma-buf: nuke dma_resv_get_excl_unlocked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heureka, that's finally not used any more. Signed-off-by: Christian König Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20210917123513.1106-27-christian.koenig@amd.com --- include/linux/dma-resv.h | 26 -------------------------- 1 file changed, 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dma-resv.h b/include/linux/dma-resv.h index 09c6063b199a..eebf04325b34 100644 --- a/include/linux/dma-resv.h +++ b/include/linux/dma-resv.h @@ -440,32 +440,6 @@ dma_resv_excl_fence(struct dma_resv *obj) return rcu_dereference_check(obj->fence_excl, dma_resv_held(obj)); } -/** - * dma_resv_get_excl_unlocked - get the reservation object's - * exclusive fence, without lock held. - * @obj: the reservation object - * - * If there is an exclusive fence, this atomically increments it's - * reference count and returns it. - * - * RETURNS - * The exclusive fence or NULL if none - */ -static inline struct dma_fence * -dma_resv_get_excl_unlocked(struct dma_resv *obj) -{ - struct dma_fence *fence; - - if (!rcu_access_pointer(obj->fence_excl)) - return NULL; - - rcu_read_lock(); - fence = dma_fence_get_rcu_safe(&obj->fence_excl); - rcu_read_unlock(); - - return fence; -} - /** * dma_resv_shared_list - get the reservation object's shared fence list * @obj: the reservation object -- cgit v1.2.3-59-g8ed1b From 2fb75e1b642f49253d8848c9e47e8942f5366221 Mon Sep 17 00:00:00 2001 From: Liu Xinpeng Date: Mon, 25 Oct 2021 11:46:26 +0800 Subject: psi: Add a missing SPDX license header Add the missing SPDX license header to include/linux/psi.h include/linux/psi_types.h kernel/sched/psi.c Signed-off-by: Liu Xinpeng Signed-off-by: Peter Zijlstra (Intel) Acked-by: Johannes Weiner Link: https://lore.kernel.org/r/1635133586-84611-2-git-send-email-liuxp11@chinatelecom.cn --- include/linux/psi.h | 1 + include/linux/psi_types.h | 1 + kernel/sched/psi.c | 1 + 3 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/psi.h b/include/linux/psi.h index 65eb1476ac70..a70ca833c6d7 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_PSI_H #define _LINUX_PSI_H diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index 0a23300d49af..bf50068d5d4b 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_PSI_TYPES_H #define _LINUX_PSI_TYPES_H diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 526af84ab852..3397fa001157 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Pressure stall information for CPU, memory and IO * -- cgit v1.2.3-59-g8ed1b From 4feee7d12603deca8775f9f9ae5e121093837444 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Mon, 18 Oct 2021 13:34:28 -0700 Subject: sched/core: Forced idle accounting Adds accounting for "forced idle" time, which is time where a cookie'd task forces its SMT sibling to idle, despite the presence of runnable tasks. Forced idle time is one means to measure the cost of enabling core scheduling (ie. the capacity lost due to the need to force idle). Forced idle time is attributed to the thread responsible for causing the forced idle. A few details: - Forced idle time is displayed via /proc/PID/sched. It also requires that schedstats is enabled. - Forced idle is only accounted when a sibling hyperthread is held idle despite the presence of runnable tasks. No time is charged if a sibling is idle but has no runnable tasks. - Tasks with 0 cookie are never charged forced idle. - For SMT > 2, we scale the amount of forced idle charged based on the number of forced idle siblings. Additionally, we split the time up and evenly charge it to all running tasks, as each is equally responsible for the forced idle. Signed-off-by: Josh Don Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20211018203428.2025792-1-joshdon@google.com --- include/linux/sched.h | 4 +++ kernel/sched/core.c | 82 +++++++++++++++++++++++++++++++++++------------ kernel/sched/core_sched.c | 66 +++++++++++++++++++++++++++++++++++++- kernel/sched/debug.c | 4 +++ kernel/sched/fair.c | 2 +- kernel/sched/sched.h | 32 ++++++++++++++++-- 6 files changed, 166 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 78c351e35fec..d2e261adb8ea 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -523,7 +523,11 @@ struct sched_statistics { u64 nr_wakeups_affine_attempts; u64 nr_wakeups_passive; u64 nr_wakeups_idle; + +#ifdef CONFIG_SCHED_CORE + u64 core_forceidle_sum; #endif +#endif /* CONFIG_SCHEDSTATS */ } ____cacheline_aligned; struct sched_entity { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3c9b0fda64ac..beaa8be6241e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -144,7 +144,7 @@ static inline bool __sched_core_less(struct task_struct *a, struct task_struct * return false; /* flip prio, so high prio is leftmost */ - if (prio_less(b, a, task_rq(a)->core->core_forceidle)) + if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) return true; return false; @@ -181,15 +181,23 @@ void sched_core_enqueue(struct rq *rq, struct task_struct *p) rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); } -void sched_core_dequeue(struct rq *rq, struct task_struct *p) +void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { rq->core->core_task_seq++; - if (!sched_core_enqueued(p)) - return; + if (sched_core_enqueued(p)) { + rb_erase(&p->core_node, &rq->core_tree); + RB_CLEAR_NODE(&p->core_node); + } - rb_erase(&p->core_node, &rq->core_tree); - RB_CLEAR_NODE(&p->core_node); + /* + * Migrating the last task off the cpu, with the cpu in forced idle + * state. Reschedule to create an accounting edge for forced idle, + * and re-examine whether the core is still in forced idle state. + */ + if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && + rq->core->core_forceidle_count && rq->curr == rq->idle) + resched_curr(rq); } /* @@ -280,6 +288,8 @@ static void __sched_core_flip(bool enabled) for_each_cpu(t, smt_mask) cpu_rq(t)->core_enabled = enabled; + cpu_rq(cpu)->core->core_forceidle_start = 0; + sched_core_unlock(cpu, &flags); cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); @@ -364,7 +374,8 @@ void sched_core_put(void) #else /* !CONFIG_SCHED_CORE */ static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } -static inline void sched_core_dequeue(struct rq *rq, struct task_struct *p) { } +static inline void +sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } #endif /* CONFIG_SCHED_CORE */ @@ -2005,7 +2016,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) { if (sched_core_enabled(rq)) - sched_core_dequeue(rq, p); + sched_core_dequeue(rq, p, flags); if (!(flags & DEQUEUE_NOCLOCK)) update_rq_clock(rq); @@ -5244,6 +5255,7 @@ void scheduler_tick(void) if (sched_feat(LATENCY_WARN)) resched_latency = cpu_resched_latency(rq); calc_global_load_tick(rq); + sched_core_tick(rq); rq_unlock(rq, &rf); @@ -5656,6 +5668,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) struct task_struct *next, *p, *max = NULL; const struct cpumask *smt_mask; bool fi_before = false; + bool core_clock_updated = (rq == rq->core); unsigned long cookie; int i, cpu, occ = 0; struct rq *rq_i; @@ -5708,10 +5721,18 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) /* reset state */ rq->core->core_cookie = 0UL; - if (rq->core->core_forceidle) { + if (rq->core->core_forceidle_count) { + if (!core_clock_updated) { + update_rq_clock(rq->core); + core_clock_updated = true; + } + sched_core_account_forceidle(rq); + /* reset after accounting force idle */ + rq->core->core_forceidle_start = 0; + rq->core->core_forceidle_count = 0; + rq->core->core_forceidle_occupation = 0; need_sync = true; fi_before = true; - rq->core->core_forceidle = false; } /* @@ -5753,7 +5774,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) for_each_cpu_wrap(i, smt_mask, cpu) { rq_i = cpu_rq(i); - if (i != cpu) + /* + * Current cpu always has its clock updated on entrance to + * pick_next_task(). If the current cpu is not the core, + * the core may also have been updated above. + */ + if (i != cpu && (rq_i != rq->core || !core_clock_updated)) update_rq_clock(rq_i); p = rq_i->core_pick = pick_task(rq_i); @@ -5783,7 +5809,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) if (p == rq_i->idle) { if (rq_i->nr_running) { - rq->core->core_forceidle = true; + rq->core->core_forceidle_count++; if (!fi_before) rq->core->core_forceidle_seq++; } @@ -5792,6 +5818,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } } + if (schedstat_enabled() && rq->core->core_forceidle_count) { + if (cookie) + rq->core->core_forceidle_start = rq_clock(rq->core); + rq->core->core_forceidle_occupation = occ; + } + rq->core->core_pick_seq = rq->core->core_task_seq; next = rq->core_pick; rq->core_sched_seq = rq->core->core_pick_seq; @@ -5828,8 +5860,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) * 1 0 1 * 1 1 0 */ - if (!(fi_before && rq->core->core_forceidle)) - task_vruntime_update(rq_i, rq_i->core_pick, rq->core->core_forceidle); + if (!(fi_before && rq->core->core_forceidle_count)) + task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); rq_i->core_pick->core_occupation = occ; @@ -6033,11 +6065,19 @@ static void sched_core_cpu_deactivate(unsigned int cpu) goto unlock; /* copy the shared state to the new leader */ - core_rq->core_task_seq = rq->core_task_seq; - core_rq->core_pick_seq = rq->core_pick_seq; - core_rq->core_cookie = rq->core_cookie; - core_rq->core_forceidle = rq->core_forceidle; - core_rq->core_forceidle_seq = rq->core_forceidle_seq; + core_rq->core_task_seq = rq->core_task_seq; + core_rq->core_pick_seq = rq->core_pick_seq; + core_rq->core_cookie = rq->core_cookie; + core_rq->core_forceidle_count = rq->core_forceidle_count; + core_rq->core_forceidle_seq = rq->core_forceidle_seq; + core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; + + /* + * Accounting edge for forced idle is handled in pick_next_task(). + * Don't need another one here, since the hotplug thread shouldn't + * have a cookie. + */ + core_rq->core_forceidle_start = 0; /* install new leader */ for_each_cpu(t, smt_mask) { @@ -9413,7 +9453,9 @@ void __init sched_init(void) rq->core_pick = NULL; rq->core_enabled = 0; rq->core_tree = RB_ROOT; - rq->core_forceidle = false; + rq->core_forceidle_count = 0; + rq->core_forceidle_occupation = 0; + rq->core_forceidle_start = 0; rq->core_cookie = 0UL; #endif diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index 517f72b008f5..1fb45672ec85 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -73,7 +73,7 @@ static unsigned long sched_core_update_cookie(struct task_struct *p, enqueued = sched_core_enqueued(p); if (enqueued) - sched_core_dequeue(rq, p); + sched_core_dequeue(rq, p, DEQUEUE_SAVE); old_cookie = p->core_cookie; p->core_cookie = cookie; @@ -85,6 +85,10 @@ static unsigned long sched_core_update_cookie(struct task_struct *p, * If task is currently running, it may not be compatible anymore after * the cookie change, so enter the scheduler on its CPU to schedule it * away. + * + * Note that it is possible that as a result of this cookie change, the + * core has now entered/left forced idle state. Defer accounting to the + * next scheduling edge, rather than always forcing a reschedule here. */ if (task_running(rq, p)) resched_curr(rq); @@ -232,3 +236,63 @@ out: return err; } +#ifdef CONFIG_SCHEDSTATS + +/* REQUIRES: rq->core's clock recently updated. */ +void __sched_core_account_forceidle(struct rq *rq) +{ + const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); + u64 delta, now = rq_clock(rq->core); + struct rq *rq_i; + struct task_struct *p; + int i; + + lockdep_assert_rq_held(rq); + + WARN_ON_ONCE(!rq->core->core_forceidle_count); + + if (rq->core->core_forceidle_start == 0) + return; + + delta = now - rq->core->core_forceidle_start; + if (unlikely((s64)delta <= 0)) + return; + + rq->core->core_forceidle_start = now; + + if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) { + /* can't be forced idle without a running task */ + } else if (rq->core->core_forceidle_count > 1 || + rq->core->core_forceidle_occupation > 1) { + /* + * For larger SMT configurations, we need to scale the charged + * forced idle amount since there can be more than one forced + * idle sibling and more than one running cookied task. + */ + delta *= rq->core->core_forceidle_count; + delta = div_u64(delta, rq->core->core_forceidle_occupation); + } + + for_each_cpu(i, smt_mask) { + rq_i = cpu_rq(i); + p = rq_i->core_pick ?: rq_i->curr; + + if (!p->core_cookie) + continue; + + __schedstat_add(p->stats.core_forceidle_sum, delta); + } +} + +void __sched_core_tick(struct rq *rq) +{ + if (!rq->core->core_forceidle_count) + return; + + if (rq != rq->core) + update_rq_clock(rq->core); + + __sched_core_account_forceidle(rq); +} + +#endif /* CONFIG_SCHEDSTATS */ diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 7dcbaa31c5d9..aa29211de1bf 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -1023,6 +1023,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, __PN(avg_atom); __PN(avg_per_cpu); + +#ifdef CONFIG_SCHED_CORE + PN_SCHEDSTAT(core_forceidle_sum); +#endif } __P(nr_switches); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6e476f6d9435..884f29d07963 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11068,7 +11068,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle && rq->cfs.nr_running == 1 && + if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0e66749486e7..eb971151e7e4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1111,8 +1111,10 @@ struct rq { unsigned int core_task_seq; unsigned int core_pick_seq; unsigned long core_cookie; - unsigned char core_forceidle; + unsigned int core_forceidle_count; unsigned int core_forceidle_seq; + unsigned int core_forceidle_occupation; + u64 core_forceidle_start; #endif }; @@ -1253,7 +1255,7 @@ static inline bool sched_core_enqueued(struct task_struct *p) } extern void sched_core_enqueue(struct rq *rq, struct task_struct *p); -extern void sched_core_dequeue(struct rq *rq, struct task_struct *p); +extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags); extern void sched_core_get(void); extern void sched_core_put(void); @@ -1854,6 +1856,32 @@ static inline void flush_smp_call_function_from_idle(void) { } #include "stats.h" #include "autogroup.h" +#if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) + +extern void __sched_core_account_forceidle(struct rq *rq); + +static inline void sched_core_account_forceidle(struct rq *rq) +{ + if (schedstat_enabled()) + __sched_core_account_forceidle(rq); +} + +extern void __sched_core_tick(struct rq *rq); + +static inline void sched_core_tick(struct rq *rq) +{ + if (sched_core_enabled(rq) && schedstat_enabled()) + __sched_core_tick(rq); +} + +#else + +static inline void sched_core_account_forceidle(struct rq *rq) {} + +static inline void sched_core_tick(struct rq *rq) {} + +#endif /* CONFIG_SCHED_CORE && CONFIG_SCHEDSTATS */ + #ifdef CONFIG_CGROUP_SCHED /* -- cgit v1.2.3-59-g8ed1b From cb0e52b7748737b2cf6481fdd9b920ce7e1ebbdf Mon Sep 17 00:00:00 2001 From: Brian Chen Date: Wed, 10 Nov 2021 21:33:12 +0000 Subject: psi: Fix PSI_MEM_FULL state when tasks are in memstall and doing reclaim We've noticed cases where tasks in a cgroup are stalled on memory but there is little memory FULL pressure since tasks stay on the runqueue in reclaim. A simple example involves a single threaded program that keeps leaking and touching large amounts of memory. It runs in a cgroup with swap enabled, memory.high set at 10M and cpu.max ratio set at 5%. Though there is significant CPU pressure and memory SOME, there is barely any memory FULL since the task enters reclaim and stays on the runqueue. However, this memory-bound task is effectively stalled on memory and we expect memory FULL to match memory SOME in this scenario. The code is confused about memstall && running, thinking there is a stalled task and a productive task when there's only one task: a reclaimer that's counted as both. To fix this, we redefine the condition for PSI_MEM_FULL to check that all running tasks are in an active memstall instead of checking that there are no running tasks. case PSI_MEM_FULL: - return unlikely(tasks[NR_MEMSTALL] && !tasks[NR_RUNNING]); + return unlikely(tasks[NR_MEMSTALL] && + tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]); This will capture reclaimers. It will also capture tasks that called psi_memstall_enter() and are about to sleep, but this should be negligible noise. Signed-off-by: Brian Chen Signed-off-by: Peter Zijlstra (Intel) Acked-by: Johannes Weiner Link: https://lore.kernel.org/r/20211110213312.310243-1-brianchen118@gmail.com --- include/linux/psi_types.h | 13 ++++++++++++- kernel/sched/psi.c | 45 ++++++++++++++++++++++++++++----------------- kernel/sched/stats.h | 5 ++++- 3 files changed, 44 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index bf50068d5d4b..516c0fe836fd 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -22,7 +22,17 @@ enum psi_task_count { * don't have to special case any state tracking for it. */ NR_ONCPU, - NR_PSI_TASK_COUNTS = 4, + /* + * For IO and CPU stalls the presence of running/oncpu tasks + * in the domain means a partial rather than a full stall. + * For memory it's not so simple because of page reclaimers: + * they are running/oncpu while representing a stall. To tell + * whether a domain has productivity left or not, we need to + * distinguish between regular running (i.e. productive) + * threads and memstall ones. + */ + NR_MEMSTALL_RUNNING, + NR_PSI_TASK_COUNTS = 5, }; /* Task state bitmasks */ @@ -30,6 +40,7 @@ enum psi_task_count { #define TSK_MEMSTALL (1 << NR_MEMSTALL) #define TSK_RUNNING (1 << NR_RUNNING) #define TSK_ONCPU (1 << NR_ONCPU) +#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING) /* Resources that workloads could be stalled on */ enum psi_res { diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 3397fa001157..a679613a7cb7 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -35,13 +35,19 @@ * delayed on that resource such that nobody is advancing and the CPU * goes idle. This leaves both workload and CPU unproductive. * - * Naturally, the FULL state doesn't exist for the CPU resource at the - * system level, but exist at the cgroup level, means all non-idle tasks - * in a cgroup are delayed on the CPU resource which used by others outside - * of the cgroup or throttled by the cgroup cpu.max configuration. - * * SOME = nr_delayed_tasks != 0 - * FULL = nr_delayed_tasks != 0 && nr_running_tasks == 0 + * FULL = nr_delayed_tasks != 0 && nr_productive_tasks == 0 + * + * What it means for a task to be productive is defined differently + * for each resource. For IO, productive means a running task. For + * memory, productive means a running task that isn't a reclaimer. For + * CPU, productive means an oncpu task. + * + * Naturally, the FULL state doesn't exist for the CPU resource at the + * system level, but exist at the cgroup level. At the cgroup level, + * FULL means all non-idle tasks in the cgroup are delayed on the CPU + * resource which is being used by others outside of the cgroup or + * throttled by the cgroup cpu.max configuration. * * The percentage of wallclock time spent in those compound stall * states gives pressure numbers between 0 and 100 for each resource, @@ -82,13 +88,13 @@ * * threads = min(nr_nonidle_tasks, nr_cpus) * SOME = min(nr_delayed_tasks / threads, 1) - * FULL = (threads - min(nr_running_tasks, threads)) / threads + * FULL = (threads - min(nr_productive_tasks, threads)) / threads * * For the 257 number crunchers on 256 CPUs, this yields: * * threads = min(257, 256) * SOME = min(1 / 256, 1) = 0.4% - * FULL = (256 - min(257, 256)) / 256 = 0% + * FULL = (256 - min(256, 256)) / 256 = 0% * * For the 1 out of 4 memory-delayed tasks, this yields: * @@ -113,7 +119,7 @@ * For each runqueue, we track: * * tSOME[cpu] = time(nr_delayed_tasks[cpu] != 0) - * tFULL[cpu] = time(nr_delayed_tasks[cpu] && !nr_running_tasks[cpu]) + * tFULL[cpu] = time(nr_delayed_tasks[cpu] && !nr_productive_tasks[cpu]) * tNONIDLE[cpu] = time(nr_nonidle_tasks[cpu] != 0) * * and then periodically aggregate: @@ -234,7 +240,8 @@ static bool test_state(unsigned int *tasks, enum psi_states state) case PSI_MEM_SOME: return unlikely(tasks[NR_MEMSTALL]); case PSI_MEM_FULL: - return unlikely(tasks[NR_MEMSTALL] && !tasks[NR_RUNNING]); + return unlikely(tasks[NR_MEMSTALL] && + tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]); case PSI_CPU_SOME: return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]); case PSI_CPU_FULL: @@ -711,10 +718,11 @@ static void psi_group_change(struct psi_group *group, int cpu, if (groupc->tasks[t]) { groupc->tasks[t]--; } else if (!psi_bug) { - printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n", + printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n", cpu, t, groupc->tasks[0], groupc->tasks[1], groupc->tasks[2], - groupc->tasks[3], clear, set); + groupc->tasks[3], groupc->tasks[4], + clear, set); psi_bug = 1; } } @@ -854,12 +862,15 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, int clear = TSK_ONCPU, set = 0; /* - * When we're going to sleep, psi_dequeue() lets us handle - * TSK_RUNNING and TSK_IOWAIT here, where we can combine it - * with TSK_ONCPU and save walking common ancestors twice. + * When we're going to sleep, psi_dequeue() lets us + * handle TSK_RUNNING, TSK_MEMSTALL_RUNNING and + * TSK_IOWAIT here, where we can combine it with + * TSK_ONCPU and save walking common ancestors twice. */ if (sleep) { clear |= TSK_RUNNING; + if (prev->in_memstall) + clear |= TSK_MEMSTALL_RUNNING; if (prev->in_iowait) set |= TSK_IOWAIT; } @@ -908,7 +919,7 @@ void psi_memstall_enter(unsigned long *flags) rq = this_rq_lock_irq(&rf); current->in_memstall = 1; - psi_task_change(current, 0, TSK_MEMSTALL); + psi_task_change(current, 0, TSK_MEMSTALL | TSK_MEMSTALL_RUNNING); rq_unlock_irq(rq, &rf); } @@ -937,7 +948,7 @@ void psi_memstall_leave(unsigned long *flags) rq = this_rq_lock_irq(&rf); current->in_memstall = 0; - psi_task_change(current, TSK_MEMSTALL, 0); + psi_task_change(current, TSK_MEMSTALL | TSK_MEMSTALL_RUNNING, 0); rq_unlock_irq(rq, &rf); } diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index cfb0893a83d4..3a3c826dd83a 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -118,6 +118,9 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup) if (static_branch_likely(&psi_disabled)) return; + if (p->in_memstall) + set |= TSK_MEMSTALL_RUNNING; + if (!wakeup || p->sched_psi_wake_requeue) { if (p->in_memstall) set |= TSK_MEMSTALL; @@ -148,7 +151,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep) return; if (p->in_memstall) - clear |= TSK_MEMSTALL; + clear |= (TSK_MEMSTALL | TSK_MEMSTALL_RUNNING); psi_task_change(p, clear, 0); } -- cgit v1.2.3-59-g8ed1b From ff083a2d972f56bebfd82409ca62e5dfce950961 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:22 +0000 Subject: perf: Protect perf_guest_cbs with RCU Protect perf_guest_cbs with RCU to fix multiple possible errors. Luckily, all paths that read perf_guest_cbs already require RCU protection, e.g. to protect the callback chains, so only the direct perf_guest_cbs touchpoints need to be modified. Bug #1 is a simple lack of WRITE_ONCE/READ_ONCE behavior to ensure perf_guest_cbs isn't reloaded between a !NULL check and a dereference. Fixed via the READ_ONCE() in rcu_dereference(). Bug #2 is that on weakly-ordered architectures, updates to the callbacks themselves are not guaranteed to be visible before the pointer is made visible to readers. Fixed by the smp_store_release() in rcu_assign_pointer() when the new pointer is non-NULL. Bug #3 is that, because the callbacks are global, it's possible for readers to run in parallel with an unregisters, and thus a module implementing the callbacks can be unloaded while readers are in flight, resulting in a use-after-free. Fixed by a synchronize_rcu() call when unregistering callbacks. Bug #1 escaped notice because it's extremely unlikely a compiler will reload perf_guest_cbs in this sequence. perf_guest_cbs does get reloaded for future derefs, e.g. for ->is_user_mode(), but the ->is_in_guest() guard all but guarantees the consumer will win the race, e.g. to nullify perf_guest_cbs, KVM has to completely exit the guest and teardown down all VMs before KVM start its module unload / unregister sequence. This also makes it all but impossible to encounter bug #3. Bug #2 has not been a problem because all architectures that register callbacks are strongly ordered and/or have a static set of callbacks. But with help, unloading kvm_intel can trigger bug #1 e.g. wrapping perf_guest_cbs with READ_ONCE in perf_misc_flags() while spamming kvm_intel module load/unload leads to: BUG: kernel NULL pointer dereference, address: 0000000000000000 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP CPU: 6 PID: 1825 Comm: stress Not tainted 5.14.0-rc2+ #459 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:perf_misc_flags+0x1c/0x70 Call Trace: perf_prepare_sample+0x53/0x6b0 perf_event_output_forward+0x67/0x160 __perf_event_overflow+0x52/0xf0 handle_pmi_common+0x207/0x300 intel_pmu_handle_irq+0xcf/0x410 perf_event_nmi_handler+0x28/0x50 nmi_handle+0xc7/0x260 default_do_nmi+0x6b/0x170 exc_nmi+0x103/0x130 asm_exc_nmi+0x76/0xbf Fixes: 39447b386c84 ("perf: Enhance perf to allow for guest statistic collection from host") Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20211111020738.2512932-2-seanjc@google.com --- arch/arm/kernel/perf_callchain.c | 17 +++++++++++------ arch/arm64/kernel/perf_callchain.c | 18 ++++++++++++------ arch/csky/kernel/perf_callchain.c | 6 ++++-- arch/nds32/kernel/perf_event_cpu.c | 17 +++++++++++------ arch/riscv/kernel/perf_callchain.c | 7 +++++-- arch/x86/events/core.c | 17 +++++++++++------ arch/x86/events/intel/core.c | 9 ++++++--- include/linux/perf_event.h | 13 ++++++++++++- kernel/events/core.c | 13 ++++++++++--- 9 files changed, 82 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c index 3b69a76d341e..1626dfc6f6ce 100644 --- a/arch/arm/kernel/perf_callchain.c +++ b/arch/arm/kernel/perf_callchain.c @@ -62,9 +62,10 @@ user_backtrace(struct frame_tail __user *tail, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct frame_tail __user *tail; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -98,9 +99,10 @@ callchain_trace(struct stackframe *fr, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe fr; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -111,18 +113,21 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re unsigned long perf_instruction_pointer(struct pt_regs *regs) { - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return perf_guest_cbs->get_guest_ip(); + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + if (guest_cbs && guest_cbs->is_in_guest()) + return guest_cbs->get_guest_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); int misc = 0; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - if (perf_guest_cbs->is_user_mode()) + if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs->is_user_mode()) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 4a72c2727309..86d9f2013172 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -102,7 +102,9 @@ compat_user_backtrace(struct compat_frame_tail __user *tail, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -147,9 +149,10 @@ static bool callchain_trace(void *data, unsigned long pc) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe frame; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -160,18 +163,21 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, unsigned long perf_instruction_pointer(struct pt_regs *regs) { - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return perf_guest_cbs->get_guest_ip(); + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + if (guest_cbs && guest_cbs->is_in_guest()) + return guest_cbs->get_guest_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); int misc = 0; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - if (perf_guest_cbs->is_user_mode()) + if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs->is_user_mode()) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c index ab55e98ee8f6..35318a635a5f 100644 --- a/arch/csky/kernel/perf_callchain.c +++ b/arch/csky/kernel/perf_callchain.c @@ -86,10 +86,11 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); unsigned long fp = 0; /* C-SKY does not support virtualization. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) + if (guest_cbs && guest_cbs->is_in_guest()) return; fp = regs->regs[4]; @@ -110,10 +111,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe fr; /* C-SKY does not support virtualization. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { pr_warn("C-SKY does not support perf in guest mode!"); return; } diff --git a/arch/nds32/kernel/perf_event_cpu.c b/arch/nds32/kernel/perf_event_cpu.c index 0ce6f9f307e6..f38791960781 100644 --- a/arch/nds32/kernel/perf_event_cpu.c +++ b/arch/nds32/kernel/perf_event_cpu.c @@ -1363,6 +1363,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); unsigned long fp = 0; unsigned long gp = 0; unsigned long lp = 0; @@ -1371,7 +1372,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, leaf_fp = 0; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -1479,9 +1480,10 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe fr; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* We don't support guest os callchain now */ return; } @@ -1493,20 +1495,23 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, unsigned long perf_instruction_pointer(struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + /* However, NDS32 does not support virtualization */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return perf_guest_cbs->get_guest_ip(); + if (guest_cbs && guest_cbs->is_in_guest()) + return guest_cbs->get_guest_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); int misc = 0; /* However, NDS32 does not support virtualization */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - if (perf_guest_cbs->is_user_mode()) + if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs->is_user_mode()) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 0bb1854dce83..8ecfc4c128bc 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -56,10 +56,11 @@ static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); unsigned long fp = 0; /* RISC-V does not support perf in guest mode. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) + if (guest_cbs && guest_cbs->is_in_guest()) return; fp = regs->s0; @@ -78,8 +79,10 @@ static bool fill_callchain(void *entry, unsigned long pc) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + /* RISC-V does not support perf in guest mode. */ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { pr_warn("RISC-V does not support perf in guest mode!"); return; } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 38b2c779146f..32cec290d3ad 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2768,10 +2768,11 @@ static bool perf_hw_regs(struct pt_regs *regs) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct unwind_state state; unsigned long addr; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2871,10 +2872,11 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stack_frame frame; const struct stack_frame __user *fp; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->is_in_guest()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2951,18 +2953,21 @@ static unsigned long code_segment_base(struct pt_regs *regs) unsigned long perf_instruction_pointer(struct pt_regs *regs) { - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return perf_guest_cbs->get_guest_ip(); + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + if (guest_cbs && guest_cbs->is_in_guest()) + return guest_cbs->get_guest_ip(); return regs->ip + code_segment_base(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); int misc = 0; - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - if (perf_guest_cbs->is_user_mode()) + if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs->is_user_mode()) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 42cf01ecdd13..2258e02ca350 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2837,6 +2837,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) { struct perf_sample_data data; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct perf_guest_info_callbacks *guest_cbs; int bit; int handled = 0; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); @@ -2903,9 +2904,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) { handled++; - if (unlikely(perf_guest_cbs && perf_guest_cbs->is_in_guest() && - perf_guest_cbs->handle_intel_pt_intr)) - perf_guest_cbs->handle_intel_pt_intr(); + + guest_cbs = perf_get_guest_cbs(); + if (unlikely(guest_cbs && guest_cbs->is_in_guest() && + guest_cbs->handle_intel_pt_intr)) + guest_cbs->handle_intel_pt_intr(); else intel_pt_interrupt(); } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 0dcfd265beed..318c489b735b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1240,7 +1240,18 @@ extern void perf_event_bpf_event(struct bpf_prog *prog, enum perf_bpf_event_type type, u16 flags); -extern struct perf_guest_info_callbacks *perf_guest_cbs; +extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs; +static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) +{ + /* + * Callbacks are RCU-protected and must be READ_ONCE to avoid reloading + * the callbacks between a !NULL check and dereferences, to ensure + * pending stores/changes to the callback pointers are visible before a + * non-NULL perf_guest_cbs is visible to readers, and to prevent a + * module from unloading callbacks while readers are active. + */ + return rcu_dereference(perf_guest_cbs); +} extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); diff --git a/kernel/events/core.c b/kernel/events/core.c index 523106a506ee..c552e1bfcaea 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6526,18 +6526,25 @@ static void perf_pending_event(struct irq_work *entry) * Later on, we might change it to a list if there is * another virtualization implementation supporting the callbacks. */ -struct perf_guest_info_callbacks *perf_guest_cbs; +struct perf_guest_info_callbacks __rcu *perf_guest_cbs; int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) { - perf_guest_cbs = cbs; + if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs))) + return -EBUSY; + + rcu_assign_pointer(perf_guest_cbs, cbs); return 0; } EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) { - perf_guest_cbs = NULL; + if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs)) + return -EINVAL; + + rcu_assign_pointer(perf_guest_cbs, NULL); + synchronize_rcu(); return 0; } EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); -- cgit v1.2.3-59-g8ed1b From 2934e3d09350c1a7ca2433fbeabfcd831e48a575 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:25 +0000 Subject: perf: Stop pretending that perf can handle multiple guest callbacks Drop the 'int' return value from the perf (un)register callbacks helpers and stop pretending perf can support multiple callbacks. The 'int' returns are not future proofing anything as none of the callers take action on an error. It's also not obvious that there will ever be co-tenant hypervisors, and if there are, that allowing multiple callbacks to be registered is desirable or even correct. Opportunistically rename callbacks=>cbs in the affected declarations to match their definitions. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-5-seanjc@google.com --- arch/arm64/include/asm/kvm_host.h | 4 ++-- arch/arm64/kvm/perf.c | 8 ++++---- include/linux/perf_event.h | 12 ++++++------ kernel/events/core.c | 15 ++++----------- 4 files changed, 16 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 2a5f7f38006f..f680f303ba7c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -675,8 +675,8 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); int kvm_handle_mmio_return(struct kvm_vcpu *vcpu); int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); -int kvm_perf_init(void); -int kvm_perf_teardown(void); +void kvm_perf_init(void); +void kvm_perf_teardown(void); long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/perf.c b/arch/arm64/kvm/perf.c index c84fe24b2ea1..a0d660cf889e 100644 --- a/arch/arm64/kvm/perf.c +++ b/arch/arm64/kvm/perf.c @@ -48,12 +48,12 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = { .get_guest_ip = kvm_get_guest_ip, }; -int kvm_perf_init(void) +void kvm_perf_init(void) { - return perf_register_guest_info_callbacks(&kvm_guest_cbs); + perf_register_guest_info_callbacks(&kvm_guest_cbs); } -int kvm_perf_teardown(void) +void kvm_perf_teardown(void) { - return perf_unregister_guest_info_callbacks(&kvm_guest_cbs); + perf_unregister_guest_info_callbacks(&kvm_guest_cbs); } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 318c489b735b..98c204488496 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1252,8 +1252,8 @@ static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) */ return rcu_dereference(perf_guest_cbs); } -extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); -extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); +extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); +extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); extern void perf_event_exec(void); extern void perf_event_comm(struct task_struct *tsk, bool exec); @@ -1497,10 +1497,10 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { } static inline void perf_bp_event(struct perf_event *event, void *data) { } -static inline int perf_register_guest_info_callbacks -(struct perf_guest_info_callbacks *callbacks) { return 0; } -static inline int perf_unregister_guest_info_callbacks -(struct perf_guest_info_callbacks *callbacks) { return 0; } +static inline void perf_register_guest_info_callbacks +(struct perf_guest_info_callbacks *cbs) { } +static inline void perf_unregister_guest_info_callbacks +(struct perf_guest_info_callbacks *cbs) { } static inline void perf_event_mmap(struct vm_area_struct *vma) { } diff --git a/kernel/events/core.c b/kernel/events/core.c index c552e1bfcaea..17e5b20762c5 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6521,31 +6521,24 @@ static void perf_pending_event(struct irq_work *entry) perf_swevent_put_recursion_context(rctx); } -/* - * We assume there is only KVM supporting the callbacks. - * Later on, we might change it to a list if there is - * another virtualization implementation supporting the callbacks. - */ struct perf_guest_info_callbacks __rcu *perf_guest_cbs; -int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) +void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) { if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs))) - return -EBUSY; + return; rcu_assign_pointer(perf_guest_cbs, cbs); - return 0; } EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); -int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) +void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) { if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs)) - return -EINVAL; + return; rcu_assign_pointer(perf_guest_cbs, NULL); synchronize_rcu(); - return 0; } EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); -- cgit v1.2.3-59-g8ed1b From b9f5621c9547dd787900f005a9e1c3d5712de512 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Thu, 11 Nov 2021 02:07:27 +0000 Subject: perf/core: Rework guest callbacks to prepare for static_call support To prepare for using static_calls to optimize perf's guest callbacks, replace ->is_in_guest and ->is_user_mode with a new multiplexed hook ->state, tweak ->handle_intel_pt_intr to play nice with being called when there is no active guest, and drop "guest" from ->get_guest_ip. Return '0' from ->state and ->handle_intel_pt_intr to indicate "not in guest" so that DEFINE_STATIC_CALL_RET0 can be used to define the static calls, i.e. no callback == !guest. [sean: extracted from static_call patch, fixed get_ip() bug, wrote changelog] Suggested-by: Peter Zijlstra (Intel) Originally-by: Peter Zijlstra (Intel) Signed-off-by: Like Xu Signed-off-by: Zhu Lingshan Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Boris Ostrovsky Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-7-seanjc@google.com --- arch/arm64/kernel/perf_callchain.c | 13 +++++++------ arch/arm64/kvm/perf.c | 35 ++++++++++++++------------------- arch/x86/events/core.c | 13 +++++++------ arch/x86/events/intel/core.c | 5 +---- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/x86.c | 40 +++++++++++++++++++++----------------- arch/x86/xen/pmu.c | 32 +++++++++++++----------------- include/linux/perf_event.h | 10 ++++++---- 9 files changed, 73 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 86d9f2013172..274dc3e11b6d 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -104,7 +104,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, { struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->state()) { /* We don't support guest os callchain now */ return; } @@ -152,7 +152,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe frame; - if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->state()) { /* We don't support guest os callchain now */ return; } @@ -165,8 +165,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) { struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - if (guest_cbs && guest_cbs->is_in_guest()) - return guest_cbs->get_guest_ip(); + if (guest_cbs && guest_cbs->state()) + return guest_cbs->get_ip(); return instruction_pointer(regs); } @@ -174,10 +174,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) unsigned long perf_misc_flags(struct pt_regs *regs) { struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0; int misc = 0; - if (guest_cbs && guest_cbs->is_in_guest()) { - if (guest_cbs->is_user_mode()) + if (guest_state) { + if (guest_state & PERF_GUEST_USER) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/arm64/kvm/perf.c b/arch/arm64/kvm/perf.c index a0d660cf889e..dfa9bce8559e 100644 --- a/arch/arm64/kvm/perf.c +++ b/arch/arm64/kvm/perf.c @@ -13,39 +13,34 @@ DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available); -static int kvm_is_in_guest(void) +static unsigned int kvm_guest_state(void) { - return kvm_get_running_vcpu() != NULL; -} - -static int kvm_is_user_mode(void) -{ - struct kvm_vcpu *vcpu; + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + unsigned int state; - vcpu = kvm_get_running_vcpu(); + if (!vcpu) + return 0; - if (vcpu) - return !vcpu_mode_priv(vcpu); + state = PERF_GUEST_ACTIVE; + if (!vcpu_mode_priv(vcpu)) + state |= PERF_GUEST_USER; - return 0; + return state; } static unsigned long kvm_get_guest_ip(void) { - struct kvm_vcpu *vcpu; - - vcpu = kvm_get_running_vcpu(); + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); - if (vcpu) - return *vcpu_pc(vcpu); + if (WARN_ON_ONCE(!vcpu)) + return 0; - return 0; + return *vcpu_pc(vcpu); } static struct perf_guest_info_callbacks kvm_guest_cbs = { - .is_in_guest = kvm_is_in_guest, - .is_user_mode = kvm_is_user_mode, - .get_guest_ip = kvm_get_guest_ip, + .state = kvm_guest_state, + .get_ip = kvm_get_guest_ip, }; void kvm_perf_init(void) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 32cec290d3ad..e29312a1003a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2772,7 +2772,7 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re struct unwind_state state; unsigned long addr; - if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2876,7 +2876,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs struct stack_frame frame; const struct stack_frame __user *fp; - if (guest_cbs && guest_cbs->is_in_guest()) { + if (guest_cbs && guest_cbs->state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2955,8 +2955,8 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) { struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - if (guest_cbs && guest_cbs->is_in_guest()) - return guest_cbs->get_guest_ip(); + if (guest_cbs && guest_cbs->state()) + return guest_cbs->get_ip(); return regs->ip + code_segment_base(regs); } @@ -2964,10 +2964,11 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs) unsigned long perf_misc_flags(struct pt_regs *regs) { struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0; int misc = 0; - if (guest_cbs && guest_cbs->is_in_guest()) { - if (guest_cbs->is_user_mode()) + if (guest_state) { + if (guest_state & PERF_GUEST_USER) misc |= PERF_RECORD_MISC_GUEST_USER; else misc |= PERF_RECORD_MISC_GUEST_KERNEL; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2258e02ca350..7ff24d1ecdb7 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2906,10 +2906,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) handled++; guest_cbs = perf_get_guest_cbs(); - if (unlikely(guest_cbs && guest_cbs->is_in_guest() && - guest_cbs->handle_intel_pt_intr)) - guest_cbs->handle_intel_pt_intr(); - else + if (likely(!guest_cbs || !guest_cbs->handle_intel_pt_intr())) intel_pt_interrupt(); } diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 41e858df5795..fa1b1a209945 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1895,7 +1895,7 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu); -int kvm_is_in_guest(void); +unsigned int kvm_guest_state(void); void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 09873f6488f7..b2520b3e9e89 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -87,7 +87,7 @@ static void kvm_perf_overflow_intr(struct perf_event *perf_event, * woken up. So we should wake it, but this is impossible from * NMI context. Do it from irq work instead. */ - if (!kvm_is_in_guest()) + if (!kvm_guest_state()) irq_work_queue(&pmc_to_pmu(pmc)->irq_work); else kvm_make_request(KVM_REQ_PMI, pmc->vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 760c4e3a8326..2011a1cfb42d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8472,44 +8472,48 @@ static void kvm_timer_init(void) DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu); -int kvm_is_in_guest(void) +unsigned int kvm_guest_state(void) { - return __this_cpu_read(current_vcpu) != NULL; -} + struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu); + unsigned int state; -static int kvm_is_user_mode(void) -{ - int user_mode = 3; + if (!vcpu) + return 0; - if (__this_cpu_read(current_vcpu)) - user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu)); + state = PERF_GUEST_ACTIVE; + if (static_call(kvm_x86_get_cpl)(vcpu)) + state |= PERF_GUEST_USER; - return user_mode != 0; + return state; } -static unsigned long kvm_get_guest_ip(void) +static unsigned long kvm_guest_get_ip(void) { - unsigned long ip = 0; + struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu); - if (__this_cpu_read(current_vcpu)) - ip = kvm_rip_read(__this_cpu_read(current_vcpu)); + if (WARN_ON_ONCE(!vcpu)) + return 0; - return ip; + return kvm_rip_read(vcpu); } -static void kvm_handle_intel_pt_intr(void) +static unsigned int kvm_handle_intel_pt_intr(void) { struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu); + /* '0' on failure so that the !PT case can use a RET0 static call. */ + if (!vcpu) + return 0; + kvm_make_request(KVM_REQ_PMI, vcpu); __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, (unsigned long *)&vcpu->arch.pmu.global_status); + return 1; } static struct perf_guest_info_callbacks kvm_guest_cbs = { - .is_in_guest = kvm_is_in_guest, - .is_user_mode = kvm_is_user_mode, - .get_guest_ip = kvm_get_guest_ip, + .state = kvm_guest_state, + .get_ip = kvm_guest_get_ip, .handle_intel_pt_intr = NULL, }; diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index e13b0b49fcdf..89dd6b1708b0 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -413,34 +413,29 @@ int pmu_apic_update(uint32_t val) } /* perf callbacks */ -static int xen_is_in_guest(void) +static unsigned int xen_guest_state(void) { const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + unsigned int state = 0; if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); - return 0; + return state; } if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) - return 0; + return state; - return 1; -} - -static int xen_is_user_mode(void) -{ - const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + state |= PERF_GUEST_ACTIVE; - if (!xenpmu_data) { - pr_warn_once("%s: pmudata not initialized\n", __func__); - return 0; + if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) { + if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER) + state |= PERF_GUEST_USER; + } else if (xenpmu_data->pmu.r.regs.cpl & 3) { + state |= PERF_GUEST_USER; } - if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) - return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER); - else - return !!(xenpmu_data->pmu.r.regs.cpl & 3); + return state; } static unsigned long xen_get_guest_ip(void) @@ -456,9 +451,8 @@ static unsigned long xen_get_guest_ip(void) } static struct perf_guest_info_callbacks xen_guest_cbs = { - .is_in_guest = xen_is_in_guest, - .is_user_mode = xen_is_user_mode, - .get_guest_ip = xen_get_guest_ip, + .state = xen_guest_state, + .get_ip = xen_get_guest_ip, }; /* Convert registers from Xen's format to Linux' */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 98c204488496..5e6b346d62a7 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -26,11 +26,13 @@ # include #endif +#define PERF_GUEST_ACTIVE 0x01 +#define PERF_GUEST_USER 0x02 + struct perf_guest_info_callbacks { - int (*is_in_guest)(void); - int (*is_user_mode)(void); - unsigned long (*get_guest_ip)(void); - void (*handle_intel_pt_intr)(void); + unsigned int (*state)(void); + unsigned long (*get_ip)(void); + unsigned int (*handle_intel_pt_intr)(void); }; #ifdef CONFIG_HAVE_HW_BREAKPOINT -- cgit v1.2.3-59-g8ed1b From 1c3430516b0732d923de9fd3bfb3e2e537eeb235 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:28 +0000 Subject: perf: Add wrappers for invoking guest callbacks Add helpers for the guest callbacks to prepare for burying the callbacks behind a Kconfig (it's a lot easier to provide a few stubs than to #ifdef piles of code), and also to prepare for converting the callbacks to static_call(). perf_instruction_pointer() in particular will have subtle semantics with static_call(), as the "no callbacks" case will return 0 if the callbacks are unregistered between querying guest state and getting the IP. Implement the change now to avoid a functional change when adding static_call() support, and because the new helper needs to return _something_ in this case. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-8-seanjc@google.com --- arch/arm64/kernel/perf_callchain.c | 16 +++++----------- arch/x86/events/core.c | 15 +++++---------- arch/x86/events/intel/core.c | 5 +---- include/linux/perf_event.h | 24 ++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 274dc3e11b6d..db04a55cee7e 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -102,9 +102,7 @@ compat_user_backtrace(struct compat_frame_tail __user *tail, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->state()) { + if (perf_guest_state()) { /* We don't support guest os callchain now */ return; } @@ -149,10 +147,9 @@ static bool callchain_trace(void *data, unsigned long pc) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stackframe frame; - if (guest_cbs && guest_cbs->state()) { + if (perf_guest_state()) { /* We don't support guest os callchain now */ return; } @@ -163,18 +160,15 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, unsigned long perf_instruction_pointer(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->state()) - return guest_cbs->get_ip(); + if (perf_guest_state()) + return perf_guest_get_ip(); return instruction_pointer(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0; + unsigned int guest_state = perf_guest_state(); int misc = 0; if (guest_state) { diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index e29312a1003a..620347398027 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2768,11 +2768,10 @@ static bool perf_hw_regs(struct pt_regs *regs) void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct unwind_state state; unsigned long addr; - if (guest_cbs && guest_cbs->state()) { + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2872,11 +2871,10 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); struct stack_frame frame; const struct stack_frame __user *fp; - if (guest_cbs && guest_cbs->state()) { + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; } @@ -2953,18 +2951,15 @@ static unsigned long code_segment_base(struct pt_regs *regs) unsigned long perf_instruction_pointer(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->state()) - return guest_cbs->get_ip(); + if (perf_guest_state()) + return perf_guest_get_ip(); return regs->ip + code_segment_base(regs); } unsigned long perf_misc_flags(struct pt_regs *regs) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0; + unsigned int guest_state = perf_guest_state(); int misc = 0; if (guest_state) { diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 7ff24d1ecdb7..f7af8029664f 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2837,7 +2837,6 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) { struct perf_sample_data data; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - struct perf_guest_info_callbacks *guest_cbs; int bit; int handled = 0; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); @@ -2904,9 +2903,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) { handled++; - - guest_cbs = perf_get_guest_cbs(); - if (likely(!guest_cbs || !guest_cbs->handle_intel_pt_intr())) + if (!perf_guest_handle_intel_pt_intr()) intel_pt_interrupt(); } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5e6b346d62a7..346d5aff5804 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1254,6 +1254,30 @@ static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) */ return rcu_dereference(perf_guest_cbs); } +static inline unsigned int perf_guest_state(void) +{ + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + return guest_cbs ? guest_cbs->state() : 0; +} +static inline unsigned long perf_guest_get_ip(void) +{ + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + /* + * Arbitrarily return '0' in the unlikely scenario that the callbacks + * are unregistered between checking guest state and getting the IP. + */ + return guest_cbs ? guest_cbs->get_ip() : 0; +} +static inline unsigned int perf_guest_handle_intel_pt_intr(void) +{ + struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); + + if (guest_cbs && guest_cbs->handle_intel_pt_intr) + return guest_cbs->handle_intel_pt_intr(); + return 0; +} extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); -- cgit v1.2.3-59-g8ed1b From 2aef6f306b39bbe74e2287d6e2ee07c4867d87d0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:29 +0000 Subject: perf: Force architectures to opt-in to guest callbacks Introduce GUEST_PERF_EVENTS and require architectures to select it to allow registering and using guest callbacks in perf. This will hopefully make it more difficult for new architectures to add useless "support" for guest callbacks, e.g. via copy+paste. Stubbing out the helpers has the happy bonus of avoiding a load of perf_guest_cbs when GUEST_PERF_EVENTS=n on arm64/x86. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-9-seanjc@google.com --- arch/arm64/kvm/Kconfig | 1 + arch/x86/kvm/Kconfig | 1 + arch/x86/xen/Kconfig | 1 + include/linux/perf_event.h | 6 ++++++ init/Kconfig | 4 ++++ kernel/events/core.c | 2 ++ 6 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 8ffcbe29395e..e9761d84f982 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -39,6 +39,7 @@ menuconfig KVM select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO + select GUEST_PERF_EVENTS if PERF_EVENTS help Support hosting virtualized guest machines. diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 619186138176..47bdbe705a76 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -36,6 +36,7 @@ config KVM select KVM_MMIO select SCHED_INFO select PERF_EVENTS + select GUEST_PERF_EVENTS select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT select HAVE_KVM_NO_POLL diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index 6bcd3d8ca6ac..85246dd9faa1 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -23,6 +23,7 @@ config XEN_PV select PARAVIRT_XXL select XEN_HAVE_PVMMU select XEN_HAVE_VPMU + select GUEST_PERF_EVENTS help Support running as a Xen PV guest. diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 346d5aff5804..ea47ef616ee0 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1242,6 +1242,7 @@ extern void perf_event_bpf_event(struct bpf_prog *prog, enum perf_bpf_event_type type, u16 flags); +#ifdef CONFIG_GUEST_PERF_EVENTS extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs; static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) { @@ -1280,6 +1281,11 @@ static inline unsigned int perf_guest_handle_intel_pt_intr(void) } extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); +#else +static inline unsigned int perf_guest_state(void) { return 0; } +static inline unsigned long perf_guest_get_ip(void) { return 0; } +static inline unsigned int perf_guest_handle_intel_pt_intr(void) { return 0; } +#endif /* CONFIG_GUEST_PERF_EVENTS */ extern void perf_event_exec(void); extern void perf_event_comm(struct task_struct *tsk, bool exec); diff --git a/init/Kconfig b/init/Kconfig index 036b750e8d8a..72d40b3b5805 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1804,6 +1804,10 @@ config HAVE_PERF_EVENTS help See tools/perf/design.txt for details. +config GUEST_PERF_EVENTS + bool + depends on HAVE_PERF_EVENTS + config PERF_USE_VMALLOC bool help diff --git a/kernel/events/core.c b/kernel/events/core.c index 17e5b20762c5..5a3502cd5362 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6521,6 +6521,7 @@ static void perf_pending_event(struct irq_work *entry) perf_swevent_put_recursion_context(rctx); } +#ifdef CONFIG_GUEST_PERF_EVENTS struct perf_guest_info_callbacks __rcu *perf_guest_cbs; void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) @@ -6541,6 +6542,7 @@ void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) synchronize_rcu(); } EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); +#endif static void perf_output_sample_regs(struct perf_output_handle *handle, -- cgit v1.2.3-59-g8ed1b From 87b940a0675e25261f022ac3e53e0dfff9cdb995 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:30 +0000 Subject: perf/core: Use static_call to optimize perf_guest_info_callbacks Use static_call to optimize perf's guest callbacks on arm64 and x86, which are now the only architectures that define the callbacks. Use DEFINE_STATIC_CALL_RET0 as the default/NULL for all guest callbacks, as the callback semantics are that a return value '0' means "not in guest". static_call obviously avoids the overhead of CONFIG_RETPOLINE=y, but is also advantageous versus other solutions, e.g. per-cpu callbacks, in that a per-cpu memory load is not needed to detect the !guest case. Based on code from Peter and Like. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-10-seanjc@google.com --- include/linux/perf_event.h | 34 ++++++++-------------------------- kernel/events/core.c | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ea47ef616ee0..0ac7d867ca0c 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1244,40 +1244,22 @@ extern void perf_event_bpf_event(struct bpf_prog *prog, #ifdef CONFIG_GUEST_PERF_EVENTS extern struct perf_guest_info_callbacks __rcu *perf_guest_cbs; -static inline struct perf_guest_info_callbacks *perf_get_guest_cbs(void) -{ - /* - * Callbacks are RCU-protected and must be READ_ONCE to avoid reloading - * the callbacks between a !NULL check and dereferences, to ensure - * pending stores/changes to the callback pointers are visible before a - * non-NULL perf_guest_cbs is visible to readers, and to prevent a - * module from unloading callbacks while readers are active. - */ - return rcu_dereference(perf_guest_cbs); -} + +DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state); +DECLARE_STATIC_CALL(__perf_guest_get_ip, *perf_guest_cbs->get_ip); +DECLARE_STATIC_CALL(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr); + static inline unsigned int perf_guest_state(void) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - return guest_cbs ? guest_cbs->state() : 0; + return static_call(__perf_guest_state)(); } static inline unsigned long perf_guest_get_ip(void) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - /* - * Arbitrarily return '0' in the unlikely scenario that the callbacks - * are unregistered between checking guest state and getting the IP. - */ - return guest_cbs ? guest_cbs->get_ip() : 0; + return static_call(__perf_guest_get_ip)(); } static inline unsigned int perf_guest_handle_intel_pt_intr(void) { - struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs(); - - if (guest_cbs && guest_cbs->handle_intel_pt_intr) - return guest_cbs->handle_intel_pt_intr(); - return 0; + return static_call(__perf_guest_handle_intel_pt_intr)(); } extern void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); extern void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs); diff --git a/kernel/events/core.c b/kernel/events/core.c index 5a3502cd5362..3b3297a57228 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6524,12 +6524,23 @@ static void perf_pending_event(struct irq_work *entry) #ifdef CONFIG_GUEST_PERF_EVENTS struct perf_guest_info_callbacks __rcu *perf_guest_cbs; +DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state); +DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip); +DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr); + void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) { if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs))) return; rcu_assign_pointer(perf_guest_cbs, cbs); + static_call_update(__perf_guest_state, cbs->state); + static_call_update(__perf_guest_get_ip, cbs->get_ip); + + /* Implementing ->handle_intel_pt_intr is optional. */ + if (cbs->handle_intel_pt_intr) + static_call_update(__perf_guest_handle_intel_pt_intr, + cbs->handle_intel_pt_intr); } EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); @@ -6539,6 +6550,10 @@ void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) return; rcu_assign_pointer(perf_guest_cbs, NULL); + static_call_update(__perf_guest_state, (void *)&__static_call_return0); + static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0); + static_call_update(__perf_guest_handle_intel_pt_intr, + (void *)&__static_call_return0); synchronize_rcu(); } EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); -- cgit v1.2.3-59-g8ed1b From e1bfc24577cc65c95dc519d7621a9c985b97e567 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:33 +0000 Subject: KVM: Move x86's perf guest info callbacks to generic KVM Move x86's perf guest callbacks into common KVM, as they are semantically identical to arm64's callbacks (the only other such KVM callbacks). arm64 will convert to the common versions in a future patch. Implement the necessary arm64 arch hooks now to avoid having to provide stubs or a temporary #define (from x86) to avoid arm64 compilation errors when CONFIG_GUEST_PERF_EVENTS=y. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Acked-by: Marc Zyngier Link: https://lore.kernel.org/r/20211111020738.2512932-13-seanjc@google.com --- arch/arm64/include/asm/kvm_host.h | 10 ++++++++ arch/arm64/kvm/arm.c | 5 ++++ arch/x86/include/asm/kvm_host.h | 3 +++ arch/x86/kvm/x86.c | 53 ++++++++------------------------------- include/linux/kvm_host.h | 10 ++++++++ virt/kvm/kvm_main.c | 44 ++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index f680f303ba7c..aa28b8e0e5d0 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -678,6 +678,16 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); void kvm_perf_init(void); void kvm_perf_teardown(void); +/* + * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, + * arrived in guest context. For arm64, any event that arrives while a vCPU is + * loaded is considered to be "in guest". + */ +static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) +{ + return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; +} + long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); void kvm_update_stolen_time(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 2f03cbfefe67..b400be996040 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -496,6 +496,11 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) return vcpu_mode_priv(vcpu); } +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) +{ + return *vcpu_pc(vcpu); +} + /* Just ensure a guest exit from a particular CPU */ static void exit_vm_noop(void *info) { diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 38f01b00d82a..89576549b1d5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1567,6 +1567,9 @@ static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm) return -ENOTSUPP; } +#define kvm_arch_pmi_in_guest(vcpu) \ + ((vcpu) && (vcpu)->arch.handling_intr_from_guest) + int kvm_mmu_module_init(void); void kvm_mmu_module_exit(void); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ab032ef7879f..32cb6f9ca077 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8469,43 +8469,12 @@ static void kvm_timer_init(void) kvmclock_cpu_online, kvmclock_cpu_down_prep); } -static inline bool kvm_pmi_in_guest(struct kvm_vcpu *vcpu) -{ - return vcpu && vcpu->arch.handling_intr_from_guest; -} - -static unsigned int kvm_guest_state(void) -{ - struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); - unsigned int state; - - if (!kvm_pmi_in_guest(vcpu)) - return 0; - - state = PERF_GUEST_ACTIVE; - if (static_call(kvm_x86_get_cpl)(vcpu)) - state |= PERF_GUEST_USER; - - return state; -} - -static unsigned long kvm_guest_get_ip(void) -{ - struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); - - /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */ - if (WARN_ON_ONCE(!kvm_pmi_in_guest(vcpu))) - return 0; - - return kvm_rip_read(vcpu); -} - static unsigned int kvm_handle_intel_pt_intr(void) { struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); /* '0' on failure so that the !PT case can use a RET0 static call. */ - if (!kvm_pmi_in_guest(vcpu)) + if (!kvm_arch_pmi_in_guest(vcpu)) return 0; kvm_make_request(KVM_REQ_PMI, vcpu); @@ -8514,12 +8483,6 @@ static unsigned int kvm_handle_intel_pt_intr(void) return 1; } -static struct perf_guest_info_callbacks kvm_guest_cbs = { - .state = kvm_guest_state, - .get_ip = kvm_guest_get_ip, - .handle_intel_pt_intr = NULL, -}; - #ifdef CONFIG_X86_64 static void pvclock_gtod_update_fn(struct work_struct *work) { @@ -11229,9 +11192,11 @@ int kvm_arch_hardware_setup(void *opaque) memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); kvm_ops_static_call_update(); + /* Temporary ugliness. */ if (ops->intel_pt_intr_in_guest && ops->intel_pt_intr_in_guest()) - kvm_guest_cbs.handle_intel_pt_intr = kvm_handle_intel_pt_intr; - perf_register_guest_info_callbacks(&kvm_guest_cbs); + kvm_register_perf_callbacks(kvm_handle_intel_pt_intr); + else + kvm_register_perf_callbacks(NULL); if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) supported_xss = 0; @@ -11260,8 +11225,7 @@ int kvm_arch_hardware_setup(void *opaque) void kvm_arch_hardware_unsetup(void) { - perf_unregister_guest_info_callbacks(&kvm_guest_cbs); - kvm_guest_cbs.handle_intel_pt_intr = NULL; + kvm_unregister_perf_callbacks(); static_call(kvm_x86_hardware_unsetup)(); } @@ -11852,6 +11816,11 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) return vcpu->arch.preempted_in_kernel; } +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) +{ + return kvm_rip_read(vcpu); +} + int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) { return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 9e0667e3723e..9df7ab2d7530 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1170,6 +1170,16 @@ static inline bool kvm_arch_intc_initialized(struct kvm *kvm) } #endif +#ifdef CONFIG_GUEST_PERF_EVENTS +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu); + +void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void)); +void kvm_unregister_perf_callbacks(void); +#else +static inline void kvm_register_perf_callbacks(void *ign) {} +static inline void kvm_unregister_perf_callbacks(void) {} +#endif /* CONFIG_GUEST_PERF_EVENTS */ + int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); void kvm_arch_destroy_vm(struct kvm *kvm); void kvm_arch_sync_events(struct kvm *kvm); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d31724500501..76778dd2351f 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5479,6 +5479,50 @@ struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void) return &kvm_running_vcpu; } +#ifdef CONFIG_GUEST_PERF_EVENTS +static unsigned int kvm_guest_state(void) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + unsigned int state; + + if (!kvm_arch_pmi_in_guest(vcpu)) + return 0; + + state = PERF_GUEST_ACTIVE; + if (!kvm_arch_vcpu_in_kernel(vcpu)) + state |= PERF_GUEST_USER; + + return state; +} + +static unsigned long kvm_guest_get_ip(void) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + + /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */ + if (WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu))) + return 0; + + return kvm_arch_vcpu_get_ip(vcpu); +} + +static struct perf_guest_info_callbacks kvm_guest_cbs = { + .state = kvm_guest_state, + .get_ip = kvm_guest_get_ip, + .handle_intel_pt_intr = NULL, +}; + +void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void)) +{ + kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler; + perf_register_guest_info_callbacks(&kvm_guest_cbs); +} +void kvm_unregister_perf_callbacks(void) +{ + perf_unregister_guest_info_callbacks(&kvm_guest_cbs); +} +#endif + struct kvm_cpu_compat_check { void *opaque; int *ret; -- cgit v1.2.3-59-g8ed1b From a9f4a6e92b3b319296fb078da2615f618f6cd80c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 11 Nov 2021 02:07:38 +0000 Subject: perf: Drop guest callback (un)register stubs Drop perf's stubs for (un)registering guest callbacks now that KVM registration of callbacks is hidden behind GUEST_PERF_EVENTS=y. The only other user is x86 XEN_PV, and x86 unconditionally selects PERF_EVENTS. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20211111020738.2512932-18-seanjc@google.com --- include/linux/perf_event.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 0ac7d867ca0c..7b7525e9155f 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1511,11 +1511,6 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { } static inline void perf_bp_event(struct perf_event *event, void *data) { } -static inline void perf_register_guest_info_callbacks -(struct perf_guest_info_callbacks *cbs) { } -static inline void perf_unregister_guest_info_callbacks -(struct perf_guest_info_callbacks *cbs) { } - static inline void perf_event_mmap(struct vm_area_struct *vma) { } typedef int (perf_ksymbol_get_name_f)(char *name, int name_len, void *data); -- cgit v1.2.3-59-g8ed1b From 8160fb43d55d26d64607fd32fe69185a5f5fe41f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 16 Nov 2021 19:29:21 -0800 Subject: net: use an atomic_long_t for queue->trans_timeout tx_timeout_show() assumed dev_watchdog() would stop all the queues, to fetch queue->trans_timeout under protection of the queue->_xmit_lock. As we want to no longer disrupt transmits, we use an atomic_long_t instead. Signed-off-by: Eric Dumazet Cc: david decotigny Signed-off-by: David S. Miller --- include/linux/netdevice.h | 2 +- net/core/net-sysfs.c | 6 +----- net/sched/sch_generic.c | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index dd328364dfe9..1d22483cf78c 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -592,7 +592,7 @@ struct netdev_queue { * Number of TX timeouts for this queue * (/sys/class/net/DEV/Q/trans_timeout) */ - unsigned long trans_timeout; + atomic_long_t trans_timeout; /* Subordinate device that the queue has been assigned to */ struct net_device *sb_dev; diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 9c01c642cf9e..addbef5419fb 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1201,11 +1201,7 @@ static const struct sysfs_ops netdev_queue_sysfs_ops = { static ssize_t tx_timeout_show(struct netdev_queue *queue, char *buf) { - unsigned long trans_timeout; - - spin_lock_irq(&queue->_xmit_lock); - trans_timeout = queue->trans_timeout; - spin_unlock_irq(&queue->_xmit_lock); + unsigned long trans_timeout = atomic_long_read(&queue->trans_timeout); return sprintf(buf, fmt_ulong, trans_timeout); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 3b0f62095803..1b4328bd495d 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -467,7 +467,7 @@ static void dev_watchdog(struct timer_list *t) time_after(jiffies, (trans_start + dev->watchdog_timeo))) { some_queue_timedout = 1; - txq->trans_timeout++; + atomic_long_inc(&txq->trans_timeout); break; } } -- cgit v1.2.3-59-g8ed1b From 5337824f4dc4bb26f38fbbba4ffb425a92803f15 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 16 Nov 2021 19:29:22 -0800 Subject: net: annotate accesses to queue->trans_start In following patches, dev_watchdog() will no longer stop all queues. It will read queue->trans_start locklessly. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/ethernet/apm/xgene/xgene_enet_main.c | 2 +- drivers/net/ethernet/atheros/ag71xx.c | 2 +- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 4 ++-- drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 2 +- drivers/net/ethernet/ibm/ibmvnic.c | 2 +- drivers/net/ethernet/intel/igb/igb_main.c | 4 ++-- drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c | 2 +- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 6 +++--- drivers/net/ethernet/ti/am65-cpsw-nuss.c | 2 +- drivers/net/virtio_net.c | 2 +- drivers/net/wireless/marvell/mwifiex/init.c | 2 +- drivers/staging/rtl8192e/rtllib_softmac.c | 2 +- include/linux/netdevice.h | 16 +++++++++++++--- net/sched/sch_generic.c | 8 ++++---- 14 files changed, 33 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c index 220dc42af31a..ff2d099aab21 100644 --- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c +++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c @@ -869,7 +869,7 @@ static void xgene_enet_timeout(struct net_device *ndev, unsigned int txqueue) for (i = 0; i < pdata->txq_cnt; i++) { txq = netdev_get_tx_queue(ndev, i); - txq->trans_start = jiffies; + txq_trans_cond_update(txq); netif_tx_start_queue(txq); } } diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index 88d2ab748399..e4f30bb7498f 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -766,7 +766,7 @@ static bool ag71xx_check_dma_stuck(struct ag71xx *ag) unsigned long timestamp; u32 rx_sm, tx_sm, rx_fd; - timestamp = netdev_get_tx_queue(ag->ndev, 0)->trans_start; + timestamp = READ_ONCE(netdev_get_tx_queue(ag->ndev, 0)->trans_start); if (likely(time_before(jiffies, timestamp + HZ / 10))) return false; diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 6b2927d863e2..d6871437d951 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2325,7 +2325,7 @@ dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev) txq = netdev_get_tx_queue(net_dev, queue_mapping); /* LLTX requires to do our own update of trans_start */ - txq->trans_start = jiffies; + txq_trans_cond_update(txq); if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { fd.cmd |= cpu_to_be32(FM_FD_CMD_UPD); @@ -2531,7 +2531,7 @@ static int dpaa_xdp_xmit_frame(struct net_device *net_dev, /* Bump the trans_start */ txq = netdev_get_tx_queue(net_dev, smp_processor_id()); - txq->trans_start = jiffies; + txq_trans_cond_update(txq); err = dpaa_xmit(priv, percpu_stats, smp_processor_id(), &fd); if (err) { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 13835a37b3a2..d5100179f8d5 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -2679,7 +2679,7 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) unsigned long trans_start; q = netdev_get_tx_queue(ndev, i); - trans_start = q->trans_start; + trans_start = READ_ONCE(q->trans_start); if (netif_xmit_stopped(q) && time_after(jiffies, (trans_start + ndev->watchdog_timeo))) { diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 3cca51735421..c327fc8860da 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2058,7 +2058,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev) tx_packets++; tx_bytes += skb->len; - txq->trans_start = jiffies; + txq_trans_cond_update(txq); ret = NETDEV_TX_OK; goto out; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 836be0d3b291..18a019a47182 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -2927,7 +2927,7 @@ static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp) nq = txring_txq(tx_ring); __netif_tx_lock(nq, cpu); /* Avoid transmit queue timeout since we share it with the slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); __netif_tx_unlock(nq); @@ -2961,7 +2961,7 @@ static int igb_xdp_xmit(struct net_device *dev, int n, __netif_tx_lock(nq, cpu); /* Avoid transmit queue timeout since we share it with the slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); for (i = 0; i < n; i++) { struct xdp_frame *xdpf = frames[i]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 4f4bc8726ec4..860605133287 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -565,7 +565,7 @@ int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) snprintf(err_str, sizeof(err_str), "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u", sq->ch_ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, - jiffies_to_usecs(jiffies - sq->txq->trans_start)); + jiffies_to_usecs(jiffies - READ_ONCE(sq->txq->trans_start))); mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); return to_ctx.status; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 033c35c09a54..389d125310c1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2356,7 +2356,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) bool work_done = true; /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq->trans_start); budget = min(budget, stmmac_tx_avail(priv, queue)); @@ -4657,7 +4657,7 @@ static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, __netif_tx_lock(nq, cpu); /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq->trans_start); res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); if (res == STMMAC_XDP_TX) @@ -6293,7 +6293,7 @@ static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, __netif_tx_lock(nq, cpu); /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); for (i = 0; i < num_frames; i++) { int res; diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index c092cb61416a..750cea23e9cd 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -345,7 +345,7 @@ static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev, netif_txq = netdev_get_tx_queue(ndev, txqueue); tx_chn = &common->tx_chns[txqueue]; - trans_start = netif_txq->trans_start; + trans_start = READ_ONCE(netif_txq->trans_start); netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n", txqueue, diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 1771d6e5224f..03e38e38ee4b 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2694,7 +2694,7 @@ static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", txqueue, sq->name, sq->vq->index, sq->vq->name, - jiffies_to_usecs(jiffies - txq->trans_start)); + jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); } static const struct net_device_ops virtnet_netdev = { diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c index f006a3d72b40..88c72d1827a0 100644 --- a/drivers/net/wireless/marvell/mwifiex/init.c +++ b/drivers/net/wireless/marvell/mwifiex/init.c @@ -332,7 +332,7 @@ void mwifiex_set_trans_start(struct net_device *dev) int i; for (i = 0; i < dev->num_tx_queues; i++) - netdev_get_tx_queue(dev, i)->trans_start = jiffies; + txq_trans_cond_update(netdev_get_tx_queue(dev, i)); netif_trans_update(dev); } diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c index d2726d01c757..aabbea48223d 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac.c +++ b/drivers/staging/rtl8192e/rtllib_softmac.c @@ -2515,7 +2515,7 @@ void rtllib_stop_all_queues(struct rtllib_device *ieee) unsigned int i; for (i = 0; i < ieee->dev->num_tx_queues; i++) - netdev_get_tx_queue(ieee->dev, i)->trans_start = jiffies; + txq_trans_cond_update(netdev_get_tx_queue(ieee->dev, i)); netif_tx_stop_all_queues(ieee->dev); } diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1d22483cf78c..279409ef2b18 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4095,10 +4095,21 @@ static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) spin_unlock_bh(&txq->_xmit_lock); } +/* + * txq->trans_start can be read locklessly from dev_watchdog() + */ static inline void txq_trans_update(struct netdev_queue *txq) { if (txq->xmit_lock_owner != -1) - txq->trans_start = jiffies; + WRITE_ONCE(txq->trans_start, jiffies); +} + +static inline void txq_trans_cond_update(struct netdev_queue *txq) +{ + unsigned long now = jiffies; + + if (READ_ONCE(txq->trans_start) != now) + WRITE_ONCE(txq->trans_start, now); } /* legacy drivers only, netdev_start_xmit() sets txq->trans_start */ @@ -4106,8 +4117,7 @@ static inline void netif_trans_update(struct net_device *dev) { struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); - if (txq->trans_start != jiffies) - txq->trans_start = jiffies; + txq_trans_cond_update(txq); } /** diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 1b4328bd495d..02c46041f76e 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -434,9 +434,9 @@ unsigned long dev_trans_start(struct net_device *dev) dev = vlan_dev_real_dev(dev); else if (netif_is_macvlan(dev)) dev = macvlan_dev_real_dev(dev); - res = netdev_get_tx_queue(dev, 0)->trans_start; + res = READ_ONCE(netdev_get_tx_queue(dev, 0)->trans_start); for (i = 1; i < dev->num_tx_queues; i++) { - val = netdev_get_tx_queue(dev, i)->trans_start; + val = READ_ONCE(netdev_get_tx_queue(dev, i)->trans_start); if (val && time_after(val, res)) res = val; } @@ -462,7 +462,7 @@ static void dev_watchdog(struct timer_list *t) struct netdev_queue *txq; txq = netdev_get_tx_queue(dev, i); - trans_start = txq->trans_start; + trans_start = READ_ONCE(txq->trans_start); if (netif_xmit_stopped(txq) && time_after(jiffies, (trans_start + dev->watchdog_timeo))) { @@ -1148,7 +1148,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - dev_queue->trans_start = 0; + WRITE_ONCE(dev_queue->trans_start, 0); *need_watchdog_p = 1; } } -- cgit v1.2.3-59-g8ed1b From dab8fe320726b38a6b1dc6a7ca6e386c5f7779e8 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 16 Nov 2021 19:29:23 -0800 Subject: net: do not inline netif_tx_lock()/netif_tx_unlock() These are not fast path, there is no point in inlining them. Also provide netif_freeze_queues()/netif_unfreeze_queues() so that we can use them from dev_watchdog() in the following patch. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 39 ++---------------------------------- net/sched/sch_generic.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 279409ef2b18..4f4a299e92de 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4126,27 +4126,7 @@ static inline void netif_trans_update(struct net_device *dev) * * Get network device transmit lock */ -static inline void netif_tx_lock(struct net_device *dev) -{ - unsigned int i; - int cpu; - - spin_lock(&dev->tx_global_lock); - cpu = smp_processor_id(); - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - - /* We are the only thread of execution doing a - * freeze, but we have to grab the _xmit_lock in - * order to synchronize with threads which are in - * the ->hard_start_xmit() handler and already - * checked the frozen bit. - */ - __netif_tx_lock(txq, cpu); - set_bit(__QUEUE_STATE_FROZEN, &txq->state); - __netif_tx_unlock(txq); - } -} +void netif_tx_lock(struct net_device *dev); static inline void netif_tx_lock_bh(struct net_device *dev) { @@ -4154,22 +4134,7 @@ static inline void netif_tx_lock_bh(struct net_device *dev) netif_tx_lock(dev); } -static inline void netif_tx_unlock(struct net_device *dev) -{ - unsigned int i; - - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - - /* No need to grab the _xmit_lock here. If the - * queue is not stopped for another reason, we - * force a schedule. - */ - clear_bit(__QUEUE_STATE_FROZEN, &txq->state); - netif_schedule_queue(txq); - } - spin_unlock(&dev->tx_global_lock); -} +void netif_tx_unlock(struct net_device *dev); static inline void netif_tx_unlock_bh(struct net_device *dev) { diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 02c46041f76e..389e0d8fc68d 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -445,6 +445,57 @@ unsigned long dev_trans_start(struct net_device *dev) } EXPORT_SYMBOL(dev_trans_start); +static void netif_freeze_queues(struct net_device *dev) +{ + unsigned int i; + int cpu; + + cpu = smp_processor_id(); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + + /* We are the only thread of execution doing a + * freeze, but we have to grab the _xmit_lock in + * order to synchronize with threads which are in + * the ->hard_start_xmit() handler and already + * checked the frozen bit. + */ + __netif_tx_lock(txq, cpu); + set_bit(__QUEUE_STATE_FROZEN, &txq->state); + __netif_tx_unlock(txq); + } +} + +void netif_tx_lock(struct net_device *dev) +{ + spin_lock(&dev->tx_global_lock); + netif_freeze_queues(dev); +} +EXPORT_SYMBOL(netif_tx_lock); + +static void netif_unfreeze_queues(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + + /* No need to grab the _xmit_lock here. If the + * queue is not stopped for another reason, we + * force a schedule. + */ + clear_bit(__QUEUE_STATE_FROZEN, &txq->state); + netif_schedule_queue(txq); + } +} + +void netif_tx_unlock(struct net_device *dev) +{ + netif_unfreeze_queues(dev); + spin_unlock(&dev->tx_global_lock); +} +EXPORT_SYMBOL(netif_tx_unlock); + static void dev_watchdog(struct timer_list *t) { struct net_device *dev = from_timer(dev, t, watchdog_timer); -- cgit v1.2.3-59-g8ed1b From 1881eadb2041889d74d60c074eb04189c4a07dad Mon Sep 17 00:00:00 2001 From: Abhyuday Godhasara Date: Mon, 25 Oct 2021 21:25:20 -0700 Subject: firmware: xilinx: add register notifier in zynqmp firmware In zynqmp-firmware, register notifier is not supported, add support of register notifier in zynqmp-firmware. Acked-by: Michal Simek Signed-off-by: Tejas Patel Signed-off-by: Abhyuday Godhasara Link: https://lore.kernel.org/r/20211026042525.26612-2-abhyuday.godhasara@xilinx.com Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/xilinx/zynqmp.c | 23 +++++++++++++++++++++++ include/linux/firmware/xlnx-zynqmp.h | 11 ++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 3dd45a7420dc..30edcd233786 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -1116,6 +1116,29 @@ int zynqmp_pm_aes_engine(const u64 address, u32 *out) } EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine); +/** + * zynqmp_pm_register_notifier() - PM API for register a subsystem + * to be notified about specific + * event/error. + * @node: Node ID to which the event is related. + * @event: Event Mask of Error events for which wants to get notified. + * @wake: Wake subsystem upon capturing the event if value 1 + * @enable: Enable the registration for value 1, disable for value 0 + * + * This function is used to register/un-register for particular node-event + * combination in firmware. + * + * Return: Returns status, either success or error+reason + */ + +int zynqmp_pm_register_notifier(const u32 node, const u32 event, + const u32 wake, const u32 enable) +{ + return zynqmp_pm_invoke_fn(PM_REGISTER_NOTIFIER, node, event, + wake, enable, NULL); +} +EXPORT_SYMBOL_GPL(zynqmp_pm_register_notifier); + /** * zynqmp_pm_system_shutdown - PM call to request a system shutdown or restart * @type: Shutdown or restart? 0 for shutdown, 1 for restart diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 47fd4e52a423..d30d39dc8cb4 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -2,7 +2,7 @@ /* * Xilinx Zynq MPSoC Firmware layer * - * Copyright (C) 2014-2019 Xilinx + * Copyright (C) 2014-2021 Xilinx * * Michal Simek * Davorin Mista @@ -66,6 +66,7 @@ enum pm_api_id { PM_GET_API_VERSION = 1, + PM_REGISTER_NOTIFIER = 5, PM_SYSTEM_SHUTDOWN = 12, PM_REQUEST_NODE = 13, PM_RELEASE_NODE = 14, @@ -427,6 +428,8 @@ int zynqmp_pm_pinctrl_get_config(const u32 pin, const u32 param, int zynqmp_pm_pinctrl_set_config(const u32 pin, const u32 param, u32 value); int zynqmp_pm_load_pdi(const u32 src, const u64 address); +int zynqmp_pm_register_notifier(const u32 node, const u32 event, + const u32 wake, const u32 enable); #else static inline int zynqmp_pm_get_api_version(u32 *version) { @@ -658,6 +661,12 @@ static inline int zynqmp_pm_load_pdi(const u32 src, const u64 address) { return -ENODEV; } + +static inline int zynqmp_pm_register_notifier(const u32 node, const u32 event, + const u32 wake, const u32 enable) +{ + return -ENODEV; +} #endif #endif /* __FIRMWARE_ZYNQMP_H__ */ -- cgit v1.2.3-59-g8ed1b From fbce9f14055e547d270046f61758c29c957e675d Mon Sep 17 00:00:00 2001 From: Abhyuday Godhasara Date: Mon, 25 Oct 2021 21:25:21 -0700 Subject: firmware: xilinx: add macros of node ids for error event Add macros for the Node-Id of Error events. Move supported api callback ids from zynqmp-power to zynqmp-firmware. Acked-by: Michal Simek Signed-off-by: Rajan Vaja Signed-off-by: Abhyuday Godhasara Link: https://lore.kernel.org/r/20211026042525.26612-3-abhyuday.godhasara@xilinx.com Signed-off-by: Greg Kroah-Hartman --- drivers/soc/xilinx/zynqmp_power.c | 6 ------ include/linux/firmware/xlnx-zynqmp.h | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c index c556623dae02..76478fe2301f 100644 --- a/drivers/soc/xilinx/zynqmp_power.c +++ b/drivers/soc/xilinx/zynqmp_power.c @@ -46,12 +46,6 @@ static const char *const suspend_modes[] = { static enum pm_suspend_mode suspend_mode = PM_SUSPEND_MODE_STD; -enum pm_api_cb_id { - PM_INIT_SUSPEND_CB = 30, - PM_ACKNOWLEDGE_CB, - PM_NOTIFY_CB, -}; - static void zynqmp_pm_get_callback_data(u32 *buf) { zynqmp_pm_invoke_fn(GET_CALLBACK_DATA, 0, 0, 0, 0, buf); diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index d30d39dc8cb4..b0a38091db71 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -64,6 +64,20 @@ #define XILINX_ZYNQMP_PM_FPGA_FULL 0x0U #define XILINX_ZYNQMP_PM_FPGA_PARTIAL BIT(0) +/* + * Node IDs for the Error Events. + */ +#define EVENT_ERROR_PMC_ERR1 (0x28100000U) +#define EVENT_ERROR_PMC_ERR2 (0x28104000U) +#define EVENT_ERROR_PSM_ERR1 (0x28108000U) +#define EVENT_ERROR_PSM_ERR2 (0x2810C000U) + +enum pm_api_cb_id { + PM_INIT_SUSPEND_CB = 30, + PM_ACKNOWLEDGE_CB = 31, + PM_NOTIFY_CB = 32, +}; + enum pm_api_id { PM_GET_API_VERSION = 1, PM_REGISTER_NOTIFIER = 5, -- cgit v1.2.3-59-g8ed1b From f4d77525679e289d4976ca03b620ac4cc5403205 Mon Sep 17 00:00:00 2001 From: Abhyuday Godhasara Date: Mon, 25 Oct 2021 21:25:22 -0700 Subject: firmware: xilinx: export the feature check of zynqmp firmware Export the zynqmp_pm_feature(), so it can be use by other as to get API version available in firmware. Acked-by: Michal Simek Signed-off-by: Rajan Vaja Signed-off-by: Abhyuday Godhasara Link: https://lore.kernel.org/r/20211026042525.26612-4-abhyuday.godhasara@xilinx.com Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/xilinx/zynqmp.c | 3 ++- include/linux/firmware/xlnx-zynqmp.h | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 30edcd233786..c2828ee6d4cf 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -160,7 +160,7 @@ static noinline int do_fw_call_hvc(u64 arg0, u64 arg1, u64 arg2, * * Return: Returns status, either success or error+reason */ -static int zynqmp_pm_feature(u32 api_id) +int zynqmp_pm_feature(const u32 api_id) { int ret; u32 ret_payload[PAYLOAD_ARG_CNT]; @@ -197,6 +197,7 @@ static int zynqmp_pm_feature(u32 api_id) return ret; } +EXPORT_SYMBOL_GPL(zynqmp_pm_feature); /** * zynqmp_pm_invoke_fn() - Invoke the system-level platform management layer diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index b0a38091db71..077e894bb340 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -444,6 +444,7 @@ int zynqmp_pm_pinctrl_set_config(const u32 pin, const u32 param, int zynqmp_pm_load_pdi(const u32 src, const u64 address); int zynqmp_pm_register_notifier(const u32 node, const u32 event, const u32 wake, const u32 enable); +int zynqmp_pm_feature(const u32 api_id); #else static inline int zynqmp_pm_get_api_version(u32 *version) { @@ -681,6 +682,11 @@ static inline int zynqmp_pm_register_notifier(const u32 node, const u32 event, { return -ENODEV; } + +static inline int zynqmp_pm_feature(const u32 api_id) +{ + return -ENODEV; +} #endif #endif /* __FIRMWARE_ZYNQMP_H__ */ -- cgit v1.2.3-59-g8ed1b From 5768d8906bc23d512b1a736c1e198aa833a6daa4 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 15 Nov 2021 13:47:13 -0600 Subject: signal: Requeue signals in the appropriate queue In the event that a tracer changes which signal needs to be delivered and that signal is currently blocked then the signal needs to be requeued for later delivery. With the advent of CLONE_THREAD the kernel has 2 signal queues per task. The per process queue and the per task queue. Update the code so that if the signal is removed from the per process queue it is requeued on the per process queue. This is necessary to make it appear the signal was never dequeued. The rr debugger reasonably believes that the state of the process from the last ptrace_stop it observed until PTRACE_EVENT_EXIT can be recreated by simply letting a process run. If a SIGKILL interrupts a ptrace_stop this is not true today. So return signals to their original queue in ptrace_signal so that signals that are not delivered appear like they were never dequeued. Fixes: 794aa320b79d ("[PATCH] sigfix-2.5.40-D6") History Tree: https://git.kernel.org/pub/scm/linux/kernel/git/tglx/history.gi Reviewed-by: Kees Cook Link: https://lkml.kernel.org/r/87zgq4d5r4.fsf_-_@email.froward.int.ebiederm.org Signed-off-by: "Eric W. Biederman" --- fs/signalfd.c | 5 +++-- include/linux/sched/signal.h | 7 ++++--- kernel/signal.c | 21 ++++++++++++++------- 3 files changed, 21 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/fs/signalfd.c b/fs/signalfd.c index 040e1cf90528..74f134cd1ff6 100644 --- a/fs/signalfd.c +++ b/fs/signalfd.c @@ -165,11 +165,12 @@ static int signalfd_copyinfo(struct signalfd_siginfo __user *uinfo, static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info, int nonblock) { + enum pid_type type; ssize_t ret; DECLARE_WAITQUEUE(wait, current); spin_lock_irq(¤t->sighand->siglock); - ret = dequeue_signal(current, &ctx->sigmask, info); + ret = dequeue_signal(current, &ctx->sigmask, info, &type); switch (ret) { case 0: if (!nonblock) @@ -184,7 +185,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info add_wait_queue(¤t->sighand->signalfd_wqh, &wait); for (;;) { set_current_state(TASK_INTERRUPTIBLE); - ret = dequeue_signal(current, &ctx->sigmask, info); + ret = dequeue_signal(current, &ctx->sigmask, info, &type); if (ret != 0) break; if (signal_pending(current)) { diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 23505394ef70..167995d471da 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -286,17 +286,18 @@ static inline int signal_group_exit(const struct signal_struct *sig) extern void flush_signals(struct task_struct *); extern void ignore_signals(struct task_struct *); extern void flush_signal_handlers(struct task_struct *, int force_default); -extern int dequeue_signal(struct task_struct *task, - sigset_t *mask, kernel_siginfo_t *info); +extern int dequeue_signal(struct task_struct *task, sigset_t *mask, + kernel_siginfo_t *info, enum pid_type *type); static inline int kernel_dequeue_signal(void) { struct task_struct *task = current; kernel_siginfo_t __info; + enum pid_type __type; int ret; spin_lock_irq(&task->sighand->siglock); - ret = dequeue_signal(task, &task->blocked, &__info); + ret = dequeue_signal(task, &task->blocked, &__info, &__type); spin_unlock_irq(&task->sighand->siglock); return ret; diff --git a/kernel/signal.c b/kernel/signal.c index 986fa69c15c5..43e8b7e362b0 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -626,7 +626,8 @@ static int __dequeue_signal(struct sigpending *pending, sigset_t *mask, * * All callers have to hold the siglock. */ -int dequeue_signal(struct task_struct *tsk, sigset_t *mask, kernel_siginfo_t *info) +int dequeue_signal(struct task_struct *tsk, sigset_t *mask, + kernel_siginfo_t *info, enum pid_type *type) { bool resched_timer = false; int signr; @@ -634,8 +635,10 @@ int dequeue_signal(struct task_struct *tsk, sigset_t *mask, kernel_siginfo_t *in /* We only dequeue private signals from ourselves, we don't let * signalfd steal them */ + *type = PIDTYPE_PID; signr = __dequeue_signal(&tsk->pending, mask, info, &resched_timer); if (!signr) { + *type = PIDTYPE_TGID; signr = __dequeue_signal(&tsk->signal->shared_pending, mask, info, &resched_timer); #ifdef CONFIG_POSIX_TIMERS @@ -2522,7 +2525,7 @@ static void do_freezer_trap(void) freezable_schedule(); } -static int ptrace_signal(int signr, kernel_siginfo_t *info) +static int ptrace_signal(int signr, kernel_siginfo_t *info, enum pid_type type) { /* * We do not check sig_kernel_stop(signr) but set this marker @@ -2563,7 +2566,7 @@ static int ptrace_signal(int signr, kernel_siginfo_t *info) /* If the (new) signal is now blocked, requeue it. */ if (sigismember(¤t->blocked, signr)) { - send_signal(signr, info, current, PIDTYPE_PID); + send_signal(signr, info, current, type); signr = 0; } @@ -2664,6 +2667,7 @@ relock: for (;;) { struct k_sigaction *ka; + enum pid_type type; /* Has this task already been marked for death? */ if (signal_group_exit(signal)) { @@ -2706,16 +2710,18 @@ relock: * so that the instruction pointer in the signal stack * frame points to the faulting instruction. */ + type = PIDTYPE_PID; signr = dequeue_synchronous_signal(&ksig->info); if (!signr) - signr = dequeue_signal(current, ¤t->blocked, &ksig->info); + signr = dequeue_signal(current, ¤t->blocked, + &ksig->info, &type); if (!signr) break; /* will return 0 */ if (unlikely(current->ptrace) && (signr != SIGKILL) && !(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) { - signr = ptrace_signal(signr, &ksig->info); + signr = ptrace_signal(signr, &ksig->info, type); if (!signr) continue; } @@ -3540,6 +3546,7 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, ktime_t *to = NULL, timeout = KTIME_MAX; struct task_struct *tsk = current; sigset_t mask = *which; + enum pid_type type; int sig, ret = 0; if (ts) { @@ -3556,7 +3563,7 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, signotset(&mask); spin_lock_irq(&tsk->sighand->siglock); - sig = dequeue_signal(tsk, &mask, info); + sig = dequeue_signal(tsk, &mask, info, &type); if (!sig && timeout) { /* * None ready, temporarily unblock those we're interested @@ -3575,7 +3582,7 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, spin_lock_irq(&tsk->sighand->siglock); __set_task_blocked(tsk, &tsk->real_blocked); sigemptyset(&tsk->real_blocked); - sig = dequeue_signal(tsk, &mask, info); + sig = dequeue_signal(tsk, &mask, info, &type); } spin_unlock_irq(&tsk->sighand->siglock); -- cgit v1.2.3-59-g8ed1b From e0dbd7b0ed021fb9250f7ba4d759325678efefb5 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 16 Nov 2021 23:44:28 +0100 Subject: power: supply: core: Add kerneldoc to battery struct This complements the struct power_supply_battery_info with extensive kerneldoc explaining the different semantics of the fields, including an overview of the CC/CV charging concepts implicit in some of the struct members. This is done to first establish semantics before I can add more charging methods by breaking out the CC/CV parameters to its own struct. Tested-by: Randy Dunlap Acked-by: Randy Dunlap Reviewed-by: Matti Vaittinen Signed-off-by: Linus Walleij Signed-off-by: Sebastian Reichel --- include/linux/power_supply.h | 215 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 192 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 2d1318fe2455..f6e94eae4f28 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -343,37 +343,206 @@ struct power_supply_resistance_temp_table { #define POWER_SUPPLY_OCV_TEMP_MAX 20 -/* +/** + * struct power_supply_battery_info - information about batteries + * @technology: from the POWER_SUPPLY_TECHNOLOGY_* enum + * @energy_full_design_uwh: energy content when fully charged in microwatt + * hours + * @charge_full_design_uah: charge content when fully charged in microampere + * hours + * @voltage_min_design_uv: minimum voltage across the poles when the battery + * is at minimum voltage level in microvolts. If the voltage drops below this + * level the battery will need precharging when using CC/CV charging. + * @voltage_max_design_uv: voltage across the poles when the battery is fully + * charged in microvolts. This is the "nominal voltage" i.e. the voltage + * printed on the label of the battery. + * @tricklecharge_current_ua: the tricklecharge current used when trickle + * charging the battery in microamperes. This is the charging phase when the + * battery is completely empty and we need to carefully trickle in some + * charge until we reach the precharging voltage. + * @precharge_current_ua: current to use in the precharge phase in microamperes, + * the precharge rate is limited by limiting the current to this value. + * @precharge_voltage_max_uv: the maximum voltage allowed when precharging in + * microvolts. When we pass this voltage we will nominally switch over to the + * CC (constant current) charging phase defined by constant_charge_current_ua + * and constant_charge_voltage_max_uv. + * @charge_term_current_ua: when the current in the CV (constant voltage) + * charging phase drops below this value in microamperes the charging will + * terminate completely and not restart until the voltage over the battery + * poles reach charge_restart_voltage_uv unless we use maintenance charging. + * @charge_restart_voltage_uv: when the battery has been fully charged by + * CC/CV charging and charging has been disabled, and the voltage subsequently + * drops below this value in microvolts, the charging will be restarted + * (typically using CV charging). + * @overvoltage_limit_uv: If the voltage exceeds the nominal voltage + * voltage_max_design_uv and we reach this voltage level, all charging must + * stop and emergency procedures take place, such as shutting down the system + * in some cases. + * @constant_charge_current_max_ua: current in microamperes to use in the CC + * (constant current) charging phase. The charging rate is limited + * by this current. This is the main charging phase and as the current is + * constant into the battery the voltage slowly ascends to + * constant_charge_voltage_max_uv. + * @constant_charge_voltage_max_uv: voltage in microvolts signifying the end of + * the CC (constant current) charging phase and the beginning of the CV + * (constant voltage) charging phase. + * @factory_internal_resistance_uohm: the internal resistance of the battery + * at fabrication time, expressed in microohms. This resistance will vary + * depending on the lifetime and charge of the battery, so this is just a + * nominal ballpark figure. + * @ocv_temp: array indicating the open circuit voltage (OCV) capacity + * temperature indices. This is an array of temperatures in degrees Celsius + * indicating which capacity table to use for a certain temperature, since + * the capacity for reasons of chemistry will be different at different + * temperatures. Determining capacity is a multivariate problem and the + * temperature is the first variable we determine. + * @temp_ambient_alert_min: the battery will go outside of operating conditions + * when the ambient temperature goes below this temperature in degrees + * Celsius. + * @temp_ambient_alert_max: the battery will go outside of operating conditions + * when the ambient temperature goes above this temperature in degrees + * Celsius. + * @temp_alert_min: the battery should issue an alert if the internal + * temperature goes below this temperature in degrees Celsius. + * @temp_alert_max: the battery should issue an alert if the internal + * temperature goes above this temperature in degrees Celsius. + * @temp_min: the battery will go outside of operating conditions when + * the internal temperature goes below this temperature in degrees Celsius. + * Normally this means the system should shut down. + * @temp_max: the battery will go outside of operating conditions when + * the internal temperature goes above this temperature in degrees Celsius. + * Normally this means the system should shut down. + * @ocv_table: for each entry in ocv_temp there is a corresponding entry in + * ocv_table and a size for each entry in ocv_table_size. These arrays + * determine the capacity in percent in relation to the voltage in microvolts + * at the indexed temperature. + * @ocv_table_size: for each entry in ocv_temp this array is giving the size of + * each entry in the array of capacity arrays in ocv_table. + * @resist_table: this is a table that correlates a battery temperature to the + * expected internal resistance at this temperature. The resistance is given + * as a percentage of factory_internal_resistance_uohm. Knowing the + * resistance of the battery is usually necessary for calculating the open + * circuit voltage (OCV) that is then used with the ocv_table to calculate + * the capacity of the battery. The resist_table must be ordered descending + * by temperature: highest temperature with lowest resistance first, lowest + * temperature with highest resistance last. + * @resist_table_size: the number of items in the resist_table. + * * This is the recommended struct to manage static battery parameters, * populated by power_supply_get_battery_info(). Most platform drivers should * use these for consistency. + * * Its field names must correspond to elements in enum power_supply_property. * The default field value is -EINVAL. - * Power supply class itself doesn't use this. + * + * The charging parameters here assume a CC/CV charging scheme. This method + * is most common with Lithium Ion batteries (other methods are possible) and + * looks as follows: + * + * ^ Battery voltage + * | --- overvoltage_limit_uv + * | + * | ................................................... + * | .. constant_charge_voltage_max_uv + * | .. + * | . + * | . + * | . + * | . + * | . + * | .. precharge_voltage_max_uv + * | .. + * |. (trickle charging) + * +------------------------------------------------------------------> time + * + * ^ Current into the battery + * | + * | ............. constant_charge_current_max_ua + * | . . + * | . . + * | . . + * | . . + * | . .. + * | . .... + * | . ..... + * | ... precharge_current_ua ....... charge_term_current_ua + * | . . + * | . . + * |.... tricklecharge_current_ua . + * | . + * +-----------------------------------------------------------------> time + * + * These diagrams are synchronized on time and the voltage and current + * follow each other. + * + * With CC/CV charging commence over time like this for an empty battery: + * + * 1. When the battery is completely empty it may need to be charged with + * an especially small current so that electrons just "trickle in", + * this is the tricklecharge_current_ua. + * + * 2. Next a small initial pre-charge current (precharge_current_ua) + * is applied if the voltage is below precharge_voltage_max_uv until we + * reach precharge_voltage_max_uv. CAUTION: in some texts this is referred + * to as "trickle charging" but the use in the Linux kernel is different + * see below! + * + * 3. Then the main charging current is applied, which is called the constant + * current (CC) phase. A current regulator is set up to allow + * constant_charge_current_max_ua of current to flow into the battery. + * The chemical reaction in the battery will make the voltage go up as + * charge goes into the battery. This current is applied until we reach + * the constant_charge_voltage_max_uv voltage. + * + * 4. At this voltage we switch over to the constant voltage (CV) phase. This + * means we allow current to go into the battery, but we keep the voltage + * fixed. This current will continue to charge the battery while keeping + * the voltage the same. A chemical reaction in the battery goes on + * storing energy without affecting the voltage. Over time the current + * will slowly drop and when we reach charge_term_current_ua we will + * end the constant voltage phase. + * + * After this the battery is fully charged, and if we do not support maintenance + * charging, the charging will not restart until power dissipation makes the + * voltage fall so that we reach charge_restart_voltage_uv and at this point + * we restart charging at the appropriate phase, usually this will be inside + * the CV phase. + * + * If we support maintenance charging the voltage is however kept high after + * the CV phase with a very low current. This is meant to let the same charge + * go in for usage while the charger is still connected, mainly for + * dissipation for the power consuming entity while connected to the + * charger. + * + * All charging MUST terminate if the overvoltage_limit_uv is ever reached. + * Overcharging Lithium Ion cells can be DANGEROUS and lead to fire or + * explosions. + * + * The power supply class itself doesn't use this struct as of now. */ struct power_supply_battery_info { - unsigned int technology; /* from the enum above */ - int energy_full_design_uwh; /* microWatt-hours */ - int charge_full_design_uah; /* microAmp-hours */ - int voltage_min_design_uv; /* microVolts */ - int voltage_max_design_uv; /* microVolts */ - int tricklecharge_current_ua; /* microAmps */ - int precharge_current_ua; /* microAmps */ - int precharge_voltage_max_uv; /* microVolts */ - int charge_term_current_ua; /* microAmps */ - int charge_restart_voltage_uv; /* microVolts */ - int overvoltage_limit_uv; /* microVolts */ - int constant_charge_current_max_ua; /* microAmps */ - int constant_charge_voltage_max_uv; /* microVolts */ - int factory_internal_resistance_uohm; /* microOhms */ - int ocv_temp[POWER_SUPPLY_OCV_TEMP_MAX];/* celsius */ - int temp_ambient_alert_min; /* celsius */ - int temp_ambient_alert_max; /* celsius */ - int temp_alert_min; /* celsius */ - int temp_alert_max; /* celsius */ - int temp_min; /* celsius */ - int temp_max; /* celsius */ + unsigned int technology; + int energy_full_design_uwh; + int charge_full_design_uah; + int voltage_min_design_uv; + int voltage_max_design_uv; + int tricklecharge_current_ua; + int precharge_current_ua; + int precharge_voltage_max_uv; + int charge_term_current_ua; + int charge_restart_voltage_uv; + int overvoltage_limit_uv; + int constant_charge_current_max_ua; + int constant_charge_voltage_max_uv; + int factory_internal_resistance_uohm; + int ocv_temp[POWER_SUPPLY_OCV_TEMP_MAX]; + int temp_ambient_alert_min; + int temp_ambient_alert_max; + int temp_alert_min; + int temp_alert_max; + int temp_min; + int temp_max; struct power_supply_battery_ocv_table *ocv_table[POWER_SUPPLY_OCV_TEMP_MAX]; int ocv_table_size[POWER_SUPPLY_OCV_TEMP_MAX]; struct power_supply_resistance_temp_table *resist_table; -- cgit v1.2.3-59-g8ed1b From 6bb835f3d00467c9a5e35f4955afa29df96a404e Mon Sep 17 00:00:00 2001 From: Andriy Tryshnivskyy Date: Sun, 24 Oct 2021 12:16:26 +0300 Subject: iio: core: Introduce IIO_VAL_INT_64. Introduce IIO_VAL_INT_64 to read 64-bit value for channel attribute. Val is used as lower 32 bits. Signed-off-by: Andriy Tryshnivskyy Link: https://lore.kernel.org/r/20211024091627.28031-2-andriy.tryshnivskyy@opensynergy.com Signed-off-by: Jonathan Cameron --- drivers/iio/industrialio-core.c | 3 +++ include/linux/iio/types.h | 1 + 2 files changed, 4 insertions(+) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 463a63d5bf56..d94d26b11473 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -702,6 +702,9 @@ static ssize_t __iio_format_value(char *buf, size_t offset, unsigned int type, } case IIO_VAL_CHAR: return sysfs_emit_at(buf, offset, "%c", (char)vals[0]); + case IIO_VAL_INT_64: + tmp2 = (s64)((((u64)vals[1]) << 32) | (u32)vals[0]); + return sysfs_emit_at(buf, offset, "%lld", tmp2); default: return 0; } diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h index 84b3f8175cc6..a7aa91f3a8dc 100644 --- a/include/linux/iio/types.h +++ b/include/linux/iio/types.h @@ -24,6 +24,7 @@ enum iio_event_info { #define IIO_VAL_INT_PLUS_NANO 3 #define IIO_VAL_INT_PLUS_MICRO_DB 4 #define IIO_VAL_INT_MULTIPLE 5 +#define IIO_VAL_INT_64 6 /* 64-bit data, val is lower 32 bits */ #define IIO_VAL_FRACTIONAL 10 #define IIO_VAL_FRACTIONAL_LOG2 11 #define IIO_VAL_CHAR 12 -- cgit v1.2.3-59-g8ed1b From 2925748eadc33cba3bded7b69475a1b002b124ac Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 17 Nov 2021 13:22:53 +0000 Subject: firmware: cs_dsp: Add version checks on coefficient loading The firmware coefficient files contain version information that is currently ignored by the cs_dsp code. This information specifies which version of the firmware the coefficient were generated for. Add a check into the code which prints a warning in the case the coefficient and firmware differ in version, in many cases this will be ok but it is not always, so best to let the user know there is a potential issue. Co-authored-by: Simon Trimmer Signed-off-by: Simon Trimmer Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20211117132300.1290-3-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 49 +++++++++++++++++++++++++--------- include/linux/firmware/cirrus/cs_dsp.h | 2 ++ 2 files changed, 38 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 0d1ba7d8efa4..0da454a8498d 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -1569,7 +1569,7 @@ EXPORT_SYMBOL_GPL(cs_dsp_find_alg_region); static struct cs_dsp_alg_region *cs_dsp_create_region(struct cs_dsp *dsp, int type, __be32 id, - __be32 base) + __be32 ver, __be32 base) { struct cs_dsp_alg_region *alg_region; @@ -1579,6 +1579,7 @@ static struct cs_dsp_alg_region *cs_dsp_create_region(struct cs_dsp *dsp, alg_region->type = type; alg_region->alg = be32_to_cpu(id); + alg_region->ver = be32_to_cpu(ver); alg_region->base = be32_to_cpu(base); list_add_tail(&alg_region->list, &dsp->alg_regions); @@ -1628,14 +1629,14 @@ static void cs_dsp_parse_wmfw_v3_id_header(struct cs_dsp *dsp, nalgs); } -static int cs_dsp_create_regions(struct cs_dsp *dsp, __be32 id, int nregions, - const int *type, __be32 *base) +static int cs_dsp_create_regions(struct cs_dsp *dsp, __be32 id, __be32 ver, + int nregions, const int *type, __be32 *base) { struct cs_dsp_alg_region *alg_region; int i; for (i = 0; i < nregions; i++) { - alg_region = cs_dsp_create_region(dsp, type[i], id, base[i]); + alg_region = cs_dsp_create_region(dsp, type[i], id, ver, base[i]); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); } @@ -1670,12 +1671,14 @@ static int cs_dsp_adsp1_setup_algs(struct cs_dsp *dsp) cs_dsp_parse_wmfw_id_header(dsp, &adsp1_id.fw, n_algs); alg_region = cs_dsp_create_region(dsp, WMFW_ADSP1_ZM, - adsp1_id.fw.id, adsp1_id.zm); + adsp1_id.fw.id, adsp1_id.fw.ver, + adsp1_id.zm); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); alg_region = cs_dsp_create_region(dsp, WMFW_ADSP1_DM, - adsp1_id.fw.id, adsp1_id.dm); + adsp1_id.fw.id, adsp1_id.fw.ver, + adsp1_id.dm); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); @@ -1698,6 +1701,7 @@ static int cs_dsp_adsp1_setup_algs(struct cs_dsp *dsp) alg_region = cs_dsp_create_region(dsp, WMFW_ADSP1_DM, adsp1_alg[i].alg.id, + adsp1_alg[i].alg.ver, adsp1_alg[i].dm); if (IS_ERR(alg_region)) { ret = PTR_ERR(alg_region); @@ -1719,6 +1723,7 @@ static int cs_dsp_adsp1_setup_algs(struct cs_dsp *dsp) alg_region = cs_dsp_create_region(dsp, WMFW_ADSP1_ZM, adsp1_alg[i].alg.id, + adsp1_alg[i].alg.ver, adsp1_alg[i].zm); if (IS_ERR(alg_region)) { ret = PTR_ERR(alg_region); @@ -1771,17 +1776,20 @@ static int cs_dsp_adsp2_setup_algs(struct cs_dsp *dsp) cs_dsp_parse_wmfw_id_header(dsp, &adsp2_id.fw, n_algs); alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_XM, - adsp2_id.fw.id, adsp2_id.xm); + adsp2_id.fw.id, adsp2_id.fw.ver, + adsp2_id.xm); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_YM, - adsp2_id.fw.id, adsp2_id.ym); + adsp2_id.fw.id, adsp2_id.fw.ver, + adsp2_id.ym); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_ZM, - adsp2_id.fw.id, adsp2_id.zm); + adsp2_id.fw.id, adsp2_id.fw.ver, + adsp2_id.zm); if (IS_ERR(alg_region)) return PTR_ERR(alg_region); @@ -1806,6 +1814,7 @@ static int cs_dsp_adsp2_setup_algs(struct cs_dsp *dsp) alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_XM, adsp2_alg[i].alg.id, + adsp2_alg[i].alg.ver, adsp2_alg[i].xm); if (IS_ERR(alg_region)) { ret = PTR_ERR(alg_region); @@ -1827,6 +1836,7 @@ static int cs_dsp_adsp2_setup_algs(struct cs_dsp *dsp) alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_YM, adsp2_alg[i].alg.id, + adsp2_alg[i].alg.ver, adsp2_alg[i].ym); if (IS_ERR(alg_region)) { ret = PTR_ERR(alg_region); @@ -1848,6 +1858,7 @@ static int cs_dsp_adsp2_setup_algs(struct cs_dsp *dsp) alg_region = cs_dsp_create_region(dsp, WMFW_ADSP2_ZM, adsp2_alg[i].alg.id, + adsp2_alg[i].alg.ver, adsp2_alg[i].zm); if (IS_ERR(alg_region)) { ret = PTR_ERR(alg_region); @@ -1873,7 +1884,7 @@ out: return ret; } -static int cs_dsp_halo_create_regions(struct cs_dsp *dsp, __be32 id, +static int cs_dsp_halo_create_regions(struct cs_dsp *dsp, __be32 id, __be32 ver, __be32 xm_base, __be32 ym_base) { static const int types[] = { @@ -1882,7 +1893,7 @@ static int cs_dsp_halo_create_regions(struct cs_dsp *dsp, __be32 id, }; __be32 bases[] = { xm_base, xm_base, ym_base, ym_base }; - return cs_dsp_create_regions(dsp, id, ARRAY_SIZE(types), types, bases); + return cs_dsp_create_regions(dsp, id, ver, ARRAY_SIZE(types), types, bases); } static int cs_dsp_halo_setup_algs(struct cs_dsp *dsp) @@ -1910,7 +1921,7 @@ static int cs_dsp_halo_setup_algs(struct cs_dsp *dsp) cs_dsp_parse_wmfw_v3_id_header(dsp, &halo_id.fw, n_algs); - ret = cs_dsp_halo_create_regions(dsp, halo_id.fw.id, + ret = cs_dsp_halo_create_regions(dsp, halo_id.fw.id, halo_id.fw.ver, halo_id.xm_base, halo_id.ym_base); if (ret) return ret; @@ -1934,6 +1945,7 @@ static int cs_dsp_halo_setup_algs(struct cs_dsp *dsp) be32_to_cpu(halo_alg[i].ym_base)); ret = cs_dsp_halo_create_regions(dsp, halo_alg[i].alg.id, + halo_alg[i].alg.ver, halo_alg[i].xm_base, halo_alg[i].ym_base); if (ret) @@ -1955,7 +1967,7 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware const struct cs_dsp_region *mem; struct cs_dsp_alg_region *alg_region; const char *region_name; - int ret, pos, blocks, type, offset, reg; + int ret, pos, blocks, type, offset, reg, version; struct cs_dsp_buf *buf; if (!firmware) @@ -1999,6 +2011,7 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware type = le16_to_cpu(blk->type); offset = le16_to_cpu(blk->offset); + version = le32_to_cpu(blk->ver) >> 8; cs_dsp_dbg(dsp, "%s.%d: %x v%d.%d.%d\n", file, blocks, le32_to_cpu(blk->id), @@ -2056,6 +2069,16 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware alg_region = cs_dsp_find_alg_region(dsp, type, le32_to_cpu(blk->id)); if (alg_region) { + if (version != alg_region->ver) + cs_dsp_warn(dsp, + "Algorithm coefficient version %d.%d.%d but expected %d.%d.%d\n", + (version >> 16) & 0xFF, + (version >> 8) & 0xFF, + version & 0xFF, + (alg_region->ver >> 16) & 0xFF, + (alg_region->ver >> 8) & 0xFF, + alg_region->ver & 0xFF); + reg = alg_region->base; reg = dsp->ops->region_to_reg(mem, reg); reg += offset; diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 3a54b1afc48f..ce54705e2bec 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -54,12 +54,14 @@ struct cs_dsp_region { * struct cs_dsp_alg_region - Describes a logical algorithm region in DSP address space * @list: List node for internal use * @alg: Algorithm id + * @ver: Expected algorithm version * @type: Memory region type * @base: Address of region */ struct cs_dsp_alg_region { struct list_head list; unsigned int alg; + unsigned int ver; int type; unsigned int base; }; -- cgit v1.2.3-59-g8ed1b From 14055b5a3a23204c4702ae5d3f2a819ee081ce33 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 17 Nov 2021 13:22:54 +0000 Subject: firmware: cs_dsp: Add pre_run callback The code already has a post_run callback, add a matching pre_run callback to the client_ops that is called before execution is started. This callback provides a convenient place for the client code to set DSP controls or hardware that requires configuration before the DSP core actually starts execution. Note that placing this callback before cs_dsp_coeff_sync_controls is important to ensure that any control values are then correctly synced out to the chip. Co-authored-by: Simon Trimmer Signed-off-by: Simon Trimmer Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20211117132300.1290-4-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 6 ++++++ include/linux/firmware/cirrus/cs_dsp.h | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 0da454a8498d..ef7afadea42d 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -2627,6 +2627,12 @@ int cs_dsp_run(struct cs_dsp *dsp) goto err; } + if (dsp->client_ops->pre_run) { + ret = dsp->client_ops->pre_run(dsp); + if (ret) + goto err; + } + /* Sync set controls */ ret = cs_dsp_coeff_sync_controls(dsp); if (ret != 0) diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index ce54705e2bec..0bf849baeaa5 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -187,7 +187,8 @@ struct cs_dsp { * struct cs_dsp_client_ops - client callbacks * @control_add: Called under the pwr_lock when a control is created * @control_remove: Called under the pwr_lock when a control is destroyed - * @post_run: Called under the pwr_lock by cs_dsp_run() + * @pre_run: Called under the pwr_lock by cs_dsp_run() before the core is started + * @post_run: Called under the pwr_lock by cs_dsp_run() after the core is started * @post_stop: Called under the pwr_lock by cs_dsp_stop() * @watchdog_expired: Called when a watchdog expiry is detected * @@ -197,6 +198,7 @@ struct cs_dsp { struct cs_dsp_client_ops { int (*control_add)(struct cs_dsp_coeff_ctl *ctl); void (*control_remove)(struct cs_dsp_coeff_ctl *ctl); + int (*pre_run)(struct cs_dsp *dsp); int (*post_run)(struct cs_dsp *dsp); void (*post_stop)(struct cs_dsp *dsp); void (*watchdog_expired)(struct cs_dsp *dsp); -- cgit v1.2.3-59-g8ed1b From b329b3d39497a9fdb175d7e4fd77ae7170d5d26c Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 17 Nov 2021 13:22:58 +0000 Subject: firmware: cs_dsp: Clarify some kernel doc comments Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20211117132300.1290-8-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 4 ++-- include/linux/firmware/cirrus/cs_dsp.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 9eecd1626537..d1bcade2efe2 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -746,7 +746,7 @@ static int cs_dsp_coeff_write_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, * cs_dsp_coeff_write_ctrl() - Writes the given buffer to the given coefficient control * @ctl: pointer to coefficient control * @buf: the buffer to write to the given control - * @len: the length of the buffer + * @len: the length of the buffer in bytes * * Must be called with pwr_lock held. * @@ -808,7 +808,7 @@ static int cs_dsp_coeff_read_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, void *buf, s * cs_dsp_coeff_read_ctrl() - Reads the given coefficient control into the given buffer * @ctl: pointer to coefficient control * @buf: the buffer to store to the given control - * @len: the length of the buffer + * @len: the length of the buffer in bytes * * Must be called with pwr_lock held. * diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 0bf849baeaa5..1ad1b173417a 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -76,8 +76,8 @@ struct cs_dsp_alg_region { * @enabled: Flag indicating whether control is enabled * @list: List node for internal use * @cache: Cached value of the control - * @offset: Offset of control within alg_region - * @len: Length of the cached value + * @offset: Offset of control within alg_region in words + * @len: Length of the cached value in bytes * @set: Flag indicating the value has been written by the user * @flags: Bitfield of WMFW_CTL_FLAG_ control flags defined in wmfw.h * @type: One of the WMFW_CTL_TYPE_ control types defined in wmfw.h -- cgit v1.2.3-59-g8ed1b From f444da38ac924748de696c393327a44c4b8d727e Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 17 Nov 2021 13:22:59 +0000 Subject: firmware: cs_dsp: Add offset to cs_dsp read/write Provide a mechanism to access only part of a control through the cs_dsp interface. Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20211117132300.1290-9-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 44 +++++++++++++++++++++------------- include/linux/firmware/cirrus/cs_dsp.h | 6 +++-- sound/soc/codecs/wm_adsp.c | 14 +++++------ 3 files changed, 39 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index d1bcade2efe2..5fe08de91ecd 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -616,7 +616,8 @@ static void cs_dsp_halo_show_fw_status(struct cs_dsp *dsp) offs[0], offs[1], offs[2], offs[3]); } -static int cs_dsp_coeff_base_reg(struct cs_dsp_coeff_ctl *ctl, unsigned int *reg) +static int cs_dsp_coeff_base_reg(struct cs_dsp_coeff_ctl *ctl, unsigned int *reg, + unsigned int off) { const struct cs_dsp_alg_region *alg_region = &ctl->alg_region; struct cs_dsp *dsp = ctl->dsp; @@ -629,7 +630,7 @@ static int cs_dsp_coeff_base_reg(struct cs_dsp_coeff_ctl *ctl, unsigned int *reg return -EINVAL; } - *reg = dsp->ops->region_to_reg(mem, ctl->alg_region.base + ctl->offset); + *reg = dsp->ops->region_to_reg(mem, ctl->alg_region.base + ctl->offset + off); return 0; } @@ -658,7 +659,7 @@ int cs_dsp_coeff_write_acked_control(struct cs_dsp_coeff_ctl *ctl, unsigned int if (!dsp->running) return -EPERM; - ret = cs_dsp_coeff_base_reg(ctl, ®); + ret = cs_dsp_coeff_base_reg(ctl, ®, 0); if (ret) return ret; @@ -712,14 +713,14 @@ int cs_dsp_coeff_write_acked_control(struct cs_dsp_coeff_ctl *ctl, unsigned int EXPORT_SYMBOL_GPL(cs_dsp_coeff_write_acked_control); static int cs_dsp_coeff_write_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, - const void *buf, size_t len) + unsigned int off, const void *buf, size_t len) { struct cs_dsp *dsp = ctl->dsp; void *scratch; int ret; unsigned int reg; - ret = cs_dsp_coeff_base_reg(ctl, ®); + ret = cs_dsp_coeff_base_reg(ctl, ®, off); if (ret) return ret; @@ -745,6 +746,7 @@ static int cs_dsp_coeff_write_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, /** * cs_dsp_coeff_write_ctrl() - Writes the given buffer to the given coefficient control * @ctl: pointer to coefficient control + * @off: word offset at which data should be written * @buf: the buffer to write to the given control * @len: the length of the buffer in bytes * @@ -752,7 +754,8 @@ static int cs_dsp_coeff_write_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, * * Return: Zero for success, a negative number on error. */ -int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, const void *buf, size_t len) +int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, + unsigned int off, const void *buf, size_t len) { int ret = 0; @@ -761,27 +764,31 @@ int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, const void *buf, size_ if (!ctl) return -ENOENT; + if (len + off * sizeof(u32) > ctl->len) + return -EINVAL; + if (ctl->flags & WMFW_CTL_FLAG_VOLATILE) ret = -EPERM; else if (buf != ctl->cache) - memcpy(ctl->cache, buf, len); + memcpy(ctl->cache + off * sizeof(u32), buf, len); ctl->set = 1; if (ctl->enabled && ctl->dsp->running) - ret = cs_dsp_coeff_write_ctrl_raw(ctl, buf, len); + ret = cs_dsp_coeff_write_ctrl_raw(ctl, off, buf, len); return ret; } EXPORT_SYMBOL_GPL(cs_dsp_coeff_write_ctrl); -static int cs_dsp_coeff_read_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, void *buf, size_t len) +static int cs_dsp_coeff_read_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, + unsigned int off, void *buf, size_t len) { struct cs_dsp *dsp = ctl->dsp; void *scratch; int ret; unsigned int reg; - ret = cs_dsp_coeff_base_reg(ctl, ®); + ret = cs_dsp_coeff_base_reg(ctl, ®, off); if (ret) return ret; @@ -807,6 +814,7 @@ static int cs_dsp_coeff_read_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, void *buf, s /** * cs_dsp_coeff_read_ctrl() - Reads the given coefficient control into the given buffer * @ctl: pointer to coefficient control + * @off: word offset at which data should be read * @buf: the buffer to store to the given control * @len: the length of the buffer in bytes * @@ -814,7 +822,8 @@ static int cs_dsp_coeff_read_ctrl_raw(struct cs_dsp_coeff_ctl *ctl, void *buf, s * * Return: Zero for success, a negative number on error. */ -int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, void *buf, size_t len) +int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, + unsigned int off, void *buf, size_t len) { int ret = 0; @@ -823,17 +832,20 @@ int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, void *buf, size_t len) if (!ctl) return -ENOENT; + if (len + off * sizeof(u32) > ctl->len) + return -EINVAL; + if (ctl->flags & WMFW_CTL_FLAG_VOLATILE) { if (ctl->enabled && ctl->dsp->running) - return cs_dsp_coeff_read_ctrl_raw(ctl, buf, len); + return cs_dsp_coeff_read_ctrl_raw(ctl, off, buf, len); else return -EPERM; } else { if (!ctl->flags && ctl->enabled && ctl->dsp->running) - ret = cs_dsp_coeff_read_ctrl_raw(ctl, ctl->cache, ctl->len); + ret = cs_dsp_coeff_read_ctrl_raw(ctl, 0, ctl->cache, ctl->len); if (buf != ctl->cache) - memcpy(buf, ctl->cache, len); + memcpy(buf, ctl->cache + off * sizeof(u32), len); } return ret; @@ -857,7 +869,7 @@ static int cs_dsp_coeff_init_control_caches(struct cs_dsp *dsp) * created so we don't need to do anything. */ if (!ctl->flags || (ctl->flags & WMFW_CTL_FLAG_READABLE)) { - ret = cs_dsp_coeff_read_ctrl_raw(ctl, ctl->cache, ctl->len); + ret = cs_dsp_coeff_read_ctrl_raw(ctl, 0, ctl->cache, ctl->len); if (ret < 0) return ret; } @@ -875,7 +887,7 @@ static int cs_dsp_coeff_sync_controls(struct cs_dsp *dsp) if (!ctl->enabled) continue; if (ctl->set && !(ctl->flags & WMFW_CTL_FLAG_VOLATILE)) { - ret = cs_dsp_coeff_write_ctrl_raw(ctl, ctl->cache, + ret = cs_dsp_coeff_write_ctrl_raw(ctl, 0, ctl->cache, ctl->len); if (ret < 0) return ret; diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 1ad1b173417a..38b4da3ddfe4 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -232,8 +232,10 @@ void cs_dsp_init_debugfs(struct cs_dsp *dsp, struct dentry *debugfs_root); void cs_dsp_cleanup_debugfs(struct cs_dsp *dsp); int cs_dsp_coeff_write_acked_control(struct cs_dsp_coeff_ctl *ctl, unsigned int event_id); -int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, const void *buf, size_t len); -int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, void *buf, size_t len); +int cs_dsp_coeff_write_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off, + const void *buf, size_t len); +int cs_dsp_coeff_read_ctrl(struct cs_dsp_coeff_ctl *ctl, unsigned int off, + void *buf, size_t len); struct cs_dsp_coeff_ctl *cs_dsp_get_ctl(struct cs_dsp *dsp, const char *name, int type, unsigned int alg); diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c index 404717e30f44..f084b093cff6 100644 --- a/sound/soc/codecs/wm_adsp.c +++ b/sound/soc/codecs/wm_adsp.c @@ -401,7 +401,7 @@ static int wm_coeff_put(struct snd_kcontrol *kctl, int ret = 0; mutex_lock(&cs_ctl->dsp->pwr_lock); - ret = cs_dsp_coeff_write_ctrl(cs_ctl, p, cs_ctl->len); + ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, p, cs_ctl->len); mutex_unlock(&cs_ctl->dsp->pwr_lock); return ret; @@ -421,7 +421,7 @@ static int wm_coeff_tlv_put(struct snd_kcontrol *kctl, if (copy_from_user(cs_ctl->cache, bytes, size)) ret = -EFAULT; else - ret = cs_dsp_coeff_write_ctrl(cs_ctl, cs_ctl->cache, size); + ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, cs_ctl->cache, size); mutex_unlock(&cs_ctl->dsp->pwr_lock); @@ -464,7 +464,7 @@ static int wm_coeff_get(struct snd_kcontrol *kctl, int ret; mutex_lock(&cs_ctl->dsp->pwr_lock); - ret = cs_dsp_coeff_read_ctrl(cs_ctl, p, cs_ctl->len); + ret = cs_dsp_coeff_read_ctrl(cs_ctl, 0, p, cs_ctl->len); mutex_unlock(&cs_ctl->dsp->pwr_lock); return ret; @@ -481,7 +481,7 @@ static int wm_coeff_tlv_get(struct snd_kcontrol *kctl, mutex_lock(&cs_ctl->dsp->pwr_lock); - ret = cs_dsp_coeff_read_ctrl(cs_ctl, cs_ctl->cache, size); + ret = cs_dsp_coeff_read_ctrl(cs_ctl, 0, cs_ctl->cache, size); if (!ret && copy_to_user(bytes, cs_ctl->cache, size)) ret = -EFAULT; @@ -684,7 +684,7 @@ int wm_adsp_write_ctl(struct wm_adsp *dsp, const char *name, int type, if (len > cs_ctl->len) return -EINVAL; - ret = cs_dsp_coeff_write_ctrl(cs_ctl, buf, len); + ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, buf, len); if (ret) return ret; @@ -723,7 +723,7 @@ int wm_adsp_read_ctl(struct wm_adsp *dsp, const char *name, int type, if (len > cs_ctl->len) return -EINVAL; - return cs_dsp_coeff_read_ctrl(cs_ctl, buf, len); + return cs_dsp_coeff_read_ctrl(cs_ctl, 0, buf, len); } EXPORT_SYMBOL_GPL(wm_adsp_read_ctl); @@ -1432,7 +1432,7 @@ static int wm_adsp_buffer_parse_coeff(struct cs_dsp_coeff_ctl *cs_ctl) int ret, i; for (i = 0; i < 5; ++i) { - ret = cs_dsp_coeff_read_ctrl(cs_ctl, &coeff_v1, sizeof(coeff_v1)); + ret = cs_dsp_coeff_read_ctrl(cs_ctl, 0, &coeff_v1, sizeof(coeff_v1)); if (ret < 0) return ret; -- cgit v1.2.3-59-g8ed1b From 5c903f64ce97172d63f7591cfa9e37cba58867b2 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 17 Nov 2021 13:23:00 +0000 Subject: firmware: cs_dsp: Allow creation of event controls Some firmwares contain controls intended to convey firmware state back to the host. Whilst more infrastructure will probably be needed for these in time, as a first step allow creation of the controls, so said firmwares arn't completely rejected. Signed-off-by: Charles Keepax Link: https://lore.kernel.org/r/20211117132300.1290-10-ckeepax@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/firmware/cirrus/cs_dsp.c | 1 + include/linux/firmware/cirrus/wmfw.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c index 5fe08de91ecd..3814cbba0a54 100644 --- a/drivers/firmware/cirrus/cs_dsp.c +++ b/drivers/firmware/cirrus/cs_dsp.c @@ -1177,6 +1177,7 @@ static int cs_dsp_parse_coeff(struct cs_dsp *dsp, return -EINVAL; break; case WMFW_CTL_TYPE_HOSTEVENT: + case WMFW_CTL_TYPE_FWEVENT: ret = cs_dsp_check_coeff_flags(dsp, &coeff_blk, WMFW_CTL_FLAG_SYS | WMFW_CTL_FLAG_VOLATILE | diff --git a/include/linux/firmware/cirrus/wmfw.h b/include/linux/firmware/cirrus/wmfw.h index a19bf7c6fc8b..74e5a4f6c13a 100644 --- a/include/linux/firmware/cirrus/wmfw.h +++ b/include/linux/firmware/cirrus/wmfw.h @@ -29,6 +29,7 @@ #define WMFW_CTL_TYPE_ACKED 0x1000 /* acked control */ #define WMFW_CTL_TYPE_HOSTEVENT 0x1001 /* event control */ #define WMFW_CTL_TYPE_HOST_BUFFER 0x1002 /* host buffer pointer */ +#define WMFW_CTL_TYPE_FWEVENT 0x1004 /* firmware event control */ struct wmfw_header { char magic[4]; -- cgit v1.2.3-59-g8ed1b From df6160deb3debe6f964c16349f9431157ff67dda Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 17 Nov 2021 17:57:29 -0800 Subject: tcp: add missing htmldocs for skb->ll_node and sk->defer_list Add missing entries to fix these "make htmldocs" warnings. ./include/linux/skbuff.h:953: warning: Function parameter or member 'll_node' not described in 'sk_buff' ./include/net/sock.h:540: warning: Function parameter or member 'defer_list' not described in 'sock' Fixes: f35f821935d8 ("tcp: defer skb freeing after socket lock is released") Signed-off-by: Eric Dumazet Reported-by: Stephen Rothwell Signed-off-by: David S. Miller --- include/linux/skbuff.h | 1 + include/net/sock.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b8b806512e16..100fd604fbc9 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -627,6 +627,7 @@ typedef unsigned char *sk_buff_data_t; * for retransmit timer * @rbnode: RB tree node, alternative to next/prev for netem/tcp * @list: queue head + * @ll_node: anchor in an llist (eg socket defer_list) * @sk: Socket we are owned by * @ip_defrag_offset: (aka @sk) alternate use of @sk, used in * fragmentation management diff --git a/include/net/sock.h b/include/net/sock.h index f09c0c4736c4..a79fc772324e 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -292,6 +292,7 @@ struct bpf_local_storage; * @sk_pacing_shift: scaling factor for TCP Small Queues * @sk_lingertime: %SO_LINGER l_linger setting * @sk_backlog: always used with the per-socket spinlock held + * @defer_list: head of llist storing skbs to be freed * @sk_callback_lock: used with the callbacks in the end of this struct * @sk_error_queue: rarely used * @sk_prot_creator: sk_prot of original sock creator (see ipv6_setsockopt, -- cgit v1.2.3-59-g8ed1b From 8b6e88555971eac384b89fb0bd6c72ee4e1e6a6a Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Thu, 18 Nov 2021 13:48:47 +0200 Subject: regulator: rohm-regulator: add helper for restricted voltage setting Few ROHM PMICs have regulators where voltage setting can be done only when regulator is disabled. Add helper for those PMICs. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/6f51871e9fea611d133b5dd2560f4a7ee1ede9cd.1637233864.git.matti.vaittinen@fi.rohmeurope.com Signed-off-by: Mark Brown --- drivers/regulator/rohm-regulator.c | 16 ++++++++++++++++ include/linux/mfd/rohm-generic.h | 7 +++++++ 2 files changed, 23 insertions(+) (limited to 'include/linux') diff --git a/drivers/regulator/rohm-regulator.c b/drivers/regulator/rohm-regulator.c index 6e0d9c08ec1c..f97a9a51ee76 100644 --- a/drivers/regulator/rohm-regulator.c +++ b/drivers/regulator/rohm-regulator.c @@ -112,6 +112,22 @@ int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, } EXPORT_SYMBOL(rohm_regulator_set_dvs_levels); +/* + * Few ROHM PMIC ICs have constrains on voltage changing: + * BD71837 - only buck 1-4 voltages can be changed when they are enabled. + * Other bucks and all LDOs must be disabled when voltage is changed. + * BD96801 - LDO voltage levels can be changed when LDOs are disabled. + */ +int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, + unsigned int sel) +{ + if (rdev->desc->ops->is_enabled(rdev)) + return -EBUSY; + + return regulator_set_voltage_sel_regmap(rdev, sel); +} +EXPORT_SYMBOL_GPL(rohm_regulator_set_voltage_sel_restricted); + MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Matti Vaittinen "); MODULE_DESCRIPTION("Generic helpers for ROHM PMIC regulator drivers"); diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h index 35b392a0d73a..35c5866f48b7 100644 --- a/include/linux/mfd/rohm-generic.h +++ b/include/linux/mfd/rohm-generic.h @@ -80,6 +80,8 @@ int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, const struct regulator_desc *desc, struct regmap *regmap); +int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, + unsigned int sel); #else static inline int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, struct device_node *np, @@ -88,6 +90,11 @@ static inline int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dv { return 0; } +static int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, + unsigned int sel) +{ + return 0; +} #endif #endif -- cgit v1.2.3-59-g8ed1b From 92b1348277f8893671e5354adde64fe3cf462821 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Thu, 18 Nov 2021 13:49:30 +0200 Subject: regulator: Add units to limit documentation The documentation for limits used at protection level setting did not mention the units. Fix the units in documentation to match values passed in from device-tree (uV, uA, Kelvin) to avoid confusion. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/111114aca991e41e49a32f89b74e95285f07c1e3.1637233864.git.matti.vaittinen@fi.rohmeurope.com Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index bd7a73db2e66..1cb8071fee34 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -101,11 +101,13 @@ enum regulator_detection_severity { * is requested. * @set_over_voltage_protection: Support enabling of and setting limits for over * voltage situation detection. Detection can be configured for same - * severities as over current protection. + * severities as over current protection. Units of uV. * @set_under_voltage_protection: Support enabling of and setting limits for - * under situation detection. + * under voltage situation detection. Detection can be configured for same + * severities as over current protection. Units of uV. * @set_thermal_protection: Support enabling of and setting limits for over - * temperature situation detection. + * temperature situation detection.Detection can be configured for same + * severities as over current protection. Units of degree Kelvin. * * @set_active_discharge: Set active discharge enable/disable of regulators. * -- cgit v1.2.3-59-g8ed1b From 418e0a3551bbef5b221705b0e5b8412cdc0afd39 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 5 Nov 2021 14:42:24 +0200 Subject: lib/string_helpers: Introduce kasprintf_strarray() We have a few users already that basically want to have array of sequential strings to be allocated and filled. Provide a helper for them (basically adjusted version from gpio-mockup.c). Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij --- include/linux/string_helpers.h | 1 + lib/string_helpers.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/include/linux/string_helpers.h b/include/linux/string_helpers.h index 4ba39e1403b2..f67a94013c87 100644 --- a/include/linux/string_helpers.h +++ b/include/linux/string_helpers.h @@ -100,6 +100,7 @@ char *kstrdup_quotable(const char *src, gfp_t gfp); char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp); char *kstrdup_quotable_file(struct file *file, gfp_t gfp); +char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n); void kfree_strarray(char **array, size_t n); #endif diff --git a/lib/string_helpers.c b/lib/string_helpers.c index d5d008f5b1d9..9758997c465e 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -674,6 +674,39 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp) } EXPORT_SYMBOL_GPL(kstrdup_quotable_file); +/** + * kasprintf_strarray - allocate and fill array of sequential strings + * @gfp: flags for the slab allocator + * @prefix: prefix to be used + * @n: amount of lines to be allocated and filled + * + * Allocates and fills @n strings using pattern "%s-%zu", where prefix + * is provided by caller. The caller is responsible to free them with + * kfree_strarray() after use. + * + * Returns array of strings or NULL when memory can't be allocated. + */ +char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n) +{ + char **names; + size_t i; + + names = kcalloc(n + 1, sizeof(char *), gfp); + if (!names) + return NULL; + + for (i = 0; i < n; i++) { + names[i] = kasprintf(gfp, "%s-%zu", prefix, i); + if (!names[i]) { + kfree_strarray(names, i); + return NULL; + } + } + + return names; +} +EXPORT_SYMBOL_GPL(kasprintf_strarray); + /** * kfree_strarray - free a number of dynamically allocated strings contained * in an array and the array itself -- cgit v1.2.3-59-g8ed1b From acdb89b6c87a2d7b5c48a82756e6f5c6f599f60a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 5 Nov 2021 14:42:25 +0200 Subject: lib/string_helpers: Introduce managed variant of kasprintf_strarray() Some of the users want to have easy way to allocate array of strings that will be automatically cleaned when associated device is gone. Introduce managed variant of kasprintf_strarray() for such use cases. Signed-off-by: Andy Shevchenko --- include/linux/string_helpers.h | 3 +++ lib/string_helpers.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/include/linux/string_helpers.h b/include/linux/string_helpers.h index f67a94013c87..7a22921c9db7 100644 --- a/include/linux/string_helpers.h +++ b/include/linux/string_helpers.h @@ -7,6 +7,7 @@ #include #include +struct device; struct file; struct task_struct; @@ -103,4 +104,6 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp); char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n); void kfree_strarray(char **array, size_t n); +char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n); + #endif diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 9758997c465e..90f9f1b7afec 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -730,6 +731,36 @@ void kfree_strarray(char **array, size_t n) } EXPORT_SYMBOL_GPL(kfree_strarray); +struct strarray { + char **array; + size_t n; +}; + +static void devm_kfree_strarray(struct device *dev, void *res) +{ + struct strarray *array = res; + + kfree_strarray(array->array, array->n); +} + +char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) +{ + struct strarray *ptr; + + ptr = devres_alloc(devm_kfree_strarray, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n); + if (!ptr->array) { + devres_free(ptr); + return ERR_PTR(-ENOMEM); + } + + return ptr->array; +} +EXPORT_SYMBOL_GPL(devm_kasprintf_strarray); + /** * strscpy_pad() - Copy a C-string into a sized buffer * @dest: Where to copy the string to -- cgit v1.2.3-59-g8ed1b From 57bdeef4716689d9b0e3571034d65cf420f6efcd Mon Sep 17 00:00:00 2001 From: Naveen Naidu Date: Thu, 18 Nov 2021 19:33:11 +0530 Subject: PCI: Add PCI_ERROR_RESPONSE and related definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A config or MMIO read from a PCI device that doesn't exist or doesn't respond causes a PCI error. There's no real data to return to satisfy the CPU read, so most hardware fabricates ~0 data. Add a PCI_ERROR_RESPONSE definition for that and use it where appropriate to make these checks consistent and easier to find. Also add helper definitions PCI_SET_ERROR_RESPONSE() and PCI_POSSIBLE_ERROR() to make the code more readable. Suggested-by: Bjorn Helgaas Link: https://lore.kernel.org/r/55563bf4dfc5d3fdc96695373c659d099bf175b1.1637243717.git.naveennaidu479@gmail.com Signed-off-by: Naveen Naidu Signed-off-by: Bjorn Helgaas Reviewed-by: Pali Rohár --- include/linux/pci.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pci.h b/include/linux/pci.h index 18a75c8e615c..0ce26850470e 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -154,6 +154,15 @@ enum pci_interrupt_pin { /* The number of legacy PCI INTx interrupts */ #define PCI_NUM_INTX 4 +/* + * Reading from a device that doesn't respond typically returns ~0. A + * successful read from a device may also return ~0, so you need additional + * information to reliably identify errors. + */ +#define PCI_ERROR_RESPONSE (~0ULL) +#define PCI_SET_ERROR_RESPONSE(val) (*(val) = ((typeof(*(val))) PCI_ERROR_RESPONSE)) +#define PCI_POSSIBLE_ERROR(val) ((val) == ((typeof(val)) PCI_ERROR_RESPONSE)) + /* * pci_power_t values must match the bits in the Capabilities PME_Support * and Control/Status PowerState fields in the Power Management capability. -- cgit v1.2.3-59-g8ed1b From c4f5b30dda01f2f6979a9681142de454991182ee Mon Sep 17 00:00:00 2001 From: Biju Das Date: Fri, 12 Nov 2021 18:44:10 +0000 Subject: reset: Add of_reset_control_get_optional_exclusive() Add optional variant of of_reset_control_get_exclusive(). If the requested reset is not specified in the device tree, this function returns NULL instead of an error. Suggested-by: Philipp Zabel Signed-off-by: Biju Das Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20211112184413.4391-2-biju.das.jz@bp.renesas.com Signed-off-by: Philipp Zabel --- include/linux/reset.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/reset.h b/include/linux/reset.h index db0e6115a2f6..8a21b5756c3e 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -454,6 +454,26 @@ static inline struct reset_control *of_reset_control_get_exclusive( return __of_reset_control_get(node, id, 0, false, false, true); } +/** + * of_reset_control_get_optional_exclusive - Lookup and obtain an optional exclusive + * reference to a reset controller. + * @node: device to be reset by the controller + * @id: reset line name + * + * Optional variant of of_reset_control_get_exclusive(). If the requested reset + * is not specified in the device tree, this function returns NULL instead of + * an error. + * + * Returns a struct reset_control or IS_ERR() condition containing errno. + * + * Use of id names is optional. + */ +static inline struct reset_control *of_reset_control_get_optional_exclusive( + struct device_node *node, const char *id) +{ + return __of_reset_control_get(node, id, 0, false, true, true); +} + /** * of_reset_control_get_shared - Lookup and obtain a shared reference * to a reset controller. -- cgit v1.2.3-59-g8ed1b From d8466f73010faf71effb21228ae1cbf577dab130 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Sat, 16 Oct 2021 14:22:27 +0100 Subject: mtd: rawnand: Export nand_read_page_hwecc_oob_first() Move the function nand_read_page_hwecc_oob_first() (previously nand_davinci_read_page_hwecc_oob_first()) to nand_base.c, and export it as a GPL symbol, so that it can be used by more modules. Cc: # v5.2 Fixes: a0ac778eb82c ("mtd: rawnand: ingenic: Add support for the JZ4740") Signed-off-by: Paul Cercueil Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20211016132228.40254-4-paul@crapouillou.net --- drivers/mtd/nand/raw/davinci_nand.c | 69 +------------------------------------ drivers/mtd/nand/raw/nand_base.c | 67 +++++++++++++++++++++++++++++++++++ include/linux/mtd/rawnand.h | 2 ++ 3 files changed, 70 insertions(+), 68 deletions(-) (limited to 'include/linux') diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c index fe2511cdcae3..45fec8c192ab 100644 --- a/drivers/mtd/nand/raw/davinci_nand.c +++ b/drivers/mtd/nand/raw/davinci_nand.c @@ -371,73 +371,6 @@ correct: return corrected; } -/** - * nand_davinci_read_page_hwecc_oob_first - Hardware ECC page read with ECC - * data read from OOB area - * @chip: nand chip info structure - * @buf: buffer to store read data - * @oob_required: caller requires OOB data read to chip->oob_poi - * @page: page number to read - * - * Hardware ECC for large page chips, which requires the ECC data to be - * extracted from the OOB before the actual data is read. - */ -static int nand_davinci_read_page_hwecc_oob_first(struct nand_chip *chip, - uint8_t *buf, - int oob_required, int page) -{ - struct mtd_info *mtd = nand_to_mtd(chip); - int i, eccsize = chip->ecc.size, ret; - int eccbytes = chip->ecc.bytes; - int eccsteps = chip->ecc.steps; - uint8_t *p = buf; - uint8_t *ecc_code = chip->ecc.code_buf; - unsigned int max_bitflips = 0; - - /* Read the OOB area first */ - ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); - if (ret) - return ret; - - /* Move read cursor to start of page */ - ret = nand_change_read_column_op(chip, 0, NULL, 0, false); - if (ret) - return ret; - - ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, - chip->ecc.total); - if (ret) - return ret; - - for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { - int stat; - - chip->ecc.hwctl(chip, NAND_ECC_READ); - - ret = nand_read_data_op(chip, p, eccsize, false, false); - if (ret) - return ret; - - stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL); - if (stat == -EBADMSG && - (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { - /* check for empty pages with bitflips */ - stat = nand_check_erased_ecc_chunk(p, eccsize, - &ecc_code[i], - eccbytes, NULL, 0, - chip->ecc.strength); - } - - if (stat < 0) { - mtd->ecc_stats.failed++; - } else { - mtd->ecc_stats.corrected += stat; - max_bitflips = max_t(unsigned int, max_bitflips, stat); - } - } - return max_bitflips; -} - /*----------------------------------------------------------------------*/ /* An ECC layout for using 4-bit ECC with small-page flash, storing @@ -647,7 +580,7 @@ static int davinci_nand_attach_chip(struct nand_chip *chip) } else if (chunks == 4 || chunks == 8) { mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); - chip->ecc.read_page = nand_davinci_read_page_hwecc_oob_first; + chip->ecc.read_page = nand_read_page_hwecc_oob_first; } else { return -EIO; } diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 3d6c6e880520..113a2e9f43b1 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -3160,6 +3160,73 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf, return max_bitflips; } +/** + * nand_read_page_hwecc_oob_first - Hardware ECC page read with ECC + * data read from OOB area + * @chip: nand chip info structure + * @buf: buffer to store read data + * @oob_required: caller requires OOB data read to chip->oob_poi + * @page: page number to read + * + * Hardware ECC for large page chips, which requires the ECC data to be + * extracted from the OOB before the actual data is read. + */ +int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int i, eccsize = chip->ecc.size, ret; + int eccbytes = chip->ecc.bytes; + int eccsteps = chip->ecc.steps; + uint8_t *p = buf; + uint8_t *ecc_code = chip->ecc.code_buf; + unsigned int max_bitflips = 0; + + /* Read the OOB area first */ + ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); + if (ret) + return ret; + + /* Move read cursor to start of page */ + ret = nand_change_read_column_op(chip, 0, NULL, 0, false); + if (ret) + return ret; + + ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, + chip->ecc.total); + if (ret) + return ret; + + for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { + int stat; + + chip->ecc.hwctl(chip, NAND_ECC_READ); + + ret = nand_read_data_op(chip, p, eccsize, false, false); + if (ret) + return ret; + + stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL); + if (stat == -EBADMSG && + (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) { + /* check for empty pages with bitflips */ + stat = nand_check_erased_ecc_chunk(p, eccsize, + &ecc_code[i], + eccbytes, NULL, 0, + chip->ecc.strength); + } + + if (stat < 0) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += stat; + max_bitflips = max_t(unsigned int, max_bitflips, stat); + } + } + return max_bitflips; +} +EXPORT_SYMBOL_GPL(nand_read_page_hwecc_oob_first); + /** * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read * @chip: nand chip info structure diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h index b2f9dd3cbd69..5b88cd51fadb 100644 --- a/include/linux/mtd/rawnand.h +++ b/include/linux/mtd/rawnand.h @@ -1539,6 +1539,8 @@ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len, bool force_8bit, bool check_only); int nand_write_data_op(struct nand_chip *chip, const void *buf, unsigned int len, bool force_8bit); +int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf, + int oob_required, int page); /* Scan and identify a NAND device */ int nand_scan_with_ids(struct nand_chip *chip, unsigned int max_chips, -- cgit v1.2.3-59-g8ed1b From adeef3e32146a8d2a73c399dc6f5d76a449131b1 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 19 Nov 2021 06:21:51 -0800 Subject: net: constify netdev->dev_addr Commit 406f42fa0d3c ("net-next: When a bond have a massive amount of VLANs...") introduced a rbtree for faster Ethernet address look up. We converted all users to make modifications via appropriate helpers, make netdev->dev_addr const. The update helpers need to upcast from the buffer to struct netdev_hw_addr. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/linux/netdevice.h | 14 +++++--------- net/core/dev_addr_lists.c | 10 ++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 4f4a299e92de..2462195784a9 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2117,7 +2117,7 @@ struct net_device { * Cache lines mostly used on receive path (including eth_type_trans()) */ /* Interface address info used in eth_type_trans() */ - unsigned char *dev_addr; + const unsigned char *dev_addr; struct netdev_rx_queue *_rx; unsigned int num_rx_queues; @@ -4268,10 +4268,13 @@ void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list, void __hw_addr_init(struct netdev_hw_addr_list *list); /* Functions used for device addresses handling */ +void dev_addr_mod(struct net_device *dev, unsigned int offset, + const void *addr, size_t len); + static inline void __dev_addr_set(struct net_device *dev, const void *addr, size_t len) { - memcpy(dev->dev_addr, addr, len); + dev_addr_mod(dev, 0, addr, len); } static inline void dev_addr_set(struct net_device *dev, const u8 *addr) @@ -4279,13 +4282,6 @@ static inline void dev_addr_set(struct net_device *dev, const u8 *addr) __dev_addr_set(dev, addr, dev->addr_len); } -static inline void -dev_addr_mod(struct net_device *dev, unsigned int offset, - const void *addr, size_t len) -{ - memcpy(&dev->dev_addr[offset], addr, len); -} - int dev_addr_add(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); int dev_addr_del(struct net_device *dev, const unsigned char *addr, diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index f0cb38344126..ae8b1ef00fec 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -549,6 +549,16 @@ int dev_addr_init(struct net_device *dev) } EXPORT_SYMBOL(dev_addr_init); +void dev_addr_mod(struct net_device *dev, unsigned int offset, + const void *addr, size_t len) +{ + struct netdev_hw_addr *ha; + + ha = container_of(dev->dev_addr, struct netdev_hw_addr, addr[0]); + memcpy(&ha->addr[offset], addr, len); +} +EXPORT_SYMBOL(dev_addr_mod); + /** * dev_addr_add - Add a device address * @dev: device -- cgit v1.2.3-59-g8ed1b From d07b26f5bbea9ade34dfd6abea7b3ca056c03cd1 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 19 Nov 2021 06:21:53 -0800 Subject: dev_addr: add a modification check netdev->dev_addr should only be modified via helpers, but someone may be casting off the const. Add a runtime check to catch abuses. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/linux/netdevice.h | 5 +++++ net/core/dev.c | 1 + net/core/dev_addr_lists.c | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 2462195784a9..cb7f2661d187 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1942,6 +1942,8 @@ enum netdev_ml_priv_type { * @unlink_list: As netif_addr_lock() can be called recursively, * keep a list of interfaces to be deleted. * + * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. + * * FIXME: cleanup struct net_device such that network protocol info * moves out. */ @@ -2268,6 +2270,8 @@ struct net_device { /* protected by rtnl_lock */ struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; + + u8 dev_addr_shadow[MAX_ADDR_LEN]; }; #define to_net_dev(d) container_of(d, struct net_device, dev) @@ -4288,6 +4292,7 @@ int dev_addr_del(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); void dev_addr_flush(struct net_device *dev); int dev_addr_init(struct net_device *dev); +void dev_addr_check(struct net_device *dev); /* Functions used for unicast addresses handling */ int dev_uc_add(struct net_device *dev, const unsigned char *addr); diff --git a/net/core/dev.c b/net/core/dev.c index 92c9258cbf28..9219e319e901 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1377,6 +1377,7 @@ static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) int ret; ASSERT_RTNL(); + dev_addr_check(dev); if (!netif_device_present(dev)) { /* may be detached because parent is runtime-suspended */ diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index a23a83ac18e5..969942734951 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -498,6 +498,21 @@ EXPORT_SYMBOL(__hw_addr_init); * Device addresses handling functions */ +/* Check that netdev->dev_addr is not written to directly as this would + * break the rbtree layout. All changes should go thru dev_addr_set() and co. + * Remove this check in mid-2024. + */ +void dev_addr_check(struct net_device *dev) +{ + if (!memcmp(dev->dev_addr, dev->dev_addr_shadow, MAX_ADDR_LEN)) + return; + + netdev_warn(dev, "Current addr: %*ph\n", MAX_ADDR_LEN, dev->dev_addr); + netdev_warn(dev, "Expected addr: %*ph\n", + MAX_ADDR_LEN, dev->dev_addr_shadow); + netdev_WARN(dev, "Incorrect netdev->dev_addr\n"); +} + /** * dev_addr_flush - Flush device address list * @dev: device @@ -509,6 +524,7 @@ EXPORT_SYMBOL(__hw_addr_init); void dev_addr_flush(struct net_device *dev) { /* rtnl_mutex must be held here */ + dev_addr_check(dev); __hw_addr_flush(&dev->dev_addrs); dev->dev_addr = NULL; @@ -552,8 +568,11 @@ void dev_addr_mod(struct net_device *dev, unsigned int offset, { struct netdev_hw_addr *ha; + dev_addr_check(dev); + ha = container_of(dev->dev_addr, struct netdev_hw_addr, addr[0]); memcpy(&ha->addr[offset], addr, len); + memcpy(&dev->dev_addr_shadow[offset], addr, len); } EXPORT_SYMBOL(dev_addr_mod); -- cgit v1.2.3-59-g8ed1b From f65b8132092699e4f672111836f3f51c00c354f2 Mon Sep 17 00:00:00 2001 From: Elyes HAOUAS Date: Thu, 28 Oct 2021 23:05:17 +0200 Subject: include/linux/efi.h: Remove unneeded whitespaces before tabs Signed-off-by: Elyes HAOUAS Signed-off-by: Ard Biesheuvel --- include/linux/efi.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index dbd39b20e034..de36fb547602 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -570,8 +570,8 @@ extern struct efi { unsigned long flags; } efi; -#define EFI_RT_SUPPORTED_GET_TIME 0x0001 -#define EFI_RT_SUPPORTED_SET_TIME 0x0002 +#define EFI_RT_SUPPORTED_GET_TIME 0x0001 +#define EFI_RT_SUPPORTED_SET_TIME 0x0002 #define EFI_RT_SUPPORTED_GET_WAKEUP_TIME 0x0004 #define EFI_RT_SUPPORTED_SET_WAKEUP_TIME 0x0008 #define EFI_RT_SUPPORTED_GET_VARIABLE 0x0010 @@ -838,7 +838,7 @@ extern int efi_status_to_err(efi_status_t status); #define EFI_VARIABLE_TIME_BASED_AUTHENTICATED_WRITE_ACCESS 0x0000000000000020 #define EFI_VARIABLE_APPEND_WRITE 0x0000000000000040 -#define EFI_VARIABLE_MASK (EFI_VARIABLE_NON_VOLATILE | \ +#define EFI_VARIABLE_MASK (EFI_VARIABLE_NON_VOLATILE | \ EFI_VARIABLE_BOOTSERVICE_ACCESS | \ EFI_VARIABLE_RUNTIME_ACCESS | \ EFI_VARIABLE_HARDWARE_ERROR_RECORD | \ -- cgit v1.2.3-59-g8ed1b From 3218910fd5858842a1dd98ce92b602f0878f8210 Mon Sep 17 00:00:00 2001 From: Adrian Larumbe Date: Mon, 1 Nov 2021 18:08:24 +0000 Subject: dmaengine: Add core function and capability check for DMA_MEMCPY_SG This is the old DMA_SG interface that was removed in commit c678fa66341c ("dmaengine: remove DMA_SG as it is dead code in kernel"). It has been renamed to DMA_MEMCPY_SG to better match the MEMSET and MEMSET_SG naming convention. It should only be used for mem2mem copies, either main system memory or CPU-addressable device memory (like video memory on a PCI graphics card). Bringing back this interface was prompted by the need to use the Xilinx CDMA device for mem2mem SG transfers. Signed-off-by: Adrian Larumbe Link: https://lore.kernel.org/r/20211101180825.241048-3-adrianml@alumnos.upm.es Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.c | 7 +++++++ include/linux/dmaengine.h | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index d9f7c097cfd6..2cfa8458b51b 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -1159,6 +1159,13 @@ int dma_async_device_register(struct dma_device *device) return -EIO; } + if (dma_has_cap(DMA_MEMCPY_SG, device->cap_mask) && !device->device_prep_dma_memcpy_sg) { + dev_err(device->dev, + "Device claims capability %s, but op is not defined\n", + "DMA_MEMCPY_SG"); + return -EIO; + } + if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) { dev_err(device->dev, "Device claims capability %s, but op is not defined\n", diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 9000f3ffce8b..554a86665de9 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -50,6 +50,7 @@ enum dma_status { */ enum dma_transaction_type { DMA_MEMCPY, + DMA_MEMCPY_SG, DMA_XOR, DMA_PQ, DMA_XOR_VAL, @@ -891,6 +892,11 @@ struct dma_device { struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags); + struct dma_async_tx_descriptor *(*device_prep_dma_memcpy_sg)( + struct dma_chan *chan, + struct scatterlist *dst_sg, unsigned int dst_nents, + struct scatterlist *src_sg, unsigned int src_nents, + unsigned long flags); struct dma_async_tx_descriptor *(*device_prep_dma_xor)( struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags); @@ -1051,6 +1057,20 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy( len, flags); } +static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy_sg( + struct dma_chan *chan, + struct scatterlist *dst_sg, unsigned int dst_nents, + struct scatterlist *src_sg, unsigned int src_nents, + unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_memcpy_sg) + return NULL; + + return chan->device->device_prep_dma_memcpy_sg(chan, dst_sg, dst_nents, + src_sg, src_nents, + flags); +} + static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan, enum dma_desc_metadata_mode mode) { -- cgit v1.2.3-59-g8ed1b From 0b70c256eba8448b072d25c95ee65e59da8970de Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 18 Nov 2021 20:12:42 +0800 Subject: ethtool: add support to set/get rx buf len via ethtool Add support to set rx buf len via ethtool -G parameter and get rx buf len via ethtool -g parameter. Signed-off-by: Hao Chen Signed-off-by: Guangbin Huang Signed-off-by: David S. Miller --- Documentation/networking/ethtool-netlink.rst | 10 ++++++---- include/linux/ethtool.h | 18 ++++++++++++++++++ include/uapi/linux/ethtool_netlink.h | 1 + net/ethtool/netlink.h | 2 +- net/ethtool/rings.c | 23 +++++++++++++++++++++-- 5 files changed, 47 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 7b598c7e3912..9d98e0511249 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -849,7 +849,7 @@ Request contents: Kernel response contents: - ==================================== ====== ========================== + ==================================== ====== =========================== ``ETHTOOL_A_RINGS_HEADER`` nested reply header ``ETHTOOL_A_RINGS_RX_MAX`` u32 max size of RX ring ``ETHTOOL_A_RINGS_RX_MINI_MAX`` u32 max size of RX mini ring @@ -859,7 +859,8 @@ Kernel response contents: ``ETHTOOL_A_RINGS_RX_MINI`` u32 size of RX mini ring ``ETHTOOL_A_RINGS_RX_JUMBO`` u32 size of RX jumbo ring ``ETHTOOL_A_RINGS_TX`` u32 size of TX ring - ==================================== ====== ========================== + ``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring + ==================================== ====== =========================== RINGS_SET @@ -869,13 +870,14 @@ Sets ring sizes like ``ETHTOOL_SRINGPARAM`` ioctl request. Request contents: - ==================================== ====== ========================== + ==================================== ====== =========================== ``ETHTOOL_A_RINGS_HEADER`` nested reply header ``ETHTOOL_A_RINGS_RX`` u32 size of RX ring ``ETHTOOL_A_RINGS_RX_MINI`` u32 size of RX mini ring ``ETHTOOL_A_RINGS_RX_JUMBO`` u32 size of RX jumbo ring ``ETHTOOL_A_RINGS_TX`` u32 size of TX ring - ==================================== ====== ========================== + ``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring + ==================================== ====== =========================== Kernel checks that requested ring sizes do not exceed limits reported by driver. Driver may impose additional constraints and may not suspport all diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 845a0ffc16ee..0b252b82988b 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -67,6 +67,22 @@ enum { ETH_RSS_HASH_FUNCS_COUNT }; +/** + * struct kernel_ethtool_ringparam - RX/TX ring configuration + * @rx_buf_len: Current length of buffers on the rx ring. + */ +struct kernel_ethtool_ringparam { + u32 rx_buf_len; +}; + +/** + * enum ethtool_supported_ring_param - indicator caps for setting ring params + * @ETHTOOL_RING_USE_RX_BUF_LEN: capture for setting rx_buf_len + */ +enum ethtool_supported_ring_param { + ETHTOOL_RING_USE_RX_BUF_LEN = BIT(0), +}; + #define __ETH_RSS_HASH_BIT(bit) ((u32)1 << (bit)) #define __ETH_RSS_HASH(name) __ETH_RSS_HASH_BIT(ETH_RSS_HASH_##name##_BIT) @@ -432,6 +448,7 @@ struct ethtool_module_power_mode_params { * @cap_link_lanes_supported: indicates if the driver supports lanes * parameter. * @supported_coalesce_params: supported types of interrupt coalescing. + * @supported_ring_params: supported ring params. * @get_drvinfo: Report driver/device information. Should only set the * @driver, @version, @fw_version and @bus_info fields. If not * implemented, the @driver and @bus_info fields will be filled in @@ -613,6 +630,7 @@ struct ethtool_module_power_mode_params { struct ethtool_ops { u32 cap_link_lanes_supported:1; u32 supported_coalesce_params; + u32 supported_ring_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); int (*get_regs_len)(struct net_device *); void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index 999777d32dcf..cca6e474a085 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -329,6 +329,7 @@ enum { ETHTOOL_A_RINGS_RX_MINI, /* u32 */ ETHTOOL_A_RINGS_RX_JUMBO, /* u32 */ ETHTOOL_A_RINGS_TX, /* u32 */ + ETHTOOL_A_RINGS_RX_BUF_LEN, /* u32 */ /* add new constants above here */ __ETHTOOL_A_RINGS_CNT, diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 836ee7157848..490598e5eedd 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -356,7 +356,7 @@ extern const struct nla_policy ethnl_features_set_policy[ETHTOOL_A_FEATURES_WANT extern const struct nla_policy ethnl_privflags_get_policy[ETHTOOL_A_PRIVFLAGS_HEADER + 1]; extern const struct nla_policy ethnl_privflags_set_policy[ETHTOOL_A_PRIVFLAGS_FLAGS + 1]; extern const struct nla_policy ethnl_rings_get_policy[ETHTOOL_A_RINGS_HEADER + 1]; -extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_TX + 1]; +extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_RX_BUF_LEN + 1]; extern const struct nla_policy ethnl_channels_get_policy[ETHTOOL_A_CHANNELS_HEADER + 1]; extern const struct nla_policy ethnl_channels_set_policy[ETHTOOL_A_CHANNELS_COMBINED_COUNT + 1]; extern const struct nla_policy ethnl_coalesce_get_policy[ETHTOOL_A_COALESCE_HEADER + 1]; diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c index 4e097812a967..bd8e9a7530eb 100644 --- a/net/ethtool/rings.c +++ b/net/ethtool/rings.c @@ -10,6 +10,7 @@ struct rings_req_info { struct rings_reply_data { struct ethnl_reply_data base; struct ethtool_ringparam ringparam; + struct kernel_ethtool_ringparam kernel_ringparam; }; #define RINGS_REPDATA(__reply_base) \ @@ -49,7 +50,8 @@ static int rings_reply_size(const struct ethnl_req_info *req_base, nla_total_size(sizeof(u32)) + /* _RINGS_RX */ nla_total_size(sizeof(u32)) + /* _RINGS_RX_MINI */ nla_total_size(sizeof(u32)) + /* _RINGS_RX_JUMBO */ - nla_total_size(sizeof(u32)); /* _RINGS_TX */ + nla_total_size(sizeof(u32)) + /* _RINGS_TX */ + nla_total_size(sizeof(u32)); /* _RINGS_RX_BUF_LEN */ } static int rings_fill_reply(struct sk_buff *skb, @@ -57,6 +59,7 @@ static int rings_fill_reply(struct sk_buff *skb, const struct ethnl_reply_data *reply_base) { const struct rings_reply_data *data = RINGS_REPDATA(reply_base); + const struct kernel_ethtool_ringparam *kernel_ringparam = &data->kernel_ringparam; const struct ethtool_ringparam *ringparam = &data->ringparam; if ((ringparam->rx_max_pending && @@ -78,7 +81,10 @@ static int rings_fill_reply(struct sk_buff *skb, (nla_put_u32(skb, ETHTOOL_A_RINGS_TX_MAX, ringparam->tx_max_pending) || nla_put_u32(skb, ETHTOOL_A_RINGS_TX, - ringparam->tx_pending)))) + ringparam->tx_pending))) || + (kernel_ringparam->rx_buf_len && + (nla_put_u32(skb, ETHTOOL_A_RINGS_RX_BUF_LEN, + kernel_ringparam->rx_buf_len)))) return -EMSGSIZE; return 0; @@ -105,10 +111,12 @@ const struct nla_policy ethnl_rings_set_policy[] = { [ETHTOOL_A_RINGS_RX_MINI] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_RX_JUMBO] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_TX] = { .type = NLA_U32 }, + [ETHTOOL_A_RINGS_RX_BUF_LEN] = NLA_POLICY_MIN(NLA_U32, 1), }; int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) { + struct kernel_ethtool_ringparam kernel_ringparam = {}; struct ethtool_ringparam ringparam = {}; struct ethnl_req_info req_info = {}; struct nlattr **tb = info->attrs; @@ -142,6 +150,8 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) ethnl_update_u32(&ringparam.rx_jumbo_pending, tb[ETHTOOL_A_RINGS_RX_JUMBO], &mod); ethnl_update_u32(&ringparam.tx_pending, tb[ETHTOOL_A_RINGS_TX], &mod); + ethnl_update_u32(&kernel_ringparam.rx_buf_len, + tb[ETHTOOL_A_RINGS_RX_BUF_LEN], &mod); ret = 0; if (!mod) goto out_ops; @@ -164,6 +174,15 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) goto out_ops; } + if (kernel_ringparam.rx_buf_len != 0 && + !(ops->supported_ring_params & ETHTOOL_RING_USE_RX_BUF_LEN)) { + ret = -EOPNOTSUPP; + NL_SET_ERR_MSG_ATTR(info->extack, + tb[ETHTOOL_A_RINGS_RX_BUF_LEN], + "setting rx buf len not supported"); + goto out_ops; + } + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam); if (ret < 0) goto out_ops; -- cgit v1.2.3-59-g8ed1b From 7462494408cd3de8b0bc1e79670bf213288501d0 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 18 Nov 2021 20:12:43 +0800 Subject: ethtool: extend ringparam setting/getting API with rx_buf_len Add two new parameters kernel_ringparam and extack for .get_ringparam and .set_ringparam to extend more ring params through netlink. Signed-off-by: Hao Chen Signed-off-by: Guangbin Huang Signed-off-by: David S. Miller --- arch/um/drivers/vector_kern.c | 4 +++- drivers/net/can/c_can/c_can_ethtool.c | 4 +++- drivers/net/ethernet/3com/typhoon.c | 4 +++- drivers/net/ethernet/amazon/ena/ena_ethtool.c | 8 +++++-- drivers/net/ethernet/amd/pcnet32.c | 8 +++++-- drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c | 11 +++++++--- .../net/ethernet/aquantia/atlantic/aq_ethtool.c | 8 +++++-- drivers/net/ethernet/atheros/atlx/atl1.c | 8 +++++-- drivers/net/ethernet/broadcom/b44.c | 8 +++++-- drivers/net/ethernet/broadcom/bcm63xx_enet.c | 25 ++++++++++++++++------ drivers/net/ethernet/broadcom/bnx2.c | 8 +++++-- .../net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c | 8 +++++-- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 8 +++++-- drivers/net/ethernet/broadcom/tg3.c | 10 +++++++-- drivers/net/ethernet/brocade/bna/bnad_ethtool.c | 8 +++++-- drivers/net/ethernet/cadence/macb_main.c | 8 +++++-- drivers/net/ethernet/cavium/liquidio/lio_ethtool.c | 11 +++++++--- .../net/ethernet/cavium/thunder/nicvf_ethtool.c | 8 +++++-- drivers/net/ethernet/chelsio/cxgb/cxgb2.c | 8 +++++-- drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c | 8 +++++-- drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c | 8 +++++-- .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 8 +++++-- drivers/net/ethernet/cisco/enic/enic_ethtool.c | 8 +++++-- drivers/net/ethernet/cortina/gemini.c | 8 +++++-- drivers/net/ethernet/emulex/benet/be_ethtool.c | 4 +++- drivers/net/ethernet/ethoc.c | 8 +++++-- drivers/net/ethernet/faraday/ftgmac100.c | 14 ++++++++---- .../net/ethernet/freescale/enetc/enetc_ethtool.c | 4 +++- drivers/net/ethernet/freescale/gianfar_ethtool.c | 8 +++++-- drivers/net/ethernet/freescale/ucc_geth_ethtool.c | 8 +++++-- drivers/net/ethernet/google/gve/gve_ethtool.c | 4 +++- drivers/net/ethernet/hisilicon/hns/hns_ethtool.c | 6 +++++- drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c | 8 +++++-- drivers/net/ethernet/huawei/hinic/hinic_ethtool.c | 8 +++++-- drivers/net/ethernet/ibm/emac/core.c | 7 ++++-- drivers/net/ethernet/ibm/ibmvnic.c | 8 +++++-- drivers/net/ethernet/intel/e100.c | 8 +++++-- drivers/net/ethernet/intel/e1000/e1000_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/e1000e/ethtool.c | 8 +++++-- drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/iavf/iavf_ethtool.c | 12 +++++++++-- drivers/net/ethernet/intel/ice/ice_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/igb/igb_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/igbvf/ethtool.c | 8 +++++-- drivers/net/ethernet/intel/igc/igc_ethtool.c | 14 ++++++++---- drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 8 +++++-- drivers/net/ethernet/intel/ixgbevf/ethtool.c | 8 +++++-- drivers/net/ethernet/marvell/mv643xx_eth.c | 8 +++++-- drivers/net/ethernet/marvell/mvneta.c | 14 ++++++++---- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 14 ++++++++---- .../ethernet/marvell/octeontx2/nic/otx2_ethtool.c | 8 +++++-- drivers/net/ethernet/marvell/skge.c | 8 +++++-- drivers/net/ethernet/marvell/sky2.c | 8 +++++-- drivers/net/ethernet/mellanox/mlx4/en_ethtool.c | 8 +++++-- .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 8 +++++-- drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 14 ++++++++---- .../ethernet/mellanox/mlx5/core/ipoib/ethtool.c | 8 +++++-- .../mellanox/mlxbf_gige/mlxbf_gige_ethtool.c | 7 ++++-- drivers/net/ethernet/micrel/ksz884x.c | 6 +++++- drivers/net/ethernet/myricom/myri10ge/myri10ge.c | 4 +++- drivers/net/ethernet/neterion/s2io.c | 7 ++++-- .../net/ethernet/netronome/nfp/nfp_net_ethtool.c | 8 +++++-- drivers/net/ethernet/nvidia/forcedeth.c | 10 +++++++-- .../ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c | 12 +++++++++-- drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c | 4 +++- .../net/ethernet/pensando/ionic/ionic_ethtool.c | 8 +++++-- .../ethernet/qlogic/netxen/netxen_nic_ethtool.c | 8 +++++-- drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 8 +++++-- .../net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c | 8 +++++-- drivers/net/ethernet/qualcomm/emac/emac-ethtool.c | 8 +++++-- drivers/net/ethernet/qualcomm/qca_debug.c | 8 +++++-- drivers/net/ethernet/realtek/8139cp.c | 4 +++- drivers/net/ethernet/realtek/r8169_main.c | 4 +++- drivers/net/ethernet/renesas/ravb_main.c | 8 +++++-- drivers/net/ethernet/renesas/sh_eth.c | 8 +++++-- drivers/net/ethernet/sfc/ef100_ethtool.c | 7 ++++-- drivers/net/ethernet/sfc/ethtool.c | 14 ++++++++---- drivers/net/ethernet/sfc/falcon/ethtool.c | 14 ++++++++---- .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c | 8 +++++-- drivers/net/ethernet/tehuti/tehuti.c | 12 +++++++++-- drivers/net/ethernet/ti/am65-cpsw-ethtool.c | 7 ++++-- drivers/net/ethernet/ti/cpmac.c | 8 +++++-- drivers/net/ethernet/ti/cpsw_ethtool.c | 8 +++++-- drivers/net/ethernet/ti/cpsw_priv.h | 8 +++++-- drivers/net/ethernet/toshiba/spider_net_ethtool.c | 4 +++- drivers/net/ethernet/xilinx/ll_temac_main.c | 14 ++++++++---- drivers/net/ethernet/xilinx/xilinx_axienet_main.c | 14 ++++++++---- drivers/net/hyperv/netvsc_drv.c | 8 +++++-- drivers/net/netdevsim/ethtool.c | 8 +++++-- drivers/net/usb/r8152.c | 8 +++++-- drivers/net/virtio_net.c | 4 +++- drivers/net/vmxnet3/vmxnet3_ethtool.c | 10 +++++---- drivers/s390/net/qeth_ethtool.c | 4 +++- include/linux/ethtool.h | 8 +++++-- net/ethtool/ioctl.c | 10 ++++++--- net/ethtool/rings.c | 9 +++++--- net/mac80211/ethtool.c | 8 +++++-- 99 files changed, 617 insertions(+), 212 deletions(-) (limited to 'include/linux') diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index cde6db184c26..4fc1a5d70dcf 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1441,7 +1441,9 @@ flash_fail: } static void vector_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct vector_private *vp = netdev_priv(netdev); diff --git a/drivers/net/can/c_can/c_can_ethtool.c b/drivers/net/can/c_can/c_can_ethtool.c index 377c7d2e7612..6655146294fc 100644 --- a/drivers/net/can/c_can/c_can_ethtool.c +++ b/drivers/net/can/c_can/c_can_ethtool.c @@ -20,7 +20,9 @@ static void c_can_get_drvinfo(struct net_device *netdev, } static void c_can_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct c_can_priv *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/3com/typhoon.c b/drivers/net/ethernet/3com/typhoon.c index 05e15b6e5e2c..481f1df3106c 100644 --- a/drivers/net/ethernet/3com/typhoon.c +++ b/drivers/net/ethernet/3com/typhoon.c @@ -1138,7 +1138,9 @@ typhoon_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) } static void -typhoon_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +typhoon_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { ering->rx_max_pending = RXENT_ENTRIES; ering->tx_max_pending = TXLO_ENTRIES - 1; diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 13e745cf3781..6b9b43e422c1 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -465,7 +465,9 @@ static void ena_get_drvinfo(struct net_device *dev, } static void ena_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ena_adapter *adapter = netdev_priv(netdev); @@ -476,7 +478,9 @@ static void ena_get_ringparam(struct net_device *netdev, } static int ena_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ena_adapter *adapter = netdev_priv(netdev); u32 new_tx_size, new_rx_size; diff --git a/drivers/net/ethernet/amd/pcnet32.c b/drivers/net/ethernet/amd/pcnet32.c index f5c50ff377ff..c20c369c7eb8 100644 --- a/drivers/net/ethernet/amd/pcnet32.c +++ b/drivers/net/ethernet/amd/pcnet32.c @@ -860,7 +860,9 @@ static int pcnet32_nway_reset(struct net_device *dev) } static void pcnet32_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pcnet32_private *lp = netdev_priv(dev); @@ -871,7 +873,9 @@ static void pcnet32_get_ringparam(struct net_device *dev, } static int pcnet32_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pcnet32_private *lp = netdev_priv(dev); unsigned long flags; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 94879cf8b420..6ceb1cdf6eba 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -619,8 +619,11 @@ static int xgbe_get_module_eeprom(struct net_device *netdev, return pdata->phy_if.module_eeprom(pdata, eeprom, data); } -static void xgbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) +static void +xgbe_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct xgbe_prv_data *pdata = netdev_priv(netdev); @@ -631,7 +634,9 @@ static void xgbe_get_ringparam(struct net_device *netdev, } static int xgbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct xgbe_prv_data *pdata = netdev_priv(netdev); unsigned int rx, tx; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index a9ef0544e30f..a418238f6309 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -812,7 +812,9 @@ static int aq_ethtool_set_pauseparam(struct net_device *ndev, } static void aq_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct aq_nic_s *aq_nic = netdev_priv(ndev); struct aq_nic_cfg_s *cfg; @@ -827,7 +829,9 @@ static void aq_get_ringparam(struct net_device *ndev, } static int aq_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct aq_nic_s *aq_nic = netdev_priv(ndev); const struct aq_hw_caps_s *hw_caps; diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index b4c9e805e981..6a969969d221 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -3438,7 +3438,9 @@ static void atl1_get_regs(struct net_device *netdev, struct ethtool_regs *regs, } static void atl1_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct atl1_adapter *adapter = netdev_priv(netdev); struct atl1_tpd_ring *txdr = &adapter->tpd_ring; @@ -3451,7 +3453,9 @@ static void atl1_get_ringparam(struct net_device *netdev, } static int atl1_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct atl1_adapter *adapter = netdev_priv(netdev); struct atl1_tpd_ring *tpdr = &adapter->tpd_ring; diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index 969591bbc066..e5857e88c207 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1961,7 +1961,9 @@ static int b44_set_link_ksettings(struct net_device *dev, } static void b44_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct b44 *bp = netdev_priv(dev); @@ -1972,7 +1974,9 @@ static void b44_get_ringparam(struct net_device *dev, } static int b44_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct b44 *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index a568994a03a6..b04e423c446a 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -1497,8 +1497,11 @@ static int bcm_enet_set_link_ksettings(struct net_device *dev, } } -static void bcm_enet_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +bcm_enet_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; @@ -1512,7 +1515,9 @@ static void bcm_enet_get_ringparam(struct net_device *dev, } static int bcm_enet_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; int was_running; @@ -2579,8 +2584,11 @@ static void bcm_enetsw_get_ethtool_stats(struct net_device *netdev, } } -static void bcm_enetsw_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +bcm_enetsw_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; @@ -2595,8 +2603,11 @@ static void bcm_enetsw_get_ringparam(struct net_device *dev, ering->tx_pending = priv->tx_ring_size; } -static int bcm_enetsw_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static int +bcm_enetsw_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; int was_running; diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index babc955ba64e..e20aafeb4ca9 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -7318,7 +7318,9 @@ static int bnx2_set_coalesce(struct net_device *dev, } static void -bnx2_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +bnx2_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2 *bp = netdev_priv(dev); @@ -7389,7 +7391,9 @@ bnx2_change_ring_size(struct bnx2 *bp, u32 rx, u32 tx, bool reset_irq) } static int -bnx2_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +bnx2_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2 *bp = netdev_priv(dev); int rc; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 472a3a478038..0e319ac7799f 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -1914,7 +1914,9 @@ static int bnx2x_set_coalesce(struct net_device *dev, } static void bnx2x_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2x *bp = netdev_priv(dev); @@ -1938,7 +1940,9 @@ static void bnx2x_get_ringparam(struct net_device *dev, } static int bnx2x_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2x *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 8188d55722e4..15253396096a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -775,7 +775,9 @@ skip_tpa_stats: } static void bnxt_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnxt *bp = netdev_priv(dev); @@ -794,7 +796,9 @@ static void bnxt_get_ringparam(struct net_device *dev, } static int bnxt_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnxt *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 85ca3909859d..283f3c1f1195 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -12390,7 +12390,10 @@ static int tg3_nway_reset(struct net_device *dev) return r; } -static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +static void tg3_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct tg3 *tp = netdev_priv(dev); @@ -12411,7 +12414,10 @@ static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam * ering->tx_pending = tp->napi[0].tx_pending; } -static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +static int tg3_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct tg3 *tp = netdev_priv(dev); int i, irq_sync = 0, err = 0; diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c index 391b85f25141..4b4ad731897f 100644 --- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c +++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c @@ -405,7 +405,9 @@ static int bnad_set_coalesce(struct net_device *netdev, static void bnad_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct bnad *bnad = netdev_priv(netdev); @@ -418,7 +420,9 @@ bnad_get_ringparam(struct net_device *netdev, static int bnad_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { int i, current_err, err = 0; struct bnad *bnad = netdev_priv(netdev); diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 57c5f48d19a4..7b68f5a1720c 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -3135,7 +3135,9 @@ static int macb_set_link_ksettings(struct net_device *netdev, } static void macb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); @@ -3147,7 +3149,9 @@ static void macb_get_ringparam(struct net_device *netdev, } static int macb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); u32 new_rx_size, new_tx_size; diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c index 2b9747867d4c..2c10ae3f7fc1 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c @@ -947,7 +947,9 @@ static int lio_set_phys_id(struct net_device *netdev, static void lio_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct lio *lio = GET_LIO(netdev); struct octeon_device *oct = lio->oct_dev; @@ -1252,8 +1254,11 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs) return 0; } -static int lio_ethtool_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static int +lio_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { u32 rx_count, tx_count, rx_count_old, tx_count_old; struct lio *lio = GET_LIO(netdev); diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c index 7f2882109b16..5a9fad61e9ea 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c @@ -467,7 +467,9 @@ static int nicvf_get_coalesce(struct net_device *netdev, } static void nicvf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nicvf *nic = netdev_priv(netdev); struct queue_set *qs = nic->qs; @@ -479,7 +481,9 @@ static void nicvf_get_ringparam(struct net_device *netdev, } static int nicvf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nicvf *nic = netdev_priv(netdev); struct queue_set *qs = nic->qs; diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c index 609820e214a3..18acd7cf3d6d 100644 --- a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c +++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c @@ -710,7 +710,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct adapter *adapter = dev->ml_priv; int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; @@ -724,7 +726,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = adapter->params.sge.cmdQ_size[0]; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct adapter *adapter = dev->ml_priv; int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index bfffcaeee624..e2637bd2f423 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -1948,7 +1948,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; @@ -1964,7 +1966,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = q->txq_size[0]; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 129352bbe114..0e4ec4079741 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -890,7 +890,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); const struct sge *s = &pi->adapter->sge; @@ -906,7 +908,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = s->ethtxq[pi->first_qset].q.size; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { int i; const struct port_info *pi = netdev_priv(dev); diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index 64479c464b4e..61808c2c26a6 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -1591,7 +1591,9 @@ static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel) * first Queue Set. */ static void cxgb4vf_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); const struct sge *s = &pi->adapter->sge; @@ -1614,7 +1616,9 @@ static void cxgb4vf_get_ringparam(struct net_device *dev, * device -- after vetting them of course! */ static int cxgb4vf_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index 6ded4d9fa32a..6c11f9d62526 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -177,7 +177,9 @@ static void enic_get_strings(struct net_device *netdev, u32 stringset, } static void enic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enic *enic = netdev_priv(netdev); struct vnic_enet_config *c = &enic->config; @@ -189,7 +191,9 @@ static void enic_get_ringparam(struct net_device *netdev, } static int enic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enic *enic = netdev_priv(netdev); struct vnic_enet_config *c = &enic->config; diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 941f175fb911..07add311f65d 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -2105,7 +2105,9 @@ static void gmac_get_pauseparam(struct net_device *netdev, } static void gmac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct gemini_ethernet_port *port = netdev_priv(netdev); @@ -2123,7 +2125,9 @@ static void gmac_get_ringparam(struct net_device *netdev, } static int gmac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct gemini_ethernet_port *port = netdev_priv(netdev); int err = 0; diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index f9955308b93d..dfa784339781 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -683,7 +683,9 @@ static int be_get_link_ksettings(struct net_device *netdev, } static void be_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct be_adapter *adapter = netdev_priv(netdev); diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c index b1c8ffea6ad2..d618a8b785b0 100644 --- a/drivers/net/ethernet/ethoc.c +++ b/drivers/net/ethernet/ethoc.c @@ -945,7 +945,9 @@ static void ethoc_get_regs(struct net_device *dev, struct ethtool_regs *regs, } static void ethoc_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ethoc *priv = netdev_priv(dev); @@ -961,7 +963,9 @@ static void ethoc_get_ringparam(struct net_device *dev, } static int ethoc_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ethoc *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 97c5d70de76e..691605c15265 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1178,8 +1178,11 @@ static void ftgmac100_get_drvinfo(struct net_device *netdev, strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); } -static void ftgmac100_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static void +ftgmac100_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct ftgmac100 *priv = netdev_priv(netdev); @@ -1190,8 +1193,11 @@ static void ftgmac100_get_ringparam(struct net_device *netdev, ering->tx_pending = priv->tx_q_entries; } -static int ftgmac100_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static int +ftgmac100_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct ftgmac100 *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 910b9f722504..fa5b4f885b17 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -562,7 +562,9 @@ static int enetc_set_rxfh(struct net_device *ndev, const u32 *indir, } static void enetc_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enetc_ndev_priv *priv = netdev_priv(ndev); diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index 7b32ed29bf4c..ff756265d58f 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -372,7 +372,9 @@ static int gfar_scoalesce(struct net_device *dev, * rx, rx_mini, and rx_jumbo rings are the same size, as mini and * jumbo are ignored by the driver */ static void gfar_gringparam(struct net_device *dev, - struct ethtool_ringparam *rvals) + struct ethtool_ringparam *rvals, + struct kernel_ethtool_ringparam *kernel_rvals, + struct netlink_ext_ack *extack) { struct gfar_private *priv = netdev_priv(dev); struct gfar_priv_tx_q *tx_queue = NULL; @@ -399,7 +401,9 @@ static void gfar_gringparam(struct net_device *dev, * necessary so that we don't mess things up while we're in motion. */ static int gfar_sringparam(struct net_device *dev, - struct ethtool_ringparam *rvals) + struct ethtool_ringparam *rvals, + struct kernel_ethtool_ringparam *kernel_rvals, + struct netlink_ext_ack *extack) { struct gfar_private *priv = netdev_priv(dev); int err = 0, i; diff --git a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c index 14c08a868190..69b2b98b1525 100644 --- a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c +++ b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c @@ -207,7 +207,9 @@ uec_get_regs(struct net_device *netdev, static void uec_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ucc_geth_private *ugeth = netdev_priv(netdev); struct ucc_geth_info *ug_info = ugeth->ug_info; @@ -226,7 +228,9 @@ uec_get_ringparam(struct net_device *netdev, static int uec_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ucc_geth_private *ugeth = netdev_priv(netdev); struct ucc_geth_info *ug_info = ugeth->ug_info; diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index c8df47a97fa4..fd2d2c705391 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -419,7 +419,9 @@ static int gve_set_channels(struct net_device *netdev, } static void gve_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *cmd) + struct ethtool_ringparam *cmd, + struct kernel_ethtool_ringparam *kernel_cmd, + struct netlink_ext_ack *extack) { struct gve_priv *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c index ab7390225942..d7a27c244d48 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c @@ -663,9 +663,13 @@ static void hns_nic_get_drvinfo(struct net_device *net_dev, * hns_get_ringparam - get ring parameter * @net_dev: net device * @param: ethtool parameter + * @kernel_param: ethtool external parameter + * @extack: netlink extended ACK report struct */ static void hns_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns_nic_priv *priv = netdev_priv(net_dev); struct hnae_ae_ops *ops; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 9a816dbec613..e35a2661b45d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -643,7 +643,9 @@ static u32 hns3_get_link(struct net_device *netdev) } static void hns3_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns3_nic_priv *priv = netdev_priv(netdev); struct hnae3_handle *h = priv->ae_handle; @@ -1081,7 +1083,9 @@ static int hns3_check_ringparam(struct net_device *ndev, } static int hns3_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = priv->ae_handle; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index a35a80f9a234..93192f58ac88 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -547,7 +547,9 @@ static void hinic_get_drvinfo(struct net_device *netdev, } static void hinic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct hinic_dev *nic_dev = netdev_priv(netdev); @@ -580,7 +582,9 @@ static int check_ringparam_valid(struct hinic_dev *nic_dev, } static int hinic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct hinic_dev *nic_dev = netdev_priv(netdev); u16 new_sq_depth, new_rq_depth; diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index 6b3fc8823c54..fbea9f7efe8c 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -2137,8 +2137,11 @@ emac_ethtool_set_link_ksettings(struct net_device *ndev, return 0; } -static void emac_ethtool_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *rp) +static void +emac_ethtool_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF; rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF; diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index c327fc8860da..ad9a2819e202 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -3088,7 +3088,9 @@ static u32 ibmvnic_get_link(struct net_device *netdev) } static void ibmvnic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); @@ -3108,7 +3110,9 @@ static void ibmvnic_get_ringparam(struct net_device *netdev, } static int ibmvnic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); int ret; diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c index 0bf3d47bb90d..4a8013f20152 100644 --- a/drivers/net/ethernet/intel/e100.c +++ b/drivers/net/ethernet/intel/e100.c @@ -2557,7 +2557,9 @@ static int e100_set_eeprom(struct net_device *netdev, } static void e100_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nic *nic = netdev_priv(netdev); struct param_range *rfds = &nic->params.rfds; @@ -2570,7 +2572,9 @@ static void e100_get_ringparam(struct net_device *netdev, } static int e100_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nic *nic = netdev_priv(netdev); struct param_range *rfds = &nic->params.rfds; diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index 0a57172dfcbc..32803b0cf1e8 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -539,7 +539,9 @@ static void e1000_get_drvinfo(struct net_device *netdev, } static void e1000_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -556,7 +558,9 @@ static void e1000_get_ringparam(struct net_device *netdev, } static int e1000_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index 8515e00d1b40..b80ae9a82224 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -655,7 +655,9 @@ static void e1000_get_drvinfo(struct net_device *netdev, } static void e1000_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); @@ -666,7 +668,9 @@ static void e1000_get_ringparam(struct net_device *netdev, } static int e1000_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_ring *temp_tx = NULL, *temp_rx = NULL; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c index 0d37f011d0ce..d53369e30040 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c @@ -502,7 +502,9 @@ static void fm10k_set_msglevel(struct net_device *netdev, u32 data) } static void fm10k_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fm10k_intfc *interface = netdev_priv(netdev); @@ -517,7 +519,9 @@ static void fm10k_get_ringparam(struct net_device *netdev, } static int fm10k_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fm10k_intfc *interface = netdev_priv(netdev); struct fm10k_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 513ba6974355..091f36adbbe1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -1916,7 +1916,9 @@ static void i40e_get_drvinfo(struct net_device *netdev, } static void i40e_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_pf *pf = np->vsi->back; @@ -1944,7 +1946,9 @@ static bool i40e_active_tx_ring_index(struct i40e_vsi *vsi, u16 index) } static int i40e_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct i40e_ring *tx_rings = NULL, *rx_rings = NULL; struct i40e_netdev_priv *np = netdev_priv(netdev); diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 144a77679359..d327f576136f 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -580,12 +580,16 @@ static void iavf_get_drvinfo(struct net_device *netdev, * iavf_get_ringparam - Get ring parameters * @netdev: network interface device structure * @ring: ethtool ringparam structure + * @kernel_ring: ethtool extenal ringparam structure + * @extack: netlink extended ACK report struct * * Returns current ring parameters. TX and RX rings are reported separately, * but the number of rings is not reported. **/ static void iavf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -599,12 +603,16 @@ static void iavf_get_ringparam(struct net_device *netdev, * iavf_set_ringparam - Set ring parameters * @netdev: network interface device structure * @ring: ethtool ringparam structure + * @kernel_ring: ethtool external ringparam structure + * @extack: netlink extended ACK report struct * * Sets ring parameters. TX and RX rings are controlled separately, but the * number of rings is not specified, so all rings get the same settings. **/ static int iavf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct iavf_adapter *adapter = netdev_priv(netdev); u32 new_rx_count, new_tx_count; diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 572519e402f4..5af2faaa21e1 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -2686,7 +2686,9 @@ ice_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, } static void -ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; @@ -2704,7 +2706,9 @@ ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) } static int -ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_tx_ring *xdp_rings = NULL; diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index fb1029352c3e..51a2dcaf553d 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -864,7 +864,9 @@ static void igb_get_drvinfo(struct net_device *netdev, } static void igb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igb_adapter *adapter = netdev_priv(netdev); @@ -875,7 +877,9 @@ static void igb_get_ringparam(struct net_device *netdev, } static int igb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igb_adapter *adapter = netdev_priv(netdev); struct igb_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c index 06e5bd646a0e..9d4322b74163 100644 --- a/drivers/net/ethernet/intel/igbvf/ethtool.c +++ b/drivers/net/ethernet/intel/igbvf/ethtool.c @@ -175,7 +175,9 @@ static void igbvf_get_drvinfo(struct net_device *netdev, } static void igbvf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igbvf_adapter *adapter = netdev_priv(netdev); struct igbvf_ring *tx_ring = adapter->tx_ring; @@ -188,7 +190,9 @@ static void igbvf_get_ringparam(struct net_device *netdev, } static int igbvf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igbvf_adapter *adapter = netdev_priv(netdev); struct igbvf_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index e0a76ac1bbbc..8cc077b712ad 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -567,8 +567,11 @@ static int igc_ethtool_set_eeprom(struct net_device *netdev, return ret_val; } -static void igc_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +igc_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -578,8 +581,11 @@ static void igc_ethtool_get_ringparam(struct net_device *netdev, ring->tx_pending = adapter->tx_ring_count; } -static int igc_ethtool_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static int +igc_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c index 582099a5ad41..46efcfab7234 100644 --- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c +++ b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c @@ -464,7 +464,9 @@ ixgb_get_drvinfo(struct net_device *netdev, static void ixgb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgb_adapter *adapter = netdev_priv(netdev); struct ixgb_desc_ring *txdr = &adapter->tx_ring; @@ -478,7 +480,9 @@ ixgb_get_ringparam(struct net_device *netdev, static int ixgb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgb_adapter *adapter = netdev_priv(netdev); struct ixgb_desc_ring *txdr = &adapter->tx_ring; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 8362822316a9..f70967c32116 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1118,7 +1118,9 @@ static void ixgbe_get_drvinfo(struct net_device *netdev, } static void ixgbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_ring *tx_ring = adapter->tx_ring[0]; @@ -1131,7 +1133,9 @@ static void ixgbe_get_ringparam(struct net_device *netdev, } static int ixgbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 8380f905e708..3b41f83c8dff 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -225,7 +225,9 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev, } static void ixgbevf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); @@ -236,7 +238,9 @@ static void ixgbevf_get_ringparam(struct net_device *netdev, } static int ixgbevf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); struct ixgbevf_ring *tx_ring = NULL, *rx_ring = NULL; diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index bb14fa2241a3..116919730237 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -1638,7 +1638,9 @@ static int mv643xx_eth_set_coalesce(struct net_device *dev, } static void -mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); @@ -1650,7 +1652,9 @@ mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) } static int -mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 67a644177880..350d24d7ead0 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -4510,8 +4510,11 @@ static void mvneta_ethtool_get_drvinfo(struct net_device *dev, } -static void mvneta_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +mvneta_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(netdev); @@ -4521,8 +4524,11 @@ static void mvneta_ethtool_get_ringparam(struct net_device *netdev, ring->tx_pending = pp->tx_ring_size; } -static int mvneta_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvneta_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(dev); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index df6c793f4b1b..8b1ca0b13e88 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5431,8 +5431,11 @@ static void mvpp2_ethtool_get_drvinfo(struct net_device *dev, sizeof(drvinfo->bus_info)); } -static void mvpp2_ethtool_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static void +mvpp2_ethtool_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); @@ -5442,8 +5445,11 @@ static void mvpp2_ethtool_get_ringparam(struct net_device *dev, ring->tx_pending = port->tx_ring_size; } -static int mvpp2_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvpp2_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); u16 prev_rx_ring_size = port->rx_ring_size; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 80d4ce61f442..d85db90632d6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -360,7 +360,9 @@ static int otx2_set_pauseparam(struct net_device *netdev, } static void otx2_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); struct otx2_qset *qs = &pfvf->qset; @@ -372,7 +374,9 @@ static void otx2_get_ringparam(struct net_device *netdev, } static int otx2_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); bool if_up = netif_running(netdev); diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 0c864e5bf0a6..cf03c67fbf40 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -492,7 +492,9 @@ static void skge_get_strings(struct net_device *dev, u32 stringset, u8 *data) } static void skge_get_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); @@ -504,7 +506,9 @@ static void skge_get_ring_param(struct net_device *dev, } static int skge_set_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); int err = 0; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index d53ce826befd..ea16b1dd6a98 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -4149,7 +4149,9 @@ static unsigned long roundup_ring_size(unsigned long pending) } static void sky2_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); @@ -4161,7 +4163,9 @@ static void sky2_get_ringparam(struct net_device *dev, } static int sky2_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 066d79e4ecfc..e97d97e94231 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -1141,7 +1141,9 @@ static void mlx4_en_get_pauseparam(struct net_device *dev, } static int mlx4_en_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; @@ -1208,7 +1210,9 @@ out: } static void mlx4_en_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx4_en_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 45bdfcb3dcc7..c8757c5a812b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -314,7 +314,9 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, } static void mlx5e_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -380,7 +382,9 @@ unlock: } static int mlx5e_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 8c81aeba07db..87c762066adb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -219,16 +219,22 @@ static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset) } } -static void mlx5e_rep_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) +static void +mlx5e_rep_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); mlx5e_ethtool_get_ringparam(priv, param); } -static int mlx5e_rep_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) +static int +mlx5e_rep_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index f23e33ac9c6b..f4f7eaf16446 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -67,7 +67,9 @@ static void mlx5i_get_ethtool_stats(struct net_device *dev, } static int mlx5i_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = mlx5i_epriv(dev); @@ -75,7 +77,9 @@ static int mlx5i_set_ringparam(struct net_device *dev, } static void mlx5i_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = mlx5i_epriv(dev); diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c index 92b798f8e73a..ceeb7f4c3f6c 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c @@ -33,8 +33,11 @@ static void mlxbf_gige_get_regs(struct net_device *netdev, memcpy_fromio(p, priv->base, MLXBF_GIGE_MMIO_REG_SZ); } -static void mlxbf_gige_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static void +mlxbf_gige_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct mlxbf_gige *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index 99c0c1491af2..d024983815da 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -6317,11 +6317,15 @@ static int netdev_set_pauseparam(struct net_device *dev, * netdev_get_ringparam - get tx/rx ring parameters * @dev: Network device. * @ring: Ethtool RING settings data structure. + * @kernel_ring: Ethtool external RING settings data structure. + * @extack: Netlink handle. * * This procedure returns the TX/RX ring settings. */ static void netdev_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct dev_priv *priv = netdev_priv(dev); struct dev_info *hw_priv = priv->adapter; diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 5736fcdafd7a..83a5e29c836a 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -1704,7 +1704,9 @@ myri10ge_set_pauseparam(struct net_device *netdev, static void myri10ge_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct myri10ge_priv *mgp = netdev_priv(netdev); diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index d1c32c65db05..d6ae44f164a9 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -5461,8 +5461,11 @@ static int s2io_ethtool_set_led(struct net_device *dev, return 0; } -static void s2io_ethtool_gringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +s2io_ethtool_gringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct s2io_nic *sp = netdev_priv(dev); int i, tx_desc_count = 0, rx_desc_count = 0; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index 1de076f55740..93fa8e677e05 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -381,7 +381,9 @@ err_bad_set: } static void nfp_net_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nfp_net *nn = netdev_priv(netdev); @@ -406,7 +408,9 @@ static int nfp_net_set_ring_size(struct nfp_net *nn, u32 rxd_cnt, u32 txd_cnt) } static int nfp_net_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nfp_net *nn = netdev_priv(netdev); u32 rxd_cnt, txd_cnt; diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 9b530d7509a4..660013f716d4 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -4651,7 +4651,10 @@ static int nv_nway_reset(struct net_device *dev) return ret; } -static void nv_get_ringparam(struct net_device *dev, struct ethtool_ringparam* ring) +static void nv_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fe_priv *np = netdev_priv(dev); @@ -4662,7 +4665,10 @@ static void nv_get_ringparam(struct net_device *dev, struct ethtool_ringparam* r ring->tx_pending = np->tx_ring_size; } -static int nv_set_ringparam(struct net_device *dev, struct ethtool_ringparam* ring) +static int nv_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fe_priv *np = netdev_priv(dev); u8 __iomem *base = get_hwbase(dev); diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c index 660b07cb5b92..84cc79e928c8 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c @@ -270,9 +270,13 @@ static int pch_gbe_nway_reset(struct net_device *netdev) * pch_gbe_get_ringparam - Report ring sizes * @netdev: Network interface device structure * @ring: Ring param structure + * @kernel_ring: Ring external param structure + * @extack: netlink handle */ static void pch_gbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct pch_gbe_adapter *adapter = netdev_priv(netdev); struct pch_gbe_tx_ring *txdr = adapter->tx_ring; @@ -288,12 +292,16 @@ static void pch_gbe_get_ringparam(struct net_device *netdev, * pch_gbe_set_ringparam - Set ring sizes * @netdev: Network interface device structure * @ring: Ring param structure + * @kernel_ring: Ring external param structure + * @extack: netlink handle * Returns * 0: Successful. * Negative value: Failed. */ static int pch_gbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct pch_gbe_adapter *adapter = netdev_priv(netdev); struct pch_gbe_tx_ring *txdr, *tx_old; diff --git a/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c b/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c index e1a304886a3c..4c7e0c991105 100644 --- a/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c +++ b/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c @@ -69,7 +69,9 @@ pasemi_mac_ethtool_set_msglevel(struct net_device *netdev, static void pasemi_mac_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pasemi_mac *mac = netdev_priv(netdev); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index c54d735b9e2e..386a5cf1e224 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -512,7 +512,9 @@ static int ionic_set_coalesce(struct net_device *netdev, } static void ionic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ionic_lif *lif = netdev_priv(netdev); @@ -523,7 +525,9 @@ static void ionic_get_ringparam(struct net_device *netdev, } static int ionic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ionic_lif *lif = netdev_priv(netdev); struct ionic_queue_params qparam; diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c index a075643f5826..3c4a84ea6321 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c @@ -392,7 +392,9 @@ netxen_nic_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, static void netxen_nic_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netxen_adapter *adapter = netdev_priv(dev); @@ -430,7 +432,9 @@ netxen_validate_ringparam(u32 val, u32 min, u32 max, char *r_name) static int netxen_nic_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netxen_adapter *adapter = netdev_priv(dev); u16 max_rcv_desc = MAX_RCV_DESCRIPTORS_10G; diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 8284c4c1528f..100c9c52c20b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -888,7 +888,9 @@ int qede_set_coalesce(struct net_device *dev, struct ethtool_coalesce *coal, } static void qede_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct qede_dev *edev = netdev_priv(dev); @@ -899,7 +901,9 @@ static void qede_get_ringparam(struct net_device *dev, } static int qede_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct qede_dev *edev = netdev_priv(dev); diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c index fc364b4ab6eb..e10fe071a40f 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c @@ -632,7 +632,9 @@ qlcnic_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, static void qlcnic_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qlcnic_adapter *adapter = netdev_priv(dev); @@ -663,7 +665,9 @@ qlcnic_validate_ringparam(u32 val, u32 min, u32 max, char *r_name) static int qlcnic_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qlcnic_adapter *adapter = netdev_priv(dev); u16 num_rxd, num_jumbo_rxd, num_txd; diff --git a/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c b/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c index f72e13b83869..f502db9cdea9 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c @@ -133,7 +133,9 @@ static int emac_nway_reset(struct net_device *netdev) } static void emac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct emac_adapter *adpt = netdev_priv(netdev); @@ -144,7 +146,9 @@ static void emac_get_ringparam(struct net_device *netdev, } static int emac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct emac_adapter *adpt = netdev_priv(netdev); diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c index d59fff2fbcc6..792ce9a323cd 100644 --- a/drivers/net/ethernet/qualcomm/qca_debug.c +++ b/drivers/net/ethernet/qualcomm/qca_debug.c @@ -246,7 +246,9 @@ qcaspi_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *p) } static void -qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) +qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qcaspi *qca = netdev_priv(dev); @@ -257,7 +259,9 @@ qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) } static int -qcaspi_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) +qcaspi_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; struct qcaspi *qca = netdev_priv(dev); diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 4f39f843bb3a..ad7b9e9d7f95 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -1388,7 +1388,9 @@ static void cp_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *info } static void cp_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { ring->rx_max_pending = CP_RX_RING_SIZE; ring->tx_max_pending = CP_TX_RING_SIZE; diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index e896e5eca804..14dd04b2fbed 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -1873,7 +1873,9 @@ static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data) } static void rtl8169_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *data) + struct ethtool_ringparam *data, + struct kernel_ethtool_ringparam *kernel_data, + struct netlink_ext_ack *extack) { data->rx_max_pending = NUM_RX_DESC; data->rx_pending = NUM_RX_DESC; diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 151cce2fe36d..ce09bd45527e 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1604,7 +1604,9 @@ static void ravb_get_strings(struct net_device *ndev, u32 stringset, u8 *data) } static void ravb_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ravb_private *priv = netdev_priv(ndev); @@ -1615,7 +1617,9 @@ static void ravb_get_ringparam(struct net_device *ndev, } static int ravb_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ravb_private *priv = netdev_priv(ndev); const struct ravb_hw_info *info = priv->info; diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index a3fbb2221c9a..223626290ce0 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -2296,7 +2296,9 @@ static void sh_eth_get_strings(struct net_device *ndev, u32 stringset, u8 *data) } static void sh_eth_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct sh_eth_private *mdp = netdev_priv(ndev); @@ -2307,7 +2309,9 @@ static void sh_eth_get_ringparam(struct net_device *ndev, } static int sh_eth_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct sh_eth_private *mdp = netdev_priv(ndev); int ret; diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c index 835c838b7dfa..5dba4125d953 100644 --- a/drivers/net/ethernet/sfc/ef100_ethtool.c +++ b/drivers/net/ethernet/sfc/ef100_ethtool.c @@ -20,8 +20,11 @@ /* This is the maximum number of descriptor rings supported by the QDMA */ #define EFX_EF100_MAX_DMAQ_SIZE 16384UL -static void ef100_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +ef100_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c index e002ce21788d..48506373721a 100644 --- a/drivers/net/ethernet/sfc/ethtool.c +++ b/drivers/net/ethernet/sfc/ethtool.c @@ -157,8 +157,11 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev, return 0; } -static void efx_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +efx_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); @@ -168,8 +171,11 @@ static void efx_ethtool_get_ringparam(struct net_device *net_dev, ring->tx_pending = efx->txq_entries; } -static int efx_ethtool_set_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static int +efx_ethtool_set_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); u32 txq_entries; diff --git a/drivers/net/ethernet/sfc/falcon/ethtool.c b/drivers/net/ethernet/sfc/falcon/ethtool.c index 137e8a7aeaa1..907254b36663 100644 --- a/drivers/net/ethernet/sfc/falcon/ethtool.c +++ b/drivers/net/ethernet/sfc/falcon/ethtool.c @@ -637,8 +637,11 @@ static int ef4_ethtool_set_coalesce(struct net_device *net_dev, return 0; } -static void ef4_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +ef4_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ef4_nic *efx = netdev_priv(net_dev); @@ -648,8 +651,11 @@ static void ef4_ethtool_get_ringparam(struct net_device *net_dev, ring->tx_pending = efx->txq_entries; } -static int ef4_ethtool_set_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static int +ef4_ethtool_set_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ef4_nic *efx = netdev_priv(net_dev); u32 txq_entries; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index d89455803bed..eead45369eb9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -463,7 +463,9 @@ static int stmmac_nway_reset(struct net_device *dev) } static void stmmac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct stmmac_priv *priv = netdev_priv(netdev); @@ -474,7 +476,9 @@ static void stmmac_get_ringparam(struct net_device *netdev, } static int stmmac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { if (ring->rx_mini_pending || ring->rx_jumbo_pending || ring->rx_pending < DMA_MIN_RX_SIZE || diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c index 0775a5542f2f..89bc1602661c 100644 --- a/drivers/net/ethernet/tehuti/tehuti.c +++ b/drivers/net/ethernet/tehuti/tehuti.c @@ -2245,9 +2245,13 @@ static inline int bdx_tx_fifo_size_to_packets(int tx_size) * bdx_get_ringparam - report ring sizes * @netdev * @ring + * @kernel_ring + * @extack */ static void -bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct bdx_priv *priv = netdev_priv(netdev); @@ -2262,9 +2266,13 @@ bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) * bdx_set_ringparam - set ring sizes * @netdev * @ring + * @kernel_ring + * @extack */ static int -bdx_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +bdx_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct bdx_priv *priv = netdev_priv(netdev); int rx_size = 0; diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c index b05de9b61ad6..d45b6bb86f0b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c +++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c @@ -453,8 +453,11 @@ static int am65_cpsw_set_channels(struct net_device *ndev, return am65_cpsw_nuss_update_tx_chns(common, chs->tx_count); } -static void am65_cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +am65_cpsw_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct am65_cpsw_common *common = am65_ndev_to_common(ndev); diff --git a/drivers/net/ethernet/ti/cpmac.c b/drivers/net/ethernet/ti/cpmac.c index 7449436fc87c..bef5e68dac31 100644 --- a/drivers/net/ethernet/ti/cpmac.c +++ b/drivers/net/ethernet/ti/cpmac.c @@ -817,7 +817,9 @@ static void cpmac_tx_timeout(struct net_device *dev, unsigned int txqueue) } static void cpmac_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct cpmac_priv *priv = netdev_priv(dev); @@ -833,7 +835,9 @@ static void cpmac_get_ringparam(struct net_device *dev, } static int cpmac_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct cpmac_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c index 158c8d3793f4..aa42141be3c0 100644 --- a/drivers/net/ethernet/ti/cpsw_ethtool.c +++ b/drivers/net/ethernet/ti/cpsw_ethtool.c @@ -658,7 +658,9 @@ err: } void cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct cpsw_priv *priv = netdev_priv(ndev); struct cpsw_common *cpsw = priv->cpsw; @@ -671,7 +673,9 @@ void cpsw_get_ringparam(struct net_device *ndev, } int cpsw_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct cpsw_common *cpsw = ndev_to_cpsw(ndev); int descs_num, ret; diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index 435668ee542d..f33c882eb70e 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -491,9 +491,13 @@ int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata); int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata); int cpsw_nway_reset(struct net_device *ndev); void cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering); + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack); int cpsw_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering); + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack); int cpsw_set_channels_common(struct net_device *ndev, struct ethtool_channels *chs, cpdma_handler_fn rx_handler); diff --git a/drivers/net/ethernet/toshiba/spider_net_ethtool.c b/drivers/net/ethernet/toshiba/spider_net_ethtool.c index 54f655a68148..93110dba0bfa 100644 --- a/drivers/net/ethernet/toshiba/spider_net_ethtool.c +++ b/drivers/net/ethernet/toshiba/spider_net_ethtool.c @@ -110,7 +110,9 @@ spider_net_ethtool_nway_reset(struct net_device *netdev) static void spider_net_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct spider_net_card *card = netdev_priv(netdev); diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index e7065c9a8e38..b900ab5aef2a 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -1276,8 +1276,11 @@ static const struct attribute_group temac_attr_group = { * ethtool support */ -static void ll_temac_ethtools_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +ll_temac_ethtools_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct temac_local *lp = netdev_priv(ndev); @@ -1291,8 +1294,11 @@ static void ll_temac_ethtools_get_ringparam(struct net_device *ndev, ering->tx_pending = lp->tx_bd_num; } -static int ll_temac_ethtools_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static int +ll_temac_ethtools_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct temac_local *lp = netdev_priv(ndev); diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 3dabc1901416..e39356364f33 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -1323,8 +1323,11 @@ static void axienet_ethtools_get_regs(struct net_device *ndev, data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET); } -static void axienet_ethtools_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +axienet_ethtools_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct axienet_local *lp = netdev_priv(ndev); @@ -1338,8 +1341,11 @@ static void axienet_ethtools_get_ringparam(struct net_device *ndev, ering->tx_pending = lp->tx_bd_num; } -static int axienet_ethtools_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static int +axienet_ethtools_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct axienet_local *lp = netdev_priv(ndev); diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 7e66ae1d2a59..efa963b7af54 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -1858,7 +1858,9 @@ static void __netvsc_get_ringparam(struct netvsc_device *nvdev, } static void netvsc_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct net_device_context *ndevctx = netdev_priv(ndev); struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); @@ -1870,7 +1872,9 @@ static void netvsc_get_ringparam(struct net_device *ndev, } static int netvsc_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct net_device_context *ndevctx = netdev_priv(ndev); struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); diff --git a/drivers/net/netdevsim/ethtool.c b/drivers/net/netdevsim/ethtool.c index 0ab6a40be611..2b84169bf3a2 100644 --- a/drivers/net/netdevsim/ethtool.c +++ b/drivers/net/netdevsim/ethtool.c @@ -65,7 +65,9 @@ static int nsim_set_coalesce(struct net_device *dev, } static void nsim_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netdevsim *ns = netdev_priv(dev); @@ -73,7 +75,9 @@ static void nsim_get_ringparam(struct net_device *dev, } static int nsim_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netdevsim *ns = netdev_priv(dev); diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index f9877a3e83ac..b88f07c77a6b 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -8983,7 +8983,9 @@ static int rtl8152_set_tunable(struct net_device *netdev, } static void rtl8152_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct r8152 *tp = netdev_priv(netdev); @@ -8992,7 +8994,9 @@ static void rtl8152_get_ringparam(struct net_device *netdev, } static int rtl8152_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct r8152 *tp = netdev_priv(netdev); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 03e38e38ee4b..c74af526d79b 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2174,7 +2174,9 @@ static void virtnet_cpu_notif_remove(struct virtnet_info *vi) } static void virtnet_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c index 16f3a2057b90..3172d46c0335 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethtool.c +++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c @@ -575,10 +575,11 @@ vmxnet3_get_link_ksettings(struct net_device *netdev, return 0; } - static void vmxnet3_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); @@ -595,10 +596,11 @@ vmxnet3_get_ringparam(struct net_device *netdev, param->rx_jumbo_pending = adapter->rx_ring2_size; } - static int vmxnet3_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); u32 new_tx_ring_size, new_rx_ring_size, new_rx_ring2_size; diff --git a/drivers/s390/net/qeth_ethtool.c b/drivers/s390/net/qeth_ethtool.c index 46d0fe0d0e8a..b0b36b2132fe 100644 --- a/drivers/s390/net/qeth_ethtool.c +++ b/drivers/s390/net/qeth_ethtool.c @@ -144,7 +144,9 @@ static int qeth_set_coalesce(struct net_device *dev, } static void qeth_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct qeth_card *card = dev->ml_priv; diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 0b252b82988b..a26f37a27167 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -656,9 +656,13 @@ struct ethtool_ops { struct kernel_ethtool_coalesce *, struct netlink_ext_ack *); void (*get_ringparam)(struct net_device *, - struct ethtool_ringparam *); + struct ethtool_ringparam *, + struct kernel_ethtool_ringparam *, + struct netlink_ext_ack *); int (*set_ringparam)(struct net_device *, - struct ethtool_ringparam *); + struct ethtool_ringparam *, + struct kernel_ethtool_ringparam *, + struct netlink_ext_ack *); void (*get_pause_stats)(struct net_device *dev, struct ethtool_pause_stats *pause_stats); void (*get_pauseparam)(struct net_device *, diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 5c5f170a9851..af2d4e022076 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1743,11 +1743,13 @@ static noinline_for_stack int ethtool_set_coalesce(struct net_device *dev, static int ethtool_get_ringparam(struct net_device *dev, void __user *useraddr) { struct ethtool_ringparam ringparam = { .cmd = ETHTOOL_GRINGPARAM }; + struct kernel_ethtool_ringparam kernel_ringparam = {}; if (!dev->ethtool_ops->get_ringparam) return -EOPNOTSUPP; - dev->ethtool_ops->get_ringparam(dev, &ringparam); + dev->ethtool_ops->get_ringparam(dev, &ringparam, + &kernel_ringparam, NULL); if (copy_to_user(useraddr, &ringparam, sizeof(ringparam))) return -EFAULT; @@ -1757,6 +1759,7 @@ static int ethtool_get_ringparam(struct net_device *dev, void __user *useraddr) static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) { struct ethtool_ringparam ringparam, max = { .cmd = ETHTOOL_GRINGPARAM }; + struct kernel_ethtool_ringparam kernel_ringparam; int ret; if (!dev->ethtool_ops->set_ringparam || !dev->ethtool_ops->get_ringparam) @@ -1765,7 +1768,7 @@ static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) if (copy_from_user(&ringparam, useraddr, sizeof(ringparam))) return -EFAULT; - dev->ethtool_ops->get_ringparam(dev, &max); + dev->ethtool_ops->get_ringparam(dev, &max, &kernel_ringparam, NULL); /* ensure new ring parameters are within the maximums */ if (ringparam.rx_pending > max.rx_max_pending || @@ -1774,7 +1777,8 @@ static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) ringparam.tx_pending > max.tx_max_pending) return -EINVAL; - ret = dev->ethtool_ops->set_ringparam(dev, &ringparam); + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, + &kernel_ringparam, NULL); if (!ret) ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF, NULL); return ret; diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c index bd8e9a7530eb..450b8866373d 100644 --- a/net/ethtool/rings.c +++ b/net/ethtool/rings.c @@ -26,6 +26,7 @@ static int rings_prepare_data(const struct ethnl_req_info *req_base, struct genl_info *info) { struct rings_reply_data *data = RINGS_REPDATA(reply_base); + struct netlink_ext_ack *extack = info ? info->extack : NULL; struct net_device *dev = reply_base->dev; int ret; @@ -34,7 +35,8 @@ static int rings_prepare_data(const struct ethnl_req_info *req_base, ret = ethnl_ops_begin(dev); if (ret < 0) return ret; - dev->ethtool_ops->get_ringparam(dev, &data->ringparam); + dev->ethtool_ops->get_ringparam(dev, &data->ringparam, + &data->kernel_ringparam, extack); ethnl_ops_complete(dev); return 0; @@ -142,7 +144,7 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) ret = ethnl_ops_begin(dev); if (ret < 0) goto out_rtnl; - ops->get_ringparam(dev, &ringparam); + ops->get_ringparam(dev, &ringparam, &kernel_ringparam, info->extack); ethnl_update_u32(&ringparam.rx_pending, tb[ETHTOOL_A_RINGS_RX], &mod); ethnl_update_u32(&ringparam.rx_mini_pending, @@ -183,7 +185,8 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) goto out_ops; } - ret = dev->ethtool_ops->set_ringparam(dev, &ringparam); + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, + &kernel_ringparam, info->extack); if (ret < 0) goto out_ops; ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF, NULL); diff --git a/net/mac80211/ethtool.c b/net/mac80211/ethtool.c index 99a2e30b3833..b2253df54413 100644 --- a/net/mac80211/ethtool.c +++ b/net/mac80211/ethtool.c @@ -14,7 +14,9 @@ #include "driver-ops.h" static int ieee80211_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct ieee80211_local *local = wiphy_priv(dev->ieee80211_ptr->wiphy); @@ -25,7 +27,9 @@ static int ieee80211_set_ringparam(struct net_device *dev, } static void ieee80211_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct ieee80211_local *local = wiphy_priv(dev->ieee80211_ptr->wiphy); -- cgit v1.2.3-59-g8ed1b From 4b66d2161b8125b6caa6971815e85631cf3cf36f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 19 Nov 2021 07:43:31 -0800 Subject: net: annotate accesses to dev->gso_max_size dev->gso_max_size is written under RTNL protection, or when the device is not yet visible, but is read locklessly. Add the READ_ONCE()/WRITE_ONCE() pairs, and use netif_set_gso_max_size() where we can to better document what is going on. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/ethernet/netronome/nfp/nfp_net_repr.c | 2 +- drivers/net/ethernet/realtek/r8169_main.c | 4 ++-- drivers/net/ethernet/sfc/ef100_nic.c | 2 +- drivers/net/ipvlan/ipvlan_main.c | 4 ++-- drivers/net/macvlan.c | 4 ++-- drivers/net/veth.c | 2 +- drivers/net/vxlan.c | 2 +- include/linux/netdevice.h | 3 ++- net/8021q/vlan.c | 2 +- net/8021q/vlan_dev.c | 2 +- net/bridge/br_if.c | 2 +- net/core/sock.c | 3 ++- net/sctp/output.c | 2 +- 13 files changed, 18 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index 369f6ae700c7..927ddbdf650b 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -286,7 +286,7 @@ nfp_repr_transfer_features(struct net_device *netdev, struct net_device *lower) if (repr->dst->u.port_info.lower_dev != lower) return; - netdev->gso_max_size = lower->gso_max_size; + netif_set_gso_max_size(netdev, lower->gso_max_size); netdev->gso_max_segs = lower->gso_max_segs; netdev_update_features(netdev); diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 14dd04b2fbed..ff222c54299d 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -5390,11 +5390,11 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (rtl_chip_supports_csum_v2(tp)) { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6; - dev->gso_max_size = RTL_GSO_MAX_SIZE_V2; + netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V2); dev->gso_max_segs = RTL_GSO_MAX_SEGS_V2; } else { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO; - dev->gso_max_size = RTL_GSO_MAX_SIZE_V1; + netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V1); dev->gso_max_segs = RTL_GSO_MAX_SEGS_V1; } diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index 6aa81229b68a..d9015a77a6b8 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -993,7 +993,7 @@ static int ef100_process_design_param(struct efx_nic *efx, return 0; case ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_LEN: nic_data->tso_max_payload_len = min_t(u64, reader->value, GSO_MAX_SIZE); - efx->net_dev->gso_max_size = nic_data->tso_max_payload_len; + netif_set_gso_max_size(efx->net_dev, nic_data->tso_max_payload_len); return 0; case ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_NUM_SEGS: nic_data->tso_max_payload_num_segs = min_t(u64, reader->value, 0xffff); diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 1d2f4e7d7324..0ba63eccefa5 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -140,7 +140,7 @@ static int ipvlan_init(struct net_device *dev) dev->vlan_features = phy_dev->vlan_features & IPVLAN_FEATURES; dev->vlan_features |= IPVLAN_ALWAYS_ON_OFLOADS; dev->hw_enc_features |= dev->features; - dev->gso_max_size = phy_dev->gso_max_size; + netif_set_gso_max_size(dev, phy_dev->gso_max_size); dev->gso_max_segs = phy_dev->gso_max_segs; dev->hard_header_len = phy_dev->hard_header_len; @@ -763,7 +763,7 @@ static int ipvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(ipvlan, &port->ipvlans, pnode) { - ipvlan->dev->gso_max_size = dev->gso_max_size; + netif_set_gso_max_size(ipvlan->dev, dev->gso_max_size); ipvlan->dev->gso_max_segs = dev->gso_max_segs; netdev_update_features(ipvlan->dev); } diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index d2f830ec2969..506e9559df80 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -900,7 +900,7 @@ static int macvlan_init(struct net_device *dev) dev->vlan_features = lowerdev->vlan_features & MACVLAN_FEATURES; dev->vlan_features |= ALWAYS_ON_OFFLOADS; dev->hw_enc_features |= dev->features; - dev->gso_max_size = lowerdev->gso_max_size; + netif_set_gso_max_size(dev, lowerdev->gso_max_size); dev->gso_max_segs = lowerdev->gso_max_segs; dev->hard_header_len = lowerdev->hard_header_len; macvlan_set_lockdep_class(dev); @@ -1748,7 +1748,7 @@ static int macvlan_device_event(struct notifier_block *unused, break; case NETDEV_FEAT_CHANGE: list_for_each_entry(vlan, &port->vlans, list) { - vlan->dev->gso_max_size = dev->gso_max_size; + netif_set_gso_max_size(vlan->dev, dev->gso_max_size); vlan->dev->gso_max_segs = dev->gso_max_segs; netdev_update_features(vlan->dev); } diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 50eb43e5bf45..f2f66fb293fd 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1689,7 +1689,7 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, if (ifmp && (dev->ifindex != 0)) peer->ifindex = ifmp->ifi_index; - peer->gso_max_size = dev->gso_max_size; + netif_set_gso_max_size(peer, dev->gso_max_size); peer->gso_max_segs = dev->gso_max_segs; err = register_netdevice(peer); diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 563f86de0e0d..df4f1712be95 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -3811,7 +3811,7 @@ static void vxlan_config_apply(struct net_device *dev, if (lowerdev) { dst->remote_ifindex = conf->remote_ifindex; - dev->gso_max_size = lowerdev->gso_max_size; + netif_set_gso_max_size(dev, lowerdev->gso_max_size); dev->gso_max_segs = lowerdev->gso_max_segs; needed_headroom = lowerdev->hard_header_len; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index cb7f2661d187..14eeb58ed197 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4731,7 +4731,8 @@ static inline bool netif_needs_gso(struct sk_buff *skb, static inline void netif_set_gso_max_size(struct net_device *dev, unsigned int size) { - dev->gso_max_size = size; + /* dev->gso_max_size is read locklessly from sk_setup_caps() */ + WRITE_ONCE(dev->gso_max_size, size); } static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol, diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index a3a0a5e994f5..34c0ffa81e5f 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -322,7 +322,7 @@ static void vlan_transfer_features(struct net_device *dev, { struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); - vlandev->gso_max_size = dev->gso_max_size; + netif_set_gso_max_size(vlandev, dev->gso_max_size); vlandev->gso_max_segs = dev->gso_max_segs; if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index ab6dee28536d..208f6845f6dd 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -573,7 +573,7 @@ static int vlan_dev_init(struct net_device *dev) NETIF_F_ALL_FCOE; dev->features |= dev->hw_features | NETIF_F_LLTX; - dev->gso_max_size = real_dev->gso_max_size; + netif_set_gso_max_size(dev, real_dev->gso_max_size); dev->gso_max_segs = real_dev->gso_max_segs; if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index 64b2d4fb50f5..d3e1c2276551 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -525,7 +525,7 @@ static void br_set_gso_limits(struct net_bridge *br) gso_max_size = min(gso_max_size, p->dev->gso_max_size); gso_max_segs = min(gso_max_segs, p->dev->gso_max_segs); } - br->dev->gso_max_size = gso_max_size; + netif_set_gso_max_size(br->dev, gso_max_size); br->dev->gso_max_segs = gso_max_segs; } diff --git a/net/core/sock.c b/net/core/sock.c index 1f9be266e586..4418e2a07c34 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -2257,7 +2257,8 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) sk->sk_route_caps &= ~NETIF_F_GSO_MASK; } else { sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; - sk->sk_gso_max_size = dst->dev->gso_max_size; + /* pairs with the WRITE_ONCE() in netif_set_gso_max_size() */ + sk->sk_gso_max_size = READ_ONCE(dst->dev->gso_max_size); max_segs = max_t(u32, dst->dev->gso_max_segs, 1); } } diff --git a/net/sctp/output.c b/net/sctp/output.c index cdfdbd353c67..72fe6669c50d 100644 --- a/net/sctp/output.c +++ b/net/sctp/output.c @@ -134,7 +134,7 @@ void sctp_packet_config(struct sctp_packet *packet, __u32 vtag, dst_hold(tp->dst); sk_setup_caps(sk, tp->dst); } - packet->max_size = sk_can_gso(sk) ? tp->dst->dev->gso_max_size + packet->max_size = sk_can_gso(sk) ? READ_ONCE(tp->dst->dev->gso_max_size) : asoc->pathmtu; rcu_read_unlock(); } -- cgit v1.2.3-59-g8ed1b From 6d872df3e3b91532b142de9044e5b4984017a55f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 19 Nov 2021 07:43:32 -0800 Subject: net: annotate accesses to dev->gso_max_segs dev->gso_max_segs is written under RTNL protection, or when the device is not yet visible, but is read locklessly. Add netif_set_gso_max_segs() helper. Add the READ_ONCE()/WRITE_ONCE() pairs, and use netif_set_gso_max_segs() where we can to better document what is going on. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 2 +- drivers/net/ethernet/freescale/fec_main.c | 2 +- drivers/net/ethernet/marvell/mv643xx_eth.c | 2 +- drivers/net/ethernet/marvell/mvneta.c | 2 +- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 2 +- drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c | 2 +- drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c | 2 +- drivers/net/ethernet/netronome/nfp/nfp_net_common.c | 2 +- drivers/net/ethernet/netronome/nfp/nfp_net_repr.c | 4 ++-- drivers/net/ethernet/realtek/r8169_main.c | 4 ++-- drivers/net/ethernet/sfc/ef100_nic.c | 4 ++-- drivers/net/ethernet/sfc/efx.c | 2 +- drivers/net/ethernet/sfc/falcon/efx.c | 2 +- drivers/net/ipvlan/ipvlan_main.c | 4 ++-- drivers/net/macvlan.c | 4 ++-- drivers/net/veth.c | 2 +- drivers/net/vxlan.c | 2 +- include/linux/netdevice.h | 7 +++++++ net/8021q/vlan.c | 2 +- net/8021q/vlan_dev.c | 2 +- net/bridge/br_if.c | 2 +- net/core/dev.c | 2 +- net/core/rtnetlink.c | 4 ++-- net/core/sock.c | 3 ++- 24 files changed, 37 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index ff8da720a33a..cf73eacdda91 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1460,7 +1460,7 @@ done: bond_dev->hw_enc_features |= xfrm_features; #endif /* CONFIG_XFRM_OFFLOAD */ bond_dev->mpls_features = mpls_features; - bond_dev->gso_max_segs = gso_max_segs; + netif_set_gso_max_segs(bond_dev, gso_max_segs); netif_set_gso_max_size(bond_dev, gso_max_size); bond_dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index bc418b910999..f38e70692514 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -3558,7 +3558,7 @@ static int fec_enet_init(struct net_device *ndev) ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; if (fep->quirks & FEC_QUIRK_HAS_CSUM) { - ndev->gso_max_segs = FEC_MAX_TSO_SEGS; + netif_set_gso_max_segs(ndev, FEC_MAX_TSO_SEGS); /* enable hw accelerator */ ndev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 116919730237..105247582684 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -3201,7 +3201,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) dev->hw_features = dev->features; dev->priv_flags |= IFF_UNICAST_FLT; - dev->gso_max_segs = MV643XX_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MV643XX_MAX_TSO_SEGS); /* MTU range: 64 - 9500 */ dev->min_mtu = 64; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 350d24d7ead0..80e4b500695e 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -5335,7 +5335,7 @@ static int mvneta_probe(struct platform_device *pdev) dev->hw_features |= dev->features; dev->vlan_features |= dev->features; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; - dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MVNETA_MAX_TSO_SEGS); /* MTU range: 68 - 9676 */ dev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 8b1ca0b13e88..49fc31fe0db8 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -6857,7 +6857,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, mvpp2_set_hw_csum(port, port->pool_long->id); dev->vlan_features |= features; - dev->gso_max_segs = MVPP2_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MVPP2_MAX_TSO_SEGS); dev->priv_flags |= IFF_UNICAST_FLT; /* MTU range: 68 - 9704 */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 1e0d0c9c1dac..1333edf1c361 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -2741,7 +2741,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_gso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2_netdev_ops; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 78944ad3492f..254bebffe8c1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -663,7 +663,7 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_NTUPLE; netdev->hw_features |= NETIF_F_RXALL; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_gso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2vf_netdev_ops; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 850bfdf83d0a..6e38da4ad554 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -4097,7 +4097,7 @@ static void nfp_net_netdev_init(struct nfp_net *nn) netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = nn->max_mtu; - netdev->gso_max_segs = NFP_NET_LSO_MAX_SEGS; + netif_set_gso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS); netif_carrier_off(netdev); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index 927ddbdf650b..181ac8e789a3 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -287,7 +287,7 @@ nfp_repr_transfer_features(struct net_device *netdev, struct net_device *lower) return; netif_set_gso_max_size(netdev, lower->gso_max_size); - netdev->gso_max_segs = lower->gso_max_segs; + netif_set_gso_max_segs(netdev, lower->gso_max_segs); netdev_update_features(netdev); } @@ -381,7 +381,7 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev, /* Advertise but disable TSO by default. */ netdev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6); - netdev->gso_max_segs = NFP_NET_LSO_MAX_SEGS; + netif_set_gso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS); netdev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; netdev->features |= NETIF_F_LLTX; diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index ff222c54299d..1f6313c8d308 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -5391,11 +5391,11 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (rtl_chip_supports_csum_v2(tp)) { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6; netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V2); - dev->gso_max_segs = RTL_GSO_MAX_SEGS_V2; + netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V2); } else { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO; netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V1); - dev->gso_max_segs = RTL_GSO_MAX_SEGS_V1; + netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V1); } dev->hw_features |= NETIF_F_RXALL; diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index d9015a77a6b8..5a6fed33a1d7 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -997,7 +997,7 @@ static int ef100_process_design_param(struct efx_nic *efx, return 0; case ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_NUM_SEGS: nic_data->tso_max_payload_num_segs = min_t(u64, reader->value, 0xffff); - efx->net_dev->gso_max_segs = nic_data->tso_max_payload_num_segs; + netif_set_gso_max_segs(efx->net_dev, nic_data->tso_max_payload_num_segs); return 0; case ESE_EF100_DP_GZ_TSO_MAX_NUM_FRAMES: nic_data->tso_max_frames = min_t(u64, reader->value, 0xffff); @@ -1122,7 +1122,7 @@ static int ef100_probe_main(struct efx_nic *efx) nic_data->tso_max_frames = ESE_EF100_DP_GZ_TSO_MAX_NUM_FRAMES_DEFAULT; nic_data->tso_max_payload_num_segs = ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_NUM_SEGS_DEFAULT; nic_data->tso_max_payload_len = ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_LEN_DEFAULT; - net_dev->gso_max_segs = ESE_EF100_DP_GZ_TSO_MAX_HDR_NUM_SEGS_DEFAULT; + netif_set_gso_max_segs(net_dev, ESE_EF100_DP_GZ_TSO_MAX_HDR_NUM_SEGS_DEFAULT); /* Read design parameters */ rc = ef100_check_design_params(efx); if (rc) { diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 6960a2fe2b53..a8c252e2b252 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -709,7 +709,7 @@ static int efx_register_netdev(struct efx_nic *efx) if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) net_dev->priv_flags |= IFF_UNICAST_FLT; net_dev->ethtool_ops = &efx_ethtool_ops; - net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; + netif_set_gso_max_segs(net_dev, EFX_TSO_MAX_SEGS); net_dev->min_mtu = EFX_MIN_MTU; net_dev->max_mtu = EFX_MAX_MTU; diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c index 314c9c69eb0e..60c595ef7589 100644 --- a/drivers/net/ethernet/sfc/falcon/efx.c +++ b/drivers/net/ethernet/sfc/falcon/efx.c @@ -2267,7 +2267,7 @@ static int ef4_register_netdev(struct ef4_nic *efx) net_dev->irq = efx->pci_dev->irq; net_dev->netdev_ops = &ef4_netdev_ops; net_dev->ethtool_ops = &ef4_ethtool_ops; - net_dev->gso_max_segs = EF4_TSO_MAX_SEGS; + netif_set_gso_max_segs(net_dev, EF4_TSO_MAX_SEGS); net_dev->min_mtu = EF4_MIN_MTU; net_dev->max_mtu = EF4_MAX_MTU; diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 0ba63eccefa5..20da0b2bd246 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -141,7 +141,7 @@ static int ipvlan_init(struct net_device *dev) dev->vlan_features |= IPVLAN_ALWAYS_ON_OFLOADS; dev->hw_enc_features |= dev->features; netif_set_gso_max_size(dev, phy_dev->gso_max_size); - dev->gso_max_segs = phy_dev->gso_max_segs; + netif_set_gso_max_segs(dev, phy_dev->gso_max_segs); dev->hard_header_len = phy_dev->hard_header_len; netdev_lockdep_set_classes(dev); @@ -764,7 +764,7 @@ static int ipvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(ipvlan, &port->ipvlans, pnode) { netif_set_gso_max_size(ipvlan->dev, dev->gso_max_size); - ipvlan->dev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_segs(ipvlan->dev, dev->gso_max_segs); netdev_update_features(ipvlan->dev); } break; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 506e9559df80..75b453acd778 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -901,7 +901,7 @@ static int macvlan_init(struct net_device *dev) dev->vlan_features |= ALWAYS_ON_OFFLOADS; dev->hw_enc_features |= dev->features; netif_set_gso_max_size(dev, lowerdev->gso_max_size); - dev->gso_max_segs = lowerdev->gso_max_segs; + netif_set_gso_max_segs(dev, lowerdev->gso_max_segs); dev->hard_header_len = lowerdev->hard_header_len; macvlan_set_lockdep_class(dev); @@ -1749,7 +1749,7 @@ static int macvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(vlan, &port->vlans, list) { netif_set_gso_max_size(vlan->dev, dev->gso_max_size); - vlan->dev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_segs(vlan->dev, dev->gso_max_segs); netdev_update_features(vlan->dev); } break; diff --git a/drivers/net/veth.c b/drivers/net/veth.c index f2f66fb293fd..5ca0a899101d 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1690,7 +1690,7 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, peer->ifindex = ifmp->ifi_index; netif_set_gso_max_size(peer, dev->gso_max_size); - peer->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_segs(peer, dev->gso_max_segs); err = register_netdevice(peer); put_net(net); diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index df4f1712be95..82bd794fceb3 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -3812,7 +3812,7 @@ static void vxlan_config_apply(struct net_device *dev, dst->remote_ifindex = conf->remote_ifindex; netif_set_gso_max_size(dev, lowerdev->gso_max_size); - dev->gso_max_segs = lowerdev->gso_max_segs; + netif_set_gso_max_segs(dev, lowerdev->gso_max_segs); needed_headroom = lowerdev->hard_header_len; needed_headroom += lowerdev->needed_headroom; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 14eeb58ed197..df049864661d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4735,6 +4735,13 @@ static inline void netif_set_gso_max_size(struct net_device *dev, WRITE_ONCE(dev->gso_max_size, size); } +static inline void netif_set_gso_max_segs(struct net_device *dev, + unsigned int segs) +{ + /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ + WRITE_ONCE(dev->gso_max_segs, segs); +} + static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol, int pulled_hlen, u16 mac_offset, int mac_len) diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 34c0ffa81e5f..0c00200c9cb2 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -323,7 +323,7 @@ static void vlan_transfer_features(struct net_device *dev, struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); netif_set_gso_max_size(vlandev, dev->gso_max_size); - vlandev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_segs(vlandev, dev->gso_max_segs); if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) vlandev->hard_header_len = dev->hard_header_len; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 208f6845f6dd..e9499a7c19fe 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -574,7 +574,7 @@ static int vlan_dev_init(struct net_device *dev) dev->features |= dev->hw_features | NETIF_F_LLTX; netif_set_gso_max_size(dev, real_dev->gso_max_size); - dev->gso_max_segs = real_dev->gso_max_segs; + netif_set_gso_max_segs(dev, real_dev->gso_max_segs); if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index d3e1c2276551..3915832a03c2 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -526,7 +526,7 @@ static void br_set_gso_limits(struct net_bridge *br) gso_max_segs = min(gso_max_segs, p->dev->gso_max_segs); } netif_set_gso_max_size(br->dev, gso_max_size); - br->dev->gso_max_segs = gso_max_segs; + netif_set_gso_max_segs(br->dev, gso_max_segs); } /* diff --git a/net/core/dev.c b/net/core/dev.c index 9219e319e901..9c4fc8c3f981 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3396,7 +3396,7 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, { u16 gso_segs = skb_shinfo(skb)->gso_segs; - if (gso_segs > dev->gso_max_segs) + if (gso_segs > READ_ONCE(dev->gso_max_segs)) return features & ~NETIF_F_GSO_MASK; if (!skb_shinfo(skb)->gso_type) { diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 2af8aeeadadf..fd030e02f16d 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2768,7 +2768,7 @@ static int do_setlink(const struct sk_buff *skb, } if (dev->gso_max_segs ^ max_segs) { - dev->gso_max_segs = max_segs; + netif_set_gso_max_segs(dev, max_segs); status |= DO_SETLINK_MODIFIED; } } @@ -3222,7 +3222,7 @@ struct net_device *rtnl_create_link(struct net *net, const char *ifname, if (tb[IFLA_GSO_MAX_SIZE]) netif_set_gso_max_size(dev, nla_get_u32(tb[IFLA_GSO_MAX_SIZE])); if (tb[IFLA_GSO_MAX_SEGS]) - dev->gso_max_segs = nla_get_u32(tb[IFLA_GSO_MAX_SEGS]); + netif_set_gso_max_segs(dev, nla_get_u32(tb[IFLA_GSO_MAX_SEGS])); return dev; } diff --git a/net/core/sock.c b/net/core/sock.c index 4418e2a07c34..31a2b79c9b38 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -2259,7 +2259,8 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; /* pairs with the WRITE_ONCE() in netif_set_gso_max_size() */ sk->sk_gso_max_size = READ_ONCE(dst->dev->gso_max_size); - max_segs = max_t(u32, dst->dev->gso_max_segs, 1); + /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ + max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1); } } sk->sk_gso_max_segs = max_segs; -- cgit v1.2.3-59-g8ed1b From 291dcae39bc482f7edc91a50d97027327e0cf5f9 Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Fri, 19 Nov 2021 10:58:09 -0500 Subject: net: phylink: Add helpers for c22 registers without MDIO Some devices expose memory-mapped c22-compliant PHYs. Because these devices do not have an MDIO bus, we cannot use the existing helpers. Refactor the existing helpers to allow supplying the values for c22 registers directly, instead of using MDIO to access them. Only get_state and set_advertisement are converted, since they contain the most complex logic. Because set_advertisement is never actually used outside phylink_mii_c22_pcs_config, move the MDIO-writing part into that function. Because some modes do not need the advertisement register set at all, we use -EINVAL for this purpose. Additionally, a new function phylink_pcs_enable_an is provided to determine whether to enable autonegotiation. Signed-off-by: Sean Anderson Reviewed-by: Russell King (Oracle) Signed-off-by: David S. Miller --- drivers/net/phy/phylink.c | 104 ++++++++++++++++++++++++++-------------------- include/linux/phylink.h | 7 ++-- 2 files changed, 64 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index da17b874a5e7..2d201a795775 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -2815,31 +2815,22 @@ void phylink_decode_usxgmii_word(struct phylink_link_state *state, EXPORT_SYMBOL_GPL(phylink_decode_usxgmii_word); /** - * phylink_mii_c22_pcs_get_state() - read the MAC PCS state - * @pcs: a pointer to a &struct mdio_device. + * phylink_mii_c22_pcs_decode_state() - Decode MAC PCS state from MII registers * @state: a pointer to a &struct phylink_link_state. + * @bmsr: The value of the %MII_BMSR register + * @lpa: The value of the %MII_LPA register * * Helper for MAC PCS supporting the 802.3 clause 22 register set for * clause 37 negotiation and/or SGMII control. * - * Read the MAC PCS state from the MII device configured in @config and - * parse the Clause 37 or Cisco SGMII link partner negotiation word into - * the phylink @state structure. This is suitable to be directly plugged - * into the mac_pcs_get_state() member of the struct phylink_mac_ops - * structure. + * Parse the Clause 37 or Cisco SGMII link partner negotiation word into + * the phylink @state structure. This is suitable to be used for implementing + * the mac_pcs_get_state() member of the struct phylink_mac_ops structure if + * accessing @bmsr and @lpa cannot be done with MDIO directly. */ -void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, - struct phylink_link_state *state) +void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, + u16 bmsr, u16 lpa) { - int bmsr, lpa; - - bmsr = mdiodev_read(pcs, MII_BMSR); - lpa = mdiodev_read(pcs, MII_LPA); - if (bmsr < 0 || lpa < 0) { - state->link = false; - return; - } - state->link = !!(bmsr & BMSR_LSTATUS); state->an_complete = !!(bmsr & BMSR_ANEGCOMPLETE); /* If there is no link or autonegotiation is disabled, the LP advertisement @@ -2867,28 +2858,54 @@ void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, break; } } +EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_decode_state); + +/** + * phylink_mii_c22_pcs_get_state() - read the MAC PCS state + * @pcs: a pointer to a &struct mdio_device. + * @state: a pointer to a &struct phylink_link_state. + * + * Helper for MAC PCS supporting the 802.3 clause 22 register set for + * clause 37 negotiation and/or SGMII control. + * + * Read the MAC PCS state from the MII device configured in @config and + * parse the Clause 37 or Cisco SGMII link partner negotiation word into + * the phylink @state structure. This is suitable to be directly plugged + * into the mac_pcs_get_state() member of the struct phylink_mac_ops + * structure. + */ +void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, + struct phylink_link_state *state) +{ + int bmsr, lpa; + + bmsr = mdiodev_read(pcs, MII_BMSR); + lpa = mdiodev_read(pcs, MII_LPA); + if (bmsr < 0 || lpa < 0) { + state->link = false; + return; + } + + phylink_mii_c22_pcs_decode_state(state, bmsr, lpa); +} EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_get_state); /** - * phylink_mii_c22_pcs_set_advertisement() - configure the clause 37 PCS + * phylink_mii_c22_pcs_encode_advertisement() - configure the clause 37 PCS * advertisement - * @pcs: a pointer to a &struct mdio_device. * @interface: the PHY interface mode being configured * @advertising: the ethtool advertisement mask * * Helper for MAC PCS supporting the 802.3 clause 22 register set for * clause 37 negotiation and/or SGMII control. * - * Configure the clause 37 PCS advertisement as specified by @state. This - * does not trigger a renegotiation; phylink will do that via the - * mac_an_restart() method of the struct phylink_mac_ops structure. + * Encode the clause 37 PCS advertisement as specified by @interface and + * @advertising. * - * Returns negative error code on failure to configure the advertisement, - * zero if no change has been made, or one if the advertisement has changed. + * Return: The new value for @adv, or ``-EINVAL`` if it should not be changed. */ -int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, - phy_interface_t interface, - const unsigned long *advertising) +int phylink_mii_c22_pcs_encode_advertisement(phy_interface_t interface, + const unsigned long *advertising) { u16 adv; @@ -2902,18 +2919,15 @@ int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, if (linkmode_test_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, advertising)) adv |= ADVERTISE_1000XPSE_ASYM; - - return mdiodev_modify_changed(pcs, MII_ADVERTISE, 0xffff, adv); - + return adv; case PHY_INTERFACE_MODE_SGMII: - return mdiodev_modify_changed(pcs, MII_ADVERTISE, 0xffff, 0x0001); - + return 0x0001; default: /* Nothing to do for other modes */ - return 0; + return -EINVAL; } } -EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_set_advertisement); +EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_encode_advertisement); /** * phylink_mii_c22_pcs_config() - configure clause 22 PCS @@ -2931,16 +2945,18 @@ int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising) { - bool changed; + bool changed = 0; u16 bmcr; - int ret; + int ret, adv; - ret = phylink_mii_c22_pcs_set_advertisement(pcs, interface, - advertising); - if (ret < 0) - return ret; - - changed = ret > 0; + adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (adv >= 0) { + ret = mdiobus_modify_changed(pcs->bus, pcs->addr, + MII_ADVERTISE, 0xffff, adv); + if (ret < 0) + return ret; + changed = ret; + } /* Ensure ISOLATE bit is disabled */ if (mode == MLO_AN_INBAND && @@ -2953,7 +2969,7 @@ int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, if (ret < 0) return ret; - return changed ? 1 : 0; + return changed; } EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_config); diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 3563820a1765..01224235df0f 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -527,11 +527,12 @@ void phylink_set_port_modes(unsigned long *bits); void phylink_set_10g_modes(unsigned long *mask); void phylink_helper_basex_speed(struct phylink_link_state *state); +void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, + u16 bmsr, u16 lpa); void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, struct phylink_link_state *state); -int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, - phy_interface_t interface, - const unsigned long *advertising); +int phylink_mii_c22_pcs_encode_advertisement(phy_interface_t interface, + const unsigned long *advertising); int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising); -- cgit v1.2.3-59-g8ed1b From c4804670026b93f4ebddda30af89fd737bf93931 Mon Sep 17 00:00:00 2001 From: M Chetan Kumar Date: Sat, 20 Nov 2021 21:51:54 +0530 Subject: net: wwan: common debugfs base dir for wwan device This patch set brings in a common debugfs base directory i.e. /sys/kernel/debugfs/wwan/ in WWAN Subsystem for a WWAN device instance. So that it avoids driver polluting debugfs root with unrelated directories & possible name collusion. Having a common debugfs base directory for WWAN drivers eases user to match control devices with debugfs entries. WWAN Subsystem creates dentry (/sys/kernel/debugfs/wwan) on module load & removes dentry on module unload. When driver registers a new wwan device, dentry (wwanX) is created for WWAN device instance & on driver unregister dentry is removed. New API is introduced to return the wwan device instance dentry so that driver can create debugfs entries under it. Signed-off-by: M Chetan Kumar Reviewed-by: Loic Poulain Signed-off-by: David S. Miller --- drivers/net/wwan/wwan_core.c | 30 ++++++++++++++++++++++++++++-- include/linux/wwan.h | 2 ++ 2 files changed, 30 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index d293ab688044..5bf62dc35ac7 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ static DEFINE_IDA(minors); /* minors for WWAN port chardevs */ static DEFINE_IDA(wwan_dev_ids); /* for unique WWAN device IDs */ static struct class *wwan_class; static int wwan_major; +static struct dentry *wwan_debugfs_dir; #define to_wwan_dev(d) container_of(d, struct wwan_device, dev) #define to_wwan_port(d) container_of(d, struct wwan_port, dev) @@ -40,6 +42,7 @@ static int wwan_major; * @port_id: Current available port ID to pick. * @ops: wwan device ops * @ops_ctxt: context to pass to ops + * @debugfs_dir: WWAN device debugfs dir */ struct wwan_device { unsigned int id; @@ -47,6 +50,7 @@ struct wwan_device { atomic_t port_id; const struct wwan_ops *ops; void *ops_ctxt; + struct dentry *debugfs_dir; }; /** @@ -142,6 +146,18 @@ static struct wwan_device *wwan_dev_get_by_name(const char *name) return to_wwan_dev(dev); } +struct dentry *wwan_get_debugfs_dir(struct device *parent) +{ + struct wwan_device *wwandev; + + wwandev = wwan_dev_get_by_parent(parent); + if (IS_ERR(wwandev)) + return ERR_CAST(wwandev); + + return wwandev->debugfs_dir; +} +EXPORT_SYMBOL_GPL(wwan_get_debugfs_dir); + /* This function allocates and registers a new WWAN device OR if a WWAN device * already exist for the given parent, it gets a reference and return it. * This function is not exported (for now), it is called indirectly via @@ -150,6 +166,7 @@ static struct wwan_device *wwan_dev_get_by_name(const char *name) static struct wwan_device *wwan_create_dev(struct device *parent) { struct wwan_device *wwandev; + const char *wwandev_name; int err, id; /* The 'find-alloc-register' operation must be protected against @@ -189,6 +206,10 @@ static struct wwan_device *wwan_create_dev(struct device *parent) goto done_unlock; } + wwandev_name = kobject_name(&wwandev->dev.kobj); + wwandev->debugfs_dir = debugfs_create_dir(wwandev_name, + wwan_debugfs_dir); + done_unlock: mutex_unlock(&wwan_register_lock); @@ -218,10 +239,12 @@ static void wwan_remove_dev(struct wwan_device *wwandev) else ret = device_for_each_child(&wwandev->dev, NULL, is_wwan_child); - if (!ret) + if (!ret) { + debugfs_remove_recursive(wwandev->debugfs_dir); device_unregister(&wwandev->dev); - else + } else { put_device(&wwandev->dev); + } mutex_unlock(&wwan_register_lock); } @@ -1117,6 +1140,8 @@ static int __init wwan_init(void) goto destroy; } + wwan_debugfs_dir = debugfs_create_dir("wwan", NULL); + return 0; destroy: @@ -1128,6 +1153,7 @@ unregister: static void __exit wwan_exit(void) { + debugfs_remove_recursive(wwan_debugfs_dir); __unregister_chrdev(wwan_major, 0, WWAN_MAX_MINORS, "wwan_port"); rtnl_link_unregister(&wwan_rtnl_link_ops); class_destroy(wwan_class); diff --git a/include/linux/wwan.h b/include/linux/wwan.h index 9fac819f92e3..1646aa3e6779 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -171,4 +171,6 @@ int wwan_register_ops(struct device *parent, const struct wwan_ops *ops, void wwan_unregister_ops(struct device *parent); +struct dentry *wwan_get_debugfs_dir(struct device *parent); + #endif /* __WWAN_H */ -- cgit v1.2.3-59-g8ed1b From fba84957e2e2e201cf4e352efe0c7cac0fbb5d5d Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sat, 20 Nov 2021 16:31:48 -0800 Subject: skbuff: Move conditional preprocessor directives out of struct sk_buff In preparation for using the struct_group() macro in struct sk_buff, move the conditional preprocessor directives out of the region of struct sk_buff that will be enclosed by struct_group(). While GCC and Clang are happy with conditional preprocessor directives here, sparse is not, even under -Wno-directive-within-macro[1], as would be seen under a C=1 build: net/core/filter.c: note: in included file (through include/linux/netlink.h, include/linux/sock_diag.h): ./include/linux/skbuff.h:820:1: warning: directive in macro's argument list ./include/linux/skbuff.h:822:1: warning: directive in macro's argument list ./include/linux/skbuff.h:846:1: warning: directive in macro's argument list ./include/linux/skbuff.h:848:1: warning: directive in macro's argument list Additionally remove empty macro argument definitions and usage. "objdump -d" shows no object code differences. [1] https://www.spinics.net/lists/linux-sparse/msg10857.html Signed-off-by: Kees Cook Signed-off-by: David S. Miller --- include/linux/skbuff.h | 36 +++++++++++++++++++----------------- net/core/filter.c | 10 +++++----- 2 files changed, 24 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 059b6266dcd7..c7889afe0d0d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -795,7 +795,7 @@ struct sk_buff { #else #define CLONED_MASK 1 #endif -#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset) +#define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset) /* private: */ __u8 __cloned_offset[0]; @@ -818,18 +818,10 @@ struct sk_buff { __u32 headers_start[0]; /* public: */ -/* if you move pkt_type around you also must adapt those constants */ -#ifdef __BIG_ENDIAN_BITFIELD -#define PKT_TYPE_MAX (7 << 5) -#else -#define PKT_TYPE_MAX 7 -#endif -#define PKT_TYPE_OFFSET() offsetof(struct sk_buff, __pkt_type_offset) - /* private: */ __u8 __pkt_type_offset[0]; /* public: */ - __u8 pkt_type:3; + __u8 pkt_type:3; /* see PKT_TYPE_MAX */ __u8 ignore_df:1; __u8 nf_trace:1; __u8 ip_summed:2; @@ -845,16 +837,10 @@ struct sk_buff { __u8 encap_hdr_csum:1; __u8 csum_valid:1; -#ifdef __BIG_ENDIAN_BITFIELD -#define PKT_VLAN_PRESENT_BIT 7 -#else -#define PKT_VLAN_PRESENT_BIT 0 -#endif -#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset) /* private: */ __u8 __pkt_vlan_present_offset[0]; /* public: */ - __u8 vlan_present:1; + __u8 vlan_present:1; /* See PKT_VLAN_PRESENT_BIT */ __u8 csum_complete_sw:1; __u8 csum_level:2; __u8 csum_not_inet:1; @@ -953,6 +939,22 @@ struct sk_buff { #endif }; +/* if you move pkt_type around you also must adapt those constants */ +#ifdef __BIG_ENDIAN_BITFIELD +#define PKT_TYPE_MAX (7 << 5) +#else +#define PKT_TYPE_MAX 7 +#endif +#define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset) + +/* if you move pkt_vlan_present around you also must adapt these constants */ +#ifdef __BIG_ENDIAN_BITFIELD +#define PKT_VLAN_PRESENT_BIT 7 +#else +#define PKT_VLAN_PRESENT_BIT 0 +#endif +#define PKT_VLAN_PRESENT_OFFSET offsetof(struct sk_buff, __pkt_vlan_present_offset) + #ifdef __KERNEL__ /* * Handling routines are only of interest to the kernel diff --git a/net/core/filter.c b/net/core/filter.c index efebbaf4e00b..8271624a19aa 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -301,7 +301,7 @@ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, break; case SKF_AD_PKTTYPE: - *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX); #ifdef __BIG_ENDIAN_BITFIELD *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5); @@ -323,7 +323,7 @@ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, offsetof(struct sk_buff, vlan_tci)); break; case SKF_AD_VLAN_TAG_PRESENT: - *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET); if (PKT_VLAN_PRESENT_BIT) *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT); if (PKT_VLAN_PRESENT_BIT < 7) @@ -8029,7 +8029,7 @@ static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write, * (Fast-path, otherwise approximation that we might be * a clone, do the rest in helper.) */ - *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK); *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7); @@ -8617,7 +8617,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, case offsetof(struct __sk_buff, pkt_type): *target_size = 1; *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, - PKT_TYPE_OFFSET()); + PKT_TYPE_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX); #ifdef __BIG_ENDIAN_BITFIELD *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5); @@ -8642,7 +8642,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, case offsetof(struct __sk_buff, vlan_present): *target_size = 1; *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, - PKT_VLAN_PRESENT_OFFSET()); + PKT_VLAN_PRESENT_OFFSET); if (PKT_VLAN_PRESENT_BIT) *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT); if (PKT_VLAN_PRESENT_BIT < 7) -- cgit v1.2.3-59-g8ed1b From 03f61041c17914355dde7261be9ccdc821ddd454 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sat, 20 Nov 2021 16:31:49 -0800 Subject: skbuff: Switch structure bounds to struct_group() In preparation for FORTIFY_SOURCE performing compile-time and run-time field bounds checking for memcpy(), memmove(), and memset(), avoid intentionally writing across neighboring fields. Replace the existing empty member position markers "headers_start" and "headers_end" with a struct_group(). This will allow memcpy() and sizeof() to more easily reason about sizes, and improve readability. "pahole" shows no size nor member offset changes to struct sk_buff. "objdump -d" shows no object code changes (outside of WARNs affected by source line number changes). Signed-off-by: Kees Cook Reviewed-by: Gustavo A. R. Silva Reviewed-by: Jason A. Donenfeld # drivers/net/wireguard/* Link: https://lore.kernel.org/lkml/20210728035006.GD35706@embeddedor Signed-off-by: David S. Miller --- drivers/net/wireguard/queueing.h | 4 +--- include/linux/skbuff.h | 10 +++------- net/core/skbuff.c | 14 +++++--------- 3 files changed, 9 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index 4ef2944a68bc..52da5e963003 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -79,9 +79,7 @@ static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) u8 sw_hash = skb->sw_hash; u32 hash = skb->hash; skb_scrub_packet(skb, true); - memset(&skb->headers_start, 0, - offsetof(struct sk_buff, headers_end) - - offsetof(struct sk_buff, headers_start)); + memset(&skb->headers, 0, sizeof(skb->headers)); if (encapsulating) { skb->l4_hash = l4_hash; skb->sw_hash = sw_hash; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c7889afe0d0d..eba256af64a5 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -811,12 +811,10 @@ struct sk_buff { __u8 active_extensions; #endif - /* fields enclosed in headers_start/headers_end are copied + /* Fields enclosed in headers group are copied * using a single memcpy() in __copy_skb_header() */ - /* private: */ - __u32 headers_start[0]; - /* public: */ + struct_group(headers, /* private: */ __u8 __pkt_type_offset[0]; @@ -921,9 +919,7 @@ struct sk_buff { u64 kcov_handle; #endif - /* private: */ - __u32 headers_end[0]; - /* public: */ + ); /* end headers group */ /* These elements must be at the end, see alloc_skb() for details. */ sk_buff_data_t tail; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 73720bc1daa3..a33247fdb8f5 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -992,12 +992,10 @@ void napi_consume_skb(struct sk_buff *skb, int budget) } EXPORT_SYMBOL(napi_consume_skb); -/* Make sure a field is enclosed inside headers_start/headers_end section */ +/* Make sure a field is contained by headers group */ #define CHECK_SKB_FIELD(field) \ - BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ - offsetof(struct sk_buff, headers_start)); \ - BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ - offsetof(struct sk_buff, headers_end)); \ + BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ + offsetof(struct sk_buff, headers.field)); \ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) { @@ -1009,14 +1007,12 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) __skb_ext_copy(new, old); __nf_copy(new, old, false); - /* Note : this field could be in headers_start/headers_end section + /* Note : this field could be in the headers group. * It is not yet because we do not want to have a 16 bit hole */ new->queue_mapping = old->queue_mapping; - memcpy(&new->headers_start, &old->headers_start, - offsetof(struct sk_buff, headers_end) - - offsetof(struct sk_buff, headers_start)); + memcpy(&new->headers, &old->headers, sizeof(new->headers)); CHECK_SKB_FIELD(protocol); CHECK_SKB_FIELD(csum); CHECK_SKB_FIELD(hash); -- cgit v1.2.3-59-g8ed1b From e523af4ee56090fbdd9cf474752448d35930bcd4 Mon Sep 17 00:00:00 2001 From: Shiraz Saleem Date: Mon, 18 Oct 2021 18:16:02 -0500 Subject: net/ice: Add support for enable_iwarp and enable_roce devlink param Allow support for 'enable_iwarp' and 'enable_roce' devlink params to turn on/off iWARP or RoCE protocol support for E800 devices. For example, a user can turn on iWARP functionality with, devlink dev param set pci/0000:07:00.0 name enable_iwarp value true cmode runtime This add an iWARP auxiliary rdma device, ice.iwarp.<>, under this PF. A user request to enable both iWARP and RoCE under the same PF is rejected since this device does not support both protocols simultaneously on the same port. Signed-off-by: Shiraz Saleem Tested-by: Leszek Kaliszczuk Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/ice/ice.h | 1 + drivers/net/ethernet/intel/ice/ice_devlink.c | 144 +++++++++++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_devlink.h | 6 ++ drivers/net/ethernet/intel/ice/ice_idc.c | 4 +- drivers/net/ethernet/intel/ice/ice_main.c | 9 +- include/linux/net/intel/iidc.h | 7 +- 6 files changed, 166 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index b2db39ee5f85..b67ad51cbcc9 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -576,6 +576,7 @@ struct ice_pf { struct ice_hw_port_stats stats_prev; struct ice_hw hw; u8 stat_prev_loaded:1; /* has previous stats been loaded */ + u8 rdma_mode; u16 dcbx_cap; u32 tx_timeout_count; unsigned long tx_timeout_last_recovery; diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index b9bd9f9472f6..478412b28a76 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -430,6 +430,120 @@ static const struct devlink_ops ice_devlink_ops = { .flash_update = ice_devlink_flash_update, }; +static int +ice_devlink_enable_roce_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + + ctx->val.vbool = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2; + + return 0; +} + +static int +ice_devlink_enable_roce_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + bool roce_ena = ctx->val.vbool; + int ret; + + if (!roce_ena) { + ice_unplug_aux_dev(pf); + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_ROCEV2; + return 0; + } + + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_ROCEV2; + ret = ice_plug_aux_dev(pf); + if (ret) + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_ROCEV2; + + return ret; +} + +static int +ice_devlink_enable_roce_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct ice_pf *pf = devlink_priv(devlink); + + if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + return -EOPNOTSUPP; + + if (pf->rdma_mode & IIDC_RDMA_PROTOCOL_IWARP) { + NL_SET_ERR_MSG_MOD(extack, "iWARP is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int +ice_devlink_enable_iw_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + + ctx->val.vbool = pf->rdma_mode & IIDC_RDMA_PROTOCOL_IWARP; + + return 0; +} + +static int +ice_devlink_enable_iw_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + bool iw_ena = ctx->val.vbool; + int ret; + + if (!iw_ena) { + ice_unplug_aux_dev(pf); + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_IWARP; + return 0; + } + + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_IWARP; + ret = ice_plug_aux_dev(pf); + if (ret) + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_IWARP; + + return ret; +} + +static int +ice_devlink_enable_iw_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct ice_pf *pf = devlink_priv(devlink); + + if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + return -EOPNOTSUPP; + + if (pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2) { + NL_SET_ERR_MSG_MOD(extack, "RoCEv2 is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously"); + return -EOPNOTSUPP; + } + + return 0; +} + +static const struct devlink_param ice_devlink_params[] = { + DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME), + ice_devlink_enable_roce_get, + ice_devlink_enable_roce_set, + ice_devlink_enable_roce_validate), + DEVLINK_PARAM_GENERIC(ENABLE_IWARP, BIT(DEVLINK_PARAM_CMODE_RUNTIME), + ice_devlink_enable_iw_get, + ice_devlink_enable_iw_set, + ice_devlink_enable_iw_validate), + +}; + static void ice_devlink_free(void *devlink_ptr) { devlink_free((struct devlink *)devlink_ptr); @@ -484,6 +598,36 @@ void ice_devlink_unregister(struct ice_pf *pf) devlink_unregister(priv_to_devlink(pf)); } +int ice_devlink_register_params(struct ice_pf *pf) +{ + struct devlink *devlink = priv_to_devlink(pf); + union devlink_param_value value; + int err; + + err = devlink_params_register(devlink, ice_devlink_params, + ARRAY_SIZE(ice_devlink_params)); + if (err) + return err; + + value.vbool = false; + devlink_param_driverinit_value_set(devlink, + DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP, + value); + + value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags) ? true : false; + devlink_param_driverinit_value_set(devlink, + DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, + value); + + return 0; +} + +void ice_devlink_unregister_params(struct ice_pf *pf) +{ + devlink_params_unregister(priv_to_devlink(pf), ice_devlink_params, + ARRAY_SIZE(ice_devlink_params)); +} + /** * ice_devlink_create_pf_port - Create a devlink port for this PF * @pf: the PF to create a devlink port for diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.h b/drivers/net/ethernet/intel/ice/ice_devlink.h index b7f9551e4fc4..faea757fcf5d 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.h +++ b/drivers/net/ethernet/intel/ice/ice_devlink.h @@ -4,10 +4,16 @@ #ifndef _ICE_DEVLINK_H_ #define _ICE_DEVLINK_H_ +enum ice_devlink_param_id { + ICE_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, +}; + struct ice_pf *ice_allocate_pf(struct device *dev); void ice_devlink_register(struct ice_pf *pf); void ice_devlink_unregister(struct ice_pf *pf); +int ice_devlink_register_params(struct ice_pf *pf); +void ice_devlink_unregister_params(struct ice_pf *pf); int ice_devlink_create_pf_port(struct ice_pf *pf); void ice_devlink_destroy_pf_port(struct ice_pf *pf); int ice_devlink_create_vf_port(struct ice_vf *vf); diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c index adcc9a251595..fc3580167e7b 100644 --- a/drivers/net/ethernet/intel/ice/ice_idc.c +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -288,7 +288,7 @@ int ice_plug_aux_dev(struct ice_pf *pf) adev->id = pf->aux_idx; adev->dev.release = ice_adev_release; adev->dev.parent = &pf->pdev->dev; - adev->name = IIDC_RDMA_ROCE_NAME; + adev->name = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? "roce" : "iwarp"; ret = auxiliary_device_init(adev); if (ret) { @@ -335,6 +335,6 @@ int ice_init_rdma(struct ice_pf *pf) dev_err(dev, "failed to reserve vectors for RDMA\n"); return ret; } - + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_ROCEV2; return ice_plug_aux_dev(pf); } diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index f099797f35e3..f2a5f2f965d1 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -4705,6 +4705,10 @@ probe_done: if (err) goto err_netdev_reg; + err = ice_devlink_register_params(pf); + if (err) + goto err_netdev_reg; + /* ready to go, so clear down state bit */ clear_bit(ICE_DOWN, pf->state); if (ice_is_aux_ena(pf)) { @@ -4712,7 +4716,7 @@ probe_done: if (pf->aux_idx < 0) { dev_err(dev, "Failed to allocate device ID for AUX driver\n"); err = -ENOMEM; - goto err_netdev_reg; + goto err_devlink_reg_param; } err = ice_init_rdma(pf); @@ -4731,6 +4735,8 @@ probe_done: err_init_aux_unroll: pf->adev = NULL; ida_free(&ice_aux_ida, pf->aux_idx); +err_devlink_reg_param: + ice_devlink_unregister_params(pf); err_netdev_reg: err_send_version_unroll: ice_vsi_release_all(pf); @@ -4845,6 +4851,7 @@ static void ice_remove(struct pci_dev *pdev) ice_unplug_aux_dev(pf); if (pf->aux_idx >= 0) ida_free(&ice_aux_ida, pf->aux_idx); + ice_devlink_unregister_params(pf); set_bit(ICE_DOWN, pf->state); mutex_destroy(&(&pf->hw)->fdir_fltr_lock); diff --git a/include/linux/net/intel/iidc.h b/include/linux/net/intel/iidc.h index e32f6712aee0..1289593411d3 100644 --- a/include/linux/net/intel/iidc.h +++ b/include/linux/net/intel/iidc.h @@ -26,6 +26,11 @@ enum iidc_reset_type { IIDC_GLOBR, }; +enum iidc_rdma_protocol { + IIDC_RDMA_PROTOCOL_IWARP = BIT(0), + IIDC_RDMA_PROTOCOL_ROCEV2 = BIT(1), +}; + #define IIDC_MAX_USER_PRIORITY 8 /* Struct to hold per RDMA Qset info */ @@ -70,8 +75,6 @@ int ice_rdma_request_reset(struct ice_pf *pf, enum iidc_reset_type reset_type); int ice_rdma_update_vsi_filter(struct ice_pf *pf, u16 vsi_id, bool enable); void ice_get_qos_params(struct ice_pf *pf, struct iidc_qos_params *qos); -#define IIDC_RDMA_ROCE_NAME "roce" - /* Structure representing auxiliary driver tailored information about the core * PCI dev, each auxiliary driver using the IIDC interface will have an * instance of this struct dedicated to it. -- cgit v1.2.3-59-g8ed1b From 6326948f940dc3f77066d5cdc44ba6afe67830c0 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 29 Sep 2021 11:01:21 -0400 Subject: lsm: security_task_getsecid_subj() -> security_current_getsecid_subj() The security_task_getsecid_subj() LSM hook invites misuse by allowing callers to specify a task even though the hook is only safe when the current task is referenced. Fix this by removing the task_struct argument to the hook, requiring LSM implementations to use the current task. While we are changing the hook declaration we also rename the function to security_current_getsecid_subj() in an effort to reinforce that the hook captures the subjective credentials of the current task and not an arbitrary task on the system. Reviewed-by: Serge Hallyn Reviewed-by: Casey Schaufler Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 3 +-- include/linux/lsm_hooks.h | 8 +++----- include/linux/security.h | 4 ++-- kernel/audit.c | 4 ++-- kernel/auditfilter.c | 3 +-- kernel/auditsc.c | 11 ++++++++++- net/netlabel/netlabel_unlabeled.c | 2 +- net/netlabel/netlabel_user.h | 2 +- security/apparmor/lsm.c | 13 ++++++++++--- security/integrity/ima/ima_appraise.c | 2 +- security/integrity/ima/ima_main.c | 14 +++++++------- security/security.c | 6 +++--- security/selinux/hooks.c | 19 +++---------------- security/smack/smack.h | 16 ---------------- security/smack/smack_lsm.c | 9 ++++----- 15 files changed, 49 insertions(+), 67 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index df8de62f4710..ae2228f0711d 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -206,8 +206,7 @@ LSM_HOOK(int, 0, task_fix_setgid, struct cred *new, const struct cred * old, LSM_HOOK(int, 0, task_setpgid, struct task_struct *p, pid_t pgid) LSM_HOOK(int, 0, task_getpgid, struct task_struct *p) LSM_HOOK(int, 0, task_getsid, struct task_struct *p) -LSM_HOOK(void, LSM_RET_VOID, task_getsecid_subj, - struct task_struct *p, u32 *secid) +LSM_HOOK(void, LSM_RET_VOID, current_getsecid_subj, u32 *secid) LSM_HOOK(void, LSM_RET_VOID, task_getsecid_obj, struct task_struct *p, u32 *secid) LSM_HOOK(int, 0, task_setnice, struct task_struct *p, int nice) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index d45b6f6e27fd..52c1990644b9 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -719,11 +719,9 @@ * @p. * @p contains the task_struct for the process. * Return 0 if permission is granted. - * @task_getsecid_subj: - * Retrieve the subjective security identifier of the task_struct in @p - * and return it in @secid. Special care must be taken to ensure that @p - * is the either the "current" task, or the caller has exclusive access - * to @p. + * @current_getsecid_subj: + * Retrieve the subjective security identifier of the current task and + * return it in @secid. * In case of failure, @secid will be set to zero. * @task_getsecid_obj: * Retrieve the objective security identifier of the task_struct in @p diff --git a/include/linux/security.h b/include/linux/security.h index bbf44a466832..bb301963e333 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -418,7 +418,7 @@ int security_task_fix_setgid(struct cred *new, const struct cred *old, int security_task_setpgid(struct task_struct *p, pid_t pgid); int security_task_getpgid(struct task_struct *p); int security_task_getsid(struct task_struct *p); -void security_task_getsecid_subj(struct task_struct *p, u32 *secid); +void security_current_getsecid_subj(u32 *secid); void security_task_getsecid_obj(struct task_struct *p, u32 *secid); int security_task_setnice(struct task_struct *p, int nice); int security_task_setioprio(struct task_struct *p, int ioprio); @@ -1119,7 +1119,7 @@ static inline int security_task_getsid(struct task_struct *p) return 0; } -static inline void security_task_getsecid_subj(struct task_struct *p, u32 *secid) +static inline void security_current_getsecid_subj(u32 *secid) { *secid = 0; } diff --git a/kernel/audit.c b/kernel/audit.c index 121d37e700a6..d4084751cfe6 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -2132,7 +2132,7 @@ int audit_log_task_context(struct audit_buffer *ab) int error; u32 sid; - security_task_getsecid_subj(current, &sid); + security_current_getsecid_subj(&sid); if (!sid) return 0; @@ -2353,7 +2353,7 @@ int audit_signal_info(int sig, struct task_struct *t) audit_sig_uid = auid; else audit_sig_uid = uid; - security_task_getsecid_subj(current, &audit_sig_sid); + security_current_getsecid_subj(&audit_sig_sid); } return audit_signal_info_syscall(t); diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index d75acb014ccd..4173e771650c 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -1368,8 +1368,7 @@ int audit_filter(int msgtype, unsigned int listtype) case AUDIT_SUBJ_SEN: case AUDIT_SUBJ_CLR: if (f->lsm_rule) { - security_task_getsecid_subj(current, - &sid); + security_current_getsecid_subj(&sid); result = security_audit_rule_match(sid, f->type, f->op, f->lsm_rule); } diff --git a/kernel/auditsc.c b/kernel/auditsc.c index b517947bfa48..fce5d43a933f 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -666,7 +666,16 @@ static int audit_filter_rules(struct task_struct *tsk, logged upon error */ if (f->lsm_rule) { if (need_sid) { - security_task_getsecid_subj(tsk, &sid); + /* @tsk should always be equal to + * @current with the exception of + * fork()/copy_process() in which case + * the new @tsk creds are still a dup + * of @current's creds so we can still + * use security_current_getsecid_subj() + * here even though it always refs + * @current's creds + */ + security_current_getsecid_subj(&sid); need_sid = 0; } result = security_audit_rule_match(sid, f->type, diff --git a/net/netlabel/netlabel_unlabeled.c b/net/netlabel/netlabel_unlabeled.c index 566ba4397ee4..8490e46359ae 100644 --- a/net/netlabel/netlabel_unlabeled.c +++ b/net/netlabel/netlabel_unlabeled.c @@ -1537,7 +1537,7 @@ int __init netlbl_unlabel_defconf(void) /* Only the kernel is allowed to call this function and the only time * it is called is at bootup before the audit subsystem is reporting * messages so don't worry to much about these values. */ - security_task_getsecid_subj(current, &audit_info.secid); + security_current_getsecid_subj(&audit_info.secid); audit_info.loginuid = GLOBAL_ROOT_UID; audit_info.sessionid = 0; diff --git a/net/netlabel/netlabel_user.h b/net/netlabel/netlabel_user.h index 6190cbf94bf0..d6c5b31eb4eb 100644 --- a/net/netlabel/netlabel_user.h +++ b/net/netlabel/netlabel_user.h @@ -32,7 +32,7 @@ */ static inline void netlbl_netlink_auditinfo(struct netlbl_audit *audit_info) { - security_task_getsecid_subj(current, &audit_info->secid); + security_current_getsecid_subj(&audit_info->secid); audit_info->loginuid = audit_get_loginuid(current); audit_info->sessionid = audit_get_sessionid(current); } diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 0d6585056f3d..4f0eecb67dde 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -728,7 +728,14 @@ static void apparmor_bprm_committed_creds(struct linux_binprm *bprm) return; } -static void apparmor_task_getsecid(struct task_struct *p, u32 *secid) +static void apparmor_current_getsecid_subj(u32 *secid) +{ + struct aa_label *label = aa_get_current_label(); + *secid = label->secid; + aa_put_label(label); +} + +static void apparmor_task_getsecid_obj(struct task_struct *p, u32 *secid) { struct aa_label *label = aa_get_task_label(p); *secid = label->secid; @@ -1252,8 +1259,8 @@ static struct security_hook_list apparmor_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(task_free, apparmor_task_free), LSM_HOOK_INIT(task_alloc, apparmor_task_alloc), - LSM_HOOK_INIT(task_getsecid_subj, apparmor_task_getsecid), - LSM_HOOK_INIT(task_getsecid_obj, apparmor_task_getsecid), + LSM_HOOK_INIT(current_getsecid_subj, apparmor_current_getsecid_subj), + LSM_HOOK_INIT(task_getsecid_obj, apparmor_task_getsecid_obj), LSM_HOOK_INIT(task_setrlimit, apparmor_task_setrlimit), LSM_HOOK_INIT(task_kill, apparmor_task_kill), diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index dbba51583e7c..17232bbfb9f9 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -76,7 +76,7 @@ int ima_must_appraise(struct user_namespace *mnt_userns, struct inode *inode, if (!ima_appraise) return 0; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); return ima_match_policy(mnt_userns, inode, current_cred(), secid, func, mask, IMA_APPRAISE | IMA_HASH, NULL, NULL, NULL, NULL); diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 465865412100..8c6e4514d494 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -408,7 +408,7 @@ int ima_file_mmap(struct file *file, unsigned long prot) u32 secid; if (file && (prot & PROT_EXEC)) { - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); return process_measurement(file, current_cred(), secid, NULL, 0, MAY_EXEC, MMAP_CHECK); } @@ -446,7 +446,7 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) !(prot & PROT_EXEC) || (vma->vm_flags & VM_EXEC)) return 0; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); inode = file_inode(vma->vm_file); action = ima_get_action(file_mnt_user_ns(vma->vm_file), inode, current_cred(), secid, MAY_EXEC, MMAP_CHECK, @@ -487,7 +487,7 @@ int ima_bprm_check(struct linux_binprm *bprm) int ret; u32 secid; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); ret = process_measurement(bprm->file, current_cred(), secid, NULL, 0, MAY_EXEC, BPRM_CHECK); if (ret) @@ -512,7 +512,7 @@ int ima_file_check(struct file *file, int mask) { u32 secid; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); return process_measurement(file, current_cred(), secid, NULL, 0, mask & (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND), FILE_CHECK); @@ -709,7 +709,7 @@ int ima_read_file(struct file *file, enum kernel_read_file_id read_id, /* Read entire file for all partial reads. */ func = read_idmap[read_id] ?: FILE_CHECK; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); return process_measurement(file, current_cred(), secid, NULL, 0, MAY_READ, func); } @@ -752,7 +752,7 @@ int ima_post_read_file(struct file *file, void *buf, loff_t size, } func = read_idmap[read_id] ?: FILE_CHECK; - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); return process_measurement(file, current_cred(), secid, buf, size, MAY_READ, func); } @@ -905,7 +905,7 @@ int process_buffer_measurement(struct user_namespace *mnt_userns, * buffer measurements. */ if (func) { - security_task_getsecid_subj(current, &secid); + security_current_getsecid_subj(&secid); action = ima_get_action(mnt_userns, inode, current_cred(), secid, 0, func, &pcr, &template, func_data, NULL); diff --git a/security/security.c b/security/security.c index c88167a414b4..edb922b8bf4a 100644 --- a/security/security.c +++ b/security/security.c @@ -1808,12 +1808,12 @@ int security_task_getsid(struct task_struct *p) return call_int_hook(task_getsid, 0, p); } -void security_task_getsecid_subj(struct task_struct *p, u32 *secid) +void security_current_getsecid_subj(u32 *secid) { *secid = 0; - call_void_hook(task_getsecid_subj, p, secid); + call_void_hook(current_getsecid_subj, secid); } -EXPORT_SYMBOL(security_task_getsecid_subj); +EXPORT_SYMBOL(security_current_getsecid_subj); void security_task_getsecid_obj(struct task_struct *p, u32 *secid) { diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 62d30c0a30c2..726175254f60 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -229,19 +229,6 @@ static inline u32 cred_sid(const struct cred *cred) return tsec->sid; } -/* - * get the subjective security ID of a task - */ -static inline u32 task_sid_subj(const struct task_struct *task) -{ - u32 sid; - - rcu_read_lock(); - sid = cred_sid(rcu_dereference(task->cred)); - rcu_read_unlock(); - return sid; -} - /* * get the objective security ID of a task */ @@ -4205,9 +4192,9 @@ static int selinux_task_getsid(struct task_struct *p) PROCESS__GETSESSION, NULL); } -static void selinux_task_getsecid_subj(struct task_struct *p, u32 *secid) +static void selinux_current_getsecid_subj(u32 *secid) { - *secid = task_sid_subj(p); + *secid = current_sid(); } static void selinux_task_getsecid_obj(struct task_struct *p, u32 *secid) @@ -7159,7 +7146,7 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(task_setpgid, selinux_task_setpgid), LSM_HOOK_INIT(task_getpgid, selinux_task_getpgid), LSM_HOOK_INIT(task_getsid, selinux_task_getsid), - LSM_HOOK_INIT(task_getsecid_subj, selinux_task_getsecid_subj), + LSM_HOOK_INIT(current_getsecid_subj, selinux_current_getsecid_subj), LSM_HOOK_INIT(task_getsecid_obj, selinux_task_getsecid_obj), LSM_HOOK_INIT(task_setnice, selinux_task_setnice), LSM_HOOK_INIT(task_setioprio, selinux_task_setioprio), diff --git a/security/smack/smack.h b/security/smack/smack.h index 99c3422596ab..fc837dcebf96 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -389,22 +389,6 @@ static inline struct smack_known *smk_of_task(const struct task_smack *tsp) return tsp->smk_task; } -static inline struct smack_known *smk_of_task_struct_subj( - const struct task_struct *t) -{ - struct smack_known *skp; - const struct cred *cred; - - rcu_read_lock(); - - cred = rcu_dereference(t->cred); - skp = smk_of_task(smack_cred(cred)); - - rcu_read_unlock(); - - return skp; -} - static inline struct smack_known *smk_of_task_struct_obj( const struct task_struct *t) { diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index efd35b07c7f8..14b279cc75c9 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -2067,15 +2067,14 @@ static int smack_task_getsid(struct task_struct *p) } /** - * smack_task_getsecid_subj - get the subjective secid of the task - * @p: the task + * smack_current_getsecid_subj - get the subjective secid of the current task * @secid: where to put the result * * Sets the secid to contain a u32 version of the task's subjective smack label. */ -static void smack_task_getsecid_subj(struct task_struct *p, u32 *secid) +static void smack_current_getsecid_subj(u32 *secid) { - struct smack_known *skp = smk_of_task_struct_subj(p); + struct smack_known *skp = smk_of_current(); *secid = skp->smk_secid; } @@ -4807,7 +4806,7 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(task_setpgid, smack_task_setpgid), LSM_HOOK_INIT(task_getpgid, smack_task_getpgid), LSM_HOOK_INIT(task_getsid, smack_task_getsid), - LSM_HOOK_INIT(task_getsecid_subj, smack_task_getsecid_subj), + LSM_HOOK_INIT(current_getsecid_subj, smack_current_getsecid_subj), LSM_HOOK_INIT(task_getsecid_obj, smack_task_getsecid_obj), LSM_HOOK_INIT(task_setnice, smack_task_setnice), LSM_HOOK_INIT(task_setioprio, smack_task_setioprio), -- cgit v1.2.3-59-g8ed1b From 91389c390521a02ecfb91270f5b9d7fae4312ae5 Mon Sep 17 00:00:00 2001 From: Samuel Holland Date: Thu, 18 Nov 2021 21:33:37 -0600 Subject: clk: sunxi-ng: Allow the CCU core to be built as a module Like the individual CCU drivers, it can be beneficial for memory consumption of cross-platform configurations to only load the CCU core on the relevant platform. For example, a generic arm64 kernel sees the following improvement when building the CCU core and drivers as modules: before: text data bss dec hex filename 13882360 5251670 360800 19494830 12977ae vmlinux after: text data bss dec hex filename 13734787 5086442 360800 19182029 124b1cd vmlinux So the result is a 390KB total reduction in kernel image size. The one early clock provider (sun5i) requires the core to be built in. Now that loading the MMC driver will trigger loading the CCU core, the MMC timing mode functions do not need a compile-time fallback. Signed-off-by: Samuel Holland Signed-off-by: Maxime Ripard Link: https://lore.kernel.org/r/20211119033338.25486-5-samuel@sholland.org --- drivers/clk/Makefile | 2 +- drivers/clk/sunxi-ng/Kconfig | 3 ++- drivers/clk/sunxi-ng/Makefile | 33 ++++++++++++++++++--------------- drivers/clk/sunxi-ng/ccu_common.c | 3 +++ drivers/mmc/host/Kconfig | 1 + include/linux/clk/sunxi-ng.h | 15 --------------- 6 files changed, 25 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index e42312121e51..6afe36bd2c0a 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -110,7 +110,7 @@ obj-$(CONFIG_PLAT_SPEAR) += spear/ obj-y += sprd/ obj-$(CONFIG_ARCH_STI) += st/ obj-$(CONFIG_ARCH_SUNXI) += sunxi/ -obj-$(CONFIG_SUNXI_CCU) += sunxi-ng/ +obj-y += sunxi-ng/ obj-$(CONFIG_ARCH_TEGRA) += tegra/ obj-y += ti/ obj-$(CONFIG_CLK_UNIPHIER) += uniphier/ diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index de88b6e0ec69..727ff755eca4 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config SUNXI_CCU - bool "Clock support for Allwinner SoCs" + tristate "Clock support for Allwinner SoCs" depends on ARCH_SUNXI || COMPILE_TEST select RESET_CONTROLLER default ARCH_SUNXI @@ -52,6 +52,7 @@ config SUN5I_CCU bool "Support for the Allwinner sun5i family CCM" default MACH_SUN5I depends on MACH_SUN5I || COMPILE_TEST + depends on SUNXI_CCU=y config SUN6I_A31_CCU tristate "Support for the Allwinner A31/A31s CCU" diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index 1020ed49a588..659d55150c32 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -1,24 +1,27 @@ # SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SUNXI_CCU) += sunxi-ccu.o + # Common objects -obj-y += ccu_common.o -obj-y += ccu_mmc_timing.o -obj-y += ccu_reset.o +sunxi-ccu-y += ccu_common.o +sunxi-ccu-y += ccu_mmc_timing.o +sunxi-ccu-y += ccu_reset.o # Base clock types -obj-y += ccu_div.o -obj-y += ccu_frac.o -obj-y += ccu_gate.o -obj-y += ccu_mux.o -obj-y += ccu_mult.o -obj-y += ccu_phase.o -obj-y += ccu_sdm.o +sunxi-ccu-y += ccu_div.o +sunxi-ccu-y += ccu_frac.o +sunxi-ccu-y += ccu_gate.o +sunxi-ccu-y += ccu_mux.o +sunxi-ccu-y += ccu_mult.o +sunxi-ccu-y += ccu_phase.o +sunxi-ccu-y += ccu_sdm.o # Multi-factor clocks -obj-y += ccu_nk.o -obj-y += ccu_nkm.o -obj-y += ccu_nkmp.o -obj-y += ccu_nm.o -obj-y += ccu_mp.o +sunxi-ccu-y += ccu_nk.o +sunxi-ccu-y += ccu_nkm.o +sunxi-ccu-y += ccu_nkmp.o +sunxi-ccu-y += ccu_nm.o +sunxi-ccu-y += ccu_mp.o # SoC support obj-$(CONFIG_SUNIV_F1C100S_CCU) += suniv-f1c100s-ccu.o diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c index 6afdedbce6a2..8d28a7a079d0 100644 --- a/drivers/clk/sunxi-ng/ccu_common.c +++ b/drivers/clk/sunxi-ng/ccu_common.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "ccu_common.h" @@ -214,3 +215,5 @@ void of_sunxi_ccu_probe(struct device_node *node, void __iomem *reg, kfree(ccu); } } + +MODULE_LICENSE("GPL"); diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 5af8494c31b5..52b0b27a6839 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -966,6 +966,7 @@ config MMC_REALTEK_USB config MMC_SUNXI tristate "Allwinner sunxi SD/MMC Host Controller support" depends on ARCH_SUNXI || COMPILE_TEST + depends on SUNXI_CCU help This selects support for the SD/MMC Host Controller on Allwinner sunxi SoCs. diff --git a/include/linux/clk/sunxi-ng.h b/include/linux/clk/sunxi-ng.h index 3cd14acde0a1..cf32123b39f5 100644 --- a/include/linux/clk/sunxi-ng.h +++ b/include/linux/clk/sunxi-ng.h @@ -6,22 +6,7 @@ #ifndef _LINUX_CLK_SUNXI_NG_H_ #define _LINUX_CLK_SUNXI_NG_H_ -#include - -#ifdef CONFIG_SUNXI_CCU int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, bool new_mode); int sunxi_ccu_get_mmc_timing_mode(struct clk *clk); -#else -static inline int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, - bool new_mode) -{ - return -ENOTSUPP; -} - -static inline int sunxi_ccu_get_mmc_timing_mode(struct clk *clk) -{ - return -ENOTSUPP; -} -#endif #endif -- cgit v1.2.3-59-g8ed1b From c214f124161d446b340597e7c968e0a2dc149142 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Tue, 9 Nov 2021 19:57:10 +0000 Subject: arch_topology: Introduce thermal pressure update function The thermal pressure is a mechanism which is used for providing information about reduced CPU performance to the scheduler. Usually code has to convert the value from frequency units into capacity units, which are understandable by the scheduler. Create a common conversion code which can be just used via a handy API. Internally, the topology_update_thermal_pressure() operates on frequency in MHz and max CPU frequency is taken from 'freq_factor' (per-cpu). Signed-off-by: Lukasz Luba Reviewed-by: Thara Gopinath Signed-off-by: Viresh Kumar --- arch/arm/include/asm/topology.h | 1 + arch/arm64/include/asm/topology.h | 1 + drivers/base/arch_topology.c | 43 ++++++++++++++++++++++++++++++++++++++- include/linux/arch_topology.h | 3 +++ include/linux/sched/topology.h | 7 +++++++ 5 files changed, 54 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 470299ee2fba..f1eafacc9a30 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -24,6 +24,7 @@ /* Replace task scheduler's default thermal pressure API */ #define arch_scale_thermal_pressure topology_get_thermal_pressure #define arch_set_thermal_pressure topology_set_thermal_pressure +#define arch_update_thermal_pressure topology_update_thermal_pressure #else diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index ec2db3419c41..7a421cbc99ed 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -33,6 +33,7 @@ void update_freq_counters_refs(void); /* Replace task scheduler's default thermal pressure API */ #define arch_scale_thermal_pressure topology_get_thermal_pressure #define arch_set_thermal_pressure topology_set_thermal_pressure +#define arch_update_thermal_pressure topology_update_thermal_pressure #include diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index ff16a36a908b..a3ae0b4eea09 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -22,6 +22,7 @@ static DEFINE_PER_CPU(struct scale_freq_data __rcu *, sft_data); static struct cpumask scale_freq_counters_mask; static bool scale_freq_invariant; +static DEFINE_PER_CPU(u32, freq_factor) = 1; static bool supports_scale_freq_counters(const struct cpumask *cpus) { @@ -165,6 +166,47 @@ void topology_set_thermal_pressure(const struct cpumask *cpus, } EXPORT_SYMBOL_GPL(topology_set_thermal_pressure); +/** + * topology_update_thermal_pressure() - Update thermal pressure for CPUs + * @cpus : The related CPUs for which capacity has been reduced + * @capped_freq : The maximum allowed frequency that CPUs can run at + * + * Update the value of thermal pressure for all @cpus in the mask. The + * cpumask should include all (online+offline) affected CPUs, to avoid + * operating on stale data when hot-plug is used for some CPUs. The + * @capped_freq reflects the currently allowed max CPUs frequency due to + * thermal capping. It might be also a boost frequency value, which is bigger + * than the internal 'freq_factor' max frequency. In such case the pressure + * value should simply be removed, since this is an indication that there is + * no thermal throttling. The @capped_freq must be provided in kHz. + */ +void topology_update_thermal_pressure(const struct cpumask *cpus, + unsigned long capped_freq) +{ + unsigned long max_capacity, capacity; + u32 max_freq; + int cpu; + + cpu = cpumask_first(cpus); + max_capacity = arch_scale_cpu_capacity(cpu); + max_freq = per_cpu(freq_factor, cpu); + + /* Convert to MHz scale which is used in 'freq_factor' */ + capped_freq /= 1000; + + /* + * Handle properly the boost frequencies, which should simply clean + * the thermal pressure value. + */ + if (max_freq <= capped_freq) + capacity = max_capacity; + else + capacity = mult_frac(max_capacity, capped_freq, max_freq); + + arch_set_thermal_pressure(cpus, max_capacity - capacity); +} +EXPORT_SYMBOL_GPL(topology_update_thermal_pressure); + static ssize_t cpu_capacity_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -217,7 +259,6 @@ static void update_topology_flags_workfn(struct work_struct *work) update_topology = 0; } -static DEFINE_PER_CPU(u32, freq_factor) = 1; static u32 *raw_capacity; static int free_raw_capacity(void) diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index b97cea83b25e..ace1e5dcf773 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -59,6 +59,9 @@ static inline unsigned long topology_get_thermal_pressure(int cpu) void topology_set_thermal_pressure(const struct cpumask *cpus, unsigned long th_pressure); +void topology_update_thermal_pressure(const struct cpumask *cpus, + unsigned long capped_freq); + struct cpu_topology { int thread_id; int core_id; diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index c07bfa2d80f2..6e89a8e43aa7 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -273,6 +273,13 @@ void arch_set_thermal_pressure(const struct cpumask *cpus, { } #endif +#ifndef arch_update_thermal_pressure +static __always_inline +void arch_update_thermal_pressure(const struct cpumask *cpus, + unsigned long capped_frequency) +{ } +#endif + static inline int task_node(const struct task_struct *p) { return cpu_to_node(task_cpu(p)); -- cgit v1.2.3-59-g8ed1b From 7e97b3dc2556743dd02612c92a8de7026e8d7dc9 Mon Sep 17 00:00:00 2001 From: Lukasz Luba Date: Tue, 9 Nov 2021 19:57:14 +0000 Subject: arch_topology: Remove unused topology_set_thermal_pressure() and related There is no need of this function (and related) since code has been converted to use the new arch_update_thermal_pressure() API. The old code can be removed. Signed-off-by: Lukasz Luba Signed-off-by: Viresh Kumar --- arch/arm/include/asm/topology.h | 1 - arch/arm64/include/asm/topology.h | 1 - drivers/base/arch_topology.c | 17 +++++------------ include/linux/arch_topology.h | 3 --- include/linux/sched/topology.h | 7 ------- init/Kconfig | 2 +- 6 files changed, 6 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index f1eafacc9a30..c7d2510e5a78 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -23,7 +23,6 @@ /* Replace task scheduler's default thermal pressure API */ #define arch_scale_thermal_pressure topology_get_thermal_pressure -#define arch_set_thermal_pressure topology_set_thermal_pressure #define arch_update_thermal_pressure topology_update_thermal_pressure #else diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 7a421cbc99ed..f386b90a79c8 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -32,7 +32,6 @@ void update_freq_counters_refs(void); /* Replace task scheduler's default thermal pressure API */ #define arch_scale_thermal_pressure topology_get_thermal_pressure -#define arch_set_thermal_pressure topology_set_thermal_pressure #define arch_update_thermal_pressure topology_update_thermal_pressure #include diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index a3ae0b4eea09..976154140f0b 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -156,16 +156,6 @@ void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity) DEFINE_PER_CPU(unsigned long, thermal_pressure); -void topology_set_thermal_pressure(const struct cpumask *cpus, - unsigned long th_pressure) -{ - int cpu; - - for_each_cpu(cpu, cpus) - WRITE_ONCE(per_cpu(thermal_pressure, cpu), th_pressure); -} -EXPORT_SYMBOL_GPL(topology_set_thermal_pressure); - /** * topology_update_thermal_pressure() - Update thermal pressure for CPUs * @cpus : The related CPUs for which capacity has been reduced @@ -183,7 +173,7 @@ EXPORT_SYMBOL_GPL(topology_set_thermal_pressure); void topology_update_thermal_pressure(const struct cpumask *cpus, unsigned long capped_freq) { - unsigned long max_capacity, capacity; + unsigned long max_capacity, capacity, th_pressure; u32 max_freq; int cpu; @@ -203,7 +193,10 @@ void topology_update_thermal_pressure(const struct cpumask *cpus, else capacity = mult_frac(max_capacity, capped_freq, max_freq); - arch_set_thermal_pressure(cpus, max_capacity - capacity); + th_pressure = max_capacity - capacity; + + for_each_cpu(cpu, cpus) + WRITE_ONCE(per_cpu(thermal_pressure, cpu), th_pressure); } EXPORT_SYMBOL_GPL(topology_update_thermal_pressure); diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index ace1e5dcf773..cce6136b300a 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -56,9 +56,6 @@ static inline unsigned long topology_get_thermal_pressure(int cpu) return per_cpu(thermal_pressure, cpu); } -void topology_set_thermal_pressure(const struct cpumask *cpus, - unsigned long th_pressure); - void topology_update_thermal_pressure(const struct cpumask *cpus, unsigned long capped_freq); diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 6e89a8e43aa7..8054641c0a7b 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -266,13 +266,6 @@ unsigned long arch_scale_thermal_pressure(int cpu) } #endif -#ifndef arch_set_thermal_pressure -static __always_inline -void arch_set_thermal_pressure(const struct cpumask *cpus, - unsigned long th_pressure) -{ } -#endif - #ifndef arch_update_thermal_pressure static __always_inline void arch_update_thermal_pressure(const struct cpumask *cpus, diff --git a/init/Kconfig b/init/Kconfig index 036b750e8d8a..086fd11348e0 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -550,7 +550,7 @@ config SCHED_THERMAL_PRESSURE i.e. put less load on throttled CPUs than on non/less throttled ones. This requires the architecture to implement - arch_set_thermal_pressure() and arch_scale_thermal_pressure(). + arch_update_thermal_pressure() and arch_scale_thermal_pressure(). config BSD_PROCESS_ACCT bool "BSD Process Accounting" -- cgit v1.2.3-59-g8ed1b From cff6f593251cdf5398dc3c57f7032b8e9dcb633e Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Tue, 23 Nov 2021 12:36:47 +0200 Subject: regulator: rohm-generic: iniline stub function The function rohm_regulator_set_voltage_sel_restricted() has a stub implementation. Linux-next testing spot following: include/linux/mfd/rohm-generic.h:93:12: error: 'rohm_regulator_set_voltage_sel_restricted' defined but not used Fix this by inlining the stub. Fixes: 8b6e88555971 ("regulator: rohm-regulator: add helper for restricted voltage setting") Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/YZzEP3S7U15bTDAI@fedora Signed-off-by: Mark Brown --- include/linux/mfd/rohm-generic.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h index 35c5866f48b7..080d60adcd5f 100644 --- a/include/linux/mfd/rohm-generic.h +++ b/include/linux/mfd/rohm-generic.h @@ -90,7 +90,8 @@ static inline int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dv { return 0; } -static int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, + +static inline int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, unsigned int sel) { return 0; -- cgit v1.2.3-59-g8ed1b From 2106efda785b55a8957efed9a52dfa28ee0d7280 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 22 Nov 2021 17:24:47 -0800 Subject: net: remove .ndo_change_proto_down .ndo_change_proto_down was added seemingly to enable out-of-tree implementations. Over 2.5yrs later we still have no real users upstream. Hardwire the generic implementation for now, we can revert once real users materialize. (rocker is a test vehicle, not a user.) We need to drop the optimization on the sysfs side, because unlike ndos priv_flags will be changed at runtime, so we'd need READ_ONCE/WRITE_ONCE everywhere.. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- drivers/net/ethernet/rocker/rocker_main.c | 12 ------------ drivers/net/macvlan.c | 3 +-- drivers/net/vxlan.c | 3 +-- include/linux/netdevice.h | 12 +++--------- net/8021q/vlanproc.c | 2 +- net/core/dev.c | 26 ++++---------------------- net/core/net-sysfs.c | 8 -------- net/core/rtnetlink.c | 3 +-- 8 files changed, 11 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index ba4062881eed..b620470c7905 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -1995,17 +1995,6 @@ static int rocker_port_get_phys_port_name(struct net_device *dev, return err ? -EOPNOTSUPP : 0; } -static int rocker_port_change_proto_down(struct net_device *dev, - bool proto_down) -{ - struct rocker_port *rocker_port = netdev_priv(dev); - - if (rocker_port->dev->flags & IFF_UP) - rocker_port_set_enable(rocker_port, !proto_down); - rocker_port->dev->proto_down = proto_down; - return 0; -} - static void rocker_port_neigh_destroy(struct net_device *dev, struct neighbour *n) { @@ -2037,7 +2026,6 @@ static const struct net_device_ops rocker_port_netdev_ops = { .ndo_set_mac_address = rocker_port_set_mac_address, .ndo_change_mtu = rocker_port_change_mtu, .ndo_get_phys_port_name = rocker_port_get_phys_port_name, - .ndo_change_proto_down = rocker_port_change_proto_down, .ndo_neigh_destroy = rocker_port_neigh_destroy, .ndo_get_port_parent_id = rocker_port_get_port_parent_id, }; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 75b453acd778..6ef5f77be4d0 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1171,7 +1171,6 @@ static const struct net_device_ops macvlan_netdev_ops = { #endif .ndo_get_iflink = macvlan_dev_get_iflink, .ndo_features_check = passthru_features_check, - .ndo_change_proto_down = dev_change_proto_down_generic, }; void macvlan_common_setup(struct net_device *dev) @@ -1182,7 +1181,7 @@ void macvlan_common_setup(struct net_device *dev) dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; netif_keep_dst(dev); - dev->priv_flags |= IFF_UNICAST_FLT; + dev->priv_flags |= IFF_UNICAST_FLT | IFF_CHANGE_PROTO_DOWN; dev->netdev_ops = &macvlan_netdev_ops; dev->needs_free_netdev = true; dev->header_ops = &macvlan_hard_header_ops; diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 82bd794fceb3..fecff0a46612 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -3234,7 +3234,6 @@ static const struct net_device_ops vxlan_netdev_ether_ops = { .ndo_fdb_dump = vxlan_fdb_dump, .ndo_fdb_get = vxlan_fdb_get, .ndo_fill_metadata_dst = vxlan_fill_metadata_dst, - .ndo_change_proto_down = dev_change_proto_down_generic, }; static const struct net_device_ops vxlan_netdev_raw_ops = { @@ -3305,7 +3304,7 @@ static void vxlan_setup(struct net_device *dev) dev->hw_features |= NETIF_F_RXCSUM; dev->hw_features |= NETIF_F_GSO_SOFTWARE; netif_keep_dst(dev); - dev->priv_flags |= IFF_NO_QUEUE; + dev->priv_flags |= IFF_NO_QUEUE | IFF_CHANGE_PROTO_DOWN; /* MTU range: 68 - 65535 */ dev->min_mtu = ETH_MIN_MTU; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index df049864661d..db3bff1ae7fd 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1297,11 +1297,6 @@ struct netdev_net_notifier { * TX queue. * int (*ndo_get_iflink)(const struct net_device *dev); * Called to get the iflink value of this device. - * void (*ndo_change_proto_down)(struct net_device *dev, - * bool proto_down); - * This function is used to pass protocol port error state information - * to the switch driver. The switch driver can react to the proto_down - * by doing a phys down on the associated switch port. * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb); * This function is used to get egress tunnel information for given skb. * This is useful for retrieving outer tunnel header parameters while @@ -1542,8 +1537,6 @@ struct net_device_ops { int queue_index, u32 maxrate); int (*ndo_get_iflink)(const struct net_device *dev); - int (*ndo_change_proto_down)(struct net_device *dev, - bool proto_down); int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb); void (*ndo_set_rx_headroom)(struct net_device *dev, @@ -1612,6 +1605,7 @@ struct net_device_ops { * @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with * skb_headlen(skb) == 0 (data starts from frag0) + * @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN */ enum netdev_priv_flags { IFF_802_1Q_VLAN = 1<<0, @@ -1646,6 +1640,7 @@ enum netdev_priv_flags { IFF_L3MDEV_RX_HANDLER = 1<<29, IFF_LIVE_RENAME_OK = 1<<30, IFF_TX_SKB_NO_LINEAR = 1<<31, + IFF_CHANGE_PROTO_DOWN = BIT_ULL(32), }; #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN @@ -1982,7 +1977,7 @@ struct net_device { /* Read-mostly cache-line for fast-path access */ unsigned int flags; - unsigned int priv_flags; + unsigned long long priv_flags; const struct net_device_ops *netdev_ops; int ifindex; unsigned short gflags; @@ -3735,7 +3730,6 @@ int dev_get_port_parent_id(struct net_device *dev, struct netdev_phys_item_id *ppid, bool recurse); bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b); int dev_change_proto_down(struct net_device *dev, bool proto_down); -int dev_change_proto_down_generic(struct net_device *dev, bool proto_down); void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, u32 value); struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again); diff --git a/net/8021q/vlanproc.c b/net/8021q/vlanproc.c index ec87dea23719..08bf6c839e25 100644 --- a/net/8021q/vlanproc.c +++ b/net/8021q/vlanproc.c @@ -252,7 +252,7 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset) stats = dev_get_stats(vlandev, &temp); seq_printf(seq, - "%s VID: %d REORDER_HDR: %i dev->priv_flags: %hx\n", + "%s VID: %d REORDER_HDR: %i dev->priv_flags: %llx\n", vlandev->name, vlan->vlan_id, (int)(vlan->flags & 1), vlandev->priv_flags); diff --git a/net/core/dev.c b/net/core/dev.c index 9c4fc8c3f981..823917de0d2b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -8558,35 +8558,17 @@ bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) EXPORT_SYMBOL(netdev_port_same_parent_id); /** - * dev_change_proto_down - update protocol port state information + * dev_change_proto_down - set carrier according to proto_down. + * * @dev: device * @proto_down: new value - * - * This info can be used by switch drivers to set the phys state of the - * port. */ int dev_change_proto_down(struct net_device *dev, bool proto_down) { - const struct net_device_ops *ops = dev->netdev_ops; - - if (!ops->ndo_change_proto_down) + if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) return -EOPNOTSUPP; if (!netif_device_present(dev)) return -ENODEV; - return ops->ndo_change_proto_down(dev, proto_down); -} -EXPORT_SYMBOL(dev_change_proto_down); - -/** - * dev_change_proto_down_generic - generic implementation for - * ndo_change_proto_down that sets carrier according to - * proto_down. - * - * @dev: device - * @proto_down: new value - */ -int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) -{ if (proto_down) netif_carrier_off(dev); else @@ -8594,7 +8576,7 @@ int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) dev->proto_down = proto_down; return 0; } -EXPORT_SYMBOL(dev_change_proto_down_generic); +EXPORT_SYMBOL(dev_change_proto_down); /** * dev_change_proto_down_reason - proto down reason diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 4edd58d34f16..affe34d71d31 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -488,14 +488,6 @@ static ssize_t proto_down_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { - struct net_device *netdev = to_net_dev(dev); - - /* The check is also done in change_proto_down; this helps returning - * early without hitting the trylock/restart in netdev_store. - */ - if (!netdev->netdev_ops->ndo_change_proto_down) - return -EOPNOTSUPP; - return netdev_store(dev, attr, buf, len, change_proto_down); } NETDEVICE_SHOW_RW(proto_down, fmt_dec); diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index fd030e02f16d..6f25c0a8aebe 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2539,13 +2539,12 @@ static int do_set_proto_down(struct net_device *dev, struct netlink_ext_ack *extack) { struct nlattr *pdreason[IFLA_PROTO_DOWN_REASON_MAX + 1]; - const struct net_device_ops *ops = dev->netdev_ops; unsigned long mask = 0; u32 value; bool proto_down; int err; - if (!ops->ndo_change_proto_down) { + if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) { NL_SET_ERR_MSG(extack, "Protodown not supported by device"); return -EOPNOTSUPP; } -- cgit v1.2.3-59-g8ed1b From c6d5f1933085f9a92ed5c256a859ab31c7a35f88 Mon Sep 17 00:00:00 2001 From: Kurt Kanzenbach Date: Mon, 22 Nov 2021 12:19:31 +0100 Subject: net: stmmac: Calculate CDC error only once The clock domain crossing error (CDC) is calculated at every fetch of Tx or Rx timestamps. It includes a division. Especially on arm32 based systems it is expensive. It also requires two conditionals in the hotpath. Add a compensation value cache to struct plat_stmmacenet_data and subtract it unconditionally in the RX/TX functions which spares the conditionals. The value is initialized to 0 and if supported calculated in the PTP initialization code. Suggested-by: Thomas Gleixner Signed-off-by: Kurt Kanzenbach Link: https://lore.kernel.org/r/20211122111931.135135-1-kurt@linutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 12 ++---------- drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c | 5 +++++ include/linux/stmmac.h | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 21111df73719..340076b5bb38 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -511,14 +511,6 @@ bool stmmac_eee_init(struct stmmac_priv *priv) return true; } -static inline u32 stmmac_cdc_adjust(struct stmmac_priv *priv) -{ - /* Correct the clk domain crossing(CDC) error */ - if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) - return (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; - return 0; -} - /* stmmac_get_tx_hwtstamp - get HW TX timestamps * @priv: driver private structure * @p : descriptor pointer @@ -550,7 +542,7 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, } if (found) { - ns -= stmmac_cdc_adjust(priv); + ns -= priv->plat->cdc_error_adj; memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); shhwtstamp.hwtstamp = ns_to_ktime(ns); @@ -587,7 +579,7 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); - ns -= stmmac_cdc_adjust(priv); + ns -= priv->plat->cdc_error_adj; netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); shhwtstamp = skb_hwtstamps(skb); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 580cc035536b..e14c97c04317 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -309,6 +309,11 @@ void stmmac_ptp_register(struct stmmac_priv *priv) if (priv->plat->ptp_max_adj) stmmac_ptp_clock_ops.max_adj = priv->plat->ptp_max_adj; + /* Calculate the clock domain crossing (CDC) error if necessary */ + priv->plat->cdc_error_adj = 0; + if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) + priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; + stmmac_ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num; stmmac_ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n; diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index a6f03b36fc4f..89b8e208cd7b 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -241,6 +241,7 @@ struct plat_stmmacenet_data { unsigned int clk_ref_rate; unsigned int mult_fact_100ns; s32 ptp_max_adj; + u32 cdc_error_adj; struct reset_control *stmmac_rst; struct reset_control *stmmac_ahb_rst; struct stmmac_axi *axi; -- cgit v1.2.3-59-g8ed1b From 393c3714081a53795bbff0e985d24146def6f57f Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Thu, 18 Nov 2021 15:00:08 -0800 Subject: kernfs: switch global kernfs_rwsem lock to per-fs lock The kernfs implementation has big lock granularity(kernfs_rwsem) so every kernfs-based(e.g., sysfs, cgroup) fs are able to compete the lock. It makes trouble for some cases to wait the global lock for a long time even though they are totally independent contexts each other. A general example is process A goes under direct reclaim with holding the lock when it accessed the file in sysfs and process B is waiting the lock with exclusive mode and then process C is waiting the lock until process B could finish the job after it gets the lock from process A. This patch switches the global kernfs_rwsem to per-fs lock, which put the rwsem into kernfs_root. Suggested-by: Tejun Heo Acked-by: Tejun Heo Signed-off-by: Minchan Kim Link: https://lore.kernel.org/r/20211118230008.2679780-1-minchan@kernel.org Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/dir.c | 110 +++++++++++++++++++++++++++++-------------------- fs/kernfs/file.c | 6 ++- fs/kernfs/inode.c | 22 ++++++---- fs/kernfs/mount.c | 15 ++++--- fs/kernfs/symlink.c | 5 ++- include/linux/kernfs.h | 2 + 6 files changed, 97 insertions(+), 63 deletions(-) (limited to 'include/linux') diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 8e0a1378a4b1..13cae0ccce74 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -17,7 +17,6 @@ #include "kernfs-internal.h" -DECLARE_RWSEM(kernfs_rwsem); static DEFINE_SPINLOCK(kernfs_rename_lock); /* kn->parent and ->name */ static char kernfs_pr_cont_buf[PATH_MAX]; /* protected by rename_lock */ static DEFINE_SPINLOCK(kernfs_idr_lock); /* root->ino_idr */ @@ -26,7 +25,7 @@ static DEFINE_SPINLOCK(kernfs_idr_lock); /* root->ino_idr */ static bool kernfs_active(struct kernfs_node *kn) { - lockdep_assert_held(&kernfs_rwsem); + lockdep_assert_held(&kernfs_root(kn)->kernfs_rwsem); return atomic_read(&kn->active) >= 0; } @@ -457,14 +456,15 @@ void kernfs_put_active(struct kernfs_node *kn) * return after draining is complete. */ static void kernfs_drain(struct kernfs_node *kn) - __releases(&kernfs_rwsem) __acquires(&kernfs_rwsem) + __releases(&kernfs_root(kn)->kernfs_rwsem) + __acquires(&kernfs_root(kn)->kernfs_rwsem) { struct kernfs_root *root = kernfs_root(kn); - lockdep_assert_held_write(&kernfs_rwsem); + lockdep_assert_held_write(&root->kernfs_rwsem); WARN_ON_ONCE(kernfs_active(kn)); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); if (kernfs_lockdep(kn)) { rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_); @@ -483,7 +483,7 @@ static void kernfs_drain(struct kernfs_node *kn) kernfs_drain_open_files(kn); - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); } /** @@ -718,11 +718,12 @@ err_unlock: int kernfs_add_one(struct kernfs_node *kn) { struct kernfs_node *parent = kn->parent; + struct kernfs_root *root = kernfs_root(parent); struct kernfs_iattrs *ps_iattr; bool has_ns; int ret; - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); ret = -EINVAL; has_ns = kernfs_ns_enabled(parent); @@ -753,7 +754,7 @@ int kernfs_add_one(struct kernfs_node *kn) ps_iattr->ia_mtime = ps_iattr->ia_ctime; } - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); /* * Activate the new node unless CREATE_DEACTIVATED is requested. @@ -767,7 +768,7 @@ int kernfs_add_one(struct kernfs_node *kn) return 0; out_unlock: - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); return ret; } @@ -788,7 +789,7 @@ static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent, bool has_ns = kernfs_ns_enabled(parent); unsigned int hash; - lockdep_assert_held(&kernfs_rwsem); + lockdep_assert_held(&kernfs_root(parent)->kernfs_rwsem); if (has_ns != (bool)ns) { WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n", @@ -820,7 +821,7 @@ static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent, size_t len; char *p, *name; - lockdep_assert_held_read(&kernfs_rwsem); + lockdep_assert_held_read(&kernfs_root(parent)->kernfs_rwsem); /* grab kernfs_rename_lock to piggy back on kernfs_pr_cont_buf */ spin_lock_irq(&kernfs_rename_lock); @@ -859,11 +860,12 @@ struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent, const char *name, const void *ns) { struct kernfs_node *kn; + struct kernfs_root *root = kernfs_root(parent); - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); kn = kernfs_find_ns(parent, name, ns); kernfs_get(kn); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return kn; } @@ -883,11 +885,12 @@ struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent, const char *path, const void *ns) { struct kernfs_node *kn; + struct kernfs_root *root = kernfs_root(parent); - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); kn = kernfs_walk_ns(parent, path, ns); kernfs_get(kn); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return kn; } @@ -912,6 +915,7 @@ struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops, return ERR_PTR(-ENOMEM); idr_init(&root->ino_idr); + init_rwsem(&root->kernfs_rwsem); INIT_LIST_HEAD(&root->supers); /* @@ -1035,6 +1039,7 @@ struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) { struct kernfs_node *kn; + struct kernfs_root *root; if (flags & LOOKUP_RCU) return -ECHILD; @@ -1046,18 +1051,19 @@ static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) /* If the kernfs parent node has changed discard and * proceed to ->lookup. */ - down_read(&kernfs_rwsem); spin_lock(&dentry->d_lock); parent = kernfs_dentry_node(dentry->d_parent); if (parent) { + spin_unlock(&dentry->d_lock); + root = kernfs_root(parent); + down_read(&root->kernfs_rwsem); if (kernfs_dir_changed(parent, dentry)) { - spin_unlock(&dentry->d_lock); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return 0; } - } - spin_unlock(&dentry->d_lock); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); + } else + spin_unlock(&dentry->d_lock); /* The kernfs parent node hasn't changed, leave the * dentry negative and return success. @@ -1066,7 +1072,8 @@ static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) } kn = kernfs_dentry_node(dentry); - down_read(&kernfs_rwsem); + root = kernfs_root(kn); + down_read(&root->kernfs_rwsem); /* The kernfs node has been deactivated */ if (!kernfs_active(kn)) @@ -1085,10 +1092,10 @@ static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags) kernfs_info(dentry->d_sb)->ns != kn->ns) goto out_bad; - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return 1; out_bad: - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return 0; } @@ -1102,10 +1109,12 @@ static struct dentry *kernfs_iop_lookup(struct inode *dir, { struct kernfs_node *parent = dir->i_private; struct kernfs_node *kn; + struct kernfs_root *root; struct inode *inode = NULL; const void *ns = NULL; - down_read(&kernfs_rwsem); + root = kernfs_root(parent); + down_read(&root->kernfs_rwsem); if (kernfs_ns_enabled(parent)) ns = kernfs_info(dir->i_sb)->ns; @@ -1116,7 +1125,7 @@ static struct dentry *kernfs_iop_lookup(struct inode *dir, * create a negative. */ if (!kernfs_active(kn)) { - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return NULL; } inode = kernfs_get_inode(dir->i_sb, kn); @@ -1131,7 +1140,7 @@ static struct dentry *kernfs_iop_lookup(struct inode *dir, */ if (!IS_ERR(inode)) kernfs_set_rev(parent, dentry); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); /* instantiate and hash (possibly negative) dentry */ return d_splice_alias(inode, dentry); @@ -1254,7 +1263,7 @@ static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos, { struct rb_node *rbn; - lockdep_assert_held_write(&kernfs_rwsem); + lockdep_assert_held_write(&kernfs_root(root)->kernfs_rwsem); /* if first iteration, visit leftmost descendant which may be root */ if (!pos) @@ -1289,8 +1298,9 @@ static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos, void kernfs_activate(struct kernfs_node *kn) { struct kernfs_node *pos; + struct kernfs_root *root = kernfs_root(kn); - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); pos = NULL; while ((pos = kernfs_next_descendant_post(pos, kn))) { @@ -1304,14 +1314,14 @@ void kernfs_activate(struct kernfs_node *kn) pos->flags |= KERNFS_ACTIVATED; } - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); } static void __kernfs_remove(struct kernfs_node *kn) { struct kernfs_node *pos; - lockdep_assert_held_write(&kernfs_rwsem); + lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem); /* * Short-circuit if non-root @kn has already finished removal. @@ -1381,9 +1391,11 @@ static void __kernfs_remove(struct kernfs_node *kn) */ void kernfs_remove(struct kernfs_node *kn) { - down_write(&kernfs_rwsem); + struct kernfs_root *root = kernfs_root(kn); + + down_write(&root->kernfs_rwsem); __kernfs_remove(kn); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); } /** @@ -1469,8 +1481,9 @@ void kernfs_unbreak_active_protection(struct kernfs_node *kn) bool kernfs_remove_self(struct kernfs_node *kn) { bool ret; + struct kernfs_root *root = kernfs_root(kn); - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); kernfs_break_active_protection(kn); /* @@ -1498,9 +1511,9 @@ bool kernfs_remove_self(struct kernfs_node *kn) atomic_read(&kn->active) == KN_DEACTIVATED_BIAS) break; - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); schedule(); - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); } finish_wait(waitq, &wait); WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb)); @@ -1513,7 +1526,7 @@ bool kernfs_remove_self(struct kernfs_node *kn) */ kernfs_unbreak_active_protection(kn); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); return ret; } @@ -1530,6 +1543,7 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name, const void *ns) { struct kernfs_node *kn; + struct kernfs_root *root; if (!parent) { WARN(1, KERN_WARNING "kernfs: can not remove '%s', no directory\n", @@ -1537,13 +1551,14 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name, return -ENOENT; } - down_write(&kernfs_rwsem); + root = kernfs_root(parent); + down_write(&root->kernfs_rwsem); kn = kernfs_find_ns(parent, name, ns); if (kn) __kernfs_remove(kn); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); if (kn) return 0; @@ -1562,6 +1577,7 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, const char *new_name, const void *new_ns) { struct kernfs_node *old_parent; + struct kernfs_root *root; const char *old_name = NULL; int error; @@ -1569,7 +1585,8 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, if (!kn->parent) return -EINVAL; - down_write(&kernfs_rwsem); + root = kernfs_root(kn); + down_write(&root->kernfs_rwsem); error = -ENOENT; if (!kernfs_active(kn) || !kernfs_active(new_parent) || @@ -1623,7 +1640,7 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, error = 0; out: - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); return error; } @@ -1694,11 +1711,14 @@ static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx) struct dentry *dentry = file->f_path.dentry; struct kernfs_node *parent = kernfs_dentry_node(dentry); struct kernfs_node *pos = file->private_data; + struct kernfs_root *root; const void *ns = NULL; if (!dir_emit_dots(file, ctx)) return 0; - down_read(&kernfs_rwsem); + + root = kernfs_root(parent); + down_read(&root->kernfs_rwsem); if (kernfs_ns_enabled(parent)) ns = kernfs_info(dentry->d_sb)->ns; @@ -1715,12 +1735,12 @@ static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx) file->private_data = pos; kernfs_get(pos); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); if (!dir_emit(ctx, name, len, ino, type)) return 0; - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); } - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); file->private_data = NULL; ctx->pos = INT_MAX; return 0; diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 60e2a86c535e..9414a7a60a9f 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -847,6 +847,7 @@ static void kernfs_notify_workfn(struct work_struct *work) { struct kernfs_node *kn; struct kernfs_super_info *info; + struct kernfs_root *root; repeat: /* pop one off the notify_list */ spin_lock_irq(&kernfs_notify_lock); @@ -859,8 +860,9 @@ repeat: kn->attr.notify_next = NULL; spin_unlock_irq(&kernfs_notify_lock); + root = kernfs_root(kn); /* kick fsnotify */ - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); list_for_each_entry(info, &kernfs_root(kn)->supers, node) { struct kernfs_node *parent; @@ -898,7 +900,7 @@ repeat: iput(inode); } - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); kernfs_put(kn); goto repeat; } diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index c0eae1725435..3d783d80f5da 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -99,10 +99,11 @@ int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) { int ret; + struct kernfs_root *root = kernfs_root(kn); - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); ret = __kernfs_setattr(kn, iattr); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); return ret; } @@ -111,12 +112,14 @@ int kernfs_iop_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, { struct inode *inode = d_inode(dentry); struct kernfs_node *kn = inode->i_private; + struct kernfs_root *root; int error; if (!kn) return -EINVAL; - down_write(&kernfs_rwsem); + root = kernfs_root(kn); + down_write(&root->kernfs_rwsem); error = setattr_prepare(&init_user_ns, dentry, iattr); if (error) goto out; @@ -129,7 +132,7 @@ int kernfs_iop_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, setattr_copy(&init_user_ns, inode, iattr); out: - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); return error; } @@ -184,13 +187,14 @@ int kernfs_iop_getattr(struct user_namespace *mnt_userns, { struct inode *inode = d_inode(path->dentry); struct kernfs_node *kn = inode->i_private; + struct kernfs_root *root = kernfs_root(kn); - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); spin_lock(&inode->i_lock); kernfs_refresh_inode(kn, inode); generic_fillattr(&init_user_ns, inode, stat); spin_unlock(&inode->i_lock); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return 0; } @@ -274,19 +278,21 @@ int kernfs_iop_permission(struct user_namespace *mnt_userns, struct inode *inode, int mask) { struct kernfs_node *kn; + struct kernfs_root *root; int ret; if (mask & MAY_NOT_BLOCK) return -ECHILD; kn = inode->i_private; + root = kernfs_root(kn); - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); spin_lock(&inode->i_lock); kernfs_refresh_inode(kn, inode); ret = generic_permission(&init_user_ns, inode, mask); spin_unlock(&inode->i_lock); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return ret; } diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index f2f909d09f52..cfa79715fc1a 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -236,6 +236,7 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, static int kernfs_fill_super(struct super_block *sb, struct kernfs_fs_context *kfc) { struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *kf_root = kfc->root; struct inode *inode; struct dentry *root; @@ -255,9 +256,9 @@ static int kernfs_fill_super(struct super_block *sb, struct kernfs_fs_context *k sb->s_shrink.seeks = 0; /* get root inode, initialize and unlock it */ - down_read(&kernfs_rwsem); + down_read(&kf_root->kernfs_rwsem); inode = kernfs_get_inode(sb, info->root->kn); - up_read(&kernfs_rwsem); + up_read(&kf_root->kernfs_rwsem); if (!inode) { pr_debug("kernfs: could not get root inode\n"); return -ENOMEM; @@ -334,6 +335,7 @@ int kernfs_get_tree(struct fs_context *fc) if (!sb->s_root) { struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *root = kfc->root; kfc->new_sb_created = true; @@ -344,9 +346,9 @@ int kernfs_get_tree(struct fs_context *fc) } sb->s_flags |= SB_ACTIVE; - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); list_add(&info->node, &info->root->supers); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); } fc->root = dget(sb->s_root); @@ -371,10 +373,11 @@ void kernfs_free_fs_context(struct fs_context *fc) void kernfs_kill_sb(struct super_block *sb) { struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *root = info->root; - down_write(&kernfs_rwsem); + down_write(&root->kernfs_rwsem); list_del(&info->node); - up_write(&kernfs_rwsem); + up_write(&root->kernfs_rwsem); /* * Remove the superblock from fs_supers/s_instances diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c index 19a6c71c6ff5..0ab13824822f 100644 --- a/fs/kernfs/symlink.c +++ b/fs/kernfs/symlink.c @@ -113,11 +113,12 @@ static int kernfs_getlink(struct inode *inode, char *path) struct kernfs_node *kn = inode->i_private; struct kernfs_node *parent = kn->parent; struct kernfs_node *target = kn->symlink.target_kn; + struct kernfs_root *root = kernfs_root(parent); int error; - down_read(&kernfs_rwsem); + down_read(&root->kernfs_rwsem); error = kernfs_get_target_path(parent, target, path); - up_read(&kernfs_rwsem); + up_read(&root->kernfs_rwsem); return error; } diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 3ccce6f24548..9f650986a81b 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -16,6 +16,7 @@ #include #include #include +#include struct file; struct dentry; @@ -197,6 +198,7 @@ struct kernfs_root { struct list_head supers; wait_queue_head_t deactivate_waitq; + struct rw_semaphore kernfs_rwsem; }; struct kernfs_open_file { -- cgit v1.2.3-59-g8ed1b From 1b6ed6bf32fb22ef8e3572fc9c0f6454adf1ca40 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Wed, 24 Nov 2021 09:17:37 +0200 Subject: regulator: Drop unnecessary struct member The irq_flags from the regulator IRQ helper description struct was never used. The IRQ flags are passed as parameters to helper registration instead. Remove the unnecessary struct field. Fixes: 7111c6d1b31b ("regulator: IRQ based event/error notification helpers") Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/5f6371e178453fa2b165da50452f7db4e986debb.1637736436.git.matti.vaittinen@fi.rohmeurope.com Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index 1cb8071fee34..53b25cd7ead0 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -554,7 +554,6 @@ struct regulator_irq_data { */ struct regulator_irq_desc { const char *name; - int irq_flags; int fatal_cnt; int reread_ms; int irq_off_ms; -- cgit v1.2.3-59-g8ed1b From 6fadec4c5561e2fbe1dfa8a7da9bc58d094a8f04 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Wed, 24 Nov 2021 09:16:45 +0200 Subject: regulator: Add regulator_err2notif() helper Help drivers avoid storing both supported notification and supported error flags by supporting conversion from regulator error to notification. This may help saving some bytes. Add helper for finding the regulator notification corresponding to a regulator error. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/eb1755ac0569ff07ffa466cf8912c6fd50e7c7c6.1637736436.git.matti.vaittinen@fi.rohmeurope.com Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index 53b25cd7ead0..6c6ec9658c30 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -645,6 +645,40 @@ struct regulator_dev { spinlock_t err_lock; }; +/* + * Convert error flags to corresponding notifications. + * + * Can be used by drivers which use the notification helpers to + * find out correct notification flags based on the error flags. Drivers + * can avoid storing both supported notification and error flags which + * may save few bytes. + */ +static inline int regulator_err2notif(int err) +{ + switch (err) { + case REGULATOR_ERROR_UNDER_VOLTAGE: + return REGULATOR_EVENT_UNDER_VOLTAGE; + case REGULATOR_ERROR_OVER_CURRENT: + return REGULATOR_EVENT_OVER_CURRENT; + case REGULATOR_ERROR_REGULATION_OUT: + return REGULATOR_EVENT_REGULATION_OUT; + case REGULATOR_ERROR_FAIL: + return REGULATOR_EVENT_FAIL; + case REGULATOR_ERROR_OVER_TEMP: + return REGULATOR_EVENT_OVER_TEMP; + case REGULATOR_ERROR_UNDER_VOLTAGE_WARN: + return REGULATOR_EVENT_UNDER_VOLTAGE_WARN; + case REGULATOR_ERROR_OVER_CURRENT_WARN: + return REGULATOR_EVENT_OVER_CURRENT_WARN; + case REGULATOR_ERROR_OVER_VOLTAGE_WARN: + return REGULATOR_EVENT_OVER_VOLTAGE_WARN; + case REGULATOR_ERROR_OVER_TEMP_WARN: + return REGULATOR_EVENT_OVER_TEMP_WARN; + } + return 0; +} + + struct regulator_dev * regulator_register(const struct regulator_desc *regulator_desc, const struct regulator_config *config); -- cgit v1.2.3-59-g8ed1b From a764ff77d697a4a13e69b3379cc613f7409c6b9a Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Wed, 24 Nov 2021 09:17:13 +0200 Subject: regulator: irq_helper: Provide helper for trivial IRQ notifications Provide a generic map_event helper for regulators which have a notification IRQ with single, well defined purpose. Eg, IRQ always indicates exactly one event for exactly one regulator device. For such IRQs the mapping is trivial. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/603b7ed1938013a00371c1e7ccc63dfb16982b87.1637736436.git.matti.vaittinen@fi.rohmeurope.com Signed-off-by: Mark Brown --- drivers/regulator/irq_helpers.c | 41 +++++++++++++++++++++++++++++++++++++++- include/linux/regulator/driver.h | 2 ++ 2 files changed, 42 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/regulator/irq_helpers.c b/drivers/regulator/irq_helpers.c index 522764435575..fe7ae0f3f46a 100644 --- a/drivers/regulator/irq_helpers.c +++ b/drivers/regulator/irq_helpers.c @@ -320,7 +320,9 @@ static void init_rdev_errors(struct regulator_irq *h) * IRQF_ONESHOT when requesting the (threaded) irq. * @common_errs: Errors which can be flagged by this IRQ for all rdevs. * When IRQ is re-enabled these errors will be cleared - * from all associated regulators + * from all associated regulators. Use this instead of the + * per_rdev_errs if you use + * regulator_irq_map_event_simple() for event mapping. * @per_rdev_errs: Optional error flag array describing errors specific * for only some of the regulators. These errors will be * or'ed with common errors. If this is given the array @@ -395,3 +397,40 @@ void regulator_irq_helper_cancel(void **handle) } } EXPORT_SYMBOL_GPL(regulator_irq_helper_cancel); + +/** + * regulator_irq_map_event_simple - regulator IRQ notification for trivial IRQs + * + * @irq: Number of IRQ that occurred + * @rid: Information about the event IRQ indicates + * @dev_mask: mask indicating the regulator originating the IRQ + * + * Regulators whose IRQ has single, well defined purpose (always indicate + * exactly one event, and are relevant to exactly one regulator device) can + * use this function as their map_event callbac for their regulator IRQ + * notification helperk. Exactly one rdev and exactly one error (in + * "common_errs"-field) can be given at IRQ helper registration for + * regulator_irq_map_event_simple() to be viable. + */ +int regulator_irq_map_event_simple(int irq, struct regulator_irq_data *rid, + unsigned long *dev_mask) +{ + int err = rid->states[0].possible_errs; + + *dev_mask = 1; + /* + * This helper should only be used in a situation where the IRQ + * can indicate only one type of problem for one specific rdev. + * Something fishy is going on if we are having multiple rdevs or ERROR + * flags here. + */ + if (WARN_ON(rid->num_states != 1 || hweight32(err) != 1)) + return 0; + + rid->states[0].errors = err; + rid->states[0].notifs = regulator_err2notif(err); + + return 0; +} +EXPORT_SYMBOL_GPL(regulator_irq_map_event_simple); + diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index 6c6ec9658c30..4078c7776453 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -700,6 +700,8 @@ void *regulator_irq_helper(struct device *dev, int irq_flags, int common_errs, int *per_rdev_errs, struct regulator_dev **rdev, int rdev_amount); void regulator_irq_helper_cancel(void **handle); +int regulator_irq_map_event_simple(int irq, struct regulator_irq_data *rid, + unsigned long *dev_mask); void *rdev_get_drvdata(struct regulator_dev *rdev); struct device *rdev_get_dev(struct regulator_dev *rdev); -- cgit v1.2.3-59-g8ed1b From 432dd1fc134ef902b049b26839edfd3fdc1f8dc0 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Wed, 24 Nov 2021 07:57:49 +0200 Subject: regulator: rohm-generic: remove unused dummies Function rohm_regulator_set_voltage_sel_restricted() and rohm_regulator_set_dvs_levels() had inlined dummy implementations for cases when the real implementation was not configured in. All of the drivers who issue the call to these functions do SELECT the real implementation from the Kconfig. There should be no cases where the real implementation was not selected by the drivers using these functions - such a situation is likely to be an error which deserves to be noticed at compile-time. These dummies could in theory be used for compile-testing the drivers only (without the generic rohm regulator pieces). However, for such compile testing we should manually drop the selection from KConfig - and I guess that if it does not work out-of-the-box, then it is not going to happen. Especially when there should be no reason to omit compile-testing the generic rohm_regulator part. Crash test dummies. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/YZ3UXXrk/Efe7Scj@fedora Signed-off-by: Mark Brown --- include/linux/mfd/rohm-generic.h | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h index 080d60adcd5f..5ed97a1d0908 100644 --- a/include/linux/mfd/rohm-generic.h +++ b/include/linux/mfd/rohm-generic.h @@ -82,20 +82,6 @@ int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, unsigned int sel); -#else -static inline int rohm_regulator_set_dvs_levels(const struct rohm_dvs_config *dvs, - struct device_node *np, - const struct regulator_desc *desc, - struct regmap *regmap) -{ - return 0; -} - -static inline int rohm_regulator_set_voltage_sel_restricted(struct regulator_dev *rdev, - unsigned int sel) -{ - return 0; -} #endif #endif -- cgit v1.2.3-59-g8ed1b From 2338e7bcef445059a99848a3eddde0b556277663 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Mon, 15 Nov 2021 15:10:01 +0300 Subject: device property: Remove device_add_properties() API There are no more users for it. Reviewed-by: Andy Shevchenko Signed-off-by: Heikki Krogerus Signed-off-by: Rafael J. Wysocki --- drivers/base/property.c | 48 ------------------------------------------------ include/linux/property.h | 4 ---- 2 files changed, 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index f1f35b48ab8b..d0960a9e8974 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -507,54 +507,6 @@ struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode, } EXPORT_SYMBOL_GPL(fwnode_find_reference); -/** - * device_remove_properties - Remove properties from a device object. - * @dev: Device whose properties to remove. - * - * The function removes properties previously associated to the device - * firmware node with device_add_properties(). Memory allocated to the - * properties will also be released. - */ -void device_remove_properties(struct device *dev) -{ - struct fwnode_handle *fwnode = dev_fwnode(dev); - - if (!fwnode) - return; - - if (is_software_node(fwnode->secondary)) { - fwnode_remove_software_node(fwnode->secondary); - set_secondary_fwnode(dev, NULL); - } -} -EXPORT_SYMBOL_GPL(device_remove_properties); - -/** - * device_add_properties - Add a collection of properties to a device object. - * @dev: Device to add properties to. - * @properties: Collection of properties to add. - * - * Associate a collection of device properties represented by @properties with - * @dev. The function takes a copy of @properties. - * - * WARNING: The callers should not use this function if it is known that there - * is no real firmware node associated with @dev! In that case the callers - * should create a software node and assign it to @dev directly. - */ -int device_add_properties(struct device *dev, - const struct property_entry *properties) -{ - struct fwnode_handle *fwnode; - - fwnode = fwnode_create_software_node(properties, NULL); - if (IS_ERR(fwnode)) - return PTR_ERR(fwnode); - - set_secondary_fwnode(dev, fwnode); - return 0; -} -EXPORT_SYMBOL_GPL(device_add_properties); - /** * fwnode_get_name - Return the name of a node * @fwnode: The firmware node diff --git a/include/linux/property.h b/include/linux/property.h index 88fa726a76df..16f736c698a2 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -378,10 +378,6 @@ property_entries_dup(const struct property_entry *properties); void property_entries_free(const struct property_entry *properties); -int device_add_properties(struct device *dev, - const struct property_entry *properties); -void device_remove_properties(struct device *dev); - bool device_dma_supported(struct device *dev); enum dev_dma_attr device_get_dma_attr(struct device *dev); -- cgit v1.2.3-59-g8ed1b From 635e4172bd0a43af943fb164799965fc9a9a705d Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Tue, 2 Nov 2021 07:38:10 +0100 Subject: arm: remove zte zx platform left-over Commit 89d4f98ae90d ("ARM: remove zte zx platform") missed to remove some definitions for this platform's debug and serial, e.g., code dependent on the config DEBUG_ZTE_ZX. Fortunately, ./scripts/checkkconfigsymbols.py detects this and warns: DEBUG_ZTE_ZX Referencing files: arch/arm/include/debug/pl01x.S Further review by Arnd Bergmann identified even more dead code in the amba serial driver. Remove all this left-over from the zte zx platform. Reviewed-by: Arnd Bergmann Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20211102063810.932-1-lukas.bulwahn@gmail.com Signed-off-by: Greg Kroah-Hartman --- arch/arm/include/debug/pl01x.S | 7 ------- drivers/tty/serial/amba-pl011.c | 37 ------------------------------------- include/linux/amba/bus.h | 6 ------ 3 files changed, 50 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/include/debug/pl01x.S b/arch/arm/include/debug/pl01x.S index 0c7bfa4c10db..c7e02d0628bf 100644 --- a/arch/arm/include/debug/pl01x.S +++ b/arch/arm/include/debug/pl01x.S @@ -8,13 +8,6 @@ */ #include -#ifdef CONFIG_DEBUG_ZTE_ZX -#undef UART01x_DR -#undef UART01x_FR -#define UART01x_DR 0x04 -#define UART01x_FR 0x14 -#endif - #ifdef CONFIG_DEBUG_UART_PHYS .macro addruart, rp, rv, tmp ldr \rp, =CONFIG_DEBUG_UART_PHYS diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index d361cd84ff8c..c9534e229166 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -188,38 +188,6 @@ static struct vendor_data vendor_st = { .get_fifosize = get_fifosize_st, }; -static const u16 pl011_zte_offsets[REG_ARRAY_SIZE] = { - [REG_DR] = ZX_UART011_DR, - [REG_FR] = ZX_UART011_FR, - [REG_LCRH_RX] = ZX_UART011_LCRH, - [REG_LCRH_TX] = ZX_UART011_LCRH, - [REG_IBRD] = ZX_UART011_IBRD, - [REG_FBRD] = ZX_UART011_FBRD, - [REG_CR] = ZX_UART011_CR, - [REG_IFLS] = ZX_UART011_IFLS, - [REG_IMSC] = ZX_UART011_IMSC, - [REG_RIS] = ZX_UART011_RIS, - [REG_MIS] = ZX_UART011_MIS, - [REG_ICR] = ZX_UART011_ICR, - [REG_DMACR] = ZX_UART011_DMACR, -}; - -static unsigned int get_fifosize_zte(struct amba_device *dev) -{ - return 16; -} - -static struct vendor_data vendor_zte = { - .reg_offset = pl011_zte_offsets, - .access_32b = true, - .ifls = UART011_IFLS_RX4_8|UART011_IFLS_TX4_8, - .fr_busy = ZX_UART01x_FR_BUSY, - .fr_dsr = ZX_UART01x_FR_DSR, - .fr_cts = ZX_UART01x_FR_CTS, - .fr_ri = ZX_UART011_FR_RI, - .get_fifosize = get_fifosize_zte, -}; - /* Deals with DMA transactions */ struct pl011_sgbuf { @@ -2974,11 +2942,6 @@ static const struct amba_id pl011_ids[] = { .mask = 0x00ffffff, .data = &vendor_st, }, - { - .id = AMBA_LINUX_ID(0x00, 0x1, 0xffe), - .mask = 0x00ffffff, - .data = &vendor_zte, - }, { 0, 0 }, }; diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index edfcf7a14dcd..6c7f47846971 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -90,14 +90,8 @@ enum amba_vendor { AMBA_VENDOR_ST = 0x80, AMBA_VENDOR_QCOM = 0x51, AMBA_VENDOR_LSI = 0xb6, - AMBA_VENDOR_LINUX = 0xfe, /* This value is not official */ }; -/* This is used to generate pseudo-ID for AMBA device */ -#define AMBA_LINUX_ID(conf, rev, part) \ - (((conf) & 0xff) << 24 | ((rev) & 0xf) << 20 | \ - AMBA_VENDOR_LINUX << 12 | ((part) & 0xfff)) - extern struct bus_type amba_bustype; #define to_amba_device(d) container_of(d, struct amba_device, dev) -- cgit v1.2.3-59-g8ed1b From 4167bd25ec3bc221387ec6811c05eadfe3cf1d3e Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Thu, 18 Nov 2021 08:31:24 +0100 Subject: mxser: move ids from pci_ids.h here There is no point having MOXA PCI device IDs in include/linux/pci_ids.h. Move them to the driver and sort them all by the ID. Cc: Bjorn Helgaas Cc: linux-pci@vger.kernel.org Acked-by: Bjorn Helgaas Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211118073125.12283-19-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/mxser.c | 20 +++++++++++++++++++- include/linux/pci_ids.h | 18 ------------------ 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c index f6842089459a..ba96ffed193a 100644 --- a/drivers/tty/mxser.c +++ b/drivers/tty/mxser.c @@ -159,14 +159,32 @@ #define MXSER_BAUD_BASE 921600 #define MXSER_CUSTOM_DIVISOR (MXSER_BAUD_BASE * 16) +#define PCI_DEVICE_ID_MOXA_RC7000 0x0001 +#define PCI_DEVICE_ID_MOXA_CP102 0x1020 +#define PCI_DEVICE_ID_MOXA_CP102UL 0x1021 +#define PCI_DEVICE_ID_MOXA_CP102U 0x1022 +#define PCI_DEVICE_ID_MOXA_CP102UF 0x1023 +#define PCI_DEVICE_ID_MOXA_C104 0x1040 +#define PCI_DEVICE_ID_MOXA_CP104U 0x1041 +#define PCI_DEVICE_ID_MOXA_CP104JU 0x1042 +#define PCI_DEVICE_ID_MOXA_CP104EL 0x1043 #define PCI_DEVICE_ID_MOXA_POS104UL 0x1044 #define PCI_DEVICE_ID_MOXA_CB108 0x1080 -#define PCI_DEVICE_ID_MOXA_CP102UF 0x1023 #define PCI_DEVICE_ID_MOXA_CP112UL 0x1120 +#define PCI_DEVICE_ID_MOXA_CT114 0x1140 +#define PCI_DEVICE_ID_MOXA_CP114 0x1141 #define PCI_DEVICE_ID_MOXA_CB114 0x1142 #define PCI_DEVICE_ID_MOXA_CP114UL 0x1143 +#define PCI_DEVICE_ID_MOXA_CP118U 0x1180 +#define PCI_DEVICE_ID_MOXA_CP118EL 0x1181 +#define PCI_DEVICE_ID_MOXA_CP132 0x1320 +#define PCI_DEVICE_ID_MOXA_CP132U 0x1321 +#define PCI_DEVICE_ID_MOXA_CP134U 0x1340 #define PCI_DEVICE_ID_MOXA_CB134I 0x1341 #define PCI_DEVICE_ID_MOXA_CP138U 0x1380 +#define PCI_DEVICE_ID_MOXA_C168 0x1680 +#define PCI_DEVICE_ID_MOXA_CP168U 0x1681 +#define PCI_DEVICE_ID_MOXA_CP168EL 0x1682 #define MXSER_NPORTS(ddata) ((ddata) & 0xffU) #define MXSER_HIGHBAUD 0x0100 diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 011f2f1ea5bb..c389a9c0f290 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -1964,24 +1964,6 @@ #define PCI_DEVICE_ID_APPLICOM_PCI2000PFB 0x0003 #define PCI_VENDOR_ID_MOXA 0x1393 -#define PCI_DEVICE_ID_MOXA_RC7000 0x0001 -#define PCI_DEVICE_ID_MOXA_CP102 0x1020 -#define PCI_DEVICE_ID_MOXA_CP102UL 0x1021 -#define PCI_DEVICE_ID_MOXA_CP102U 0x1022 -#define PCI_DEVICE_ID_MOXA_C104 0x1040 -#define PCI_DEVICE_ID_MOXA_CP104U 0x1041 -#define PCI_DEVICE_ID_MOXA_CP104JU 0x1042 -#define PCI_DEVICE_ID_MOXA_CP104EL 0x1043 -#define PCI_DEVICE_ID_MOXA_CT114 0x1140 -#define PCI_DEVICE_ID_MOXA_CP114 0x1141 -#define PCI_DEVICE_ID_MOXA_CP118U 0x1180 -#define PCI_DEVICE_ID_MOXA_CP118EL 0x1181 -#define PCI_DEVICE_ID_MOXA_CP132 0x1320 -#define PCI_DEVICE_ID_MOXA_CP132U 0x1321 -#define PCI_DEVICE_ID_MOXA_CP134U 0x1340 -#define PCI_DEVICE_ID_MOXA_C168 0x1680 -#define PCI_DEVICE_ID_MOXA_CP168U 0x1681 -#define PCI_DEVICE_ID_MOXA_CP168EL 0x1682 #define PCI_DEVICE_ID_MOXA_CP204J 0x2040 #define PCI_DEVICE_ID_MOXA_C218 0x2180 #define PCI_DEVICE_ID_MOXA_C320 0x3200 -- cgit v1.2.3-59-g8ed1b From 5db96ef23bda6c2a61a51693c85b78b52d03f654 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Mon, 22 Nov 2021 12:16:48 +0100 Subject: tty: drop tty_schedule_flip() Since commit a9c3f68f3cd8d (tty: Fix low_latency BUG) in 2014, tty_flip_buffer_push() is only a wrapper to tty_schedule_flip(). All users were converted in the previous patches, so remove tty_schedule_flip() completely while inlining its body into tty_flip_buffer_push(). One less exported function. Reviewed-by: Johan Hovold Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211122111648.30379-4-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_buffer.c | 30 ++++++++---------------------- include/linux/tty_flip.h | 1 - 2 files changed, 8 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index 6c7e65b1d9a1..5b6875057ce2 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -401,27 +401,6 @@ int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag) } EXPORT_SYMBOL(__tty_insert_flip_char); -/** - * tty_schedule_flip - push characters to ldisc - * @port: tty port to push from - * - * Takes any pending buffers and transfers their ownership to the - * ldisc side of the queue. It then schedules those characters for - * processing by the line discipline. - */ - -void tty_schedule_flip(struct tty_port *port) -{ - struct tty_bufhead *buf = &port->buf; - - /* paired w/ acquire in flush_to_ldisc(); ensures - * flush_to_ldisc() sees buffer data. - */ - smp_store_release(&buf->tail->commit, buf->tail->used); - queue_work(system_unbound_wq, &buf->work); -} -EXPORT_SYMBOL(tty_schedule_flip); - /** * tty_prepare_flip_string - make room for characters * @port: tty port @@ -566,7 +545,14 @@ static void flush_to_ldisc(struct work_struct *work) void tty_flip_buffer_push(struct tty_port *port) { - tty_schedule_flip(port); + struct tty_bufhead *buf = &port->buf; + + /* + * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees + * buffer data. + */ + smp_store_release(&buf->tail->commit, buf->tail->used); + queue_work(system_unbound_wq, &buf->work); } EXPORT_SYMBOL(tty_flip_buffer_push); diff --git a/include/linux/tty_flip.h b/include/linux/tty_flip.h index 9916acb5de49..483d41cbcbb7 100644 --- a/include/linux/tty_flip.h +++ b/include/linux/tty_flip.h @@ -17,7 +17,6 @@ int tty_insert_flip_string_fixed_flag(struct tty_port *port, int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars, size_t size); void tty_flip_buffer_push(struct tty_port *port); -void tty_schedule_flip(struct tty_port *port); int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag); static inline int tty_insert_flip_char(struct tty_port *port, -- cgit v1.2.3-59-g8ed1b From d78328bcc4d0e677f2ff83f4ae1f43c933fbd143 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Mon, 22 Nov 2021 10:45:29 +0100 Subject: tty: remove file from tty_ldisc_ops::ioctl and compat_ioctl After the previous patches, noone needs 'file' parameter in neither ioctl hook from tty_ldisc_ops. So remove 'file' from both of them. Cc: Marcel Holtmann Cc: Johan Hedberg Cc: Luiz Augusto von Dentz Cc: Wolfgang Grandegger Cc: Marc Kleine-Budde Cc: "David S. Miller" Cc: Jakub Kicinski Cc: Andreas Koensgen Cc: Paul Mackerras Acked-by: Krzysztof Kozlowski [NFC] Acked-by: Dmitry Torokhov Acked-by: Marc Kleine-Budde Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211122094529.24171-1-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/bluetooth/hci_ldisc.c | 5 ++--- drivers/input/serio/serport.c | 5 ++--- drivers/net/can/slcan.c | 4 ++-- drivers/net/hamradio/6pack.c | 4 ++-- drivers/net/hamradio/mkiss.c | 4 ++-- drivers/net/ppp/ppp_async.c | 3 +-- drivers/net/ppp/ppp_synctty.c | 3 +-- drivers/net/slip/slip.c | 4 ++-- drivers/tty/n_gsm.c | 4 ++-- drivers/tty/n_hdlc.c | 5 ++--- drivers/tty/n_tty.c | 4 ++-- drivers/tty/tty_io.c | 8 ++++---- include/linux/tty_ldisc.h | 15 +++++++-------- net/nfc/nci/uart.c | 5 ++--- 14 files changed, 33 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index ecdf8e034351..f537673ede17 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -739,14 +739,13 @@ static int hci_uart_set_flags(struct hci_uart *hu, unsigned long flags) * Arguments: * * tty pointer to tty instance data - * file pointer to open file object for device * cmd IOCTL command code * arg argument for IOCTL call (cmd dependent) * * Return Value: Command dependent */ -static int hci_uart_tty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int hci_uart_tty_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct hci_uart *hu = tty->disc_data; int err = 0; diff --git a/drivers/input/serio/serport.c b/drivers/input/serio/serport.c index 17eb8f2aa48d..669a728095b8 100644 --- a/drivers/input/serio/serport.c +++ b/drivers/input/serio/serport.c @@ -207,8 +207,8 @@ static void serport_set_type(struct tty_struct *tty, unsigned long type) * serport_ldisc_ioctl() allows to set the port protocol, and device ID */ -static int serport_ldisc_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int serport_ldisc_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { if (cmd == SPIOCSTYPE) { unsigned long type; @@ -226,7 +226,6 @@ static int serport_ldisc_ioctl(struct tty_struct *tty, struct file *file, #ifdef CONFIG_COMPAT #define COMPAT_SPIOCSTYPE _IOW('q', 0x01, compat_ulong_t) static int serport_ldisc_compat_ioctl(struct tty_struct *tty, - struct file *file, unsigned int cmd, unsigned long arg) { if (cmd == COMPAT_SPIOCSTYPE) { diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c index 9a4ebda30510..113763790ac9 100644 --- a/drivers/net/can/slcan.c +++ b/drivers/net/can/slcan.c @@ -670,8 +670,8 @@ static void slcan_hangup(struct tty_struct *tty) } /* Perform I/O control on an active SLCAN channel. */ -static int slcan_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int slcan_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct slcan *sl = (struct slcan *) tty->disc_data; unsigned int tmp; diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c index 8a19a06b505d..b1fc153125d9 100644 --- a/drivers/net/hamradio/6pack.c +++ b/drivers/net/hamradio/6pack.c @@ -681,8 +681,8 @@ static void sixpack_close(struct tty_struct *tty) } /* Perform I/O control on an active 6pack channel. */ -static int sixpack_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int sixpack_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct sixpack *sp = sp_get(tty); struct net_device *dev; diff --git a/drivers/net/hamradio/mkiss.c b/drivers/net/hamradio/mkiss.c index e2b332b54f06..894b5f92b85f 100644 --- a/drivers/net/hamradio/mkiss.c +++ b/drivers/net/hamradio/mkiss.c @@ -804,8 +804,8 @@ static void mkiss_close(struct tty_struct *tty) } /* Perform I/O control on an active ax25 channel. */ -static int mkiss_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int mkiss_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct mkiss *ax = mkiss_get(tty); struct net_device *dev; diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c index f4429b93a9c8..15a179631903 100644 --- a/drivers/net/ppp/ppp_async.c +++ b/drivers/net/ppp/ppp_async.c @@ -281,8 +281,7 @@ ppp_asynctty_write(struct tty_struct *tty, struct file *file, */ static int -ppp_asynctty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +ppp_asynctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct asyncppp *ap = ap_get(tty); int err, val; diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c index b3a71b409a80..18283b7b94bc 100644 --- a/drivers/net/ppp/ppp_synctty.c +++ b/drivers/net/ppp/ppp_synctty.c @@ -274,8 +274,7 @@ ppp_sync_write(struct tty_struct *tty, struct file *file, } static int -ppp_synctty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +ppp_synctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct syncppp *ap = sp_get(tty); int __user *p = (int __user *)arg; diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c index 9f3b4c1aa5ce..98f586f910fb 100644 --- a/drivers/net/slip/slip.c +++ b/drivers/net/slip/slip.c @@ -1072,8 +1072,8 @@ static void slip_unesc6(struct slip *sl, unsigned char s) #endif /* CONFIG_SLIP_MODE_SLIP6 */ /* Perform I/O control on an active SLIP channel. */ -static int slip_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int slip_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct slip *sl = tty->disc_data; unsigned int tmp; diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 68e6df27d2e3..ba27b274c967 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -2687,8 +2687,8 @@ static __poll_t gsmld_poll(struct tty_struct *tty, struct file *file, return mask; } -static int gsmld_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int gsmld_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct gsm_config c; struct gsm_mux *gsm = tty->disc_data; diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c index 7e0884ecc74f..a66915032e7e 100644 --- a/drivers/tty/n_hdlc.c +++ b/drivers/tty/n_hdlc.c @@ -572,14 +572,13 @@ static ssize_t n_hdlc_tty_write(struct tty_struct *tty, struct file *file, /** * n_hdlc_tty_ioctl - process IOCTL system call for the tty device. * @tty: pointer to tty instance data - * @file: pointer to open file object for device * @cmd: IOCTL command code * @arg: argument for IOCTL call (cmd dependent) * * Returns command dependent result. */ -static int n_hdlc_tty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int n_hdlc_tty_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct n_hdlc *n_hdlc = tty->disc_data; int error = 0; diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c index 5be6d02dc690..5b0f50373fc6 100644 --- a/drivers/tty/n_tty.c +++ b/drivers/tty/n_tty.c @@ -2400,8 +2400,8 @@ static unsigned long inq_canon(struct n_tty_data *ldata) return nr; } -static int n_tty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int n_tty_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct n_tty_data *ldata = tty->disc_data; int retval; diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 99cad1560876..3c2349b2089c 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -2811,7 +2811,7 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return hung_up_tty_ioctl(file, cmd, arg); retval = -EINVAL; if (ld->ops->ioctl) { - retval = ld->ops->ioctl(tty, file, cmd, arg); + retval = ld->ops->ioctl(tty, cmd, arg); if (retval == -ENOIOCTLCMD) retval = -ENOTTY; } @@ -2990,10 +2990,10 @@ static long tty_compat_ioctl(struct file *file, unsigned int cmd, if (!ld) return hung_up_tty_compat_ioctl(file, cmd, arg); if (ld->ops->compat_ioctl) - retval = ld->ops->compat_ioctl(tty, file, cmd, arg); + retval = ld->ops->compat_ioctl(tty, cmd, arg); if (retval == -ENOIOCTLCMD && ld->ops->ioctl) - retval = ld->ops->ioctl(tty, file, - (unsigned long)compat_ptr(cmd), arg); + retval = ld->ops->ioctl(tty, (unsigned long)compat_ptr(cmd), + arg); tty_ldisc_deref(ld); return retval; diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index b85d84fb5f49..25f07017bbad 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -45,8 +45,7 @@ struct tty_struct; * some processing on the characters first. If this function is * not defined, the user will receive an EIO error. * - * int (*ioctl)(struct tty_struct * tty, struct file * file, - * unsigned int cmd, unsigned long arg); + * int (*ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); * * This function is called when the user requests an ioctl which * is not handled by the tty layer or the low-level tty driver. @@ -56,8 +55,8 @@ struct tty_struct; * low-level driver can "grab" an ioctl request before the line * discpline has a chance to see it. * - * int (*compat_ioctl)(struct tty_struct * tty, struct file * file, - * unsigned int cmd, unsigned long arg); + * int (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, + * unsigned long arg); * * Process ioctl calls from 32-bit process on 64-bit system * @@ -192,10 +191,10 @@ struct tty_ldisc_ops { void **cookie, unsigned long offset); ssize_t (*write)(struct tty_struct *tty, struct file *file, const unsigned char *buf, size_t nr); - int (*ioctl)(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg); - int (*compat_ioctl)(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg); + int (*ioctl)(struct tty_struct *tty, unsigned int cmd, + unsigned long arg); + int (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, + unsigned long arg); void (*set_termios)(struct tty_struct *tty, struct ktermios *old); __poll_t (*poll)(struct tty_struct *, struct file *, struct poll_table_struct *); diff --git a/net/nfc/nci/uart.c b/net/nfc/nci/uart.c index c027c76d493c..cc8fa9e36159 100644 --- a/net/nfc/nci/uart.c +++ b/net/nfc/nci/uart.c @@ -317,14 +317,13 @@ static void nci_uart_tty_receive(struct tty_struct *tty, const u8 *data, * Arguments: * * tty pointer to tty instance data - * file pointer to open file object for device * cmd IOCTL command code * arg argument for IOCTL call (cmd dependent) * * Return Value: Command dependent */ -static int nci_uart_tty_ioctl(struct tty_struct *tty, struct file *file, - unsigned int cmd, unsigned long arg) +static int nci_uart_tty_ioctl(struct tty_struct *tty, unsigned int cmd, + unsigned long arg) { struct nci_uart *nu = (void *)tty->disc_data; int err = 0; -- cgit v1.2.3-59-g8ed1b From 29c3002644bdd653f6ec6407d25135d0a4f7cefb Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 24 Nov 2021 12:24:46 -0800 Subject: net: optimize skb_postpull_rcsum() Remove one pair of add/adc instructions and their dependency against carry flag. We can leverage third argument to csum_partial(): X = csum_block_sub(X, csum_partial(start, len, 0), 0); --> X = csum_block_add(X, ~csum_partial(start, len, 0), 0); --> X = ~csum_partial(start, len, ~X); Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index eba256af64a5..eae4bd3237a4 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3485,7 +3485,11 @@ __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, static inline void skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len) { - __skb_postpull_rcsum(skb, start, len, 0); + if (skb->ip_summed == CHECKSUM_COMPLETE) + skb->csum = ~csum_partial(start, len, ~skb->csum); + else if (skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_start_offset(skb) < 0) + skb->ip_summed = CHECKSUM_NONE; } static __always_inline void -- cgit v1.2.3-59-g8ed1b From b4c80629c5c9d48880c5ad99943374f9ab72432e Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Sun, 23 May 2021 22:49:57 +0200 Subject: include/linux/byteorder/generic.h: fix index variables In cpu_to_be32_array() and be32_to_cpu_array() the length of the array is given by variable len of type size_t. An index variable of type int is used to iterate over the array. This is bound to fail for len > INT_MAX and lets GCC add instructions for sign extension. Correct the type of the index variable. Signed-off-by: Heinrich Schuchardt Link: https://lore.kernel.org/r/20210523204958.64575-1-xypron.glpk@gmx.de Signed-off-by: Greg Kroah-Hartman --- include/linux/byteorder/generic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/byteorder/generic.h b/include/linux/byteorder/generic.h index 4b13e0a3e15b..c9a4c96c9943 100644 --- a/include/linux/byteorder/generic.h +++ b/include/linux/byteorder/generic.h @@ -190,7 +190,7 @@ static inline void be64_add_cpu(__be64 *var, u64 val) static inline void cpu_to_be32_array(__be32 *dst, const u32 *src, size_t len) { - int i; + size_t i; for (i = 0; i < len; i++) dst[i] = cpu_to_be32(src[i]); @@ -198,7 +198,7 @@ static inline void cpu_to_be32_array(__be32 *dst, const u32 *src, size_t len) static inline void be32_to_cpu_array(u32 *dst, const __be32 *src, size_t len) { - int i; + size_t i; for (i = 0; i < len; i++) dst[i] = be32_to_cpu(src[i]); -- cgit v1.2.3-59-g8ed1b From 18e6c0751cf9ae0631a2623e31af2bf504f72c30 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:49 +0100 Subject: tty: finish kernel-doc of tty_struct members There are already pieces of kernel-doc documentation for struct tty_struct in tty.h. Finish the documentation for the members which were undocumented yet. It also includes tuning the already existing pieces like flow and ctrl, especially adding highlights to them. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-2-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty.h | 79 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty.h b/include/linux/tty.h index 5dbd7c5afac7..da49ad9be281 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -122,33 +122,84 @@ struct tty_operations; /** * struct tty_struct - state associated with a tty while open * - * @flow.lock: lock for flow members - * @flow.stopped: tty stopped/started by tty_stop/tty_start - * @flow.tco_stopped: tty stopped/started by TCOOFF/TCOON ioctls (it has - * precedense over @flow.stopped) + * @magic: magic value set early in @alloc_tty_struct to %TTY_MAGIC, for + * debugging purposes + * @kref: reference counting by tty_kref_get() and tty_kref_put(), reaching zero + * frees the structure + * @dev: class device or %NULL (e.g. ptys, serdev) + * @driver: &struct tty_driver operating this tty + * @ops: &struct tty_operations of @driver for this tty (open, close, etc.) + * @index: index of this tty (e.g. to construct @name like tty12) + * @ldisc_sem: protects line discipline changes (@ldisc) -- lock tty not pty + * @ldisc: the current line discipline for this tty (n_tty by default) + * @atomic_write_lock: protects against concurrent writers, i.e. locks + * @write_cnt, @write_buf and similar + * @legacy_mutex: leftover from history (BKL -> BTM -> @legacy_mutex), + * protecting several operations on this tty + * @throttle_mutex: protects against concurrent tty_throttle_safe() and + * tty_unthrottle_safe() (but not tty_unthrottle()) + * @termios_rwsem: protects @termios and @termios_locked + * @winsize_mutex: protects @winsize + * @termios: termios for the current tty, copied from/to @driver.termios + * @termios_locked: locked termios (by %TIOCGLCKTRMIOS and %TIOCSLCKTRMIOS + * ioctls) + * @name: name of the tty constructed by tty_line_name() (e.g. ttyS3) + * @flags: bitwise OR of %TTY_THROTTLED, %TTY_IO_ERROR, ... + * @count: count of open processes, reaching zero cancels all the work for + * this tty and drops a @kref too (but does not free this tty) + * @winsize: size of the terminal "window" (cf. @winsize_mutex) + * @flow: flow settings grouped together, see also @flow.unused + * @flow.lock: lock for @flow members + * @flow.stopped: tty stopped/started by stop_tty()/start_tty() + * @flow.tco_stopped: tty stopped/started by %TCOOFF/%TCOON ioctls (it has + * precedence over @flow.stopped) * @flow.unused: alignment for Alpha, so that no members other than @flow.* are * modified by the same 64b word store. The @flow's __aligned is * there for the very same reason. - * @ctrl.lock: lock for ctrl members + * @ctrl: control settings grouped together, see also @ctrl.unused + * @ctrl.lock: lock for @ctrl members * @ctrl.pgrp: process group of this tty (setpgrp(2)) * @ctrl.session: session of this tty (setsid(2)). Writes are protected by both - * @ctrl.lock and legacy mutex, readers must use at least one of + * @ctrl.lock and @legacy_mutex, readers must use at least one of * them. - * @ctrl.pktstatus: packet mode status (bitwise OR of TIOCPKT_* constants) + * @ctrl.pktstatus: packet mode status (bitwise OR of %TIOCPKT_ constants) * @ctrl.packet: packet mode enabled + * @ctrl.unused: alignment for Alpha, see @flow.unused for explanation + * @hw_stopped: not controlled by the tty layer, under @driver's control for CTS + * handling + * @receive_room: bytes permitted to feed to @ldisc without any being lost + * @flow_change: controls behavior of throttling, see tty_throttle_safe() and + * tty_unthrottle_safe() + * @link: link to another pty (master -> slave and vice versa) + * @fasync: state for %O_ASYNC (for %SIGIO); managed by fasync_helper() + * @write_wait: concurrent writers are waiting in this queue until they are + * allowed to write + * @read_wait: readers wait for data in this queue + * @hangup_work: normally a work to perform a hangup (do_tty_hangup()); while + * freeing the tty, (re)used to release_one_tty() + * @disc_data: pointer to @ldisc's private data (e.g. to &struct n_tty_data) + * @driver_data: pointer to @driver's private data (e.g. &struct uart_state) + * @files_lock: protects @tty_files list + * @tty_files: list of (re)openers of this tty (i.e. linked &struct + * tty_file_private) + * @closing: when set during close, n_tty processes only START & STOP chars + * @write_buf: temporary buffer used during tty_write() to copy user data to + * @write_cnt: count of bytes written in tty_write() to @write_buf + * @SAK_work: if the tty has a pending do_SAK, it is queued here + * @port: persistent storage for this device (i.e. &struct tty_port) * * All of the state associated with a tty while the tty is open. Persistent - * storage for tty devices is referenced here as @port in struct tty_port. + * storage for tty devices is referenced here as @port and is documented in + * &struct tty_port. */ struct tty_struct { int magic; struct kref kref; - struct device *dev; /* class device or NULL (e.g. ptys, serdev) */ + struct device *dev; struct tty_driver *driver; const struct tty_operations *ops; int index; - /* Protects ldisc changes: Lock tty not pty */ struct ld_semaphore ldisc_sem; struct tty_ldisc *ldisc; @@ -157,12 +208,11 @@ struct tty_struct { struct mutex throttle_mutex; struct rw_semaphore termios_rwsem; struct mutex winsize_mutex; - /* Termios values are protected by the termios rwsem */ struct ktermios termios, termios_locked; char name[64]; unsigned long flags; int count; - struct winsize winsize; /* winsize_mutex */ + struct winsize winsize; struct { spinlock_t lock; @@ -181,7 +231,7 @@ struct tty_struct { } __aligned(sizeof(unsigned long)) ctrl; int hw_stopped; - unsigned int receive_room; /* Bytes free for queue */ + unsigned int receive_room; int flow_change; struct tty_struct *link; @@ -191,7 +241,7 @@ struct tty_struct { struct work_struct hangup_work; void *disc_data; void *driver_data; - spinlock_t files_lock; /* protects tty_files list */ + spinlock_t files_lock; struct list_head tty_files; #define N_TTY_BUF_SIZE 4096 @@ -199,7 +249,6 @@ struct tty_struct { int closing; unsigned char *write_buf; int write_cnt; - /* If the tty has a pending do_SAK, queue it here - akpm */ struct work_struct SAK_work; struct tty_port *port; } __randomize_layout; -- cgit v1.2.3-59-g8ed1b From 61c83addb77c65f498a8db0e7113dc3acf753c45 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:50 +0100 Subject: tty: add kernel-doc for tty_port tty_port used to have only short comments along its members. Convert them into proper kernel-doc comments in front of the structure. And add some more explanation to them where needed. The whole structure purpose and handling is documented at the end too -- some pieces of preexisting text moved to this place. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-3-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_port.h | 104 +++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 6e86e9e118b6..9091e1c8de4c 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -7,17 +7,6 @@ #include #include -/* - * Port level information. Each device keeps its own port level information - * so provide a common structure for those ports wanting to use common support - * routines. - * - * The tty port has a different lifetime to the tty so must be kept apart. - * In addition be careful as tty -> port mappings are valid for the life - * of the tty object but in many cases port -> tty mappings are valid only - * until a hangup so don't use the wrong path. - */ - struct attribute_group; struct tty_driver; struct tty_port; @@ -48,30 +37,77 @@ struct tty_port_client_operations { extern const struct tty_port_client_operations tty_port_default_client_ops; +/** + * struct tty_port -- port level information + * + * @buf: buffer for this port, locked internally + * @tty: back pointer to &struct tty_struct, valid only if the tty is open. Use + * tty_port_tty_get() to obtain it (and tty_kref_put() to release). + * @itty: internal back pointer to &struct tty_struct. Avoid this. It should be + * eliminated in the long term. + * @ops: tty port operations (like activate, shutdown), see &struct + * tty_port_operations + * @client_ops: tty port client operations (like receive_buf, write_wakeup). + * By default, tty_port_default_client_ops is used. + * @lock: lock protecting @tty + * @blocked_open: # of procs waiting for open in tty_port_block_til_ready() + * @count: usage count + * @open_wait: open waiters queue (waiting e.g. for a carrier) + * @delta_msr_wait: modem status change queue (waiting for MSR changes) + * @flags: user TTY flags (%ASYNC_) + * @iflags: internal flags (%TTY_PORT_) + * @console: when set, the port is a console + * @mutex: locking, for open, shutdown and other port operations + * @buf_mutex: @xmit_buf alloc lock + * @xmit_buf: optional xmit buffer used by some drivers + * @close_delay: delay in jiffies to wait when closing the port + * @closing_wait: delay in jiffies for output to be sent before closing + * @drain_delay: set to zero if no pure time based drain is needed else set to + * size of fifo + * @kref: references counter. Reaching zero calls @ops->destruct() if non-%NULL + * or frees the port otherwise. + * @client_data: pointer to private data, for @client_ops + * + * Each device keeps its own port level information. &struct tty_port was + * introduced as a common structure for such information. As every TTY device + * shall have a backing tty_port structure, every driver can use these members. + * + * The tty port has a different lifetime to the tty so must be kept apart. + * In addition be careful as tty -> port mappings are valid for the life + * of the tty object but in many cases port -> tty mappings are valid only + * until a hangup so don't use the wrong path. + * + * Tty port shall be initialized by tty_port_init() and shut down either by + * tty_port_destroy() (refcounting not used), or tty_port_put() (refcounting). + * + * There is a lot of helpers around &struct tty_port too. To name the most + * significant ones: tty_port_open(), tty_port_close() (or + * tty_port_close_start() and tty_port_close_end() separately if need be), and + * tty_port_hangup(). These call @ops->activate() and @ops->shutdown() as + * needed. + */ struct tty_port { - struct tty_bufhead buf; /* Locked internally */ - struct tty_struct *tty; /* Back pointer */ - struct tty_struct *itty; /* internal back ptr */ - const struct tty_port_operations *ops; /* Port operations */ - const struct tty_port_client_operations *client_ops; /* Port client operations */ - spinlock_t lock; /* Lock protecting tty field */ - int blocked_open; /* Waiting to open */ - int count; /* Usage count */ - wait_queue_head_t open_wait; /* Open waiters */ - wait_queue_head_t delta_msr_wait; /* Modem status change */ - unsigned long flags; /* User TTY flags ASYNC_ */ - unsigned long iflags; /* Internal flags TTY_PORT_ */ - unsigned char console:1; /* port is a console */ - struct mutex mutex; /* Locking */ - struct mutex buf_mutex; /* Buffer alloc lock */ - unsigned char *xmit_buf; /* Optional buffer */ - unsigned int close_delay; /* Close port delay */ - unsigned int closing_wait; /* Delay for output */ - int drain_delay; /* Set to zero if no pure time - based drain is needed else - set to size of fifo */ - struct kref kref; /* Ref counter */ - void *client_data; + struct tty_bufhead buf; + struct tty_struct *tty; + struct tty_struct *itty; + const struct tty_port_operations *ops; + const struct tty_port_client_operations *client_ops; + spinlock_t lock; + int blocked_open; + int count; + wait_queue_head_t open_wait; + wait_queue_head_t delta_msr_wait; + unsigned long flags; + unsigned long iflags; + unsigned char console:1; + struct mutex mutex; + struct mutex buf_mutex; + unsigned char *xmit_buf; + unsigned int close_delay; + unsigned int closing_wait; + int drain_delay; + struct kref kref; + void *client_data; }; /* tty_port::iflags bits -- use atomic bit ops */ -- cgit v1.2.3-59-g8ed1b From a6563830215226aae0e7e6802955c77a6a7b7547 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:51 +0100 Subject: tty: add kernel-doc for tty_driver tty_driver used to have only short comments along its members. Convert them into proper kernel-doc comments in front of the structure. And add some more explanation to them where needed. The whole structure handling is documented at the end too. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-4-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_driver.h | 62 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 795b94ccdeb6..3622404a678d 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -291,23 +291,61 @@ struct tty_operations { int (*proc_show)(struct seq_file *, void *); } __randomize_layout; +/** + * struct tty_driver -- driver for TTY devices + * + * @magic: set to %TTY_DRIVER_MAGIC in __tty_alloc_driver() + * @kref: reference counting. Reaching zero frees all the internals and the + * driver. + * @cdevs: allocated/registered character /dev devices + * @owner: modules owning this driver. Used drivers cannot be rmmod'ed. + * Automatically set by tty_alloc_driver(). + * @driver_name: name of the driver used in /proc/tty + * @name: used for constructing /dev node name + * @name_base: used as a number base for constructing /dev node name + * @major: major /dev device number (zero for autoassignment) + * @minor_start: the first minor /dev device number + * @num: number of devices allocated + * @type: type of tty driver (%TTY_DRIVER_TYPE_) + * @subtype: subtype of tty driver (%SYSTEM_TYPE_, %PTY_TYPE_, %SERIAL_TYPE_) + * @init_termios: termios to set to each tty initially (e.g. %tty_std_termios) + * @flags: tty driver flags (%TTY_DRIVER_) + * @proc_entry: proc fs entry, used internally + * @other: driver of the linked tty; only used for the PTY driver + * @ttys: array of active &struct tty_struct, set by tty_standard_install() + * @ports: array of &struct tty_port; can be set during initialization by + * tty_port_link_device() and similar + * @termios: storage for termios at each TTY close for the next open + * @driver_state: pointer to driver's arbitrary data + * @ops: driver hooks for TTYs. Set them using tty_set_operations(). Use &struct + * tty_port helpers in them as much as possible. + * @tty_drivers: used internally to link tty_drivers together + * + * The usual handling of &struct tty_driver is to allocate it by + * tty_alloc_driver(), set up all the necessary members, and register it by + * tty_register_driver(). At last, the driver is torn down by calling + * tty_unregister_driver() followed by tty_driver_kref_put(). + * + * The fields required to be set before calling tty_register_driver() include + * @driver_name, @name, @type, @subtype, @init_termios, and @ops. + */ struct tty_driver { - int magic; /* magic number for this structure */ - struct kref kref; /* Reference management */ + int magic; + struct kref kref; struct cdev **cdevs; struct module *owner; const char *driver_name; const char *name; - int name_base; /* offset of printed name */ - int major; /* major device number */ - int minor_start; /* start of minor device number */ - unsigned int num; /* number of devices allocated */ - short type; /* type of tty driver */ - short subtype; /* subtype of tty driver */ - struct ktermios init_termios; /* Initial termios */ - unsigned long flags; /* tty driver flags */ - struct proc_dir_entry *proc_entry; /* /proc fs entry */ - struct tty_driver *other; /* only used for the PTY driver */ + int name_base; + int major; + int minor_start; + unsigned int num; + short type; + short subtype; + struct ktermios init_termios; + unsigned long flags; + struct proc_dir_entry *proc_entry; + struct tty_driver *other; /* * Pointer to the tty data structures -- cgit v1.2.3-59-g8ed1b From 1fe183091753b1d7f11e70593700c0c0ef268db7 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:52 +0100 Subject: tty: add kernel-doc for tty_operations tty_operations structure was already documented in a standalone comment in the header beginning. Move it right before the structure and reformat it so it complies to kernel-doc. That way, we can include it in Documentation/ later in this series. Note that we named proc_show's parameters, so that we can reference them in the text. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-5-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_driver.h | 398 +++++++++++++++++++++++++++------------------ 1 file changed, 241 insertions(+), 157 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 3622404a678d..5611992ab26a 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -2,248 +2,332 @@ #ifndef _LINUX_TTY_DRIVER_H #define _LINUX_TTY_DRIVER_H -/* - * This structure defines the interface between the low-level tty - * driver and the tty routines. The following routines can be - * defined; unless noted otherwise, they are optional, and can be - * filled in with a null pointer. +#include +#include +#include +#include +#include +#include +#include + +struct tty_struct; +struct tty_driver; +struct serial_icounter_struct; +struct serial_struct; + +/** + * struct tty_operations -- interface between driver and tty * - * struct tty_struct * (*lookup)(struct tty_driver *self, struct file *, int idx) + * @lookup: ``struct tty_struct *()(struct tty_driver *self, struct file *, + * int idx)`` * - * Return the tty device corresponding to idx, NULL if there is not - * one currently in use and an ERR_PTR value on error. Called under - * tty_mutex (for now!) + * Return the tty device corresponding to @idx, %NULL if there is not + * one currently in use and an %ERR_PTR value on error. Called under + * %tty_mutex (for now!) * - * Optional method. Default behaviour is to use the ttys array + * Optional method. Default behaviour is to use the @self->ttys array. * - * int (*install)(struct tty_driver *self, struct tty_struct *tty) + * @install: ``int ()(struct tty_driver *self, struct tty_struct *tty)`` * - * Install a new tty into the tty driver internal tables. Used in - * conjunction with lookup and remove methods. + * Install a new @tty into the @self's internal tables. Used in + * conjunction with @lookup and @remove methods. * - * Optional method. Default behaviour is to use the ttys array + * Optional method. Default behaviour is to use the @self->ttys array. * - * void (*remove)(struct tty_driver *self, struct tty_struct *tty) + * @remove: ``void ()(struct tty_driver *self, struct tty_struct *tty)`` * - * Remove a closed tty from the tty driver internal tables. Used in - * conjunction with lookup and remove methods. + * Remove a closed @tty from the @self's internal tables. Used in + * conjunction with @lookup and @remove methods. * - * Optional method. Default behaviour is to use the ttys array + * Optional method. Default behaviour is to use the @self->ttys array. * - * int (*open)(struct tty_struct * tty, struct file * filp); + * @open: ``int ()(struct tty_struct *tty, struct file *)`` * - * This routine is called when a particular tty device is opened. - * This routine is mandatory; if this routine is not filled in, - * the attempted open will fail with ENODEV. + * This routine is called when a particular @tty device is opened. This + * routine is mandatory; if this routine is not filled in, the attempted + * open will fail with %ENODEV. * * Required method. Called with tty lock held. * - * void (*close)(struct tty_struct * tty, struct file * filp); + * @close: ``void ()(struct tty_struct *tty, struct file *)`` * - * This routine is called when a particular tty device is closed. - * Note: called even if the corresponding open() failed. + * This routine is called when a particular @tty device is closed. + * + * Remark: called even if the corresponding @open() failed. * * Required method. Called with tty lock held. * - * void (*shutdown)(struct tty_struct * tty); + * @shutdown: ``void ()(struct tty_struct *tty)`` * - * This routine is called under the tty lock when a particular tty device - * is closed for the last time. It executes before the tty resources - * are freed so may execute while another function holds a tty kref. + * This routine is called under the tty lock when a particular @tty device + * is closed for the last time. It executes before the @tty resources + * are freed so may execute while another function holds a @tty kref. * - * void (*cleanup)(struct tty_struct * tty); + * @cleanup: ``void ()(struct tty_struct *tty)`` * - * This routine is called asynchronously when a particular tty device + * This routine is called asynchronously when a particular @tty device * is closed for the last time freeing up the resources. This is * actually the second part of shutdown for routines that might sleep. * + * @write: ``int ()(struct tty_struct *tty, const unsigned char *buf, + * int count)`` * - * int (*write)(struct tty_struct * tty, - * const unsigned char *buf, int count); - * - * This routine is called by the kernel to write a series of - * characters to the tty device. The characters may come from - * user space or kernel space. This routine will return the + * This routine is called by the kernel to write a series (@count) of + * characters (@buf) to the @tty device. The characters may come from + * user space or kernel space. This routine will return the * number of characters actually accepted for writing. * * Optional: Required for writable devices. * - * int (*put_char)(struct tty_struct *tty, unsigned char ch); + * @put_char: ``int ()(struct tty_struct *tty, unsigned char ch)`` * - * This routine is called by the kernel to write a single - * character to the tty device. If the kernel uses this routine, - * it must call the flush_chars() routine (if defined) when it is - * done stuffing characters into the driver. If there is no room - * in the queue, the character is ignored. + * This routine is called by the kernel to write a single character @ch to + * the @tty device. If the kernel uses this routine, it must call the + * @flush_chars() routine (if defined) when it is done stuffing characters + * into the driver. If there is no room in the queue, the character is + * ignored. * - * Optional: Kernel will use the write method if not provided. + * Optional: Kernel will use the @write method if not provided. Do not + * call this function directly, call tty_put_char(). * - * Note: Do not call this function directly, call tty_put_char + * @flush_chars: ``void ()(struct tty_struct *tty)`` * - * void (*flush_chars)(struct tty_struct *tty); + * This routine is called by the kernel after it has written a + * series of characters to the tty device using @put_char(). * - * This routine is called by the kernel after it has written a - * series of characters to the tty device using put_char(). + * Optional. Do not call this function directly, call + * tty_driver_flush_chars(). * - * Optional: + * @write_room: ``unsigned int ()(struct tty_struct *tty)`` * - * Note: Do not call this function directly, call tty_driver_flush_chars - * - * unsigned int (*write_room)(struct tty_struct *tty); - * - * This routine returns the numbers of characters the tty driver - * will accept for queuing to be written. This number is subject - * to change as output buffers get emptied, or if the output flow + * This routine returns the numbers of characters the @tty driver + * will accept for queuing to be written. This number is subject + * to change as output buffers get emptied, or if the output flow * control is acted. * - * Required if write method is provided else not needed. + * Required if @write method is provided else not needed. Do not call this + * function directly, call tty_write_room() * - * Note: Do not call this function directly, call tty_write_room - * - * int (*ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); + * @chars_in_buffer: ``unsigned int ()(struct tty_struct *tty)`` * - * This routine allows the tty driver to implement - * device-specific ioctls. If the ioctl number passed in cmd - * is not recognized by the driver, it should return ENOIOCTLCMD. + * This routine returns the number of characters in the device private + * output queue. Used in tty_wait_until_sent() and for poll() + * implementation. * - * Optional + * Optional: if not provided, it is assumed there is no queue on the + * device. Do not call this function directly, call tty_chars_in_buffer(). * - * long (*compat_ioctl)(struct tty_struct *tty,, - * unsigned int cmd, unsigned long arg); + * @ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd, + * unsigned long arg)`` * - * implement ioctl processing for 32 bit process on 64 bit system + * This routine allows the @tty driver to implement device-specific + * ioctls. If the ioctl number passed in @cmd is not recognized by the + * driver, it should return %ENOIOCTLCMD. * - * Optional - * - * void (*set_termios)(struct tty_struct *tty, struct ktermios * old); + * Optional. * - * This routine allows the tty driver to be notified when - * device's termios settings have changed. + * @compat_ioctl: ``long ()(struct tty_struct *tty, unsigned int cmd, + * unsigned long arg)`` * - * Optional: Called under the termios lock + * Implement ioctl processing for 32 bit process on 64 bit system. * + * Optional. * - * void (*set_ldisc)(struct tty_struct *tty); + * @set_termios: ``void ()(struct tty_struct *tty, struct ktermios *old)`` * - * This routine allows the tty driver to be notified when the - * device's termios settings have changed. + * This routine allows the @tty driver to be notified when device's + * termios settings have changed. * - * Optional: Called under BKL (currently) - * - * void (*throttle)(struct tty_struct * tty); + * Optional: Called under the @tty->termios_rwsem. * - * This routine notifies the tty driver that input buffers for - * the line discipline are close to full, and it should somehow - * signal that no more characters should be sent to the tty. + * @set_ldisc: ``void ()(struct tty_struct *tty)`` * - * Optional: Always invoke via tty_throttle_safe(), called under the - * termios lock. - * - * void (*unthrottle)(struct tty_struct * tty); + * This routine allows the @tty driver to be notified when the device's + * line discipline is being changed. * - * This routine notifies the tty drivers that it should signals - * that characters can now be sent to the tty without fear of - * overrunning the input buffers of the line disciplines. - * - * Optional: Always invoke via tty_unthrottle(), called under the - * termios lock. + * Optional. Called under the @tty->ldisc_sem and @tty->termios_rwsem. * - * void (*stop)(struct tty_struct *tty); + * @throttle: ``void ()(struct tty_struct *tty)`` * - * This routine notifies the tty driver that it should stop - * outputting characters to the tty device. + * This routine notifies the @tty driver that input buffers for the line + * discipline are close to full, and it should somehow signal that no more + * characters should be sent to the @tty. * - * Called with ->flow.lock held. Serialized with start() method. + * Optional: Always invoke via tty_throttle_safe(). Called under the + * @tty->termios_rwsem. * - * Optional: + * @unthrottle: ``void ()(struct tty_struct *tty)`` * - * Note: Call stop_tty not this method. - * - * void (*start)(struct tty_struct *tty); + * This routine notifies the @tty driver that it should signal that + * characters can now be sent to the @tty without fear of overrunning the + * input buffers of the line disciplines. * - * This routine notifies the tty driver that it resume sending + * Optional. Always invoke via tty_unthrottle(). Called under the + * @tty->termios_rwsem. + * + * @stop: ``void ()(struct tty_struct *tty)`` + * + * This routine notifies the @tty driver that it should stop outputting * characters to the tty device. * - * Called with ->flow.lock held. Serialized with stop() method. + * Called with @tty->flow.lock held. Serialized with @start() method. * - * Optional: + * Optional. Always invoke via stop_tty(). * - * Note: Call start_tty not this method. - * - * void (*hangup)(struct tty_struct *tty); + * @start: ``void ()(struct tty_struct *tty)`` * - * This routine notifies the tty driver that it should hang up the - * tty device. + * This routine notifies the @tty driver that it resumed sending + * characters to the @tty device. * - * Optional: + * Called with @tty->flow.lock held. Serialized with stop() method. * - * Called with tty lock held. + * Optional. Always invoke via start_tty(). * - * int (*break_ctl)(struct tty_struct *tty, int state); + * @hangup: ``void ()(struct tty_struct *tty)`` * - * This optional routine requests the tty driver to turn on or - * off BREAK status on the RS-232 port. If state is -1, - * then the BREAK status should be turned on; if state is 0, then - * BREAK should be turned off. + * This routine notifies the @tty driver that it should hang up the @tty + * device. * - * If this routine is implemented, the high-level tty driver will - * handle the following ioctls: TCSBRK, TCSBRKP, TIOCSBRK, - * TIOCCBRK. + * Optional. Called with tty lock held. * - * If the driver sets TTY_DRIVER_HARDWARE_BREAK then the interface - * will also be called with actual times and the hardware is expected - * to do the delay work itself. 0 and -1 are still used for on/off. + * @break_ctl: ``int ()(struct tty_struct *tty, int state)`` * - * Optional: Required for TCSBRK/BRKP/etc handling. + * This optional routine requests the @tty driver to turn on or off BREAK + * status on the RS-232 port. If @state is -1, then the BREAK status + * should be turned on; if @state is 0, then BREAK should be turned off. * - * void (*wait_until_sent)(struct tty_struct *tty, int timeout); - * - * This routine waits until the device has written out all of the - * characters in its transmitter FIFO. + * If this routine is implemented, the high-level tty driver will handle + * the following ioctls: %TCSBRK, %TCSBRKP, %TIOCSBRK, %TIOCCBRK. + * + * If the driver sets %TTY_DRIVER_HARDWARE_BREAK in tty_alloc_driver(), + * then the interface will also be called with actual times and the + * hardware is expected to do the delay work itself. 0 and -1 are still + * used for on/off. + * + * Optional: Required for %TCSBRK/%BRKP/etc. handling. + * + * @flush_buffer: ``void ()(struct tty_struct *tty)`` + * + * This routine discards device private output buffer. Invoked on close, + * hangup, to implement %TCOFLUSH ioctl and similar. + * + * Optional: if not provided, it is assumed there is no queue on the + * device. Do not call this function directly, call + * tty_driver_flush_buffer(). + * + * @wait_until_sent: ``void ()(struct tty_struct *tty, int timeout)`` + * + * This routine waits until the device has written out all of the + * characters in its transmitter FIFO. Or until @timeout (in jiffies) is + * reached. + * + * Optional: If not provided, the device is assumed to have no FIFO. + * Usually correct to invoke via tty_wait_until_sent(). + * + * @send_xchar: ``void ()(struct tty_struct *tty, char ch)`` + * + * This routine is used to send a high-priority XON/XOFF character (@ch) + * to the @tty device. * - * Optional: If not provided the device is assumed to have no FIFO + * Optional: If not provided, then the @write method is called under + * the @tty->atomic_write_lock to keep it serialized with the ldisc. * - * Note: Usually correct to call tty_wait_until_sent + * @tiocmget: ``int ()(struct tty_struct *tty)`` * - * void (*send_xchar)(struct tty_struct *tty, char ch); + * This routine is used to obtain the modem status bits from the @tty + * driver. * - * This routine is used to send a high-priority XON/XOFF - * character to the device. + * Optional: If not provided, then %ENOTTY is returned from the %TIOCMGET + * ioctl. Do not call this function directly, call tty_tiocmget(). * - * Optional: If not provided then the write method is called under - * the atomic write lock to keep it serialized with the ldisc. + * @tiocmset: ``int ()(struct tty_struct *tty, + * unsigned int set, unsigned int clear)`` * - * int (*resize)(struct tty_struct *tty, struct winsize *ws) + * This routine is used to set the modem status bits to the @tty driver. + * First, @clear bits should be cleared, then @set bits set. * - * Called when a termios request is issued which changes the - * requested terminal geometry. + * Optional: If not provided, then %ENOTTY is returned from the %TIOCMSET + * ioctl. Do not call this function directly, call tty_tiocmset(). + * + * @resize: ``int ()(struct tty_struct *tty, struct winsize *ws)`` + * + * Called when a termios request is issued which changes the requested + * terminal geometry to @ws. * * Optional: the default action is to update the termios structure * without error. This is usually the correct behaviour. Drivers should - * not force errors here if they are not resizable objects (eg a serial + * not force errors here if they are not resizable objects (e.g. a serial * line). See tty_do_resize() if you need to wrap the standard method - * in your own logic - the usual case. + * in your own logic -- the usual case. + * + * @get_icount: ``int ()(struct tty_struct *tty, + * struct serial_icounter *icount)`` + * + * Called when the @tty device receives a %TIOCGICOUNT ioctl. Passed a + * kernel structure @icount to complete. + * + * Optional: called only if provided, otherwise %ENOTTY will be returned. + * + * @get_serial: ``int ()(struct tty_struct *tty, struct serial_struct *p)`` + * + * Called when the @tty device receives a %TIOCGSERIAL ioctl. Passed a + * kernel structure @p (&struct serial_struct) to complete. + * + * Optional: called only if provided, otherwise %ENOTTY will be returned. + * Do not call this function directly, call tty_tiocgserial(). + * + * @set_serial: ``int ()(struct tty_struct *tty, struct serial_struct *p)`` + * + * Called when the @tty device receives a %TIOCSSERIAL ioctl. Passed a + * kernel structure @p (&struct serial_struct) to set the values from. + * + * Optional: called only if provided, otherwise %ENOTTY will be returned. + * Do not call this function directly, call tty_tiocsserial(). + * + * @show_fdinfo: ``void ()(struct tty_struct *tty, struct seq_file *m)`` + * + * Called when the @tty device file descriptor receives a fdinfo request + * from VFS (to show in /proc//fdinfo/). @m should be filled with + * information. + * + * Optional: called only if provided, otherwise nothing is written to @m. + * Do not call this function directly, call tty_show_fdinfo(). + * + * @poll_init: ``int ()(struct tty_driver *driver, int line, char *options)`` + * + * kgdboc support (Documentation/dev-tools/kgdb.rst). This routine is + * called to initialize the HW for later use by calling @poll_get_char or + * @poll_put_char. + * + * Optional: called only if provided, otherwise skipped as a non-polling + * driver. + * + * @poll_get_char: ``int ()(struct tty_driver *driver, int line)`` + * + * kgdboc support (see @poll_init). @driver should read a character from a + * tty identified by @line and return it. + * + * Optional: called only if @poll_init provided. * - * int (*get_icount)(struct tty_struct *tty, struct serial_icounter *icount); + * @poll_put_char: ``void ()(struct tty_driver *driver, int line, char ch)`` * - * Called when the device receives a TIOCGICOUNT ioctl. Passed a kernel - * structure to complete. This method is optional and will only be called - * if provided (otherwise ENOTTY will be returned). + * kgdboc support (see @poll_init). @driver should write character @ch to + * a tty identified by @line. + * + * Optional: called only if @poll_init provided. + * + * @proc_show: ``int ()(struct seq_file *m, void *driver)`` + * + * Driver @driver (cast to &struct tty_driver) can show additional info in + * /proc/tty/driver/. It is enough to fill in the information + * into @m. + * + * Optional: called only if provided, otherwise no /proc entry created. + * + * This structure defines the interface between the low-level tty driver and + * the tty routines. These routines can be defined. Unless noted otherwise, + * they are optional, and can be filled in with a %NULL pointer. */ - -#include -#include -#include -#include -#include -#include -#include - -struct tty_struct; -struct tty_driver; -struct serial_icounter_struct; -struct serial_struct; - struct tty_operations { struct tty_struct * (*lookup)(struct tty_driver *driver, struct file *filp, int idx); @@ -288,7 +372,7 @@ struct tty_operations { int (*poll_get_char)(struct tty_driver *driver, int line); void (*poll_put_char)(struct tty_driver *driver, int line, char ch); #endif - int (*proc_show)(struct seq_file *, void *); + int (*proc_show)(struct seq_file *m, void *driver); } __randomize_layout; /** -- cgit v1.2.3-59-g8ed1b From 630bf86d15778fd3e5df17cb6e00839d0f44a707 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:53 +0100 Subject: tty: add kernel-doc for tty_port_operations tty_port_operations used to have only comments along its members. Convert them into proper kernel-doc comments in front of the structure. And add some more explanation to them where needed. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-6-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_port.h | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 9091e1c8de4c..d3ea9ed0b98e 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -12,21 +12,28 @@ struct tty_driver; struct tty_port; struct tty_struct; +/** + * struct tty_port_operations -- operations on tty_port + * @carrier_raised: return 1 if the carrier is raised on @port + * @dtr_rts: raise the DTR line if @raise is nonzero, otherwise lower DTR + * @shutdown: called when the last close completes or a hangup finishes IFF the + * port was initialized. Do not use to free resources. Turn off the device + * only. Called under the port mutex to serialize against @activate and + * @shutdown. + * @activate: called under the port mutex from tty_port_open(), serialized using + * the port mutex. Supposed to turn on the device. + * + * FIXME: long term getting the tty argument *out* of this would be good + * for consoles. + * + * @destruct: called on the final put of a port. Free resources, possibly incl. + * the port itself. + */ struct tty_port_operations { - /* Return 1 if the carrier is raised */ int (*carrier_raised)(struct tty_port *port); - /* Control the DTR line */ void (*dtr_rts)(struct tty_port *port, int raise); - /* Called when the last close completes or a hangup finishes - IFF the port was initialized. Do not use to free resources. Called - under the port mutex to serialize against activate/shutdowns */ void (*shutdown)(struct tty_port *port); - /* Called under the port mutex from tty_port_open, serialized using - the port mutex */ - /* FIXME: long term getting the tty argument *out* of this would be - good for consoles */ int (*activate)(struct tty_port *port, struct tty_struct *tty); - /* Called on the final put of a port */ void (*destruct)(struct tty_port *port); }; -- cgit v1.2.3-59-g8ed1b From 0c6119f9f7dc03a53bd35ca5a77926eef3c33d10 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:54 +0100 Subject: tty: add kernel-doc for tty_ldisc_ops tty_ldisc_ops structure was already partially documented in a standalone comment in the header beginning. Move it right before the structure and reformat it so it complies to kernel-doc. That way, we can include it in Documentation/ later in this series. And add the documentation for the members where missing too. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-7-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_ldisc.h | 259 +++++++++++++++++++++++----------------------- 1 file changed, 130 insertions(+), 129 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index 25f07017bbad..e0da0ba02de9 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -4,126 +4,6 @@ struct tty_struct; -/* - * This structure defines the interface between the tty line discipline - * implementation and the tty routines. The following routines can be - * defined; unless noted otherwise, they are optional, and can be - * filled in with a null pointer. - * - * int (*open)(struct tty_struct *); - * - * This function is called when the line discipline is associated - * with the tty. The line discipline can use this as an - * opportunity to initialize any state needed by the ldisc routines. - * - * void (*close)(struct tty_struct *); - * - * This function is called when the line discipline is being - * shutdown, either because the tty is being closed or because - * the tty is being changed to use a new line discipline - * - * void (*flush_buffer)(struct tty_struct *tty); - * - * This function instructs the line discipline to clear its - * buffers of any input characters it may have queued to be - * delivered to the user mode process. - * - * ssize_t (*read)(struct tty_struct * tty, struct file * file, - * unsigned char * buf, size_t nr); - * - * This function is called when the user requests to read from - * the tty. The line discipline will return whatever characters - * it has buffered up for the user. If this function is not - * defined, the user will receive an EIO error. - * - * ssize_t (*write)(struct tty_struct * tty, struct file * file, - * const unsigned char * buf, size_t nr); - * - * This function is called when the user requests to write to the - * tty. The line discipline will deliver the characters to the - * low-level tty device for transmission, optionally performing - * some processing on the characters first. If this function is - * not defined, the user will receive an EIO error. - * - * int (*ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); - * - * This function is called when the user requests an ioctl which - * is not handled by the tty layer or the low-level tty driver. - * It is intended for ioctls which affect line discpline - * operation. Note that the search order for ioctls is (1) tty - * layer, (2) tty low-level driver, (3) line discpline. So a - * low-level driver can "grab" an ioctl request before the line - * discpline has a chance to see it. - * - * int (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, - * unsigned long arg); - * - * Process ioctl calls from 32-bit process on 64-bit system - * - * NOTE: only ioctls that are neither "pointer to compatible - * structure" nor tty-generic. Something private that takes - * an integer or a pointer to wordsize-sensitive structure - * belongs here, but most of ldiscs will happily leave - * it NULL. - * - * void (*set_termios)(struct tty_struct *tty, struct ktermios * old); - * - * This function notifies the line discpline that a change has - * been made to the termios structure. - * - * int (*poll)(struct tty_struct * tty, struct file * file, - * poll_table *wait); - * - * This function is called when a user attempts to select/poll on a - * tty device. It is solely the responsibility of the line - * discipline to handle poll requests. - * - * void (*receive_buf)(struct tty_struct *, const unsigned char *cp, - * char *fp, int count); - * - * This function is called by the low-level tty driver to send - * characters received by the hardware to the line discpline for - * processing. is a pointer to the buffer of input - * character received by the device. is a pointer to a - * pointer of flag bytes which indicate whether a character was - * received with a parity error, etc. may be NULL to indicate - * all data received is TTY_NORMAL. - * - * void (*write_wakeup)(struct tty_struct *); - * - * This function is called by the low-level tty driver to signal - * that line discpline should try to send more characters to the - * low-level driver for transmission. If the line discpline does - * not have any more data to send, it can just return. If the line - * discipline does have some data to send, please arise a tasklet - * or workqueue to do the real data transfer. Do not send data in - * this hook, it may leads to a deadlock. - * - * int (*hangup)(struct tty_struct *) - * - * Called on a hangup. Tells the discipline that it should - * cease I/O to the tty driver. Can sleep. The driver should - * seek to perform this action quickly but should wait until - * any pending driver I/O is completed. - * - * void (*dcd_change)(struct tty_struct *tty, unsigned int status) - * - * Tells the discipline that the DCD pin has changed its status. - * Used exclusively by the N_PPS (Pulse-Per-Second) line discipline. - * - * int (*receive_buf2)(struct tty_struct *, const unsigned char *cp, - * char *fp, int count); - * - * This function is called by the low-level tty driver to send - * characters received by the hardware to the line discpline for - * processing. is a pointer to the buffer of input - * character received by the device. is a pointer to a - * pointer of flag bytes which indicate whether a character was - * received with a parity error, etc. may be NULL to indicate - * all data received is TTY_NORMAL. - * If assigned, prefer this function for automatic flow control. - */ - #include #include #include @@ -175,7 +55,128 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, ldsem_down_write(sem, timeout) #endif - +/** + * struct tty_ldisc_ops - ldisc operations + * + * @name: name of this ldisc rendered in /proc/tty/ldiscs + * @num: ``N_*`` number (%N_TTY, %N_HDLC, ...) reserved to this ldisc + * + * @open: ``int ()(struct tty_struct *tty)`` + * + * This function is called when the line discipline is associated with the + * @tty. The line discipline can use this as an opportunity to initialize + * any state needed by the ldisc routines. + * + * @close: ``void ()(struct tty_struct *tty)`` + * + * This function is called when the line discipline is being shutdown, + * either because the @tty is being closed or because the @tty is being + * changed to use a new line discipline + * + * @flush_buffer: ``void ()(struct tty_struct *tty)`` + * + * This function instructs the line discipline to clear its buffers of any + * input characters it may have queued to be delivered to the user mode + * process. + * + * @read: ``ssize_t ()(struct tty_struct *tty, struct file *file, + * unsigned char *buf, size_t nr)`` + * + * This function is called when the user requests to read from the @tty. + * The line discipline will return whatever characters it has buffered up + * for the user. If this function is not defined, the user will receive + * an %EIO error. + * + * @write: ``ssize_t ()(struct tty_struct *tty, struct file *file, + * const unsigned char *buf, size_t nr)`` + * + * This function is called when the user requests to write to the @tty. + * The line discipline will deliver the characters to the low-level tty + * device for transmission, optionally performing some processing on the + * characters first. If this function is not defined, the user will + * receive an %EIO error. + * + * @ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd, + * unsigned long arg)`` + * + * This function is called when the user requests an ioctl which is not + * handled by the tty layer or the low-level tty driver. It is intended + * for ioctls which affect line discpline operation. Note that the search + * order for ioctls is (1) tty layer, (2) tty low-level driver, (3) line + * discpline. So a low-level driver can "grab" an ioctl request before + * the line discpline has a chance to see it. + * + * @compat_ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd, + * unsigned long arg)`` + * + * Process ioctl calls from 32-bit process on 64-bit system. + * + * Note that only ioctls that are neither "pointer to compatible + * structure" nor tty-generic. Something private that takes an integer or + * a pointer to wordsize-sensitive structure belongs here, but most of + * ldiscs will happily leave it %NULL. + * + * @set_termios: ``void ()(struct tty_struct *tty, struct ktermios *old)`` + * + * This function notifies the line discpline that a change has been made + * to the termios structure. + * + * @poll: ``int ()(struct tty_struct *tty, struct file *file, + * struct poll_table_struct *wait)`` + * + * This function is called when a user attempts to select/poll on a @tty + * device. It is solely the responsibility of the line discipline to + * handle poll requests. + * + * @hangup: ``void ()(struct tty_struct *tty)`` + * + * Called on a hangup. Tells the discipline that it should cease I/O to + * the tty driver. Can sleep. The driver should seek to perform this + * action quickly but should wait until any pending driver I/O is + * completed. + * + * @receive_buf: ``void ()(struct tty_struct *tty, const unsigned char *cp, + * const char *fp, int count)`` + * + * This function is called by the low-level tty driver to send characters + * received by the hardware to the line discpline for processing. @cp is + * a pointer to the buffer of input character received by the device. @fp + * is a pointer to an array of flag bytes which indicate whether a + * character was received with a parity error, etc. @fp may be %NULL to + * indicate all data received is %TTY_NORMAL. + * + * @write_wakeup: ``void ()(struct tty_struct *tty)`` + * + * This function is called by the low-level tty driver to signal that line + * discpline should try to send more characters to the low-level driver + * for transmission. If the line discpline does not have any more data to + * send, it can just return. If the line discipline does have some data to + * send, please arise a tasklet or workqueue to do the real data transfer. + * Do not send data in this hook, it may lead to a deadlock. + * + * @dcd_change: ``void ()(struct tty_struct *tty, unsigned int status)`` + * + * Tells the discipline that the DCD pin has changed its status. Used + * exclusively by the %N_PPS (Pulse-Per-Second) line discipline. + * + * @receive_buf2: ``int ()(struct tty_struct *tty, const unsigned char *cp, + * const char *fp, int count)`` + * + * This function is called by the low-level tty driver to send characters + * received by the hardware to the line discpline for processing. @cp is a + * pointer to the buffer of input character received by the device. @fp + * is a pointer to an array of flag bytes which indicate whether a + * character was received with a parity error, etc. @fp may be %NULL to + * indicate all data received is %TTY_NORMAL. If assigned, prefer this + * function for automatic flow control. + * + * @owner: module containting this ldisc (for reference counting) + * + * This structure defines the interface between the tty line discipline + * implementation and the tty routines. The above routines can be defined. + * Unless noted otherwise, they are optional, and can be filled in with a %NULL + * pointer. + */ struct tty_ldisc_ops { char *name; int num; @@ -183,8 +184,8 @@ struct tty_ldisc_ops { /* * The following routines are called from above. */ - int (*open)(struct tty_struct *); - void (*close)(struct tty_struct *); + int (*open)(struct tty_struct *tty); + void (*close)(struct tty_struct *tty); void (*flush_buffer)(struct tty_struct *tty); ssize_t (*read)(struct tty_struct *tty, struct file *file, unsigned char *buf, size_t nr, @@ -196,18 +197,18 @@ struct tty_ldisc_ops { int (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); void (*set_termios)(struct tty_struct *tty, struct ktermios *old); - __poll_t (*poll)(struct tty_struct *, struct file *, - struct poll_table_struct *); + __poll_t (*poll)(struct tty_struct *tty, struct file *file, + struct poll_table_struct *wait); void (*hangup)(struct tty_struct *tty); /* * The following routines are called from below. */ - void (*receive_buf)(struct tty_struct *, const unsigned char *cp, + void (*receive_buf)(struct tty_struct *tty, const unsigned char *cp, const char *fp, int count); - void (*write_wakeup)(struct tty_struct *); - void (*dcd_change)(struct tty_struct *, unsigned int); - int (*receive_buf2)(struct tty_struct *, const unsigned char *cp, + void (*write_wakeup)(struct tty_struct *tty); + void (*dcd_change)(struct tty_struct *tty, unsigned int status); + int (*receive_buf2)(struct tty_struct *tty, const unsigned char *cp, const char *fp, int count); struct module *owner; -- cgit v1.2.3-59-g8ed1b From 29d5ef685948369602ccd5c04d2a215449c4b943 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:55 +0100 Subject: tty: combine tty_operations triple docs into kernel-doc In Documentation/driver-api/serial/tty.rst, there are triplicated texts about some struct tty_operations' hooks. Combine them into existing kernel-doc comments of struct tty_operations and drop them from the Documentation/. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-8-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/serial/tty.rst | 134 +------------------------------- include/linux/tty_driver.h | 36 ++++++--- 2 files changed, 29 insertions(+), 141 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/serial/tty.rst b/Documentation/driver-api/serial/tty.rst index 4b709f392713..f7ef10c6f458 100644 --- a/Documentation/driver-api/serial/tty.rst +++ b/Documentation/driver-api/serial/tty.rst @@ -132,74 +132,8 @@ dcd_change() Report to the tty line the current DCD pin status Driver Access ^^^^^^^^^^^^^ -Line discipline methods can call the following methods of the underlying -hardware driver through the function pointers within the tty->driver -structure: - -======================= ======================================================= -write() Write a block of characters to the tty device. - Returns the number of characters accepted. The - character buffer passed to this method is already - in kernel space. - -put_char() Queues a character for writing to the tty device. - If there is no room in the queue, the character is - ignored. - -flush_chars() (Optional) If defined, must be called after - queueing characters with put_char() in order to - start transmission. - -write_room() Returns the numbers of characters the tty driver - will accept for queueing to be written. - -ioctl() Invoke device specific ioctl. - Expects data pointers to refer to userspace. - Returns ENOIOCTLCMD for unrecognized ioctl numbers. - -set_termios() Notify the tty driver that the device's termios - settings have changed. New settings are in - tty->termios. Previous settings should be passed in - the "old" argument. - - The API is defined such that the driver should return - the actual modes selected. This means that the - driver function is responsible for modifying any - bits in the request it cannot fulfill to indicate - the actual modes being used. A device with no - hardware capability for change (e.g. a USB dongle or - virtual port) can provide NULL for this method. - -throttle() Notify the tty driver that input buffers for the - line discipline are close to full, and it should - somehow signal that no more characters should be - sent to the tty. - -unthrottle() Notify the tty driver that characters can now be - sent to the tty without fear of overrunning the - input buffers of the line disciplines. - -stop() Ask the tty driver to stop outputting characters - to the tty device. - -start() Ask the tty driver to resume sending characters - to the tty device. - -hangup() Ask the tty driver to hang up the tty device. - -break_ctl() (Optional) Ask the tty driver to turn on or off - BREAK status on the RS-232 port. If state is -1, - then the BREAK status should be turned on; if - state is 0, then BREAK should be turned off. - If this routine is not implemented, use ioctls - TIOCSBRK / TIOCCBRK instead. - -wait_until_sent() Waits until the device has written out all of the - characters in its transmitter FIFO. - -send_xchar() Send a high-priority XON/XOFF character to the device. -======================= ======================================================= - +Line discipline methods can call the methods of the underlying hardware driver. +These are documented as a part of struct tty_operations. Flags ^^^^^ @@ -262,67 +196,3 @@ A caution: The ldisc->open(), ldisc->close() and driver->set_ldisc functions are called with the ldisc unavailable. Thus tty_ldisc_ref will fail in this situation if used within these functions. Ldisc and driver code calling its own functions must be careful in this case. - - -Driver Interface ----------------- - -======================= ======================================================= -open() Called when a device is opened. May sleep - -close() Called when a device is closed. At the point of - return from this call the driver must make no - further ldisc calls of any kind. May sleep - -write() Called to write bytes to the device. May not - sleep. May occur in parallel in special cases. - Because this includes panic paths drivers generally - shouldn't try and do clever locking here. - -put_char() Stuff a single character onto the queue. The - driver is guaranteed following up calls to - flush_chars. - -flush_chars() Ask the kernel to write put_char queue - -write_room() Return the number of characters that can be stuffed - into the port buffers without overflow (or less). - The ldisc is responsible for being intelligent - about multi-threading of write_room/write calls - -ioctl() Called when an ioctl may be for the driver - -set_termios() Called on termios change, serialized against - itself by a semaphore. May sleep. - -set_ldisc() Notifier for discipline change. At the point this - is done the discipline is not yet usable. Can now - sleep (I think) - -throttle() Called by the ldisc to ask the driver to do flow - control. Serialization including with unthrottle - is the job of the ldisc layer. - -unthrottle() Called by the ldisc to ask the driver to stop flow - control. - -stop() Ldisc notifier to the driver to stop output. As with - throttle the serializations with start() are down - to the ldisc layer. - -start() Ldisc notifier to the driver to start output. - -hangup() Ask the tty driver to cause a hangup initiated - from the host side. [Can sleep ??] - -break_ctl() Send RS232 break. Can sleep. Can get called in - parallel, driver must serialize (for now), and - with write calls. - -wait_until_sent() Wait for characters to exit the hardware queue - of the driver. Can sleep - -send_xchar() Send XON/XOFF and if possible jump the queue with - it in order to get fast flow control responses. - Cannot sleep ?? -======================= ======================================================= diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 5611992ab26a..41274d551e28 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -47,15 +47,17 @@ struct serial_struct; * routine is mandatory; if this routine is not filled in, the attempted * open will fail with %ENODEV. * - * Required method. Called with tty lock held. + * Required method. Called with tty lock held. May sleep. * * @close: ``void ()(struct tty_struct *tty, struct file *)`` * - * This routine is called when a particular @tty device is closed. + * This routine is called when a particular @tty device is closed. At the + * point of return from this call the driver must make no further ldisc + * calls of any kind. * * Remark: called even if the corresponding @open() failed. * - * Required method. Called with tty lock held. + * Required method. Called with tty lock held. May sleep. * * @shutdown: ``void ()(struct tty_struct *tty)`` * @@ -77,7 +79,10 @@ struct serial_struct; * user space or kernel space. This routine will return the * number of characters actually accepted for writing. * - * Optional: Required for writable devices. + * May occur in parallel in special cases. Because this includes panic + * paths drivers generally shouldn't try and do clever locking here. + * + * Optional: Required for writable devices. May not sleep. * * @put_char: ``int ()(struct tty_struct *tty, unsigned char ch)`` * @@ -105,6 +110,9 @@ struct serial_struct; * to change as output buffers get emptied, or if the output flow * control is acted. * + * The ldisc is responsible for being intelligent about multi-threading of + * write_room/write calls + * * Required if @write method is provided else not needed. Do not call this * function directly, call tty_write_room() * @@ -136,14 +144,21 @@ struct serial_struct; * @set_termios: ``void ()(struct tty_struct *tty, struct ktermios *old)`` * * This routine allows the @tty driver to be notified when device's - * termios settings have changed. + * termios settings have changed. New settings are in @tty->termios. + * Previous settings are passed in the @old argument. * - * Optional: Called under the @tty->termios_rwsem. + * The API is defined such that the driver should return the actual modes + * selected. This means that the driver is responsible for modifying any + * bits in @tty->termios it cannot fulfill to indicate the actual modes + * being used. + * + * Optional. Called under the @tty->termios_rwsem. May sleep. * * @set_ldisc: ``void ()(struct tty_struct *tty)`` * * This routine allows the @tty driver to be notified when the device's - * line discipline is being changed. + * line discipline is being changed. At the point this is done the + * discipline is not yet usable. * * Optional. Called under the @tty->ldisc_sem and @tty->termios_rwsem. * @@ -153,6 +168,9 @@ struct serial_struct; * discipline are close to full, and it should somehow signal that no more * characters should be sent to the @tty. * + * Serialization including with @unthrottle() is the job of the ldisc + * layer. + * * Optional: Always invoke via tty_throttle_safe(). Called under the * @tty->termios_rwsem. * @@ -204,7 +222,7 @@ struct serial_struct; * hardware is expected to do the delay work itself. 0 and -1 are still * used for on/off. * - * Optional: Required for %TCSBRK/%BRKP/etc. handling. + * Optional: Required for %TCSBRK/%BRKP/etc. handling. May sleep. * * @flush_buffer: ``void ()(struct tty_struct *tty)`` * @@ -222,7 +240,7 @@ struct serial_struct; * reached. * * Optional: If not provided, the device is assumed to have no FIFO. - * Usually correct to invoke via tty_wait_until_sent(). + * Usually correct to invoke via tty_wait_until_sent(). May sleep. * * @send_xchar: ``void ()(struct tty_struct *tty, char ch)`` * -- cgit v1.2.3-59-g8ed1b From 40f4268cddb93d17a11579920d940c2dca8b9445 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:56 +0100 Subject: tty: combine tty_ldisc_ops docs into kernel-doc In Documentation/driver-api/serial/tty.rst, there are duplicated texts about some struct tty_ldisc_ops' hooks. Combine them into existing kernel-doc comments of struct tty_ldisc_ops and drop them from the Documentation/. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-9-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/serial/tty.rst | 95 +-------------------------------- include/linux/tty_ldisc.h | 67 ++++++++++++++--------- 2 files changed, 45 insertions(+), 117 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/serial/tty.rst b/Documentation/driver-api/serial/tty.rst index f7ef10c6f458..333a5923f6e1 100644 --- a/Documentation/driver-api/serial/tty.rst +++ b/Documentation/driver-api/serial/tty.rst @@ -35,99 +35,8 @@ about to enter and exit although this detail matters not). Line Discipline Methods ----------------------- -TTY side interfaces -^^^^^^^^^^^^^^^^^^^ - -======================= ======================================================= -open() Called when the line discipline is attached to - the terminal. No other call into the line - discipline for this tty will occur until it - completes successfully. Should initialize any - state needed by the ldisc, and set receive_room - in the tty_struct to the maximum amount of data - the line discipline is willing to accept from the - driver with a single call to receive_buf(). - Returning an error will prevent the ldisc from - being attached. Can sleep. - -close() This is called on a terminal when the line - discipline is being unplugged. At the point of - execution no further users will enter the - ldisc code for this tty. Can sleep. - -hangup() Called when the tty line is hung up. - The line discipline should cease I/O to the tty. - No further calls into the ldisc code will occur. - Can sleep. - -read() (optional) A process requests reading data from - the line. Multiple read calls may occur in parallel - and the ldisc must deal with serialization issues. - If not defined, the process will receive an EIO - error. May sleep. - -write() (optional) A process requests writing data to the - line. Multiple write calls are serialized by the - tty layer for the ldisc. If not defined, the - process will receive an EIO error. May sleep. - -flush_buffer() (optional) May be called at any point between - open and close, and instructs the line discipline - to empty its input buffer. - -set_termios() (optional) Called on termios structure changes. - The caller passes the old termios data and the - current data is in the tty. Called under the - termios semaphore so allowed to sleep. Serialized - against itself only. - -poll() (optional) Check the status for the poll/select - calls. Multiple poll calls may occur in parallel. - May sleep. - -ioctl() (optional) Called when an ioctl is handed to the - tty layer that might be for the ldisc. Multiple - ioctl calls may occur in parallel. May sleep. - -compat_ioctl() (optional) Called when a 32 bit ioctl is handed - to the tty layer that might be for the ldisc. - Multiple ioctl calls may occur in parallel. - May sleep. -======================= ======================================================= - -Driver Side Interfaces -^^^^^^^^^^^^^^^^^^^^^^ - -======================= ======================================================= -receive_buf() (optional) Called by the low-level driver to hand - a buffer of received bytes to the ldisc for - processing. The number of bytes is guaranteed not - to exceed the current value of tty->receive_room. - All bytes must be processed. - -receive_buf2() (optional) Called by the low-level driver to hand - a buffer of received bytes to the ldisc for - processing. Returns the number of bytes processed. - - If both receive_buf() and receive_buf2() are - defined, receive_buf2() should be preferred. - -write_wakeup() May be called at any point between open and close. - The TTY_DO_WRITE_WAKEUP flag indicates if a call - is needed but always races versus calls. Thus the - ldisc must be careful about setting order and to - handle unexpected calls. Must not sleep. - - The driver is forbidden from calling this directly - from the ->write call from the ldisc as the ldisc - is permitted to call the driver write method from - this function. In such a situation defer it. - -dcd_change() Report to the tty line the current DCD pin status - changes and the relative timestamp. The timestamp - cannot be NULL. -======================= ======================================================= - +.. kernel-doc:: include/linux/tty_ldisc.h + :identifiers: tty_ldisc_ops Driver Access ^^^^^^^^^^^^^ diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index e0da0ba02de9..e85002b56752 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -61,33 +61,45 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * @name: name of this ldisc rendered in /proc/tty/ldiscs * @num: ``N_*`` number (%N_TTY, %N_HDLC, ...) reserved to this ldisc * - * @open: ``int ()(struct tty_struct *tty)`` + * @open: [TTY] ``int ()(struct tty_struct *tty)`` * * This function is called when the line discipline is associated with the - * @tty. The line discipline can use this as an opportunity to initialize - * any state needed by the ldisc routines. + * @tty. No other call into the line discipline for this tty will occur + * until it completes successfully. It should initialize any state needed + * by the ldisc, and set @tty->receive_room to the maximum amount of data + * the line discipline is willing to accept from the driver with a single + * call to @receive_buf(). Returning an error will prevent the ldisc from + * being attached. * - * @close: ``void ()(struct tty_struct *tty)`` + * Can sleep. + * + * @close: [TTY] ``void ()(struct tty_struct *tty)`` * * This function is called when the line discipline is being shutdown, * either because the @tty is being closed or because the @tty is being - * changed to use a new line discipline + * changed to use a new line discipline. At the point of execution no + * further users will enter the ldisc code for this tty. + * + * Can sleep. * - * @flush_buffer: ``void ()(struct tty_struct *tty)`` + * @flush_buffer: [TTY] ``void ()(struct tty_struct *tty)`` * * This function instructs the line discipline to clear its buffers of any * input characters it may have queued to be delivered to the user mode - * process. + * process. It may be called at any point between open and close. * - * @read: ``ssize_t ()(struct tty_struct *tty, struct file *file, + * @read: [TTY] ``ssize_t ()(struct tty_struct *tty, struct file *file, * unsigned char *buf, size_t nr)`` * * This function is called when the user requests to read from the @tty. * The line discipline will return whatever characters it has buffered up * for the user. If this function is not defined, the user will receive - * an %EIO error. + * an %EIO error. Multiple read calls may occur in parallel and the ldisc + * must deal with serialization issues. + * + * Can sleep. * - * @write: ``ssize_t ()(struct tty_struct *tty, struct file *file, + * @write: [TTY] ``ssize_t ()(struct tty_struct *tty, struct file *file, * const unsigned char *buf, size_t nr)`` * * This function is called when the user requests to write to the @tty. @@ -96,7 +108,9 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * characters first. If this function is not defined, the user will * receive an %EIO error. * - * @ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd, + * Can sleep. + * + * @ioctl: [TTY] ``int ()(struct tty_struct *tty, unsigned int cmd, * unsigned long arg)`` * * This function is called when the user requests an ioctl which is not @@ -106,7 +120,7 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * discpline. So a low-level driver can "grab" an ioctl request before * the line discpline has a chance to see it. * - * @compat_ioctl: ``int ()(struct tty_struct *tty, unsigned int cmd, + * @compat_ioctl: [TTY] ``int ()(struct tty_struct *tty, unsigned int cmd, * unsigned long arg)`` * * Process ioctl calls from 32-bit process on 64-bit system. @@ -116,27 +130,29 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * a pointer to wordsize-sensitive structure belongs here, but most of * ldiscs will happily leave it %NULL. * - * @set_termios: ``void ()(struct tty_struct *tty, struct ktermios *old)`` + * @set_termios: [TTY] ``void ()(struct tty_struct *tty, struct ktermios *old)`` * * This function notifies the line discpline that a change has been made * to the termios structure. * - * @poll: ``int ()(struct tty_struct *tty, struct file *file, + * @poll: [TTY] ``int ()(struct tty_struct *tty, struct file *file, * struct poll_table_struct *wait)`` * * This function is called when a user attempts to select/poll on a @tty * device. It is solely the responsibility of the line discipline to * handle poll requests. * - * @hangup: ``void ()(struct tty_struct *tty)`` + * @hangup: [TTY] ``void ()(struct tty_struct *tty)`` * * Called on a hangup. Tells the discipline that it should cease I/O to - * the tty driver. Can sleep. The driver should seek to perform this - * action quickly but should wait until any pending driver I/O is - * completed. + * the tty driver. The driver should seek to perform this action quickly + * but should wait until any pending driver I/O is completed. No further + * calls into the ldisc code will occur. + * + * Can sleep. * - * @receive_buf: ``void ()(struct tty_struct *tty, const unsigned char *cp, - * const char *fp, int count)`` + * @receive_buf: [DRV] ``void ()(struct tty_struct *tty, + * const unsigned char *cp, const char *fp, int count)`` * * This function is called by the low-level tty driver to send characters * received by the hardware to the line discpline for processing. @cp is @@ -145,7 +161,7 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * character was received with a parity error, etc. @fp may be %NULL to * indicate all data received is %TTY_NORMAL. * - * @write_wakeup: ``void ()(struct tty_struct *tty)`` + * @write_wakeup: [DRV] ``void ()(struct tty_struct *tty)`` * * This function is called by the low-level tty driver to signal that line * discpline should try to send more characters to the low-level driver @@ -154,13 +170,13 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * send, please arise a tasklet or workqueue to do the real data transfer. * Do not send data in this hook, it may lead to a deadlock. * - * @dcd_change: ``void ()(struct tty_struct *tty, unsigned int status)`` + * @dcd_change: [DRV] ``void ()(struct tty_struct *tty, unsigned int status)`` * * Tells the discipline that the DCD pin has changed its status. Used * exclusively by the %N_PPS (Pulse-Per-Second) line discipline. * - * @receive_buf2: ``int ()(struct tty_struct *tty, const unsigned char *cp, - * const char *fp, int count)`` + * @receive_buf2: [DRV] ``int ()(struct tty_struct *tty, + * const unsigned char *cp, const char *fp, int count)`` * * This function is called by the low-level tty driver to send characters * received by the hardware to the line discpline for processing. @cp is a @@ -176,6 +192,9 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * implementation and the tty routines. The above routines can be defined. * Unless noted otherwise, they are optional, and can be filled in with a %NULL * pointer. + * + * Hooks marked [TTY] are invoked from the TTY core, the [DRV] ones from the + * tty_driver side. */ struct tty_ldisc_ops { char *name; -- cgit v1.2.3-59-g8ed1b From 4072254f96f954ec0d34899f15d987803b6d76a2 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:57 +0100 Subject: tty: reformat tty_struct::flags into kernel-doc Move the partial tty_struct::flags documentation from tty_ldisc to the tty.h header and combine it with the one-liners present there. Convert all those to kernel-doc. This way, we can simply reference the documentation in Documentation while the text is still along the definitions. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-10-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/serial/tty.rst | 24 +---------- include/linux/tty.h | 74 ++++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/serial/tty.rst b/Documentation/driver-api/serial/tty.rst index 333a5923f6e1..65f971e3aada 100644 --- a/Documentation/driver-api/serial/tty.rst +++ b/Documentation/driver-api/serial/tty.rst @@ -47,28 +47,8 @@ These are documented as a part of struct tty_operations. Flags ^^^^^ -Line discipline methods have access to tty->flags field containing the -following interesting flags: - -======================= ======================================================= -TTY_THROTTLED Driver input is throttled. The ldisc should call - tty->driver->unthrottle() in order to resume - reception when it is ready to process more data. - -TTY_DO_WRITE_WAKEUP If set, causes the driver to call the ldisc's - write_wakeup() method in order to resume - transmission when it can accept more data - to transmit. - -TTY_IO_ERROR If set, causes all subsequent userspace read/write - calls on the tty to fail, returning -EIO. - -TTY_OTHER_CLOSED Device is a pty and the other side has closed. - -TTY_NO_WRITE_SPLIT Prevent driver from splitting up writes into - smaller chunks. -======================= ======================================================= - +Line discipline methods have access to :c:member:`tty_struct.flags` field. See +:doc:`tty_struct`. Locking ^^^^^^^ diff --git a/include/linux/tty.h b/include/linux/tty.h index da49ad9be281..7b0a5d478ef6 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -263,26 +263,72 @@ struct tty_file_private { /* tty magic number */ #define TTY_MAGIC 0x5401 -/* - * These bits are used in the flags field of the tty structure. +/** + * DOC: TTY Struct Flags + * + * These bits are used in the :c:member:`tty_struct.flags` field. * * So that interrupts won't be able to mess up the queues, * copy_to_cooked must be atomic with respect to itself, as must * tty->write. Thus, you must use the inline functions set_bit() and * clear_bit() to make things atomic. + * + * TTY_THROTTLED + * Driver input is throttled. The ldisc should call + * :c:member:`tty_driver.unthrottle()` in order to resume reception when + * it is ready to process more data (at threshold min). + * + * TTY_IO_ERROR + * If set, causes all subsequent userspace read/write calls on the tty to + * fail, returning -%EIO. (May be no ldisc too.) + * + * TTY_OTHER_CLOSED + * Device is a pty and the other side has closed. + * + * TTY_EXCLUSIVE + * Exclusive open mode (a single opener). + * + * TTY_DO_WRITE_WAKEUP + * If set, causes the driver to call the + * :c:member:`tty_ldisc_ops.write_wakeup()` method in order to resume + * transmission when it can accept more data to transmit. + * + * TTY_LDISC_OPEN + * Indicates that a line discipline is open. For debugging purposes only. + * + * TTY_PTY_LOCK + * A flag private to pty code to implement %TIOCSPTLCK/%TIOCGPTLCK logic. + * + * TTY_NO_WRITE_SPLIT + * Prevent driver from splitting up writes into smaller chunks (preserve + * write boundaries to driver). + * + * TTY_HUPPED + * The TTY was hung up. This is set post :c:member:`tty_driver.hangup()`. + * + * TTY_HUPPING + * The TTY is in the process of hanging up to abort potential readers. + * + * TTY_LDISC_CHANGING + * Line discipline for this TTY is being changed. I/O should not block + * when this is set. Use tty_io_nonblock() to check. + * + * TTY_LDISC_HALTED + * Line discipline for this TTY was stopped. No work should be queued to + * this ldisc. */ -#define TTY_THROTTLED 0 /* Call unthrottle() at threshold min */ -#define TTY_IO_ERROR 1 /* Cause an I/O error (may be no ldisc too) */ -#define TTY_OTHER_CLOSED 2 /* Other side (if any) has closed */ -#define TTY_EXCLUSIVE 3 /* Exclusive open mode */ -#define TTY_DO_WRITE_WAKEUP 5 /* Call write_wakeup after queuing new */ -#define TTY_LDISC_OPEN 11 /* Line discipline is open */ -#define TTY_PTY_LOCK 16 /* pty private */ -#define TTY_NO_WRITE_SPLIT 17 /* Preserve write boundaries to driver */ -#define TTY_HUPPED 18 /* Post driver->hangup() */ -#define TTY_HUPPING 19 /* Hangup in progress */ -#define TTY_LDISC_CHANGING 20 /* Change pending - non-block IO */ -#define TTY_LDISC_HALTED 22 /* Line discipline is halted */ +#define TTY_THROTTLED 0 +#define TTY_IO_ERROR 1 +#define TTY_OTHER_CLOSED 2 +#define TTY_EXCLUSIVE 3 +#define TTY_DO_WRITE_WAKEUP 5 +#define TTY_LDISC_OPEN 11 +#define TTY_PTY_LOCK 16 +#define TTY_NO_WRITE_SPLIT 17 +#define TTY_HUPPED 18 +#define TTY_HUPPING 19 +#define TTY_LDISC_CHANGING 20 +#define TTY_LDISC_HALTED 22 static inline bool tty_io_nonblock(struct tty_struct *tty, struct file *file) { -- cgit v1.2.3-59-g8ed1b From 34d809f8b4ff68f63e8d7f71d93d150382c6bb8b Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Fri, 26 Nov 2021 09:15:58 +0100 Subject: tty: reformat TTY_DRIVER_ flags into kernel-doc We want to reference TTY_DRIVER_* flags in Documentation/ later in this series. But the current documentation in the TTY_DRIVER_*'s header does not allow that. Reformat it to kernel-doc using "DOC" directive and line-feeds, so that we can include it as it is. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20211126081611.11001-11-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_driver.h | 90 ++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 41274d551e28..4841d8069c07 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -492,49 +492,53 @@ static inline void tty_set_operations(struct tty_driver *driver, /* tty driver magic number */ #define TTY_DRIVER_MAGIC 0x5402 -/* - * tty driver flags - * - * TTY_DRIVER_RESET_TERMIOS --- requests the tty layer to reset the - * termios setting when the last process has closed the device. - * Used for PTY's, in particular. - * - * TTY_DRIVER_REAL_RAW --- if set, indicates that the driver will - * guarantee never to set any special character handling - * flags if ((IGNBRK || (!BRKINT && !PARMRK)) && (IGNPAR || - * !INPCK)). That is, if there is no reason for the driver to - * send notifications of parity and break characters up to the - * line driver, it won't do so. This allows the line driver to - * optimize for this case if this flag is set. (Note that there - * is also a promise, if the above case is true, not to signal - * overruns, either.) - * - * TTY_DRIVER_DYNAMIC_DEV --- if set, the individual tty devices need - * to be registered with a call to tty_register_device() when the - * device is found in the system and unregistered with a call to - * tty_unregister_device() so the devices will be show up - * properly in sysfs. If not set, driver->num entries will be - * created by the tty core in sysfs when tty_register_driver() is - * called. This is to be used by drivers that have tty devices - * that can appear and disappear while the main tty driver is - * registered with the tty core. - * - * TTY_DRIVER_DEVPTS_MEM -- don't use the standard arrays, instead - * use dynamic memory keyed through the devpts filesystem. This - * is only applicable to the pty driver. - * - * TTY_DRIVER_HARDWARE_BREAK -- hardware handles break signals. Pass - * the requested timeout to the caller instead of using a simple - * on/off interface. - * - * TTY_DRIVER_DYNAMIC_ALLOC -- do not allocate structures which are - * needed per line for this driver as it would waste memory. - * The driver will take care. - * - * TTY_DRIVER_UNNUMBERED_NODE -- do not create numbered /dev nodes. In - * other words create /dev/ttyprintk and not /dev/ttyprintk0. - * Applicable only when a driver for a single tty device is - * being allocated. +/** + * DOC: TTY Driver Flags + * + * TTY_DRIVER_RESET_TERMIOS + * Requests the tty layer to reset the termios setting when the last + * process has closed the device. Used for PTYs, in particular. + * + * TTY_DRIVER_REAL_RAW + * Indicates that the driver will guarantee not to set any special + * character handling flags if this is set for the tty: + * + * ``(IGNBRK || (!BRKINT && !PARMRK)) && (IGNPAR || !INPCK)`` + * + * That is, if there is no reason for the driver to + * send notifications of parity and break characters up to the line + * driver, it won't do so. This allows the line driver to optimize for + * this case if this flag is set. (Note that there is also a promise, if + * the above case is true, not to signal overruns, either.) + * + * TTY_DRIVER_DYNAMIC_DEV + * The individual tty devices need to be registered with a call to + * tty_register_device() when the device is found in the system and + * unregistered with a call to tty_unregister_device() so the devices will + * be show up properly in sysfs. If not set, all &tty_driver.num entries + * will be created by the tty core in sysfs when tty_register_driver() is + * called. This is to be used by drivers that have tty devices that can + * appear and disappear while the main tty driver is registered with the + * tty core. + * + * TTY_DRIVER_DEVPTS_MEM + * Don't use the standard arrays (&tty_driver.ttys and + * &tty_driver.termios), instead use dynamic memory keyed through the + * devpts filesystem. This is only applicable to the PTY driver. + * + * TTY_DRIVER_HARDWARE_BREAK + * Hardware handles break signals. Pass the requested timeout to the + * &tty_operations.break_ctl instead of using a simple on/off interface. + * + * TTY_DRIVER_DYNAMIC_ALLOC + * Do not allocate structures which are needed per line for this driver + * (&tty_driver.ports) as it would waste memory. The driver will take + * care. This is only applicable to the PTY driver. + * + * TTY_DRIVER_UNNUMBERED_NODE + * Do not create numbered ``/dev`` nodes. For example, create + * ``/dev/ttyprintk`` and not ``/dev/ttyprintk0``. Applicable only when a + * driver for a single tty device is being allocated. */ #define TTY_DRIVER_INSTALLED 0x0001 #define TTY_DRIVER_RESET_TERMIOS 0x0002 -- cgit v1.2.3-59-g8ed1b From df0e68c1e9945e2ee86d266ce45597bbd8299b06 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Wed, 17 Nov 2021 12:05:59 +0000 Subject: comedi: Move the main COMEDI headers Move the main COMEDI driver headers out of "drivers/comedi/" into new directory "include/linux/comedi/". These are "comedidev.h", "comedilib.h", "comedi_pci.h", "comedi_pcmcia.h", and "comedi_usb.h". Additionally, move the user-space API header "comedi.h" into "include/uapi/linux/" and add "WITH Linux-syscall-note" to its SPDX-License-Identifier. Update the "COMEDI DRIVERS" section of the MAINTAINERS file to account for these changes. Signed-off-by: Ian Abbott Link: https://lore.kernel.org/r/20211117120604.117740-2-abbotti@mev.co.uk Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 2 + drivers/comedi/comedi.h | 1528 -------------------- drivers/comedi/comedi_buf.c | 3 +- drivers/comedi/comedi_fops.c | 2 +- drivers/comedi/comedi_pci.c | 3 +- drivers/comedi/comedi_pci.h | 57 - drivers/comedi/comedi_pcmcia.c | 3 +- drivers/comedi/comedi_pcmcia.h | 49 - drivers/comedi/comedi_usb.c | 3 +- drivers/comedi/comedi_usb.h | 42 - drivers/comedi/comedidev.h | 1054 -------------- drivers/comedi/comedilib.h | 26 - drivers/comedi/drivers.c | 3 +- drivers/comedi/drivers/8255.c | 2 +- drivers/comedi/drivers/8255_pci.c | 3 +- drivers/comedi/drivers/addi_apci_1032.c | 2 +- drivers/comedi/drivers/addi_apci_1500.c | 2 +- drivers/comedi/drivers/addi_apci_1516.c | 2 +- drivers/comedi/drivers/addi_apci_1564.c | 2 +- drivers/comedi/drivers/addi_apci_16xx.c | 3 +- drivers/comedi/drivers/addi_apci_2032.c | 2 +- drivers/comedi/drivers/addi_apci_2200.c | 2 +- drivers/comedi/drivers/addi_apci_3120.c | 2 +- drivers/comedi/drivers/addi_apci_3501.c | 2 +- drivers/comedi/drivers/addi_apci_3xxx.c | 3 +- drivers/comedi/drivers/addi_watchdog.c | 2 +- drivers/comedi/drivers/adl_pci6208.c | 3 +- drivers/comedi/drivers/adl_pci7x3x.c | 3 +- drivers/comedi/drivers/adl_pci8164.c | 3 +- drivers/comedi/drivers/adl_pci9111.c | 3 +- drivers/comedi/drivers/adl_pci9118.c | 3 +- drivers/comedi/drivers/adq12b.c | 3 +- drivers/comedi/drivers/adv_pci1710.c | 3 +- drivers/comedi/drivers/adv_pci1720.c | 3 +- drivers/comedi/drivers/adv_pci1723.c | 3 +- drivers/comedi/drivers/adv_pci1724.c | 3 +- drivers/comedi/drivers/adv_pci1760.c | 3 +- drivers/comedi/drivers/adv_pci_dio.c | 3 +- drivers/comedi/drivers/aio_aio12_8.c | 2 +- drivers/comedi/drivers/aio_iiro_16.c | 3 +- drivers/comedi/drivers/amplc_dio200.c | 2 +- drivers/comedi/drivers/amplc_dio200_common.c | 3 +- drivers/comedi/drivers/amplc_dio200_pci.c | 3 +- drivers/comedi/drivers/amplc_pc236.c | 3 +- drivers/comedi/drivers/amplc_pc236_common.c | 3 +- drivers/comedi/drivers/amplc_pc263.c | 2 +- drivers/comedi/drivers/amplc_pci224.c | 3 +- drivers/comedi/drivers/amplc_pci230.c | 3 +- drivers/comedi/drivers/amplc_pci236.c | 3 +- drivers/comedi/drivers/amplc_pci263.c | 3 +- drivers/comedi/drivers/c6xdigio.c | 3 +- drivers/comedi/drivers/cb_das16_cs.c | 3 +- drivers/comedi/drivers/cb_pcidas.c | 3 +- drivers/comedi/drivers/cb_pcidas64.c | 3 +- drivers/comedi/drivers/cb_pcidda.c | 3 +- drivers/comedi/drivers/cb_pcimdas.c | 3 +- drivers/comedi/drivers/cb_pcimdda.c | 3 +- drivers/comedi/drivers/comedi_8254.c | 3 +- drivers/comedi/drivers/comedi_8255.c | 2 +- drivers/comedi/drivers/comedi_bond.c | 6 +- drivers/comedi/drivers/comedi_isadma.c | 3 +- drivers/comedi/drivers/comedi_parport.c | 3 +- drivers/comedi/drivers/comedi_test.c | 4 +- drivers/comedi/drivers/contec_pci_dio.c | 3 +- drivers/comedi/drivers/dac02.c | 3 +- drivers/comedi/drivers/daqboard2000.c | 3 +- drivers/comedi/drivers/das08.c | 3 +- drivers/comedi/drivers/das08_cs.c | 3 +- drivers/comedi/drivers/das08_isa.c | 2 +- drivers/comedi/drivers/das08_pci.c | 3 +- drivers/comedi/drivers/das16.c | 3 +- drivers/comedi/drivers/das16m1.c | 2 +- drivers/comedi/drivers/das1800.c | 3 +- drivers/comedi/drivers/das6402.c | 3 +- drivers/comedi/drivers/das800.c | 3 +- drivers/comedi/drivers/dmm32at.c | 2 +- drivers/comedi/drivers/dt2801.c | 2 +- drivers/comedi/drivers/dt2811.c | 3 +- drivers/comedi/drivers/dt2814.c | 3 +- drivers/comedi/drivers/dt2815.c | 3 +- drivers/comedi/drivers/dt2817.c | 2 +- drivers/comedi/drivers/dt282x.c | 3 +- drivers/comedi/drivers/dt3000.c | 3 +- drivers/comedi/drivers/dt9812.c | 3 +- drivers/comedi/drivers/dyna_pci10xx.c | 3 +- drivers/comedi/drivers/fl512.c | 3 +- drivers/comedi/drivers/gsc_hpdi.c | 3 +- drivers/comedi/drivers/icp_multi.c | 3 +- drivers/comedi/drivers/ii_pci20kc.c | 2 +- drivers/comedi/drivers/jr3_pci.c | 3 +- drivers/comedi/drivers/ke_counter.c | 3 +- drivers/comedi/drivers/me4000.c | 3 +- drivers/comedi/drivers/me_daq.c | 3 +- drivers/comedi/drivers/mf6x4.c | 3 +- drivers/comedi/drivers/mite.c | 3 +- drivers/comedi/drivers/mpc624.c | 3 +- drivers/comedi/drivers/multiq3.c | 3 +- drivers/comedi/drivers/ni_6527.c | 3 +- drivers/comedi/drivers/ni_65xx.c | 3 +- drivers/comedi/drivers/ni_660x.c | 3 +- drivers/comedi/drivers/ni_670x.c | 3 +- drivers/comedi/drivers/ni_at_a2150.c | 3 +- drivers/comedi/drivers/ni_at_ao.c | 3 +- drivers/comedi/drivers/ni_atmio.c | 3 +- drivers/comedi/drivers/ni_atmio16d.c | 2 +- drivers/comedi/drivers/ni_daq_700.c | 3 +- drivers/comedi/drivers/ni_daq_dio24.c | 2 +- drivers/comedi/drivers/ni_labpc.c | 3 +- drivers/comedi/drivers/ni_labpc_common.c | 3 +- drivers/comedi/drivers/ni_labpc_cs.c | 3 +- drivers/comedi/drivers/ni_labpc_isadma.c | 3 +- drivers/comedi/drivers/ni_labpc_pci.c | 3 +- drivers/comedi/drivers/ni_mio_cs.c | 2 +- drivers/comedi/drivers/ni_pcidio.c | 3 +- drivers/comedi/drivers/ni_pcimio.c | 4 +- drivers/comedi/drivers/ni_routes.c | 3 +- drivers/comedi/drivers/ni_routes.h | 2 +- .../comedi/drivers/ni_routing/ni_route_values.h | 2 +- drivers/comedi/drivers/ni_tio.h | 2 +- drivers/comedi/drivers/ni_usb6501.c | 3 +- drivers/comedi/drivers/pcl711.c | 3 +- drivers/comedi/drivers/pcl724.c | 2 +- drivers/comedi/drivers/pcl726.c | 3 +- drivers/comedi/drivers/pcl730.c | 2 +- drivers/comedi/drivers/pcl812.c | 3 +- drivers/comedi/drivers/pcl816.c | 3 +- drivers/comedi/drivers/pcl818.c | 3 +- drivers/comedi/drivers/pcm3724.c | 2 +- drivers/comedi/drivers/pcmad.c | 2 +- drivers/comedi/drivers/pcmda12.c | 2 +- drivers/comedi/drivers/pcmmio.c | 3 +- drivers/comedi/drivers/pcmuio.c | 3 +- drivers/comedi/drivers/quatech_daqp_cs.c | 3 +- drivers/comedi/drivers/rtd520.c | 3 +- drivers/comedi/drivers/rti800.c | 2 +- drivers/comedi/drivers/rti802.c | 2 +- drivers/comedi/drivers/s526.c | 2 +- drivers/comedi/drivers/s626.c | 3 +- drivers/comedi/drivers/ssv_dnp.c | 2 +- drivers/comedi/drivers/usbdux.c | 3 +- drivers/comedi/drivers/usbduxfast.c | 2 +- drivers/comedi/drivers/usbduxsigma.c | 3 +- drivers/comedi/drivers/vmk80xx.c | 3 +- drivers/comedi/kcomedilib/kcomedilib_main.c | 6 +- drivers/comedi/proc.c | 2 +- drivers/comedi/range.c | 2 +- include/linux/comedi/comedi_pci.h | 56 + include/linux/comedi/comedi_pcmcia.h | 48 + include/linux/comedi/comedi_usb.h | 41 + include/linux/comedi/comedidev.h | 1053 ++++++++++++++ include/linux/comedi/comedilib.h | 26 + include/uapi/linux/comedi.h | 1528 ++++++++++++++++++++ 152 files changed, 2897 insertions(+), 2999 deletions(-) delete mode 100644 drivers/comedi/comedi.h delete mode 100644 drivers/comedi/comedi_pci.h delete mode 100644 drivers/comedi/comedi_pcmcia.h delete mode 100644 drivers/comedi/comedi_usb.h delete mode 100644 drivers/comedi/comedidev.h delete mode 100644 drivers/comedi/comedilib.h create mode 100644 include/linux/comedi/comedi_pci.h create mode 100644 include/linux/comedi/comedi_pcmcia.h create mode 100644 include/linux/comedi/comedi_usb.h create mode 100644 include/linux/comedi/comedidev.h create mode 100644 include/linux/comedi/comedilib.h create mode 100644 include/uapi/linux/comedi.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 7a2345ce8521..f9b50d136c90 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4713,6 +4713,8 @@ M: Ian Abbott M: H Hartley Sweeten S: Odd Fixes F: drivers/comedi/ +F: include/linux/comedi/ +F: include/uapi/linux/comedi.h COMMON CLK FRAMEWORK M: Michael Turquette diff --git a/drivers/comedi/comedi.h b/drivers/comedi/comedi.h deleted file mode 100644 index b5d00a006dbb..000000000000 --- a/drivers/comedi/comedi.h +++ /dev/null @@ -1,1528 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.0+ */ -/* - * comedi.h - * header file for COMEDI user API - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1998-2001 David A. Schleef - */ - -#ifndef _COMEDI_H -#define _COMEDI_H - -#define COMEDI_MAJORVERSION 0 -#define COMEDI_MINORVERSION 7 -#define COMEDI_MICROVERSION 76 -#define VERSION "0.7.76" - -/* comedi's major device number */ -#define COMEDI_MAJOR 98 - -/* - * maximum number of minor devices. This can be increased, although - * kernel structures are currently statically allocated, thus you - * don't want this to be much more than you actually use. - */ -#define COMEDI_NDEVICES 16 - -/* number of config options in the config structure */ -#define COMEDI_NDEVCONFOPTS 32 - -/* - * NOTE: 'comedi_config --init-data' is deprecated - * - * The following indexes in the config options were used by - * comedi_config to pass firmware blobs from user space to the - * comedi drivers. The request_firmware() hotplug interface is - * now used by all comedi drivers instead. - */ - -/* length of nth chunk of firmware data -*/ -#define COMEDI_DEVCONF_AUX_DATA3_LENGTH 25 -#define COMEDI_DEVCONF_AUX_DATA2_LENGTH 26 -#define COMEDI_DEVCONF_AUX_DATA1_LENGTH 27 -#define COMEDI_DEVCONF_AUX_DATA0_LENGTH 28 -/* most significant 32 bits of pointer address (if needed) */ -#define COMEDI_DEVCONF_AUX_DATA_HI 29 -/* least significant 32 bits of pointer address */ -#define COMEDI_DEVCONF_AUX_DATA_LO 30 -#define COMEDI_DEVCONF_AUX_DATA_LENGTH 31 /* total data length */ - -/* max length of device and driver names */ -#define COMEDI_NAMELEN 20 - -/* packs and unpacks a channel/range number */ - -#define CR_PACK(chan, rng, aref) \ - ((((aref) & 0x3) << 24) | (((rng) & 0xff) << 16) | (chan)) -#define CR_PACK_FLAGS(chan, range, aref, flags) \ - (CR_PACK(chan, range, aref) | ((flags) & CR_FLAGS_MASK)) - -#define CR_CHAN(a) ((a) & 0xffff) -#define CR_RANGE(a) (((a) >> 16) & 0xff) -#define CR_AREF(a) (((a) >> 24) & 0x03) - -#define CR_FLAGS_MASK 0xfc000000 -#define CR_ALT_FILTER 0x04000000 -#define CR_DITHER CR_ALT_FILTER -#define CR_DEGLITCH CR_ALT_FILTER -#define CR_ALT_SOURCE 0x08000000 -#define CR_EDGE 0x40000000 -#define CR_INVERT 0x80000000 - -#define AREF_GROUND 0x00 /* analog ref = analog ground */ -#define AREF_COMMON 0x01 /* analog ref = analog common */ -#define AREF_DIFF 0x02 /* analog ref = differential */ -#define AREF_OTHER 0x03 /* analog ref = other (undefined) */ - -/* counters -- these are arbitrary values */ -#define GPCT_RESET 0x0001 -#define GPCT_SET_SOURCE 0x0002 -#define GPCT_SET_GATE 0x0004 -#define GPCT_SET_DIRECTION 0x0008 -#define GPCT_SET_OPERATION 0x0010 -#define GPCT_ARM 0x0020 -#define GPCT_DISARM 0x0040 -#define GPCT_GET_INT_CLK_FRQ 0x0080 - -#define GPCT_INT_CLOCK 0x0001 -#define GPCT_EXT_PIN 0x0002 -#define GPCT_NO_GATE 0x0004 -#define GPCT_UP 0x0008 -#define GPCT_DOWN 0x0010 -#define GPCT_HWUD 0x0020 -#define GPCT_SIMPLE_EVENT 0x0040 -#define GPCT_SINGLE_PERIOD 0x0080 -#define GPCT_SINGLE_PW 0x0100 -#define GPCT_CONT_PULSE_OUT 0x0200 -#define GPCT_SINGLE_PULSE_OUT 0x0400 - -/* instructions */ - -#define INSN_MASK_WRITE 0x8000000 -#define INSN_MASK_READ 0x4000000 -#define INSN_MASK_SPECIAL 0x2000000 - -#define INSN_READ (0 | INSN_MASK_READ) -#define INSN_WRITE (1 | INSN_MASK_WRITE) -#define INSN_BITS (2 | INSN_MASK_READ | INSN_MASK_WRITE) -#define INSN_CONFIG (3 | INSN_MASK_READ | INSN_MASK_WRITE) -#define INSN_DEVICE_CONFIG (INSN_CONFIG | INSN_MASK_SPECIAL) -#define INSN_GTOD (4 | INSN_MASK_READ | INSN_MASK_SPECIAL) -#define INSN_WAIT (5 | INSN_MASK_WRITE | INSN_MASK_SPECIAL) -#define INSN_INTTRIG (6 | INSN_MASK_WRITE | INSN_MASK_SPECIAL) - -/* command flags */ -/* These flags are used in comedi_cmd structures */ - -#define CMDF_BOGUS 0x00000001 /* do the motions */ - -/* try to use a real-time interrupt while performing command */ -#define CMDF_PRIORITY 0x00000008 - -/* wake up on end-of-scan events */ -#define CMDF_WAKE_EOS 0x00000020 - -#define CMDF_WRITE 0x00000040 - -#define CMDF_RAWDATA 0x00000080 - -/* timer rounding definitions */ -#define CMDF_ROUND_MASK 0x00030000 -#define CMDF_ROUND_NEAREST 0x00000000 -#define CMDF_ROUND_DOWN 0x00010000 -#define CMDF_ROUND_UP 0x00020000 -#define CMDF_ROUND_UP_NEXT 0x00030000 - -#define COMEDI_EV_START 0x00040000 -#define COMEDI_EV_SCAN_BEGIN 0x00080000 -#define COMEDI_EV_CONVERT 0x00100000 -#define COMEDI_EV_SCAN_END 0x00200000 -#define COMEDI_EV_STOP 0x00400000 - -/* compatibility definitions */ -#define TRIG_BOGUS CMDF_BOGUS -#define TRIG_RT CMDF_PRIORITY -#define TRIG_WAKE_EOS CMDF_WAKE_EOS -#define TRIG_WRITE CMDF_WRITE -#define TRIG_ROUND_MASK CMDF_ROUND_MASK -#define TRIG_ROUND_NEAREST CMDF_ROUND_NEAREST -#define TRIG_ROUND_DOWN CMDF_ROUND_DOWN -#define TRIG_ROUND_UP CMDF_ROUND_UP -#define TRIG_ROUND_UP_NEXT CMDF_ROUND_UP_NEXT - -/* trigger sources */ - -#define TRIG_ANY 0xffffffff -#define TRIG_INVALID 0x00000000 - -#define TRIG_NONE 0x00000001 /* never trigger */ -#define TRIG_NOW 0x00000002 /* trigger now + N ns */ -#define TRIG_FOLLOW 0x00000004 /* trigger on next lower level trig */ -#define TRIG_TIME 0x00000008 /* trigger at time N ns */ -#define TRIG_TIMER 0x00000010 /* trigger at rate N ns */ -#define TRIG_COUNT 0x00000020 /* trigger when count reaches N */ -#define TRIG_EXT 0x00000040 /* trigger on external signal N */ -#define TRIG_INT 0x00000080 /* trigger on comedi-internal signal N */ -#define TRIG_OTHER 0x00000100 /* driver defined */ - -/* subdevice flags */ - -#define SDF_BUSY 0x0001 /* device is busy */ -#define SDF_BUSY_OWNER 0x0002 /* device is busy with your job */ -#define SDF_LOCKED 0x0004 /* subdevice is locked */ -#define SDF_LOCK_OWNER 0x0008 /* you own lock */ -#define SDF_MAXDATA 0x0010 /* maxdata depends on channel */ -#define SDF_FLAGS 0x0020 /* flags depend on channel */ -#define SDF_RANGETYPE 0x0040 /* range type depends on channel */ -#define SDF_PWM_COUNTER 0x0080 /* PWM can automatically switch off */ -#define SDF_PWM_HBRIDGE 0x0100 /* PWM is signed (H-bridge) */ -#define SDF_CMD 0x1000 /* can do commands (deprecated) */ -#define SDF_SOFT_CALIBRATED 0x2000 /* subdevice uses software calibration */ -#define SDF_CMD_WRITE 0x4000 /* can do output commands */ -#define SDF_CMD_READ 0x8000 /* can do input commands */ - -/* subdevice can be read (e.g. analog input) */ -#define SDF_READABLE 0x00010000 -/* subdevice can be written (e.g. analog output) */ -#define SDF_WRITABLE 0x00020000 -#define SDF_WRITEABLE SDF_WRITABLE /* spelling error in API */ -/* subdevice does not have externally visible lines */ -#define SDF_INTERNAL 0x00040000 -#define SDF_GROUND 0x00100000 /* can do aref=ground */ -#define SDF_COMMON 0x00200000 /* can do aref=common */ -#define SDF_DIFF 0x00400000 /* can do aref=diff */ -#define SDF_OTHER 0x00800000 /* can do aref=other */ -#define SDF_DITHER 0x01000000 /* can do dithering */ -#define SDF_DEGLITCH 0x02000000 /* can do deglitching */ -#define SDF_MMAP 0x04000000 /* can do mmap() */ -#define SDF_RUNNING 0x08000000 /* subdevice is acquiring data */ -#define SDF_LSAMPL 0x10000000 /* subdevice uses 32-bit samples */ -#define SDF_PACKED 0x20000000 /* subdevice can do packed DIO */ - -/* subdevice types */ - -/** - * enum comedi_subdevice_type - COMEDI subdevice types - * @COMEDI_SUBD_UNUSED: Unused subdevice. - * @COMEDI_SUBD_AI: Analog input. - * @COMEDI_SUBD_AO: Analog output. - * @COMEDI_SUBD_DI: Digital input. - * @COMEDI_SUBD_DO: Digital output. - * @COMEDI_SUBD_DIO: Digital input/output. - * @COMEDI_SUBD_COUNTER: Counter. - * @COMEDI_SUBD_TIMER: Timer. - * @COMEDI_SUBD_MEMORY: Memory, EEPROM, DPRAM. - * @COMEDI_SUBD_CALIB: Calibration DACs. - * @COMEDI_SUBD_PROC: Processor, DSP. - * @COMEDI_SUBD_SERIAL: Serial I/O. - * @COMEDI_SUBD_PWM: Pulse-Width Modulation output. - */ -enum comedi_subdevice_type { - COMEDI_SUBD_UNUSED, - COMEDI_SUBD_AI, - COMEDI_SUBD_AO, - COMEDI_SUBD_DI, - COMEDI_SUBD_DO, - COMEDI_SUBD_DIO, - COMEDI_SUBD_COUNTER, - COMEDI_SUBD_TIMER, - COMEDI_SUBD_MEMORY, - COMEDI_SUBD_CALIB, - COMEDI_SUBD_PROC, - COMEDI_SUBD_SERIAL, - COMEDI_SUBD_PWM -}; - -/* configuration instructions */ - -/** - * enum comedi_io_direction - COMEDI I/O directions - * @COMEDI_INPUT: Input. - * @COMEDI_OUTPUT: Output. - * @COMEDI_OPENDRAIN: Open-drain (or open-collector) output. - * - * These are used by the %INSN_CONFIG_DIO_QUERY configuration instruction to - * report a direction. They may also be used in other places where a direction - * needs to be specified. - */ -enum comedi_io_direction { - COMEDI_INPUT = 0, - COMEDI_OUTPUT = 1, - COMEDI_OPENDRAIN = 2 -}; - -/** - * enum configuration_ids - COMEDI configuration instruction codes - * @INSN_CONFIG_DIO_INPUT: Configure digital I/O as input. - * @INSN_CONFIG_DIO_OUTPUT: Configure digital I/O as output. - * @INSN_CONFIG_DIO_OPENDRAIN: Configure digital I/O as open-drain (or open - * collector) output. - * @INSN_CONFIG_ANALOG_TRIG: Configure analog trigger. - * @INSN_CONFIG_ALT_SOURCE: Configure alternate input source. - * @INSN_CONFIG_DIGITAL_TRIG: Configure digital trigger. - * @INSN_CONFIG_BLOCK_SIZE: Configure block size for DMA transfers. - * @INSN_CONFIG_TIMER_1: Configure divisor for external clock. - * @INSN_CONFIG_FILTER: Configure a filter. - * @INSN_CONFIG_CHANGE_NOTIFY: Configure change notification for digital - * inputs. (New drivers should use - * %INSN_CONFIG_DIGITAL_TRIG instead.) - * @INSN_CONFIG_SERIAL_CLOCK: Configure clock for serial I/O. - * @INSN_CONFIG_BIDIRECTIONAL_DATA: Send and receive byte over serial I/O. - * @INSN_CONFIG_DIO_QUERY: Query direction of digital I/O channel. - * @INSN_CONFIG_PWM_OUTPUT: Configure pulse-width modulator output. - * @INSN_CONFIG_GET_PWM_OUTPUT: Get pulse-width modulator output configuration. - * @INSN_CONFIG_ARM: Arm a subdevice or channel. - * @INSN_CONFIG_DISARM: Disarm a subdevice or channel. - * @INSN_CONFIG_GET_COUNTER_STATUS: Get counter status. - * @INSN_CONFIG_RESET: Reset a subdevice or channel. - * @INSN_CONFIG_GPCT_SINGLE_PULSE_GENERATOR: Configure counter/timer as - * single pulse generator. - * @INSN_CONFIG_GPCT_PULSE_TRAIN_GENERATOR: Configure counter/timer as - * pulse train generator. - * @INSN_CONFIG_GPCT_QUADRATURE_ENCODER: Configure counter as a quadrature - * encoder. - * @INSN_CONFIG_SET_GATE_SRC: Set counter/timer gate source. - * @INSN_CONFIG_GET_GATE_SRC: Get counter/timer gate source. - * @INSN_CONFIG_SET_CLOCK_SRC: Set counter/timer master clock source. - * @INSN_CONFIG_GET_CLOCK_SRC: Get counter/timer master clock source. - * @INSN_CONFIG_SET_OTHER_SRC: Set counter/timer "other" source. - * @INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE: Get size (in bytes) of subdevice's - * on-board FIFOs used during streaming - * input/output. - * @INSN_CONFIG_SET_COUNTER_MODE: Set counter/timer mode. - * @INSN_CONFIG_8254_SET_MODE: (Deprecated) Same as - * %INSN_CONFIG_SET_COUNTER_MODE. - * @INSN_CONFIG_8254_READ_STATUS: Read status of 8254 counter channel. - * @INSN_CONFIG_SET_ROUTING: Set routing for a channel. - * @INSN_CONFIG_GET_ROUTING: Get routing for a channel. - * @INSN_CONFIG_PWM_SET_PERIOD: Set PWM period in nanoseconds. - * @INSN_CONFIG_PWM_GET_PERIOD: Get PWM period in nanoseconds. - * @INSN_CONFIG_GET_PWM_STATUS: Get PWM status. - * @INSN_CONFIG_PWM_SET_H_BRIDGE: Set PWM H bridge duty cycle and polarity for - * a relay simultaneously. - * @INSN_CONFIG_PWM_GET_H_BRIDGE: Get PWM H bridge duty cycle and polarity. - * @INSN_CONFIG_GET_CMD_TIMING_CONSTRAINTS: Get the hardware timing restraints, - * regardless of trigger sources. - */ -enum configuration_ids { - INSN_CONFIG_DIO_INPUT = COMEDI_INPUT, - INSN_CONFIG_DIO_OUTPUT = COMEDI_OUTPUT, - INSN_CONFIG_DIO_OPENDRAIN = COMEDI_OPENDRAIN, - INSN_CONFIG_ANALOG_TRIG = 16, -/* INSN_CONFIG_WAVEFORM = 17, */ -/* INSN_CONFIG_TRIG = 18, */ -/* INSN_CONFIG_COUNTER = 19, */ - INSN_CONFIG_ALT_SOURCE = 20, - INSN_CONFIG_DIGITAL_TRIG = 21, - INSN_CONFIG_BLOCK_SIZE = 22, - INSN_CONFIG_TIMER_1 = 23, - INSN_CONFIG_FILTER = 24, - INSN_CONFIG_CHANGE_NOTIFY = 25, - - INSN_CONFIG_SERIAL_CLOCK = 26, /*ALPHA*/ - INSN_CONFIG_BIDIRECTIONAL_DATA = 27, - INSN_CONFIG_DIO_QUERY = 28, - INSN_CONFIG_PWM_OUTPUT = 29, - INSN_CONFIG_GET_PWM_OUTPUT = 30, - INSN_CONFIG_ARM = 31, - INSN_CONFIG_DISARM = 32, - INSN_CONFIG_GET_COUNTER_STATUS = 33, - INSN_CONFIG_RESET = 34, - INSN_CONFIG_GPCT_SINGLE_PULSE_GENERATOR = 1001, - INSN_CONFIG_GPCT_PULSE_TRAIN_GENERATOR = 1002, - INSN_CONFIG_GPCT_QUADRATURE_ENCODER = 1003, - INSN_CONFIG_SET_GATE_SRC = 2001, - INSN_CONFIG_GET_GATE_SRC = 2002, - INSN_CONFIG_SET_CLOCK_SRC = 2003, - INSN_CONFIG_GET_CLOCK_SRC = 2004, - INSN_CONFIG_SET_OTHER_SRC = 2005, - INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE = 2006, - INSN_CONFIG_SET_COUNTER_MODE = 4097, - INSN_CONFIG_8254_SET_MODE = INSN_CONFIG_SET_COUNTER_MODE, - INSN_CONFIG_8254_READ_STATUS = 4098, - INSN_CONFIG_SET_ROUTING = 4099, - INSN_CONFIG_GET_ROUTING = 4109, - INSN_CONFIG_PWM_SET_PERIOD = 5000, - INSN_CONFIG_PWM_GET_PERIOD = 5001, - INSN_CONFIG_GET_PWM_STATUS = 5002, - INSN_CONFIG_PWM_SET_H_BRIDGE = 5003, - INSN_CONFIG_PWM_GET_H_BRIDGE = 5004, - INSN_CONFIG_GET_CMD_TIMING_CONSTRAINTS = 5005, -}; - -/** - * enum device_configuration_ids - COMEDI configuration instruction codes global - * to an entire device. - * @INSN_DEVICE_CONFIG_TEST_ROUTE: Validate the possibility of a - * globally-named route - * @INSN_DEVICE_CONFIG_CONNECT_ROUTE: Connect a globally-named route - * @INSN_DEVICE_CONFIG_DISCONNECT_ROUTE:Disconnect a globally-named route - * @INSN_DEVICE_CONFIG_GET_ROUTES: Get a list of all globally-named routes - * that are valid for a particular device. - */ -enum device_config_route_ids { - INSN_DEVICE_CONFIG_TEST_ROUTE = 0, - INSN_DEVICE_CONFIG_CONNECT_ROUTE = 1, - INSN_DEVICE_CONFIG_DISCONNECT_ROUTE = 2, - INSN_DEVICE_CONFIG_GET_ROUTES = 3, -}; - -/** - * enum comedi_digital_trig_op - operations for configuring a digital trigger - * @COMEDI_DIGITAL_TRIG_DISABLE: Return digital trigger to its default, - * inactive, unconfigured state. - * @COMEDI_DIGITAL_TRIG_ENABLE_EDGES: Set rising and/or falling edge inputs - * that each can fire the trigger. - * @COMEDI_DIGITAL_TRIG_ENABLE_LEVELS: Set a combination of high and/or low - * level inputs that can fire the trigger. - * - * These are used with the %INSN_CONFIG_DIGITAL_TRIG configuration instruction. - * The data for the configuration instruction is as follows... - * - * data[%0] = %INSN_CONFIG_DIGITAL_TRIG - * - * data[%1] = trigger ID - * - * data[%2] = configuration operation - * - * data[%3] = configuration parameter 1 - * - * data[%4] = configuration parameter 2 - * - * data[%5] = configuration parameter 3 - * - * The trigger ID (data[%1]) is used to differentiate multiple digital triggers - * belonging to the same subdevice. The configuration operation (data[%2]) is - * one of the enum comedi_digital_trig_op values. The configuration - * parameters (data[%3], data[%4], and data[%5]) depend on the operation; they - * are not used with %COMEDI_DIGITAL_TRIG_DISABLE. - * - * For %COMEDI_DIGITAL_TRIG_ENABLE_EDGES and %COMEDI_DIGITAL_TRIG_ENABLE_LEVELS, - * configuration parameter 1 (data[%3]) contains a "left-shift" value that - * specifies the input corresponding to bit 0 of configuration parameters 2 - * and 3. This is useful if the trigger has more than 32 inputs. - * - * For %COMEDI_DIGITAL_TRIG_ENABLE_EDGES, configuration parameter 2 (data[%4]) - * specifies which of up to 32 inputs have rising-edge sensitivity, and - * configuration parameter 3 (data[%5]) specifies which of up to 32 inputs - * have falling-edge sensitivity that can fire the trigger. - * - * For %COMEDI_DIGITAL_TRIG_ENABLE_LEVELS, configuration parameter 2 (data[%4]) - * specifies which of up to 32 inputs must be at a high level, and - * configuration parameter 3 (data[%5]) specifies which of up to 32 inputs - * must be at a low level for the trigger to fire. - * - * Some sequences of %INSN_CONFIG_DIGITAL_TRIG instructions may have a (partly) - * accumulative effect, depending on the low-level driver. This is useful - * when setting up a trigger that has more than 32 inputs, or has a combination - * of edge- and level-triggered inputs. - */ -enum comedi_digital_trig_op { - COMEDI_DIGITAL_TRIG_DISABLE = 0, - COMEDI_DIGITAL_TRIG_ENABLE_EDGES = 1, - COMEDI_DIGITAL_TRIG_ENABLE_LEVELS = 2 -}; - -/** - * enum comedi_support_level - support level for a COMEDI feature - * @COMEDI_UNKNOWN_SUPPORT: Unspecified support for feature. - * @COMEDI_SUPPORTED: Feature is supported. - * @COMEDI_UNSUPPORTED: Feature is unsupported. - */ -enum comedi_support_level { - COMEDI_UNKNOWN_SUPPORT = 0, - COMEDI_SUPPORTED, - COMEDI_UNSUPPORTED -}; - -/** - * enum comedi_counter_status_flags - counter status bits - * @COMEDI_COUNTER_ARMED: Counter is armed. - * @COMEDI_COUNTER_COUNTING: Counter is counting. - * @COMEDI_COUNTER_TERMINAL_COUNT: Counter reached terminal count. - * - * These bitwise values are used by the %INSN_CONFIG_GET_COUNTER_STATUS - * configuration instruction to report the status of a counter. - */ -enum comedi_counter_status_flags { - COMEDI_COUNTER_ARMED = 0x1, - COMEDI_COUNTER_COUNTING = 0x2, - COMEDI_COUNTER_TERMINAL_COUNT = 0x4, -}; - -/* ioctls */ - -#define CIO 'd' -#define COMEDI_DEVCONFIG _IOW(CIO, 0, struct comedi_devconfig) -#define COMEDI_DEVINFO _IOR(CIO, 1, struct comedi_devinfo) -#define COMEDI_SUBDINFO _IOR(CIO, 2, struct comedi_subdinfo) -#define COMEDI_CHANINFO _IOR(CIO, 3, struct comedi_chaninfo) -/* _IOWR(CIO, 4, ...) is reserved */ -#define COMEDI_LOCK _IO(CIO, 5) -#define COMEDI_UNLOCK _IO(CIO, 6) -#define COMEDI_CANCEL _IO(CIO, 7) -#define COMEDI_RANGEINFO _IOR(CIO, 8, struct comedi_rangeinfo) -#define COMEDI_CMD _IOR(CIO, 9, struct comedi_cmd) -#define COMEDI_CMDTEST _IOR(CIO, 10, struct comedi_cmd) -#define COMEDI_INSNLIST _IOR(CIO, 11, struct comedi_insnlist) -#define COMEDI_INSN _IOR(CIO, 12, struct comedi_insn) -#define COMEDI_BUFCONFIG _IOR(CIO, 13, struct comedi_bufconfig) -#define COMEDI_BUFINFO _IOWR(CIO, 14, struct comedi_bufinfo) -#define COMEDI_POLL _IO(CIO, 15) -#define COMEDI_SETRSUBD _IO(CIO, 16) -#define COMEDI_SETWSUBD _IO(CIO, 17) - -/* structures */ - -/** - * struct comedi_insn - COMEDI instruction - * @insn: COMEDI instruction type (%INSN_xxx). - * @n: Length of @data[]. - * @data: Pointer to data array operated on by the instruction. - * @subdev: Subdevice index. - * @chanspec: A packed "chanspec" value consisting of channel number, - * analog range index, analog reference type, and flags. - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_INSN ioctl, and indirectly with the - * %COMEDI_INSNLIST ioctl. - */ -struct comedi_insn { - unsigned int insn; - unsigned int n; - unsigned int __user *data; - unsigned int subdev; - unsigned int chanspec; - unsigned int unused[3]; -}; - -/** - * struct comedi_insnlist - list of COMEDI instructions - * @n_insns: Number of COMEDI instructions. - * @insns: Pointer to array COMEDI instructions. - * - * This is used with the %COMEDI_INSNLIST ioctl. - */ -struct comedi_insnlist { - unsigned int n_insns; - struct comedi_insn __user *insns; -}; - -/** - * struct comedi_cmd - COMEDI asynchronous acquisition command details - * @subdev: Subdevice index. - * @flags: Command flags (%CMDF_xxx). - * @start_src: "Start acquisition" trigger source (%TRIG_xxx). - * @start_arg: "Start acquisition" trigger argument. - * @scan_begin_src: "Scan begin" trigger source. - * @scan_begin_arg: "Scan begin" trigger argument. - * @convert_src: "Convert" trigger source. - * @convert_arg: "Convert" trigger argument. - * @scan_end_src: "Scan end" trigger source. - * @scan_end_arg: "Scan end" trigger argument. - * @stop_src: "Stop acquisition" trigger source. - * @stop_arg: "Stop acquisition" trigger argument. - * @chanlist: Pointer to array of "chanspec" values, containing a - * sequence of channel numbers packed with analog range - * index, etc. - * @chanlist_len: Number of channels in sequence. - * @data: Pointer to miscellaneous set-up data (not used). - * @data_len: Length of miscellaneous set-up data. - * - * This is used with the %COMEDI_CMD or %COMEDI_CMDTEST ioctl to set-up - * or validate an asynchronous acquisition command. The ioctl may modify - * the &struct comedi_cmd and copy it back to the caller. - * - * Optional command @flags values that can be ORed together... - * - * %CMDF_BOGUS - makes %COMEDI_CMD ioctl return error %EAGAIN instead of - * starting the command. - * - * %CMDF_PRIORITY - requests "hard real-time" processing (which is not - * supported in this version of COMEDI). - * - * %CMDF_WAKE_EOS - requests the command makes data available for reading - * after every "scan" period. - * - * %CMDF_WRITE - marks the command as being in the "write" (to device) - * direction. This does not need to be specified by the caller unless the - * subdevice supports commands in either direction. - * - * %CMDF_RAWDATA - prevents the command from "munging" the data between the - * COMEDI sample format and the raw hardware sample format. - * - * %CMDF_ROUND_NEAREST - requests timing periods to be rounded to nearest - * supported values. - * - * %CMDF_ROUND_DOWN - requests timing periods to be rounded down to supported - * values (frequencies rounded up). - * - * %CMDF_ROUND_UP - requests timing periods to be rounded up to supported - * values (frequencies rounded down). - * - * Trigger source values for @start_src, @scan_begin_src, @convert_src, - * @scan_end_src, and @stop_src... - * - * %TRIG_ANY - "all ones" value used to test which trigger sources are - * supported. - * - * %TRIG_INVALID - "all zeroes" value used to indicate that all requested - * trigger sources are invalid. - * - * %TRIG_NONE - never trigger (often used as a @stop_src value). - * - * %TRIG_NOW - trigger after '_arg' nanoseconds. - * - * %TRIG_FOLLOW - trigger follows another event. - * - * %TRIG_TIMER - trigger every '_arg' nanoseconds. - * - * %TRIG_COUNT - trigger when count '_arg' is reached. - * - * %TRIG_EXT - trigger on external signal specified by '_arg'. - * - * %TRIG_INT - trigger on internal, software trigger specified by '_arg'. - * - * %TRIG_OTHER - trigger on other, driver-defined signal specified by '_arg'. - */ -struct comedi_cmd { - unsigned int subdev; - unsigned int flags; - - unsigned int start_src; - unsigned int start_arg; - - unsigned int scan_begin_src; - unsigned int scan_begin_arg; - - unsigned int convert_src; - unsigned int convert_arg; - - unsigned int scan_end_src; - unsigned int scan_end_arg; - - unsigned int stop_src; - unsigned int stop_arg; - - unsigned int *chanlist; - unsigned int chanlist_len; - - short __user *data; - unsigned int data_len; -}; - -/** - * struct comedi_chaninfo - used to retrieve per-channel information - * @subdev: Subdevice index. - * @maxdata_list: Optional pointer to per-channel maximum data values. - * @flaglist: Optional pointer to per-channel flags. - * @rangelist: Optional pointer to per-channel range types. - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_CHANINFO ioctl to get per-channel information - * for the subdevice. Use of this requires knowledge of the number of channels - * and subdevice flags obtained using the %COMEDI_SUBDINFO ioctl. - * - * The @maxdata_list member must be %NULL unless the %SDF_MAXDATA subdevice - * flag is set. The @flaglist member must be %NULL unless the %SDF_FLAGS - * subdevice flag is set. The @rangelist member must be %NULL unless the - * %SDF_RANGETYPE subdevice flag is set. Otherwise, the arrays they point to - * must be at least as long as the number of channels. - */ -struct comedi_chaninfo { - unsigned int subdev; - unsigned int __user *maxdata_list; - unsigned int __user *flaglist; - unsigned int __user *rangelist; - unsigned int unused[4]; -}; - -/** - * struct comedi_rangeinfo - used to retrieve the range table for a channel - * @range_type: Encodes subdevice index (bits 27:24), channel index - * (bits 23:16) and range table length (bits 15:0). - * @range_ptr: Pointer to array of @struct comedi_krange to be filled - * in with the range table for the channel or subdevice. - * - * This is used with the %COMEDI_RANGEINFO ioctl to retrieve the range table - * for a specific channel (if the subdevice has the %SDF_RANGETYPE flag set to - * indicate that the range table depends on the channel), or for the subdevice - * as a whole (if the %SDF_RANGETYPE flag is clear, indicating the range table - * is shared by all channels). - * - * The @range_type value is an input to the ioctl and comes from a previous - * use of the %COMEDI_SUBDINFO ioctl (if the %SDF_RANGETYPE flag is clear), - * or the %COMEDI_CHANINFO ioctl (if the %SDF_RANGETYPE flag is set). - */ -struct comedi_rangeinfo { - unsigned int range_type; - void __user *range_ptr; -}; - -/** - * struct comedi_krange - describes a range in a range table - * @min: Minimum value in millionths (1e-6) of a unit. - * @max: Maximum value in millionths (1e-6) of a unit. - * @flags: Indicates the units (in bits 7:0) OR'ed with optional flags. - * - * A range table is associated with a single channel, or with all channels in a - * subdevice, and a list of one or more ranges. A %struct comedi_krange - * describes the physical range of units for one of those ranges. Sample - * values in COMEDI are unsigned from %0 up to some 'maxdata' value. The - * mapping from sample values to physical units is assumed to be nomimally - * linear (for the purpose of describing the range), with sample value %0 - * mapping to @min, and the 'maxdata' sample value mapping to @max. - * - * The currently defined units are %UNIT_volt (%0), %UNIT_mA (%1), and - * %UNIT_none (%2). The @min and @max values are the physical range multiplied - * by 1e6, so a @max value of %1000000 (with %UNIT_volt) represents a maximal - * value of 1 volt. - * - * The only defined flag value is %RF_EXTERNAL (%0x100), indicating that the - * range needs to be multiplied by an external reference. - */ -struct comedi_krange { - int min; - int max; - unsigned int flags; -}; - -/** - * struct comedi_subdinfo - used to retrieve information about a subdevice - * @type: Type of subdevice from &enum comedi_subdevice_type. - * @n_chan: Number of channels the subdevice supports. - * @subd_flags: A mixture of static and dynamic flags describing - * aspects of the subdevice and its current state. - * @timer_type: Timer type. Always set to %5 ("nanosecond timer"). - * @len_chanlist: Maximum length of a channel list if the subdevice - * supports asynchronous acquisition commands. - * @maxdata: Maximum sample value for all channels if the - * %SDF_MAXDATA subdevice flag is clear. - * @flags: Channel flags for all channels if the %SDF_FLAGS - * subdevice flag is clear. - * @range_type: The range type for all channels if the %SDF_RANGETYPE - * subdevice flag is clear. Encodes the subdevice index - * (bits 27:24), a dummy channel index %0 (bits 23:16), - * and the range table length (bits 15:0). - * @settling_time_0: Not used. - * @insn_bits_support: Set to %COMEDI_SUPPORTED if the subdevice supports the - * %INSN_BITS instruction, or to %COMEDI_UNSUPPORTED if it - * does not. - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_SUBDINFO ioctl which copies an array of - * &struct comedi_subdinfo back to user space, with one element per subdevice. - * Use of this requires knowledge of the number of subdevices obtained from - * the %COMEDI_DEVINFO ioctl. - * - * These are the @subd_flags values that may be ORed together... - * - * %SDF_BUSY - the subdevice is busy processing an asynchronous command or a - * synchronous instruction. - * - * %SDF_BUSY_OWNER - the subdevice is busy processing an asynchronous - * acquisition command started on the current file object (the file object - * issuing the %COMEDI_SUBDINFO ioctl). - * - * %SDF_LOCKED - the subdevice is locked by a %COMEDI_LOCK ioctl. - * - * %SDF_LOCK_OWNER - the subdevice is locked by a %COMEDI_LOCK ioctl from the - * current file object. - * - * %SDF_MAXDATA - maximum sample values are channel-specific. - * - * %SDF_FLAGS - channel flags are channel-specific. - * - * %SDF_RANGETYPE - range types are channel-specific. - * - * %SDF_PWM_COUNTER - PWM can switch off automatically. - * - * %SDF_PWM_HBRIDGE - or PWM is signed (H-bridge). - * - * %SDF_CMD - the subdevice supports asynchronous commands. - * - * %SDF_SOFT_CALIBRATED - the subdevice uses software calibration. - * - * %SDF_CMD_WRITE - the subdevice supports asynchronous commands in the output - * ("write") direction. - * - * %SDF_CMD_READ - the subdevice supports asynchronous commands in the input - * ("read") direction. - * - * %SDF_READABLE - the subdevice is readable (e.g. analog input). - * - * %SDF_WRITABLE (aliased as %SDF_WRITEABLE) - the subdevice is writable (e.g. - * analog output). - * - * %SDF_INTERNAL - the subdevice has no externally visible lines. - * - * %SDF_GROUND - the subdevice can use ground as an analog reference. - * - * %SDF_COMMON - the subdevice can use a common analog reference. - * - * %SDF_DIFF - the subdevice can use differential inputs (or outputs). - * - * %SDF_OTHER - the subdevice can use some other analog reference. - * - * %SDF_DITHER - the subdevice can do dithering. - * - * %SDF_DEGLITCH - the subdevice can do deglitching. - * - * %SDF_MMAP - this is never set. - * - * %SDF_RUNNING - an asynchronous command is still running. - * - * %SDF_LSAMPL - the subdevice uses "long" (32-bit) samples (for asynchronous - * command data). - * - * %SDF_PACKED - the subdevice packs several DIO samples into a single sample - * (for asynchronous command data). - * - * No "channel flags" (@flags) values are currently defined. - */ -struct comedi_subdinfo { - unsigned int type; - unsigned int n_chan; - unsigned int subd_flags; - unsigned int timer_type; - unsigned int len_chanlist; - unsigned int maxdata; - unsigned int flags; - unsigned int range_type; - unsigned int settling_time_0; - unsigned int insn_bits_support; - unsigned int unused[8]; -}; - -/** - * struct comedi_devinfo - used to retrieve information about a COMEDI device - * @version_code: COMEDI version code. - * @n_subdevs: Number of subdevices the device has. - * @driver_name: Null-terminated COMEDI driver name. - * @board_name: Null-terminated COMEDI board name. - * @read_subdevice: Index of the current "read" subdevice (%-1 if none). - * @write_subdevice: Index of the current "write" subdevice (%-1 if none). - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_DEVINFO ioctl to get basic information about - * the device. - */ -struct comedi_devinfo { - unsigned int version_code; - unsigned int n_subdevs; - char driver_name[COMEDI_NAMELEN]; - char board_name[COMEDI_NAMELEN]; - int read_subdevice; - int write_subdevice; - int unused[30]; -}; - -/** - * struct comedi_devconfig - used to configure a legacy COMEDI device - * @board_name: Null-terminated string specifying the type of board - * to configure. - * @options: An array of integer configuration options. - * - * This is used with the %COMEDI_DEVCONFIG ioctl to configure a "legacy" COMEDI - * device, such as an ISA card. Not all COMEDI drivers support this. Those - * that do either expect the specified board name to match one of a list of - * names registered with the COMEDI core, or expect the specified board name - * to match the COMEDI driver name itself. The configuration options are - * handled in a driver-specific manner. - */ -struct comedi_devconfig { - char board_name[COMEDI_NAMELEN]; - int options[COMEDI_NDEVCONFOPTS]; -}; - -/** - * struct comedi_bufconfig - used to set or get buffer size for a subdevice - * @subdevice: Subdevice index. - * @flags: Not used. - * @maximum_size: Maximum allowed buffer size. - * @size: Buffer size. - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_BUFCONFIG ioctl to get or configure the - * maximum buffer size and current buffer size for a COMEDI subdevice that - * supports asynchronous commands. If the subdevice does not support - * asynchronous commands, @maximum_size and @size are ignored and set to 0. - * - * On ioctl input, non-zero values of @maximum_size and @size specify a - * new maximum size and new current size (in bytes), respectively. These - * will by rounded up to a multiple of %PAGE_SIZE. Specifying a new maximum - * size requires admin capabilities. - * - * On ioctl output, @maximum_size and @size and set to the current maximum - * buffer size and current buffer size, respectively. - */ -struct comedi_bufconfig { - unsigned int subdevice; - unsigned int flags; - - unsigned int maximum_size; - unsigned int size; - - unsigned int unused[4]; -}; - -/** - * struct comedi_bufinfo - used to manipulate buffer position for a subdevice - * @subdevice: Subdevice index. - * @bytes_read: Specify amount to advance read position for an - * asynchronous command in the input ("read") direction. - * @buf_write_ptr: Current write position (index) within the buffer. - * @buf_read_ptr: Current read position (index) within the buffer. - * @buf_write_count: Total amount written, modulo 2^32. - * @buf_read_count: Total amount read, modulo 2^32. - * @bytes_written: Specify amount to advance write position for an - * asynchronous command in the output ("write") direction. - * @unused: Reserved for future use. - * - * This is used with the %COMEDI_BUFINFO ioctl to optionally advance the - * current read or write position in an asynchronous acquisition data buffer, - * and to get the current read and write positions in the buffer. - */ -struct comedi_bufinfo { - unsigned int subdevice; - unsigned int bytes_read; - - unsigned int buf_write_ptr; - unsigned int buf_read_ptr; - unsigned int buf_write_count; - unsigned int buf_read_count; - - unsigned int bytes_written; - - unsigned int unused[4]; -}; - -/* range stuff */ - -#define __RANGE(a, b) ((((a) & 0xffff) << 16) | ((b) & 0xffff)) - -#define RANGE_OFFSET(a) (((a) >> 16) & 0xffff) -#define RANGE_LENGTH(b) ((b) & 0xffff) - -#define RF_UNIT(flags) ((flags) & 0xff) -#define RF_EXTERNAL 0x100 - -#define UNIT_volt 0 -#define UNIT_mA 1 -#define UNIT_none 2 - -#define COMEDI_MIN_SPEED 0xffffffffu - -/**********************************************************/ -/* everything after this line is ALPHA */ -/**********************************************************/ - -/* - * 8254 specific configuration. - * - * It supports two config commands: - * - * 0 ID: INSN_CONFIG_SET_COUNTER_MODE - * 1 8254 Mode - * I8254_MODE0, I8254_MODE1, ..., I8254_MODE5 - * OR'ed with: - * I8254_BCD, I8254_BINARY - * - * 0 ID: INSN_CONFIG_8254_READ_STATUS - * 1 <-- Status byte returned here. - * B7 = Output - * B6 = NULL Count - * B5 - B0 Current mode. - */ - -enum i8254_mode { - I8254_MODE0 = (0 << 1), /* Interrupt on terminal count */ - I8254_MODE1 = (1 << 1), /* Hardware retriggerable one-shot */ - I8254_MODE2 = (2 << 1), /* Rate generator */ - I8254_MODE3 = (3 << 1), /* Square wave mode */ - I8254_MODE4 = (4 << 1), /* Software triggered strobe */ - /* Hardware triggered strobe (retriggerable) */ - I8254_MODE5 = (5 << 1), - /* Use binary-coded decimal instead of binary (pretty useless) */ - I8254_BCD = 1, - I8254_BINARY = 0 -}; - -/* *** BEGIN GLOBALLY-NAMED NI TERMINALS/SIGNALS *** */ - -/* - * Common National Instruments Terminal/Signal names. - * Some of these have no NI_ prefix as they are useful for non-NI hardware, such - * as those that utilize the PXI/RTSI trigger lines. - * - * NOTE ABOUT THE CHOICE OF NAMES HERE AND THE CAMELSCRIPT: - * The choice to use CamelScript and the exact names below is for - * maintainability, clarity, similarity to manufacturer's documentation, - * _and_ a mitigation for confusion that has plagued the use of these drivers - * for years! - * - * More detail: - * There have been significant confusions over the past many years for users - * when trying to understand how to connect to/from signals and terminals on - * NI hardware using comedi. The major reason for this is that the actual - * register values were exposed and required to be used by users. Several - * major reasons exist why this caused major confusion for users: - * 1) The register values are _NOT_ in user documentation, but rather in - * arcane locations, such as a few register programming manuals that are - * increasingly hard to find and the NI MHDDK (comments in example code). - * There is no one place to find the various valid values of the registers. - * 2) The register values are _NOT_ completely consistent. There is no way to - * gain any sense of intuition of which values, or even enums one should use - * for various registers. There was some attempt in prior use of comedi to - * name enums such that a user might know which enums should be used for - * varying purposes, but the end-user had to gain a knowledge of register - * values to correctly wield this approach. - * 3) The names for signals and registers found in the various register level - * programming manuals and vendor-provided documentation are _not_ even - * close to the same names that are in the end-user documentation. - * - * Similar, albeit less, confusion plagued NI's previous version of their own - * drivers. Earlier than 2003, NI greatly simplified the situation for users - * by releasing a new API that abstracted the names of signals/terminals to a - * common and intuitive set of names. - * - * The names below mirror the names chosen and well documented by NI. These - * names are exposed to the user via the comedilib user library. By keeping - * the names below, in spite of the use of CamelScript, maintenance will be - * greatly eased and confusion for users _and_ comedi developers will be - * greatly reduced. - */ - -/* - * Base of abstracted NI names. - * The first 16 bits of *_arg are reserved for channel selection. - * Since we only actually need the first 4 or 5 bits for all register values on - * NI select registers anyways, we'll identify all values >= (1<<15) as being an - * abstracted NI signal/terminal name. - * These values are also used/returned by INSN_DEVICE_CONFIG_TEST_ROUTE, - * INSN_DEVICE_CONFIG_CONNECT_ROUTE, INSN_DEVICE_CONFIG_DISCONNECT_ROUTE, - * and INSN_DEVICE_CONFIG_GET_ROUTES. - */ -#define NI_NAMES_BASE 0x8000u - -#define _TERM_N(base, n, x) ((base) + ((x) & ((n) - 1))) - -/* - * not necessarily all allowed 64 PFIs are valid--certainly not for all devices - */ -#define NI_PFI(x) _TERM_N(NI_NAMES_BASE, 64, x) -/* 8 trigger lines by standard, Some devices cannot talk to all eight. */ -#define TRIGGER_LINE(x) _TERM_N(NI_PFI(-1) + 1, 8, x) -/* 4 RTSI shared MUXes to route signals to/from TRIGGER_LINES on NI hardware */ -#define NI_RTSI_BRD(x) _TERM_N(TRIGGER_LINE(-1) + 1, 4, x) - -/* *** Counter/timer names : 8 counters max *** */ -#define NI_MAX_COUNTERS 8 -#define NI_COUNTER_NAMES_BASE (NI_RTSI_BRD(-1) + 1) -#define NI_CtrSource(x) _TERM_N(NI_COUNTER_NAMES_BASE, NI_MAX_COUNTERS, x) -/* Gate, Aux, A,B,Z are all treated, at times as gates */ -#define NI_GATES_NAMES_BASE (NI_CtrSource(-1) + 1) -#define NI_CtrGate(x) _TERM_N(NI_GATES_NAMES_BASE, NI_MAX_COUNTERS, x) -#define NI_CtrAux(x) _TERM_N(NI_CtrGate(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_CtrA(x) _TERM_N(NI_CtrAux(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_CtrB(x) _TERM_N(NI_CtrA(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_CtrZ(x) _TERM_N(NI_CtrB(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_GATES_NAMES_MAX NI_CtrZ(-1) -#define NI_CtrArmStartTrigger(x) _TERM_N(NI_CtrZ(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_CtrInternalOutput(x) \ - _TERM_N(NI_CtrArmStartTrigger(-1) + 1, NI_MAX_COUNTERS, x) -/** external pin(s) labeled conveniently as CtrOut. */ -#define NI_CtrOut(x) _TERM_N(NI_CtrInternalOutput(-1) + 1, NI_MAX_COUNTERS, x) -/** For Buffered sampling of ctr -- x series capability. */ -#define NI_CtrSampleClock(x) _TERM_N(NI_CtrOut(-1) + 1, NI_MAX_COUNTERS, x) -#define NI_COUNTER_NAMES_MAX NI_CtrSampleClock(-1) - -enum ni_common_signal_names { - /* PXI_Star: this is a non-NI-specific signal */ - PXI_Star = NI_COUNTER_NAMES_MAX + 1, - PXI_Clk10, - PXIe_Clk100, - NI_AI_SampleClock, - NI_AI_SampleClockTimebase, - NI_AI_StartTrigger, - NI_AI_ReferenceTrigger, - NI_AI_ConvertClock, - NI_AI_ConvertClockTimebase, - NI_AI_PauseTrigger, - NI_AI_HoldCompleteEvent, - NI_AI_HoldComplete, - NI_AI_ExternalMUXClock, - NI_AI_STOP, /* pulse signal that occurs when a update is finished(?) */ - NI_AO_SampleClock, - NI_AO_SampleClockTimebase, - NI_AO_StartTrigger, - NI_AO_PauseTrigger, - NI_DI_SampleClock, - NI_DI_SampleClockTimebase, - NI_DI_StartTrigger, - NI_DI_ReferenceTrigger, - NI_DI_PauseTrigger, - NI_DI_InputBufferFull, - NI_DI_ReadyForStartEvent, - NI_DI_ReadyForTransferEventBurst, - NI_DI_ReadyForTransferEventPipelined, - NI_DO_SampleClock, - NI_DO_SampleClockTimebase, - NI_DO_StartTrigger, - NI_DO_PauseTrigger, - NI_DO_OutputBufferFull, - NI_DO_DataActiveEvent, - NI_DO_ReadyForStartEvent, - NI_DO_ReadyForTransferEvent, - NI_MasterTimebase, - NI_20MHzTimebase, - NI_80MHzTimebase, - NI_100MHzTimebase, - NI_200MHzTimebase, - NI_100kHzTimebase, - NI_10MHzRefClock, - NI_FrequencyOutput, - NI_ChangeDetectionEvent, - NI_AnalogComparisonEvent, - NI_WatchdogExpiredEvent, - NI_WatchdogExpirationTrigger, - NI_SCXI_Trig1, - NI_LogicLow, - NI_LogicHigh, - NI_ExternalStrobe, - NI_PFI_DO, - NI_CaseGround, - /* special internal signal used as variable source for RTSI bus: */ - NI_RGOUT0, - - /* just a name to make the next more convenient, regardless of above */ - _NI_NAMES_MAX_PLUS_1, - NI_NUM_NAMES = _NI_NAMES_MAX_PLUS_1 - NI_NAMES_BASE, -}; - -/* *** END GLOBALLY-NAMED NI TERMINALS/SIGNALS *** */ - -#define NI_USUAL_PFI_SELECT(x) (((x) < 10) ? (0x1 + (x)) : (0xb + (x))) -#define NI_USUAL_RTSI_SELECT(x) (((x) < 7) ? (0xb + (x)) : 0x1b) - -/* - * mode bits for NI general-purpose counters, set with - * INSN_CONFIG_SET_COUNTER_MODE - */ -#define NI_GPCT_COUNTING_MODE_SHIFT 16 -#define NI_GPCT_INDEX_PHASE_BITSHIFT 20 -#define NI_GPCT_COUNTING_DIRECTION_SHIFT 24 -enum ni_gpct_mode_bits { - NI_GPCT_GATE_ON_BOTH_EDGES_BIT = 0x4, - NI_GPCT_EDGE_GATE_MODE_MASK = 0x18, - NI_GPCT_EDGE_GATE_STARTS_STOPS_BITS = 0x0, - NI_GPCT_EDGE_GATE_STOPS_STARTS_BITS = 0x8, - NI_GPCT_EDGE_GATE_STARTS_BITS = 0x10, - NI_GPCT_EDGE_GATE_NO_STARTS_NO_STOPS_BITS = 0x18, - NI_GPCT_STOP_MODE_MASK = 0x60, - NI_GPCT_STOP_ON_GATE_BITS = 0x00, - NI_GPCT_STOP_ON_GATE_OR_TC_BITS = 0x20, - NI_GPCT_STOP_ON_GATE_OR_SECOND_TC_BITS = 0x40, - NI_GPCT_LOAD_B_SELECT_BIT = 0x80, - NI_GPCT_OUTPUT_MODE_MASK = 0x300, - NI_GPCT_OUTPUT_TC_PULSE_BITS = 0x100, - NI_GPCT_OUTPUT_TC_TOGGLE_BITS = 0x200, - NI_GPCT_OUTPUT_TC_OR_GATE_TOGGLE_BITS = 0x300, - NI_GPCT_HARDWARE_DISARM_MASK = 0xc00, - NI_GPCT_NO_HARDWARE_DISARM_BITS = 0x000, - NI_GPCT_DISARM_AT_TC_BITS = 0x400, - NI_GPCT_DISARM_AT_GATE_BITS = 0x800, - NI_GPCT_DISARM_AT_TC_OR_GATE_BITS = 0xc00, - NI_GPCT_LOADING_ON_TC_BIT = 0x1000, - NI_GPCT_LOADING_ON_GATE_BIT = 0x4000, - NI_GPCT_COUNTING_MODE_MASK = 0x7 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_NORMAL_BITS = - 0x0 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_QUADRATURE_X1_BITS = - 0x1 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_QUADRATURE_X2_BITS = - 0x2 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_QUADRATURE_X4_BITS = - 0x3 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_TWO_PULSE_BITS = - 0x4 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_COUNTING_MODE_SYNC_SOURCE_BITS = - 0x6 << NI_GPCT_COUNTING_MODE_SHIFT, - NI_GPCT_INDEX_PHASE_MASK = 0x3 << NI_GPCT_INDEX_PHASE_BITSHIFT, - NI_GPCT_INDEX_PHASE_LOW_A_LOW_B_BITS = - 0x0 << NI_GPCT_INDEX_PHASE_BITSHIFT, - NI_GPCT_INDEX_PHASE_LOW_A_HIGH_B_BITS = - 0x1 << NI_GPCT_INDEX_PHASE_BITSHIFT, - NI_GPCT_INDEX_PHASE_HIGH_A_LOW_B_BITS = - 0x2 << NI_GPCT_INDEX_PHASE_BITSHIFT, - NI_GPCT_INDEX_PHASE_HIGH_A_HIGH_B_BITS = - 0x3 << NI_GPCT_INDEX_PHASE_BITSHIFT, - NI_GPCT_INDEX_ENABLE_BIT = 0x400000, - NI_GPCT_COUNTING_DIRECTION_MASK = - 0x3 << NI_GPCT_COUNTING_DIRECTION_SHIFT, - NI_GPCT_COUNTING_DIRECTION_DOWN_BITS = - 0x00 << NI_GPCT_COUNTING_DIRECTION_SHIFT, - NI_GPCT_COUNTING_DIRECTION_UP_BITS = - 0x1 << NI_GPCT_COUNTING_DIRECTION_SHIFT, - NI_GPCT_COUNTING_DIRECTION_HW_UP_DOWN_BITS = - 0x2 << NI_GPCT_COUNTING_DIRECTION_SHIFT, - NI_GPCT_COUNTING_DIRECTION_HW_GATE_BITS = - 0x3 << NI_GPCT_COUNTING_DIRECTION_SHIFT, - NI_GPCT_RELOAD_SOURCE_MASK = 0xc000000, - NI_GPCT_RELOAD_SOURCE_FIXED_BITS = 0x0, - NI_GPCT_RELOAD_SOURCE_SWITCHING_BITS = 0x4000000, - NI_GPCT_RELOAD_SOURCE_GATE_SELECT_BITS = 0x8000000, - NI_GPCT_OR_GATE_BIT = 0x10000000, - NI_GPCT_INVERT_OUTPUT_BIT = 0x20000000 -}; - -/* - * Bits for setting a clock source with - * INSN_CONFIG_SET_CLOCK_SRC when using NI general-purpose counters. - */ -enum ni_gpct_clock_source_bits { - NI_GPCT_CLOCK_SRC_SELECT_MASK = 0x3f, - NI_GPCT_TIMEBASE_1_CLOCK_SRC_BITS = 0x0, - NI_GPCT_TIMEBASE_2_CLOCK_SRC_BITS = 0x1, - NI_GPCT_TIMEBASE_3_CLOCK_SRC_BITS = 0x2, - NI_GPCT_LOGIC_LOW_CLOCK_SRC_BITS = 0x3, - NI_GPCT_NEXT_GATE_CLOCK_SRC_BITS = 0x4, - NI_GPCT_NEXT_TC_CLOCK_SRC_BITS = 0x5, - /* NI 660x-specific */ - NI_GPCT_SOURCE_PIN_i_CLOCK_SRC_BITS = 0x6, - NI_GPCT_PXI10_CLOCK_SRC_BITS = 0x7, - NI_GPCT_PXI_STAR_TRIGGER_CLOCK_SRC_BITS = 0x8, - NI_GPCT_ANALOG_TRIGGER_OUT_CLOCK_SRC_BITS = 0x9, - NI_GPCT_PRESCALE_MODE_CLOCK_SRC_MASK = 0x30000000, - NI_GPCT_NO_PRESCALE_CLOCK_SRC_BITS = 0x0, - /* divide source by 2 */ - NI_GPCT_PRESCALE_X2_CLOCK_SRC_BITS = 0x10000000, - /* divide source by 8 */ - NI_GPCT_PRESCALE_X8_CLOCK_SRC_BITS = 0x20000000, - NI_GPCT_INVERT_CLOCK_SRC_BIT = 0x80000000 -}; - -/* NI 660x-specific */ -#define NI_GPCT_SOURCE_PIN_CLOCK_SRC_BITS(x) (0x10 + (x)) - -#define NI_GPCT_RTSI_CLOCK_SRC_BITS(x) (0x18 + (x)) - -/* no pfi on NI 660x */ -#define NI_GPCT_PFI_CLOCK_SRC_BITS(x) (0x20 + (x)) - -/* - * Possibilities for setting a gate source with - * INSN_CONFIG_SET_GATE_SRC when using NI general-purpose counters. - * May be bitwise-or'd with CR_EDGE or CR_INVERT. - */ -enum ni_gpct_gate_select { - /* m-series gates */ - NI_GPCT_TIMESTAMP_MUX_GATE_SELECT = 0x0, - NI_GPCT_AI_START2_GATE_SELECT = 0x12, - NI_GPCT_PXI_STAR_TRIGGER_GATE_SELECT = 0x13, - NI_GPCT_NEXT_OUT_GATE_SELECT = 0x14, - NI_GPCT_AI_START1_GATE_SELECT = 0x1c, - NI_GPCT_NEXT_SOURCE_GATE_SELECT = 0x1d, - NI_GPCT_ANALOG_TRIGGER_OUT_GATE_SELECT = 0x1e, - NI_GPCT_LOGIC_LOW_GATE_SELECT = 0x1f, - /* more gates for 660x */ - NI_GPCT_SOURCE_PIN_i_GATE_SELECT = 0x100, - NI_GPCT_GATE_PIN_i_GATE_SELECT = 0x101, - /* more gates for 660x "second gate" */ - NI_GPCT_UP_DOWN_PIN_i_GATE_SELECT = 0x201, - NI_GPCT_SELECTED_GATE_GATE_SELECT = 0x21e, - /* - * m-series "second gate" sources are unknown, - * we should add them here with an offset of 0x300 when - * known. - */ - NI_GPCT_DISABLED_GATE_SELECT = 0x8000, -}; - -#define NI_GPCT_GATE_PIN_GATE_SELECT(x) (0x102 + (x)) -#define NI_GPCT_RTSI_GATE_SELECT(x) NI_USUAL_RTSI_SELECT(x) -#define NI_GPCT_PFI_GATE_SELECT(x) NI_USUAL_PFI_SELECT(x) -#define NI_GPCT_UP_DOWN_PIN_GATE_SELECT(x) (0x202 + (x)) - -/* - * Possibilities for setting a source with - * INSN_CONFIG_SET_OTHER_SRC when using NI general-purpose counters. - */ -enum ni_gpct_other_index { - NI_GPCT_SOURCE_ENCODER_A, - NI_GPCT_SOURCE_ENCODER_B, - NI_GPCT_SOURCE_ENCODER_Z -}; - -enum ni_gpct_other_select { - /* m-series gates */ - /* Still unknown, probably only need NI_GPCT_PFI_OTHER_SELECT */ - NI_GPCT_DISABLED_OTHER_SELECT = 0x8000, -}; - -#define NI_GPCT_PFI_OTHER_SELECT(x) NI_USUAL_PFI_SELECT(x) - -/* - * start sources for ni general-purpose counters for use with - * INSN_CONFIG_ARM - */ -enum ni_gpct_arm_source { - NI_GPCT_ARM_IMMEDIATE = 0x0, - /* - * Start both the counter and the adjacent paired counter simultaneously - */ - NI_GPCT_ARM_PAIRED_IMMEDIATE = 0x1, - /* - * If the NI_GPCT_HW_ARM bit is set, we will pass the least significant - * bits (3 bits for 660x or 5 bits for m-series) through to the - * hardware. To select a hardware trigger, pass the appropriate select - * bit, e.g., - * NI_GPCT_HW_ARM | NI_GPCT_AI_START1_GATE_SELECT or - * NI_GPCT_HW_ARM | NI_GPCT_PFI_GATE_SELECT(pfi_number) - */ - NI_GPCT_HW_ARM = 0x1000, - NI_GPCT_ARM_UNKNOWN = NI_GPCT_HW_ARM, /* for backward compatibility */ -}; - -/* digital filtering options for ni 660x for use with INSN_CONFIG_FILTER. */ -enum ni_gpct_filter_select { - NI_GPCT_FILTER_OFF = 0x0, - NI_GPCT_FILTER_TIMEBASE_3_SYNC = 0x1, - NI_GPCT_FILTER_100x_TIMEBASE_1 = 0x2, - NI_GPCT_FILTER_20x_TIMEBASE_1 = 0x3, - NI_GPCT_FILTER_10x_TIMEBASE_1 = 0x4, - NI_GPCT_FILTER_2x_TIMEBASE_1 = 0x5, - NI_GPCT_FILTER_2x_TIMEBASE_3 = 0x6 -}; - -/* - * PFI digital filtering options for ni m-series for use with - * INSN_CONFIG_FILTER. - */ -enum ni_pfi_filter_select { - NI_PFI_FILTER_OFF = 0x0, - NI_PFI_FILTER_125ns = 0x1, - NI_PFI_FILTER_6425ns = 0x2, - NI_PFI_FILTER_2550us = 0x3 -}; - -/* master clock sources for ni mio boards and INSN_CONFIG_SET_CLOCK_SRC */ -enum ni_mio_clock_source { - NI_MIO_INTERNAL_CLOCK = 0, - /* - * Doesn't work for m-series, use NI_MIO_PLL_RTSI_CLOCK() - * the NI_MIO_PLL_* sources are m-series only - */ - NI_MIO_RTSI_CLOCK = 1, - NI_MIO_PLL_PXI_STAR_TRIGGER_CLOCK = 2, - NI_MIO_PLL_PXI10_CLOCK = 3, - NI_MIO_PLL_RTSI0_CLOCK = 4 -}; - -#define NI_MIO_PLL_RTSI_CLOCK(x) (NI_MIO_PLL_RTSI0_CLOCK + (x)) - -/* - * Signals which can be routed to an NI RTSI pin with INSN_CONFIG_SET_ROUTING. - * The numbers assigned are not arbitrary, they correspond to the bits required - * to program the board. - */ -enum ni_rtsi_routing { - NI_RTSI_OUTPUT_ADR_START1 = 0, - NI_RTSI_OUTPUT_ADR_START2 = 1, - NI_RTSI_OUTPUT_SCLKG = 2, - NI_RTSI_OUTPUT_DACUPDN = 3, - NI_RTSI_OUTPUT_DA_START1 = 4, - NI_RTSI_OUTPUT_G_SRC0 = 5, - NI_RTSI_OUTPUT_G_GATE0 = 6, - NI_RTSI_OUTPUT_RGOUT0 = 7, - NI_RTSI_OUTPUT_RTSI_BRD_0 = 8, - /* Pre-m-series always have RTSI clock on line 7 */ - NI_RTSI_OUTPUT_RTSI_OSC = 12 -}; - -#define NI_RTSI_OUTPUT_RTSI_BRD(x) (NI_RTSI_OUTPUT_RTSI_BRD_0 + (x)) - -/* - * Signals which can be routed to an NI PFI pin on an m-series board with - * INSN_CONFIG_SET_ROUTING. These numbers are also returned by - * INSN_CONFIG_GET_ROUTING on pre-m-series boards, even though their routing - * cannot be changed. The numbers assigned are not arbitrary, they correspond - * to the bits required to program the board. - */ -enum ni_pfi_routing { - NI_PFI_OUTPUT_PFI_DEFAULT = 0, - NI_PFI_OUTPUT_AI_START1 = 1, - NI_PFI_OUTPUT_AI_START2 = 2, - NI_PFI_OUTPUT_AI_CONVERT = 3, - NI_PFI_OUTPUT_G_SRC1 = 4, - NI_PFI_OUTPUT_G_GATE1 = 5, - NI_PFI_OUTPUT_AO_UPDATE_N = 6, - NI_PFI_OUTPUT_AO_START1 = 7, - NI_PFI_OUTPUT_AI_START_PULSE = 8, - NI_PFI_OUTPUT_G_SRC0 = 9, - NI_PFI_OUTPUT_G_GATE0 = 10, - NI_PFI_OUTPUT_EXT_STROBE = 11, - NI_PFI_OUTPUT_AI_EXT_MUX_CLK = 12, - NI_PFI_OUTPUT_GOUT0 = 13, - NI_PFI_OUTPUT_GOUT1 = 14, - NI_PFI_OUTPUT_FREQ_OUT = 15, - NI_PFI_OUTPUT_PFI_DO = 16, - NI_PFI_OUTPUT_I_ATRIG = 17, - NI_PFI_OUTPUT_RTSI0 = 18, - NI_PFI_OUTPUT_PXI_STAR_TRIGGER_IN = 26, - NI_PFI_OUTPUT_SCXI_TRIG1 = 27, - NI_PFI_OUTPUT_DIO_CHANGE_DETECT_RTSI = 28, - NI_PFI_OUTPUT_CDI_SAMPLE = 29, - NI_PFI_OUTPUT_CDO_UPDATE = 30 -}; - -#define NI_PFI_OUTPUT_RTSI(x) (NI_PFI_OUTPUT_RTSI0 + (x)) - -/* - * Signals which can be routed to output on a NI PFI pin on a 660x board - * with INSN_CONFIG_SET_ROUTING. The numbers assigned are - * not arbitrary, they correspond to the bits required - * to program the board. Lines 0 to 7 can only be set to - * NI_660X_PFI_OUTPUT_DIO. Lines 32 to 39 can only be set to - * NI_660X_PFI_OUTPUT_COUNTER. - */ -enum ni_660x_pfi_routing { - NI_660X_PFI_OUTPUT_COUNTER = 1, /* counter */ - NI_660X_PFI_OUTPUT_DIO = 2, /* static digital output */ -}; - -/* - * NI External Trigger lines. These values are not arbitrary, but are related - * to the bits required to program the board (offset by 1 for historical - * reasons). - */ -#define NI_EXT_PFI(x) (NI_USUAL_PFI_SELECT(x) - 1) -#define NI_EXT_RTSI(x) (NI_USUAL_RTSI_SELECT(x) - 1) - -/* - * Clock sources for CDIO subdevice on NI m-series boards. Used as the - * scan_begin_arg for a comedi_command. These sources may also be bitwise-or'd - * with CR_INVERT to change polarity. - */ -enum ni_m_series_cdio_scan_begin_src { - NI_CDIO_SCAN_BEGIN_SRC_GROUND = 0, - NI_CDIO_SCAN_BEGIN_SRC_AI_START = 18, - NI_CDIO_SCAN_BEGIN_SRC_AI_CONVERT = 19, - NI_CDIO_SCAN_BEGIN_SRC_PXI_STAR_TRIGGER = 20, - NI_CDIO_SCAN_BEGIN_SRC_G0_OUT = 28, - NI_CDIO_SCAN_BEGIN_SRC_G1_OUT = 29, - NI_CDIO_SCAN_BEGIN_SRC_ANALOG_TRIGGER = 30, - NI_CDIO_SCAN_BEGIN_SRC_AO_UPDATE = 31, - NI_CDIO_SCAN_BEGIN_SRC_FREQ_OUT = 32, - NI_CDIO_SCAN_BEGIN_SRC_DIO_CHANGE_DETECT_IRQ = 33 -}; - -#define NI_CDIO_SCAN_BEGIN_SRC_PFI(x) NI_USUAL_PFI_SELECT(x) -#define NI_CDIO_SCAN_BEGIN_SRC_RTSI(x) NI_USUAL_RTSI_SELECT(x) - -/* - * scan_begin_src for scan_begin_arg==TRIG_EXT with analog output command on NI - * boards. These scan begin sources can also be bitwise-or'd with CR_INVERT to - * change polarity. - */ -#define NI_AO_SCAN_BEGIN_SRC_PFI(x) NI_USUAL_PFI_SELECT(x) -#define NI_AO_SCAN_BEGIN_SRC_RTSI(x) NI_USUAL_RTSI_SELECT(x) - -/* - * Bits for setting a clock source with - * INSN_CONFIG_SET_CLOCK_SRC when using NI frequency output subdevice. - */ -enum ni_freq_out_clock_source_bits { - NI_FREQ_OUT_TIMEBASE_1_DIV_2_CLOCK_SRC, /* 10 MHz */ - NI_FREQ_OUT_TIMEBASE_2_CLOCK_SRC /* 100 KHz */ -}; - -/* - * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for - * 8254 counter subdevices on Amplicon DIO boards (amplc_dio200 driver). - */ -enum amplc_dio_clock_source { - /* - * Per channel external clock - * input/output pin (pin is only an - * input when clock source set to this value, - * otherwise it is an output) - */ - AMPLC_DIO_CLK_CLKN, - AMPLC_DIO_CLK_10MHZ, /* 10 MHz internal clock */ - AMPLC_DIO_CLK_1MHZ, /* 1 MHz internal clock */ - AMPLC_DIO_CLK_100KHZ, /* 100 kHz internal clock */ - AMPLC_DIO_CLK_10KHZ, /* 10 kHz internal clock */ - AMPLC_DIO_CLK_1KHZ, /* 1 kHz internal clock */ - /* - * Output of preceding counter channel - * (for channel 0, preceding counter - * channel is channel 2 on preceding - * counter subdevice, for first counter - * subdevice, preceding counter - * subdevice is the last counter - * subdevice) - */ - AMPLC_DIO_CLK_OUTNM1, - AMPLC_DIO_CLK_EXT, /* per chip external input pin */ - /* the following are "enhanced" clock sources for PCIe models */ - AMPLC_DIO_CLK_VCC, /* clock input HIGH */ - AMPLC_DIO_CLK_GND, /* clock input LOW */ - AMPLC_DIO_CLK_PAT_PRESENT, /* "pattern present" signal */ - AMPLC_DIO_CLK_20MHZ /* 20 MHz internal clock */ -}; - -/* - * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for - * timer subdevice on some Amplicon DIO PCIe boards (amplc_dio200 driver). - */ -enum amplc_dio_ts_clock_src { - AMPLC_DIO_TS_CLK_1GHZ, /* 1 ns period with 20 ns granularity */ - AMPLC_DIO_TS_CLK_1MHZ, /* 1 us period */ - AMPLC_DIO_TS_CLK_1KHZ /* 1 ms period */ -}; - -/* - * Values for setting a gate source with INSN_CONFIG_SET_GATE_SRC for - * 8254 counter subdevices on Amplicon DIO boards (amplc_dio200 driver). - */ -enum amplc_dio_gate_source { - AMPLC_DIO_GAT_VCC, /* internal high logic level */ - AMPLC_DIO_GAT_GND, /* internal low logic level */ - AMPLC_DIO_GAT_GATN, /* per channel external gate input */ - /* - * negated output of counter channel minus 2 - * (for channels 0 or 1, channel minus 2 is channel 1 or 2 on - * the preceding counter subdevice, for the first counter subdevice - * the preceding counter subdevice is the last counter subdevice) - */ - AMPLC_DIO_GAT_NOUTNM2, - AMPLC_DIO_GAT_RESERVED4, - AMPLC_DIO_GAT_RESERVED5, - AMPLC_DIO_GAT_RESERVED6, - AMPLC_DIO_GAT_RESERVED7, - /* the following are "enhanced" gate sources for PCIe models */ - AMPLC_DIO_GAT_NGATN = 6, /* negated per channel gate input */ - /* non-negated output of counter channel minus 2 */ - AMPLC_DIO_GAT_OUTNM2, - AMPLC_DIO_GAT_PAT_PRESENT, /* "pattern present" signal */ - AMPLC_DIO_GAT_PAT_OCCURRED, /* "pattern occurred" latched */ - AMPLC_DIO_GAT_PAT_GONE, /* "pattern gone away" latched */ - AMPLC_DIO_GAT_NPAT_PRESENT, /* negated "pattern present" */ - AMPLC_DIO_GAT_NPAT_OCCURRED, /* negated "pattern occurred" */ - AMPLC_DIO_GAT_NPAT_GONE /* negated "pattern gone away" */ -}; - -/* - * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for - * the counter subdevice on the Kolter Electronic PCI-Counter board - * (ke_counter driver). - */ -enum ke_counter_clock_source { - KE_CLK_20MHZ, /* internal 20MHz (default) */ - KE_CLK_4MHZ, /* internal 4MHz (option) */ - KE_CLK_EXT /* external clock on pin 21 of D-Sub */ -}; - -#endif /* _COMEDI_H */ diff --git a/drivers/comedi/comedi_buf.c b/drivers/comedi/comedi_buf.c index 06bfc859ab31..393966c09740 100644 --- a/drivers/comedi/comedi_buf.c +++ b/drivers/comedi/comedi_buf.c @@ -9,8 +9,7 @@ #include #include - -#include "comedidev.h" +#include #include "comedi_internal.h" #ifdef PAGE_KERNEL_NOCACHE diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index 763cea8418f8..55a0cae04b8d 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -23,7 +23,7 @@ #include #include #include -#include "comedidev.h" +#include #include #include diff --git a/drivers/comedi/comedi_pci.c b/drivers/comedi/comedi_pci.c index 54739af7eb71..cc2581902195 100644 --- a/drivers/comedi/comedi_pci.c +++ b/drivers/comedi/comedi_pci.c @@ -9,8 +9,7 @@ #include #include - -#include "comedi_pci.h" +#include /** * comedi_to_pci_dev() - Return PCI device attached to COMEDI device diff --git a/drivers/comedi/comedi_pci.h b/drivers/comedi/comedi_pci.h deleted file mode 100644 index 4e069440cbdc..000000000000 --- a/drivers/comedi/comedi_pci.h +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * comedi_pci.h - * header file for Comedi PCI drivers - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1997-2000 David A. Schleef - */ - -#ifndef _COMEDI_PCI_H -#define _COMEDI_PCI_H - -#include - -#include "comedidev.h" - -/* - * PCI Vendor IDs not in - */ -#define PCI_VENDOR_ID_KOLTER 0x1001 -#define PCI_VENDOR_ID_ICP 0x104c -#define PCI_VENDOR_ID_DT 0x1116 -#define PCI_VENDOR_ID_IOTECH 0x1616 -#define PCI_VENDOR_ID_CONTEC 0x1221 -#define PCI_VENDOR_ID_RTD 0x1435 -#define PCI_VENDOR_ID_HUMUSOFT 0x186c - -struct pci_dev *comedi_to_pci_dev(struct comedi_device *dev); - -int comedi_pci_enable(struct comedi_device *dev); -void comedi_pci_disable(struct comedi_device *dev); -void comedi_pci_detach(struct comedi_device *dev); - -int comedi_pci_auto_config(struct pci_dev *pcidev, struct comedi_driver *driver, - unsigned long context); -void comedi_pci_auto_unconfig(struct pci_dev *pcidev); - -int comedi_pci_driver_register(struct comedi_driver *comedi_driver, - struct pci_driver *pci_driver); -void comedi_pci_driver_unregister(struct comedi_driver *comedi_driver, - struct pci_driver *pci_driver); - -/** - * module_comedi_pci_driver() - Helper macro for registering a comedi PCI driver - * @__comedi_driver: comedi_driver struct - * @__pci_driver: pci_driver struct - * - * Helper macro for comedi PCI drivers which do not do anything special - * in module init/exit. This eliminates a lot of boilerplate. Each - * module may only use this macro once, and calling it replaces - * module_init() and module_exit() - */ -#define module_comedi_pci_driver(__comedi_driver, __pci_driver) \ - module_driver(__comedi_driver, comedi_pci_driver_register, \ - comedi_pci_driver_unregister, &(__pci_driver)) - -#endif /* _COMEDI_PCI_H */ diff --git a/drivers/comedi/comedi_pcmcia.c b/drivers/comedi/comedi_pcmcia.c index bb273bb202e6..c53aad0fc2ce 100644 --- a/drivers/comedi/comedi_pcmcia.c +++ b/drivers/comedi/comedi_pcmcia.c @@ -9,8 +9,7 @@ #include #include - -#include "comedi_pcmcia.h" +#include /** * comedi_to_pcmcia_dev() - Return PCMCIA device attached to COMEDI device diff --git a/drivers/comedi/comedi_pcmcia.h b/drivers/comedi/comedi_pcmcia.h deleted file mode 100644 index f2f6e779645b..000000000000 --- a/drivers/comedi/comedi_pcmcia.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * comedi_pcmcia.h - * header file for Comedi PCMCIA drivers - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1997-2000 David A. Schleef - */ - -#ifndef _COMEDI_PCMCIA_H -#define _COMEDI_PCMCIA_H - -#include -#include - -#include "comedidev.h" - -struct pcmcia_device *comedi_to_pcmcia_dev(struct comedi_device *dev); - -int comedi_pcmcia_enable(struct comedi_device *dev, - int (*conf_check)(struct pcmcia_device *p_dev, - void *priv_data)); -void comedi_pcmcia_disable(struct comedi_device *dev); - -int comedi_pcmcia_auto_config(struct pcmcia_device *link, - struct comedi_driver *driver); -void comedi_pcmcia_auto_unconfig(struct pcmcia_device *link); - -int comedi_pcmcia_driver_register(struct comedi_driver *comedi_driver, - struct pcmcia_driver *pcmcia_driver); -void comedi_pcmcia_driver_unregister(struct comedi_driver *comedi_driver, - struct pcmcia_driver *pcmcia_driver); - -/** - * module_comedi_pcmcia_driver() - Helper macro for registering a comedi - * PCMCIA driver - * @__comedi_driver: comedi_driver struct - * @__pcmcia_driver: pcmcia_driver struct - * - * Helper macro for comedi PCMCIA drivers which do not do anything special - * in module init/exit. This eliminates a lot of boilerplate. Each - * module may only use this macro once, and calling it replaces - * module_init() and module_exit() - */ -#define module_comedi_pcmcia_driver(__comedi_driver, __pcmcia_driver) \ - module_driver(__comedi_driver, comedi_pcmcia_driver_register, \ - comedi_pcmcia_driver_unregister, &(__pcmcia_driver)) - -#endif /* _COMEDI_PCMCIA_H */ diff --git a/drivers/comedi/comedi_usb.c b/drivers/comedi/comedi_usb.c index eea8ebf32ed0..d11ea148ebf8 100644 --- a/drivers/comedi/comedi_usb.c +++ b/drivers/comedi/comedi_usb.c @@ -8,8 +8,7 @@ */ #include - -#include "comedi_usb.h" +#include /** * comedi_to_usb_interface() - Return USB interface attached to COMEDI device diff --git a/drivers/comedi/comedi_usb.h b/drivers/comedi/comedi_usb.h deleted file mode 100644 index 601e29d3891c..000000000000 --- a/drivers/comedi/comedi_usb.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* comedi_usb.h - * header file for USB Comedi drivers - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1997-2000 David A. Schleef - */ - -#ifndef _COMEDI_USB_H -#define _COMEDI_USB_H - -#include - -#include "comedidev.h" - -struct usb_interface *comedi_to_usb_interface(struct comedi_device *dev); -struct usb_device *comedi_to_usb_dev(struct comedi_device *dev); - -int comedi_usb_auto_config(struct usb_interface *intf, - struct comedi_driver *driver, unsigned long context); -void comedi_usb_auto_unconfig(struct usb_interface *intf); - -int comedi_usb_driver_register(struct comedi_driver *comedi_driver, - struct usb_driver *usb_driver); -void comedi_usb_driver_unregister(struct comedi_driver *comedi_driver, - struct usb_driver *usb_driver); - -/** - * module_comedi_usb_driver() - Helper macro for registering a comedi USB driver - * @__comedi_driver: comedi_driver struct - * @__usb_driver: usb_driver struct - * - * Helper macro for comedi USB drivers which do not do anything special - * in module init/exit. This eliminates a lot of boilerplate. Each - * module may only use this macro once, and calling it replaces - * module_init() and module_exit() - */ -#define module_comedi_usb_driver(__comedi_driver, __usb_driver) \ - module_driver(__comedi_driver, comedi_usb_driver_register, \ - comedi_usb_driver_unregister, &(__usb_driver)) - -#endif /* _COMEDI_USB_H */ diff --git a/drivers/comedi/comedidev.h b/drivers/comedi/comedidev.h deleted file mode 100644 index 0e1b95ef9a4d..000000000000 --- a/drivers/comedi/comedidev.h +++ /dev/null @@ -1,1054 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * comedidev.h - * header file for kernel-only structures, variables, and constants - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1997-2000 David A. Schleef - */ - -#ifndef _COMEDIDEV_H -#define _COMEDIDEV_H - -#include -#include -#include -#include -#include - -#include "comedi.h" - -#define COMEDI_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) -#define COMEDI_VERSION_CODE COMEDI_VERSION(COMEDI_MAJORVERSION, \ - COMEDI_MINORVERSION, COMEDI_MICROVERSION) -#define COMEDI_RELEASE VERSION - -#define COMEDI_NUM_BOARD_MINORS 0x30 - -/** - * struct comedi_subdevice - Working data for a COMEDI subdevice - * @device: COMEDI device to which this subdevice belongs. (Initialized by - * comedi_alloc_subdevices().) - * @index: Index of this subdevice within device's array of subdevices. - * (Initialized by comedi_alloc_subdevices().) - * @type: Type of subdevice from &enum comedi_subdevice_type. (Initialized by - * the low-level driver.) - * @n_chan: Number of channels the subdevice supports. (Initialized by the - * low-level driver.) - * @subdev_flags: Various "SDF" flags indicating aspects of the subdevice to - * the COMEDI core and user application. (Initialized by the low-level - * driver.) - * @len_chanlist: Maximum length of a channel list if the subdevice supports - * asynchronous acquisition commands. (Optionally initialized by the - * low-level driver, or changed from 0 to 1 during post-configuration.) - * @private: Private data pointer which is either set by the low-level driver - * itself, or by a call to comedi_alloc_spriv() which allocates storage. - * In the latter case, the storage is automatically freed after the - * low-level driver's "detach" handler is called for the device. - * (Initialized by the low-level driver.) - * @async: Pointer to &struct comedi_async id the subdevice supports - * asynchronous acquisition commands. (Allocated and initialized during - * post-configuration if needed.) - * @lock: Pointer to a file object that performed a %COMEDI_LOCK ioctl on the - * subdevice. (Initially NULL.) - * @busy: Pointer to a file object that is performing an asynchronous - * acquisition command on the subdevice. (Initially NULL.) - * @runflags: Internal flags for use by COMEDI core, mostly indicating whether - * an asynchronous acquisition command is running. - * @spin_lock: Generic spin-lock for use by the COMEDI core and the low-level - * driver. (Initialized by comedi_alloc_subdevices().) - * @io_bits: Bit-mask indicating the channel directions for a DIO subdevice - * with no more than 32 channels. A '1' at a bit position indicates the - * corresponding channel is configured as an output. (Initialized by the - * low-level driver for a DIO subdevice. Forced to all-outputs during - * post-configuration for a digital output subdevice.) - * @maxdata: If non-zero, this is the maximum raw data value of each channel. - * If zero, the maximum data value is channel-specific. (Initialized by - * the low-level driver.) - * @maxdata_list: If the maximum data value is channel-specific, this points - * to an array of maximum data values indexed by channel index. - * (Initialized by the low-level driver.) - * @range_table: If non-NULL, this points to a COMEDI range table for the - * subdevice. If NULL, the range table is channel-specific. (Initialized - * by the low-level driver, will be set to an "invalid" range table during - * post-configuration if @range_table and @range_table_list are both - * NULL.) - * @range_table_list: If the COMEDI range table is channel-specific, this - * points to an array of pointers to COMEDI range tables indexed by - * channel number. (Initialized by the low-level driver.) - * @chanlist: Not used. - * @insn_read: Optional pointer to a handler for the %INSN_READ instruction. - * (Initialized by the low-level driver, or set to a default handler - * during post-configuration.) - * @insn_write: Optional pointer to a handler for the %INSN_WRITE instruction. - * (Initialized by the low-level driver, or set to a default handler - * during post-configuration.) - * @insn_bits: Optional pointer to a handler for the %INSN_BITS instruction - * for a digital input, digital output or digital input/output subdevice. - * (Initialized by the low-level driver, or set to a default handler - * during post-configuration.) - * @insn_config: Optional pointer to a handler for the %INSN_CONFIG - * instruction. (Initialized by the low-level driver, or set to a default - * handler during post-configuration.) - * @do_cmd: If the subdevice supports asynchronous acquisition commands, this - * points to a handler to set it up in hardware. (Initialized by the - * low-level driver.) - * @do_cmdtest: If the subdevice supports asynchronous acquisition commands, - * this points to a handler used to check and possibly tweak a prospective - * acquisition command without setting it up in hardware. (Initialized by - * the low-level driver.) - * @poll: If the subdevice supports asynchronous acquisition commands, this - * is an optional pointer to a handler for the %COMEDI_POLL ioctl which - * instructs the low-level driver to synchronize buffers. (Initialized by - * the low-level driver if needed.) - * @cancel: If the subdevice supports asynchronous acquisition commands, this - * points to a handler used to terminate a running command. (Initialized - * by the low-level driver.) - * @buf_change: If the subdevice supports asynchronous acquisition commands, - * this is an optional pointer to a handler that is called when the data - * buffer for handling asynchronous commands is allocated or reallocated. - * (Initialized by the low-level driver if needed.) - * @munge: If the subdevice supports asynchronous acquisition commands and - * uses DMA to transfer data from the hardware to the acquisition buffer, - * this points to a function used to "munge" the data values from the - * hardware into the format expected by COMEDI. (Initialized by the - * low-level driver if needed.) - * @async_dma_dir: If the subdevice supports asynchronous acquisition commands - * and uses DMA to transfer data from the hardware to the acquisition - * buffer, this sets the DMA direction for the buffer. (initialized to - * %DMA_NONE by comedi_alloc_subdevices() and changed by the low-level - * driver if necessary.) - * @state: Handy bit-mask indicating the output states for a DIO or digital - * output subdevice with no more than 32 channels. (Initialized by the - * low-level driver.) - * @class_dev: If the subdevice supports asynchronous acquisition commands, - * this points to a sysfs comediX_subdY device where X is the minor device - * number of the COMEDI device and Y is the subdevice number. The minor - * device number for the sysfs device is allocated dynamically in the - * range 48 to 255. This is used to allow the COMEDI device to be opened - * with a different default read or write subdevice. (Allocated during - * post-configuration if needed.) - * @minor: If @class_dev is set, this is its dynamically allocated minor - * device number. (Set during post-configuration if necessary.) - * @readback: Optional pointer to memory allocated by - * comedi_alloc_subdev_readback() used to hold the values written to - * analog output channels so they can be read back. The storage is - * automatically freed after the low-level driver's "detach" handler is - * called for the device. (Initialized by the low-level driver.) - * - * This is the main control structure for a COMEDI subdevice. If the subdevice - * supports asynchronous acquisition commands, additional information is stored - * in the &struct comedi_async pointed to by @async. - * - * Most of the subdevice is initialized by the low-level driver's "attach" or - * "auto_attach" handlers but parts of it are initialized by - * comedi_alloc_subdevices(), and other parts are initialized during - * post-configuration on return from that handler. - * - * A low-level driver that sets @insn_bits for a digital input, digital output, - * or DIO subdevice may leave @insn_read and @insn_write uninitialized, in - * which case they will be set to a default handler during post-configuration - * that uses @insn_bits to emulate the %INSN_READ and %INSN_WRITE instructions. - */ -struct comedi_subdevice { - struct comedi_device *device; - int index; - int type; - int n_chan; - int subdev_flags; - int len_chanlist; /* maximum length of channel/gain list */ - - void *private; - - struct comedi_async *async; - - void *lock; - void *busy; - unsigned int runflags; - spinlock_t spin_lock; /* generic spin-lock for COMEDI and drivers */ - - unsigned int io_bits; - - unsigned int maxdata; /* if maxdata==0, use list */ - const unsigned int *maxdata_list; /* list is channel specific */ - - const struct comedi_lrange *range_table; - const struct comedi_lrange *const *range_table_list; - - unsigned int *chanlist; /* driver-owned chanlist (not used) */ - - int (*insn_read)(struct comedi_device *dev, struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data); - int (*insn_write)(struct comedi_device *dev, struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data); - int (*insn_bits)(struct comedi_device *dev, struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data); - int (*insn_config)(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_insn *insn, - unsigned int *data); - - int (*do_cmd)(struct comedi_device *dev, struct comedi_subdevice *s); - int (*do_cmdtest)(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_cmd *cmd); - int (*poll)(struct comedi_device *dev, struct comedi_subdevice *s); - int (*cancel)(struct comedi_device *dev, struct comedi_subdevice *s); - - /* called when the buffer changes */ - int (*buf_change)(struct comedi_device *dev, - struct comedi_subdevice *s); - - void (*munge)(struct comedi_device *dev, struct comedi_subdevice *s, - void *data, unsigned int num_bytes, - unsigned int start_chan_index); - enum dma_data_direction async_dma_dir; - - unsigned int state; - - struct device *class_dev; - int minor; - - unsigned int *readback; -}; - -/** - * struct comedi_buf_page - Describe a page of a COMEDI buffer - * @virt_addr: Kernel address of page. - * @dma_addr: DMA address of page if in DMA coherent memory. - */ -struct comedi_buf_page { - void *virt_addr; - dma_addr_t dma_addr; -}; - -/** - * struct comedi_buf_map - Describe pages in a COMEDI buffer - * @dma_hw_dev: Low-level hardware &struct device pointer copied from the - * COMEDI device's hw_dev member. - * @page_list: Pointer to array of &struct comedi_buf_page, one for each - * page in the buffer. - * @n_pages: Number of pages in the buffer. - * @dma_dir: DMA direction used to allocate pages of DMA coherent memory, - * or %DMA_NONE if pages allocated from regular memory. - * @refcount: &struct kref reference counter used to free the buffer. - * - * A COMEDI data buffer is allocated as individual pages, either in - * conventional memory or DMA coherent memory, depending on the attached, - * low-level hardware device. (The buffer pages also get mapped into the - * kernel's contiguous virtual address space pointed to by the 'prealloc_buf' - * member of &struct comedi_async.) - * - * The buffer is normally freed when the COMEDI device is detached from the - * low-level driver (which may happen due to device removal), but if it happens - * to be mmapped at the time, the pages cannot be freed until the buffer has - * been munmapped. That is what the reference counter is for. (The virtual - * address space pointed by 'prealloc_buf' is freed when the COMEDI device is - * detached.) - */ -struct comedi_buf_map { - struct device *dma_hw_dev; - struct comedi_buf_page *page_list; - unsigned int n_pages; - enum dma_data_direction dma_dir; - struct kref refcount; -}; - -/** - * struct comedi_async - Control data for asynchronous COMEDI commands - * @prealloc_buf: Kernel virtual address of allocated acquisition buffer. - * @prealloc_bufsz: Buffer size (in bytes). - * @buf_map: Map of buffer pages. - * @max_bufsize: Maximum allowed buffer size (in bytes). - * @buf_write_count: "Write completed" count (in bytes, modulo 2**32). - * @buf_write_alloc_count: "Allocated for writing" count (in bytes, - * modulo 2**32). - * @buf_read_count: "Read completed" count (in bytes, modulo 2**32). - * @buf_read_alloc_count: "Allocated for reading" count (in bytes, - * modulo 2**32). - * @buf_write_ptr: Buffer position for writer. - * @buf_read_ptr: Buffer position for reader. - * @cur_chan: Current position in chanlist for scan (for those drivers that - * use it). - * @scans_done: The number of scans completed. - * @scan_progress: Amount received or sent for current scan (in bytes). - * @munge_chan: Current position in chanlist for "munging". - * @munge_count: "Munge" count (in bytes, modulo 2**32). - * @munge_ptr: Buffer position for "munging". - * @events: Bit-vector of events that have occurred. - * @cmd: Details of comedi command in progress. - * @wait_head: Task wait queue for file reader or writer. - * @cb_mask: Bit-vector of events that should wake waiting tasks. - * @inttrig: Software trigger function for command, or NULL. - * - * Note about the ..._count and ..._ptr members: - * - * Think of the _Count values being integers of unlimited size, indexing - * into a buffer of infinite length (though only an advancing portion - * of the buffer of fixed length prealloc_bufsz is accessible at any - * time). Then: - * - * Buf_Read_Count <= Buf_Read_Alloc_Count <= Munge_Count <= - * Buf_Write_Count <= Buf_Write_Alloc_Count <= - * (Buf_Read_Count + prealloc_bufsz) - * - * (Those aren't the actual members, apart from prealloc_bufsz.) When the - * buffer is reset, those _Count values start at 0 and only increase in value, - * maintaining the above inequalities until the next time the buffer is - * reset. The buffer is divided into the following regions by the inequalities: - * - * [0, Buf_Read_Count): - * old region no longer accessible - * - * [Buf_Read_Count, Buf_Read_Alloc_Count): - * filled and munged region allocated for reading but not yet read - * - * [Buf_Read_Alloc_Count, Munge_Count): - * filled and munged region not yet allocated for reading - * - * [Munge_Count, Buf_Write_Count): - * filled region not yet munged - * - * [Buf_Write_Count, Buf_Write_Alloc_Count): - * unfilled region allocated for writing but not yet written - * - * [Buf_Write_Alloc_Count, Buf_Read_Count + prealloc_bufsz): - * unfilled region not yet allocated for writing - * - * [Buf_Read_Count + prealloc_bufsz, infinity): - * unfilled region not yet accessible - * - * Data needs to be written into the buffer before it can be read out, - * and may need to be converted (or "munged") between the two - * operations. Extra unfilled buffer space may need to allocated for - * writing (advancing Buf_Write_Alloc_Count) before new data is written. - * After writing new data, the newly filled space needs to be released - * (advancing Buf_Write_Count). This also results in the new data being - * "munged" (advancing Munge_Count). Before data is read out of the - * buffer, extra space may need to be allocated for reading (advancing - * Buf_Read_Alloc_Count). After the data has been read out, the space - * needs to be released (advancing Buf_Read_Count). - * - * The actual members, buf_read_count, buf_read_alloc_count, - * munge_count, buf_write_count, and buf_write_alloc_count take the - * value of the corresponding capitalized _Count values modulo 2^32 - * (UINT_MAX+1). Subtracting a "higher" _count value from a "lower" - * _count value gives the same answer as subtracting a "higher" _Count - * value from a lower _Count value because prealloc_bufsz < UINT_MAX+1. - * The modulo operation is done implicitly. - * - * The buf_read_ptr, munge_ptr, and buf_write_ptr members take the value - * of the corresponding capitalized _Count values modulo prealloc_bufsz. - * These correspond to byte indices in the physical buffer. The modulo - * operation is done by subtracting prealloc_bufsz when the value - * exceeds prealloc_bufsz (assuming prealloc_bufsz plus the increment is - * less than or equal to UINT_MAX). - */ -struct comedi_async { - void *prealloc_buf; - unsigned int prealloc_bufsz; - struct comedi_buf_map *buf_map; - unsigned int max_bufsize; - unsigned int buf_write_count; - unsigned int buf_write_alloc_count; - unsigned int buf_read_count; - unsigned int buf_read_alloc_count; - unsigned int buf_write_ptr; - unsigned int buf_read_ptr; - unsigned int cur_chan; - unsigned int scans_done; - unsigned int scan_progress; - unsigned int munge_chan; - unsigned int munge_count; - unsigned int munge_ptr; - unsigned int events; - struct comedi_cmd cmd; - wait_queue_head_t wait_head; - unsigned int cb_mask; - int (*inttrig)(struct comedi_device *dev, struct comedi_subdevice *s, - unsigned int x); -}; - -/** - * enum comedi_cb - &struct comedi_async callback "events" - * @COMEDI_CB_EOS: end-of-scan - * @COMEDI_CB_EOA: end-of-acquisition/output - * @COMEDI_CB_BLOCK: data has arrived, wakes up read() / write() - * @COMEDI_CB_EOBUF: DEPRECATED: end of buffer - * @COMEDI_CB_ERROR: card error during acquisition - * @COMEDI_CB_OVERFLOW: buffer overflow/underflow - * @COMEDI_CB_ERROR_MASK: events that indicate an error has occurred - * @COMEDI_CB_CANCEL_MASK: events that will cancel an async command - */ -enum comedi_cb { - COMEDI_CB_EOS = BIT(0), - COMEDI_CB_EOA = BIT(1), - COMEDI_CB_BLOCK = BIT(2), - COMEDI_CB_EOBUF = BIT(3), - COMEDI_CB_ERROR = BIT(4), - COMEDI_CB_OVERFLOW = BIT(5), - /* masks */ - COMEDI_CB_ERROR_MASK = (COMEDI_CB_ERROR | COMEDI_CB_OVERFLOW), - COMEDI_CB_CANCEL_MASK = (COMEDI_CB_EOA | COMEDI_CB_ERROR_MASK) -}; - -/** - * struct comedi_driver - COMEDI driver registration - * @driver_name: Name of driver. - * @module: Owning module. - * @attach: The optional "attach" handler for manually configured COMEDI - * devices. - * @detach: The "detach" handler for deconfiguring COMEDI devices. - * @auto_attach: The optional "auto_attach" handler for automatically - * configured COMEDI devices. - * @num_names: Optional number of "board names" supported. - * @board_name: Optional pointer to a pointer to a board name. The pointer - * to a board name is embedded in an element of a driver-defined array - * of static, read-only board type information. - * @offset: Optional size of each element of the driver-defined array of - * static, read-only board type information, i.e. the offset between each - * pointer to a board name. - * - * This is used with comedi_driver_register() and comedi_driver_unregister() to - * register and unregister a low-level COMEDI driver with the COMEDI core. - * - * If @num_names is non-zero, @board_name should be non-NULL, and @offset - * should be at least sizeof(*board_name). These are used by the handler for - * the %COMEDI_DEVCONFIG ioctl to match a hardware device and its driver by - * board name. If @num_names is zero, the %COMEDI_DEVCONFIG ioctl matches a - * hardware device and its driver by driver name. This is only useful if the - * @attach handler is set. If @num_names is non-zero, the driver's @attach - * handler will be called with the COMEDI device structure's board_ptr member - * pointing to the matched pointer to a board name within the driver's private - * array of static, read-only board type information. - * - * The @detach handler has two roles. If a COMEDI device was successfully - * configured by the @attach or @auto_attach handler, it is called when the - * device is being deconfigured (by the %COMEDI_DEVCONFIG ioctl, or due to - * unloading of the driver, or due to device removal). It is also called when - * the @attach or @auto_attach handler returns an error. Therefore, the - * @attach or @auto_attach handlers can defer clean-up on error until the - * @detach handler is called. If the @attach or @auto_attach handlers free - * any resources themselves, they must prevent the @detach handler from - * freeing the same resources. The @detach handler must not assume that all - * resources requested by the @attach or @auto_attach handler were - * successfully allocated. - */ -struct comedi_driver { - /* private: */ - struct comedi_driver *next; /* Next in list of COMEDI drivers. */ - /* public: */ - const char *driver_name; - struct module *module; - int (*attach)(struct comedi_device *dev, struct comedi_devconfig *it); - void (*detach)(struct comedi_device *dev); - int (*auto_attach)(struct comedi_device *dev, unsigned long context); - unsigned int num_names; - const char *const *board_name; - int offset; -}; - -/** - * struct comedi_device - Working data for a COMEDI device - * @use_count: Number of open file objects. - * @driver: Low-level COMEDI driver attached to this COMEDI device. - * @pacer: Optional pointer to a dynamically allocated acquisition pacer - * control. It is freed automatically after the COMEDI device is - * detached from the low-level driver. - * @private: Optional pointer to private data allocated by the low-level - * driver. It is freed automatically after the COMEDI device is - * detached from the low-level driver. - * @class_dev: Sysfs comediX device. - * @minor: Minor device number of COMEDI char device (0-47). - * @detach_count: Counter incremented every time the COMEDI device is detached. - * Used for checking a previous attachment is still valid. - * @hw_dev: Optional pointer to the low-level hardware &struct device. It is - * required for automatically configured COMEDI devices and optional for - * COMEDI devices configured by the %COMEDI_DEVCONFIG ioctl, although - * the bus-specific COMEDI functions only work if it is set correctly. - * It is also passed to dma_alloc_coherent() for COMEDI subdevices that - * have their 'async_dma_dir' member set to something other than - * %DMA_NONE. - * @board_name: Pointer to a COMEDI board name or a COMEDI driver name. When - * the low-level driver's "attach" handler is called by the handler for - * the %COMEDI_DEVCONFIG ioctl, it either points to a matched board name - * string if the 'num_names' member of the &struct comedi_driver is - * non-zero, otherwise it points to the low-level driver name string. - * When the low-lever driver's "auto_attach" handler is called for an - * automatically configured COMEDI device, it points to the low-level - * driver name string. The low-level driver is free to change it in its - * "attach" or "auto_attach" handler if it wishes. - * @board_ptr: Optional pointer to private, read-only board type information in - * the low-level driver. If the 'num_names' member of the &struct - * comedi_driver is non-zero, the handler for the %COMEDI_DEVCONFIG ioctl - * will point it to a pointer to a matched board name string within the - * driver's private array of static, read-only board type information when - * calling the driver's "attach" handler. The low-level driver is free to - * change it. - * @attached: Flag indicating that the COMEDI device is attached to a low-level - * driver. - * @ioenabled: Flag used to indicate that a PCI device has been enabled and - * its regions requested. - * @spinlock: Generic spin-lock for use by the low-level driver. - * @mutex: Generic mutex for use by the COMEDI core module. - * @attach_lock: &struct rw_semaphore used to guard against the COMEDI device - * being detached while an operation is in progress. The down_write() - * operation is only allowed while @mutex is held and is used when - * changing @attached and @detach_count and calling the low-level driver's - * "detach" handler. The down_read() operation is generally used without - * holding @mutex. - * @refcount: &struct kref reference counter for freeing COMEDI device. - * @n_subdevices: Number of COMEDI subdevices allocated by the low-level - * driver for this device. - * @subdevices: Dynamically allocated array of COMEDI subdevices. - * @mmio: Optional pointer to a remapped MMIO region set by the low-level - * driver. - * @iobase: Optional base of an I/O port region requested by the low-level - * driver. - * @iolen: Length of I/O port region requested at @iobase. - * @irq: Optional IRQ number requested by the low-level driver. - * @read_subdev: Optional pointer to a default COMEDI subdevice operated on by - * the read() file operation. Set by the low-level driver. - * @write_subdev: Optional pointer to a default COMEDI subdevice operated on by - * the write() file operation. Set by the low-level driver. - * @async_queue: Storage for fasync_helper(). - * @open: Optional pointer to a function set by the low-level driver to be - * called when @use_count changes from 0 to 1. - * @close: Optional pointer to a function set by the low-level driver to be - * called when @use_count changed from 1 to 0. - * @insn_device_config: Optional pointer to a handler for all sub-instructions - * except %INSN_DEVICE_CONFIG_GET_ROUTES of the %INSN_DEVICE_CONFIG - * instruction. If this is not initialized by the low-level driver, a - * default handler will be set during post-configuration. - * @get_valid_routes: Optional pointer to a handler for the - * %INSN_DEVICE_CONFIG_GET_ROUTES sub-instruction of the - * %INSN_DEVICE_CONFIG instruction set. If this is not initialized by the - * low-level driver, a default handler that copies zero routes back to the - * user will be used. - * - * This is the main control data structure for a COMEDI device (as far as the - * COMEDI core is concerned). There are two groups of COMEDI devices - - * "legacy" devices that are configured by the handler for the - * %COMEDI_DEVCONFIG ioctl, and automatically configured devices resulting - * from a call to comedi_auto_config() as a result of a bus driver probe in - * a low-level COMEDI driver. The "legacy" COMEDI devices are allocated - * during module initialization if the "comedi_num_legacy_minors" module - * parameter is non-zero and use minor device numbers from 0 to - * comedi_num_legacy_minors minus one. The automatically configured COMEDI - * devices are allocated on demand and use minor device numbers from - * comedi_num_legacy_minors to 47. - */ -struct comedi_device { - int use_count; - struct comedi_driver *driver; - struct comedi_8254 *pacer; - void *private; - - struct device *class_dev; - int minor; - unsigned int detach_count; - struct device *hw_dev; - - const char *board_name; - const void *board_ptr; - unsigned int attached:1; - unsigned int ioenabled:1; - spinlock_t spinlock; /* generic spin-lock for low-level driver */ - struct mutex mutex; /* generic mutex for COMEDI core */ - struct rw_semaphore attach_lock; - struct kref refcount; - - int n_subdevices; - struct comedi_subdevice *subdevices; - - /* dumb */ - void __iomem *mmio; - unsigned long iobase; - unsigned long iolen; - unsigned int irq; - - struct comedi_subdevice *read_subdev; - struct comedi_subdevice *write_subdev; - - struct fasync_struct *async_queue; - - int (*open)(struct comedi_device *dev); - void (*close)(struct comedi_device *dev); - int (*insn_device_config)(struct comedi_device *dev, - struct comedi_insn *insn, unsigned int *data); - unsigned int (*get_valid_routes)(struct comedi_device *dev, - unsigned int n_pairs, - unsigned int *pair_data); -}; - -/* - * function prototypes - */ - -void comedi_event(struct comedi_device *dev, struct comedi_subdevice *s); - -struct comedi_device *comedi_dev_get_from_minor(unsigned int minor); -int comedi_dev_put(struct comedi_device *dev); - -bool comedi_is_subdevice_running(struct comedi_subdevice *s); - -void *comedi_alloc_spriv(struct comedi_subdevice *s, size_t size); -void comedi_set_spriv_auto_free(struct comedi_subdevice *s); - -int comedi_check_chanlist(struct comedi_subdevice *s, - int n, - unsigned int *chanlist); - -/* range stuff */ - -#define RANGE(a, b) {(a) * 1e6, (b) * 1e6, 0} -#define RANGE_ext(a, b) {(a) * 1e6, (b) * 1e6, RF_EXTERNAL} -#define RANGE_mA(a, b) {(a) * 1e6, (b) * 1e6, UNIT_mA} -#define RANGE_unitless(a, b) {(a) * 1e6, (b) * 1e6, 0} -#define BIP_RANGE(a) {-(a) * 1e6, (a) * 1e6, 0} -#define UNI_RANGE(a) {0, (a) * 1e6, 0} - -extern const struct comedi_lrange range_bipolar10; -extern const struct comedi_lrange range_bipolar5; -extern const struct comedi_lrange range_bipolar2_5; -extern const struct comedi_lrange range_unipolar10; -extern const struct comedi_lrange range_unipolar5; -extern const struct comedi_lrange range_unipolar2_5; -extern const struct comedi_lrange range_0_20mA; -extern const struct comedi_lrange range_4_20mA; -extern const struct comedi_lrange range_0_32mA; -extern const struct comedi_lrange range_unknown; - -#define range_digital range_unipolar5 - -/** - * struct comedi_lrange - Describes a COMEDI range table - * @length: Number of entries in the range table. - * @range: Array of &struct comedi_krange, one for each range. - * - * Each element of @range[] describes the minimum and maximum physical range - * and the type of units. Typically, the type of unit is %UNIT_volt - * (i.e. volts) and the minimum and maximum are in millionths of a volt. - * There may also be a flag that indicates the minimum and maximum are merely - * scale factors for an unknown, external reference. - */ -struct comedi_lrange { - int length; - struct comedi_krange range[]; -}; - -/** - * comedi_range_is_bipolar() - Test if subdevice range is bipolar - * @s: COMEDI subdevice. - * @range: Index of range within a range table. - * - * Tests whether a range is bipolar by checking whether its minimum value - * is negative. - * - * Assumes @range is valid. Does not work for subdevices using a - * channel-specific range table list. - * - * Return: - * %true if the range is bipolar. - * %false if the range is unipolar. - */ -static inline bool comedi_range_is_bipolar(struct comedi_subdevice *s, - unsigned int range) -{ - return s->range_table->range[range].min < 0; -} - -/** - * comedi_range_is_unipolar() - Test if subdevice range is unipolar - * @s: COMEDI subdevice. - * @range: Index of range within a range table. - * - * Tests whether a range is unipolar by checking whether its minimum value - * is at least 0. - * - * Assumes @range is valid. Does not work for subdevices using a - * channel-specific range table list. - * - * Return: - * %true if the range is unipolar. - * %false if the range is bipolar. - */ -static inline bool comedi_range_is_unipolar(struct comedi_subdevice *s, - unsigned int range) -{ - return s->range_table->range[range].min >= 0; -} - -/** - * comedi_range_is_external() - Test if subdevice range is external - * @s: COMEDI subdevice. - * @range: Index of range within a range table. - * - * Tests whether a range is externally reference by checking whether its - * %RF_EXTERNAL flag is set. - * - * Assumes @range is valid. Does not work for subdevices using a - * channel-specific range table list. - * - * Return: - * %true if the range is external. - * %false if the range is internal. - */ -static inline bool comedi_range_is_external(struct comedi_subdevice *s, - unsigned int range) -{ - return !!(s->range_table->range[range].flags & RF_EXTERNAL); -} - -/** - * comedi_chan_range_is_bipolar() - Test if channel-specific range is bipolar - * @s: COMEDI subdevice. - * @chan: The channel number. - * @range: Index of range within a range table. - * - * Tests whether a range is bipolar by checking whether its minimum value - * is negative. - * - * Assumes @chan and @range are valid. Only works for subdevices with a - * channel-specific range table list. - * - * Return: - * %true if the range is bipolar. - * %false if the range is unipolar. - */ -static inline bool comedi_chan_range_is_bipolar(struct comedi_subdevice *s, - unsigned int chan, - unsigned int range) -{ - return s->range_table_list[chan]->range[range].min < 0; -} - -/** - * comedi_chan_range_is_unipolar() - Test if channel-specific range is unipolar - * @s: COMEDI subdevice. - * @chan: The channel number. - * @range: Index of range within a range table. - * - * Tests whether a range is unipolar by checking whether its minimum value - * is at least 0. - * - * Assumes @chan and @range are valid. Only works for subdevices with a - * channel-specific range table list. - * - * Return: - * %true if the range is unipolar. - * %false if the range is bipolar. - */ -static inline bool comedi_chan_range_is_unipolar(struct comedi_subdevice *s, - unsigned int chan, - unsigned int range) -{ - return s->range_table_list[chan]->range[range].min >= 0; -} - -/** - * comedi_chan_range_is_external() - Test if channel-specific range is external - * @s: COMEDI subdevice. - * @chan: The channel number. - * @range: Index of range within a range table. - * - * Tests whether a range is externally reference by checking whether its - * %RF_EXTERNAL flag is set. - * - * Assumes @chan and @range are valid. Only works for subdevices with a - * channel-specific range table list. - * - * Return: - * %true if the range is bipolar. - * %false if the range is unipolar. - */ -static inline bool comedi_chan_range_is_external(struct comedi_subdevice *s, - unsigned int chan, - unsigned int range) -{ - return !!(s->range_table_list[chan]->range[range].flags & RF_EXTERNAL); -} - -/** - * comedi_offset_munge() - Convert between offset binary and 2's complement - * @s: COMEDI subdevice. - * @val: Value to be converted. - * - * Toggles the highest bit of a sample value to toggle between offset binary - * and 2's complement. Assumes that @s->maxdata is a power of 2 minus 1. - * - * Return: The converted value. - */ -static inline unsigned int comedi_offset_munge(struct comedi_subdevice *s, - unsigned int val) -{ - return val ^ s->maxdata ^ (s->maxdata >> 1); -} - -/** - * comedi_bytes_per_sample() - Determine subdevice sample size - * @s: COMEDI subdevice. - * - * The sample size will be 4 (sizeof int) or 2 (sizeof short) depending on - * whether the %SDF_LSAMPL subdevice flag is set or not. - * - * Return: The subdevice sample size. - */ -static inline unsigned int comedi_bytes_per_sample(struct comedi_subdevice *s) -{ - return s->subdev_flags & SDF_LSAMPL ? sizeof(int) : sizeof(short); -} - -/** - * comedi_sample_shift() - Determine log2 of subdevice sample size - * @s: COMEDI subdevice. - * - * The sample size will be 4 (sizeof int) or 2 (sizeof short) depending on - * whether the %SDF_LSAMPL subdevice flag is set or not. The log2 of the - * sample size will be 2 or 1 and can be used as the right operand of a - * bit-shift operator to multiply or divide something by the sample size. - * - * Return: log2 of the subdevice sample size. - */ -static inline unsigned int comedi_sample_shift(struct comedi_subdevice *s) -{ - return s->subdev_flags & SDF_LSAMPL ? 2 : 1; -} - -/** - * comedi_bytes_to_samples() - Convert a number of bytes to a number of samples - * @s: COMEDI subdevice. - * @nbytes: Number of bytes - * - * Return: The number of bytes divided by the subdevice sample size. - */ -static inline unsigned int comedi_bytes_to_samples(struct comedi_subdevice *s, - unsigned int nbytes) -{ - return nbytes >> comedi_sample_shift(s); -} - -/** - * comedi_samples_to_bytes() - Convert a number of samples to a number of bytes - * @s: COMEDI subdevice. - * @nsamples: Number of samples. - * - * Return: The number of samples multiplied by the subdevice sample size. - * (Does not check for arithmetic overflow.) - */ -static inline unsigned int comedi_samples_to_bytes(struct comedi_subdevice *s, - unsigned int nsamples) -{ - return nsamples << comedi_sample_shift(s); -} - -/** - * comedi_check_trigger_src() - Trivially validate a comedi_cmd trigger source - * @src: Pointer to the trigger source to validate. - * @flags: Bitmask of valid %TRIG_* for the trigger. - * - * This is used in "step 1" of the do_cmdtest functions of comedi drivers - * to validate the comedi_cmd triggers. The mask of the @src against the - * @flags allows the userspace comedilib to pass all the comedi_cmd - * triggers as %TRIG_ANY and get back a bitmask of the valid trigger sources. - * - * Return: - * 0 if trigger sources in *@src are all supported. - * -EINVAL if any trigger source in *@src is unsupported. - */ -static inline int comedi_check_trigger_src(unsigned int *src, - unsigned int flags) -{ - unsigned int orig_src = *src; - - *src = orig_src & flags; - if (*src == TRIG_INVALID || *src != orig_src) - return -EINVAL; - return 0; -} - -/** - * comedi_check_trigger_is_unique() - Make sure a trigger source is unique - * @src: The trigger source to check. - * - * Return: - * 0 if no more than one trigger source is set. - * -EINVAL if more than one trigger source is set. - */ -static inline int comedi_check_trigger_is_unique(unsigned int src) -{ - /* this test is true if more than one _src bit is set */ - if ((src & (src - 1)) != 0) - return -EINVAL; - return 0; -} - -/** - * comedi_check_trigger_arg_is() - Trivially validate a trigger argument - * @arg: Pointer to the trigger arg to validate. - * @val: The value the argument should be. - * - * Forces *@arg to be @val. - * - * Return: - * 0 if *@arg was already @val. - * -EINVAL if *@arg differed from @val. - */ -static inline int comedi_check_trigger_arg_is(unsigned int *arg, - unsigned int val) -{ - if (*arg != val) { - *arg = val; - return -EINVAL; - } - return 0; -} - -/** - * comedi_check_trigger_arg_min() - Trivially validate a trigger argument min - * @arg: Pointer to the trigger arg to validate. - * @val: The minimum value the argument should be. - * - * Forces *@arg to be at least @val, setting it to @val if necessary. - * - * Return: - * 0 if *@arg was already at least @val. - * -EINVAL if *@arg was less than @val. - */ -static inline int comedi_check_trigger_arg_min(unsigned int *arg, - unsigned int val) -{ - if (*arg < val) { - *arg = val; - return -EINVAL; - } - return 0; -} - -/** - * comedi_check_trigger_arg_max() - Trivially validate a trigger argument max - * @arg: Pointer to the trigger arg to validate. - * @val: The maximum value the argument should be. - * - * Forces *@arg to be no more than @val, setting it to @val if necessary. - * - * Return: - * 0 if*@arg was already no more than @val. - * -EINVAL if *@arg was greater than @val. - */ -static inline int comedi_check_trigger_arg_max(unsigned int *arg, - unsigned int val) -{ - if (*arg > val) { - *arg = val; - return -EINVAL; - } - return 0; -} - -/* - * Must set dev->hw_dev if you wish to dma directly into comedi's buffer. - * Also useful for retrieving a previously configured hardware device of - * known bus type. Set automatically for auto-configured devices. - * Automatically set to NULL when detaching hardware device. - */ -int comedi_set_hw_dev(struct comedi_device *dev, struct device *hw_dev); - -/** - * comedi_buf_n_bytes_ready - Determine amount of unread data in buffer - * @s: COMEDI subdevice. - * - * Determines the number of bytes of unread data in the asynchronous - * acquisition data buffer for a subdevice. The data in question might not - * have been fully "munged" yet. - * - * Returns: The amount of unread data in bytes. - */ -static inline unsigned int comedi_buf_n_bytes_ready(struct comedi_subdevice *s) -{ - return s->async->buf_write_count - s->async->buf_read_count; -} - -unsigned int comedi_buf_write_alloc(struct comedi_subdevice *s, unsigned int n); -unsigned int comedi_buf_write_free(struct comedi_subdevice *s, unsigned int n); - -unsigned int comedi_buf_read_n_available(struct comedi_subdevice *s); -unsigned int comedi_buf_read_alloc(struct comedi_subdevice *s, unsigned int n); -unsigned int comedi_buf_read_free(struct comedi_subdevice *s, unsigned int n); - -unsigned int comedi_buf_write_samples(struct comedi_subdevice *s, - const void *data, unsigned int nsamples); -unsigned int comedi_buf_read_samples(struct comedi_subdevice *s, - void *data, unsigned int nsamples); - -/* drivers.c - general comedi driver functions */ - -#define COMEDI_TIMEOUT_MS 1000 - -int comedi_timeout(struct comedi_device *dev, struct comedi_subdevice *s, - struct comedi_insn *insn, - int (*cb)(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned long context), - unsigned long context); - -unsigned int comedi_handle_events(struct comedi_device *dev, - struct comedi_subdevice *s); - -int comedi_dio_insn_config(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data, - unsigned int mask); -unsigned int comedi_dio_update_state(struct comedi_subdevice *s, - unsigned int *data); -unsigned int comedi_bytes_per_scan_cmd(struct comedi_subdevice *s, - struct comedi_cmd *cmd); -unsigned int comedi_bytes_per_scan(struct comedi_subdevice *s); -unsigned int comedi_nscans_left(struct comedi_subdevice *s, - unsigned int nscans); -unsigned int comedi_nsamples_left(struct comedi_subdevice *s, - unsigned int nsamples); -void comedi_inc_scan_progress(struct comedi_subdevice *s, - unsigned int num_bytes); - -void *comedi_alloc_devpriv(struct comedi_device *dev, size_t size); -int comedi_alloc_subdevices(struct comedi_device *dev, int num_subdevices); -int comedi_alloc_subdev_readback(struct comedi_subdevice *s); - -int comedi_readback_insn_read(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data); - -int comedi_load_firmware(struct comedi_device *dev, struct device *hw_dev, - const char *name, - int (*cb)(struct comedi_device *dev, - const u8 *data, size_t size, - unsigned long context), - unsigned long context); - -int __comedi_request_region(struct comedi_device *dev, - unsigned long start, unsigned long len); -int comedi_request_region(struct comedi_device *dev, - unsigned long start, unsigned long len); -void comedi_legacy_detach(struct comedi_device *dev); - -int comedi_auto_config(struct device *hardware_device, - struct comedi_driver *driver, unsigned long context); -void comedi_auto_unconfig(struct device *hardware_device); - -int comedi_driver_register(struct comedi_driver *driver); -void comedi_driver_unregister(struct comedi_driver *driver); - -/** - * module_comedi_driver() - Helper macro for registering a comedi driver - * @__comedi_driver: comedi_driver struct - * - * Helper macro for comedi drivers which do not do anything special in module - * init/exit. This eliminates a lot of boilerplate. Each module may only use - * this macro once, and calling it replaces module_init() and module_exit(). - */ -#define module_comedi_driver(__comedi_driver) \ - module_driver(__comedi_driver, comedi_driver_register, \ - comedi_driver_unregister) - -#endif /* _COMEDIDEV_H */ diff --git a/drivers/comedi/comedilib.h b/drivers/comedi/comedilib.h deleted file mode 100644 index 0223c9cd9215..000000000000 --- a/drivers/comedi/comedilib.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * comedilib.h - * Header file for kcomedilib - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1998-2001 David A. Schleef - */ - -#ifndef _LINUX_COMEDILIB_H -#define _LINUX_COMEDILIB_H - -struct comedi_device *comedi_open(const char *path); -int comedi_close(struct comedi_device *dev); -int comedi_dio_get_config(struct comedi_device *dev, unsigned int subdev, - unsigned int chan, unsigned int *io); -int comedi_dio_config(struct comedi_device *dev, unsigned int subdev, - unsigned int chan, unsigned int io); -int comedi_dio_bitfield2(struct comedi_device *dev, unsigned int subdev, - unsigned int mask, unsigned int *bits, - unsigned int base_channel); -int comedi_find_subdevice_by_type(struct comedi_device *dev, int type, - unsigned int subd); -int comedi_get_n_channels(struct comedi_device *dev, unsigned int subdevice); - -#endif diff --git a/drivers/comedi/drivers.c b/drivers/comedi/drivers.c index 750a6ff3c03c..8eb1f699a857 100644 --- a/drivers/comedi/drivers.c +++ b/drivers/comedi/drivers.c @@ -17,8 +17,7 @@ #include #include #include - -#include "comedidev.h" +#include #include "comedi_internal.h" struct comedi_driver *comedi_drivers; diff --git a/drivers/comedi/drivers/8255.c b/drivers/comedi/drivers/8255.c index e23335c75867..f23a52b7c919 100644 --- a/drivers/comedi/drivers/8255.c +++ b/drivers/comedi/drivers/8255.c @@ -40,7 +40,7 @@ */ #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/8255_pci.c b/drivers/comedi/drivers/8255_pci.c index 5a810f0e532a..76b8b4762bae 100644 --- a/drivers/comedi/drivers/8255_pci.c +++ b/drivers/comedi/drivers/8255_pci.c @@ -53,8 +53,7 @@ */ #include - -#include "../comedi_pci.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/addi_apci_1032.c b/drivers/comedi/drivers/addi_apci_1032.c index 81a246fbcc01..8eec6d9402de 100644 --- a/drivers/comedi/drivers/addi_apci_1032.c +++ b/drivers/comedi/drivers/addi_apci_1032.c @@ -63,8 +63,8 @@ #include #include +#include -#include "../comedi_pci.h" #include "amcc_s5933.h" /* diff --git a/drivers/comedi/drivers/addi_apci_1500.c b/drivers/comedi/drivers/addi_apci_1500.c index b04c15dcfb57..c94c78588889 100644 --- a/drivers/comedi/drivers/addi_apci_1500.c +++ b/drivers/comedi/drivers/addi_apci_1500.c @@ -14,8 +14,8 @@ #include #include +#include -#include "../comedi_pci.h" #include "amcc_s5933.h" #include "z8536.h" diff --git a/drivers/comedi/drivers/addi_apci_1516.c b/drivers/comedi/drivers/addi_apci_1516.c index 274ec9fb030c..3c48b72dad9d 100644 --- a/drivers/comedi/drivers/addi_apci_1516.c +++ b/drivers/comedi/drivers/addi_apci_1516.c @@ -14,8 +14,8 @@ */ #include +#include -#include "../comedi_pci.h" #include "addi_watchdog.h" /* diff --git a/drivers/comedi/drivers/addi_apci_1564.c b/drivers/comedi/drivers/addi_apci_1564.c index 06fc7ed96200..0cd40948bee7 100644 --- a/drivers/comedi/drivers/addi_apci_1564.c +++ b/drivers/comedi/drivers/addi_apci_1564.c @@ -68,8 +68,8 @@ #include #include +#include -#include "../comedi_pci.h" #include "addi_tcw.h" #include "addi_watchdog.h" diff --git a/drivers/comedi/drivers/addi_apci_16xx.c b/drivers/comedi/drivers/addi_apci_16xx.c index c306aa41df97..ec2c321d2431 100644 --- a/drivers/comedi/drivers/addi_apci_16xx.c +++ b/drivers/comedi/drivers/addi_apci_16xx.c @@ -14,8 +14,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/addi_apci_2032.c b/drivers/comedi/drivers/addi_apci_2032.c index e9a2b37a4ae0..e048dfc3ec77 100644 --- a/drivers/comedi/drivers/addi_apci_2032.c +++ b/drivers/comedi/drivers/addi_apci_2032.c @@ -16,8 +16,8 @@ #include #include #include +#include -#include "../comedi_pci.h" #include "addi_watchdog.h" /* diff --git a/drivers/comedi/drivers/addi_apci_2200.c b/drivers/comedi/drivers/addi_apci_2200.c index 4c5aee784bd9..00378c9dddc8 100644 --- a/drivers/comedi/drivers/addi_apci_2200.c +++ b/drivers/comedi/drivers/addi_apci_2200.c @@ -14,8 +14,8 @@ */ #include +#include -#include "../comedi_pci.h" #include "addi_watchdog.h" /* diff --git a/drivers/comedi/drivers/addi_apci_3120.c b/drivers/comedi/drivers/addi_apci_3120.c index 1ed3b33d1a30..28a242e69721 100644 --- a/drivers/comedi/drivers/addi_apci_3120.c +++ b/drivers/comedi/drivers/addi_apci_3120.c @@ -14,8 +14,8 @@ #include #include +#include -#include "../comedi_pci.h" #include "amcc_s5933.h" /* diff --git a/drivers/comedi/drivers/addi_apci_3501.c b/drivers/comedi/drivers/addi_apci_3501.c index f0c9642f3f1a..ecb5552f1785 100644 --- a/drivers/comedi/drivers/addi_apci_3501.c +++ b/drivers/comedi/drivers/addi_apci_3501.c @@ -41,8 +41,8 @@ */ #include +#include -#include "../comedi_pci.h" #include "amcc_s5933.h" /* diff --git a/drivers/comedi/drivers/addi_apci_3xxx.c b/drivers/comedi/drivers/addi_apci_3xxx.c index a90d59377e18..bc72273e6a29 100644 --- a/drivers/comedi/drivers/addi_apci_3xxx.c +++ b/drivers/comedi/drivers/addi_apci_3xxx.c @@ -15,8 +15,7 @@ #include #include - -#include "../comedi_pci.h" +#include #define CONV_UNIT_NS BIT(0) #define CONV_UNIT_US BIT(1) diff --git a/drivers/comedi/drivers/addi_watchdog.c b/drivers/comedi/drivers/addi_watchdog.c index 69b323fb869f..ed87ab432020 100644 --- a/drivers/comedi/drivers/addi_watchdog.c +++ b/drivers/comedi/drivers/addi_watchdog.c @@ -10,7 +10,7 @@ */ #include -#include "../comedidev.h" +#include #include "addi_tcw.h" #include "addi_watchdog.h" diff --git a/drivers/comedi/drivers/adl_pci6208.c b/drivers/comedi/drivers/adl_pci6208.c index 9ae4cc523dd4..b27354a51f5c 100644 --- a/drivers/comedi/drivers/adl_pci6208.c +++ b/drivers/comedi/drivers/adl_pci6208.c @@ -24,8 +24,7 @@ #include #include - -#include "../comedi_pci.h" +#include /* * PCI-6208/6216-GL register map diff --git a/drivers/comedi/drivers/adl_pci7x3x.c b/drivers/comedi/drivers/adl_pci7x3x.c index 8fc45638ff59..e9f22de9b6f1 100644 --- a/drivers/comedi/drivers/adl_pci7x3x.c +++ b/drivers/comedi/drivers/adl_pci7x3x.c @@ -46,8 +46,7 @@ */ #include - -#include "../comedi_pci.h" +#include #include "plx9052.h" diff --git a/drivers/comedi/drivers/adl_pci8164.c b/drivers/comedi/drivers/adl_pci8164.c index d5e1bda81557..0c513a67a264 100644 --- a/drivers/comedi/drivers/adl_pci8164.c +++ b/drivers/comedi/drivers/adl_pci8164.c @@ -19,8 +19,7 @@ #include #include - -#include "../comedi_pci.h" +#include #define PCI8164_AXIS(x) ((x) * 0x08) #define PCI8164_CMD_MSTS_REG 0x00 diff --git a/drivers/comedi/drivers/adl_pci9111.c b/drivers/comedi/drivers/adl_pci9111.c index a062c5ab20e9..65454f3ecc91 100644 --- a/drivers/comedi/drivers/adl_pci9111.c +++ b/drivers/comedi/drivers/adl_pci9111.c @@ -42,8 +42,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "plx9052.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/adl_pci9118.c b/drivers/comedi/drivers/adl_pci9118.c index cda3a4267dca..248cec3d894f 100644 --- a/drivers/comedi/drivers/adl_pci9118.c +++ b/drivers/comedi/drivers/adl_pci9118.c @@ -78,8 +78,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "amcc_s5933.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/adq12b.c b/drivers/comedi/drivers/adq12b.c index d719f76709ef..19d765182006 100644 --- a/drivers/comedi/drivers/adq12b.c +++ b/drivers/comedi/drivers/adq12b.c @@ -48,8 +48,7 @@ #include #include - -#include "../comedidev.h" +#include /* address scheme (page 2.17 of the manual) */ #define ADQ12B_CTREG 0x00 diff --git a/drivers/comedi/drivers/adv_pci1710.c b/drivers/comedi/drivers/adv_pci1710.c index 090607760be6..47a800d72e58 100644 --- a/drivers/comedi/drivers/adv_pci1710.c +++ b/drivers/comedi/drivers/adv_pci1710.c @@ -30,8 +30,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "amcc_s5933.h" diff --git a/drivers/comedi/drivers/adv_pci1720.c b/drivers/comedi/drivers/adv_pci1720.c index 2fcd7e8e7d85..2619591ba301 100644 --- a/drivers/comedi/drivers/adv_pci1720.c +++ b/drivers/comedi/drivers/adv_pci1720.c @@ -42,8 +42,7 @@ #include #include - -#include "../comedi_pci.h" +#include /* * PCI BAR2 Register map (dev->iobase) diff --git a/drivers/comedi/drivers/adv_pci1723.c b/drivers/comedi/drivers/adv_pci1723.c index 23660a9fdb9c..e2aedb152068 100644 --- a/drivers/comedi/drivers/adv_pci1723.c +++ b/drivers/comedi/drivers/adv_pci1723.c @@ -32,8 +32,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * PCI Bar 2 I/O Register map (dev->iobase) diff --git a/drivers/comedi/drivers/adv_pci1724.c b/drivers/comedi/drivers/adv_pci1724.c index e8ab573c839f..bb43b7deeb56 100644 --- a/drivers/comedi/drivers/adv_pci1724.c +++ b/drivers/comedi/drivers/adv_pci1724.c @@ -38,8 +38,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * PCI bar 2 Register I/O map (dev->iobase) diff --git a/drivers/comedi/drivers/adv_pci1760.c b/drivers/comedi/drivers/adv_pci1760.c index 6de8ab97d346..fcfc2e299110 100644 --- a/drivers/comedi/drivers/adv_pci1760.c +++ b/drivers/comedi/drivers/adv_pci1760.c @@ -22,8 +22,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * PCI-1760 Register Map diff --git a/drivers/comedi/drivers/adv_pci_dio.c b/drivers/comedi/drivers/adv_pci_dio.c index 54c7419c8ca6..5947f08b9a1e 100644 --- a/drivers/comedi/drivers/adv_pci_dio.c +++ b/drivers/comedi/drivers/adv_pci_dio.c @@ -23,8 +23,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "8255.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/aio_aio12_8.c b/drivers/comedi/drivers/aio_aio12_8.c index 4829115921a3..36c3a2d8a352 100644 --- a/drivers/comedi/drivers/aio_aio12_8.c +++ b/drivers/comedi/drivers/aio_aio12_8.c @@ -22,7 +22,7 @@ */ #include -#include "../comedidev.h" +#include #include "comedi_8254.h" #include "8255.h" diff --git a/drivers/comedi/drivers/aio_iiro_16.c b/drivers/comedi/drivers/aio_iiro_16.c index fe3876235075..b00fab0b89d4 100644 --- a/drivers/comedi/drivers/aio_iiro_16.c +++ b/drivers/comedi/drivers/aio_iiro_16.c @@ -30,8 +30,7 @@ #include #include - -#include "../comedidev.h" +#include #define AIO_IIRO_16_RELAY_0_7 0x00 #define AIO_IIRO_16_INPUT_0_7 0x01 diff --git a/drivers/comedi/drivers/amplc_dio200.c b/drivers/comedi/drivers/amplc_dio200.c index fa19c9e7c56b..4544bcdd8a70 100644 --- a/drivers/comedi/drivers/amplc_dio200.c +++ b/drivers/comedi/drivers/amplc_dio200.c @@ -185,7 +185,7 @@ */ #include -#include "../comedidev.h" +#include #include "amplc_dio200.h" diff --git a/drivers/comedi/drivers/amplc_dio200_common.c b/drivers/comedi/drivers/amplc_dio200_common.c index a3454130d5f8..950c50be4ff3 100644 --- a/drivers/comedi/drivers/amplc_dio200_common.c +++ b/drivers/comedi/drivers/amplc_dio200_common.c @@ -12,8 +12,7 @@ #include #include - -#include "../comedidev.h" +#include #include "amplc_dio200.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/amplc_dio200_pci.c b/drivers/comedi/drivers/amplc_dio200_pci.c index 1bd7a42c8464..527994d82a1f 100644 --- a/drivers/comedi/drivers/amplc_dio200_pci.c +++ b/drivers/comedi/drivers/amplc_dio200_pci.c @@ -214,8 +214,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "amplc_dio200.h" diff --git a/drivers/comedi/drivers/amplc_pc236.c b/drivers/comedi/drivers/amplc_pc236.c index c377af1d5246..b21e0c906aab 100644 --- a/drivers/comedi/drivers/amplc_pc236.c +++ b/drivers/comedi/drivers/amplc_pc236.c @@ -32,8 +32,7 @@ */ #include - -#include "../comedidev.h" +#include #include "amplc_pc236.h" diff --git a/drivers/comedi/drivers/amplc_pc236_common.c b/drivers/comedi/drivers/amplc_pc236_common.c index 981d281e87a1..b8b0a624f72b 100644 --- a/drivers/comedi/drivers/amplc_pc236_common.c +++ b/drivers/comedi/drivers/amplc_pc236_common.c @@ -11,8 +11,7 @@ #include #include - -#include "../comedidev.h" +#include #include "amplc_pc236.h" #include "8255.h" diff --git a/drivers/comedi/drivers/amplc_pc263.c b/drivers/comedi/drivers/amplc_pc263.c index 68da6098ee84..d7f088a8a5e3 100644 --- a/drivers/comedi/drivers/amplc_pc263.c +++ b/drivers/comedi/drivers/amplc_pc263.c @@ -25,7 +25,7 @@ */ #include -#include "../comedidev.h" +#include /* PC263 registers */ #define PC263_DO_0_7_REG 0x00 diff --git a/drivers/comedi/drivers/amplc_pci224.c b/drivers/comedi/drivers/amplc_pci224.c index bcf6d61af863..3cf1b7fa565d 100644 --- a/drivers/comedi/drivers/amplc_pci224.c +++ b/drivers/comedi/drivers/amplc_pci224.c @@ -96,8 +96,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/amplc_pci230.c b/drivers/comedi/drivers/amplc_pci230.c index 8911dc2bd2c6..554ee40e321f 100644 --- a/drivers/comedi/drivers/amplc_pci230.c +++ b/drivers/comedi/drivers/amplc_pci230.c @@ -174,8 +174,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "8255.h" diff --git a/drivers/comedi/drivers/amplc_pci236.c b/drivers/comedi/drivers/amplc_pci236.c index e7f6fa4d101a..482eb261c333 100644 --- a/drivers/comedi/drivers/amplc_pci236.c +++ b/drivers/comedi/drivers/amplc_pci236.c @@ -34,8 +34,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "amplc_pc236.h" #include "plx9052.h" diff --git a/drivers/comedi/drivers/amplc_pci263.c b/drivers/comedi/drivers/amplc_pci263.c index 9217973f1141..1609665c4b18 100644 --- a/drivers/comedi/drivers/amplc_pci263.c +++ b/drivers/comedi/drivers/amplc_pci263.c @@ -24,8 +24,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* PCI263 registers */ #define PCI263_DO_0_7_REG 0x00 diff --git a/drivers/comedi/drivers/c6xdigio.c b/drivers/comedi/drivers/c6xdigio.c index 786fd15698df..14b90d1c64dc 100644 --- a/drivers/comedi/drivers/c6xdigio.c +++ b/drivers/comedi/drivers/c6xdigio.c @@ -30,8 +30,7 @@ #include #include #include - -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/cb_das16_cs.c b/drivers/comedi/drivers/cb_das16_cs.c index a5d171e71c33..190d73a7d12c 100644 --- a/drivers/comedi/drivers/cb_das16_cs.c +++ b/drivers/comedi/drivers/cb_das16_cs.c @@ -27,8 +27,7 @@ #include #include #include - -#include "../comedi_pcmcia.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/cb_pcidas.c b/drivers/comedi/drivers/cb_pcidas.c index 2f20bd56ec6c..9b603532a4e7 100644 --- a/drivers/comedi/drivers/cb_pcidas.c +++ b/drivers/comedi/drivers/cb_pcidas.c @@ -54,8 +54,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "8255.h" diff --git a/drivers/comedi/drivers/cb_pcidas64.c b/drivers/comedi/drivers/cb_pcidas64.c index 41a8fea7f48a..7d4808faa1fb 100644 --- a/drivers/comedi/drivers/cb_pcidas64.c +++ b/drivers/comedi/drivers/cb_pcidas64.c @@ -73,8 +73,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "8255.h" #include "plx9080.h" diff --git a/drivers/comedi/drivers/cb_pcidda.c b/drivers/comedi/drivers/cb_pcidda.c index 78cf1603638c..4ed3bcf47973 100644 --- a/drivers/comedi/drivers/cb_pcidda.c +++ b/drivers/comedi/drivers/cb_pcidda.c @@ -27,8 +27,7 @@ */ #include - -#include "../comedi_pci.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/cb_pcimdas.c b/drivers/comedi/drivers/cb_pcimdas.c index 2292f69da4f4..64c7d72c7956 100644 --- a/drivers/comedi/drivers/cb_pcimdas.c +++ b/drivers/comedi/drivers/cb_pcimdas.c @@ -34,8 +34,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "plx9052.h" diff --git a/drivers/comedi/drivers/cb_pcimdda.c b/drivers/comedi/drivers/cb_pcimdda.c index 21fc7b3c5f60..69d7803b0e58 100644 --- a/drivers/comedi/drivers/cb_pcimdda.c +++ b/drivers/comedi/drivers/cb_pcimdda.c @@ -67,8 +67,7 @@ */ #include - -#include "../comedi_pci.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/comedi_8254.c b/drivers/comedi/drivers/comedi_8254.c index 4bf5daa9e885..fac81567133d 100644 --- a/drivers/comedi/drivers/comedi_8254.c +++ b/drivers/comedi/drivers/comedi_8254.c @@ -116,8 +116,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/comedi_8255.c b/drivers/comedi/drivers/comedi_8255.c index b7ca465933ee..10614603d677 100644 --- a/drivers/comedi/drivers/comedi_8255.c +++ b/drivers/comedi/drivers/comedi_8255.c @@ -29,7 +29,7 @@ */ #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/comedi_bond.c b/drivers/comedi/drivers/comedi_bond.c index 4392b5927a99..78c39fa84177 100644 --- a/drivers/comedi/drivers/comedi_bond.c +++ b/drivers/comedi/drivers/comedi_bond.c @@ -40,9 +40,9 @@ #include #include #include -#include "../comedi.h" -#include "../comedilib.h" -#include "../comedidev.h" +#include +#include +#include struct bonded_device { struct comedi_device *dev; diff --git a/drivers/comedi/drivers/comedi_isadma.c b/drivers/comedi/drivers/comedi_isadma.c index 479b58e209ba..63457bd4ff78 100644 --- a/drivers/comedi/drivers/comedi_isadma.c +++ b/drivers/comedi/drivers/comedi_isadma.c @@ -9,8 +9,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" diff --git a/drivers/comedi/drivers/comedi_parport.c b/drivers/comedi/drivers/comedi_parport.c index 5338b5eea440..098738a688fe 100644 --- a/drivers/comedi/drivers/comedi_parport.c +++ b/drivers/comedi/drivers/comedi_parport.c @@ -57,8 +57,7 @@ #include #include - -#include "../comedidev.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/comedi_test.c b/drivers/comedi/drivers/comedi_test.c index cbc225eb1991..0b5c0af1cebf 100644 --- a/drivers/comedi/drivers/comedi_test.c +++ b/drivers/comedi/drivers/comedi_test.c @@ -45,10 +45,8 @@ */ #include -#include "../comedidev.h" - +#include #include - #include #include #include diff --git a/drivers/comedi/drivers/contec_pci_dio.c b/drivers/comedi/drivers/contec_pci_dio.c index b8fdd9c1f166..41d42ff14144 100644 --- a/drivers/comedi/drivers/contec_pci_dio.c +++ b/drivers/comedi/drivers/contec_pci_dio.c @@ -18,8 +18,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/dac02.c b/drivers/comedi/drivers/dac02.c index 5ef8114c2c85..4b011d66d7b0 100644 --- a/drivers/comedi/drivers/dac02.c +++ b/drivers/comedi/drivers/dac02.c @@ -25,8 +25,7 @@ */ #include - -#include "../comedidev.h" +#include /* * The output range is selected by jumpering pins on the I/O connector. diff --git a/drivers/comedi/drivers/daqboard2000.c b/drivers/comedi/drivers/daqboard2000.c index f64e747078bd..52e4bf16cbda 100644 --- a/drivers/comedi/drivers/daqboard2000.c +++ b/drivers/comedi/drivers/daqboard2000.c @@ -96,8 +96,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "8255.h" #include "plx9080.h" diff --git a/drivers/comedi/drivers/das08.c b/drivers/comedi/drivers/das08.c index b50743c5b822..c146a168f43b 100644 --- a/drivers/comedi/drivers/das08.c +++ b/drivers/comedi/drivers/das08.c @@ -10,8 +10,7 @@ */ #include - -#include "../comedidev.h" +#include #include "8255.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/das08_cs.c b/drivers/comedi/drivers/das08_cs.c index 223479f9ea3c..6075efcf10d6 100644 --- a/drivers/comedi/drivers/das08_cs.c +++ b/drivers/comedi/drivers/das08_cs.c @@ -30,8 +30,7 @@ */ #include - -#include "../comedi_pcmcia.h" +#include #include "das08.h" diff --git a/drivers/comedi/drivers/das08_isa.c b/drivers/comedi/drivers/das08_isa.c index 8c4cfa821423..3d43b77cc9f4 100644 --- a/drivers/comedi/drivers/das08_isa.c +++ b/drivers/comedi/drivers/das08_isa.c @@ -29,7 +29,7 @@ */ #include -#include "../comedidev.h" +#include #include "das08.h" diff --git a/drivers/comedi/drivers/das08_pci.c b/drivers/comedi/drivers/das08_pci.c index 1cd903336a4c..982f3ab0ccbd 100644 --- a/drivers/comedi/drivers/das08_pci.c +++ b/drivers/comedi/drivers/das08_pci.c @@ -23,8 +23,7 @@ */ #include - -#include "../comedi_pci.h" +#include #include "das08.h" diff --git a/drivers/comedi/drivers/das16.c b/drivers/comedi/drivers/das16.c index 4ac2622b0fac..362232ad4409 100644 --- a/drivers/comedi/drivers/das16.c +++ b/drivers/comedi/drivers/das16.c @@ -63,8 +63,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/das16m1.c b/drivers/comedi/drivers/das16m1.c index 75f3dbbe97ac..cc79e318cb2d 100644 --- a/drivers/comedi/drivers/das16m1.c +++ b/drivers/comedi/drivers/das16m1.c @@ -42,7 +42,7 @@ #include #include #include -#include "../comedidev.h" +#include #include "8255.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/das1800.c b/drivers/comedi/drivers/das1800.c index f50891a6ee7d..768803742350 100644 --- a/drivers/comedi/drivers/das1800.c +++ b/drivers/comedi/drivers/das1800.c @@ -73,8 +73,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/das6402.c b/drivers/comedi/drivers/das6402.c index 96f4107b8054..d411ab7cf37c 100644 --- a/drivers/comedi/drivers/das6402.c +++ b/drivers/comedi/drivers/das6402.c @@ -24,8 +24,7 @@ #include #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/das800.c b/drivers/comedi/drivers/das800.c index bc08324f422f..c95e0fcb94a4 100644 --- a/drivers/comedi/drivers/das800.c +++ b/drivers/comedi/drivers/das800.c @@ -46,8 +46,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/dmm32at.c b/drivers/comedi/drivers/dmm32at.c index 56682f01242f..0f2bea88b8a7 100644 --- a/drivers/comedi/drivers/dmm32at.c +++ b/drivers/comedi/drivers/dmm32at.c @@ -29,7 +29,7 @@ #include #include #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/dt2801.c b/drivers/comedi/drivers/dt2801.c index 0d571d817b4e..230d25010f58 100644 --- a/drivers/comedi/drivers/dt2801.c +++ b/drivers/comedi/drivers/dt2801.c @@ -31,7 +31,7 @@ */ #include -#include "../comedidev.h" +#include #include #define DT2801_TIMEOUT 1000 diff --git a/drivers/comedi/drivers/dt2811.c b/drivers/comedi/drivers/dt2811.c index 0eb5e6ba6916..dbb9f38da289 100644 --- a/drivers/comedi/drivers/dt2811.c +++ b/drivers/comedi/drivers/dt2811.c @@ -40,8 +40,7 @@ #include #include #include - -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/dt2814.c b/drivers/comedi/drivers/dt2814.c index ed44ce0d151b..c98a5a4a7aec 100644 --- a/drivers/comedi/drivers/dt2814.c +++ b/drivers/comedi/drivers/dt2814.c @@ -27,8 +27,7 @@ #include #include -#include "../comedidev.h" - +#include #include #define DT2814_CSR 0 diff --git a/drivers/comedi/drivers/dt2815.c b/drivers/comedi/drivers/dt2815.c index 5906f32aa01f..03ba2fd18a21 100644 --- a/drivers/comedi/drivers/dt2815.c +++ b/drivers/comedi/drivers/dt2815.c @@ -43,8 +43,7 @@ */ #include -#include "../comedidev.h" - +#include #include #define DT2815_DATA 0 diff --git a/drivers/comedi/drivers/dt2817.c b/drivers/comedi/drivers/dt2817.c index 7c1463e835d3..6738045c7531 100644 --- a/drivers/comedi/drivers/dt2817.c +++ b/drivers/comedi/drivers/dt2817.c @@ -25,7 +25,7 @@ */ #include -#include "../comedidev.h" +#include #define DT2817_CR 0 #define DT2817_DATA 1 diff --git a/drivers/comedi/drivers/dt282x.c b/drivers/comedi/drivers/dt282x.c index 2656b4b0e3d0..078f8fba7183 100644 --- a/drivers/comedi/drivers/dt282x.c +++ b/drivers/comedi/drivers/dt282x.c @@ -51,8 +51,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" diff --git a/drivers/comedi/drivers/dt3000.c b/drivers/comedi/drivers/dt3000.c index ec27aa4730d4..fc6e9c30e522 100644 --- a/drivers/comedi/drivers/dt3000.c +++ b/drivers/comedi/drivers/dt3000.c @@ -43,8 +43,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include /* * PCI BAR0 - dual-ported RAM location definitions (dev->mmio) diff --git a/drivers/comedi/drivers/dt9812.c b/drivers/comedi/drivers/dt9812.c index 704b04d2980d..b37b9d8eca0d 100644 --- a/drivers/comedi/drivers/dt9812.c +++ b/drivers/comedi/drivers/dt9812.c @@ -34,8 +34,7 @@ #include #include #include - -#include "../comedi_usb.h" +#include #define DT9812_DIAGS_BOARD_INFO_ADDR 0xFBFF #define DT9812_MAX_WRITE_CMD_PIPE_SIZE 32 diff --git a/drivers/comedi/drivers/dyna_pci10xx.c b/drivers/comedi/drivers/dyna_pci10xx.c index c224422bb126..407a038fb3e0 100644 --- a/drivers/comedi/drivers/dyna_pci10xx.c +++ b/drivers/comedi/drivers/dyna_pci10xx.c @@ -26,8 +26,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #define READ_TIMEOUT 50 diff --git a/drivers/comedi/drivers/fl512.c b/drivers/comedi/drivers/fl512.c index b715f30659fa..139e801fc358 100644 --- a/drivers/comedi/drivers/fl512.c +++ b/drivers/comedi/drivers/fl512.c @@ -21,8 +21,7 @@ */ #include -#include "../comedidev.h" - +#include #include /* diff --git a/drivers/comedi/drivers/gsc_hpdi.c b/drivers/comedi/drivers/gsc_hpdi.c index e35e4a743714..c09d135df38d 100644 --- a/drivers/comedi/drivers/gsc_hpdi.c +++ b/drivers/comedi/drivers/gsc_hpdi.c @@ -34,8 +34,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "plx9080.h" diff --git a/drivers/comedi/drivers/icp_multi.c b/drivers/comedi/drivers/icp_multi.c index 16d2b78de83c..ac4b11dbd741 100644 --- a/drivers/comedi/drivers/icp_multi.c +++ b/drivers/comedi/drivers/icp_multi.c @@ -36,8 +36,7 @@ #include #include - -#include "../comedi_pci.h" +#include #define ICP_MULTI_ADC_CSR 0x00 /* R/W: ADC command/status register */ #define ICP_MULTI_ADC_CSR_ST BIT(0) /* Start ADC */ diff --git a/drivers/comedi/drivers/ii_pci20kc.c b/drivers/comedi/drivers/ii_pci20kc.c index 399255dbe388..4a19bf8462be 100644 --- a/drivers/comedi/drivers/ii_pci20kc.c +++ b/drivers/comedi/drivers/ii_pci20kc.c @@ -30,7 +30,7 @@ #include #include -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/jr3_pci.c b/drivers/comedi/drivers/jr3_pci.c index f963080dd61f..951c23fa0369 100644 --- a/drivers/comedi/drivers/jr3_pci.c +++ b/drivers/comedi/drivers/jr3_pci.c @@ -35,8 +35,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "jr3_pci.h" diff --git a/drivers/comedi/drivers/ke_counter.c b/drivers/comedi/drivers/ke_counter.c index bef1b20c1c8d..b825cf60e1e0 100644 --- a/drivers/comedi/drivers/ke_counter.c +++ b/drivers/comedi/drivers/ke_counter.c @@ -19,8 +19,7 @@ */ #include - -#include "../comedi_pci.h" +#include /* * PCI BAR 0 Register I/O map diff --git a/drivers/comedi/drivers/me4000.c b/drivers/comedi/drivers/me4000.c index 0d3d4cafce2e..c5dc8199771f 100644 --- a/drivers/comedi/drivers/me4000.c +++ b/drivers/comedi/drivers/me4000.c @@ -32,8 +32,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "plx9052.h" diff --git a/drivers/comedi/drivers/me_daq.c b/drivers/comedi/drivers/me_daq.c index ef18e387471b..076b15097afd 100644 --- a/drivers/comedi/drivers/me_daq.c +++ b/drivers/comedi/drivers/me_daq.c @@ -23,8 +23,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "plx9052.h" diff --git a/drivers/comedi/drivers/mf6x4.c b/drivers/comedi/drivers/mf6x4.c index 9da8dd748078..14f1d5e9cd59 100644 --- a/drivers/comedi/drivers/mf6x4.c +++ b/drivers/comedi/drivers/mf6x4.c @@ -18,8 +18,7 @@ #include #include - -#include "../comedi_pci.h" +#include /* Registers present in BAR0 memory region */ #define MF624_GPIOC_REG 0x54 diff --git a/drivers/comedi/drivers/mite.c b/drivers/comedi/drivers/mite.c index 70960e3ba878..88f3cd6f54f1 100644 --- a/drivers/comedi/drivers/mite.c +++ b/drivers/comedi/drivers/mite.c @@ -38,8 +38,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "mite.h" diff --git a/drivers/comedi/drivers/mpc624.c b/drivers/comedi/drivers/mpc624.c index 646f4c086204..9e51ff528ed1 100644 --- a/drivers/comedi/drivers/mpc624.c +++ b/drivers/comedi/drivers/mpc624.c @@ -44,8 +44,7 @@ */ #include -#include "../comedidev.h" - +#include #include /* Offsets of different ports */ diff --git a/drivers/comedi/drivers/multiq3.c b/drivers/comedi/drivers/multiq3.c index c1897aee9a9a..07ff5383da99 100644 --- a/drivers/comedi/drivers/multiq3.c +++ b/drivers/comedi/drivers/multiq3.c @@ -26,8 +26,7 @@ */ #include - -#include "../comedidev.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/ni_6527.c b/drivers/comedi/drivers/ni_6527.c index f1a45cf7342a..ac5820085231 100644 --- a/drivers/comedi/drivers/ni_6527.c +++ b/drivers/comedi/drivers/ni_6527.c @@ -20,8 +20,7 @@ #include #include - -#include "../comedi_pci.h" +#include /* * PCI BAR1 - Register memory map diff --git a/drivers/comedi/drivers/ni_65xx.c b/drivers/comedi/drivers/ni_65xx.c index 7cd8497420f2..58334de3b253 100644 --- a/drivers/comedi/drivers/ni_65xx.c +++ b/drivers/comedi/drivers/ni_65xx.c @@ -49,8 +49,7 @@ #include #include - -#include "../comedi_pci.h" +#include /* * PCI BAR1 Register Map diff --git a/drivers/comedi/drivers/ni_660x.c b/drivers/comedi/drivers/ni_660x.c index e60d0125bcb2..0679bc39e0bc 100644 --- a/drivers/comedi/drivers/ni_660x.c +++ b/drivers/comedi/drivers/ni_660x.c @@ -26,8 +26,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "mite.h" #include "ni_tio.h" diff --git a/drivers/comedi/drivers/ni_670x.c b/drivers/comedi/drivers/ni_670x.c index c197e47486be..c875d251c230 100644 --- a/drivers/comedi/drivers/ni_670x.c +++ b/drivers/comedi/drivers/ni_670x.c @@ -24,8 +24,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #define AO_VALUE_OFFSET 0x00 #define AO_CHAN_OFFSET 0x0c diff --git a/drivers/comedi/drivers/ni_at_a2150.c b/drivers/comedi/drivers/ni_at_a2150.c index 10ad7b88713e..ce5de58c499f 100644 --- a/drivers/comedi/drivers/ni_at_a2150.c +++ b/drivers/comedi/drivers/ni_at_a2150.c @@ -39,8 +39,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/ni_at_ao.c b/drivers/comedi/drivers/ni_at_ao.c index 2a0fb4d460db..a06dfb9da329 100644 --- a/drivers/comedi/drivers/ni_at_ao.c +++ b/drivers/comedi/drivers/ni_at_ao.c @@ -25,8 +25,7 @@ */ #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/ni_atmio.c b/drivers/comedi/drivers/ni_atmio.c index 56c78da475e7..f60a4e459a98 100644 --- a/drivers/comedi/drivers/ni_atmio.c +++ b/drivers/comedi/drivers/ni_atmio.c @@ -73,8 +73,7 @@ #include #include -#include "../comedidev.h" - +#include #include #include "ni_stc.h" diff --git a/drivers/comedi/drivers/ni_atmio16d.c b/drivers/comedi/drivers/ni_atmio16d.c index dffce1aa3e69..0bd4f88a2ac8 100644 --- a/drivers/comedi/drivers/ni_atmio16d.c +++ b/drivers/comedi/drivers/ni_atmio16d.c @@ -39,7 +39,7 @@ #include #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/ni_daq_700.c b/drivers/comedi/drivers/ni_daq_700.c index d40fc89f9cef..0ef20e9a8bc4 100644 --- a/drivers/comedi/drivers/ni_daq_700.c +++ b/drivers/comedi/drivers/ni_daq_700.c @@ -41,8 +41,7 @@ #include #include #include - -#include "../comedi_pcmcia.h" +#include /* daqcard700 registers */ #define DIO_W 0x04 /* WO 8bit */ diff --git a/drivers/comedi/drivers/ni_daq_dio24.c b/drivers/comedi/drivers/ni_daq_dio24.c index 44fb65afc218..84d78f2ee5ac 100644 --- a/drivers/comedi/drivers/ni_daq_dio24.c +++ b/drivers/comedi/drivers/ni_daq_dio24.c @@ -23,7 +23,7 @@ */ #include -#include "../comedi_pcmcia.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/ni_labpc.c b/drivers/comedi/drivers/ni_labpc.c index 1f4a07bd1d26..b25a8e117072 100644 --- a/drivers/comedi/drivers/ni_labpc.c +++ b/drivers/comedi/drivers/ni_labpc.c @@ -48,8 +48,7 @@ */ #include - -#include "../comedidev.h" +#include #include "ni_labpc.h" #include "ni_labpc_isadma.h" diff --git a/drivers/comedi/drivers/ni_labpc_common.c b/drivers/comedi/drivers/ni_labpc_common.c index dd97946eacaf..7c4687226450 100644 --- a/drivers/comedi/drivers/ni_labpc_common.c +++ b/drivers/comedi/drivers/ni_labpc_common.c @@ -12,8 +12,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" #include "8255.h" diff --git a/drivers/comedi/drivers/ni_labpc_cs.c b/drivers/comedi/drivers/ni_labpc_cs.c index 4f7e2fe21254..62fecb50ec6e 100644 --- a/drivers/comedi/drivers/ni_labpc_cs.c +++ b/drivers/comedi/drivers/ni_labpc_cs.c @@ -38,8 +38,7 @@ */ #include - -#include "../comedi_pcmcia.h" +#include #include "ni_labpc.h" diff --git a/drivers/comedi/drivers/ni_labpc_isadma.c b/drivers/comedi/drivers/ni_labpc_isadma.c index a551aca6e615..dd37ec0d9b15 100644 --- a/drivers/comedi/drivers/ni_labpc_isadma.c +++ b/drivers/comedi/drivers/ni_labpc_isadma.c @@ -10,8 +10,7 @@ #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "ni_labpc.h" diff --git a/drivers/comedi/drivers/ni_labpc_pci.c b/drivers/comedi/drivers/ni_labpc_pci.c index ec180b0fedf7..e2a44bbd9fa6 100644 --- a/drivers/comedi/drivers/ni_labpc_pci.c +++ b/drivers/comedi/drivers/ni_labpc_pci.c @@ -22,8 +22,7 @@ #include #include - -#include "../comedi_pci.h" +#include #include "ni_labpc.h" diff --git a/drivers/comedi/drivers/ni_mio_cs.c b/drivers/comedi/drivers/ni_mio_cs.c index 4f37b4e58f09..bd967cdb2036 100644 --- a/drivers/comedi/drivers/ni_mio_cs.c +++ b/drivers/comedi/drivers/ni_mio_cs.c @@ -28,8 +28,8 @@ #include #include +#include -#include "../comedi_pcmcia.h" #include "ni_stc.h" #include "8255.h" diff --git a/drivers/comedi/drivers/ni_pcidio.c b/drivers/comedi/drivers/ni_pcidio.c index 623f8d08d13a..2d58e83420e8 100644 --- a/drivers/comedi/drivers/ni_pcidio.c +++ b/drivers/comedi/drivers/ni_pcidio.c @@ -42,8 +42,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "mite.h" diff --git a/drivers/comedi/drivers/ni_pcimio.c b/drivers/comedi/drivers/ni_pcimio.c index 6c813a490ba5..0b055321023d 100644 --- a/drivers/comedi/drivers/ni_pcimio.c +++ b/drivers/comedi/drivers/ni_pcimio.c @@ -94,9 +94,7 @@ #include #include - -#include "../comedi_pci.h" - +#include #include #include "ni_stc.h" diff --git a/drivers/comedi/drivers/ni_routes.c b/drivers/comedi/drivers/ni_routes.c index f0f8cd424b30..f24eeb464eba 100644 --- a/drivers/comedi/drivers/ni_routes.c +++ b/drivers/comedi/drivers/ni_routes.c @@ -21,8 +21,7 @@ #include #include #include - -#include "../comedi.h" +#include #include "ni_routes.h" #include "ni_routing/ni_route_values.h" diff --git a/drivers/comedi/drivers/ni_routes.h b/drivers/comedi/drivers/ni_routes.h index 036982315584..cff8a463a03f 100644 --- a/drivers/comedi/drivers/ni_routes.h +++ b/drivers/comedi/drivers/ni_routes.h @@ -27,7 +27,7 @@ #include #endif -#include "../comedi.h" +#include /** * struct ni_route_set - Set of destinations with a common source. diff --git a/drivers/comedi/drivers/ni_routing/ni_route_values.h b/drivers/comedi/drivers/ni_routing/ni_route_values.h index 6e358efa6f7f..80880083ea41 100644 --- a/drivers/comedi/drivers/ni_routing/ni_route_values.h +++ b/drivers/comedi/drivers/ni_routing/ni_route_values.h @@ -20,7 +20,7 @@ #ifndef _COMEDI_DRIVERS_NI_ROUTINT_NI_ROUTE_VALUES_H #define _COMEDI_DRIVERS_NI_ROUTINT_NI_ROUTE_VALUES_H -#include "../../comedi.h" +#include #include /* diff --git a/drivers/comedi/drivers/ni_tio.h b/drivers/comedi/drivers/ni_tio.h index e7b05718df9b..9ae2221c3c18 100644 --- a/drivers/comedi/drivers/ni_tio.h +++ b/drivers/comedi/drivers/ni_tio.h @@ -8,7 +8,7 @@ #ifndef _COMEDI_NI_TIO_H #define _COMEDI_NI_TIO_H -#include "../comedidev.h" +#include enum ni_gpct_register { NITIO_G0_AUTO_INC, diff --git a/drivers/comedi/drivers/ni_usb6501.c b/drivers/comedi/drivers/ni_usb6501.c index c42987b74b1d..0dd9edf7bced 100644 --- a/drivers/comedi/drivers/ni_usb6501.c +++ b/drivers/comedi/drivers/ni_usb6501.c @@ -87,8 +87,7 @@ #include #include #include - -#include "../comedi_usb.h" +#include #define NI6501_TIMEOUT 1000 diff --git a/drivers/comedi/drivers/pcl711.c b/drivers/comedi/drivers/pcl711.c index bd6f42fe9e3c..f1c383bd9d87 100644 --- a/drivers/comedi/drivers/pcl711.c +++ b/drivers/comedi/drivers/pcl711.c @@ -29,8 +29,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/pcl724.c b/drivers/comedi/drivers/pcl724.c index 1a5799278a7a..b3f472c93e80 100644 --- a/drivers/comedi/drivers/pcl724.c +++ b/drivers/comedi/drivers/pcl724.c @@ -25,7 +25,7 @@ */ #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/pcl726.c b/drivers/comedi/drivers/pcl726.c index 88f25d7e76f7..0430630e6ebb 100644 --- a/drivers/comedi/drivers/pcl726.c +++ b/drivers/comedi/drivers/pcl726.c @@ -50,8 +50,7 @@ #include #include - -#include "../comedidev.h" +#include #define PCL726_AO_MSB_REG(x) (0x00 + ((x) * 2)) #define PCL726_AO_LSB_REG(x) (0x01 + ((x) * 2)) diff --git a/drivers/comedi/drivers/pcl730.c b/drivers/comedi/drivers/pcl730.c index 32a29129e6e8..d2733cd5383d 100644 --- a/drivers/comedi/drivers/pcl730.c +++ b/drivers/comedi/drivers/pcl730.c @@ -25,7 +25,7 @@ */ #include -#include "../comedidev.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/pcl812.c b/drivers/comedi/drivers/pcl812.c index b87ab3840eee..f00976ddfc2a 100644 --- a/drivers/comedi/drivers/pcl812.c +++ b/drivers/comedi/drivers/pcl812.c @@ -114,8 +114,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/pcl816.c b/drivers/comedi/drivers/pcl816.c index c368a337a0ae..c5acdc8913f8 100644 --- a/drivers/comedi/drivers/pcl816.c +++ b/drivers/comedi/drivers/pcl816.c @@ -35,8 +35,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/pcl818.c b/drivers/comedi/drivers/pcl818.c index f4b4a686c710..20fcd6d588f8 100644 --- a/drivers/comedi/drivers/pcl818.c +++ b/drivers/comedi/drivers/pcl818.c @@ -97,8 +97,7 @@ #include #include #include - -#include "../comedidev.h" +#include #include "comedi_isadma.h" #include "comedi_8254.h" diff --git a/drivers/comedi/drivers/pcm3724.c b/drivers/comedi/drivers/pcm3724.c index 0cb1ad060402..93ae6cffed44 100644 --- a/drivers/comedi/drivers/pcm3724.c +++ b/drivers/comedi/drivers/pcm3724.c @@ -24,7 +24,7 @@ */ #include -#include "../comedidev.h" +#include #include "8255.h" diff --git a/drivers/comedi/drivers/pcmad.c b/drivers/comedi/drivers/pcmad.c index eec89a0afb2f..976eda43881b 100644 --- a/drivers/comedi/drivers/pcmad.c +++ b/drivers/comedi/drivers/pcmad.c @@ -29,7 +29,7 @@ */ #include -#include "../comedidev.h" +#include #define PCMAD_STATUS 0 #define PCMAD_LSB 1 diff --git a/drivers/comedi/drivers/pcmda12.c b/drivers/comedi/drivers/pcmda12.c index 14ab1f0d1e9f..611f13bedca0 100644 --- a/drivers/comedi/drivers/pcmda12.c +++ b/drivers/comedi/drivers/pcmda12.c @@ -40,7 +40,7 @@ */ #include -#include "../comedidev.h" +#include /* AI range is not configurable, it's set by jumpers on the board */ static const struct comedi_lrange pcmda12_ranges = { diff --git a/drivers/comedi/drivers/pcmmio.c b/drivers/comedi/drivers/pcmmio.c index 24a9568d3378..c2402239d551 100644 --- a/drivers/comedi/drivers/pcmmio.c +++ b/drivers/comedi/drivers/pcmmio.c @@ -66,8 +66,7 @@ #include #include #include - -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/pcmuio.c b/drivers/comedi/drivers/pcmuio.c index b299d648a0eb..33b24dbbb919 100644 --- a/drivers/comedi/drivers/pcmuio.c +++ b/drivers/comedi/drivers/pcmuio.c @@ -65,8 +65,7 @@ #include #include - -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/quatech_daqp_cs.c b/drivers/comedi/drivers/quatech_daqp_cs.c index fe4408ebf6b3..2a76c75c513b 100644 --- a/drivers/comedi/drivers/quatech_daqp_cs.c +++ b/drivers/comedi/drivers/quatech_daqp_cs.c @@ -41,8 +41,7 @@ */ #include - -#include "../comedi_pcmcia.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/rtd520.c b/drivers/comedi/drivers/rtd520.c index 2d99a648b054..ee5bca2b1c09 100644 --- a/drivers/comedi/drivers/rtd520.c +++ b/drivers/comedi/drivers/rtd520.c @@ -85,8 +85,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "comedi_8254.h" #include "plx9080.h" diff --git a/drivers/comedi/drivers/rti800.c b/drivers/comedi/drivers/rti800.c index 327fd93b8b12..1b02e47bdb4c 100644 --- a/drivers/comedi/drivers/rti800.c +++ b/drivers/comedi/drivers/rti800.c @@ -42,7 +42,7 @@ #include #include #include -#include "../comedidev.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/rti802.c b/drivers/comedi/drivers/rti802.c index 195e2b1ac4c1..d66762a22258 100644 --- a/drivers/comedi/drivers/rti802.c +++ b/drivers/comedi/drivers/rti802.c @@ -22,7 +22,7 @@ */ #include -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/s526.c b/drivers/comedi/drivers/s526.c index 085cf5b449e5..9245c679a3c4 100644 --- a/drivers/comedi/drivers/s526.c +++ b/drivers/comedi/drivers/s526.c @@ -27,7 +27,7 @@ */ #include -#include "../comedidev.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/s626.c b/drivers/comedi/drivers/s626.c index e7aba937d896..0e5f9a9a7fd3 100644 --- a/drivers/comedi/drivers/s626.c +++ b/drivers/comedi/drivers/s626.c @@ -55,8 +55,7 @@ #include #include #include - -#include "../comedi_pci.h" +#include #include "s626.h" diff --git a/drivers/comedi/drivers/ssv_dnp.c b/drivers/comedi/drivers/ssv_dnp.c index 016d315aa584..813bd0853b0b 100644 --- a/drivers/comedi/drivers/ssv_dnp.c +++ b/drivers/comedi/drivers/ssv_dnp.c @@ -19,7 +19,7 @@ /* include files ----------------------------------------------------------- */ #include -#include "../comedidev.h" +#include /* Some global definitions: the registers of the DNP ----------------------- */ /* */ diff --git a/drivers/comedi/drivers/usbdux.c b/drivers/comedi/drivers/usbdux.c index 0350f303d557..92d514b3c1c3 100644 --- a/drivers/comedi/drivers/usbdux.c +++ b/drivers/comedi/drivers/usbdux.c @@ -73,8 +73,7 @@ #include #include #include - -#include "../comedi_usb.h" +#include /* constants for firmware upload and download */ #define USBDUX_FIRMWARE "usbdux_firmware.bin" diff --git a/drivers/comedi/drivers/usbduxfast.c b/drivers/comedi/drivers/usbduxfast.c index 4af012968cb6..39faae0ecb19 100644 --- a/drivers/comedi/drivers/usbduxfast.c +++ b/drivers/comedi/drivers/usbduxfast.c @@ -40,7 +40,7 @@ #include #include #include -#include "../comedi_usb.h" +#include /* * timeout for the USB-transfer diff --git a/drivers/comedi/drivers/usbduxsigma.c b/drivers/comedi/drivers/usbduxsigma.c index 54d7605e909f..2aaeaf44fbe5 100644 --- a/drivers/comedi/drivers/usbduxsigma.c +++ b/drivers/comedi/drivers/usbduxsigma.c @@ -40,8 +40,7 @@ #include #include #include - -#include "../comedi_usb.h" +#include /* timeout for the USB-transfer in ms*/ #define BULK_TIMEOUT 1000 diff --git a/drivers/comedi/drivers/vmk80xx.c b/drivers/comedi/drivers/vmk80xx.c index 4b00a9ea611a..46023adc5395 100644 --- a/drivers/comedi/drivers/vmk80xx.c +++ b/drivers/comedi/drivers/vmk80xx.c @@ -35,8 +35,7 @@ #include #include #include - -#include "../comedi_usb.h" +#include enum { DEVICE_VMK8055, diff --git a/drivers/comedi/kcomedilib/kcomedilib_main.c b/drivers/comedi/kcomedilib/kcomedilib_main.c index df9bba1b69ed..43fbe1a63b14 100644 --- a/drivers/comedi/kcomedilib/kcomedilib_main.c +++ b/drivers/comedi/kcomedilib/kcomedilib_main.c @@ -16,9 +16,9 @@ #include #include -#include "../comedi.h" -#include "../comedilib.h" -#include "../comedidev.h" +#include +#include +#include MODULE_AUTHOR("David Schleef "); MODULE_DESCRIPTION("Comedi kernel library"); diff --git a/drivers/comedi/proc.c b/drivers/comedi/proc.c index 8bc8e42beb90..2e4496633d3d 100644 --- a/drivers/comedi/proc.c +++ b/drivers/comedi/proc.c @@ -13,7 +13,7 @@ * was cool. */ -#include "comedidev.h" +#include #include "comedi_internal.h" #include #include diff --git a/drivers/comedi/range.c b/drivers/comedi/range.c index a4e6fe0fb729..8f43cf88d784 100644 --- a/drivers/comedi/range.c +++ b/drivers/comedi/range.c @@ -8,7 +8,7 @@ */ #include -#include "comedidev.h" +#include #include "comedi_internal.h" const struct comedi_lrange range_bipolar10 = { 1, {BIP_RANGE(10)} }; diff --git a/include/linux/comedi/comedi_pci.h b/include/linux/comedi/comedi_pci.h new file mode 100644 index 000000000000..2fb50663e3ed --- /dev/null +++ b/include/linux/comedi/comedi_pci.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedi_pci.h + * header file for Comedi PCI drivers + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1997-2000 David A. Schleef + */ + +#ifndef _COMEDI_PCI_H +#define _COMEDI_PCI_H + +#include +#include + +/* + * PCI Vendor IDs not in + */ +#define PCI_VENDOR_ID_KOLTER 0x1001 +#define PCI_VENDOR_ID_ICP 0x104c +#define PCI_VENDOR_ID_DT 0x1116 +#define PCI_VENDOR_ID_IOTECH 0x1616 +#define PCI_VENDOR_ID_CONTEC 0x1221 +#define PCI_VENDOR_ID_RTD 0x1435 +#define PCI_VENDOR_ID_HUMUSOFT 0x186c + +struct pci_dev *comedi_to_pci_dev(struct comedi_device *dev); + +int comedi_pci_enable(struct comedi_device *dev); +void comedi_pci_disable(struct comedi_device *dev); +void comedi_pci_detach(struct comedi_device *dev); + +int comedi_pci_auto_config(struct pci_dev *pcidev, struct comedi_driver *driver, + unsigned long context); +void comedi_pci_auto_unconfig(struct pci_dev *pcidev); + +int comedi_pci_driver_register(struct comedi_driver *comedi_driver, + struct pci_driver *pci_driver); +void comedi_pci_driver_unregister(struct comedi_driver *comedi_driver, + struct pci_driver *pci_driver); + +/** + * module_comedi_pci_driver() - Helper macro for registering a comedi PCI driver + * @__comedi_driver: comedi_driver struct + * @__pci_driver: pci_driver struct + * + * Helper macro for comedi PCI drivers which do not do anything special + * in module init/exit. This eliminates a lot of boilerplate. Each + * module may only use this macro once, and calling it replaces + * module_init() and module_exit() + */ +#define module_comedi_pci_driver(__comedi_driver, __pci_driver) \ + module_driver(__comedi_driver, comedi_pci_driver_register, \ + comedi_pci_driver_unregister, &(__pci_driver)) + +#endif /* _COMEDI_PCI_H */ diff --git a/include/linux/comedi/comedi_pcmcia.h b/include/linux/comedi/comedi_pcmcia.h new file mode 100644 index 000000000000..a33dfb65b869 --- /dev/null +++ b/include/linux/comedi/comedi_pcmcia.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedi_pcmcia.h + * header file for Comedi PCMCIA drivers + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1997-2000 David A. Schleef + */ + +#ifndef _COMEDI_PCMCIA_H +#define _COMEDI_PCMCIA_H + +#include +#include +#include + +struct pcmcia_device *comedi_to_pcmcia_dev(struct comedi_device *dev); + +int comedi_pcmcia_enable(struct comedi_device *dev, + int (*conf_check)(struct pcmcia_device *p_dev, + void *priv_data)); +void comedi_pcmcia_disable(struct comedi_device *dev); + +int comedi_pcmcia_auto_config(struct pcmcia_device *link, + struct comedi_driver *driver); +void comedi_pcmcia_auto_unconfig(struct pcmcia_device *link); + +int comedi_pcmcia_driver_register(struct comedi_driver *comedi_driver, + struct pcmcia_driver *pcmcia_driver); +void comedi_pcmcia_driver_unregister(struct comedi_driver *comedi_driver, + struct pcmcia_driver *pcmcia_driver); + +/** + * module_comedi_pcmcia_driver() - Helper macro for registering a comedi + * PCMCIA driver + * @__comedi_driver: comedi_driver struct + * @__pcmcia_driver: pcmcia_driver struct + * + * Helper macro for comedi PCMCIA drivers which do not do anything special + * in module init/exit. This eliminates a lot of boilerplate. Each + * module may only use this macro once, and calling it replaces + * module_init() and module_exit() + */ +#define module_comedi_pcmcia_driver(__comedi_driver, __pcmcia_driver) \ + module_driver(__comedi_driver, comedi_pcmcia_driver_register, \ + comedi_pcmcia_driver_unregister, &(__pcmcia_driver)) + +#endif /* _COMEDI_PCMCIA_H */ diff --git a/include/linux/comedi/comedi_usb.h b/include/linux/comedi/comedi_usb.h new file mode 100644 index 000000000000..5d17dd425bd2 --- /dev/null +++ b/include/linux/comedi/comedi_usb.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* comedi_usb.h + * header file for USB Comedi drivers + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1997-2000 David A. Schleef + */ + +#ifndef _COMEDI_USB_H +#define _COMEDI_USB_H + +#include +#include + +struct usb_interface *comedi_to_usb_interface(struct comedi_device *dev); +struct usb_device *comedi_to_usb_dev(struct comedi_device *dev); + +int comedi_usb_auto_config(struct usb_interface *intf, + struct comedi_driver *driver, unsigned long context); +void comedi_usb_auto_unconfig(struct usb_interface *intf); + +int comedi_usb_driver_register(struct comedi_driver *comedi_driver, + struct usb_driver *usb_driver); +void comedi_usb_driver_unregister(struct comedi_driver *comedi_driver, + struct usb_driver *usb_driver); + +/** + * module_comedi_usb_driver() - Helper macro for registering a comedi USB driver + * @__comedi_driver: comedi_driver struct + * @__usb_driver: usb_driver struct + * + * Helper macro for comedi USB drivers which do not do anything special + * in module init/exit. This eliminates a lot of boilerplate. Each + * module may only use this macro once, and calling it replaces + * module_init() and module_exit() + */ +#define module_comedi_usb_driver(__comedi_driver, __usb_driver) \ + module_driver(__comedi_driver, comedi_usb_driver_register, \ + comedi_usb_driver_unregister, &(__usb_driver)) + +#endif /* _COMEDI_USB_H */ diff --git a/include/linux/comedi/comedidev.h b/include/linux/comedi/comedidev.h new file mode 100644 index 000000000000..0a1150900ef3 --- /dev/null +++ b/include/linux/comedi/comedidev.h @@ -0,0 +1,1053 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedidev.h + * header file for kernel-only structures, variables, and constants + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1997-2000 David A. Schleef + */ + +#ifndef _COMEDIDEV_H +#define _COMEDIDEV_H + +#include +#include +#include +#include +#include +#include + +#define COMEDI_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) +#define COMEDI_VERSION_CODE COMEDI_VERSION(COMEDI_MAJORVERSION, \ + COMEDI_MINORVERSION, COMEDI_MICROVERSION) +#define COMEDI_RELEASE VERSION + +#define COMEDI_NUM_BOARD_MINORS 0x30 + +/** + * struct comedi_subdevice - Working data for a COMEDI subdevice + * @device: COMEDI device to which this subdevice belongs. (Initialized by + * comedi_alloc_subdevices().) + * @index: Index of this subdevice within device's array of subdevices. + * (Initialized by comedi_alloc_subdevices().) + * @type: Type of subdevice from &enum comedi_subdevice_type. (Initialized by + * the low-level driver.) + * @n_chan: Number of channels the subdevice supports. (Initialized by the + * low-level driver.) + * @subdev_flags: Various "SDF" flags indicating aspects of the subdevice to + * the COMEDI core and user application. (Initialized by the low-level + * driver.) + * @len_chanlist: Maximum length of a channel list if the subdevice supports + * asynchronous acquisition commands. (Optionally initialized by the + * low-level driver, or changed from 0 to 1 during post-configuration.) + * @private: Private data pointer which is either set by the low-level driver + * itself, or by a call to comedi_alloc_spriv() which allocates storage. + * In the latter case, the storage is automatically freed after the + * low-level driver's "detach" handler is called for the device. + * (Initialized by the low-level driver.) + * @async: Pointer to &struct comedi_async id the subdevice supports + * asynchronous acquisition commands. (Allocated and initialized during + * post-configuration if needed.) + * @lock: Pointer to a file object that performed a %COMEDI_LOCK ioctl on the + * subdevice. (Initially NULL.) + * @busy: Pointer to a file object that is performing an asynchronous + * acquisition command on the subdevice. (Initially NULL.) + * @runflags: Internal flags for use by COMEDI core, mostly indicating whether + * an asynchronous acquisition command is running. + * @spin_lock: Generic spin-lock for use by the COMEDI core and the low-level + * driver. (Initialized by comedi_alloc_subdevices().) + * @io_bits: Bit-mask indicating the channel directions for a DIO subdevice + * with no more than 32 channels. A '1' at a bit position indicates the + * corresponding channel is configured as an output. (Initialized by the + * low-level driver for a DIO subdevice. Forced to all-outputs during + * post-configuration for a digital output subdevice.) + * @maxdata: If non-zero, this is the maximum raw data value of each channel. + * If zero, the maximum data value is channel-specific. (Initialized by + * the low-level driver.) + * @maxdata_list: If the maximum data value is channel-specific, this points + * to an array of maximum data values indexed by channel index. + * (Initialized by the low-level driver.) + * @range_table: If non-NULL, this points to a COMEDI range table for the + * subdevice. If NULL, the range table is channel-specific. (Initialized + * by the low-level driver, will be set to an "invalid" range table during + * post-configuration if @range_table and @range_table_list are both + * NULL.) + * @range_table_list: If the COMEDI range table is channel-specific, this + * points to an array of pointers to COMEDI range tables indexed by + * channel number. (Initialized by the low-level driver.) + * @chanlist: Not used. + * @insn_read: Optional pointer to a handler for the %INSN_READ instruction. + * (Initialized by the low-level driver, or set to a default handler + * during post-configuration.) + * @insn_write: Optional pointer to a handler for the %INSN_WRITE instruction. + * (Initialized by the low-level driver, or set to a default handler + * during post-configuration.) + * @insn_bits: Optional pointer to a handler for the %INSN_BITS instruction + * for a digital input, digital output or digital input/output subdevice. + * (Initialized by the low-level driver, or set to a default handler + * during post-configuration.) + * @insn_config: Optional pointer to a handler for the %INSN_CONFIG + * instruction. (Initialized by the low-level driver, or set to a default + * handler during post-configuration.) + * @do_cmd: If the subdevice supports asynchronous acquisition commands, this + * points to a handler to set it up in hardware. (Initialized by the + * low-level driver.) + * @do_cmdtest: If the subdevice supports asynchronous acquisition commands, + * this points to a handler used to check and possibly tweak a prospective + * acquisition command without setting it up in hardware. (Initialized by + * the low-level driver.) + * @poll: If the subdevice supports asynchronous acquisition commands, this + * is an optional pointer to a handler for the %COMEDI_POLL ioctl which + * instructs the low-level driver to synchronize buffers. (Initialized by + * the low-level driver if needed.) + * @cancel: If the subdevice supports asynchronous acquisition commands, this + * points to a handler used to terminate a running command. (Initialized + * by the low-level driver.) + * @buf_change: If the subdevice supports asynchronous acquisition commands, + * this is an optional pointer to a handler that is called when the data + * buffer for handling asynchronous commands is allocated or reallocated. + * (Initialized by the low-level driver if needed.) + * @munge: If the subdevice supports asynchronous acquisition commands and + * uses DMA to transfer data from the hardware to the acquisition buffer, + * this points to a function used to "munge" the data values from the + * hardware into the format expected by COMEDI. (Initialized by the + * low-level driver if needed.) + * @async_dma_dir: If the subdevice supports asynchronous acquisition commands + * and uses DMA to transfer data from the hardware to the acquisition + * buffer, this sets the DMA direction for the buffer. (initialized to + * %DMA_NONE by comedi_alloc_subdevices() and changed by the low-level + * driver if necessary.) + * @state: Handy bit-mask indicating the output states for a DIO or digital + * output subdevice with no more than 32 channels. (Initialized by the + * low-level driver.) + * @class_dev: If the subdevice supports asynchronous acquisition commands, + * this points to a sysfs comediX_subdY device where X is the minor device + * number of the COMEDI device and Y is the subdevice number. The minor + * device number for the sysfs device is allocated dynamically in the + * range 48 to 255. This is used to allow the COMEDI device to be opened + * with a different default read or write subdevice. (Allocated during + * post-configuration if needed.) + * @minor: If @class_dev is set, this is its dynamically allocated minor + * device number. (Set during post-configuration if necessary.) + * @readback: Optional pointer to memory allocated by + * comedi_alloc_subdev_readback() used to hold the values written to + * analog output channels so they can be read back. The storage is + * automatically freed after the low-level driver's "detach" handler is + * called for the device. (Initialized by the low-level driver.) + * + * This is the main control structure for a COMEDI subdevice. If the subdevice + * supports asynchronous acquisition commands, additional information is stored + * in the &struct comedi_async pointed to by @async. + * + * Most of the subdevice is initialized by the low-level driver's "attach" or + * "auto_attach" handlers but parts of it are initialized by + * comedi_alloc_subdevices(), and other parts are initialized during + * post-configuration on return from that handler. + * + * A low-level driver that sets @insn_bits for a digital input, digital output, + * or DIO subdevice may leave @insn_read and @insn_write uninitialized, in + * which case they will be set to a default handler during post-configuration + * that uses @insn_bits to emulate the %INSN_READ and %INSN_WRITE instructions. + */ +struct comedi_subdevice { + struct comedi_device *device; + int index; + int type; + int n_chan; + int subdev_flags; + int len_chanlist; /* maximum length of channel/gain list */ + + void *private; + + struct comedi_async *async; + + void *lock; + void *busy; + unsigned int runflags; + spinlock_t spin_lock; /* generic spin-lock for COMEDI and drivers */ + + unsigned int io_bits; + + unsigned int maxdata; /* if maxdata==0, use list */ + const unsigned int *maxdata_list; /* list is channel specific */ + + const struct comedi_lrange *range_table; + const struct comedi_lrange *const *range_table_list; + + unsigned int *chanlist; /* driver-owned chanlist (not used) */ + + int (*insn_read)(struct comedi_device *dev, struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data); + int (*insn_write)(struct comedi_device *dev, struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data); + int (*insn_bits)(struct comedi_device *dev, struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data); + int (*insn_config)(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_insn *insn, + unsigned int *data); + + int (*do_cmd)(struct comedi_device *dev, struct comedi_subdevice *s); + int (*do_cmdtest)(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_cmd *cmd); + int (*poll)(struct comedi_device *dev, struct comedi_subdevice *s); + int (*cancel)(struct comedi_device *dev, struct comedi_subdevice *s); + + /* called when the buffer changes */ + int (*buf_change)(struct comedi_device *dev, + struct comedi_subdevice *s); + + void (*munge)(struct comedi_device *dev, struct comedi_subdevice *s, + void *data, unsigned int num_bytes, + unsigned int start_chan_index); + enum dma_data_direction async_dma_dir; + + unsigned int state; + + struct device *class_dev; + int minor; + + unsigned int *readback; +}; + +/** + * struct comedi_buf_page - Describe a page of a COMEDI buffer + * @virt_addr: Kernel address of page. + * @dma_addr: DMA address of page if in DMA coherent memory. + */ +struct comedi_buf_page { + void *virt_addr; + dma_addr_t dma_addr; +}; + +/** + * struct comedi_buf_map - Describe pages in a COMEDI buffer + * @dma_hw_dev: Low-level hardware &struct device pointer copied from the + * COMEDI device's hw_dev member. + * @page_list: Pointer to array of &struct comedi_buf_page, one for each + * page in the buffer. + * @n_pages: Number of pages in the buffer. + * @dma_dir: DMA direction used to allocate pages of DMA coherent memory, + * or %DMA_NONE if pages allocated from regular memory. + * @refcount: &struct kref reference counter used to free the buffer. + * + * A COMEDI data buffer is allocated as individual pages, either in + * conventional memory or DMA coherent memory, depending on the attached, + * low-level hardware device. (The buffer pages also get mapped into the + * kernel's contiguous virtual address space pointed to by the 'prealloc_buf' + * member of &struct comedi_async.) + * + * The buffer is normally freed when the COMEDI device is detached from the + * low-level driver (which may happen due to device removal), but if it happens + * to be mmapped at the time, the pages cannot be freed until the buffer has + * been munmapped. That is what the reference counter is for. (The virtual + * address space pointed by 'prealloc_buf' is freed when the COMEDI device is + * detached.) + */ +struct comedi_buf_map { + struct device *dma_hw_dev; + struct comedi_buf_page *page_list; + unsigned int n_pages; + enum dma_data_direction dma_dir; + struct kref refcount; +}; + +/** + * struct comedi_async - Control data for asynchronous COMEDI commands + * @prealloc_buf: Kernel virtual address of allocated acquisition buffer. + * @prealloc_bufsz: Buffer size (in bytes). + * @buf_map: Map of buffer pages. + * @max_bufsize: Maximum allowed buffer size (in bytes). + * @buf_write_count: "Write completed" count (in bytes, modulo 2**32). + * @buf_write_alloc_count: "Allocated for writing" count (in bytes, + * modulo 2**32). + * @buf_read_count: "Read completed" count (in bytes, modulo 2**32). + * @buf_read_alloc_count: "Allocated for reading" count (in bytes, + * modulo 2**32). + * @buf_write_ptr: Buffer position for writer. + * @buf_read_ptr: Buffer position for reader. + * @cur_chan: Current position in chanlist for scan (for those drivers that + * use it). + * @scans_done: The number of scans completed. + * @scan_progress: Amount received or sent for current scan (in bytes). + * @munge_chan: Current position in chanlist for "munging". + * @munge_count: "Munge" count (in bytes, modulo 2**32). + * @munge_ptr: Buffer position for "munging". + * @events: Bit-vector of events that have occurred. + * @cmd: Details of comedi command in progress. + * @wait_head: Task wait queue for file reader or writer. + * @cb_mask: Bit-vector of events that should wake waiting tasks. + * @inttrig: Software trigger function for command, or NULL. + * + * Note about the ..._count and ..._ptr members: + * + * Think of the _Count values being integers of unlimited size, indexing + * into a buffer of infinite length (though only an advancing portion + * of the buffer of fixed length prealloc_bufsz is accessible at any + * time). Then: + * + * Buf_Read_Count <= Buf_Read_Alloc_Count <= Munge_Count <= + * Buf_Write_Count <= Buf_Write_Alloc_Count <= + * (Buf_Read_Count + prealloc_bufsz) + * + * (Those aren't the actual members, apart from prealloc_bufsz.) When the + * buffer is reset, those _Count values start at 0 and only increase in value, + * maintaining the above inequalities until the next time the buffer is + * reset. The buffer is divided into the following regions by the inequalities: + * + * [0, Buf_Read_Count): + * old region no longer accessible + * + * [Buf_Read_Count, Buf_Read_Alloc_Count): + * filled and munged region allocated for reading but not yet read + * + * [Buf_Read_Alloc_Count, Munge_Count): + * filled and munged region not yet allocated for reading + * + * [Munge_Count, Buf_Write_Count): + * filled region not yet munged + * + * [Buf_Write_Count, Buf_Write_Alloc_Count): + * unfilled region allocated for writing but not yet written + * + * [Buf_Write_Alloc_Count, Buf_Read_Count + prealloc_bufsz): + * unfilled region not yet allocated for writing + * + * [Buf_Read_Count + prealloc_bufsz, infinity): + * unfilled region not yet accessible + * + * Data needs to be written into the buffer before it can be read out, + * and may need to be converted (or "munged") between the two + * operations. Extra unfilled buffer space may need to allocated for + * writing (advancing Buf_Write_Alloc_Count) before new data is written. + * After writing new data, the newly filled space needs to be released + * (advancing Buf_Write_Count). This also results in the new data being + * "munged" (advancing Munge_Count). Before data is read out of the + * buffer, extra space may need to be allocated for reading (advancing + * Buf_Read_Alloc_Count). After the data has been read out, the space + * needs to be released (advancing Buf_Read_Count). + * + * The actual members, buf_read_count, buf_read_alloc_count, + * munge_count, buf_write_count, and buf_write_alloc_count take the + * value of the corresponding capitalized _Count values modulo 2^32 + * (UINT_MAX+1). Subtracting a "higher" _count value from a "lower" + * _count value gives the same answer as subtracting a "higher" _Count + * value from a lower _Count value because prealloc_bufsz < UINT_MAX+1. + * The modulo operation is done implicitly. + * + * The buf_read_ptr, munge_ptr, and buf_write_ptr members take the value + * of the corresponding capitalized _Count values modulo prealloc_bufsz. + * These correspond to byte indices in the physical buffer. The modulo + * operation is done by subtracting prealloc_bufsz when the value + * exceeds prealloc_bufsz (assuming prealloc_bufsz plus the increment is + * less than or equal to UINT_MAX). + */ +struct comedi_async { + void *prealloc_buf; + unsigned int prealloc_bufsz; + struct comedi_buf_map *buf_map; + unsigned int max_bufsize; + unsigned int buf_write_count; + unsigned int buf_write_alloc_count; + unsigned int buf_read_count; + unsigned int buf_read_alloc_count; + unsigned int buf_write_ptr; + unsigned int buf_read_ptr; + unsigned int cur_chan; + unsigned int scans_done; + unsigned int scan_progress; + unsigned int munge_chan; + unsigned int munge_count; + unsigned int munge_ptr; + unsigned int events; + struct comedi_cmd cmd; + wait_queue_head_t wait_head; + unsigned int cb_mask; + int (*inttrig)(struct comedi_device *dev, struct comedi_subdevice *s, + unsigned int x); +}; + +/** + * enum comedi_cb - &struct comedi_async callback "events" + * @COMEDI_CB_EOS: end-of-scan + * @COMEDI_CB_EOA: end-of-acquisition/output + * @COMEDI_CB_BLOCK: data has arrived, wakes up read() / write() + * @COMEDI_CB_EOBUF: DEPRECATED: end of buffer + * @COMEDI_CB_ERROR: card error during acquisition + * @COMEDI_CB_OVERFLOW: buffer overflow/underflow + * @COMEDI_CB_ERROR_MASK: events that indicate an error has occurred + * @COMEDI_CB_CANCEL_MASK: events that will cancel an async command + */ +enum comedi_cb { + COMEDI_CB_EOS = BIT(0), + COMEDI_CB_EOA = BIT(1), + COMEDI_CB_BLOCK = BIT(2), + COMEDI_CB_EOBUF = BIT(3), + COMEDI_CB_ERROR = BIT(4), + COMEDI_CB_OVERFLOW = BIT(5), + /* masks */ + COMEDI_CB_ERROR_MASK = (COMEDI_CB_ERROR | COMEDI_CB_OVERFLOW), + COMEDI_CB_CANCEL_MASK = (COMEDI_CB_EOA | COMEDI_CB_ERROR_MASK) +}; + +/** + * struct comedi_driver - COMEDI driver registration + * @driver_name: Name of driver. + * @module: Owning module. + * @attach: The optional "attach" handler for manually configured COMEDI + * devices. + * @detach: The "detach" handler for deconfiguring COMEDI devices. + * @auto_attach: The optional "auto_attach" handler for automatically + * configured COMEDI devices. + * @num_names: Optional number of "board names" supported. + * @board_name: Optional pointer to a pointer to a board name. The pointer + * to a board name is embedded in an element of a driver-defined array + * of static, read-only board type information. + * @offset: Optional size of each element of the driver-defined array of + * static, read-only board type information, i.e. the offset between each + * pointer to a board name. + * + * This is used with comedi_driver_register() and comedi_driver_unregister() to + * register and unregister a low-level COMEDI driver with the COMEDI core. + * + * If @num_names is non-zero, @board_name should be non-NULL, and @offset + * should be at least sizeof(*board_name). These are used by the handler for + * the %COMEDI_DEVCONFIG ioctl to match a hardware device and its driver by + * board name. If @num_names is zero, the %COMEDI_DEVCONFIG ioctl matches a + * hardware device and its driver by driver name. This is only useful if the + * @attach handler is set. If @num_names is non-zero, the driver's @attach + * handler will be called with the COMEDI device structure's board_ptr member + * pointing to the matched pointer to a board name within the driver's private + * array of static, read-only board type information. + * + * The @detach handler has two roles. If a COMEDI device was successfully + * configured by the @attach or @auto_attach handler, it is called when the + * device is being deconfigured (by the %COMEDI_DEVCONFIG ioctl, or due to + * unloading of the driver, or due to device removal). It is also called when + * the @attach or @auto_attach handler returns an error. Therefore, the + * @attach or @auto_attach handlers can defer clean-up on error until the + * @detach handler is called. If the @attach or @auto_attach handlers free + * any resources themselves, they must prevent the @detach handler from + * freeing the same resources. The @detach handler must not assume that all + * resources requested by the @attach or @auto_attach handler were + * successfully allocated. + */ +struct comedi_driver { + /* private: */ + struct comedi_driver *next; /* Next in list of COMEDI drivers. */ + /* public: */ + const char *driver_name; + struct module *module; + int (*attach)(struct comedi_device *dev, struct comedi_devconfig *it); + void (*detach)(struct comedi_device *dev); + int (*auto_attach)(struct comedi_device *dev, unsigned long context); + unsigned int num_names; + const char *const *board_name; + int offset; +}; + +/** + * struct comedi_device - Working data for a COMEDI device + * @use_count: Number of open file objects. + * @driver: Low-level COMEDI driver attached to this COMEDI device. + * @pacer: Optional pointer to a dynamically allocated acquisition pacer + * control. It is freed automatically after the COMEDI device is + * detached from the low-level driver. + * @private: Optional pointer to private data allocated by the low-level + * driver. It is freed automatically after the COMEDI device is + * detached from the low-level driver. + * @class_dev: Sysfs comediX device. + * @minor: Minor device number of COMEDI char device (0-47). + * @detach_count: Counter incremented every time the COMEDI device is detached. + * Used for checking a previous attachment is still valid. + * @hw_dev: Optional pointer to the low-level hardware &struct device. It is + * required for automatically configured COMEDI devices and optional for + * COMEDI devices configured by the %COMEDI_DEVCONFIG ioctl, although + * the bus-specific COMEDI functions only work if it is set correctly. + * It is also passed to dma_alloc_coherent() for COMEDI subdevices that + * have their 'async_dma_dir' member set to something other than + * %DMA_NONE. + * @board_name: Pointer to a COMEDI board name or a COMEDI driver name. When + * the low-level driver's "attach" handler is called by the handler for + * the %COMEDI_DEVCONFIG ioctl, it either points to a matched board name + * string if the 'num_names' member of the &struct comedi_driver is + * non-zero, otherwise it points to the low-level driver name string. + * When the low-lever driver's "auto_attach" handler is called for an + * automatically configured COMEDI device, it points to the low-level + * driver name string. The low-level driver is free to change it in its + * "attach" or "auto_attach" handler if it wishes. + * @board_ptr: Optional pointer to private, read-only board type information in + * the low-level driver. If the 'num_names' member of the &struct + * comedi_driver is non-zero, the handler for the %COMEDI_DEVCONFIG ioctl + * will point it to a pointer to a matched board name string within the + * driver's private array of static, read-only board type information when + * calling the driver's "attach" handler. The low-level driver is free to + * change it. + * @attached: Flag indicating that the COMEDI device is attached to a low-level + * driver. + * @ioenabled: Flag used to indicate that a PCI device has been enabled and + * its regions requested. + * @spinlock: Generic spin-lock for use by the low-level driver. + * @mutex: Generic mutex for use by the COMEDI core module. + * @attach_lock: &struct rw_semaphore used to guard against the COMEDI device + * being detached while an operation is in progress. The down_write() + * operation is only allowed while @mutex is held and is used when + * changing @attached and @detach_count and calling the low-level driver's + * "detach" handler. The down_read() operation is generally used without + * holding @mutex. + * @refcount: &struct kref reference counter for freeing COMEDI device. + * @n_subdevices: Number of COMEDI subdevices allocated by the low-level + * driver for this device. + * @subdevices: Dynamically allocated array of COMEDI subdevices. + * @mmio: Optional pointer to a remapped MMIO region set by the low-level + * driver. + * @iobase: Optional base of an I/O port region requested by the low-level + * driver. + * @iolen: Length of I/O port region requested at @iobase. + * @irq: Optional IRQ number requested by the low-level driver. + * @read_subdev: Optional pointer to a default COMEDI subdevice operated on by + * the read() file operation. Set by the low-level driver. + * @write_subdev: Optional pointer to a default COMEDI subdevice operated on by + * the write() file operation. Set by the low-level driver. + * @async_queue: Storage for fasync_helper(). + * @open: Optional pointer to a function set by the low-level driver to be + * called when @use_count changes from 0 to 1. + * @close: Optional pointer to a function set by the low-level driver to be + * called when @use_count changed from 1 to 0. + * @insn_device_config: Optional pointer to a handler for all sub-instructions + * except %INSN_DEVICE_CONFIG_GET_ROUTES of the %INSN_DEVICE_CONFIG + * instruction. If this is not initialized by the low-level driver, a + * default handler will be set during post-configuration. + * @get_valid_routes: Optional pointer to a handler for the + * %INSN_DEVICE_CONFIG_GET_ROUTES sub-instruction of the + * %INSN_DEVICE_CONFIG instruction set. If this is not initialized by the + * low-level driver, a default handler that copies zero routes back to the + * user will be used. + * + * This is the main control data structure for a COMEDI device (as far as the + * COMEDI core is concerned). There are two groups of COMEDI devices - + * "legacy" devices that are configured by the handler for the + * %COMEDI_DEVCONFIG ioctl, and automatically configured devices resulting + * from a call to comedi_auto_config() as a result of a bus driver probe in + * a low-level COMEDI driver. The "legacy" COMEDI devices are allocated + * during module initialization if the "comedi_num_legacy_minors" module + * parameter is non-zero and use minor device numbers from 0 to + * comedi_num_legacy_minors minus one. The automatically configured COMEDI + * devices are allocated on demand and use minor device numbers from + * comedi_num_legacy_minors to 47. + */ +struct comedi_device { + int use_count; + struct comedi_driver *driver; + struct comedi_8254 *pacer; + void *private; + + struct device *class_dev; + int minor; + unsigned int detach_count; + struct device *hw_dev; + + const char *board_name; + const void *board_ptr; + unsigned int attached:1; + unsigned int ioenabled:1; + spinlock_t spinlock; /* generic spin-lock for low-level driver */ + struct mutex mutex; /* generic mutex for COMEDI core */ + struct rw_semaphore attach_lock; + struct kref refcount; + + int n_subdevices; + struct comedi_subdevice *subdevices; + + /* dumb */ + void __iomem *mmio; + unsigned long iobase; + unsigned long iolen; + unsigned int irq; + + struct comedi_subdevice *read_subdev; + struct comedi_subdevice *write_subdev; + + struct fasync_struct *async_queue; + + int (*open)(struct comedi_device *dev); + void (*close)(struct comedi_device *dev); + int (*insn_device_config)(struct comedi_device *dev, + struct comedi_insn *insn, unsigned int *data); + unsigned int (*get_valid_routes)(struct comedi_device *dev, + unsigned int n_pairs, + unsigned int *pair_data); +}; + +/* + * function prototypes + */ + +void comedi_event(struct comedi_device *dev, struct comedi_subdevice *s); + +struct comedi_device *comedi_dev_get_from_minor(unsigned int minor); +int comedi_dev_put(struct comedi_device *dev); + +bool comedi_is_subdevice_running(struct comedi_subdevice *s); + +void *comedi_alloc_spriv(struct comedi_subdevice *s, size_t size); +void comedi_set_spriv_auto_free(struct comedi_subdevice *s); + +int comedi_check_chanlist(struct comedi_subdevice *s, + int n, + unsigned int *chanlist); + +/* range stuff */ + +#define RANGE(a, b) {(a) * 1e6, (b) * 1e6, 0} +#define RANGE_ext(a, b) {(a) * 1e6, (b) * 1e6, RF_EXTERNAL} +#define RANGE_mA(a, b) {(a) * 1e6, (b) * 1e6, UNIT_mA} +#define RANGE_unitless(a, b) {(a) * 1e6, (b) * 1e6, 0} +#define BIP_RANGE(a) {-(a) * 1e6, (a) * 1e6, 0} +#define UNI_RANGE(a) {0, (a) * 1e6, 0} + +extern const struct comedi_lrange range_bipolar10; +extern const struct comedi_lrange range_bipolar5; +extern const struct comedi_lrange range_bipolar2_5; +extern const struct comedi_lrange range_unipolar10; +extern const struct comedi_lrange range_unipolar5; +extern const struct comedi_lrange range_unipolar2_5; +extern const struct comedi_lrange range_0_20mA; +extern const struct comedi_lrange range_4_20mA; +extern const struct comedi_lrange range_0_32mA; +extern const struct comedi_lrange range_unknown; + +#define range_digital range_unipolar5 + +/** + * struct comedi_lrange - Describes a COMEDI range table + * @length: Number of entries in the range table. + * @range: Array of &struct comedi_krange, one for each range. + * + * Each element of @range[] describes the minimum and maximum physical range + * and the type of units. Typically, the type of unit is %UNIT_volt + * (i.e. volts) and the minimum and maximum are in millionths of a volt. + * There may also be a flag that indicates the minimum and maximum are merely + * scale factors for an unknown, external reference. + */ +struct comedi_lrange { + int length; + struct comedi_krange range[]; +}; + +/** + * comedi_range_is_bipolar() - Test if subdevice range is bipolar + * @s: COMEDI subdevice. + * @range: Index of range within a range table. + * + * Tests whether a range is bipolar by checking whether its minimum value + * is negative. + * + * Assumes @range is valid. Does not work for subdevices using a + * channel-specific range table list. + * + * Return: + * %true if the range is bipolar. + * %false if the range is unipolar. + */ +static inline bool comedi_range_is_bipolar(struct comedi_subdevice *s, + unsigned int range) +{ + return s->range_table->range[range].min < 0; +} + +/** + * comedi_range_is_unipolar() - Test if subdevice range is unipolar + * @s: COMEDI subdevice. + * @range: Index of range within a range table. + * + * Tests whether a range is unipolar by checking whether its minimum value + * is at least 0. + * + * Assumes @range is valid. Does not work for subdevices using a + * channel-specific range table list. + * + * Return: + * %true if the range is unipolar. + * %false if the range is bipolar. + */ +static inline bool comedi_range_is_unipolar(struct comedi_subdevice *s, + unsigned int range) +{ + return s->range_table->range[range].min >= 0; +} + +/** + * comedi_range_is_external() - Test if subdevice range is external + * @s: COMEDI subdevice. + * @range: Index of range within a range table. + * + * Tests whether a range is externally reference by checking whether its + * %RF_EXTERNAL flag is set. + * + * Assumes @range is valid. Does not work for subdevices using a + * channel-specific range table list. + * + * Return: + * %true if the range is external. + * %false if the range is internal. + */ +static inline bool comedi_range_is_external(struct comedi_subdevice *s, + unsigned int range) +{ + return !!(s->range_table->range[range].flags & RF_EXTERNAL); +} + +/** + * comedi_chan_range_is_bipolar() - Test if channel-specific range is bipolar + * @s: COMEDI subdevice. + * @chan: The channel number. + * @range: Index of range within a range table. + * + * Tests whether a range is bipolar by checking whether its minimum value + * is negative. + * + * Assumes @chan and @range are valid. Only works for subdevices with a + * channel-specific range table list. + * + * Return: + * %true if the range is bipolar. + * %false if the range is unipolar. + */ +static inline bool comedi_chan_range_is_bipolar(struct comedi_subdevice *s, + unsigned int chan, + unsigned int range) +{ + return s->range_table_list[chan]->range[range].min < 0; +} + +/** + * comedi_chan_range_is_unipolar() - Test if channel-specific range is unipolar + * @s: COMEDI subdevice. + * @chan: The channel number. + * @range: Index of range within a range table. + * + * Tests whether a range is unipolar by checking whether its minimum value + * is at least 0. + * + * Assumes @chan and @range are valid. Only works for subdevices with a + * channel-specific range table list. + * + * Return: + * %true if the range is unipolar. + * %false if the range is bipolar. + */ +static inline bool comedi_chan_range_is_unipolar(struct comedi_subdevice *s, + unsigned int chan, + unsigned int range) +{ + return s->range_table_list[chan]->range[range].min >= 0; +} + +/** + * comedi_chan_range_is_external() - Test if channel-specific range is external + * @s: COMEDI subdevice. + * @chan: The channel number. + * @range: Index of range within a range table. + * + * Tests whether a range is externally reference by checking whether its + * %RF_EXTERNAL flag is set. + * + * Assumes @chan and @range are valid. Only works for subdevices with a + * channel-specific range table list. + * + * Return: + * %true if the range is bipolar. + * %false if the range is unipolar. + */ +static inline bool comedi_chan_range_is_external(struct comedi_subdevice *s, + unsigned int chan, + unsigned int range) +{ + return !!(s->range_table_list[chan]->range[range].flags & RF_EXTERNAL); +} + +/** + * comedi_offset_munge() - Convert between offset binary and 2's complement + * @s: COMEDI subdevice. + * @val: Value to be converted. + * + * Toggles the highest bit of a sample value to toggle between offset binary + * and 2's complement. Assumes that @s->maxdata is a power of 2 minus 1. + * + * Return: The converted value. + */ +static inline unsigned int comedi_offset_munge(struct comedi_subdevice *s, + unsigned int val) +{ + return val ^ s->maxdata ^ (s->maxdata >> 1); +} + +/** + * comedi_bytes_per_sample() - Determine subdevice sample size + * @s: COMEDI subdevice. + * + * The sample size will be 4 (sizeof int) or 2 (sizeof short) depending on + * whether the %SDF_LSAMPL subdevice flag is set or not. + * + * Return: The subdevice sample size. + */ +static inline unsigned int comedi_bytes_per_sample(struct comedi_subdevice *s) +{ + return s->subdev_flags & SDF_LSAMPL ? sizeof(int) : sizeof(short); +} + +/** + * comedi_sample_shift() - Determine log2 of subdevice sample size + * @s: COMEDI subdevice. + * + * The sample size will be 4 (sizeof int) or 2 (sizeof short) depending on + * whether the %SDF_LSAMPL subdevice flag is set or not. The log2 of the + * sample size will be 2 or 1 and can be used as the right operand of a + * bit-shift operator to multiply or divide something by the sample size. + * + * Return: log2 of the subdevice sample size. + */ +static inline unsigned int comedi_sample_shift(struct comedi_subdevice *s) +{ + return s->subdev_flags & SDF_LSAMPL ? 2 : 1; +} + +/** + * comedi_bytes_to_samples() - Convert a number of bytes to a number of samples + * @s: COMEDI subdevice. + * @nbytes: Number of bytes + * + * Return: The number of bytes divided by the subdevice sample size. + */ +static inline unsigned int comedi_bytes_to_samples(struct comedi_subdevice *s, + unsigned int nbytes) +{ + return nbytes >> comedi_sample_shift(s); +} + +/** + * comedi_samples_to_bytes() - Convert a number of samples to a number of bytes + * @s: COMEDI subdevice. + * @nsamples: Number of samples. + * + * Return: The number of samples multiplied by the subdevice sample size. + * (Does not check for arithmetic overflow.) + */ +static inline unsigned int comedi_samples_to_bytes(struct comedi_subdevice *s, + unsigned int nsamples) +{ + return nsamples << comedi_sample_shift(s); +} + +/** + * comedi_check_trigger_src() - Trivially validate a comedi_cmd trigger source + * @src: Pointer to the trigger source to validate. + * @flags: Bitmask of valid %TRIG_* for the trigger. + * + * This is used in "step 1" of the do_cmdtest functions of comedi drivers + * to validate the comedi_cmd triggers. The mask of the @src against the + * @flags allows the userspace comedilib to pass all the comedi_cmd + * triggers as %TRIG_ANY and get back a bitmask of the valid trigger sources. + * + * Return: + * 0 if trigger sources in *@src are all supported. + * -EINVAL if any trigger source in *@src is unsupported. + */ +static inline int comedi_check_trigger_src(unsigned int *src, + unsigned int flags) +{ + unsigned int orig_src = *src; + + *src = orig_src & flags; + if (*src == TRIG_INVALID || *src != orig_src) + return -EINVAL; + return 0; +} + +/** + * comedi_check_trigger_is_unique() - Make sure a trigger source is unique + * @src: The trigger source to check. + * + * Return: + * 0 if no more than one trigger source is set. + * -EINVAL if more than one trigger source is set. + */ +static inline int comedi_check_trigger_is_unique(unsigned int src) +{ + /* this test is true if more than one _src bit is set */ + if ((src & (src - 1)) != 0) + return -EINVAL; + return 0; +} + +/** + * comedi_check_trigger_arg_is() - Trivially validate a trigger argument + * @arg: Pointer to the trigger arg to validate. + * @val: The value the argument should be. + * + * Forces *@arg to be @val. + * + * Return: + * 0 if *@arg was already @val. + * -EINVAL if *@arg differed from @val. + */ +static inline int comedi_check_trigger_arg_is(unsigned int *arg, + unsigned int val) +{ + if (*arg != val) { + *arg = val; + return -EINVAL; + } + return 0; +} + +/** + * comedi_check_trigger_arg_min() - Trivially validate a trigger argument min + * @arg: Pointer to the trigger arg to validate. + * @val: The minimum value the argument should be. + * + * Forces *@arg to be at least @val, setting it to @val if necessary. + * + * Return: + * 0 if *@arg was already at least @val. + * -EINVAL if *@arg was less than @val. + */ +static inline int comedi_check_trigger_arg_min(unsigned int *arg, + unsigned int val) +{ + if (*arg < val) { + *arg = val; + return -EINVAL; + } + return 0; +} + +/** + * comedi_check_trigger_arg_max() - Trivially validate a trigger argument max + * @arg: Pointer to the trigger arg to validate. + * @val: The maximum value the argument should be. + * + * Forces *@arg to be no more than @val, setting it to @val if necessary. + * + * Return: + * 0 if*@arg was already no more than @val. + * -EINVAL if *@arg was greater than @val. + */ +static inline int comedi_check_trigger_arg_max(unsigned int *arg, + unsigned int val) +{ + if (*arg > val) { + *arg = val; + return -EINVAL; + } + return 0; +} + +/* + * Must set dev->hw_dev if you wish to dma directly into comedi's buffer. + * Also useful for retrieving a previously configured hardware device of + * known bus type. Set automatically for auto-configured devices. + * Automatically set to NULL when detaching hardware device. + */ +int comedi_set_hw_dev(struct comedi_device *dev, struct device *hw_dev); + +/** + * comedi_buf_n_bytes_ready - Determine amount of unread data in buffer + * @s: COMEDI subdevice. + * + * Determines the number of bytes of unread data in the asynchronous + * acquisition data buffer for a subdevice. The data in question might not + * have been fully "munged" yet. + * + * Returns: The amount of unread data in bytes. + */ +static inline unsigned int comedi_buf_n_bytes_ready(struct comedi_subdevice *s) +{ + return s->async->buf_write_count - s->async->buf_read_count; +} + +unsigned int comedi_buf_write_alloc(struct comedi_subdevice *s, unsigned int n); +unsigned int comedi_buf_write_free(struct comedi_subdevice *s, unsigned int n); + +unsigned int comedi_buf_read_n_available(struct comedi_subdevice *s); +unsigned int comedi_buf_read_alloc(struct comedi_subdevice *s, unsigned int n); +unsigned int comedi_buf_read_free(struct comedi_subdevice *s, unsigned int n); + +unsigned int comedi_buf_write_samples(struct comedi_subdevice *s, + const void *data, unsigned int nsamples); +unsigned int comedi_buf_read_samples(struct comedi_subdevice *s, + void *data, unsigned int nsamples); + +/* drivers.c - general comedi driver functions */ + +#define COMEDI_TIMEOUT_MS 1000 + +int comedi_timeout(struct comedi_device *dev, struct comedi_subdevice *s, + struct comedi_insn *insn, + int (*cb)(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned long context), + unsigned long context); + +unsigned int comedi_handle_events(struct comedi_device *dev, + struct comedi_subdevice *s); + +int comedi_dio_insn_config(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data, + unsigned int mask); +unsigned int comedi_dio_update_state(struct comedi_subdevice *s, + unsigned int *data); +unsigned int comedi_bytes_per_scan_cmd(struct comedi_subdevice *s, + struct comedi_cmd *cmd); +unsigned int comedi_bytes_per_scan(struct comedi_subdevice *s); +unsigned int comedi_nscans_left(struct comedi_subdevice *s, + unsigned int nscans); +unsigned int comedi_nsamples_left(struct comedi_subdevice *s, + unsigned int nsamples); +void comedi_inc_scan_progress(struct comedi_subdevice *s, + unsigned int num_bytes); + +void *comedi_alloc_devpriv(struct comedi_device *dev, size_t size); +int comedi_alloc_subdevices(struct comedi_device *dev, int num_subdevices); +int comedi_alloc_subdev_readback(struct comedi_subdevice *s); + +int comedi_readback_insn_read(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data); + +int comedi_load_firmware(struct comedi_device *dev, struct device *hw_dev, + const char *name, + int (*cb)(struct comedi_device *dev, + const u8 *data, size_t size, + unsigned long context), + unsigned long context); + +int __comedi_request_region(struct comedi_device *dev, + unsigned long start, unsigned long len); +int comedi_request_region(struct comedi_device *dev, + unsigned long start, unsigned long len); +void comedi_legacy_detach(struct comedi_device *dev); + +int comedi_auto_config(struct device *hardware_device, + struct comedi_driver *driver, unsigned long context); +void comedi_auto_unconfig(struct device *hardware_device); + +int comedi_driver_register(struct comedi_driver *driver); +void comedi_driver_unregister(struct comedi_driver *driver); + +/** + * module_comedi_driver() - Helper macro for registering a comedi driver + * @__comedi_driver: comedi_driver struct + * + * Helper macro for comedi drivers which do not do anything special in module + * init/exit. This eliminates a lot of boilerplate. Each module may only use + * this macro once, and calling it replaces module_init() and module_exit(). + */ +#define module_comedi_driver(__comedi_driver) \ + module_driver(__comedi_driver, comedi_driver_register, \ + comedi_driver_unregister) + +#endif /* _COMEDIDEV_H */ diff --git a/include/linux/comedi/comedilib.h b/include/linux/comedi/comedilib.h new file mode 100644 index 000000000000..0223c9cd9215 --- /dev/null +++ b/include/linux/comedi/comedilib.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedilib.h + * Header file for kcomedilib + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1998-2001 David A. Schleef + */ + +#ifndef _LINUX_COMEDILIB_H +#define _LINUX_COMEDILIB_H + +struct comedi_device *comedi_open(const char *path); +int comedi_close(struct comedi_device *dev); +int comedi_dio_get_config(struct comedi_device *dev, unsigned int subdev, + unsigned int chan, unsigned int *io); +int comedi_dio_config(struct comedi_device *dev, unsigned int subdev, + unsigned int chan, unsigned int io); +int comedi_dio_bitfield2(struct comedi_device *dev, unsigned int subdev, + unsigned int mask, unsigned int *bits, + unsigned int base_channel); +int comedi_find_subdevice_by_type(struct comedi_device *dev, int type, + unsigned int subd); +int comedi_get_n_channels(struct comedi_device *dev, unsigned int subdevice); + +#endif diff --git a/include/uapi/linux/comedi.h b/include/uapi/linux/comedi.h new file mode 100644 index 000000000000..7314e5ee0a1e --- /dev/null +++ b/include/uapi/linux/comedi.h @@ -0,0 +1,1528 @@ +/* SPDX-License-Identifier: LGPL-2.0+ WITH Linux-syscall-note */ +/* + * comedi.h + * header file for COMEDI user API + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1998-2001 David A. Schleef + */ + +#ifndef _COMEDI_H +#define _COMEDI_H + +#define COMEDI_MAJORVERSION 0 +#define COMEDI_MINORVERSION 7 +#define COMEDI_MICROVERSION 76 +#define VERSION "0.7.76" + +/* comedi's major device number */ +#define COMEDI_MAJOR 98 + +/* + * maximum number of minor devices. This can be increased, although + * kernel structures are currently statically allocated, thus you + * don't want this to be much more than you actually use. + */ +#define COMEDI_NDEVICES 16 + +/* number of config options in the config structure */ +#define COMEDI_NDEVCONFOPTS 32 + +/* + * NOTE: 'comedi_config --init-data' is deprecated + * + * The following indexes in the config options were used by + * comedi_config to pass firmware blobs from user space to the + * comedi drivers. The request_firmware() hotplug interface is + * now used by all comedi drivers instead. + */ + +/* length of nth chunk of firmware data -*/ +#define COMEDI_DEVCONF_AUX_DATA3_LENGTH 25 +#define COMEDI_DEVCONF_AUX_DATA2_LENGTH 26 +#define COMEDI_DEVCONF_AUX_DATA1_LENGTH 27 +#define COMEDI_DEVCONF_AUX_DATA0_LENGTH 28 +/* most significant 32 bits of pointer address (if needed) */ +#define COMEDI_DEVCONF_AUX_DATA_HI 29 +/* least significant 32 bits of pointer address */ +#define COMEDI_DEVCONF_AUX_DATA_LO 30 +#define COMEDI_DEVCONF_AUX_DATA_LENGTH 31 /* total data length */ + +/* max length of device and driver names */ +#define COMEDI_NAMELEN 20 + +/* packs and unpacks a channel/range number */ + +#define CR_PACK(chan, rng, aref) \ + ((((aref) & 0x3) << 24) | (((rng) & 0xff) << 16) | (chan)) +#define CR_PACK_FLAGS(chan, range, aref, flags) \ + (CR_PACK(chan, range, aref) | ((flags) & CR_FLAGS_MASK)) + +#define CR_CHAN(a) ((a) & 0xffff) +#define CR_RANGE(a) (((a) >> 16) & 0xff) +#define CR_AREF(a) (((a) >> 24) & 0x03) + +#define CR_FLAGS_MASK 0xfc000000 +#define CR_ALT_FILTER 0x04000000 +#define CR_DITHER CR_ALT_FILTER +#define CR_DEGLITCH CR_ALT_FILTER +#define CR_ALT_SOURCE 0x08000000 +#define CR_EDGE 0x40000000 +#define CR_INVERT 0x80000000 + +#define AREF_GROUND 0x00 /* analog ref = analog ground */ +#define AREF_COMMON 0x01 /* analog ref = analog common */ +#define AREF_DIFF 0x02 /* analog ref = differential */ +#define AREF_OTHER 0x03 /* analog ref = other (undefined) */ + +/* counters -- these are arbitrary values */ +#define GPCT_RESET 0x0001 +#define GPCT_SET_SOURCE 0x0002 +#define GPCT_SET_GATE 0x0004 +#define GPCT_SET_DIRECTION 0x0008 +#define GPCT_SET_OPERATION 0x0010 +#define GPCT_ARM 0x0020 +#define GPCT_DISARM 0x0040 +#define GPCT_GET_INT_CLK_FRQ 0x0080 + +#define GPCT_INT_CLOCK 0x0001 +#define GPCT_EXT_PIN 0x0002 +#define GPCT_NO_GATE 0x0004 +#define GPCT_UP 0x0008 +#define GPCT_DOWN 0x0010 +#define GPCT_HWUD 0x0020 +#define GPCT_SIMPLE_EVENT 0x0040 +#define GPCT_SINGLE_PERIOD 0x0080 +#define GPCT_SINGLE_PW 0x0100 +#define GPCT_CONT_PULSE_OUT 0x0200 +#define GPCT_SINGLE_PULSE_OUT 0x0400 + +/* instructions */ + +#define INSN_MASK_WRITE 0x8000000 +#define INSN_MASK_READ 0x4000000 +#define INSN_MASK_SPECIAL 0x2000000 + +#define INSN_READ (0 | INSN_MASK_READ) +#define INSN_WRITE (1 | INSN_MASK_WRITE) +#define INSN_BITS (2 | INSN_MASK_READ | INSN_MASK_WRITE) +#define INSN_CONFIG (3 | INSN_MASK_READ | INSN_MASK_WRITE) +#define INSN_DEVICE_CONFIG (INSN_CONFIG | INSN_MASK_SPECIAL) +#define INSN_GTOD (4 | INSN_MASK_READ | INSN_MASK_SPECIAL) +#define INSN_WAIT (5 | INSN_MASK_WRITE | INSN_MASK_SPECIAL) +#define INSN_INTTRIG (6 | INSN_MASK_WRITE | INSN_MASK_SPECIAL) + +/* command flags */ +/* These flags are used in comedi_cmd structures */ + +#define CMDF_BOGUS 0x00000001 /* do the motions */ + +/* try to use a real-time interrupt while performing command */ +#define CMDF_PRIORITY 0x00000008 + +/* wake up on end-of-scan events */ +#define CMDF_WAKE_EOS 0x00000020 + +#define CMDF_WRITE 0x00000040 + +#define CMDF_RAWDATA 0x00000080 + +/* timer rounding definitions */ +#define CMDF_ROUND_MASK 0x00030000 +#define CMDF_ROUND_NEAREST 0x00000000 +#define CMDF_ROUND_DOWN 0x00010000 +#define CMDF_ROUND_UP 0x00020000 +#define CMDF_ROUND_UP_NEXT 0x00030000 + +#define COMEDI_EV_START 0x00040000 +#define COMEDI_EV_SCAN_BEGIN 0x00080000 +#define COMEDI_EV_CONVERT 0x00100000 +#define COMEDI_EV_SCAN_END 0x00200000 +#define COMEDI_EV_STOP 0x00400000 + +/* compatibility definitions */ +#define TRIG_BOGUS CMDF_BOGUS +#define TRIG_RT CMDF_PRIORITY +#define TRIG_WAKE_EOS CMDF_WAKE_EOS +#define TRIG_WRITE CMDF_WRITE +#define TRIG_ROUND_MASK CMDF_ROUND_MASK +#define TRIG_ROUND_NEAREST CMDF_ROUND_NEAREST +#define TRIG_ROUND_DOWN CMDF_ROUND_DOWN +#define TRIG_ROUND_UP CMDF_ROUND_UP +#define TRIG_ROUND_UP_NEXT CMDF_ROUND_UP_NEXT + +/* trigger sources */ + +#define TRIG_ANY 0xffffffff +#define TRIG_INVALID 0x00000000 + +#define TRIG_NONE 0x00000001 /* never trigger */ +#define TRIG_NOW 0x00000002 /* trigger now + N ns */ +#define TRIG_FOLLOW 0x00000004 /* trigger on next lower level trig */ +#define TRIG_TIME 0x00000008 /* trigger at time N ns */ +#define TRIG_TIMER 0x00000010 /* trigger at rate N ns */ +#define TRIG_COUNT 0x00000020 /* trigger when count reaches N */ +#define TRIG_EXT 0x00000040 /* trigger on external signal N */ +#define TRIG_INT 0x00000080 /* trigger on comedi-internal signal N */ +#define TRIG_OTHER 0x00000100 /* driver defined */ + +/* subdevice flags */ + +#define SDF_BUSY 0x0001 /* device is busy */ +#define SDF_BUSY_OWNER 0x0002 /* device is busy with your job */ +#define SDF_LOCKED 0x0004 /* subdevice is locked */ +#define SDF_LOCK_OWNER 0x0008 /* you own lock */ +#define SDF_MAXDATA 0x0010 /* maxdata depends on channel */ +#define SDF_FLAGS 0x0020 /* flags depend on channel */ +#define SDF_RANGETYPE 0x0040 /* range type depends on channel */ +#define SDF_PWM_COUNTER 0x0080 /* PWM can automatically switch off */ +#define SDF_PWM_HBRIDGE 0x0100 /* PWM is signed (H-bridge) */ +#define SDF_CMD 0x1000 /* can do commands (deprecated) */ +#define SDF_SOFT_CALIBRATED 0x2000 /* subdevice uses software calibration */ +#define SDF_CMD_WRITE 0x4000 /* can do output commands */ +#define SDF_CMD_READ 0x8000 /* can do input commands */ + +/* subdevice can be read (e.g. analog input) */ +#define SDF_READABLE 0x00010000 +/* subdevice can be written (e.g. analog output) */ +#define SDF_WRITABLE 0x00020000 +#define SDF_WRITEABLE SDF_WRITABLE /* spelling error in API */ +/* subdevice does not have externally visible lines */ +#define SDF_INTERNAL 0x00040000 +#define SDF_GROUND 0x00100000 /* can do aref=ground */ +#define SDF_COMMON 0x00200000 /* can do aref=common */ +#define SDF_DIFF 0x00400000 /* can do aref=diff */ +#define SDF_OTHER 0x00800000 /* can do aref=other */ +#define SDF_DITHER 0x01000000 /* can do dithering */ +#define SDF_DEGLITCH 0x02000000 /* can do deglitching */ +#define SDF_MMAP 0x04000000 /* can do mmap() */ +#define SDF_RUNNING 0x08000000 /* subdevice is acquiring data */ +#define SDF_LSAMPL 0x10000000 /* subdevice uses 32-bit samples */ +#define SDF_PACKED 0x20000000 /* subdevice can do packed DIO */ + +/* subdevice types */ + +/** + * enum comedi_subdevice_type - COMEDI subdevice types + * @COMEDI_SUBD_UNUSED: Unused subdevice. + * @COMEDI_SUBD_AI: Analog input. + * @COMEDI_SUBD_AO: Analog output. + * @COMEDI_SUBD_DI: Digital input. + * @COMEDI_SUBD_DO: Digital output. + * @COMEDI_SUBD_DIO: Digital input/output. + * @COMEDI_SUBD_COUNTER: Counter. + * @COMEDI_SUBD_TIMER: Timer. + * @COMEDI_SUBD_MEMORY: Memory, EEPROM, DPRAM. + * @COMEDI_SUBD_CALIB: Calibration DACs. + * @COMEDI_SUBD_PROC: Processor, DSP. + * @COMEDI_SUBD_SERIAL: Serial I/O. + * @COMEDI_SUBD_PWM: Pulse-Width Modulation output. + */ +enum comedi_subdevice_type { + COMEDI_SUBD_UNUSED, + COMEDI_SUBD_AI, + COMEDI_SUBD_AO, + COMEDI_SUBD_DI, + COMEDI_SUBD_DO, + COMEDI_SUBD_DIO, + COMEDI_SUBD_COUNTER, + COMEDI_SUBD_TIMER, + COMEDI_SUBD_MEMORY, + COMEDI_SUBD_CALIB, + COMEDI_SUBD_PROC, + COMEDI_SUBD_SERIAL, + COMEDI_SUBD_PWM +}; + +/* configuration instructions */ + +/** + * enum comedi_io_direction - COMEDI I/O directions + * @COMEDI_INPUT: Input. + * @COMEDI_OUTPUT: Output. + * @COMEDI_OPENDRAIN: Open-drain (or open-collector) output. + * + * These are used by the %INSN_CONFIG_DIO_QUERY configuration instruction to + * report a direction. They may also be used in other places where a direction + * needs to be specified. + */ +enum comedi_io_direction { + COMEDI_INPUT = 0, + COMEDI_OUTPUT = 1, + COMEDI_OPENDRAIN = 2 +}; + +/** + * enum configuration_ids - COMEDI configuration instruction codes + * @INSN_CONFIG_DIO_INPUT: Configure digital I/O as input. + * @INSN_CONFIG_DIO_OUTPUT: Configure digital I/O as output. + * @INSN_CONFIG_DIO_OPENDRAIN: Configure digital I/O as open-drain (or open + * collector) output. + * @INSN_CONFIG_ANALOG_TRIG: Configure analog trigger. + * @INSN_CONFIG_ALT_SOURCE: Configure alternate input source. + * @INSN_CONFIG_DIGITAL_TRIG: Configure digital trigger. + * @INSN_CONFIG_BLOCK_SIZE: Configure block size for DMA transfers. + * @INSN_CONFIG_TIMER_1: Configure divisor for external clock. + * @INSN_CONFIG_FILTER: Configure a filter. + * @INSN_CONFIG_CHANGE_NOTIFY: Configure change notification for digital + * inputs. (New drivers should use + * %INSN_CONFIG_DIGITAL_TRIG instead.) + * @INSN_CONFIG_SERIAL_CLOCK: Configure clock for serial I/O. + * @INSN_CONFIG_BIDIRECTIONAL_DATA: Send and receive byte over serial I/O. + * @INSN_CONFIG_DIO_QUERY: Query direction of digital I/O channel. + * @INSN_CONFIG_PWM_OUTPUT: Configure pulse-width modulator output. + * @INSN_CONFIG_GET_PWM_OUTPUT: Get pulse-width modulator output configuration. + * @INSN_CONFIG_ARM: Arm a subdevice or channel. + * @INSN_CONFIG_DISARM: Disarm a subdevice or channel. + * @INSN_CONFIG_GET_COUNTER_STATUS: Get counter status. + * @INSN_CONFIG_RESET: Reset a subdevice or channel. + * @INSN_CONFIG_GPCT_SINGLE_PULSE_GENERATOR: Configure counter/timer as + * single pulse generator. + * @INSN_CONFIG_GPCT_PULSE_TRAIN_GENERATOR: Configure counter/timer as + * pulse train generator. + * @INSN_CONFIG_GPCT_QUADRATURE_ENCODER: Configure counter as a quadrature + * encoder. + * @INSN_CONFIG_SET_GATE_SRC: Set counter/timer gate source. + * @INSN_CONFIG_GET_GATE_SRC: Get counter/timer gate source. + * @INSN_CONFIG_SET_CLOCK_SRC: Set counter/timer master clock source. + * @INSN_CONFIG_GET_CLOCK_SRC: Get counter/timer master clock source. + * @INSN_CONFIG_SET_OTHER_SRC: Set counter/timer "other" source. + * @INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE: Get size (in bytes) of subdevice's + * on-board FIFOs used during streaming + * input/output. + * @INSN_CONFIG_SET_COUNTER_MODE: Set counter/timer mode. + * @INSN_CONFIG_8254_SET_MODE: (Deprecated) Same as + * %INSN_CONFIG_SET_COUNTER_MODE. + * @INSN_CONFIG_8254_READ_STATUS: Read status of 8254 counter channel. + * @INSN_CONFIG_SET_ROUTING: Set routing for a channel. + * @INSN_CONFIG_GET_ROUTING: Get routing for a channel. + * @INSN_CONFIG_PWM_SET_PERIOD: Set PWM period in nanoseconds. + * @INSN_CONFIG_PWM_GET_PERIOD: Get PWM period in nanoseconds. + * @INSN_CONFIG_GET_PWM_STATUS: Get PWM status. + * @INSN_CONFIG_PWM_SET_H_BRIDGE: Set PWM H bridge duty cycle and polarity for + * a relay simultaneously. + * @INSN_CONFIG_PWM_GET_H_BRIDGE: Get PWM H bridge duty cycle and polarity. + * @INSN_CONFIG_GET_CMD_TIMING_CONSTRAINTS: Get the hardware timing restraints, + * regardless of trigger sources. + */ +enum configuration_ids { + INSN_CONFIG_DIO_INPUT = COMEDI_INPUT, + INSN_CONFIG_DIO_OUTPUT = COMEDI_OUTPUT, + INSN_CONFIG_DIO_OPENDRAIN = COMEDI_OPENDRAIN, + INSN_CONFIG_ANALOG_TRIG = 16, +/* INSN_CONFIG_WAVEFORM = 17, */ +/* INSN_CONFIG_TRIG = 18, */ +/* INSN_CONFIG_COUNTER = 19, */ + INSN_CONFIG_ALT_SOURCE = 20, + INSN_CONFIG_DIGITAL_TRIG = 21, + INSN_CONFIG_BLOCK_SIZE = 22, + INSN_CONFIG_TIMER_1 = 23, + INSN_CONFIG_FILTER = 24, + INSN_CONFIG_CHANGE_NOTIFY = 25, + + INSN_CONFIG_SERIAL_CLOCK = 26, /*ALPHA*/ + INSN_CONFIG_BIDIRECTIONAL_DATA = 27, + INSN_CONFIG_DIO_QUERY = 28, + INSN_CONFIG_PWM_OUTPUT = 29, + INSN_CONFIG_GET_PWM_OUTPUT = 30, + INSN_CONFIG_ARM = 31, + INSN_CONFIG_DISARM = 32, + INSN_CONFIG_GET_COUNTER_STATUS = 33, + INSN_CONFIG_RESET = 34, + INSN_CONFIG_GPCT_SINGLE_PULSE_GENERATOR = 1001, + INSN_CONFIG_GPCT_PULSE_TRAIN_GENERATOR = 1002, + INSN_CONFIG_GPCT_QUADRATURE_ENCODER = 1003, + INSN_CONFIG_SET_GATE_SRC = 2001, + INSN_CONFIG_GET_GATE_SRC = 2002, + INSN_CONFIG_SET_CLOCK_SRC = 2003, + INSN_CONFIG_GET_CLOCK_SRC = 2004, + INSN_CONFIG_SET_OTHER_SRC = 2005, + INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE = 2006, + INSN_CONFIG_SET_COUNTER_MODE = 4097, + INSN_CONFIG_8254_SET_MODE = INSN_CONFIG_SET_COUNTER_MODE, + INSN_CONFIG_8254_READ_STATUS = 4098, + INSN_CONFIG_SET_ROUTING = 4099, + INSN_CONFIG_GET_ROUTING = 4109, + INSN_CONFIG_PWM_SET_PERIOD = 5000, + INSN_CONFIG_PWM_GET_PERIOD = 5001, + INSN_CONFIG_GET_PWM_STATUS = 5002, + INSN_CONFIG_PWM_SET_H_BRIDGE = 5003, + INSN_CONFIG_PWM_GET_H_BRIDGE = 5004, + INSN_CONFIG_GET_CMD_TIMING_CONSTRAINTS = 5005, +}; + +/** + * enum device_configuration_ids - COMEDI configuration instruction codes global + * to an entire device. + * @INSN_DEVICE_CONFIG_TEST_ROUTE: Validate the possibility of a + * globally-named route + * @INSN_DEVICE_CONFIG_CONNECT_ROUTE: Connect a globally-named route + * @INSN_DEVICE_CONFIG_DISCONNECT_ROUTE:Disconnect a globally-named route + * @INSN_DEVICE_CONFIG_GET_ROUTES: Get a list of all globally-named routes + * that are valid for a particular device. + */ +enum device_config_route_ids { + INSN_DEVICE_CONFIG_TEST_ROUTE = 0, + INSN_DEVICE_CONFIG_CONNECT_ROUTE = 1, + INSN_DEVICE_CONFIG_DISCONNECT_ROUTE = 2, + INSN_DEVICE_CONFIG_GET_ROUTES = 3, +}; + +/** + * enum comedi_digital_trig_op - operations for configuring a digital trigger + * @COMEDI_DIGITAL_TRIG_DISABLE: Return digital trigger to its default, + * inactive, unconfigured state. + * @COMEDI_DIGITAL_TRIG_ENABLE_EDGES: Set rising and/or falling edge inputs + * that each can fire the trigger. + * @COMEDI_DIGITAL_TRIG_ENABLE_LEVELS: Set a combination of high and/or low + * level inputs that can fire the trigger. + * + * These are used with the %INSN_CONFIG_DIGITAL_TRIG configuration instruction. + * The data for the configuration instruction is as follows... + * + * data[%0] = %INSN_CONFIG_DIGITAL_TRIG + * + * data[%1] = trigger ID + * + * data[%2] = configuration operation + * + * data[%3] = configuration parameter 1 + * + * data[%4] = configuration parameter 2 + * + * data[%5] = configuration parameter 3 + * + * The trigger ID (data[%1]) is used to differentiate multiple digital triggers + * belonging to the same subdevice. The configuration operation (data[%2]) is + * one of the enum comedi_digital_trig_op values. The configuration + * parameters (data[%3], data[%4], and data[%5]) depend on the operation; they + * are not used with %COMEDI_DIGITAL_TRIG_DISABLE. + * + * For %COMEDI_DIGITAL_TRIG_ENABLE_EDGES and %COMEDI_DIGITAL_TRIG_ENABLE_LEVELS, + * configuration parameter 1 (data[%3]) contains a "left-shift" value that + * specifies the input corresponding to bit 0 of configuration parameters 2 + * and 3. This is useful if the trigger has more than 32 inputs. + * + * For %COMEDI_DIGITAL_TRIG_ENABLE_EDGES, configuration parameter 2 (data[%4]) + * specifies which of up to 32 inputs have rising-edge sensitivity, and + * configuration parameter 3 (data[%5]) specifies which of up to 32 inputs + * have falling-edge sensitivity that can fire the trigger. + * + * For %COMEDI_DIGITAL_TRIG_ENABLE_LEVELS, configuration parameter 2 (data[%4]) + * specifies which of up to 32 inputs must be at a high level, and + * configuration parameter 3 (data[%5]) specifies which of up to 32 inputs + * must be at a low level for the trigger to fire. + * + * Some sequences of %INSN_CONFIG_DIGITAL_TRIG instructions may have a (partly) + * accumulative effect, depending on the low-level driver. This is useful + * when setting up a trigger that has more than 32 inputs, or has a combination + * of edge- and level-triggered inputs. + */ +enum comedi_digital_trig_op { + COMEDI_DIGITAL_TRIG_DISABLE = 0, + COMEDI_DIGITAL_TRIG_ENABLE_EDGES = 1, + COMEDI_DIGITAL_TRIG_ENABLE_LEVELS = 2 +}; + +/** + * enum comedi_support_level - support level for a COMEDI feature + * @COMEDI_UNKNOWN_SUPPORT: Unspecified support for feature. + * @COMEDI_SUPPORTED: Feature is supported. + * @COMEDI_UNSUPPORTED: Feature is unsupported. + */ +enum comedi_support_level { + COMEDI_UNKNOWN_SUPPORT = 0, + COMEDI_SUPPORTED, + COMEDI_UNSUPPORTED +}; + +/** + * enum comedi_counter_status_flags - counter status bits + * @COMEDI_COUNTER_ARMED: Counter is armed. + * @COMEDI_COUNTER_COUNTING: Counter is counting. + * @COMEDI_COUNTER_TERMINAL_COUNT: Counter reached terminal count. + * + * These bitwise values are used by the %INSN_CONFIG_GET_COUNTER_STATUS + * configuration instruction to report the status of a counter. + */ +enum comedi_counter_status_flags { + COMEDI_COUNTER_ARMED = 0x1, + COMEDI_COUNTER_COUNTING = 0x2, + COMEDI_COUNTER_TERMINAL_COUNT = 0x4, +}; + +/* ioctls */ + +#define CIO 'd' +#define COMEDI_DEVCONFIG _IOW(CIO, 0, struct comedi_devconfig) +#define COMEDI_DEVINFO _IOR(CIO, 1, struct comedi_devinfo) +#define COMEDI_SUBDINFO _IOR(CIO, 2, struct comedi_subdinfo) +#define COMEDI_CHANINFO _IOR(CIO, 3, struct comedi_chaninfo) +/* _IOWR(CIO, 4, ...) is reserved */ +#define COMEDI_LOCK _IO(CIO, 5) +#define COMEDI_UNLOCK _IO(CIO, 6) +#define COMEDI_CANCEL _IO(CIO, 7) +#define COMEDI_RANGEINFO _IOR(CIO, 8, struct comedi_rangeinfo) +#define COMEDI_CMD _IOR(CIO, 9, struct comedi_cmd) +#define COMEDI_CMDTEST _IOR(CIO, 10, struct comedi_cmd) +#define COMEDI_INSNLIST _IOR(CIO, 11, struct comedi_insnlist) +#define COMEDI_INSN _IOR(CIO, 12, struct comedi_insn) +#define COMEDI_BUFCONFIG _IOR(CIO, 13, struct comedi_bufconfig) +#define COMEDI_BUFINFO _IOWR(CIO, 14, struct comedi_bufinfo) +#define COMEDI_POLL _IO(CIO, 15) +#define COMEDI_SETRSUBD _IO(CIO, 16) +#define COMEDI_SETWSUBD _IO(CIO, 17) + +/* structures */ + +/** + * struct comedi_insn - COMEDI instruction + * @insn: COMEDI instruction type (%INSN_xxx). + * @n: Length of @data[]. + * @data: Pointer to data array operated on by the instruction. + * @subdev: Subdevice index. + * @chanspec: A packed "chanspec" value consisting of channel number, + * analog range index, analog reference type, and flags. + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_INSN ioctl, and indirectly with the + * %COMEDI_INSNLIST ioctl. + */ +struct comedi_insn { + unsigned int insn; + unsigned int n; + unsigned int __user *data; + unsigned int subdev; + unsigned int chanspec; + unsigned int unused[3]; +}; + +/** + * struct comedi_insnlist - list of COMEDI instructions + * @n_insns: Number of COMEDI instructions. + * @insns: Pointer to array COMEDI instructions. + * + * This is used with the %COMEDI_INSNLIST ioctl. + */ +struct comedi_insnlist { + unsigned int n_insns; + struct comedi_insn __user *insns; +}; + +/** + * struct comedi_cmd - COMEDI asynchronous acquisition command details + * @subdev: Subdevice index. + * @flags: Command flags (%CMDF_xxx). + * @start_src: "Start acquisition" trigger source (%TRIG_xxx). + * @start_arg: "Start acquisition" trigger argument. + * @scan_begin_src: "Scan begin" trigger source. + * @scan_begin_arg: "Scan begin" trigger argument. + * @convert_src: "Convert" trigger source. + * @convert_arg: "Convert" trigger argument. + * @scan_end_src: "Scan end" trigger source. + * @scan_end_arg: "Scan end" trigger argument. + * @stop_src: "Stop acquisition" trigger source. + * @stop_arg: "Stop acquisition" trigger argument. + * @chanlist: Pointer to array of "chanspec" values, containing a + * sequence of channel numbers packed with analog range + * index, etc. + * @chanlist_len: Number of channels in sequence. + * @data: Pointer to miscellaneous set-up data (not used). + * @data_len: Length of miscellaneous set-up data. + * + * This is used with the %COMEDI_CMD or %COMEDI_CMDTEST ioctl to set-up + * or validate an asynchronous acquisition command. The ioctl may modify + * the &struct comedi_cmd and copy it back to the caller. + * + * Optional command @flags values that can be ORed together... + * + * %CMDF_BOGUS - makes %COMEDI_CMD ioctl return error %EAGAIN instead of + * starting the command. + * + * %CMDF_PRIORITY - requests "hard real-time" processing (which is not + * supported in this version of COMEDI). + * + * %CMDF_WAKE_EOS - requests the command makes data available for reading + * after every "scan" period. + * + * %CMDF_WRITE - marks the command as being in the "write" (to device) + * direction. This does not need to be specified by the caller unless the + * subdevice supports commands in either direction. + * + * %CMDF_RAWDATA - prevents the command from "munging" the data between the + * COMEDI sample format and the raw hardware sample format. + * + * %CMDF_ROUND_NEAREST - requests timing periods to be rounded to nearest + * supported values. + * + * %CMDF_ROUND_DOWN - requests timing periods to be rounded down to supported + * values (frequencies rounded up). + * + * %CMDF_ROUND_UP - requests timing periods to be rounded up to supported + * values (frequencies rounded down). + * + * Trigger source values for @start_src, @scan_begin_src, @convert_src, + * @scan_end_src, and @stop_src... + * + * %TRIG_ANY - "all ones" value used to test which trigger sources are + * supported. + * + * %TRIG_INVALID - "all zeroes" value used to indicate that all requested + * trigger sources are invalid. + * + * %TRIG_NONE - never trigger (often used as a @stop_src value). + * + * %TRIG_NOW - trigger after '_arg' nanoseconds. + * + * %TRIG_FOLLOW - trigger follows another event. + * + * %TRIG_TIMER - trigger every '_arg' nanoseconds. + * + * %TRIG_COUNT - trigger when count '_arg' is reached. + * + * %TRIG_EXT - trigger on external signal specified by '_arg'. + * + * %TRIG_INT - trigger on internal, software trigger specified by '_arg'. + * + * %TRIG_OTHER - trigger on other, driver-defined signal specified by '_arg'. + */ +struct comedi_cmd { + unsigned int subdev; + unsigned int flags; + + unsigned int start_src; + unsigned int start_arg; + + unsigned int scan_begin_src; + unsigned int scan_begin_arg; + + unsigned int convert_src; + unsigned int convert_arg; + + unsigned int scan_end_src; + unsigned int scan_end_arg; + + unsigned int stop_src; + unsigned int stop_arg; + + unsigned int *chanlist; + unsigned int chanlist_len; + + short __user *data; + unsigned int data_len; +}; + +/** + * struct comedi_chaninfo - used to retrieve per-channel information + * @subdev: Subdevice index. + * @maxdata_list: Optional pointer to per-channel maximum data values. + * @flaglist: Optional pointer to per-channel flags. + * @rangelist: Optional pointer to per-channel range types. + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_CHANINFO ioctl to get per-channel information + * for the subdevice. Use of this requires knowledge of the number of channels + * and subdevice flags obtained using the %COMEDI_SUBDINFO ioctl. + * + * The @maxdata_list member must be %NULL unless the %SDF_MAXDATA subdevice + * flag is set. The @flaglist member must be %NULL unless the %SDF_FLAGS + * subdevice flag is set. The @rangelist member must be %NULL unless the + * %SDF_RANGETYPE subdevice flag is set. Otherwise, the arrays they point to + * must be at least as long as the number of channels. + */ +struct comedi_chaninfo { + unsigned int subdev; + unsigned int __user *maxdata_list; + unsigned int __user *flaglist; + unsigned int __user *rangelist; + unsigned int unused[4]; +}; + +/** + * struct comedi_rangeinfo - used to retrieve the range table for a channel + * @range_type: Encodes subdevice index (bits 27:24), channel index + * (bits 23:16) and range table length (bits 15:0). + * @range_ptr: Pointer to array of @struct comedi_krange to be filled + * in with the range table for the channel or subdevice. + * + * This is used with the %COMEDI_RANGEINFO ioctl to retrieve the range table + * for a specific channel (if the subdevice has the %SDF_RANGETYPE flag set to + * indicate that the range table depends on the channel), or for the subdevice + * as a whole (if the %SDF_RANGETYPE flag is clear, indicating the range table + * is shared by all channels). + * + * The @range_type value is an input to the ioctl and comes from a previous + * use of the %COMEDI_SUBDINFO ioctl (if the %SDF_RANGETYPE flag is clear), + * or the %COMEDI_CHANINFO ioctl (if the %SDF_RANGETYPE flag is set). + */ +struct comedi_rangeinfo { + unsigned int range_type; + void __user *range_ptr; +}; + +/** + * struct comedi_krange - describes a range in a range table + * @min: Minimum value in millionths (1e-6) of a unit. + * @max: Maximum value in millionths (1e-6) of a unit. + * @flags: Indicates the units (in bits 7:0) OR'ed with optional flags. + * + * A range table is associated with a single channel, or with all channels in a + * subdevice, and a list of one or more ranges. A %struct comedi_krange + * describes the physical range of units for one of those ranges. Sample + * values in COMEDI are unsigned from %0 up to some 'maxdata' value. The + * mapping from sample values to physical units is assumed to be nomimally + * linear (for the purpose of describing the range), with sample value %0 + * mapping to @min, and the 'maxdata' sample value mapping to @max. + * + * The currently defined units are %UNIT_volt (%0), %UNIT_mA (%1), and + * %UNIT_none (%2). The @min and @max values are the physical range multiplied + * by 1e6, so a @max value of %1000000 (with %UNIT_volt) represents a maximal + * value of 1 volt. + * + * The only defined flag value is %RF_EXTERNAL (%0x100), indicating that the + * range needs to be multiplied by an external reference. + */ +struct comedi_krange { + int min; + int max; + unsigned int flags; +}; + +/** + * struct comedi_subdinfo - used to retrieve information about a subdevice + * @type: Type of subdevice from &enum comedi_subdevice_type. + * @n_chan: Number of channels the subdevice supports. + * @subd_flags: A mixture of static and dynamic flags describing + * aspects of the subdevice and its current state. + * @timer_type: Timer type. Always set to %5 ("nanosecond timer"). + * @len_chanlist: Maximum length of a channel list if the subdevice + * supports asynchronous acquisition commands. + * @maxdata: Maximum sample value for all channels if the + * %SDF_MAXDATA subdevice flag is clear. + * @flags: Channel flags for all channels if the %SDF_FLAGS + * subdevice flag is clear. + * @range_type: The range type for all channels if the %SDF_RANGETYPE + * subdevice flag is clear. Encodes the subdevice index + * (bits 27:24), a dummy channel index %0 (bits 23:16), + * and the range table length (bits 15:0). + * @settling_time_0: Not used. + * @insn_bits_support: Set to %COMEDI_SUPPORTED if the subdevice supports the + * %INSN_BITS instruction, or to %COMEDI_UNSUPPORTED if it + * does not. + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_SUBDINFO ioctl which copies an array of + * &struct comedi_subdinfo back to user space, with one element per subdevice. + * Use of this requires knowledge of the number of subdevices obtained from + * the %COMEDI_DEVINFO ioctl. + * + * These are the @subd_flags values that may be ORed together... + * + * %SDF_BUSY - the subdevice is busy processing an asynchronous command or a + * synchronous instruction. + * + * %SDF_BUSY_OWNER - the subdevice is busy processing an asynchronous + * acquisition command started on the current file object (the file object + * issuing the %COMEDI_SUBDINFO ioctl). + * + * %SDF_LOCKED - the subdevice is locked by a %COMEDI_LOCK ioctl. + * + * %SDF_LOCK_OWNER - the subdevice is locked by a %COMEDI_LOCK ioctl from the + * current file object. + * + * %SDF_MAXDATA - maximum sample values are channel-specific. + * + * %SDF_FLAGS - channel flags are channel-specific. + * + * %SDF_RANGETYPE - range types are channel-specific. + * + * %SDF_PWM_COUNTER - PWM can switch off automatically. + * + * %SDF_PWM_HBRIDGE - or PWM is signed (H-bridge). + * + * %SDF_CMD - the subdevice supports asynchronous commands. + * + * %SDF_SOFT_CALIBRATED - the subdevice uses software calibration. + * + * %SDF_CMD_WRITE - the subdevice supports asynchronous commands in the output + * ("write") direction. + * + * %SDF_CMD_READ - the subdevice supports asynchronous commands in the input + * ("read") direction. + * + * %SDF_READABLE - the subdevice is readable (e.g. analog input). + * + * %SDF_WRITABLE (aliased as %SDF_WRITEABLE) - the subdevice is writable (e.g. + * analog output). + * + * %SDF_INTERNAL - the subdevice has no externally visible lines. + * + * %SDF_GROUND - the subdevice can use ground as an analog reference. + * + * %SDF_COMMON - the subdevice can use a common analog reference. + * + * %SDF_DIFF - the subdevice can use differential inputs (or outputs). + * + * %SDF_OTHER - the subdevice can use some other analog reference. + * + * %SDF_DITHER - the subdevice can do dithering. + * + * %SDF_DEGLITCH - the subdevice can do deglitching. + * + * %SDF_MMAP - this is never set. + * + * %SDF_RUNNING - an asynchronous command is still running. + * + * %SDF_LSAMPL - the subdevice uses "long" (32-bit) samples (for asynchronous + * command data). + * + * %SDF_PACKED - the subdevice packs several DIO samples into a single sample + * (for asynchronous command data). + * + * No "channel flags" (@flags) values are currently defined. + */ +struct comedi_subdinfo { + unsigned int type; + unsigned int n_chan; + unsigned int subd_flags; + unsigned int timer_type; + unsigned int len_chanlist; + unsigned int maxdata; + unsigned int flags; + unsigned int range_type; + unsigned int settling_time_0; + unsigned int insn_bits_support; + unsigned int unused[8]; +}; + +/** + * struct comedi_devinfo - used to retrieve information about a COMEDI device + * @version_code: COMEDI version code. + * @n_subdevs: Number of subdevices the device has. + * @driver_name: Null-terminated COMEDI driver name. + * @board_name: Null-terminated COMEDI board name. + * @read_subdevice: Index of the current "read" subdevice (%-1 if none). + * @write_subdevice: Index of the current "write" subdevice (%-1 if none). + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_DEVINFO ioctl to get basic information about + * the device. + */ +struct comedi_devinfo { + unsigned int version_code; + unsigned int n_subdevs; + char driver_name[COMEDI_NAMELEN]; + char board_name[COMEDI_NAMELEN]; + int read_subdevice; + int write_subdevice; + int unused[30]; +}; + +/** + * struct comedi_devconfig - used to configure a legacy COMEDI device + * @board_name: Null-terminated string specifying the type of board + * to configure. + * @options: An array of integer configuration options. + * + * This is used with the %COMEDI_DEVCONFIG ioctl to configure a "legacy" COMEDI + * device, such as an ISA card. Not all COMEDI drivers support this. Those + * that do either expect the specified board name to match one of a list of + * names registered with the COMEDI core, or expect the specified board name + * to match the COMEDI driver name itself. The configuration options are + * handled in a driver-specific manner. + */ +struct comedi_devconfig { + char board_name[COMEDI_NAMELEN]; + int options[COMEDI_NDEVCONFOPTS]; +}; + +/** + * struct comedi_bufconfig - used to set or get buffer size for a subdevice + * @subdevice: Subdevice index. + * @flags: Not used. + * @maximum_size: Maximum allowed buffer size. + * @size: Buffer size. + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_BUFCONFIG ioctl to get or configure the + * maximum buffer size and current buffer size for a COMEDI subdevice that + * supports asynchronous commands. If the subdevice does not support + * asynchronous commands, @maximum_size and @size are ignored and set to 0. + * + * On ioctl input, non-zero values of @maximum_size and @size specify a + * new maximum size and new current size (in bytes), respectively. These + * will by rounded up to a multiple of %PAGE_SIZE. Specifying a new maximum + * size requires admin capabilities. + * + * On ioctl output, @maximum_size and @size and set to the current maximum + * buffer size and current buffer size, respectively. + */ +struct comedi_bufconfig { + unsigned int subdevice; + unsigned int flags; + + unsigned int maximum_size; + unsigned int size; + + unsigned int unused[4]; +}; + +/** + * struct comedi_bufinfo - used to manipulate buffer position for a subdevice + * @subdevice: Subdevice index. + * @bytes_read: Specify amount to advance read position for an + * asynchronous command in the input ("read") direction. + * @buf_write_ptr: Current write position (index) within the buffer. + * @buf_read_ptr: Current read position (index) within the buffer. + * @buf_write_count: Total amount written, modulo 2^32. + * @buf_read_count: Total amount read, modulo 2^32. + * @bytes_written: Specify amount to advance write position for an + * asynchronous command in the output ("write") direction. + * @unused: Reserved for future use. + * + * This is used with the %COMEDI_BUFINFO ioctl to optionally advance the + * current read or write position in an asynchronous acquisition data buffer, + * and to get the current read and write positions in the buffer. + */ +struct comedi_bufinfo { + unsigned int subdevice; + unsigned int bytes_read; + + unsigned int buf_write_ptr; + unsigned int buf_read_ptr; + unsigned int buf_write_count; + unsigned int buf_read_count; + + unsigned int bytes_written; + + unsigned int unused[4]; +}; + +/* range stuff */ + +#define __RANGE(a, b) ((((a) & 0xffff) << 16) | ((b) & 0xffff)) + +#define RANGE_OFFSET(a) (((a) >> 16) & 0xffff) +#define RANGE_LENGTH(b) ((b) & 0xffff) + +#define RF_UNIT(flags) ((flags) & 0xff) +#define RF_EXTERNAL 0x100 + +#define UNIT_volt 0 +#define UNIT_mA 1 +#define UNIT_none 2 + +#define COMEDI_MIN_SPEED 0xffffffffu + +/**********************************************************/ +/* everything after this line is ALPHA */ +/**********************************************************/ + +/* + * 8254 specific configuration. + * + * It supports two config commands: + * + * 0 ID: INSN_CONFIG_SET_COUNTER_MODE + * 1 8254 Mode + * I8254_MODE0, I8254_MODE1, ..., I8254_MODE5 + * OR'ed with: + * I8254_BCD, I8254_BINARY + * + * 0 ID: INSN_CONFIG_8254_READ_STATUS + * 1 <-- Status byte returned here. + * B7 = Output + * B6 = NULL Count + * B5 - B0 Current mode. + */ + +enum i8254_mode { + I8254_MODE0 = (0 << 1), /* Interrupt on terminal count */ + I8254_MODE1 = (1 << 1), /* Hardware retriggerable one-shot */ + I8254_MODE2 = (2 << 1), /* Rate generator */ + I8254_MODE3 = (3 << 1), /* Square wave mode */ + I8254_MODE4 = (4 << 1), /* Software triggered strobe */ + /* Hardware triggered strobe (retriggerable) */ + I8254_MODE5 = (5 << 1), + /* Use binary-coded decimal instead of binary (pretty useless) */ + I8254_BCD = 1, + I8254_BINARY = 0 +}; + +/* *** BEGIN GLOBALLY-NAMED NI TERMINALS/SIGNALS *** */ + +/* + * Common National Instruments Terminal/Signal names. + * Some of these have no NI_ prefix as they are useful for non-NI hardware, such + * as those that utilize the PXI/RTSI trigger lines. + * + * NOTE ABOUT THE CHOICE OF NAMES HERE AND THE CAMELSCRIPT: + * The choice to use CamelScript and the exact names below is for + * maintainability, clarity, similarity to manufacturer's documentation, + * _and_ a mitigation for confusion that has plagued the use of these drivers + * for years! + * + * More detail: + * There have been significant confusions over the past many years for users + * when trying to understand how to connect to/from signals and terminals on + * NI hardware using comedi. The major reason for this is that the actual + * register values were exposed and required to be used by users. Several + * major reasons exist why this caused major confusion for users: + * 1) The register values are _NOT_ in user documentation, but rather in + * arcane locations, such as a few register programming manuals that are + * increasingly hard to find and the NI MHDDK (comments in example code). + * There is no one place to find the various valid values of the registers. + * 2) The register values are _NOT_ completely consistent. There is no way to + * gain any sense of intuition of which values, or even enums one should use + * for various registers. There was some attempt in prior use of comedi to + * name enums such that a user might know which enums should be used for + * varying purposes, but the end-user had to gain a knowledge of register + * values to correctly wield this approach. + * 3) The names for signals and registers found in the various register level + * programming manuals and vendor-provided documentation are _not_ even + * close to the same names that are in the end-user documentation. + * + * Similar, albeit less, confusion plagued NI's previous version of their own + * drivers. Earlier than 2003, NI greatly simplified the situation for users + * by releasing a new API that abstracted the names of signals/terminals to a + * common and intuitive set of names. + * + * The names below mirror the names chosen and well documented by NI. These + * names are exposed to the user via the comedilib user library. By keeping + * the names below, in spite of the use of CamelScript, maintenance will be + * greatly eased and confusion for users _and_ comedi developers will be + * greatly reduced. + */ + +/* + * Base of abstracted NI names. + * The first 16 bits of *_arg are reserved for channel selection. + * Since we only actually need the first 4 or 5 bits for all register values on + * NI select registers anyways, we'll identify all values >= (1<<15) as being an + * abstracted NI signal/terminal name. + * These values are also used/returned by INSN_DEVICE_CONFIG_TEST_ROUTE, + * INSN_DEVICE_CONFIG_CONNECT_ROUTE, INSN_DEVICE_CONFIG_DISCONNECT_ROUTE, + * and INSN_DEVICE_CONFIG_GET_ROUTES. + */ +#define NI_NAMES_BASE 0x8000u + +#define _TERM_N(base, n, x) ((base) + ((x) & ((n) - 1))) + +/* + * not necessarily all allowed 64 PFIs are valid--certainly not for all devices + */ +#define NI_PFI(x) _TERM_N(NI_NAMES_BASE, 64, x) +/* 8 trigger lines by standard, Some devices cannot talk to all eight. */ +#define TRIGGER_LINE(x) _TERM_N(NI_PFI(-1) + 1, 8, x) +/* 4 RTSI shared MUXes to route signals to/from TRIGGER_LINES on NI hardware */ +#define NI_RTSI_BRD(x) _TERM_N(TRIGGER_LINE(-1) + 1, 4, x) + +/* *** Counter/timer names : 8 counters max *** */ +#define NI_MAX_COUNTERS 8 +#define NI_COUNTER_NAMES_BASE (NI_RTSI_BRD(-1) + 1) +#define NI_CtrSource(x) _TERM_N(NI_COUNTER_NAMES_BASE, NI_MAX_COUNTERS, x) +/* Gate, Aux, A,B,Z are all treated, at times as gates */ +#define NI_GATES_NAMES_BASE (NI_CtrSource(-1) + 1) +#define NI_CtrGate(x) _TERM_N(NI_GATES_NAMES_BASE, NI_MAX_COUNTERS, x) +#define NI_CtrAux(x) _TERM_N(NI_CtrGate(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_CtrA(x) _TERM_N(NI_CtrAux(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_CtrB(x) _TERM_N(NI_CtrA(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_CtrZ(x) _TERM_N(NI_CtrB(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_GATES_NAMES_MAX NI_CtrZ(-1) +#define NI_CtrArmStartTrigger(x) _TERM_N(NI_CtrZ(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_CtrInternalOutput(x) \ + _TERM_N(NI_CtrArmStartTrigger(-1) + 1, NI_MAX_COUNTERS, x) +/** external pin(s) labeled conveniently as CtrOut. */ +#define NI_CtrOut(x) _TERM_N(NI_CtrInternalOutput(-1) + 1, NI_MAX_COUNTERS, x) +/** For Buffered sampling of ctr -- x series capability. */ +#define NI_CtrSampleClock(x) _TERM_N(NI_CtrOut(-1) + 1, NI_MAX_COUNTERS, x) +#define NI_COUNTER_NAMES_MAX NI_CtrSampleClock(-1) + +enum ni_common_signal_names { + /* PXI_Star: this is a non-NI-specific signal */ + PXI_Star = NI_COUNTER_NAMES_MAX + 1, + PXI_Clk10, + PXIe_Clk100, + NI_AI_SampleClock, + NI_AI_SampleClockTimebase, + NI_AI_StartTrigger, + NI_AI_ReferenceTrigger, + NI_AI_ConvertClock, + NI_AI_ConvertClockTimebase, + NI_AI_PauseTrigger, + NI_AI_HoldCompleteEvent, + NI_AI_HoldComplete, + NI_AI_ExternalMUXClock, + NI_AI_STOP, /* pulse signal that occurs when a update is finished(?) */ + NI_AO_SampleClock, + NI_AO_SampleClockTimebase, + NI_AO_StartTrigger, + NI_AO_PauseTrigger, + NI_DI_SampleClock, + NI_DI_SampleClockTimebase, + NI_DI_StartTrigger, + NI_DI_ReferenceTrigger, + NI_DI_PauseTrigger, + NI_DI_InputBufferFull, + NI_DI_ReadyForStartEvent, + NI_DI_ReadyForTransferEventBurst, + NI_DI_ReadyForTransferEventPipelined, + NI_DO_SampleClock, + NI_DO_SampleClockTimebase, + NI_DO_StartTrigger, + NI_DO_PauseTrigger, + NI_DO_OutputBufferFull, + NI_DO_DataActiveEvent, + NI_DO_ReadyForStartEvent, + NI_DO_ReadyForTransferEvent, + NI_MasterTimebase, + NI_20MHzTimebase, + NI_80MHzTimebase, + NI_100MHzTimebase, + NI_200MHzTimebase, + NI_100kHzTimebase, + NI_10MHzRefClock, + NI_FrequencyOutput, + NI_ChangeDetectionEvent, + NI_AnalogComparisonEvent, + NI_WatchdogExpiredEvent, + NI_WatchdogExpirationTrigger, + NI_SCXI_Trig1, + NI_LogicLow, + NI_LogicHigh, + NI_ExternalStrobe, + NI_PFI_DO, + NI_CaseGround, + /* special internal signal used as variable source for RTSI bus: */ + NI_RGOUT0, + + /* just a name to make the next more convenient, regardless of above */ + _NI_NAMES_MAX_PLUS_1, + NI_NUM_NAMES = _NI_NAMES_MAX_PLUS_1 - NI_NAMES_BASE, +}; + +/* *** END GLOBALLY-NAMED NI TERMINALS/SIGNALS *** */ + +#define NI_USUAL_PFI_SELECT(x) (((x) < 10) ? (0x1 + (x)) : (0xb + (x))) +#define NI_USUAL_RTSI_SELECT(x) (((x) < 7) ? (0xb + (x)) : 0x1b) + +/* + * mode bits for NI general-purpose counters, set with + * INSN_CONFIG_SET_COUNTER_MODE + */ +#define NI_GPCT_COUNTING_MODE_SHIFT 16 +#define NI_GPCT_INDEX_PHASE_BITSHIFT 20 +#define NI_GPCT_COUNTING_DIRECTION_SHIFT 24 +enum ni_gpct_mode_bits { + NI_GPCT_GATE_ON_BOTH_EDGES_BIT = 0x4, + NI_GPCT_EDGE_GATE_MODE_MASK = 0x18, + NI_GPCT_EDGE_GATE_STARTS_STOPS_BITS = 0x0, + NI_GPCT_EDGE_GATE_STOPS_STARTS_BITS = 0x8, + NI_GPCT_EDGE_GATE_STARTS_BITS = 0x10, + NI_GPCT_EDGE_GATE_NO_STARTS_NO_STOPS_BITS = 0x18, + NI_GPCT_STOP_MODE_MASK = 0x60, + NI_GPCT_STOP_ON_GATE_BITS = 0x00, + NI_GPCT_STOP_ON_GATE_OR_TC_BITS = 0x20, + NI_GPCT_STOP_ON_GATE_OR_SECOND_TC_BITS = 0x40, + NI_GPCT_LOAD_B_SELECT_BIT = 0x80, + NI_GPCT_OUTPUT_MODE_MASK = 0x300, + NI_GPCT_OUTPUT_TC_PULSE_BITS = 0x100, + NI_GPCT_OUTPUT_TC_TOGGLE_BITS = 0x200, + NI_GPCT_OUTPUT_TC_OR_GATE_TOGGLE_BITS = 0x300, + NI_GPCT_HARDWARE_DISARM_MASK = 0xc00, + NI_GPCT_NO_HARDWARE_DISARM_BITS = 0x000, + NI_GPCT_DISARM_AT_TC_BITS = 0x400, + NI_GPCT_DISARM_AT_GATE_BITS = 0x800, + NI_GPCT_DISARM_AT_TC_OR_GATE_BITS = 0xc00, + NI_GPCT_LOADING_ON_TC_BIT = 0x1000, + NI_GPCT_LOADING_ON_GATE_BIT = 0x4000, + NI_GPCT_COUNTING_MODE_MASK = 0x7 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_NORMAL_BITS = + 0x0 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_QUADRATURE_X1_BITS = + 0x1 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_QUADRATURE_X2_BITS = + 0x2 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_QUADRATURE_X4_BITS = + 0x3 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_TWO_PULSE_BITS = + 0x4 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_COUNTING_MODE_SYNC_SOURCE_BITS = + 0x6 << NI_GPCT_COUNTING_MODE_SHIFT, + NI_GPCT_INDEX_PHASE_MASK = 0x3 << NI_GPCT_INDEX_PHASE_BITSHIFT, + NI_GPCT_INDEX_PHASE_LOW_A_LOW_B_BITS = + 0x0 << NI_GPCT_INDEX_PHASE_BITSHIFT, + NI_GPCT_INDEX_PHASE_LOW_A_HIGH_B_BITS = + 0x1 << NI_GPCT_INDEX_PHASE_BITSHIFT, + NI_GPCT_INDEX_PHASE_HIGH_A_LOW_B_BITS = + 0x2 << NI_GPCT_INDEX_PHASE_BITSHIFT, + NI_GPCT_INDEX_PHASE_HIGH_A_HIGH_B_BITS = + 0x3 << NI_GPCT_INDEX_PHASE_BITSHIFT, + NI_GPCT_INDEX_ENABLE_BIT = 0x400000, + NI_GPCT_COUNTING_DIRECTION_MASK = + 0x3 << NI_GPCT_COUNTING_DIRECTION_SHIFT, + NI_GPCT_COUNTING_DIRECTION_DOWN_BITS = + 0x00 << NI_GPCT_COUNTING_DIRECTION_SHIFT, + NI_GPCT_COUNTING_DIRECTION_UP_BITS = + 0x1 << NI_GPCT_COUNTING_DIRECTION_SHIFT, + NI_GPCT_COUNTING_DIRECTION_HW_UP_DOWN_BITS = + 0x2 << NI_GPCT_COUNTING_DIRECTION_SHIFT, + NI_GPCT_COUNTING_DIRECTION_HW_GATE_BITS = + 0x3 << NI_GPCT_COUNTING_DIRECTION_SHIFT, + NI_GPCT_RELOAD_SOURCE_MASK = 0xc000000, + NI_GPCT_RELOAD_SOURCE_FIXED_BITS = 0x0, + NI_GPCT_RELOAD_SOURCE_SWITCHING_BITS = 0x4000000, + NI_GPCT_RELOAD_SOURCE_GATE_SELECT_BITS = 0x8000000, + NI_GPCT_OR_GATE_BIT = 0x10000000, + NI_GPCT_INVERT_OUTPUT_BIT = 0x20000000 +}; + +/* + * Bits for setting a clock source with + * INSN_CONFIG_SET_CLOCK_SRC when using NI general-purpose counters. + */ +enum ni_gpct_clock_source_bits { + NI_GPCT_CLOCK_SRC_SELECT_MASK = 0x3f, + NI_GPCT_TIMEBASE_1_CLOCK_SRC_BITS = 0x0, + NI_GPCT_TIMEBASE_2_CLOCK_SRC_BITS = 0x1, + NI_GPCT_TIMEBASE_3_CLOCK_SRC_BITS = 0x2, + NI_GPCT_LOGIC_LOW_CLOCK_SRC_BITS = 0x3, + NI_GPCT_NEXT_GATE_CLOCK_SRC_BITS = 0x4, + NI_GPCT_NEXT_TC_CLOCK_SRC_BITS = 0x5, + /* NI 660x-specific */ + NI_GPCT_SOURCE_PIN_i_CLOCK_SRC_BITS = 0x6, + NI_GPCT_PXI10_CLOCK_SRC_BITS = 0x7, + NI_GPCT_PXI_STAR_TRIGGER_CLOCK_SRC_BITS = 0x8, + NI_GPCT_ANALOG_TRIGGER_OUT_CLOCK_SRC_BITS = 0x9, + NI_GPCT_PRESCALE_MODE_CLOCK_SRC_MASK = 0x30000000, + NI_GPCT_NO_PRESCALE_CLOCK_SRC_BITS = 0x0, + /* divide source by 2 */ + NI_GPCT_PRESCALE_X2_CLOCK_SRC_BITS = 0x10000000, + /* divide source by 8 */ + NI_GPCT_PRESCALE_X8_CLOCK_SRC_BITS = 0x20000000, + NI_GPCT_INVERT_CLOCK_SRC_BIT = 0x80000000 +}; + +/* NI 660x-specific */ +#define NI_GPCT_SOURCE_PIN_CLOCK_SRC_BITS(x) (0x10 + (x)) + +#define NI_GPCT_RTSI_CLOCK_SRC_BITS(x) (0x18 + (x)) + +/* no pfi on NI 660x */ +#define NI_GPCT_PFI_CLOCK_SRC_BITS(x) (0x20 + (x)) + +/* + * Possibilities for setting a gate source with + * INSN_CONFIG_SET_GATE_SRC when using NI general-purpose counters. + * May be bitwise-or'd with CR_EDGE or CR_INVERT. + */ +enum ni_gpct_gate_select { + /* m-series gates */ + NI_GPCT_TIMESTAMP_MUX_GATE_SELECT = 0x0, + NI_GPCT_AI_START2_GATE_SELECT = 0x12, + NI_GPCT_PXI_STAR_TRIGGER_GATE_SELECT = 0x13, + NI_GPCT_NEXT_OUT_GATE_SELECT = 0x14, + NI_GPCT_AI_START1_GATE_SELECT = 0x1c, + NI_GPCT_NEXT_SOURCE_GATE_SELECT = 0x1d, + NI_GPCT_ANALOG_TRIGGER_OUT_GATE_SELECT = 0x1e, + NI_GPCT_LOGIC_LOW_GATE_SELECT = 0x1f, + /* more gates for 660x */ + NI_GPCT_SOURCE_PIN_i_GATE_SELECT = 0x100, + NI_GPCT_GATE_PIN_i_GATE_SELECT = 0x101, + /* more gates for 660x "second gate" */ + NI_GPCT_UP_DOWN_PIN_i_GATE_SELECT = 0x201, + NI_GPCT_SELECTED_GATE_GATE_SELECT = 0x21e, + /* + * m-series "second gate" sources are unknown, + * we should add them here with an offset of 0x300 when + * known. + */ + NI_GPCT_DISABLED_GATE_SELECT = 0x8000, +}; + +#define NI_GPCT_GATE_PIN_GATE_SELECT(x) (0x102 + (x)) +#define NI_GPCT_RTSI_GATE_SELECT(x) NI_USUAL_RTSI_SELECT(x) +#define NI_GPCT_PFI_GATE_SELECT(x) NI_USUAL_PFI_SELECT(x) +#define NI_GPCT_UP_DOWN_PIN_GATE_SELECT(x) (0x202 + (x)) + +/* + * Possibilities for setting a source with + * INSN_CONFIG_SET_OTHER_SRC when using NI general-purpose counters. + */ +enum ni_gpct_other_index { + NI_GPCT_SOURCE_ENCODER_A, + NI_GPCT_SOURCE_ENCODER_B, + NI_GPCT_SOURCE_ENCODER_Z +}; + +enum ni_gpct_other_select { + /* m-series gates */ + /* Still unknown, probably only need NI_GPCT_PFI_OTHER_SELECT */ + NI_GPCT_DISABLED_OTHER_SELECT = 0x8000, +}; + +#define NI_GPCT_PFI_OTHER_SELECT(x) NI_USUAL_PFI_SELECT(x) + +/* + * start sources for ni general-purpose counters for use with + * INSN_CONFIG_ARM + */ +enum ni_gpct_arm_source { + NI_GPCT_ARM_IMMEDIATE = 0x0, + /* + * Start both the counter and the adjacent paired counter simultaneously + */ + NI_GPCT_ARM_PAIRED_IMMEDIATE = 0x1, + /* + * If the NI_GPCT_HW_ARM bit is set, we will pass the least significant + * bits (3 bits for 660x or 5 bits for m-series) through to the + * hardware. To select a hardware trigger, pass the appropriate select + * bit, e.g., + * NI_GPCT_HW_ARM | NI_GPCT_AI_START1_GATE_SELECT or + * NI_GPCT_HW_ARM | NI_GPCT_PFI_GATE_SELECT(pfi_number) + */ + NI_GPCT_HW_ARM = 0x1000, + NI_GPCT_ARM_UNKNOWN = NI_GPCT_HW_ARM, /* for backward compatibility */ +}; + +/* digital filtering options for ni 660x for use with INSN_CONFIG_FILTER. */ +enum ni_gpct_filter_select { + NI_GPCT_FILTER_OFF = 0x0, + NI_GPCT_FILTER_TIMEBASE_3_SYNC = 0x1, + NI_GPCT_FILTER_100x_TIMEBASE_1 = 0x2, + NI_GPCT_FILTER_20x_TIMEBASE_1 = 0x3, + NI_GPCT_FILTER_10x_TIMEBASE_1 = 0x4, + NI_GPCT_FILTER_2x_TIMEBASE_1 = 0x5, + NI_GPCT_FILTER_2x_TIMEBASE_3 = 0x6 +}; + +/* + * PFI digital filtering options for ni m-series for use with + * INSN_CONFIG_FILTER. + */ +enum ni_pfi_filter_select { + NI_PFI_FILTER_OFF = 0x0, + NI_PFI_FILTER_125ns = 0x1, + NI_PFI_FILTER_6425ns = 0x2, + NI_PFI_FILTER_2550us = 0x3 +}; + +/* master clock sources for ni mio boards and INSN_CONFIG_SET_CLOCK_SRC */ +enum ni_mio_clock_source { + NI_MIO_INTERNAL_CLOCK = 0, + /* + * Doesn't work for m-series, use NI_MIO_PLL_RTSI_CLOCK() + * the NI_MIO_PLL_* sources are m-series only + */ + NI_MIO_RTSI_CLOCK = 1, + NI_MIO_PLL_PXI_STAR_TRIGGER_CLOCK = 2, + NI_MIO_PLL_PXI10_CLOCK = 3, + NI_MIO_PLL_RTSI0_CLOCK = 4 +}; + +#define NI_MIO_PLL_RTSI_CLOCK(x) (NI_MIO_PLL_RTSI0_CLOCK + (x)) + +/* + * Signals which can be routed to an NI RTSI pin with INSN_CONFIG_SET_ROUTING. + * The numbers assigned are not arbitrary, they correspond to the bits required + * to program the board. + */ +enum ni_rtsi_routing { + NI_RTSI_OUTPUT_ADR_START1 = 0, + NI_RTSI_OUTPUT_ADR_START2 = 1, + NI_RTSI_OUTPUT_SCLKG = 2, + NI_RTSI_OUTPUT_DACUPDN = 3, + NI_RTSI_OUTPUT_DA_START1 = 4, + NI_RTSI_OUTPUT_G_SRC0 = 5, + NI_RTSI_OUTPUT_G_GATE0 = 6, + NI_RTSI_OUTPUT_RGOUT0 = 7, + NI_RTSI_OUTPUT_RTSI_BRD_0 = 8, + /* Pre-m-series always have RTSI clock on line 7 */ + NI_RTSI_OUTPUT_RTSI_OSC = 12 +}; + +#define NI_RTSI_OUTPUT_RTSI_BRD(x) (NI_RTSI_OUTPUT_RTSI_BRD_0 + (x)) + +/* + * Signals which can be routed to an NI PFI pin on an m-series board with + * INSN_CONFIG_SET_ROUTING. These numbers are also returned by + * INSN_CONFIG_GET_ROUTING on pre-m-series boards, even though their routing + * cannot be changed. The numbers assigned are not arbitrary, they correspond + * to the bits required to program the board. + */ +enum ni_pfi_routing { + NI_PFI_OUTPUT_PFI_DEFAULT = 0, + NI_PFI_OUTPUT_AI_START1 = 1, + NI_PFI_OUTPUT_AI_START2 = 2, + NI_PFI_OUTPUT_AI_CONVERT = 3, + NI_PFI_OUTPUT_G_SRC1 = 4, + NI_PFI_OUTPUT_G_GATE1 = 5, + NI_PFI_OUTPUT_AO_UPDATE_N = 6, + NI_PFI_OUTPUT_AO_START1 = 7, + NI_PFI_OUTPUT_AI_START_PULSE = 8, + NI_PFI_OUTPUT_G_SRC0 = 9, + NI_PFI_OUTPUT_G_GATE0 = 10, + NI_PFI_OUTPUT_EXT_STROBE = 11, + NI_PFI_OUTPUT_AI_EXT_MUX_CLK = 12, + NI_PFI_OUTPUT_GOUT0 = 13, + NI_PFI_OUTPUT_GOUT1 = 14, + NI_PFI_OUTPUT_FREQ_OUT = 15, + NI_PFI_OUTPUT_PFI_DO = 16, + NI_PFI_OUTPUT_I_ATRIG = 17, + NI_PFI_OUTPUT_RTSI0 = 18, + NI_PFI_OUTPUT_PXI_STAR_TRIGGER_IN = 26, + NI_PFI_OUTPUT_SCXI_TRIG1 = 27, + NI_PFI_OUTPUT_DIO_CHANGE_DETECT_RTSI = 28, + NI_PFI_OUTPUT_CDI_SAMPLE = 29, + NI_PFI_OUTPUT_CDO_UPDATE = 30 +}; + +#define NI_PFI_OUTPUT_RTSI(x) (NI_PFI_OUTPUT_RTSI0 + (x)) + +/* + * Signals which can be routed to output on a NI PFI pin on a 660x board + * with INSN_CONFIG_SET_ROUTING. The numbers assigned are + * not arbitrary, they correspond to the bits required + * to program the board. Lines 0 to 7 can only be set to + * NI_660X_PFI_OUTPUT_DIO. Lines 32 to 39 can only be set to + * NI_660X_PFI_OUTPUT_COUNTER. + */ +enum ni_660x_pfi_routing { + NI_660X_PFI_OUTPUT_COUNTER = 1, /* counter */ + NI_660X_PFI_OUTPUT_DIO = 2, /* static digital output */ +}; + +/* + * NI External Trigger lines. These values are not arbitrary, but are related + * to the bits required to program the board (offset by 1 for historical + * reasons). + */ +#define NI_EXT_PFI(x) (NI_USUAL_PFI_SELECT(x) - 1) +#define NI_EXT_RTSI(x) (NI_USUAL_RTSI_SELECT(x) - 1) + +/* + * Clock sources for CDIO subdevice on NI m-series boards. Used as the + * scan_begin_arg for a comedi_command. These sources may also be bitwise-or'd + * with CR_INVERT to change polarity. + */ +enum ni_m_series_cdio_scan_begin_src { + NI_CDIO_SCAN_BEGIN_SRC_GROUND = 0, + NI_CDIO_SCAN_BEGIN_SRC_AI_START = 18, + NI_CDIO_SCAN_BEGIN_SRC_AI_CONVERT = 19, + NI_CDIO_SCAN_BEGIN_SRC_PXI_STAR_TRIGGER = 20, + NI_CDIO_SCAN_BEGIN_SRC_G0_OUT = 28, + NI_CDIO_SCAN_BEGIN_SRC_G1_OUT = 29, + NI_CDIO_SCAN_BEGIN_SRC_ANALOG_TRIGGER = 30, + NI_CDIO_SCAN_BEGIN_SRC_AO_UPDATE = 31, + NI_CDIO_SCAN_BEGIN_SRC_FREQ_OUT = 32, + NI_CDIO_SCAN_BEGIN_SRC_DIO_CHANGE_DETECT_IRQ = 33 +}; + +#define NI_CDIO_SCAN_BEGIN_SRC_PFI(x) NI_USUAL_PFI_SELECT(x) +#define NI_CDIO_SCAN_BEGIN_SRC_RTSI(x) NI_USUAL_RTSI_SELECT(x) + +/* + * scan_begin_src for scan_begin_arg==TRIG_EXT with analog output command on NI + * boards. These scan begin sources can also be bitwise-or'd with CR_INVERT to + * change polarity. + */ +#define NI_AO_SCAN_BEGIN_SRC_PFI(x) NI_USUAL_PFI_SELECT(x) +#define NI_AO_SCAN_BEGIN_SRC_RTSI(x) NI_USUAL_RTSI_SELECT(x) + +/* + * Bits for setting a clock source with + * INSN_CONFIG_SET_CLOCK_SRC when using NI frequency output subdevice. + */ +enum ni_freq_out_clock_source_bits { + NI_FREQ_OUT_TIMEBASE_1_DIV_2_CLOCK_SRC, /* 10 MHz */ + NI_FREQ_OUT_TIMEBASE_2_CLOCK_SRC /* 100 KHz */ +}; + +/* + * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for + * 8254 counter subdevices on Amplicon DIO boards (amplc_dio200 driver). + */ +enum amplc_dio_clock_source { + /* + * Per channel external clock + * input/output pin (pin is only an + * input when clock source set to this value, + * otherwise it is an output) + */ + AMPLC_DIO_CLK_CLKN, + AMPLC_DIO_CLK_10MHZ, /* 10 MHz internal clock */ + AMPLC_DIO_CLK_1MHZ, /* 1 MHz internal clock */ + AMPLC_DIO_CLK_100KHZ, /* 100 kHz internal clock */ + AMPLC_DIO_CLK_10KHZ, /* 10 kHz internal clock */ + AMPLC_DIO_CLK_1KHZ, /* 1 kHz internal clock */ + /* + * Output of preceding counter channel + * (for channel 0, preceding counter + * channel is channel 2 on preceding + * counter subdevice, for first counter + * subdevice, preceding counter + * subdevice is the last counter + * subdevice) + */ + AMPLC_DIO_CLK_OUTNM1, + AMPLC_DIO_CLK_EXT, /* per chip external input pin */ + /* the following are "enhanced" clock sources for PCIe models */ + AMPLC_DIO_CLK_VCC, /* clock input HIGH */ + AMPLC_DIO_CLK_GND, /* clock input LOW */ + AMPLC_DIO_CLK_PAT_PRESENT, /* "pattern present" signal */ + AMPLC_DIO_CLK_20MHZ /* 20 MHz internal clock */ +}; + +/* + * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for + * timer subdevice on some Amplicon DIO PCIe boards (amplc_dio200 driver). + */ +enum amplc_dio_ts_clock_src { + AMPLC_DIO_TS_CLK_1GHZ, /* 1 ns period with 20 ns granularity */ + AMPLC_DIO_TS_CLK_1MHZ, /* 1 us period */ + AMPLC_DIO_TS_CLK_1KHZ /* 1 ms period */ +}; + +/* + * Values for setting a gate source with INSN_CONFIG_SET_GATE_SRC for + * 8254 counter subdevices on Amplicon DIO boards (amplc_dio200 driver). + */ +enum amplc_dio_gate_source { + AMPLC_DIO_GAT_VCC, /* internal high logic level */ + AMPLC_DIO_GAT_GND, /* internal low logic level */ + AMPLC_DIO_GAT_GATN, /* per channel external gate input */ + /* + * negated output of counter channel minus 2 + * (for channels 0 or 1, channel minus 2 is channel 1 or 2 on + * the preceding counter subdevice, for the first counter subdevice + * the preceding counter subdevice is the last counter subdevice) + */ + AMPLC_DIO_GAT_NOUTNM2, + AMPLC_DIO_GAT_RESERVED4, + AMPLC_DIO_GAT_RESERVED5, + AMPLC_DIO_GAT_RESERVED6, + AMPLC_DIO_GAT_RESERVED7, + /* the following are "enhanced" gate sources for PCIe models */ + AMPLC_DIO_GAT_NGATN = 6, /* negated per channel gate input */ + /* non-negated output of counter channel minus 2 */ + AMPLC_DIO_GAT_OUTNM2, + AMPLC_DIO_GAT_PAT_PRESENT, /* "pattern present" signal */ + AMPLC_DIO_GAT_PAT_OCCURRED, /* "pattern occurred" latched */ + AMPLC_DIO_GAT_PAT_GONE, /* "pattern gone away" latched */ + AMPLC_DIO_GAT_NPAT_PRESENT, /* negated "pattern present" */ + AMPLC_DIO_GAT_NPAT_OCCURRED, /* negated "pattern occurred" */ + AMPLC_DIO_GAT_NPAT_GONE /* negated "pattern gone away" */ +}; + +/* + * Values for setting a clock source with INSN_CONFIG_SET_CLOCK_SRC for + * the counter subdevice on the Kolter Electronic PCI-Counter board + * (ke_counter driver). + */ +enum ke_counter_clock_source { + KE_CLK_20MHZ, /* internal 20MHz (default) */ + KE_CLK_4MHZ, /* internal 4MHz (option) */ + KE_CLK_EXT /* external clock on pin 21 of D-Sub */ +}; + +#endif /* _COMEDI_H */ -- cgit v1.2.3-59-g8ed1b From 631e272b12075b60f7c7fc4f84f937d78a699844 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Wed, 17 Nov 2021 12:06:01 +0000 Subject: comedi: Move and rename "8255.h" to Some of the header files in "drivers/comedi/drivers/" are common enough to be useful to out-of-tree comedi driver modules. Using them for out-of-tree module builds is hampered by the headers being outside the "include/" directory so it is desirable to move them. There are about a couple of dozen Comedi device drivers that use the "comedi_8255" module to add digital I/O subdevices based on the venerable 8255 Programmable Peripheral Interface chip. The macros and declarations to use that module are in the "8255.h" header file in the comedi "drivers" directory. Move it into "include/linux/comedi/" and rename it to "comedi_8255.h" for naming consistency reasons. Signed-off-by: Ian Abbott Link: https://lore.kernel.org/r/20211117120604.117740-4-abbotti@mev.co.uk Signed-off-by: Greg Kroah-Hartman --- drivers/comedi/drivers/8255.c | 3 +- drivers/comedi/drivers/8255.h | 42 ---------------------------- drivers/comedi/drivers/8255_pci.c | 3 +- drivers/comedi/drivers/adv_pci_dio.c | 2 +- drivers/comedi/drivers/aio_aio12_8.c | 2 +- drivers/comedi/drivers/amplc_dio200_common.c | 2 +- drivers/comedi/drivers/amplc_pc236_common.c | 2 +- drivers/comedi/drivers/amplc_pci230.c | 2 +- drivers/comedi/drivers/cb_pcidas.c | 2 +- drivers/comedi/drivers/cb_pcidas64.c | 2 +- drivers/comedi/drivers/cb_pcidda.c | 3 +- drivers/comedi/drivers/cb_pcimdas.c | 2 +- drivers/comedi/drivers/cb_pcimdda.c | 3 +- drivers/comedi/drivers/comedi_8255.c | 3 +- drivers/comedi/drivers/daqboard2000.c | 2 +- drivers/comedi/drivers/das08.c | 2 +- drivers/comedi/drivers/das16.c | 2 +- drivers/comedi/drivers/das16m1.c | 2 +- drivers/comedi/drivers/dmm32at.c | 3 +- drivers/comedi/drivers/ni_atmio.c | 2 +- drivers/comedi/drivers/ni_atmio16d.c | 3 +- drivers/comedi/drivers/ni_daq_dio24.c | 3 +- drivers/comedi/drivers/ni_labpc_common.c | 2 +- drivers/comedi/drivers/ni_mio_common.c | 2 +- drivers/comedi/drivers/ni_mio_cs.c | 2 +- drivers/comedi/drivers/pcl724.c | 3 +- drivers/comedi/drivers/pcm3724.c | 3 +- include/linux/comedi/comedi_8255.h | 42 ++++++++++++++++++++++++++++ 28 files changed, 68 insertions(+), 78 deletions(-) delete mode 100644 drivers/comedi/drivers/8255.h create mode 100644 include/linux/comedi/comedi_8255.h (limited to 'include/linux') diff --git a/drivers/comedi/drivers/8255.c b/drivers/comedi/drivers/8255.c index f23a52b7c919..ced8ea09d4fa 100644 --- a/drivers/comedi/drivers/8255.c +++ b/drivers/comedi/drivers/8255.c @@ -41,8 +41,7 @@ #include #include - -#include "8255.h" +#include static int dev_8255_attach(struct comedi_device *dev, struct comedi_devconfig *it) diff --git a/drivers/comedi/drivers/8255.h b/drivers/comedi/drivers/8255.h deleted file mode 100644 index ceae3ca52e60..000000000000 --- a/drivers/comedi/drivers/8255.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * module/8255.h - * Header file for 8255 - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 1998 David A. Schleef - */ - -#ifndef _8255_H -#define _8255_H - -#define I8255_SIZE 0x04 - -#define I8255_DATA_A_REG 0x00 -#define I8255_DATA_B_REG 0x01 -#define I8255_DATA_C_REG 0x02 -#define I8255_CTRL_REG 0x03 -#define I8255_CTRL_C_LO_IO BIT(0) -#define I8255_CTRL_B_IO BIT(1) -#define I8255_CTRL_B_MODE BIT(2) -#define I8255_CTRL_C_HI_IO BIT(3) -#define I8255_CTRL_A_IO BIT(4) -#define I8255_CTRL_A_MODE(x) ((x) << 5) -#define I8255_CTRL_CW BIT(7) - -struct comedi_device; -struct comedi_subdevice; - -int subdev_8255_init(struct comedi_device *dev, struct comedi_subdevice *s, - int (*io)(struct comedi_device *dev, int dir, int port, - int data, unsigned long regbase), - unsigned long regbase); - -int subdev_8255_mm_init(struct comedi_device *dev, struct comedi_subdevice *s, - int (*io)(struct comedi_device *dev, int dir, int port, - int data, unsigned long regbase), - unsigned long regbase); - -unsigned long subdev_8255_regbase(struct comedi_subdevice *s); - -#endif diff --git a/drivers/comedi/drivers/8255_pci.c b/drivers/comedi/drivers/8255_pci.c index 76b8b4762bae..0fec048e3a53 100644 --- a/drivers/comedi/drivers/8255_pci.c +++ b/drivers/comedi/drivers/8255_pci.c @@ -54,8 +54,7 @@ #include #include - -#include "8255.h" +#include enum pci_8255_boardid { BOARD_ADLINK_PCI7224, diff --git a/drivers/comedi/drivers/adv_pci_dio.c b/drivers/comedi/drivers/adv_pci_dio.c index 5947f08b9a1e..1ec602f8c6e1 100644 --- a/drivers/comedi/drivers/adv_pci_dio.c +++ b/drivers/comedi/drivers/adv_pci_dio.c @@ -24,8 +24,8 @@ #include #include #include +#include -#include "8255.h" #include "comedi_8254.h" /* diff --git a/drivers/comedi/drivers/aio_aio12_8.c b/drivers/comedi/drivers/aio_aio12_8.c index 36c3a2d8a352..cd797dc0f828 100644 --- a/drivers/comedi/drivers/aio_aio12_8.c +++ b/drivers/comedi/drivers/aio_aio12_8.c @@ -23,9 +23,9 @@ #include #include +#include #include "comedi_8254.h" -#include "8255.h" /* * Register map diff --git a/drivers/comedi/drivers/amplc_dio200_common.c b/drivers/comedi/drivers/amplc_dio200_common.c index 950c50be4ff3..26b4049b366c 100644 --- a/drivers/comedi/drivers/amplc_dio200_common.c +++ b/drivers/comedi/drivers/amplc_dio200_common.c @@ -13,10 +13,10 @@ #include #include #include +#include /* only for register defines */ #include "amplc_dio200.h" #include "comedi_8254.h" -#include "8255.h" /* only for register defines */ /* 200 series registers */ #define DIO200_IO_SIZE 0x20 diff --git a/drivers/comedi/drivers/amplc_pc236_common.c b/drivers/comedi/drivers/amplc_pc236_common.c index b8b0a624f72b..9f4f89b1ef23 100644 --- a/drivers/comedi/drivers/amplc_pc236_common.c +++ b/drivers/comedi/drivers/amplc_pc236_common.c @@ -12,9 +12,9 @@ #include #include #include +#include #include "amplc_pc236.h" -#include "8255.h" static void pc236_intr_update(struct comedi_device *dev, bool enable) { diff --git a/drivers/comedi/drivers/amplc_pci230.c b/drivers/comedi/drivers/amplc_pci230.c index 554ee40e321f..93f7057d5b3f 100644 --- a/drivers/comedi/drivers/amplc_pci230.c +++ b/drivers/comedi/drivers/amplc_pci230.c @@ -175,9 +175,9 @@ #include #include #include +#include #include "comedi_8254.h" -#include "8255.h" /* * PCI230 PCI configuration register information diff --git a/drivers/comedi/drivers/cb_pcidas.c b/drivers/comedi/drivers/cb_pcidas.c index 9b603532a4e7..75ff02b47959 100644 --- a/drivers/comedi/drivers/cb_pcidas.c +++ b/drivers/comedi/drivers/cb_pcidas.c @@ -55,9 +55,9 @@ #include #include #include +#include #include "comedi_8254.h" -#include "8255.h" #include "amcc_s5933.h" #define AI_BUFFER_SIZE 1024 /* max ai fifo size */ diff --git a/drivers/comedi/drivers/cb_pcidas64.c b/drivers/comedi/drivers/cb_pcidas64.c index 7d4808faa1fb..ca6038a25f26 100644 --- a/drivers/comedi/drivers/cb_pcidas64.c +++ b/drivers/comedi/drivers/cb_pcidas64.c @@ -74,8 +74,8 @@ #include #include #include +#include -#include "8255.h" #include "plx9080.h" #define TIMER_BASE 25 /* 40MHz master clock */ diff --git a/drivers/comedi/drivers/cb_pcidda.c b/drivers/comedi/drivers/cb_pcidda.c index 4ed3bcf47973..c52204a6bda4 100644 --- a/drivers/comedi/drivers/cb_pcidda.c +++ b/drivers/comedi/drivers/cb_pcidda.c @@ -28,8 +28,7 @@ #include #include - -#include "8255.h" +#include #define EEPROM_SIZE 128 /* number of entries in eeprom */ /* maximum number of ao channels for supported boards */ diff --git a/drivers/comedi/drivers/cb_pcimdas.c b/drivers/comedi/drivers/cb_pcimdas.c index 64c7d72c7956..7bc0805c69e2 100644 --- a/drivers/comedi/drivers/cb_pcimdas.c +++ b/drivers/comedi/drivers/cb_pcimdas.c @@ -35,10 +35,10 @@ #include #include #include +#include #include "comedi_8254.h" #include "plx9052.h" -#include "8255.h" /* * PCI Bar 1 Register map diff --git a/drivers/comedi/drivers/cb_pcimdda.c b/drivers/comedi/drivers/cb_pcimdda.c index 69d7803b0e58..bf8093a10315 100644 --- a/drivers/comedi/drivers/cb_pcimdda.c +++ b/drivers/comedi/drivers/cb_pcimdda.c @@ -68,8 +68,7 @@ #include #include - -#include "8255.h" +#include /* device ids of the cards we support -- currently only 1 card supported */ #define PCI_ID_PCIM_DDA06_16 0x0053 diff --git a/drivers/comedi/drivers/comedi_8255.c b/drivers/comedi/drivers/comedi_8255.c index 10614603d677..5562b9cd0a17 100644 --- a/drivers/comedi/drivers/comedi_8255.c +++ b/drivers/comedi/drivers/comedi_8255.c @@ -30,8 +30,7 @@ #include #include - -#include "8255.h" +#include struct subdev_8255_private { unsigned long regbase; diff --git a/drivers/comedi/drivers/daqboard2000.c b/drivers/comedi/drivers/daqboard2000.c index 52e4bf16cbda..c0a4e1b06fb3 100644 --- a/drivers/comedi/drivers/daqboard2000.c +++ b/drivers/comedi/drivers/daqboard2000.c @@ -97,8 +97,8 @@ #include #include #include +#include -#include "8255.h" #include "plx9080.h" #define DB2K_FIRMWARE "daqboard2000_firmware.bin" diff --git a/drivers/comedi/drivers/das08.c b/drivers/comedi/drivers/das08.c index c146a168f43b..bab868de2967 100644 --- a/drivers/comedi/drivers/das08.c +++ b/drivers/comedi/drivers/das08.c @@ -11,8 +11,8 @@ #include #include +#include -#include "8255.h" #include "comedi_8254.h" #include "das08.h" diff --git a/drivers/comedi/drivers/das16.c b/drivers/comedi/drivers/das16.c index 362232ad4409..338396736936 100644 --- a/drivers/comedi/drivers/das16.c +++ b/drivers/comedi/drivers/das16.c @@ -64,10 +64,10 @@ #include #include #include +#include #include "comedi_isadma.h" #include "comedi_8254.h" -#include "8255.h" #define DAS16_DMA_SIZE 0xff00 /* size in bytes of allocated dma buffer */ diff --git a/drivers/comedi/drivers/das16m1.c b/drivers/comedi/drivers/das16m1.c index cc79e318cb2d..ea55024d8c5a 100644 --- a/drivers/comedi/drivers/das16m1.c +++ b/drivers/comedi/drivers/das16m1.c @@ -43,8 +43,8 @@ #include #include #include +#include -#include "8255.h" #include "comedi_8254.h" /* diff --git a/drivers/comedi/drivers/dmm32at.c b/drivers/comedi/drivers/dmm32at.c index 0f2bea88b8a7..fe023c722aa3 100644 --- a/drivers/comedi/drivers/dmm32at.c +++ b/drivers/comedi/drivers/dmm32at.c @@ -30,8 +30,7 @@ #include #include #include - -#include "8255.h" +#include /* Board register addresses */ #define DMM32AT_AI_START_CONV_REG 0x00 diff --git a/drivers/comedi/drivers/ni_atmio.c b/drivers/comedi/drivers/ni_atmio.c index f60a4e459a98..8876a1d24c56 100644 --- a/drivers/comedi/drivers/ni_atmio.c +++ b/drivers/comedi/drivers/ni_atmio.c @@ -75,9 +75,9 @@ #include #include #include +#include #include "ni_stc.h" -#include "8255.h" /* AT specific setup */ static const struct ni_board_struct ni_boards[] = { diff --git a/drivers/comedi/drivers/ni_atmio16d.c b/drivers/comedi/drivers/ni_atmio16d.c index 0bd4f88a2ac8..9fa902529a8e 100644 --- a/drivers/comedi/drivers/ni_atmio16d.c +++ b/drivers/comedi/drivers/ni_atmio16d.c @@ -40,8 +40,7 @@ #include #include #include - -#include "8255.h" +#include /* Configuration and Status Registers */ #define COM_REG_1 0x00 /* wo 16 */ diff --git a/drivers/comedi/drivers/ni_daq_dio24.c b/drivers/comedi/drivers/ni_daq_dio24.c index 84d78f2ee5ac..487733111023 100644 --- a/drivers/comedi/drivers/ni_daq_dio24.c +++ b/drivers/comedi/drivers/ni_daq_dio24.c @@ -24,8 +24,7 @@ #include #include - -#include "8255.h" +#include static int dio24_auto_attach(struct comedi_device *dev, unsigned long context) diff --git a/drivers/comedi/drivers/ni_labpc_common.c b/drivers/comedi/drivers/ni_labpc_common.c index 7c4687226450..4a1269aeb371 100644 --- a/drivers/comedi/drivers/ni_labpc_common.c +++ b/drivers/comedi/drivers/ni_labpc_common.c @@ -13,9 +13,9 @@ #include #include #include +#include #include "comedi_8254.h" -#include "8255.h" #include "ni_labpc.h" #include "ni_labpc_regs.h" #include "ni_labpc_isadma.h" diff --git a/drivers/comedi/drivers/ni_mio_common.c b/drivers/comedi/drivers/ni_mio_common.c index 4f80a4991f95..d39998565808 100644 --- a/drivers/comedi/drivers/ni_mio_common.c +++ b/drivers/comedi/drivers/ni_mio_common.c @@ -43,7 +43,7 @@ #include #include #include -#include "8255.h" +#include #include "mite.h" /* A timeout count */ diff --git a/drivers/comedi/drivers/ni_mio_cs.c b/drivers/comedi/drivers/ni_mio_cs.c index bd967cdb2036..796f0b743772 100644 --- a/drivers/comedi/drivers/ni_mio_cs.c +++ b/drivers/comedi/drivers/ni_mio_cs.c @@ -29,9 +29,9 @@ #include #include #include +#include #include "ni_stc.h" -#include "8255.h" /* * AT specific setup diff --git a/drivers/comedi/drivers/pcl724.c b/drivers/comedi/drivers/pcl724.c index b3f472c93e80..948a0576c9ef 100644 --- a/drivers/comedi/drivers/pcl724.c +++ b/drivers/comedi/drivers/pcl724.c @@ -26,8 +26,7 @@ #include #include - -#include "8255.h" +#include struct pcl724_board { const char *name; diff --git a/drivers/comedi/drivers/pcm3724.c b/drivers/comedi/drivers/pcm3724.c index 93ae6cffed44..e4103f9eeced 100644 --- a/drivers/comedi/drivers/pcm3724.c +++ b/drivers/comedi/drivers/pcm3724.c @@ -25,8 +25,7 @@ #include #include - -#include "8255.h" +#include /* * Register I/O Map diff --git a/include/linux/comedi/comedi_8255.h b/include/linux/comedi/comedi_8255.h new file mode 100644 index 000000000000..b2a5bc6b3a49 --- /dev/null +++ b/include/linux/comedi/comedi_8255.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedi_8255.h + * Generic 8255 digital I/O subdevice support + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 1998 David A. Schleef + */ + +#ifndef _COMEDI_8255_H +#define _COMEDI_8255_H + +#define I8255_SIZE 0x04 + +#define I8255_DATA_A_REG 0x00 +#define I8255_DATA_B_REG 0x01 +#define I8255_DATA_C_REG 0x02 +#define I8255_CTRL_REG 0x03 +#define I8255_CTRL_C_LO_IO BIT(0) +#define I8255_CTRL_B_IO BIT(1) +#define I8255_CTRL_B_MODE BIT(2) +#define I8255_CTRL_C_HI_IO BIT(3) +#define I8255_CTRL_A_IO BIT(4) +#define I8255_CTRL_A_MODE(x) ((x) << 5) +#define I8255_CTRL_CW BIT(7) + +struct comedi_device; +struct comedi_subdevice; + +int subdev_8255_init(struct comedi_device *dev, struct comedi_subdevice *s, + int (*io)(struct comedi_device *dev, int dir, int port, + int data, unsigned long regbase), + unsigned long regbase); + +int subdev_8255_mm_init(struct comedi_device *dev, struct comedi_subdevice *s, + int (*io)(struct comedi_device *dev, int dir, int port, + int data, unsigned long regbase), + unsigned long regbase); + +unsigned long subdev_8255_regbase(struct comedi_subdevice *s); + +#endif -- cgit v1.2.3-59-g8ed1b From 44fb7affcfa4e968e9c2ede023ef0e15f06d8209 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Wed, 17 Nov 2021 12:06:02 +0000 Subject: comedi: Move "comedi_8254.h" to Some of the header files in "drivers/comedi/drivers/" are common enough to be useful to out-of-tree comedi driver modules. Using them for out-of-tree module builds is hampered by the headers being outside the "include/" directory so it is desirable to move them. There are about a couple of dozen or so Comedi device drivers that use the "comedi_8254" module to add timers based on the venerable 8254 Programmable Interval Timer chip. The macros and declarations to use that module are in the "comedi_8254.h" header file in the comedi "drivers" directory. Move it into "include/linux/comedi/". Signed-off-by: Ian Abbott Link: https://lore.kernel.org/r/20211117120604.117740-5-abbotti@mev.co.uk Signed-off-by: Greg Kroah-Hartman --- drivers/comedi/drivers/adl_pci9111.c | 2 +- drivers/comedi/drivers/adl_pci9118.c | 2 +- drivers/comedi/drivers/adv_pci1710.c | 2 +- drivers/comedi/drivers/adv_pci_dio.c | 3 +- drivers/comedi/drivers/aio_aio12_8.c | 3 +- drivers/comedi/drivers/amplc_dio200_common.c | 2 +- drivers/comedi/drivers/amplc_pci224.c | 3 +- drivers/comedi/drivers/amplc_pci230.c | 3 +- drivers/comedi/drivers/cb_das16_cs.c | 3 +- drivers/comedi/drivers/cb_pcidas.c | 2 +- drivers/comedi/drivers/cb_pcimdas.c | 2 +- drivers/comedi/drivers/comedi_8254.c | 3 +- drivers/comedi/drivers/comedi_8254.h | 134 --------------------------- drivers/comedi/drivers/das08.c | 2 +- drivers/comedi/drivers/das16.c | 2 +- drivers/comedi/drivers/das16m1.c | 3 +- drivers/comedi/drivers/das1800.c | 2 +- drivers/comedi/drivers/das6402.c | 3 +- drivers/comedi/drivers/das800.c | 3 +- drivers/comedi/drivers/me4000.c | 2 +- drivers/comedi/drivers/ni_at_a2150.c | 2 +- drivers/comedi/drivers/ni_at_ao.c | 3 +- drivers/comedi/drivers/ni_labpc_common.c | 2 +- drivers/comedi/drivers/pcl711.c | 3 +- drivers/comedi/drivers/pcl812.c | 2 +- drivers/comedi/drivers/pcl816.c | 2 +- drivers/comedi/drivers/pcl818.c | 2 +- drivers/comedi/drivers/rtd520.c | 2 +- include/linux/comedi/comedi_8254.h | 134 +++++++++++++++++++++++++++ 29 files changed, 161 insertions(+), 172 deletions(-) delete mode 100644 drivers/comedi/drivers/comedi_8254.h create mode 100644 include/linux/comedi/comedi_8254.h (limited to 'include/linux') diff --git a/drivers/comedi/drivers/adl_pci9111.c b/drivers/comedi/drivers/adl_pci9111.c index 65454f3ecc91..c50f94272a74 100644 --- a/drivers/comedi/drivers/adl_pci9111.c +++ b/drivers/comedi/drivers/adl_pci9111.c @@ -43,9 +43,9 @@ #include #include #include +#include #include "plx9052.h" -#include "comedi_8254.h" #define PCI9111_FIFO_HALF_SIZE 512 diff --git a/drivers/comedi/drivers/adl_pci9118.c b/drivers/comedi/drivers/adl_pci9118.c index 248cec3d894f..9a816c718303 100644 --- a/drivers/comedi/drivers/adl_pci9118.c +++ b/drivers/comedi/drivers/adl_pci9118.c @@ -79,9 +79,9 @@ #include #include #include +#include #include "amcc_s5933.h" -#include "comedi_8254.h" /* * PCI BAR2 Register map (dev->iobase) diff --git a/drivers/comedi/drivers/adv_pci1710.c b/drivers/comedi/drivers/adv_pci1710.c index 47a800d72e58..4f2639968260 100644 --- a/drivers/comedi/drivers/adv_pci1710.c +++ b/drivers/comedi/drivers/adv_pci1710.c @@ -31,8 +31,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "amcc_s5933.h" /* diff --git a/drivers/comedi/drivers/adv_pci_dio.c b/drivers/comedi/drivers/adv_pci_dio.c index 1ec602f8c6e1..efa3e46b554b 100644 --- a/drivers/comedi/drivers/adv_pci_dio.c +++ b/drivers/comedi/drivers/adv_pci_dio.c @@ -25,8 +25,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * Register offset definitions diff --git a/drivers/comedi/drivers/aio_aio12_8.c b/drivers/comedi/drivers/aio_aio12_8.c index cd797dc0f828..30b8a32204d8 100644 --- a/drivers/comedi/drivers/aio_aio12_8.c +++ b/drivers/comedi/drivers/aio_aio12_8.c @@ -24,8 +24,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/amplc_dio200_common.c b/drivers/comedi/drivers/amplc_dio200_common.c index 26b4049b366c..ff651f2eb86c 100644 --- a/drivers/comedi/drivers/amplc_dio200_common.c +++ b/drivers/comedi/drivers/amplc_dio200_common.c @@ -14,9 +14,9 @@ #include #include #include /* only for register defines */ +#include #include "amplc_dio200.h" -#include "comedi_8254.h" /* 200 series registers */ #define DIO200_IO_SIZE 0x20 diff --git a/drivers/comedi/drivers/amplc_pci224.c b/drivers/comedi/drivers/amplc_pci224.c index 3cf1b7fa565d..5a04e55daeea 100644 --- a/drivers/comedi/drivers/amplc_pci224.c +++ b/drivers/comedi/drivers/amplc_pci224.c @@ -97,8 +97,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * PCI224/234 i/o space 1 (PCIBAR2) registers. diff --git a/drivers/comedi/drivers/amplc_pci230.c b/drivers/comedi/drivers/amplc_pci230.c index 93f7057d5b3f..92ba8b8c0172 100644 --- a/drivers/comedi/drivers/amplc_pci230.c +++ b/drivers/comedi/drivers/amplc_pci230.c @@ -176,8 +176,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * PCI230 PCI configuration register information diff --git a/drivers/comedi/drivers/cb_das16_cs.c b/drivers/comedi/drivers/cb_das16_cs.c index 190d73a7d12c..8e0d2fa5f95d 100644 --- a/drivers/comedi/drivers/cb_das16_cs.c +++ b/drivers/comedi/drivers/cb_das16_cs.c @@ -28,8 +28,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/cb_pcidas.c b/drivers/comedi/drivers/cb_pcidas.c index 75ff02b47959..0c7576b967fc 100644 --- a/drivers/comedi/drivers/cb_pcidas.c +++ b/drivers/comedi/drivers/cb_pcidas.c @@ -56,8 +56,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "amcc_s5933.h" #define AI_BUFFER_SIZE 1024 /* max ai fifo size */ diff --git a/drivers/comedi/drivers/cb_pcimdas.c b/drivers/comedi/drivers/cb_pcimdas.c index 7bc0805c69e2..8bdb00774f11 100644 --- a/drivers/comedi/drivers/cb_pcimdas.c +++ b/drivers/comedi/drivers/cb_pcimdas.c @@ -36,8 +36,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "plx9052.h" /* diff --git a/drivers/comedi/drivers/comedi_8254.c b/drivers/comedi/drivers/comedi_8254.c index fac81567133d..b4185c1b2695 100644 --- a/drivers/comedi/drivers/comedi_8254.c +++ b/drivers/comedi/drivers/comedi_8254.c @@ -117,8 +117,7 @@ #include #include #include - -#include "comedi_8254.h" +#include static unsigned int __i8254_read(struct comedi_8254 *i8254, unsigned int reg) { diff --git a/drivers/comedi/drivers/comedi_8254.h b/drivers/comedi/drivers/comedi_8254.h deleted file mode 100644 index d8264417e53c..000000000000 --- a/drivers/comedi/drivers/comedi_8254.h +++ /dev/null @@ -1,134 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * comedi_8254.h - * Generic 8254 timer/counter support - * Copyright (C) 2014 H Hartley Sweeten - * - * COMEDI - Linux Control and Measurement Device Interface - * Copyright (C) 2000 David A. Schleef - */ - -#ifndef _COMEDI_8254_H -#define _COMEDI_8254_H - -#include - -struct comedi_device; -struct comedi_insn; -struct comedi_subdevice; - -/* - * Common oscillator base values in nanoseconds - */ -#define I8254_OSC_BASE_10MHZ 100 -#define I8254_OSC_BASE_5MHZ 200 -#define I8254_OSC_BASE_4MHZ 250 -#define I8254_OSC_BASE_2MHZ 500 -#define I8254_OSC_BASE_1MHZ 1000 -#define I8254_OSC_BASE_100KHZ 10000 -#define I8254_OSC_BASE_10KHZ 100000 -#define I8254_OSC_BASE_1KHZ 1000000 - -/* - * I/O access size used to read/write registers - */ -#define I8254_IO8 1 -#define I8254_IO16 2 -#define I8254_IO32 4 - -/* - * Register map for generic 8254 timer (I8254_IO8 with 0 regshift) - */ -#define I8254_COUNTER0_REG 0x00 -#define I8254_COUNTER1_REG 0x01 -#define I8254_COUNTER2_REG 0x02 -#define I8254_CTRL_REG 0x03 -#define I8254_CTRL_SEL_CTR(x) ((x) << 6) -#define I8254_CTRL_READBACK(x) (I8254_CTRL_SEL_CTR(3) | BIT(x)) -#define I8254_CTRL_READBACK_COUNT I8254_CTRL_READBACK(4) -#define I8254_CTRL_READBACK_STATUS I8254_CTRL_READBACK(5) -#define I8254_CTRL_READBACK_SEL_CTR(x) (2 << (x)) -#define I8254_CTRL_RW(x) (((x) & 0x3) << 4) -#define I8254_CTRL_LATCH I8254_CTRL_RW(0) -#define I8254_CTRL_LSB_ONLY I8254_CTRL_RW(1) -#define I8254_CTRL_MSB_ONLY I8254_CTRL_RW(2) -#define I8254_CTRL_LSB_MSB I8254_CTRL_RW(3) - -/* counter maps zero to 0x10000 */ -#define I8254_MAX_COUNT 0x10000 - -/** - * struct comedi_8254 - private data used by this module - * @iobase: PIO base address of the registers (in/out) - * @mmio: MMIO base address of the registers (read/write) - * @iosize: I/O size used to access the registers (b/w/l) - * @regshift: register gap shift - * @osc_base: cascaded oscillator speed in ns - * @divisor: divisor for single counter - * @divisor1: divisor loaded into first cascaded counter - * @divisor2: divisor loaded into second cascaded counter - * #next_div: next divisor for single counter - * @next_div1: next divisor to use for first cascaded counter - * @next_div2: next divisor to use for second cascaded counter - * @clock_src; current clock source for each counter (driver specific) - * @gate_src; current gate source for each counter (driver specific) - * @busy: flags used to indicate that a counter is "busy" - * @insn_config: driver specific (*insn_config) callback - */ -struct comedi_8254 { - unsigned long iobase; - void __iomem *mmio; - unsigned int iosize; - unsigned int regshift; - unsigned int osc_base; - unsigned int divisor; - unsigned int divisor1; - unsigned int divisor2; - unsigned int next_div; - unsigned int next_div1; - unsigned int next_div2; - unsigned int clock_src[3]; - unsigned int gate_src[3]; - bool busy[3]; - - int (*insn_config)(struct comedi_device *dev, - struct comedi_subdevice *s, - struct comedi_insn *insn, unsigned int *data); -}; - -unsigned int comedi_8254_status(struct comedi_8254 *i8254, - unsigned int counter); -unsigned int comedi_8254_read(struct comedi_8254 *i8254, unsigned int counter); -void comedi_8254_write(struct comedi_8254 *i8254, - unsigned int counter, unsigned int val); - -int comedi_8254_set_mode(struct comedi_8254 *i8254, - unsigned int counter, unsigned int mode); -int comedi_8254_load(struct comedi_8254 *i8254, - unsigned int counter, unsigned int val, unsigned int mode); - -void comedi_8254_pacer_enable(struct comedi_8254 *i8254, - unsigned int counter1, unsigned int counter2, - bool enable); -void comedi_8254_update_divisors(struct comedi_8254 *i8254); -void comedi_8254_cascade_ns_to_timer(struct comedi_8254 *i8254, - unsigned int *nanosec, unsigned int flags); -void comedi_8254_ns_to_timer(struct comedi_8254 *i8254, - unsigned int *nanosec, unsigned int flags); - -void comedi_8254_set_busy(struct comedi_8254 *i8254, - unsigned int counter, bool busy); - -void comedi_8254_subdevice_init(struct comedi_subdevice *s, - struct comedi_8254 *i8254); - -struct comedi_8254 *comedi_8254_init(unsigned long iobase, - unsigned int osc_base, - unsigned int iosize, - unsigned int regshift); -struct comedi_8254 *comedi_8254_mm_init(void __iomem *mmio, - unsigned int osc_base, - unsigned int iosize, - unsigned int regshift); - -#endif /* _COMEDI_8254_H */ diff --git a/drivers/comedi/drivers/das08.c b/drivers/comedi/drivers/das08.c index bab868de2967..f8ab3af2e391 100644 --- a/drivers/comedi/drivers/das08.c +++ b/drivers/comedi/drivers/das08.c @@ -12,8 +12,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "das08.h" /* diff --git a/drivers/comedi/drivers/das16.c b/drivers/comedi/drivers/das16.c index 338396736936..f6649ffa9670 100644 --- a/drivers/comedi/drivers/das16.c +++ b/drivers/comedi/drivers/das16.c @@ -65,9 +65,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" #define DAS16_DMA_SIZE 0xff00 /* size in bytes of allocated dma buffer */ diff --git a/drivers/comedi/drivers/das16m1.c b/drivers/comedi/drivers/das16m1.c index ea55024d8c5a..275effb77746 100644 --- a/drivers/comedi/drivers/das16m1.c +++ b/drivers/comedi/drivers/das16m1.c @@ -44,8 +44,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * Register map (dev->iobase) diff --git a/drivers/comedi/drivers/das1800.c b/drivers/comedi/drivers/das1800.c index 768803742350..a43d3414a122 100644 --- a/drivers/comedi/drivers/das1800.c +++ b/drivers/comedi/drivers/das1800.c @@ -74,9 +74,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" /* misc. defines */ #define DAS1800_SIZE 16 /* uses 16 io addresses */ diff --git a/drivers/comedi/drivers/das6402.c b/drivers/comedi/drivers/das6402.c index d411ab7cf37c..1af394591e74 100644 --- a/drivers/comedi/drivers/das6402.c +++ b/drivers/comedi/drivers/das6402.c @@ -25,8 +25,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/das800.c b/drivers/comedi/drivers/das800.c index c95e0fcb94a4..4ca33f46eaa7 100644 --- a/drivers/comedi/drivers/das800.c +++ b/drivers/comedi/drivers/das800.c @@ -47,8 +47,7 @@ #include #include #include - -#include "comedi_8254.h" +#include #define N_CHAN_AI 8 /* number of analog input channels */ diff --git a/drivers/comedi/drivers/me4000.c b/drivers/comedi/drivers/me4000.c index c5dc8199771f..9aea02b86ed9 100644 --- a/drivers/comedi/drivers/me4000.c +++ b/drivers/comedi/drivers/me4000.c @@ -33,8 +33,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "plx9052.h" #define ME4000_FIRMWARE "me4000_firmware.bin" diff --git a/drivers/comedi/drivers/ni_at_a2150.c b/drivers/comedi/drivers/ni_at_a2150.c index ce5de58c499f..9942d770add8 100644 --- a/drivers/comedi/drivers/ni_at_a2150.c +++ b/drivers/comedi/drivers/ni_at_a2150.c @@ -40,9 +40,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" #define A2150_DMA_BUFFER_SIZE 0xff00 /* size in bytes of dma buffer */ diff --git a/drivers/comedi/drivers/ni_at_ao.c b/drivers/comedi/drivers/ni_at_ao.c index a06dfb9da329..9f3147b72aa8 100644 --- a/drivers/comedi/drivers/ni_at_ao.c +++ b/drivers/comedi/drivers/ni_at_ao.c @@ -26,8 +26,7 @@ #include #include - -#include "comedi_8254.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/ni_labpc_common.c b/drivers/comedi/drivers/ni_labpc_common.c index 4a1269aeb371..763249653228 100644 --- a/drivers/comedi/drivers/ni_labpc_common.c +++ b/drivers/comedi/drivers/ni_labpc_common.c @@ -14,8 +14,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "ni_labpc.h" #include "ni_labpc_regs.h" #include "ni_labpc_isadma.h" diff --git a/drivers/comedi/drivers/pcl711.c b/drivers/comedi/drivers/pcl711.c index f1c383bd9d87..05172c553c8a 100644 --- a/drivers/comedi/drivers/pcl711.c +++ b/drivers/comedi/drivers/pcl711.c @@ -30,8 +30,7 @@ #include #include #include - -#include "comedi_8254.h" +#include /* * I/O port register map diff --git a/drivers/comedi/drivers/pcl812.c b/drivers/comedi/drivers/pcl812.c index f00976ddfc2a..790f54476a91 100644 --- a/drivers/comedi/drivers/pcl812.c +++ b/drivers/comedi/drivers/pcl812.c @@ -115,9 +115,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" /* * Register I/O map diff --git a/drivers/comedi/drivers/pcl816.c b/drivers/comedi/drivers/pcl816.c index c5acdc8913f8..77b30246d966 100644 --- a/drivers/comedi/drivers/pcl816.c +++ b/drivers/comedi/drivers/pcl816.c @@ -36,9 +36,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" /* * Register I/O map diff --git a/drivers/comedi/drivers/pcl818.c b/drivers/comedi/drivers/pcl818.c index 20fcd6d588f8..e5b7793cce05 100644 --- a/drivers/comedi/drivers/pcl818.c +++ b/drivers/comedi/drivers/pcl818.c @@ -98,9 +98,9 @@ #include #include #include +#include #include "comedi_isadma.h" -#include "comedi_8254.h" /* * Register I/O map diff --git a/drivers/comedi/drivers/rtd520.c b/drivers/comedi/drivers/rtd520.c index ee5bca2b1c09..7e0ec1a2a2ca 100644 --- a/drivers/comedi/drivers/rtd520.c +++ b/drivers/comedi/drivers/rtd520.c @@ -86,8 +86,8 @@ #include #include #include +#include -#include "comedi_8254.h" #include "plx9080.h" /* diff --git a/include/linux/comedi/comedi_8254.h b/include/linux/comedi/comedi_8254.h new file mode 100644 index 000000000000..d8264417e53c --- /dev/null +++ b/include/linux/comedi/comedi_8254.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * comedi_8254.h + * Generic 8254 timer/counter support + * Copyright (C) 2014 H Hartley Sweeten + * + * COMEDI - Linux Control and Measurement Device Interface + * Copyright (C) 2000 David A. Schleef + */ + +#ifndef _COMEDI_8254_H +#define _COMEDI_8254_H + +#include + +struct comedi_device; +struct comedi_insn; +struct comedi_subdevice; + +/* + * Common oscillator base values in nanoseconds + */ +#define I8254_OSC_BASE_10MHZ 100 +#define I8254_OSC_BASE_5MHZ 200 +#define I8254_OSC_BASE_4MHZ 250 +#define I8254_OSC_BASE_2MHZ 500 +#define I8254_OSC_BASE_1MHZ 1000 +#define I8254_OSC_BASE_100KHZ 10000 +#define I8254_OSC_BASE_10KHZ 100000 +#define I8254_OSC_BASE_1KHZ 1000000 + +/* + * I/O access size used to read/write registers + */ +#define I8254_IO8 1 +#define I8254_IO16 2 +#define I8254_IO32 4 + +/* + * Register map for generic 8254 timer (I8254_IO8 with 0 regshift) + */ +#define I8254_COUNTER0_REG 0x00 +#define I8254_COUNTER1_REG 0x01 +#define I8254_COUNTER2_REG 0x02 +#define I8254_CTRL_REG 0x03 +#define I8254_CTRL_SEL_CTR(x) ((x) << 6) +#define I8254_CTRL_READBACK(x) (I8254_CTRL_SEL_CTR(3) | BIT(x)) +#define I8254_CTRL_READBACK_COUNT I8254_CTRL_READBACK(4) +#define I8254_CTRL_READBACK_STATUS I8254_CTRL_READBACK(5) +#define I8254_CTRL_READBACK_SEL_CTR(x) (2 << (x)) +#define I8254_CTRL_RW(x) (((x) & 0x3) << 4) +#define I8254_CTRL_LATCH I8254_CTRL_RW(0) +#define I8254_CTRL_LSB_ONLY I8254_CTRL_RW(1) +#define I8254_CTRL_MSB_ONLY I8254_CTRL_RW(2) +#define I8254_CTRL_LSB_MSB I8254_CTRL_RW(3) + +/* counter maps zero to 0x10000 */ +#define I8254_MAX_COUNT 0x10000 + +/** + * struct comedi_8254 - private data used by this module + * @iobase: PIO base address of the registers (in/out) + * @mmio: MMIO base address of the registers (read/write) + * @iosize: I/O size used to access the registers (b/w/l) + * @regshift: register gap shift + * @osc_base: cascaded oscillator speed in ns + * @divisor: divisor for single counter + * @divisor1: divisor loaded into first cascaded counter + * @divisor2: divisor loaded into second cascaded counter + * #next_div: next divisor for single counter + * @next_div1: next divisor to use for first cascaded counter + * @next_div2: next divisor to use for second cascaded counter + * @clock_src; current clock source for each counter (driver specific) + * @gate_src; current gate source for each counter (driver specific) + * @busy: flags used to indicate that a counter is "busy" + * @insn_config: driver specific (*insn_config) callback + */ +struct comedi_8254 { + unsigned long iobase; + void __iomem *mmio; + unsigned int iosize; + unsigned int regshift; + unsigned int osc_base; + unsigned int divisor; + unsigned int divisor1; + unsigned int divisor2; + unsigned int next_div; + unsigned int next_div1; + unsigned int next_div2; + unsigned int clock_src[3]; + unsigned int gate_src[3]; + bool busy[3]; + + int (*insn_config)(struct comedi_device *dev, + struct comedi_subdevice *s, + struct comedi_insn *insn, unsigned int *data); +}; + +unsigned int comedi_8254_status(struct comedi_8254 *i8254, + unsigned int counter); +unsigned int comedi_8254_read(struct comedi_8254 *i8254, unsigned int counter); +void comedi_8254_write(struct comedi_8254 *i8254, + unsigned int counter, unsigned int val); + +int comedi_8254_set_mode(struct comedi_8254 *i8254, + unsigned int counter, unsigned int mode); +int comedi_8254_load(struct comedi_8254 *i8254, + unsigned int counter, unsigned int val, unsigned int mode); + +void comedi_8254_pacer_enable(struct comedi_8254 *i8254, + unsigned int counter1, unsigned int counter2, + bool enable); +void comedi_8254_update_divisors(struct comedi_8254 *i8254); +void comedi_8254_cascade_ns_to_timer(struct comedi_8254 *i8254, + unsigned int *nanosec, unsigned int flags); +void comedi_8254_ns_to_timer(struct comedi_8254 *i8254, + unsigned int *nanosec, unsigned int flags); + +void comedi_8254_set_busy(struct comedi_8254 *i8254, + unsigned int counter, bool busy); + +void comedi_8254_subdevice_init(struct comedi_subdevice *s, + struct comedi_8254 *i8254); + +struct comedi_8254 *comedi_8254_init(unsigned long iobase, + unsigned int osc_base, + unsigned int iosize, + unsigned int regshift); +struct comedi_8254 *comedi_8254_mm_init(void __iomem *mmio, + unsigned int osc_base, + unsigned int iosize, + unsigned int regshift); + +#endif /* _COMEDI_8254_H */ -- cgit v1.2.3-59-g8ed1b From fe7a4f5b9548456246ffda143bab59922acda9fd Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Wed, 17 Nov 2021 12:06:03 +0000 Subject: comedi: Move "comedi_isadma.h" to Some of the header files in "drivers/comedi/drivers/" are common enough to be useful to out-of-tree comedi driver modules. Using them for out-of-tree module builds is hampered by the headers being outside the "include/" directory so it is desirable to move them. There are about a half a dozen or so Comedi device drivers that use the "comedi_isadma" module to add ISA DMA support. The macros and declarations to use that module are in the "comedi_isadma.h" header file in the comedi "drivers" directory. Move it into "include/linux/comedi/". Signed-off-by: Ian Abbott Link: https://lore.kernel.org/r/20211117120604.117740-6-abbotti@mev.co.uk Signed-off-by: Greg Kroah-Hartman --- drivers/comedi/drivers/comedi_isadma.c | 3 +- drivers/comedi/drivers/comedi_isadma.h | 114 ------------------------------- drivers/comedi/drivers/das16.c | 3 +- drivers/comedi/drivers/das1800.c | 3 +- drivers/comedi/drivers/dt282x.c | 3 +- drivers/comedi/drivers/ni_at_a2150.c | 3 +- drivers/comedi/drivers/ni_labpc_isadma.c | 2 +- drivers/comedi/drivers/pcl812.c | 3 +- drivers/comedi/drivers/pcl816.c | 3 +- drivers/comedi/drivers/pcl818.c | 3 +- include/linux/comedi/comedi_isadma.h | 114 +++++++++++++++++++++++++++++++ 11 files changed, 123 insertions(+), 131 deletions(-) delete mode 100644 drivers/comedi/drivers/comedi_isadma.h create mode 100644 include/linux/comedi/comedi_isadma.h (limited to 'include/linux') diff --git a/drivers/comedi/drivers/comedi_isadma.c b/drivers/comedi/drivers/comedi_isadma.c index 63457bd4ff78..700982464c53 100644 --- a/drivers/comedi/drivers/comedi_isadma.c +++ b/drivers/comedi/drivers/comedi_isadma.c @@ -10,8 +10,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /** * comedi_isadma_program - program and enable an ISA DMA transfer diff --git a/drivers/comedi/drivers/comedi_isadma.h b/drivers/comedi/drivers/comedi_isadma.h deleted file mode 100644 index 9d2b12db7e6e..000000000000 --- a/drivers/comedi/drivers/comedi_isadma.h +++ /dev/null @@ -1,114 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * COMEDI ISA DMA support functions - * Copyright (c) 2014 H Hartley Sweeten - */ - -#ifndef _COMEDI_ISADMA_H -#define _COMEDI_ISADMA_H - -#include - -struct comedi_device; -struct device; - -/* - * These are used to avoid issues when and the DMA_MODE_ - * defines are not available. - */ -#define COMEDI_ISADMA_READ 0 -#define COMEDI_ISADMA_WRITE 1 - -/** - * struct comedi_isadma_desc - cookie for ISA DMA - * @virt_addr: virtual address of buffer - * @hw_addr: hardware (bus) address of buffer - * @chan: DMA channel - * @maxsize: allocated size of buffer (in bytes) - * @size: transfer size (in bytes) - * @mode: DMA_MODE_READ or DMA_MODE_WRITE - */ -struct comedi_isadma_desc { - void *virt_addr; - dma_addr_t hw_addr; - unsigned int chan; - unsigned int maxsize; - unsigned int size; - char mode; -}; - -/** - * struct comedi_isadma - ISA DMA data - * @dev: device to allocate non-coherent memory for - * @desc: cookie for each DMA buffer - * @n_desc: the number of cookies - * @cur_dma: the current cookie in use - * @chan: the first DMA channel requested - * @chan2: the second DMA channel requested - */ -struct comedi_isadma { - struct device *dev; - struct comedi_isadma_desc *desc; - int n_desc; - int cur_dma; - unsigned int chan; - unsigned int chan2; -}; - -#if IS_ENABLED(CONFIG_ISA_DMA_API) - -void comedi_isadma_program(struct comedi_isadma_desc *desc); -unsigned int comedi_isadma_disable(unsigned int dma_chan); -unsigned int comedi_isadma_disable_on_sample(unsigned int dma_chan, - unsigned int size); -unsigned int comedi_isadma_poll(struct comedi_isadma *dma); -void comedi_isadma_set_mode(struct comedi_isadma_desc *desc, char dma_dir); - -struct comedi_isadma *comedi_isadma_alloc(struct comedi_device *dev, - int n_desc, unsigned int dma_chan1, - unsigned int dma_chan2, - unsigned int maxsize, char dma_dir); -void comedi_isadma_free(struct comedi_isadma *dma); - -#else /* !IS_ENABLED(CONFIG_ISA_DMA_API) */ - -static inline void comedi_isadma_program(struct comedi_isadma_desc *desc) -{ -} - -static inline unsigned int comedi_isadma_disable(unsigned int dma_chan) -{ - return 0; -} - -static inline unsigned int -comedi_isadma_disable_on_sample(unsigned int dma_chan, unsigned int size) -{ - return 0; -} - -static inline unsigned int comedi_isadma_poll(struct comedi_isadma *dma) -{ - return 0; -} - -static inline void comedi_isadma_set_mode(struct comedi_isadma_desc *desc, - char dma_dir) -{ -} - -static inline struct comedi_isadma * -comedi_isadma_alloc(struct comedi_device *dev, int n_desc, - unsigned int dma_chan1, unsigned int dma_chan2, - unsigned int maxsize, char dma_dir) -{ - return NULL; -} - -static inline void comedi_isadma_free(struct comedi_isadma *dma) -{ -} - -#endif /* !IS_ENABLED(CONFIG_ISA_DMA_API) */ - -#endif /* #ifndef _COMEDI_ISADMA_H */ diff --git a/drivers/comedi/drivers/das16.c b/drivers/comedi/drivers/das16.c index f6649ffa9670..937a69ce0977 100644 --- a/drivers/comedi/drivers/das16.c +++ b/drivers/comedi/drivers/das16.c @@ -66,8 +66,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include #define DAS16_DMA_SIZE 0xff00 /* size in bytes of allocated dma buffer */ diff --git a/drivers/comedi/drivers/das1800.c b/drivers/comedi/drivers/das1800.c index a43d3414a122..f09608c0f4ff 100644 --- a/drivers/comedi/drivers/das1800.c +++ b/drivers/comedi/drivers/das1800.c @@ -75,8 +75,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /* misc. defines */ #define DAS1800_SIZE 16 /* uses 16 io addresses */ diff --git a/drivers/comedi/drivers/dt282x.c b/drivers/comedi/drivers/dt282x.c index 078f8fba7183..4ae80e6c7266 100644 --- a/drivers/comedi/drivers/dt282x.c +++ b/drivers/comedi/drivers/dt282x.c @@ -52,8 +52,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /* * Register map diff --git a/drivers/comedi/drivers/ni_at_a2150.c b/drivers/comedi/drivers/ni_at_a2150.c index 9942d770add8..df8d219e6723 100644 --- a/drivers/comedi/drivers/ni_at_a2150.c +++ b/drivers/comedi/drivers/ni_at_a2150.c @@ -41,8 +41,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include #define A2150_DMA_BUFFER_SIZE 0xff00 /* size in bytes of dma buffer */ diff --git a/drivers/comedi/drivers/ni_labpc_isadma.c b/drivers/comedi/drivers/ni_labpc_isadma.c index dd37ec0d9b15..0652ca8345b6 100644 --- a/drivers/comedi/drivers/ni_labpc_isadma.c +++ b/drivers/comedi/drivers/ni_labpc_isadma.c @@ -11,8 +11,8 @@ #include #include #include +#include -#include "comedi_isadma.h" #include "ni_labpc.h" #include "ni_labpc_regs.h" #include "ni_labpc_isadma.h" diff --git a/drivers/comedi/drivers/pcl812.c b/drivers/comedi/drivers/pcl812.c index 790f54476a91..70dbc129fcf5 100644 --- a/drivers/comedi/drivers/pcl812.c +++ b/drivers/comedi/drivers/pcl812.c @@ -116,8 +116,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/pcl816.c b/drivers/comedi/drivers/pcl816.c index 77b30246d966..a5e5320be648 100644 --- a/drivers/comedi/drivers/pcl816.c +++ b/drivers/comedi/drivers/pcl816.c @@ -37,8 +37,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /* * Register I/O map diff --git a/drivers/comedi/drivers/pcl818.c b/drivers/comedi/drivers/pcl818.c index e5b7793cce05..29e503de8267 100644 --- a/drivers/comedi/drivers/pcl818.c +++ b/drivers/comedi/drivers/pcl818.c @@ -99,8 +99,7 @@ #include #include #include - -#include "comedi_isadma.h" +#include /* * Register I/O map diff --git a/include/linux/comedi/comedi_isadma.h b/include/linux/comedi/comedi_isadma.h new file mode 100644 index 000000000000..9d2b12db7e6e --- /dev/null +++ b/include/linux/comedi/comedi_isadma.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * COMEDI ISA DMA support functions + * Copyright (c) 2014 H Hartley Sweeten + */ + +#ifndef _COMEDI_ISADMA_H +#define _COMEDI_ISADMA_H + +#include + +struct comedi_device; +struct device; + +/* + * These are used to avoid issues when and the DMA_MODE_ + * defines are not available. + */ +#define COMEDI_ISADMA_READ 0 +#define COMEDI_ISADMA_WRITE 1 + +/** + * struct comedi_isadma_desc - cookie for ISA DMA + * @virt_addr: virtual address of buffer + * @hw_addr: hardware (bus) address of buffer + * @chan: DMA channel + * @maxsize: allocated size of buffer (in bytes) + * @size: transfer size (in bytes) + * @mode: DMA_MODE_READ or DMA_MODE_WRITE + */ +struct comedi_isadma_desc { + void *virt_addr; + dma_addr_t hw_addr; + unsigned int chan; + unsigned int maxsize; + unsigned int size; + char mode; +}; + +/** + * struct comedi_isadma - ISA DMA data + * @dev: device to allocate non-coherent memory for + * @desc: cookie for each DMA buffer + * @n_desc: the number of cookies + * @cur_dma: the current cookie in use + * @chan: the first DMA channel requested + * @chan2: the second DMA channel requested + */ +struct comedi_isadma { + struct device *dev; + struct comedi_isadma_desc *desc; + int n_desc; + int cur_dma; + unsigned int chan; + unsigned int chan2; +}; + +#if IS_ENABLED(CONFIG_ISA_DMA_API) + +void comedi_isadma_program(struct comedi_isadma_desc *desc); +unsigned int comedi_isadma_disable(unsigned int dma_chan); +unsigned int comedi_isadma_disable_on_sample(unsigned int dma_chan, + unsigned int size); +unsigned int comedi_isadma_poll(struct comedi_isadma *dma); +void comedi_isadma_set_mode(struct comedi_isadma_desc *desc, char dma_dir); + +struct comedi_isadma *comedi_isadma_alloc(struct comedi_device *dev, + int n_desc, unsigned int dma_chan1, + unsigned int dma_chan2, + unsigned int maxsize, char dma_dir); +void comedi_isadma_free(struct comedi_isadma *dma); + +#else /* !IS_ENABLED(CONFIG_ISA_DMA_API) */ + +static inline void comedi_isadma_program(struct comedi_isadma_desc *desc) +{ +} + +static inline unsigned int comedi_isadma_disable(unsigned int dma_chan) +{ + return 0; +} + +static inline unsigned int +comedi_isadma_disable_on_sample(unsigned int dma_chan, unsigned int size) +{ + return 0; +} + +static inline unsigned int comedi_isadma_poll(struct comedi_isadma *dma) +{ + return 0; +} + +static inline void comedi_isadma_set_mode(struct comedi_isadma_desc *desc, + char dma_dir) +{ +} + +static inline struct comedi_isadma * +comedi_isadma_alloc(struct comedi_device *dev, int n_desc, + unsigned int dma_chan1, unsigned int dma_chan2, + unsigned int maxsize, char dma_dir) +{ + return NULL; +} + +static inline void comedi_isadma_free(struct comedi_isadma *dma) +{ +} + +#endif /* !IS_ENABLED(CONFIG_ISA_DMA_API) */ + +#endif /* #ifndef _COMEDI_ISADMA_H */ -- cgit v1.2.3-59-g8ed1b From 2cca3465147d650be3de04927a99784b30251ade Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 12 Nov 2021 08:28:09 +0200 Subject: mei: bus: add client dma interface Expose the client dma mapping via mei client bus interface. The client dma has to be mapped before the device is enabled, therefore we need to create device linking already during mapping and we need to unmap after the client is disable hence we need to postpone the unlink and flush till unmapping or when destroying the device. Signed-off-by: Alexander Usyskin Co-developed-by: Tomas Winkler Signed-off-by: Tomas Winkler Signed-off-by: Emmanuel Grumbach Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20210420172755.12178-1-emmanuel.grumbach@intel.com Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20211112062814.7502-1-emmanuel.grumbach@intel.com --- drivers/misc/mei/bus.c | 67 +++++++++++++++++++++++++++++++++++++++++++--- drivers/misc/mei/client.c | 3 +++ drivers/misc/mei/hw.h | 5 ++++ include/linux/mei_cl_bus.h | 3 +++ 4 files changed, 75 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 44bac4ad687c..46aa3554e97b 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -643,6 +643,64 @@ static void mei_cl_bus_vtag_free(struct mei_cl_device *cldev) kfree(cl_vtag); } +void *mei_cldev_dma_map(struct mei_cl_device *cldev, u8 buffer_id, size_t size) +{ + struct mei_device *bus; + struct mei_cl *cl; + int ret; + + if (!cldev || !buffer_id || !size) + return ERR_PTR(-EINVAL); + + if (!IS_ALIGNED(size, MEI_FW_PAGE_SIZE)) { + dev_err(&cldev->dev, "Map size should be aligned to %lu\n", + MEI_FW_PAGE_SIZE); + return ERR_PTR(-EINVAL); + } + + cl = cldev->cl; + bus = cldev->bus; + + mutex_lock(&bus->device_lock); + if (cl->state == MEI_FILE_UNINITIALIZED) { + ret = mei_cl_link(cl); + if (ret) + goto out; + /* update pointers */ + cl->cldev = cldev; + } + + ret = mei_cl_dma_alloc_and_map(cl, NULL, buffer_id, size); +out: + mutex_unlock(&bus->device_lock); + if (ret) + return ERR_PTR(ret); + return cl->dma.vaddr; +} +EXPORT_SYMBOL_GPL(mei_cldev_dma_map); + +int mei_cldev_dma_unmap(struct mei_cl_device *cldev) +{ + struct mei_device *bus; + struct mei_cl *cl; + int ret; + + if (!cldev) + return -EINVAL; + + cl = cldev->cl; + bus = cldev->bus; + + mutex_lock(&bus->device_lock); + ret = mei_cl_dma_unmap(cl, NULL); + + mei_cl_flush_queues(cl, NULL); + mei_cl_unlink(cl); + mutex_unlock(&bus->device_lock); + return ret; +} +EXPORT_SYMBOL_GPL(mei_cldev_dma_unmap); + /** * mei_cldev_enable - enable me client device * create connection with me client @@ -753,9 +811,11 @@ int mei_cldev_disable(struct mei_cl_device *cldev) dev_err(bus->dev, "Could not disconnect from the ME client\n"); out: - /* Flush queues and remove any pending read */ - mei_cl_flush_queues(cl, NULL); - mei_cl_unlink(cl); + /* Flush queues and remove any pending read unless we have mapped DMA */ + if (!cl->dma_mapped) { + mei_cl_flush_queues(cl, NULL); + mei_cl_unlink(cl); + } mutex_unlock(&bus->device_lock); return err; @@ -1052,6 +1112,7 @@ static void mei_cl_bus_dev_release(struct device *dev) if (!cldev) return; + mei_cl_flush_queues(cldev->cl, NULL); mei_me_cl_put(cldev->me_cl); mei_dev_bus_put(cldev->bus); mei_cl_unlink(cldev->cl); diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 96f4e59c32a5..0e90591235a6 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -700,6 +700,9 @@ int mei_cl_unlink(struct mei_cl *cl) cl_dbg(dev, cl, "unlink client"); + if (cl->state == MEI_FILE_UNINITIALIZED) + return 0; + if (dev->open_handle_count > 0) dev->open_handle_count--; diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index dfd60c916da0..b46077b17114 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -22,6 +22,11 @@ #define MEI_D0I3_TIMEOUT 5 /* D0i3 set/unset max response time */ #define MEI_HBM_TIMEOUT 1 /* 1 second */ +/* + * FW page size for DMA allocations + */ +#define MEI_FW_PAGE_SIZE 4096UL + /* * MEI Version */ diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index c6786c12b207..df1fab44ea5c 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -117,4 +117,7 @@ int mei_cldev_enable(struct mei_cl_device *cldev); int mei_cldev_disable(struct mei_cl_device *cldev); bool mei_cldev_enabled(const struct mei_cl_device *cldev); +void *mei_cldev_dma_map(struct mei_cl_device *cldev, u8 buffer_id, size_t size); +int mei_cldev_dma_unmap(struct mei_cl_device *cldev); + #endif /* _LINUX_MEI_CL_BUS_H */ -- cgit v1.2.3-59-g8ed1b From a6914afcdf0e3fb853fce0e0c04710be7427b62f Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 10 Nov 2021 12:31:28 +0200 Subject: kobject: Replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20211110103128.59888-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/kobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/kobject.h b/include/linux/kobject.h index efd56f990a46..c740062b4b1a 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -19,10 +19,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include -- cgit v1.2.3-59-g8ed1b From 6a2d2ddf2c345e0149bfbffdddc4768a9ab0a741 Mon Sep 17 00:00:00 2001 From: Javier Martinez Canillas Date: Fri, 12 Nov 2021 14:32:27 +0100 Subject: drm: Move nomodeset kernel parameter to the DRM subsystem The "nomodeset" kernel cmdline parameter is handled by the vgacon driver but the exported vgacon_text_force() symbol is only used by DRM drivers. It makes much more sense for the parameter logic to be in the subsystem of the drivers that are making use of it. Let's move the vgacon_text_force() function and related logic to the DRM subsystem. While doing that, rename it to drm_firmware_drivers_only() and make it return true if "nomodeset" was used and false otherwise. This is a better description of the condition that the drivers are testing for. Suggested-by: Daniel Vetter Signed-off-by: Javier Martinez Canillas Acked-by: Thomas Zimmermann Acked-by: Jani Nikula Acked-by: Pekka Paalanen Acked-by: Greg Kroah-Hartman Link: https://patchwork.freedesktop.org/patch/msgid/20211112133230.1595307-4-javierm@redhat.com --- drivers/gpu/drm/Makefile | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 +-- drivers/gpu/drm/ast/ast_drv.c | 3 +-- drivers/gpu/drm/i915/i915_module.c | 4 ++-- drivers/gpu/drm/mgag200/mgag200_drv.c | 3 +-- drivers/gpu/drm/nouveau/nouveau_drm.c | 4 ++-- drivers/gpu/drm/qxl/qxl_drv.c | 3 +-- drivers/gpu/drm/radeon/radeon_drv.c | 3 +-- drivers/gpu/drm/tiny/bochs.c | 3 +-- drivers/gpu/drm/tiny/cirrus.c | 4 ++-- drivers/gpu/drm/vboxvideo/vbox_drv.c | 3 +-- drivers/gpu/drm/virtio/virtgpu_drv.c | 3 +-- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 3 +-- drivers/video/console/vgacon.c | 21 --------------------- gpu/drm/drm_nomodeset.c | 26 ++++++++++++++++++++++++++ include/drm/drm_drv.h | 5 +++++ include/linux/console.h | 6 ------ 17 files changed, 48 insertions(+), 51 deletions(-) create mode 100644 gpu/drm/drm_nomodeset.c (limited to 'include/linux') diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 1c41156deb5f..c74810c285af 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -33,6 +33,8 @@ drm-$(CONFIG_DRM_PRIVACY_SCREEN) += drm_privacy_screen.o drm_privacy_screen_x86. obj-$(CONFIG_DRM_DP_AUX_BUS) += drm_dp_aux_bus.o +obj-$(CONFIG_VGA_CONSOLE) += drm_nomodeset.o + drm_cma_helper-y := drm_gem_cma_helper.o obj-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_cma_helper.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index e8c23e37c83c..438468b82eb6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -31,7 +31,6 @@ #include "amdgpu_drv.h" #include -#include #include #include #include @@ -2516,7 +2515,7 @@ static int __init amdgpu_init(void) { int r; - if (vgacon_text_force()) + if (drm_firmware_drivers_only()) return -EINVAL; r = amdgpu_sync_init(); diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c index 86d5cd7b6318..6d8613f6fe1c 100644 --- a/drivers/gpu/drm/ast/ast_drv.c +++ b/drivers/gpu/drm/ast/ast_drv.c @@ -26,7 +26,6 @@ * Authors: Dave Airlie */ -#include #include #include @@ -233,7 +232,7 @@ static struct pci_driver ast_pci_driver = { static int __init ast_init(void) { - if (vgacon_text_force() && ast_modeset == -1) + if (drm_firmware_drivers_only() && ast_modeset == -1) return -EINVAL; if (ast_modeset == 0) diff --git a/drivers/gpu/drm/i915/i915_module.c b/drivers/gpu/drm/i915/i915_module.c index ab2295dd4500..1df66bf276ce 100644 --- a/drivers/gpu/drm/i915/i915_module.c +++ b/drivers/gpu/drm/i915/i915_module.c @@ -4,7 +4,7 @@ * Copyright © 2021 Intel Corporation */ -#include +#include #include "gem/i915_gem_context.h" #include "gem/i915_gem_object.h" @@ -31,7 +31,7 @@ static int i915_check_nomodeset(void) if (i915_modparams.modeset == 0) use_kms = false; - if (vgacon_text_force() && i915_modparams.modeset == -1) + if (drm_firmware_drivers_only() && i915_modparams.modeset == -1) use_kms = false; if (!use_kms) { diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c index 6b9243713b3c..740108a006ba 100644 --- a/drivers/gpu/drm/mgag200/mgag200_drv.c +++ b/drivers/gpu/drm/mgag200/mgag200_drv.c @@ -6,7 +6,6 @@ * Dave Airlie */ -#include #include #include #include @@ -378,7 +377,7 @@ static struct pci_driver mgag200_pci_driver = { static int __init mgag200_init(void) { - if (vgacon_text_force() && mgag200_modeset == -1) + if (drm_firmware_drivers_only() && mgag200_modeset == -1) return -EINVAL; if (mgag200_modeset == 0) diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index e7efd9ede8e4..561309d447e0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -22,7 +22,6 @@ * Authors: Ben Skeggs */ -#include #include #include #include @@ -32,6 +31,7 @@ #include #include +#include #include #include #include @@ -1358,7 +1358,7 @@ nouveau_drm_init(void) nouveau_display_options(); if (nouveau_modeset == -1) { - if (vgacon_text_force()) + if (drm_firmware_drivers_only()) nouveau_modeset = 0; } diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c index fc47b0deb021..e4b16421500b 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.c +++ b/drivers/gpu/drm/qxl/qxl_drv.c @@ -29,7 +29,6 @@ #include "qxl_drv.h" -#include #include #include #include @@ -295,7 +294,7 @@ static struct drm_driver qxl_driver = { static int __init qxl_init(void) { - if (vgacon_text_force() && qxl_modeset == -1) + if (drm_firmware_drivers_only() && qxl_modeset == -1) return -EINVAL; if (qxl_modeset == 0) diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c index 380adc61e71c..956c72b5aa33 100644 --- a/drivers/gpu/drm/radeon/radeon_drv.c +++ b/drivers/gpu/drm/radeon/radeon_drv.c @@ -31,7 +31,6 @@ #include -#include #include #include #include @@ -637,7 +636,7 @@ static struct pci_driver radeon_kms_pci_driver = { static int __init radeon_module_init(void) { - if (vgacon_text_force() && radeon_modeset == -1) + if (drm_firmware_drivers_only() && radeon_modeset == -1) radeon_modeset = 0; if (radeon_modeset == 0) diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c index 2ce3bd903b70..fc26a1ce11ee 100644 --- a/drivers/gpu/drm/tiny/bochs.c +++ b/drivers/gpu/drm/tiny/bochs.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -#include #include #include @@ -719,7 +718,7 @@ static struct pci_driver bochs_pci_driver = { static int __init bochs_init(void) { - if (vgacon_text_force() && bochs_modeset == -1) + if (drm_firmware_drivers_only() && bochs_modeset == -1) return -EINVAL; if (bochs_modeset == 0) diff --git a/drivers/gpu/drm/tiny/cirrus.c b/drivers/gpu/drm/tiny/cirrus.c index 9327001d35dd..c95d9ff7d600 100644 --- a/drivers/gpu/drm/tiny/cirrus.c +++ b/drivers/gpu/drm/tiny/cirrus.c @@ -16,7 +16,6 @@ * Copyright 1999-2001 Jeff Garzik */ -#include #include #include #include @@ -636,8 +635,9 @@ static struct pci_driver cirrus_pci_driver = { static int __init cirrus_init(void) { - if (vgacon_text_force()) + if (drm_firmware_drivers_only()) return -EINVAL; + return pci_register_driver(&cirrus_pci_driver); } diff --git a/drivers/gpu/drm/vboxvideo/vbox_drv.c b/drivers/gpu/drm/vboxvideo/vbox_drv.c index fd7abb029c65..f35d9e44c6b7 100644 --- a/drivers/gpu/drm/vboxvideo/vbox_drv.c +++ b/drivers/gpu/drm/vboxvideo/vbox_drv.c @@ -7,7 +7,6 @@ * Michael Thayer */ -#include #include #include #include @@ -193,7 +192,7 @@ static const struct drm_driver driver = { static int __init vbox_init(void) { - if (vgacon_text_force() && vbox_modeset == -1) + if (drm_firmware_drivers_only() && vbox_modeset == -1) return -EINVAL; if (vbox_modeset == 0) diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c index d86e1ad4a972..c20c96a97a6c 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.c +++ b/drivers/gpu/drm/virtio/virtgpu_drv.c @@ -27,7 +27,6 @@ */ #include -#include #include #include #include @@ -104,7 +103,7 @@ static int virtio_gpu_probe(struct virtio_device *vdev) struct drm_device *dev; int ret; - if (vgacon_text_force() && virtio_gpu_modeset == -1) + if (drm_firmware_drivers_only() && virtio_gpu_modeset == -1) return -EINVAL; if (virtio_gpu_modeset == 0) diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index bfd71c86faa5..75d1e032cf16 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -25,7 +25,6 @@ * **************************************************************************/ -#include #include #include #include @@ -1651,7 +1650,7 @@ static int __init vmwgfx_init(void) { int ret; - if (vgacon_text_force()) + if (drm_firmware_drivers_only()) return -EINVAL; ret = pci_register_driver(&vmw_pci_driver); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index ef9c57ce0906..d4320b147956 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -97,30 +97,9 @@ static int vga_video_font_height; static int vga_scan_lines __read_mostly; static unsigned int vga_rolled_over; /* last vc_origin offset before wrap */ -static bool vgacon_text_mode_force; static bool vga_hardscroll_enabled; static bool vga_hardscroll_user_enable = true; -bool vgacon_text_force(void) -{ - return vgacon_text_mode_force; -} -EXPORT_SYMBOL(vgacon_text_force); - -static int __init text_mode(char *str) -{ - vgacon_text_mode_force = true; - - pr_warn("You have booted with nomodeset. This means your GPU drivers are DISABLED\n"); - pr_warn("Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly\n"); - pr_warn("Unless you actually understand what nomodeset does, you should reboot without enabling it\n"); - - return 1; -} - -/* force text mode - used by kernel modesetting */ -__setup("nomodeset", text_mode); - static int __init no_scroll(char *str) { /* diff --git a/gpu/drm/drm_nomodeset.c b/gpu/drm/drm_nomodeset.c new file mode 100644 index 000000000000..287edfb18b5d --- /dev/null +++ b/gpu/drm/drm_nomodeset.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +static bool drm_nomodeset; + +bool drm_firmware_drivers_only(void) +{ + return drm_nomodeset; +} +EXPORT_SYMBOL(drm_firmware_drivers_only); + +static int __init disable_modeset(char *str) +{ + drm_nomodeset = true; + + pr_warn("You have booted with nomodeset. This means your GPU drivers are DISABLED\n"); + pr_warn("Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly\n"); + pr_warn("Unless you actually understand what nomodeset does, you should reboot without enabling it\n"); + + return 1; +} + +/* Disable kernel modesetting */ +__setup("nomodeset", disable_modeset); diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h index a84eb4028e5b..89e26a732175 100644 --- a/include/drm/drm_drv.h +++ b/include/drm/drm_drv.h @@ -601,5 +601,10 @@ static inline bool drm_drv_uses_atomic_modeset(struct drm_device *dev) int drm_dev_set_unique(struct drm_device *dev, const char *name); +#ifdef CONFIG_VGA_CONSOLE +extern bool drm_firmware_drivers_only(void); +#else +static inline bool drm_firmware_drivers_only(void) { return false; } +#endif #endif diff --git a/include/linux/console.h b/include/linux/console.h index a97f277cfdfa..7cd758a4f44e 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -219,12 +219,6 @@ extern atomic_t ignore_console_lock_warning; #define VESA_HSYNC_SUSPEND 2 #define VESA_POWERDOWN 3 -#ifdef CONFIG_VGA_CONSOLE -extern bool vgacon_text_force(void); -#else -static inline bool vgacon_text_force(void) { return false; } -#endif - extern void console_init(void); /* For deferred console takeover */ -- cgit v1.2.3-59-g8ed1b From ed14e769f64311769dcf20dde544b82c158d01b1 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 15 Nov 2021 14:19:12 +0000 Subject: iio: buffer-dma: Remove unused iio_buffer_block struct This structure was never used anywhere, so it can safely be dropped. It will later be re-introduced as a different structure in a different header. Signed-off-by: Paul Cercueil Reviewed-by: Alexandru Ardelean Link: https://lore.kernel.org/r/20211115141925.60164-3-paul@crapouillou.net Signed-off-by: Jonathan Cameron --- include/linux/iio/buffer-dma.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iio/buffer-dma.h b/include/linux/iio/buffer-dma.h index ff15c61bf319..6564bdcdac66 100644 --- a/include/linux/iio/buffer-dma.h +++ b/include/linux/iio/buffer-dma.h @@ -17,11 +17,6 @@ struct iio_dma_buffer_queue; struct iio_dma_buffer_ops; struct device; -struct iio_buffer_block { - u32 size; - u32 bytes_used; -}; - /** * enum iio_block_state - State of a struct iio_dma_buffer_block * @IIO_BLOCK_STATE_DEQUEUED: Block is not queued -- cgit v1.2.3-59-g8ed1b From ffc7c5172a6d1f7ec468066a7172ce65baf1e3e1 Mon Sep 17 00:00:00 2001 From: Antoniu Miclaus Date: Fri, 19 Nov 2021 10:56:27 +0200 Subject: iio: expose shared parameter in IIO_ENUM_AVAILABLE The shared parameter should be configurable based on its usage, and not constrained to IIO_SHARED_BY_TYPE. This patch aims to improve the flexibility in using the IIO_ENUM_AVAILABLE define and avoid redefining custom iio enums that expose the shared parameter. An example is the ad5766.c driver where IIO_ENUM_AVAILABLE_SHARED was defined in order to achieve `shared` parameter customization. The current state of the IIO_ENUM_AVAILABLE implementation will imply similar redefinitions each time a driver will require access to the `shared` parameter. An example would be admv1013 driver which will require custom device attribute for the frequency translation modes: Quadrature I/Q mode and Intermediate Frequency mode. Signed-off-by: Antoniu Miclaus Reviewed-by: Alexandru Ardelean Link: https://lore.kernel.org/r/20211119085627.6348-1-antoniu.miclaus@analog.com Signed-off-by: Jonathan Cameron --- drivers/iio/accel/bma180.c | 2 +- drivers/iio/accel/mma9553.c | 2 +- drivers/iio/adc/ad7192.c | 3 ++- drivers/iio/adc/hi8435.c | 2 +- drivers/iio/dac/ad5064.c | 4 ++-- drivers/iio/dac/ad5380.c | 2 +- drivers/iio/dac/ad5446.c | 2 +- drivers/iio/dac/ad5504.c | 2 +- drivers/iio/dac/ad5624r_spi.c | 2 +- drivers/iio/dac/ad5686.c | 2 +- drivers/iio/dac/ad5766.c | 13 ++----------- drivers/iio/dac/ad5791.c | 2 +- drivers/iio/dac/max5821.c | 2 +- drivers/iio/dac/mcp4725.c | 8 ++++---- drivers/iio/dac/stm32-dac.c | 2 +- drivers/iio/dac/ti-dac082s085.c | 2 +- drivers/iio/dac/ti-dac5571.c | 2 +- drivers/iio/dac/ti-dac7311.c | 2 +- drivers/iio/magnetometer/hmc5843_core.c | 4 ++-- drivers/iio/trigger/stm32-timer-trigger.c | 4 ++-- include/linux/iio/iio.h | 5 +++-- 21 files changed, 31 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index 2edfcb4819b7..09496f358ad9 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -658,7 +658,7 @@ static const struct iio_chan_spec_ext_info bma023_ext_info[] = { static const struct iio_chan_spec_ext_info bma180_ext_info[] = { IIO_ENUM("power_mode", IIO_SHARED_BY_TYPE, &bma180_power_mode_enum), - IIO_ENUM_AVAILABLE("power_mode", &bma180_power_mode_enum), + IIO_ENUM_AVAILABLE("power_mode", IIO_SHARED_BY_TYPE, &bma180_power_mode_enum), IIO_MOUNT_MATRIX(IIO_SHARED_BY_DIR, bma180_accel_get_mount_matrix), { } }; diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c index ba3ecb3b57dc..0570ab1cc064 100644 --- a/drivers/iio/accel/mma9553.c +++ b/drivers/iio/accel/mma9553.c @@ -917,7 +917,7 @@ static const struct iio_enum mma9553_calibgender_enum = { static const struct iio_chan_spec_ext_info mma9553_ext_info[] = { IIO_ENUM("calibgender", IIO_SHARED_BY_TYPE, &mma9553_calibgender_enum), - IIO_ENUM_AVAILABLE("calibgender", &mma9553_calibgender_enum), + IIO_ENUM_AVAILABLE("calibgender", IIO_SHARED_BY_TYPE, &mma9553_calibgender_enum), {}, }; diff --git a/drivers/iio/adc/ad7192.c b/drivers/iio/adc/ad7192.c index 2121a812b0c3..cc990205f306 100644 --- a/drivers/iio/adc/ad7192.c +++ b/drivers/iio/adc/ad7192.c @@ -257,7 +257,8 @@ static const struct iio_chan_spec_ext_info ad7192_calibsys_ext_info[] = { }, IIO_ENUM("sys_calibration_mode", IIO_SEPARATE, &ad7192_syscalib_mode_enum), - IIO_ENUM_AVAILABLE("sys_calibration_mode", &ad7192_syscalib_mode_enum), + IIO_ENUM_AVAILABLE("sys_calibration_mode", IIO_SHARED_BY_TYPE, + &ad7192_syscalib_mode_enum), {} }; diff --git a/drivers/iio/adc/hi8435.c b/drivers/iio/adc/hi8435.c index 8b353e26668e..e665e14c6e54 100644 --- a/drivers/iio/adc/hi8435.c +++ b/drivers/iio/adc/hi8435.c @@ -350,7 +350,7 @@ static const struct iio_enum hi8435_sensing_mode = { static const struct iio_chan_spec_ext_info hi8435_ext_info[] = { IIO_ENUM("sensing_mode", IIO_SEPARATE, &hi8435_sensing_mode), - IIO_ENUM_AVAILABLE("sensing_mode", &hi8435_sensing_mode), + IIO_ENUM_AVAILABLE("sensing_mode", IIO_SHARED_BY_TYPE, &hi8435_sensing_mode), {}, }; diff --git a/drivers/iio/dac/ad5064.c b/drivers/iio/dac/ad5064.c index fd9cac4f6321..27ee2c63c5d4 100644 --- a/drivers/iio/dac/ad5064.c +++ b/drivers/iio/dac/ad5064.c @@ -377,7 +377,7 @@ static const struct iio_chan_spec_ext_info ad5064_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad5064_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5064_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5064_powerdown_mode_enum), { }, }; @@ -389,7 +389,7 @@ static const struct iio_chan_spec_ext_info ltc2617_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, <c2617_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", <c2617_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, <c2617_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5380.c b/drivers/iio/dac/ad5380.c index 8ca26bb4b62f..e38860a6a9f3 100644 --- a/drivers/iio/dac/ad5380.c +++ b/drivers/iio/dac/ad5380.c @@ -249,7 +249,7 @@ static const struct iio_chan_spec_ext_info ad5380_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5380_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5380_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5380_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5446.c b/drivers/iio/dac/ad5446.c index 3cc5513a6cbf..1c9b54c012a7 100644 --- a/drivers/iio/dac/ad5446.c +++ b/drivers/iio/dac/ad5446.c @@ -142,7 +142,7 @@ static const struct iio_chan_spec_ext_info ad5446_ext_info_powerdown[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad5446_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5446_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5446_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5504.c b/drivers/iio/dac/ad5504.c index 19cdf9890d02..b631261efa97 100644 --- a/drivers/iio/dac/ad5504.c +++ b/drivers/iio/dac/ad5504.c @@ -241,7 +241,7 @@ static const struct iio_chan_spec_ext_info ad5504_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5504_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5504_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5504_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5624r_spi.c b/drivers/iio/dac/ad5624r_spi.c index 530529feebb5..3c98941b9f99 100644 --- a/drivers/iio/dac/ad5624r_spi.c +++ b/drivers/iio/dac/ad5624r_spi.c @@ -159,7 +159,7 @@ static const struct iio_chan_spec_ext_info ad5624r_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5624r_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5624r_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5624r_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c index 8f001db775f4..e592a995f404 100644 --- a/drivers/iio/dac/ad5686.c +++ b/drivers/iio/dac/ad5686.c @@ -184,7 +184,7 @@ static const struct iio_chan_spec_ext_info ad5686_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad5686_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5686_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5686_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/ad5766.c b/drivers/iio/dac/ad5766.c index b0d220c3a126..43189af2fb1f 100644 --- a/drivers/iio/dac/ad5766.c +++ b/drivers/iio/dac/ad5766.c @@ -426,14 +426,6 @@ static ssize_t ad5766_write_ext(struct iio_dev *indio_dev, .shared = _shared, \ } -#define IIO_ENUM_AVAILABLE_SHARED(_name, _shared, _e) \ -{ \ - .name = (_name "_available"), \ - .shared = _shared, \ - .read = iio_enum_available_read, \ - .private = (uintptr_t)(_e), \ -} - static const struct iio_chan_spec_ext_info ad5766_ext_info[] = { _AD5766_CHAN_EXT_INFO("dither_enable", AD5766_DITHER_ENABLE, @@ -443,9 +435,8 @@ static const struct iio_chan_spec_ext_info ad5766_ext_info[] = { _AD5766_CHAN_EXT_INFO("dither_source", AD5766_DITHER_SOURCE, IIO_SEPARATE), IIO_ENUM("dither_scale", IIO_SEPARATE, &ad5766_dither_scale_enum), - IIO_ENUM_AVAILABLE_SHARED("dither_scale", - IIO_SEPARATE, - &ad5766_dither_scale_enum), + IIO_ENUM_AVAILABLE("dither_scale", IIO_SEPARATE, + &ad5766_dither_scale_enum), {} }; diff --git a/drivers/iio/dac/ad5791.c b/drivers/iio/dac/ad5791.c index a0923b76e8b6..7b4579d73d18 100644 --- a/drivers/iio/dac/ad5791.c +++ b/drivers/iio/dac/ad5791.c @@ -285,7 +285,7 @@ static const struct iio_chan_spec_ext_info ad5791_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5791_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &ad5791_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ad5791_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/max5821.c b/drivers/iio/dac/max5821.c index 7da4710a6408..fce640b7f1c8 100644 --- a/drivers/iio/dac/max5821.c +++ b/drivers/iio/dac/max5821.c @@ -137,7 +137,7 @@ static const struct iio_chan_spec_ext_info max5821_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &max5821_powerdown_mode_enum), - IIO_ENUM_AVAILABLE("powerdown_mode", &max5821_powerdown_mode_enum), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &max5821_powerdown_mode_enum), { }, }; diff --git a/drivers/iio/dac/mcp4725.c b/drivers/iio/dac/mcp4725.c index 34b14aafb630..98b2c2f10bf3 100644 --- a/drivers/iio/dac/mcp4725.c +++ b/drivers/iio/dac/mcp4725.c @@ -221,8 +221,8 @@ static const struct iio_chan_spec_ext_info mcp4725_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &mcp472x_powerdown_mode_enum[MCP4725]), - IIO_ENUM_AVAILABLE("powerdown_mode", - &mcp472x_powerdown_mode_enum[MCP4725]), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, + &mcp472x_powerdown_mode_enum[MCP4725]), { }, }; @@ -235,8 +235,8 @@ static const struct iio_chan_spec_ext_info mcp4726_ext_info[] = { }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &mcp472x_powerdown_mode_enum[MCP4726]), - IIO_ENUM_AVAILABLE("powerdown_mode", - &mcp472x_powerdown_mode_enum[MCP4726]), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, + &mcp472x_powerdown_mode_enum[MCP4726]), { }, }; diff --git a/drivers/iio/dac/stm32-dac.c b/drivers/iio/dac/stm32-dac.c index dd2e306824e7..cd71cc4553a7 100644 --- a/drivers/iio/dac/stm32-dac.c +++ b/drivers/iio/dac/stm32-dac.c @@ -246,7 +246,7 @@ static const struct iio_chan_spec_ext_info stm32_dac_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &stm32_dac_powerdown_mode_en), - IIO_ENUM_AVAILABLE("powerdown_mode", &stm32_dac_powerdown_mode_en), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &stm32_dac_powerdown_mode_en), {}, }; diff --git a/drivers/iio/dac/ti-dac082s085.c b/drivers/iio/dac/ti-dac082s085.c index 5c14bfb16521..6beda2193683 100644 --- a/drivers/iio/dac/ti-dac082s085.c +++ b/drivers/iio/dac/ti-dac082s085.c @@ -160,7 +160,7 @@ static const struct iio_chan_spec_ext_info ti_dac_ext_info[] = { .shared = IIO_SHARED_BY_TYPE, }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ti_dac_powerdown_mode), - IIO_ENUM_AVAILABLE("powerdown_mode", &ti_dac_powerdown_mode), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ti_dac_powerdown_mode), { }, }; diff --git a/drivers/iio/dac/ti-dac5571.c b/drivers/iio/dac/ti-dac5571.c index 546a4cf6c5ef..4a3b8d875518 100644 --- a/drivers/iio/dac/ti-dac5571.c +++ b/drivers/iio/dac/ti-dac5571.c @@ -212,7 +212,7 @@ static const struct iio_chan_spec_ext_info dac5571_ext_info[] = { .shared = IIO_SEPARATE, }, IIO_ENUM("powerdown_mode", IIO_SEPARATE, &dac5571_powerdown_mode), - IIO_ENUM_AVAILABLE("powerdown_mode", &dac5571_powerdown_mode), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &dac5571_powerdown_mode), {}, }; diff --git a/drivers/iio/dac/ti-dac7311.c b/drivers/iio/dac/ti-dac7311.c index 09218c3029f0..99f275829ec2 100644 --- a/drivers/iio/dac/ti-dac7311.c +++ b/drivers/iio/dac/ti-dac7311.c @@ -146,7 +146,7 @@ static const struct iio_chan_spec_ext_info ti_dac_ext_info[] = { .shared = IIO_SHARED_BY_TYPE, }, IIO_ENUM("powerdown_mode", IIO_SHARED_BY_TYPE, &ti_dac_powerdown_mode), - IIO_ENUM_AVAILABLE("powerdown_mode", &ti_dac_powerdown_mode), + IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &ti_dac_powerdown_mode), { }, }; diff --git a/drivers/iio/magnetometer/hmc5843_core.c b/drivers/iio/magnetometer/hmc5843_core.c index f08726bf5ec3..5a730d9bdbb0 100644 --- a/drivers/iio/magnetometer/hmc5843_core.c +++ b/drivers/iio/magnetometer/hmc5843_core.c @@ -246,7 +246,7 @@ static const struct iio_enum hmc5843_meas_conf_enum = { static const struct iio_chan_spec_ext_info hmc5843_ext_info[] = { IIO_ENUM("meas_conf", IIO_SHARED_BY_TYPE, &hmc5843_meas_conf_enum), - IIO_ENUM_AVAILABLE("meas_conf", &hmc5843_meas_conf_enum), + IIO_ENUM_AVAILABLE("meas_conf", IIO_SHARED_BY_TYPE, &hmc5843_meas_conf_enum), IIO_MOUNT_MATRIX(IIO_SHARED_BY_DIR, hmc5843_get_mount_matrix), { } }; @@ -260,7 +260,7 @@ static const struct iio_enum hmc5983_meas_conf_enum = { static const struct iio_chan_spec_ext_info hmc5983_ext_info[] = { IIO_ENUM("meas_conf", IIO_SHARED_BY_TYPE, &hmc5983_meas_conf_enum), - IIO_ENUM_AVAILABLE("meas_conf", &hmc5983_meas_conf_enum), + IIO_ENUM_AVAILABLE("meas_conf", IIO_SHARED_BY_TYPE, &hmc5983_meas_conf_enum), IIO_MOUNT_MATRIX(IIO_SHARED_BY_DIR, hmc5843_get_mount_matrix), { } }; diff --git a/drivers/iio/trigger/stm32-timer-trigger.c b/drivers/iio/trigger/stm32-timer-trigger.c index 33083877cd19..02b87b0f9d70 100644 --- a/drivers/iio/trigger/stm32-timer-trigger.c +++ b/drivers/iio/trigger/stm32-timer-trigger.c @@ -696,9 +696,9 @@ static const struct iio_chan_spec_ext_info stm32_trigger_count_info[] = { .write = stm32_count_set_preset }, IIO_ENUM("enable_mode", IIO_SEPARATE, &stm32_enable_mode_enum), - IIO_ENUM_AVAILABLE("enable_mode", &stm32_enable_mode_enum), + IIO_ENUM_AVAILABLE("enable_mode", IIO_SHARED_BY_TYPE, &stm32_enable_mode_enum), IIO_ENUM("trigger_mode", IIO_SEPARATE, &stm32_trigger_mode_enum), - IIO_ENUM_AVAILABLE("trigger_mode", &stm32_trigger_mode_enum), + IIO_ENUM_AVAILABLE("trigger_mode", IIO_SHARED_BY_TYPE, &stm32_trigger_mode_enum), {} }; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 324561b7a5e8..07025d6b3de1 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -103,15 +103,16 @@ ssize_t iio_enum_write(struct iio_dev *indio_dev, /** * IIO_ENUM_AVAILABLE() - Initialize enum available extended channel attribute * @_name: Attribute name ("_available" will be appended to the name) + * @_shared: Whether the attribute is shared between all channels * @_e: Pointer to an iio_enum struct * * Creates a read only attribute which lists all the available enum items in a * space separated list. This should usually be used together with IIO_ENUM() */ -#define IIO_ENUM_AVAILABLE(_name, _e) \ +#define IIO_ENUM_AVAILABLE(_name, _shared, _e) \ { \ .name = (_name "_available"), \ - .shared = IIO_SHARED_BY_TYPE, \ + .shared = _shared, \ .read = iio_enum_available_read, \ .private = (uintptr_t)(_e), \ } -- cgit v1.2.3-59-g8ed1b From 75c5bd68b699bbcb6d25879644d62de4da14ab92 Mon Sep 17 00:00:00 2001 From: Miri Korenblit Date: Fri, 26 Nov 2021 10:48:19 +0100 Subject: ieee80211: change HE nominal packet padding value defines It's easier to use and understand, and to extend for EHT later, if we use the values here instead of the shifted values. Unfortunately, we need to add _POS so that we can use it in places like iwlwifi/mvm where constants are needed. While at it, fix the typo ("NOMIMAL") which also helps catch any conflicts. Signed-off-by: Miri Korenblit Link: https://lore.kernel.org/r/20211126104817.7c29a05b8eb5.I2ca9faf06e177e3035bec91e2ae53c2f91d41774@changeid Signed-off-by: Johannes Berg --- drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c | 6 ++++-- drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c | 16 ++++++++-------- drivers/net/wireless/mediatek/mt76/mt7915/init.c | 3 ++- drivers/net/wireless/mediatek/mt76/mt7921/main.c | 3 ++- drivers/net/wireless/realtek/rtw89/core.c | 3 ++- drivers/net/wireless/realtek/rtw89/fw.c | 2 +- include/linux/ieee80211.h | 11 ++++++----- net/mac80211/debugfs_sta.c | 9 +++++---- 8 files changed, 30 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c index f470f9aea50f..881ede4a4aa2 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c @@ -607,7 +607,8 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { .phy_cap_info[9] = IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB | - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_RESERVED, + (IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED << + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS), .phy_cap_info[10] = IEEE80211_HE_PHY_CAP10_HE_MU_M1RU_MAX_LTF, }, @@ -664,7 +665,8 @@ static const struct ieee80211_sband_iftype_data iwl_he_capa[] = { IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI | IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242, .phy_cap_info[9] = - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_RESERVED, + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED + << IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS, }, /* * Set default Tx/Rx HE MCS NSS Support field. diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c index 9fb9c7dad314..b96ccabd1596 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c @@ -2126,24 +2126,24 @@ static void iwl_mvm_cfg_he_sta(struct iwl_mvm *mvm, } flags |= STA_CTXT_HE_PACKET_EXT; - } else if ((sta->he_cap.he_cap_elem.phy_cap_info[9] & - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_MASK) != - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_RESERVED) { + } else if (u8_get_bits(sta->he_cap.he_cap_elem.phy_cap_info[9], + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK) + != IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED) { int low_th = -1; int high_th = -1; /* Take the PPE thresholds from the nominal padding info */ - switch (sta->he_cap.he_cap_elem.phy_cap_info[9] & - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_MASK) { - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_0US: + switch (u8_get_bits(sta->he_cap.he_cap_elem.phy_cap_info[9], + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK)) { + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US: low_th = IWL_HE_PKT_EXT_NONE; high_th = IWL_HE_PKT_EXT_NONE; break; - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_8US: + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US: low_th = IWL_HE_PKT_EXT_BPSK; high_th = IWL_HE_PKT_EXT_NONE; break; - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US: + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US: low_th = IWL_HE_PKT_EXT_NONE; high_th = IWL_HE_PKT_EXT_BPSK; break; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 4fa8e7ba93e6..d054cdecd5f7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -853,7 +853,8 @@ mt7915_init_he_caps(struct mt7915_phy *phy, enum nl80211_band band, mt7915_gen_ppe_thresh(he_cap->ppe_thres, nss); } else { he_cap_elem->phy_cap_info[9] |= - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US; + u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US, + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); } idx++; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 633c6d2a57ac..e2eef9b5f63b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -140,7 +140,8 @@ mt7921_init_he_caps(struct mt7921_phy *phy, enum nl80211_band band, mt7921_gen_ppe_thresh(he_cap->ppe_thres, nss); } else { he_cap_elem->phy_cap_info[9] |= - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US; + u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US, + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); } if (band == NL80211_BAND_6GHZ) { diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index d02ec5a735cb..2bbe77ae7713 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -2115,7 +2115,8 @@ static void rtw89_init_he_cap(struct rtw89_dev *rtwdev, IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB | - IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US; + u8_encode_bits(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US, + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK); if (i == NL80211_IFTYPE_STATION) phy_cap_info[9] |= IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU; he_cap->he_mcs_nss_supp.rx_mcs_80 = cpu_to_le16(mcs_map); diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 212aaf577d3c..460d63dd6779 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -780,7 +780,7 @@ static void __get_sta_he_pkt_padding(struct rtw89_dev *rtwdev, if (!ppe_th) { u8 pad; - pad = FIELD_GET(IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_MASK, + pad = FIELD_GET(IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK, sta->he_cap.he_cap_elem.phy_cap_info[9]); for (i = 0; i < RTW89_PPE_BW_NUM; i++) diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 11d7af260f20..559b6c644938 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2258,11 +2258,12 @@ enum ieee80211_client_reg_power { #define IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU 0x08 #define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB 0x10 #define IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB 0x20 -#define IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_0US 0x00 -#define IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_8US 0x40 -#define IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US 0x80 -#define IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_RESERVED 0xc0 -#define IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_MASK 0xc0 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US 0x0 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US 0x1 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US 0x2 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_RESERVED 0x3 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_POS 6 +#define IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK 0xc0 #define IEEE80211_HE_PHY_CAP10_HE_MU_M1RU_MAX_LTF 0x01 diff --git a/net/mac80211/debugfs_sta.c b/net/mac80211/debugfs_sta.c index 481f01b0f65c..9479f2787ea7 100644 --- a/net/mac80211/debugfs_sta.c +++ b/net/mac80211/debugfs_sta.c @@ -936,14 +936,15 @@ static ssize_t sta_he_capa_read(struct file *file, char __user *userbuf, PFLAG(PHY, 9, RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB, "RX-FULL-BW-SU-USING-MU-WITH-NON-COMP-SIGB"); - switch (cap[9] & IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_MASK) { - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_0US: + switch (u8_get_bits(cap[9], + IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_MASK)) { + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_0US: PRINT("NOMINAL-PACKET-PADDING-0US"); break; - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_8US: + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_8US: PRINT("NOMINAL-PACKET-PADDING-8US"); break; - case IEEE80211_HE_PHY_CAP9_NOMIMAL_PKT_PADDING_16US: + case IEEE80211_HE_PHY_CAP9_NOMINAL_PKT_PADDING_16US: PRINT("NOMINAL-PACKET-PADDING-16US"); break; } -- cgit v1.2.3-59-g8ed1b From 4ba0b2c294fe691921271372f7b59e5cc2ce4b0f Mon Sep 17 00:00:00 2001 From: Russ Weight Date: Thu, 18 Nov 2021 17:55:51 -0800 Subject: fpga: mgr: Use standard dev_release for class driver The FPGA manager class driver data structure is being treated as a managed resource instead of using the standard dev_release call-back function to release the class data structure. This change removes the managed resource code for the freeing of the class data structure and combines the create() and register() functions into a single register() or register_full() function. The register_full() function accepts an info data structure to provide flexibility in passing optional parameters. The register() function supports the current parameter list for users that don't require the use of optional parameters. The devm_fpga_mgr_register() function is retained, and the devm_fpga_mgr_register_full() function is added. Signed-off-by: Russ Weight Reviewed-by: Xu Yilun Acked-by: Xu Yilun Signed-off-by: Moritz Fischer --- Documentation/driver-api/fpga/fpga-mgr.rst | 38 +++-- drivers/fpga/altera-cvp.c | 12 +- drivers/fpga/altera-pr-ip-core.c | 7 +- drivers/fpga/altera-ps-spi.c | 9 +- drivers/fpga/dfl-fme-mgr.c | 22 ++- drivers/fpga/fpga-mgr.c | 215 ++++++++++++----------------- drivers/fpga/ice40-spi.c | 9 +- drivers/fpga/machxo2-spi.c | 9 +- drivers/fpga/socfpga-a10.c | 16 +-- drivers/fpga/socfpga.c | 9 +- drivers/fpga/stratix10-soc.c | 16 +-- drivers/fpga/ts73xx-fpga.c | 9 +- drivers/fpga/versal-fpga.c | 9 +- drivers/fpga/xilinx-spi.c | 11 +- drivers/fpga/zynq-fpga.c | 16 +-- drivers/fpga/zynqmp-fpga.c | 9 +- include/linux/fpga/fpga-mgr.h | 62 ++++++--- 17 files changed, 217 insertions(+), 261 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/fpga/fpga-mgr.rst b/Documentation/driver-api/fpga/fpga-mgr.rst index 4d926b452cb3..42c01f396dce 100644 --- a/Documentation/driver-api/fpga/fpga-mgr.rst +++ b/Documentation/driver-api/fpga/fpga-mgr.rst @@ -24,7 +24,7 @@ How to support a new FPGA device -------------------------------- To add another FPGA manager, write a driver that implements a set of ops. The -probe function calls fpga_mgr_register(), such as:: +probe function calls fpga_mgr_register() or fpga_mgr_register_full(), such as:: static const struct fpga_manager_ops socfpga_fpga_ops = { .write_init = socfpga_fpga_ops_configure_init, @@ -49,14 +49,14 @@ probe function calls fpga_mgr_register(), such as:: * them in priv */ - mgr = devm_fpga_mgr_create(dev, "Altera SOCFPGA FPGA Manager", - &socfpga_fpga_ops, priv); - if (!mgr) - return -ENOMEM; + mgr = fpga_mgr_register(dev, "Altera SOCFPGA FPGA Manager", + &socfpga_fpga_ops, priv); + if (IS_ERR(mgr)) + return PTR_ERR(mgr); platform_set_drvdata(pdev, mgr); - return fpga_mgr_register(mgr); + return 0; } static int socfpga_fpga_remove(struct platform_device *pdev) @@ -68,6 +68,11 @@ probe function calls fpga_mgr_register(), such as:: return 0; } +Alternatively, the probe function could call one of the resource managed +register functions, devm_fpga_mgr_register() or devm_fpga_mgr_register_full(). +When these functions are used, the parameter syntax is the same, but the call +to fpga_mgr_unregister() should be removed. In the above example, the +socfpga_fpga_remove() function would not be required. The ops will implement whatever device specific register writes are needed to do the programming sequence for this particular FPGA. These ops return 0 for @@ -104,8 +109,14 @@ API for implementing a new FPGA Manager driver * ``fpga_mgr_states`` - Values for :c:expr:`fpga_manager->state`. * struct fpga_manager - the FPGA manager struct * struct fpga_manager_ops - Low level FPGA manager driver ops -* devm_fpga_mgr_create() - Allocate and init a manager struct -* fpga_mgr_register() - Register an FPGA manager +* struct fpga_manager_info - Parameter structure for fpga_mgr_register_full() +* fpga_mgr_register_full() - Create and register an FPGA manager using the + fpga_mgr_info structure to provide the full flexibility of options +* fpga_mgr_register() - Create and register an FPGA manager using standard + arguments +* devm_fpga_mgr_register_full() - Resource managed version of + fpga_mgr_register_full() +* devm_fpga_mgr_register() - Resource managed version of fpga_mgr_register() * fpga_mgr_unregister() - Unregister an FPGA manager .. kernel-doc:: include/linux/fpga/fpga-mgr.h @@ -117,11 +128,20 @@ API for implementing a new FPGA Manager driver .. kernel-doc:: include/linux/fpga/fpga-mgr.h :functions: fpga_manager_ops +.. kernel-doc:: include/linux/fpga/fpga-mgr.h + :functions: fpga_manager_info + .. kernel-doc:: drivers/fpga/fpga-mgr.c - :functions: devm_fpga_mgr_create + :functions: fpga_mgr_register_full .. kernel-doc:: drivers/fpga/fpga-mgr.c :functions: fpga_mgr_register +.. kernel-doc:: drivers/fpga/fpga-mgr.c + :functions: devm_fpga_mgr_register_full + +.. kernel-doc:: drivers/fpga/fpga-mgr.c + :functions: devm_fpga_mgr_register + .. kernel-doc:: drivers/fpga/fpga-mgr.c :functions: fpga_mgr_unregister diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c index ccf4546eff29..4ffb9da537d8 100644 --- a/drivers/fpga/altera-cvp.c +++ b/drivers/fpga/altera-cvp.c @@ -652,19 +652,15 @@ static int altera_cvp_probe(struct pci_dev *pdev, snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s @%s", ALTERA_CVP_MGR_NAME, pci_name(pdev)); - mgr = devm_fpga_mgr_create(&pdev->dev, conf->mgr_name, - &altera_cvp_ops, conf); - if (!mgr) { - ret = -ENOMEM; + mgr = fpga_mgr_register(&pdev->dev, conf->mgr_name, + &altera_cvp_ops, conf); + if (IS_ERR(mgr)) { + ret = PTR_ERR(mgr); goto err_unmap; } pci_set_drvdata(pdev, mgr); - ret = fpga_mgr_register(mgr); - if (ret) - goto err_unmap; - return 0; err_unmap: diff --git a/drivers/fpga/altera-pr-ip-core.c b/drivers/fpga/altera-pr-ip-core.c index dfdf21ed34c4..be0667968d33 100644 --- a/drivers/fpga/altera-pr-ip-core.c +++ b/drivers/fpga/altera-pr-ip-core.c @@ -191,11 +191,8 @@ int alt_pr_register(struct device *dev, void __iomem *reg_base) (val & ALT_PR_CSR_STATUS_MSK) >> ALT_PR_CSR_STATUS_SFT, (int)(val & ALT_PR_CSR_PR_START)); - mgr = devm_fpga_mgr_create(dev, dev_name(dev), &alt_pr_ops, priv); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, dev_name(dev), &alt_pr_ops, priv); + return PTR_ERR_OR_ZERO(mgr); } EXPORT_SYMBOL_GPL(alt_pr_register); diff --git a/drivers/fpga/altera-ps-spi.c b/drivers/fpga/altera-ps-spi.c index 23bfd4d1ad0f..5e1e009dba89 100644 --- a/drivers/fpga/altera-ps-spi.c +++ b/drivers/fpga/altera-ps-spi.c @@ -302,12 +302,9 @@ static int altera_ps_probe(struct spi_device *spi) snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s %s", dev_driver_string(&spi->dev), dev_name(&spi->dev)); - mgr = devm_fpga_mgr_create(&spi->dev, conf->mgr_name, - &altera_ps_ops, conf); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(&spi->dev, mgr); + mgr = devm_fpga_mgr_register(&spi->dev, conf->mgr_name, + &altera_ps_ops, conf); + return PTR_ERR_OR_ZERO(mgr); } static const struct spi_device_id altera_ps_spi_ids[] = { diff --git a/drivers/fpga/dfl-fme-mgr.c b/drivers/fpga/dfl-fme-mgr.c index 313420405d5e..af0785783b52 100644 --- a/drivers/fpga/dfl-fme-mgr.c +++ b/drivers/fpga/dfl-fme-mgr.c @@ -276,7 +276,7 @@ static void fme_mgr_get_compat_id(void __iomem *fme_pr, static int fme_mgr_probe(struct platform_device *pdev) { struct dfl_fme_mgr_pdata *pdata = dev_get_platdata(&pdev->dev); - struct fpga_compat_id *compat_id; + struct fpga_manager_info info = { 0 }; struct device *dev = &pdev->dev; struct fme_mgr_priv *priv; struct fpga_manager *mgr; @@ -296,20 +296,16 @@ static int fme_mgr_probe(struct platform_device *pdev) return PTR_ERR(priv->ioaddr); } - compat_id = devm_kzalloc(dev, sizeof(*compat_id), GFP_KERNEL); - if (!compat_id) + info.name = "DFL FME FPGA Manager"; + info.mops = &fme_mgr_ops; + info.priv = priv; + info.compat_id = devm_kzalloc(dev, sizeof(*info.compat_id), GFP_KERNEL); + if (!info.compat_id) return -ENOMEM; - fme_mgr_get_compat_id(priv->ioaddr, compat_id); - - mgr = devm_fpga_mgr_create(dev, "DFL FME FPGA Manager", - &fme_mgr_ops, priv); - if (!mgr) - return -ENOMEM; - - mgr->compat_id = compat_id; - - return devm_fpga_mgr_register(dev, mgr); + fme_mgr_get_compat_id(priv->ioaddr, info.compat_id); + mgr = devm_fpga_mgr_register_full(dev, &info); + return PTR_ERR_OR_ZERO(mgr); } static struct platform_driver fme_mgr_driver = { diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c index aa30889e2320..d49a9ce34568 100644 --- a/drivers/fpga/fpga-mgr.c +++ b/drivers/fpga/fpga-mgr.c @@ -592,49 +592,49 @@ void fpga_mgr_unlock(struct fpga_manager *mgr) EXPORT_SYMBOL_GPL(fpga_mgr_unlock); /** - * fpga_mgr_create - create and initialize an FPGA manager struct + * fpga_mgr_register_full - create and register an FPGA Manager device * @parent: fpga manager device from pdev - * @name: fpga manager name - * @mops: pointer to structure of fpga manager ops - * @priv: fpga manager private data + * @info: parameters for fpga manager * - * The caller of this function is responsible for freeing the struct with - * fpga_mgr_free(). Using devm_fpga_mgr_create() instead is recommended. + * The caller of this function is responsible for calling fpga_mgr_unregister(). + * Using devm_fpga_mgr_register_full() instead is recommended. * - * Return: pointer to struct fpga_manager or NULL + * Return: pointer to struct fpga_manager pointer or ERR_PTR() */ -struct fpga_manager *fpga_mgr_create(struct device *parent, const char *name, - const struct fpga_manager_ops *mops, - void *priv) +struct fpga_manager * +fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info) { + const struct fpga_manager_ops *mops = info->mops; struct fpga_manager *mgr; int id, ret; if (!mops) { dev_err(parent, "Attempt to register without fpga_manager_ops\n"); - return NULL; + return ERR_PTR(-EINVAL); } - if (!name || !strlen(name)) { + if (!info->name || !strlen(info->name)) { dev_err(parent, "Attempt to register with no name!\n"); - return NULL; + return ERR_PTR(-EINVAL); } mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); if (!mgr) - return NULL; + return ERR_PTR(-ENOMEM); id = ida_simple_get(&fpga_mgr_ida, 0, 0, GFP_KERNEL); - if (id < 0) + if (id < 0) { + ret = id; goto error_kfree; + } mutex_init(&mgr->ref_mutex); - mgr->name = name; - mgr->mops = mops; - mgr->priv = priv; + mgr->name = info->name; + mgr->mops = info->mops; + mgr->priv = info->priv; + mgr->compat_id = info->compat_id; - device_initialize(&mgr->dev); mgr->dev.class = fpga_mgr_class; mgr->dev.groups = mops->groups; mgr->dev.parent = parent; @@ -645,6 +645,19 @@ struct fpga_manager *fpga_mgr_create(struct device *parent, const char *name, if (ret) goto error_device; + /* + * Initialize framework state by requesting low level driver read state + * from device. FPGA may be in reset mode or may have been programmed + * by bootloader or EEPROM. + */ + mgr->state = fpga_mgr_state(mgr); + + ret = device_register(&mgr->dev); + if (ret) { + put_device(&mgr->dev); + return ERR_PTR(ret); + } + return mgr; error_device: @@ -652,96 +665,36 @@ error_device: error_kfree: kfree(mgr); - return NULL; + return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(fpga_mgr_create); +EXPORT_SYMBOL_GPL(fpga_mgr_register_full); /** - * fpga_mgr_free - free an FPGA manager created with fpga_mgr_create() - * @mgr: fpga manager struct - */ -void fpga_mgr_free(struct fpga_manager *mgr) -{ - ida_simple_remove(&fpga_mgr_ida, mgr->dev.id); - kfree(mgr); -} -EXPORT_SYMBOL_GPL(fpga_mgr_free); - -static void devm_fpga_mgr_release(struct device *dev, void *res) -{ - struct fpga_mgr_devres *dr = res; - - fpga_mgr_free(dr->mgr); -} - -/** - * devm_fpga_mgr_create - create and initialize a managed FPGA manager struct + * fpga_mgr_register - create and register an FPGA Manager device * @parent: fpga manager device from pdev * @name: fpga manager name * @mops: pointer to structure of fpga manager ops * @priv: fpga manager private data * - * This function is intended for use in an FPGA manager driver's probe function. - * After the manager driver creates the manager struct with - * devm_fpga_mgr_create(), it should register it with fpga_mgr_register(). The - * manager driver's remove function should call fpga_mgr_unregister(). The - * manager struct allocated with this function will be freed automatically on - * driver detach. This includes the case of a probe function returning error - * before calling fpga_mgr_register(), the struct will still get cleaned up. + * The caller of this function is responsible for calling fpga_mgr_unregister(). + * Using devm_fpga_mgr_register() instead is recommended. This simple + * version of the register function should be sufficient for most users. The + * fpga_mgr_register_full() function is available for users that need to pass + * additional, optional parameters. * - * Return: pointer to struct fpga_manager or NULL + * Return: pointer to struct fpga_manager pointer or ERR_PTR() */ -struct fpga_manager *devm_fpga_mgr_create(struct device *parent, const char *name, - const struct fpga_manager_ops *mops, - void *priv) +struct fpga_manager * +fpga_mgr_register(struct device *parent, const char *name, + const struct fpga_manager_ops *mops, void *priv) { - struct fpga_mgr_devres *dr; + struct fpga_manager_info info = { 0 }; - dr = devres_alloc(devm_fpga_mgr_release, sizeof(*dr), GFP_KERNEL); - if (!dr) - return NULL; + info.name = name; + info.mops = mops; + info.priv = priv; - dr->mgr = fpga_mgr_create(parent, name, mops, priv); - if (!dr->mgr) { - devres_free(dr); - return NULL; - } - - devres_add(parent, dr); - - return dr->mgr; -} -EXPORT_SYMBOL_GPL(devm_fpga_mgr_create); - -/** - * fpga_mgr_register - register an FPGA manager - * @mgr: fpga manager struct - * - * Return: 0 on success, negative error code otherwise. - */ -int fpga_mgr_register(struct fpga_manager *mgr) -{ - int ret; - - /* - * Initialize framework state by requesting low level driver read state - * from device. FPGA may be in reset mode or may have been programmed - * by bootloader or EEPROM. - */ - mgr->state = fpga_mgr_state(mgr); - - ret = device_add(&mgr->dev); - if (ret) - goto error_device; - - dev_info(&mgr->dev, "%s registered\n", mgr->name); - - return 0; - -error_device: - ida_simple_remove(&fpga_mgr_ida, mgr->dev.id); - - return ret; + return fpga_mgr_register_full(parent, &info); } EXPORT_SYMBOL_GPL(fpga_mgr_register); @@ -765,14 +718,6 @@ void fpga_mgr_unregister(struct fpga_manager *mgr) } EXPORT_SYMBOL_GPL(fpga_mgr_unregister); -static int fpga_mgr_devres_match(struct device *dev, void *res, - void *match_data) -{ - struct fpga_mgr_devres *dr = res; - - return match_data == dr->mgr; -} - static void devm_fpga_mgr_unregister(struct device *dev, void *res) { struct fpga_mgr_devres *dr = res; @@ -781,45 +726,67 @@ static void devm_fpga_mgr_unregister(struct device *dev, void *res) } /** - * devm_fpga_mgr_register - resource managed variant of fpga_mgr_register() - * @dev: managing device for this FPGA manager - * @mgr: fpga manager struct + * devm_fpga_mgr_register_full - resource managed variant of fpga_mgr_register() + * @parent: fpga manager device from pdev + * @info: parameters for fpga manager * - * This is the devres variant of fpga_mgr_register() for which the unregister + * This is the devres variant of fpga_mgr_register_full() for which the unregister * function will be called automatically when the managing device is detached. */ -int devm_fpga_mgr_register(struct device *dev, struct fpga_manager *mgr) +struct fpga_manager * +devm_fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info) { struct fpga_mgr_devres *dr; - int ret; - - /* - * Make sure that the struct fpga_manager * that is passed in is - * managed itself. - */ - if (WARN_ON(!devres_find(dev, devm_fpga_mgr_release, - fpga_mgr_devres_match, mgr))) - return -EINVAL; + struct fpga_manager *mgr; dr = devres_alloc(devm_fpga_mgr_unregister, sizeof(*dr), GFP_KERNEL); if (!dr) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - ret = fpga_mgr_register(mgr); - if (ret) { + mgr = fpga_mgr_register_full(parent, info); + if (IS_ERR(mgr)) { devres_free(dr); - return ret; + return mgr; } dr->mgr = mgr; - devres_add(dev, dr); + devres_add(parent, dr); - return 0; + return mgr; +} +EXPORT_SYMBOL_GPL(devm_fpga_mgr_register_full); + +/** + * devm_fpga_mgr_register - resource managed variant of fpga_mgr_register() + * @parent: fpga manager device from pdev + * @name: fpga manager name + * @mops: pointer to structure of fpga manager ops + * @priv: fpga manager private data + * + * This is the devres variant of fpga_mgr_register() for which the + * unregister function will be called automatically when the managing + * device is detached. + */ +struct fpga_manager * +devm_fpga_mgr_register(struct device *parent, const char *name, + const struct fpga_manager_ops *mops, void *priv) +{ + struct fpga_manager_info info = { 0 }; + + info.name = name; + info.mops = mops; + info.priv = priv; + + return devm_fpga_mgr_register_full(parent, &info); } EXPORT_SYMBOL_GPL(devm_fpga_mgr_register); static void fpga_mgr_dev_release(struct device *dev) { + struct fpga_manager *mgr = to_fpga_manager(dev); + + ida_simple_remove(&fpga_mgr_ida, mgr->dev.id); + kfree(mgr); } static int __init fpga_mgr_class_init(void) diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c index 029d3cdb918d..7cbb3558b844 100644 --- a/drivers/fpga/ice40-spi.c +++ b/drivers/fpga/ice40-spi.c @@ -178,12 +178,9 @@ static int ice40_fpga_probe(struct spi_device *spi) return ret; } - mgr = devm_fpga_mgr_create(dev, "Lattice iCE40 FPGA Manager", - &ice40_fpga_ops, priv); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, "Lattice iCE40 FPGA Manager", + &ice40_fpga_ops, priv); + return PTR_ERR_OR_ZERO(mgr); } static const struct of_device_id ice40_fpga_of_match[] = { diff --git a/drivers/fpga/machxo2-spi.c b/drivers/fpga/machxo2-spi.c index ea2ec3c6815c..905607992a12 100644 --- a/drivers/fpga/machxo2-spi.c +++ b/drivers/fpga/machxo2-spi.c @@ -370,12 +370,9 @@ static int machxo2_spi_probe(struct spi_device *spi) return -EINVAL; } - mgr = devm_fpga_mgr_create(dev, "Lattice MachXO2 SPI FPGA Manager", - &machxo2_ops, spi); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, "Lattice MachXO2 SPI FPGA Manager", + &machxo2_ops, spi); + return PTR_ERR_OR_ZERO(mgr); } #ifdef CONFIG_OF diff --git a/drivers/fpga/socfpga-a10.c b/drivers/fpga/socfpga-a10.c index 573d88bdf730..ac8e89b8a5cc 100644 --- a/drivers/fpga/socfpga-a10.c +++ b/drivers/fpga/socfpga-a10.c @@ -508,19 +508,15 @@ static int socfpga_a10_fpga_probe(struct platform_device *pdev) return -EBUSY; } - mgr = devm_fpga_mgr_create(dev, "SoCFPGA Arria10 FPGA Manager", - &socfpga_a10_fpga_mgr_ops, priv); - if (!mgr) - return -ENOMEM; - - platform_set_drvdata(pdev, mgr); - - ret = fpga_mgr_register(mgr); - if (ret) { + mgr = fpga_mgr_register(dev, "SoCFPGA Arria10 FPGA Manager", + &socfpga_a10_fpga_mgr_ops, priv); + if (IS_ERR(mgr)) { clk_disable_unprepare(priv->clk); - return ret; + return PTR_ERR(mgr); } + platform_set_drvdata(pdev, mgr); + return 0; } diff --git a/drivers/fpga/socfpga.c b/drivers/fpga/socfpga.c index 1f467173fc1f..7e0741f99696 100644 --- a/drivers/fpga/socfpga.c +++ b/drivers/fpga/socfpga.c @@ -571,12 +571,9 @@ static int socfpga_fpga_probe(struct platform_device *pdev) if (ret) return ret; - mgr = devm_fpga_mgr_create(dev, "Altera SOCFPGA FPGA Manager", - &socfpga_fpga_ops, priv); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, "Altera SOCFPGA FPGA Manager", + &socfpga_fpga_ops, priv); + return PTR_ERR_OR_ZERO(mgr); } #ifdef CONFIG_OF diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c index 047fd7f23706..737d14c6e0de 100644 --- a/drivers/fpga/stratix10-soc.c +++ b/drivers/fpga/stratix10-soc.c @@ -419,18 +419,11 @@ static int s10_probe(struct platform_device *pdev) init_completion(&priv->status_return_completion); - mgr = fpga_mgr_create(dev, "Stratix10 SOC FPGA Manager", - &s10_ops, priv); - if (!mgr) { - dev_err(dev, "unable to create FPGA manager\n"); - ret = -ENOMEM; - goto probe_err; - } - - ret = fpga_mgr_register(mgr); - if (ret) { + mgr = fpga_mgr_register(dev, "Stratix10 SOC FPGA Manager", + &s10_ops, priv); + if (IS_ERR(mgr)) { dev_err(dev, "unable to register FPGA manager\n"); - fpga_mgr_free(mgr); + ret = PTR_ERR(mgr); goto probe_err; } @@ -448,7 +441,6 @@ static int s10_remove(struct platform_device *pdev) struct s10_priv *priv = mgr->priv; fpga_mgr_unregister(mgr); - fpga_mgr_free(mgr); stratix10_svc_free_channel(priv->chan); return 0; diff --git a/drivers/fpga/ts73xx-fpga.c b/drivers/fpga/ts73xx-fpga.c index 167abb0b08d4..8e6e9c840d9d 100644 --- a/drivers/fpga/ts73xx-fpga.c +++ b/drivers/fpga/ts73xx-fpga.c @@ -116,12 +116,9 @@ static int ts73xx_fpga_probe(struct platform_device *pdev) if (IS_ERR(priv->io_base)) return PTR_ERR(priv->io_base); - mgr = devm_fpga_mgr_create(kdev, "TS-73xx FPGA Manager", - &ts73xx_fpga_ops, priv); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(kdev, mgr); + mgr = devm_fpga_mgr_register(kdev, "TS-73xx FPGA Manager", + &ts73xx_fpga_ops, priv); + return PTR_ERR_OR_ZERO(mgr); } static struct platform_driver ts73xx_fpga_driver = { diff --git a/drivers/fpga/versal-fpga.c b/drivers/fpga/versal-fpga.c index 5b0dda304bd2..e1601b3a345b 100644 --- a/drivers/fpga/versal-fpga.c +++ b/drivers/fpga/versal-fpga.c @@ -54,12 +54,9 @@ static int versal_fpga_probe(struct platform_device *pdev) return ret; } - mgr = devm_fpga_mgr_create(dev, "Xilinx Versal FPGA Manager", - &versal_fpga_ops, NULL); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, "Xilinx Versal FPGA Manager", + &versal_fpga_ops, NULL); + return PTR_ERR_OR_ZERO(mgr); } static const struct of_device_id versal_fpga_of_match[] = { diff --git a/drivers/fpga/xilinx-spi.c b/drivers/fpga/xilinx-spi.c index b6bcf1d9233d..e1a227e7ff2a 100644 --- a/drivers/fpga/xilinx-spi.c +++ b/drivers/fpga/xilinx-spi.c @@ -247,13 +247,10 @@ static int xilinx_spi_probe(struct spi_device *spi) return dev_err_probe(&spi->dev, PTR_ERR(conf->done), "Failed to get DONE gpio\n"); - mgr = devm_fpga_mgr_create(&spi->dev, - "Xilinx Slave Serial FPGA Manager", - &xilinx_spi_ops, conf); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(&spi->dev, mgr); + mgr = devm_fpga_mgr_register(&spi->dev, + "Xilinx Slave Serial FPGA Manager", + &xilinx_spi_ops, conf); + return PTR_ERR_OR_ZERO(mgr); } #ifdef CONFIG_OF diff --git a/drivers/fpga/zynq-fpga.c b/drivers/fpga/zynq-fpga.c index 9b75bd4f93d8..426aa34c6a0d 100644 --- a/drivers/fpga/zynq-fpga.c +++ b/drivers/fpga/zynq-fpga.c @@ -609,20 +609,16 @@ static int zynq_fpga_probe(struct platform_device *pdev) clk_disable(priv->clk); - mgr = devm_fpga_mgr_create(dev, "Xilinx Zynq FPGA Manager", - &zynq_fpga_ops, priv); - if (!mgr) - return -ENOMEM; - - platform_set_drvdata(pdev, mgr); - - err = fpga_mgr_register(mgr); - if (err) { + mgr = fpga_mgr_register(dev, "Xilinx Zynq FPGA Manager", + &zynq_fpga_ops, priv); + if (IS_ERR(mgr)) { dev_err(dev, "unable to register FPGA manager\n"); clk_unprepare(priv->clk); - return err; + return PTR_ERR(mgr); } + platform_set_drvdata(pdev, mgr); + return 0; } diff --git a/drivers/fpga/zynqmp-fpga.c b/drivers/fpga/zynqmp-fpga.c index 7d3d5650c322..c60f20949c47 100644 --- a/drivers/fpga/zynqmp-fpga.c +++ b/drivers/fpga/zynqmp-fpga.c @@ -95,12 +95,9 @@ static int zynqmp_fpga_probe(struct platform_device *pdev) priv->dev = dev; - mgr = devm_fpga_mgr_create(dev, "Xilinx ZynqMP FPGA Manager", - &zynqmp_fpga_ops, priv); - if (!mgr) - return -ENOMEM; - - return devm_fpga_mgr_register(dev, mgr); + mgr = devm_fpga_mgr_register(dev, "Xilinx ZynqMP FPGA Manager", + &zynqmp_fpga_ops, priv); + return PTR_ERR_OR_ZERO(mgr); } #ifdef CONFIG_OF diff --git a/include/linux/fpga/fpga-mgr.h b/include/linux/fpga/fpga-mgr.h index 474c1f506307..0f9468771bb9 100644 --- a/include/linux/fpga/fpga-mgr.h +++ b/include/linux/fpga/fpga-mgr.h @@ -105,6 +105,36 @@ struct fpga_image_info { #endif }; +/** + * struct fpga_compat_id - id for compatibility check + * + * @id_h: high 64bit of the compat_id + * @id_l: low 64bit of the compat_id + */ +struct fpga_compat_id { + u64 id_h; + u64 id_l; +}; + +/** + * struct fpga_manager_info - collection of parameters for an FPGA Manager + * @name: fpga manager name + * @compat_id: FPGA manager id for compatibility check. + * @mops: pointer to structure of fpga manager ops + * @priv: fpga manager private data + * + * fpga_manager_info contains parameters for the register_full function. + * These are separated into an info structure because they some are optional + * others could be added to in the future. The info structure facilitates + * maintaining a stable API. + */ +struct fpga_manager_info { + const char *name; + struct fpga_compat_id *compat_id; + const struct fpga_manager_ops *mops; + void *priv; +}; + /** * struct fpga_manager_ops - ops for low level fpga manager drivers * @initial_header_size: Maximum number of bytes that should be passed into write_init @@ -143,17 +173,6 @@ struct fpga_manager_ops { #define FPGA_MGR_STATUS_IP_PROTOCOL_ERR BIT(3) #define FPGA_MGR_STATUS_FIFO_OVERFLOW_ERR BIT(4) -/** - * struct fpga_compat_id - id for compatibility check - * - * @id_h: high 64bit of the compat_id - * @id_l: low 64bit of the compat_id - */ -struct fpga_compat_id { - u64 id_h; - u64 id_l; -}; - /** * struct fpga_manager - fpga manager structure * @name: name of low level fpga manager @@ -191,17 +210,18 @@ struct fpga_manager *fpga_mgr_get(struct device *dev); void fpga_mgr_put(struct fpga_manager *mgr); -struct fpga_manager *fpga_mgr_create(struct device *dev, const char *name, - const struct fpga_manager_ops *mops, - void *priv); -void fpga_mgr_free(struct fpga_manager *mgr); -int fpga_mgr_register(struct fpga_manager *mgr); -void fpga_mgr_unregister(struct fpga_manager *mgr); +struct fpga_manager * +fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info); -int devm_fpga_mgr_register(struct device *dev, struct fpga_manager *mgr); +struct fpga_manager * +fpga_mgr_register(struct device *parent, const char *name, + const struct fpga_manager_ops *mops, void *priv); +void fpga_mgr_unregister(struct fpga_manager *mgr); -struct fpga_manager *devm_fpga_mgr_create(struct device *dev, const char *name, - const struct fpga_manager_ops *mops, - void *priv); +struct fpga_manager * +devm_fpga_mgr_register_full(struct device *parent, const struct fpga_manager_info *info); +struct fpga_manager * +devm_fpga_mgr_register(struct device *parent, const char *name, + const struct fpga_manager_ops *mops, void *priv); #endif /*_LINUX_FPGA_MGR_H */ -- cgit v1.2.3-59-g8ed1b From 0d70af3c2530a70f1b2c197feaa63fbd3548ce34 Mon Sep 17 00:00:00 2001 From: Russ Weight Date: Thu, 18 Nov 2021 17:55:52 -0800 Subject: fpga: bridge: Use standard dev_release for class driver The FPGA bridge class driver data structure is being treated as a managed resource instead of using the standard dev_release call-back function to release the class data structure. This change removes the managed resource code and combines the create() and register() functions into a single register() function. Signed-off-by: Russ Weight Reviewed-by: Xu Yilun Acked-by: Xu Yilun Signed-off-by: Moritz Fischer --- Documentation/driver-api/fpga/fpga-bridge.rst | 6 +- drivers/fpga/altera-fpga2sdram.c | 12 +-- drivers/fpga/altera-freeze-bridge.c | 10 +-- drivers/fpga/altera-hps2fpga.c | 12 +-- drivers/fpga/dfl-fme-br.c | 10 +-- drivers/fpga/fpga-bridge.c | 122 ++++++-------------------- drivers/fpga/xilinx-pr-decoupler.c | 17 ++-- include/linux/fpga/fpga-bridge.h | 30 +++++-- 8 files changed, 74 insertions(+), 145 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/fpga/fpga-bridge.rst b/Documentation/driver-api/fpga/fpga-bridge.rst index 8d650b4e2ce6..604208534095 100644 --- a/Documentation/driver-api/fpga/fpga-bridge.rst +++ b/Documentation/driver-api/fpga/fpga-bridge.rst @@ -6,8 +6,7 @@ API to implement a new FPGA bridge * struct fpga_bridge - The FPGA Bridge structure * struct fpga_bridge_ops - Low level Bridge driver ops -* devm_fpga_bridge_create() - Allocate and init a bridge struct -* fpga_bridge_register() - Register a bridge +* fpga_bridge_register() - Create and register a bridge * fpga_bridge_unregister() - Unregister a bridge .. kernel-doc:: include/linux/fpga/fpga-bridge.h @@ -16,9 +15,6 @@ API to implement a new FPGA bridge .. kernel-doc:: include/linux/fpga/fpga-bridge.h :functions: fpga_bridge_ops -.. kernel-doc:: drivers/fpga/fpga-bridge.c - :functions: devm_fpga_bridge_create - .. kernel-doc:: drivers/fpga/fpga-bridge.c :functions: fpga_bridge_register diff --git a/drivers/fpga/altera-fpga2sdram.c b/drivers/fpga/altera-fpga2sdram.c index a78e49c63c64..ff3a646fd9e3 100644 --- a/drivers/fpga/altera-fpga2sdram.c +++ b/drivers/fpga/altera-fpga2sdram.c @@ -121,17 +121,13 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev) /* Get f2s bridge configuration saved in handoff register */ regmap_read(sysmgr, SYSMGR_ISWGRP_HANDOFF3, &priv->mask); - br = devm_fpga_bridge_create(dev, F2S_BRIDGE_NAME, - &altera_fpga2sdram_br_ops, priv); - if (!br) - return -ENOMEM; + br = fpga_bridge_register(dev, F2S_BRIDGE_NAME, + &altera_fpga2sdram_br_ops, priv); + if (IS_ERR(br)) + return PTR_ERR(br); platform_set_drvdata(pdev, br); - ret = fpga_bridge_register(br); - if (ret) - return ret; - dev_info(dev, "driver initialized with handoff %08x\n", priv->mask); if (!of_property_read_u32(dev->of_node, "bridge-enable", &enable)) { diff --git a/drivers/fpga/altera-freeze-bridge.c b/drivers/fpga/altera-freeze-bridge.c index 7d22a44d652e..445f4b011167 100644 --- a/drivers/fpga/altera-freeze-bridge.c +++ b/drivers/fpga/altera-freeze-bridge.c @@ -246,14 +246,14 @@ static int altera_freeze_br_probe(struct platform_device *pdev) priv->base_addr = base_addr; - br = devm_fpga_bridge_create(dev, FREEZE_BRIDGE_NAME, - &altera_freeze_br_br_ops, priv); - if (!br) - return -ENOMEM; + br = fpga_bridge_register(dev, FREEZE_BRIDGE_NAME, + &altera_freeze_br_br_ops, priv); + if (IS_ERR(br)) + return PTR_ERR(br); platform_set_drvdata(pdev, br); - return fpga_bridge_register(br); + return 0; } static int altera_freeze_br_remove(struct platform_device *pdev) diff --git a/drivers/fpga/altera-hps2fpga.c b/drivers/fpga/altera-hps2fpga.c index 77b95f251821..aa758426c22b 100644 --- a/drivers/fpga/altera-hps2fpga.c +++ b/drivers/fpga/altera-hps2fpga.c @@ -180,19 +180,15 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev) } } - br = devm_fpga_bridge_create(dev, priv->name, - &altera_hps2fpga_br_ops, priv); - if (!br) { - ret = -ENOMEM; + br = fpga_bridge_register(dev, priv->name, + &altera_hps2fpga_br_ops, priv); + if (IS_ERR(br)) { + ret = PTR_ERR(br); goto err; } platform_set_drvdata(pdev, br); - ret = fpga_bridge_register(br); - if (ret) - goto err; - return 0; err: diff --git a/drivers/fpga/dfl-fme-br.c b/drivers/fpga/dfl-fme-br.c index 3ff9f3a687ce..808d1f4d76df 100644 --- a/drivers/fpga/dfl-fme-br.c +++ b/drivers/fpga/dfl-fme-br.c @@ -68,14 +68,14 @@ static int fme_br_probe(struct platform_device *pdev) priv->pdata = dev_get_platdata(dev); - br = devm_fpga_bridge_create(dev, "DFL FPGA FME Bridge", - &fme_bridge_ops, priv); - if (!br) - return -ENOMEM; + br = fpga_bridge_register(dev, "DFL FPGA FME Bridge", + &fme_bridge_ops, priv); + if (IS_ERR(br)) + return PTR_ERR(br); platform_set_drvdata(pdev, br); - return fpga_bridge_register(br); + return 0; } static int fme_br_remove(struct platform_device *pdev) diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 798f55670646..16f2b164a178 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -312,36 +312,41 @@ static struct attribute *fpga_bridge_attrs[] = { ATTRIBUTE_GROUPS(fpga_bridge); /** - * fpga_bridge_create - create and initialize a struct fpga_bridge + * fpga_bridge_register - create and register an FPGA Bridge device * @parent: FPGA bridge device from pdev * @name: FPGA bridge name * @br_ops: pointer to structure of fpga bridge ops * @priv: FPGA bridge private data * - * The caller of this function is responsible for freeing the bridge with - * fpga_bridge_free(). Using devm_fpga_bridge_create() instead is recommended. - * - * Return: struct fpga_bridge or NULL + * Return: struct fpga_bridge pointer or ERR_PTR() */ -struct fpga_bridge *fpga_bridge_create(struct device *parent, const char *name, - const struct fpga_bridge_ops *br_ops, - void *priv) +struct fpga_bridge * +fpga_bridge_register(struct device *parent, const char *name, + const struct fpga_bridge_ops *br_ops, + void *priv) { struct fpga_bridge *bridge; int id, ret; + if (!br_ops) { + dev_err(parent, "Attempt to register without fpga_bridge_ops\n"); + return ERR_PTR(-EINVAL); + } + if (!name || !strlen(name)) { dev_err(parent, "Attempt to register with no name!\n"); - return NULL; + return ERR_PTR(-EINVAL); } bridge = kzalloc(sizeof(*bridge), GFP_KERNEL); if (!bridge) - return NULL; + return ERR_PTR(-ENOMEM); id = ida_simple_get(&fpga_bridge_ida, 0, 0, GFP_KERNEL); - if (id < 0) + if (id < 0) { + ret = id; goto error_kfree; + } mutex_init(&bridge->mutex); INIT_LIST_HEAD(&bridge->node); @@ -350,17 +355,23 @@ struct fpga_bridge *fpga_bridge_create(struct device *parent, const char *name, bridge->br_ops = br_ops; bridge->priv = priv; - device_initialize(&bridge->dev); bridge->dev.groups = br_ops->groups; bridge->dev.class = fpga_bridge_class; bridge->dev.parent = parent; bridge->dev.of_node = parent->of_node; bridge->dev.id = id; + of_platform_populate(bridge->dev.of_node, NULL, NULL, &bridge->dev); ret = dev_set_name(&bridge->dev, "br%d", id); if (ret) goto error_device; + ret = device_register(&bridge->dev); + if (ret) { + put_device(&bridge->dev); + return ERR_PTR(ret); + } + return bridge; error_device: @@ -368,88 +379,7 @@ error_device: error_kfree: kfree(bridge); - return NULL; -} -EXPORT_SYMBOL_GPL(fpga_bridge_create); - -/** - * fpga_bridge_free - free an fpga bridge created by fpga_bridge_create() - * @bridge: FPGA bridge struct - */ -void fpga_bridge_free(struct fpga_bridge *bridge) -{ - ida_simple_remove(&fpga_bridge_ida, bridge->dev.id); - kfree(bridge); -} -EXPORT_SYMBOL_GPL(fpga_bridge_free); - -static void devm_fpga_bridge_release(struct device *dev, void *res) -{ - struct fpga_bridge *bridge = *(struct fpga_bridge **)res; - - fpga_bridge_free(bridge); -} - -/** - * devm_fpga_bridge_create - create and init a managed struct fpga_bridge - * @parent: FPGA bridge device from pdev - * @name: FPGA bridge name - * @br_ops: pointer to structure of fpga bridge ops - * @priv: FPGA bridge private data - * - * This function is intended for use in an FPGA bridge driver's probe function. - * After the bridge driver creates the struct with devm_fpga_bridge_create(), it - * should register the bridge with fpga_bridge_register(). The bridge driver's - * remove function should call fpga_bridge_unregister(). The bridge struct - * allocated with this function will be freed automatically on driver detach. - * This includes the case of a probe function returning error before calling - * fpga_bridge_register(), the struct will still get cleaned up. - * - * Return: struct fpga_bridge or NULL - */ -struct fpga_bridge -*devm_fpga_bridge_create(struct device *parent, const char *name, - const struct fpga_bridge_ops *br_ops, void *priv) -{ - struct fpga_bridge **ptr, *bridge; - - ptr = devres_alloc(devm_fpga_bridge_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return NULL; - - bridge = fpga_bridge_create(parent, name, br_ops, priv); - if (!bridge) { - devres_free(ptr); - } else { - *ptr = bridge; - devres_add(parent, ptr); - } - - return bridge; -} -EXPORT_SYMBOL_GPL(devm_fpga_bridge_create); - -/** - * fpga_bridge_register - register an FPGA bridge - * - * @bridge: FPGA bridge struct - * - * Return: 0 for success, error code otherwise. - */ -int fpga_bridge_register(struct fpga_bridge *bridge) -{ - struct device *dev = &bridge->dev; - int ret; - - ret = device_add(dev); - if (ret) - return ret; - - of_platform_populate(dev->of_node, NULL, NULL, dev); - - dev_info(dev->parent, "fpga bridge [%s] registered\n", bridge->name); - - return 0; + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(fpga_bridge_register); @@ -475,6 +405,10 @@ EXPORT_SYMBOL_GPL(fpga_bridge_unregister); static void fpga_bridge_dev_release(struct device *dev) { + struct fpga_bridge *bridge = to_fpga_bridge(dev); + + ida_simple_remove(&fpga_bridge_ida, bridge->dev.id); + kfree(bridge); } static int __init fpga_bridge_dev_init(void) diff --git a/drivers/fpga/xilinx-pr-decoupler.c b/drivers/fpga/xilinx-pr-decoupler.c index e986ed47c4ed..2d9c491f7be9 100644 --- a/drivers/fpga/xilinx-pr-decoupler.c +++ b/drivers/fpga/xilinx-pr-decoupler.c @@ -140,22 +140,17 @@ static int xlnx_pr_decoupler_probe(struct platform_device *pdev) clk_disable(priv->clk); - br = devm_fpga_bridge_create(&pdev->dev, priv->ipconfig->name, - &xlnx_pr_decoupler_br_ops, priv); - if (!br) { - err = -ENOMEM; - goto err_clk; - } - - platform_set_drvdata(pdev, br); - - err = fpga_bridge_register(br); - if (err) { + br = fpga_bridge_register(&pdev->dev, priv->ipconfig->name, + &xlnx_pr_decoupler_br_ops, priv); + if (IS_ERR(br)) { + err = PTR_ERR(br); dev_err(&pdev->dev, "unable to register %s", priv->ipconfig->name); goto err_clk; } + platform_set_drvdata(pdev, br); + return 0; err_clk: diff --git a/include/linux/fpga/fpga-bridge.h b/include/linux/fpga/fpga-bridge.h index 6c3c28806ff1..223da48a6d18 100644 --- a/include/linux/fpga/fpga-bridge.h +++ b/include/linux/fpga/fpga-bridge.h @@ -22,6 +22,23 @@ struct fpga_bridge_ops { const struct attribute_group **groups; }; +/** + * struct fpga_bridge_info - collection of parameters an FPGA Bridge + * @name: fpga bridge name + * @br_ops: pointer to structure of fpga bridge ops + * @priv: fpga bridge private data + * + * fpga_bridge_info contains parameters for the register function. These + * are separated into an info structure because they some are optional + * others could be added to in the future. The info structure facilitates + * maintaining a stable API. + */ +struct fpga_bridge_info { + const char *name; + const struct fpga_bridge_ops *br_ops; + void *priv; +}; + /** * struct fpga_bridge - FPGA bridge structure * @name: name of low level FPGA bridge @@ -62,15 +79,10 @@ int of_fpga_bridge_get_to_list(struct device_node *np, struct fpga_image_info *info, struct list_head *bridge_list); -struct fpga_bridge *fpga_bridge_create(struct device *dev, const char *name, - const struct fpga_bridge_ops *br_ops, - void *priv); -void fpga_bridge_free(struct fpga_bridge *br); -int fpga_bridge_register(struct fpga_bridge *br); +struct fpga_bridge * +fpga_bridge_register(struct device *parent, const char *name, + const struct fpga_bridge_ops *br_ops, + void *priv); void fpga_bridge_unregister(struct fpga_bridge *br); -struct fpga_bridge -*devm_fpga_bridge_create(struct device *dev, const char *name, - const struct fpga_bridge_ops *br_ops, void *priv); - #endif /* _LINUX_FPGA_BRIDGE_H */ -- cgit v1.2.3-59-g8ed1b From 8886a579744fbfa53e69aa453ed10ae3b1f9abac Mon Sep 17 00:00:00 2001 From: Russ Weight Date: Thu, 18 Nov 2021 17:55:53 -0800 Subject: fpga: region: Use standard dev_release for class driver The FPGA region class driver data structure is being treated as a managed resource instead of using the standard dev_release call-back function to release the class data structure. This change removes the managed resource code and combines the create() and register() functions into a single register() or register_full() function. The register_full() function accepts an info data structure to provide flexibility in passing optional parameters. The register() function supports the current parameter list for users that don't require the use of optional parameters. Signed-off-by: Russ Weight Reviewed-by: Xu Yilun Acked-by: Xu Yilun Signed-off-by: Moritz Fischer --- Documentation/driver-api/fpga/fpga-region.rst | 12 ++- drivers/fpga/dfl-fme-region.c | 17 ++-- drivers/fpga/dfl.c | 12 +-- drivers/fpga/fpga-region.c | 119 ++++++++++---------------- drivers/fpga/of-fpga-region.c | 10 +-- include/linux/fpga/fpga-region.h | 36 ++++++-- 6 files changed, 95 insertions(+), 111 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/fpga/fpga-region.rst b/Documentation/driver-api/fpga/fpga-region.rst index 2636a27c11b2..dc55d60a0b4a 100644 --- a/Documentation/driver-api/fpga/fpga-region.rst +++ b/Documentation/driver-api/fpga/fpga-region.rst @@ -46,8 +46,11 @@ API to add a new FPGA region ---------------------------- * struct fpga_region - The FPGA region struct -* devm_fpga_region_create() - Allocate and init a region struct -* fpga_region_register() - Register an FPGA region +* struct fpga_region_info - Parameter structure for fpga_region_register_full() +* fpga_region_register_full() - Create and register an FPGA region using the + fpga_region_info structure to provide the full flexibility of options +* fpga_region_register() - Create and register an FPGA region using standard + arguments * fpga_region_unregister() - Unregister an FPGA region The FPGA region's probe function will need to get a reference to the FPGA @@ -75,8 +78,11 @@ following APIs to handle building or tearing down that list. .. kernel-doc:: include/linux/fpga/fpga-region.h :functions: fpga_region +.. kernel-doc:: include/linux/fpga/fpga-region.h + :functions: fpga_region_info + .. kernel-doc:: drivers/fpga/fpga-region.c - :functions: devm_fpga_region_create + :functions: fpga_region_register_full .. kernel-doc:: drivers/fpga/fpga-region.c :functions: fpga_region_register diff --git a/drivers/fpga/dfl-fme-region.c b/drivers/fpga/dfl-fme-region.c index 1eeb42af1012..4aebde0a7f1c 100644 --- a/drivers/fpga/dfl-fme-region.c +++ b/drivers/fpga/dfl-fme-region.c @@ -30,6 +30,7 @@ static int fme_region_get_bridges(struct fpga_region *region) static int fme_region_probe(struct platform_device *pdev) { struct dfl_fme_region_pdata *pdata = dev_get_platdata(&pdev->dev); + struct fpga_region_info info = { 0 }; struct device *dev = &pdev->dev; struct fpga_region *region; struct fpga_manager *mgr; @@ -39,20 +40,18 @@ static int fme_region_probe(struct platform_device *pdev) if (IS_ERR(mgr)) return -EPROBE_DEFER; - region = devm_fpga_region_create(dev, mgr, fme_region_get_bridges); - if (!region) { - ret = -ENOMEM; + info.mgr = mgr; + info.compat_id = mgr->compat_id; + info.get_bridges = fme_region_get_bridges; + info.priv = pdata; + region = fpga_region_register_full(dev, &info); + if (IS_ERR(region)) { + ret = PTR_ERR(region); goto eprobe_mgr_put; } - region->priv = pdata; - region->compat_id = mgr->compat_id; platform_set_drvdata(pdev, region); - ret = fpga_region_register(region); - if (ret) - goto eprobe_mgr_put; - dev_dbg(dev, "DFL FME FPGA Region probed\n"); return 0; diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index f86666cf2c6a..599bb21d86af 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -1407,19 +1407,15 @@ dfl_fpga_feature_devs_enumerate(struct dfl_fpga_enum_info *info) if (!cdev) return ERR_PTR(-ENOMEM); - cdev->region = devm_fpga_region_create(info->dev, NULL, NULL); - if (!cdev->region) { - ret = -ENOMEM; - goto free_cdev_exit; - } - cdev->parent = info->dev; mutex_init(&cdev->lock); INIT_LIST_HEAD(&cdev->port_dev_list); - ret = fpga_region_register(cdev->region); - if (ret) + cdev->region = fpga_region_register(info->dev, NULL, NULL); + if (IS_ERR(cdev->region)) { + ret = PTR_ERR(cdev->region); goto free_cdev_exit; + } /* create and init build info for enumeration */ binfo = devm_kzalloc(info->dev, sizeof(*binfo), GFP_KERNEL); diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c index a4838715221f..b0ac18de4885 100644 --- a/drivers/fpga/fpga-region.c +++ b/drivers/fpga/fpga-region.c @@ -180,39 +180,42 @@ static struct attribute *fpga_region_attrs[] = { ATTRIBUTE_GROUPS(fpga_region); /** - * fpga_region_create - alloc and init a struct fpga_region + * fpga_region_register_full - create and register an FPGA Region device * @parent: device parent - * @mgr: manager that programs this region - * @get_bridges: optional function to get bridges to a list - * - * The caller of this function is responsible for freeing the resulting region - * struct with fpga_region_free(). Using devm_fpga_region_create() instead is - * recommended. + * @info: parameters for FPGA Region * - * Return: struct fpga_region or NULL + * Return: struct fpga_region or ERR_PTR() */ -struct fpga_region -*fpga_region_create(struct device *parent, - struct fpga_manager *mgr, - int (*get_bridges)(struct fpga_region *)) +struct fpga_region * +fpga_region_register_full(struct device *parent, const struct fpga_region_info *info) { struct fpga_region *region; int id, ret = 0; + if (!info) { + dev_err(parent, + "Attempt to register without required info structure\n"); + return ERR_PTR(-EINVAL); + } + region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) - return NULL; + return ERR_PTR(-ENOMEM); id = ida_simple_get(&fpga_region_ida, 0, 0, GFP_KERNEL); - if (id < 0) + if (id < 0) { + ret = id; goto err_free; + } + + region->mgr = info->mgr; + region->compat_id = info->compat_id; + region->priv = info->priv; + region->get_bridges = info->get_bridges; - region->mgr = mgr; - region->get_bridges = get_bridges; mutex_init(®ion->mutex); INIT_LIST_HEAD(®ion->bridge_list); - device_initialize(®ion->dev); region->dev.class = fpga_region_class; region->dev.parent = parent; region->dev.of_node = parent->of_node; @@ -222,6 +225,12 @@ struct fpga_region if (ret) goto err_remove; + ret = device_register(®ion->dev); + if (ret) { + put_device(®ion->dev); + return ERR_PTR(ret); + } + return region; err_remove: @@ -229,76 +238,32 @@ err_remove: err_free: kfree(region); - return NULL; -} -EXPORT_SYMBOL_GPL(fpga_region_create); - -/** - * fpga_region_free - free an FPGA region created by fpga_region_create() - * @region: FPGA region - */ -void fpga_region_free(struct fpga_region *region) -{ - ida_simple_remove(&fpga_region_ida, region->dev.id); - kfree(region); -} -EXPORT_SYMBOL_GPL(fpga_region_free); - -static void devm_fpga_region_release(struct device *dev, void *res) -{ - struct fpga_region *region = *(struct fpga_region **)res; - - fpga_region_free(region); + return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(fpga_region_register_full); /** - * devm_fpga_region_create - create and initialize a managed FPGA region struct + * fpga_region_register - create and register an FPGA Region device * @parent: device parent * @mgr: manager that programs this region * @get_bridges: optional function to get bridges to a list * - * This function is intended for use in an FPGA region driver's probe function. - * After the region driver creates the region struct with - * devm_fpga_region_create(), it should register it with fpga_region_register(). - * The region driver's remove function should call fpga_region_unregister(). - * The region struct allocated with this function will be freed automatically on - * driver detach. This includes the case of a probe function returning error - * before calling fpga_region_register(), the struct will still get cleaned up. + * This simple version of the register function should be sufficient for most users. + * The fpga_region_register_full() function is available for users that need to + * pass additional, optional parameters. * - * Return: struct fpga_region or NULL + * Return: struct fpga_region or ERR_PTR() */ -struct fpga_region -*devm_fpga_region_create(struct device *parent, - struct fpga_manager *mgr, - int (*get_bridges)(struct fpga_region *)) +struct fpga_region * +fpga_region_register(struct device *parent, struct fpga_manager *mgr, + int (*get_bridges)(struct fpga_region *)) { - struct fpga_region **ptr, *region; - - ptr = devres_alloc(devm_fpga_region_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return NULL; + struct fpga_region_info info = { 0 }; - region = fpga_region_create(parent, mgr, get_bridges); - if (!region) { - devres_free(ptr); - } else { - *ptr = region; - devres_add(parent, ptr); - } + info.mgr = mgr; + info.get_bridges = get_bridges; - return region; -} -EXPORT_SYMBOL_GPL(devm_fpga_region_create); - -/** - * fpga_region_register - register an FPGA region - * @region: FPGA region - * - * Return: 0 or -errno - */ -int fpga_region_register(struct fpga_region *region) -{ - return device_add(®ion->dev); + return fpga_region_register_full(parent, &info); } EXPORT_SYMBOL_GPL(fpga_region_register); @@ -316,6 +281,10 @@ EXPORT_SYMBOL_GPL(fpga_region_unregister); static void fpga_region_dev_release(struct device *dev) { + struct fpga_region *region = to_fpga_region(dev); + + ida_simple_remove(&fpga_region_ida, region->dev.id); + kfree(region); } /** diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c index e3c25576b6b9..9c662db1c508 100644 --- a/drivers/fpga/of-fpga-region.c +++ b/drivers/fpga/of-fpga-region.c @@ -405,16 +405,12 @@ static int of_fpga_region_probe(struct platform_device *pdev) if (IS_ERR(mgr)) return -EPROBE_DEFER; - region = devm_fpga_region_create(dev, mgr, of_fpga_region_get_bridges); - if (!region) { - ret = -ENOMEM; + region = fpga_region_register(dev, mgr, of_fpga_region_get_bridges); + if (IS_ERR(region)) { + ret = PTR_ERR(region); goto eprobe_mgr_put; } - ret = fpga_region_register(region); - if (ret) - goto eprobe_mgr_put; - of_platform_populate(np, fpga_region_of_match, NULL, ®ion->dev); platform_set_drvdata(pdev, region); diff --git a/include/linux/fpga/fpga-region.h b/include/linux/fpga/fpga-region.h index 27cb706275db..3b87f232425c 100644 --- a/include/linux/fpga/fpga-region.h +++ b/include/linux/fpga/fpga-region.h @@ -7,6 +7,27 @@ #include #include +struct fpga_region; + +/** + * struct fpga_region_info - collection of parameters an FPGA Region + * @mgr: fpga region manager + * @compat_id: FPGA region id for compatibility check. + * @priv: fpga region private data + * @get_bridges: optional function to get bridges to a list + * + * fpga_region_info contains parameters for the register_full function. + * These are separated into an info structure because they some are optional + * others could be added to in the future. The info structure facilitates + * maintaining a stable API. + */ +struct fpga_region_info { + struct fpga_manager *mgr; + struct fpga_compat_id *compat_id; + void *priv; + int (*get_bridges)(struct fpga_region *region); +}; + /** * struct fpga_region - FPGA Region structure * @dev: FPGA Region device @@ -37,15 +58,12 @@ struct fpga_region *fpga_region_class_find( int fpga_region_program_fpga(struct fpga_region *region); -struct fpga_region -*fpga_region_create(struct device *dev, struct fpga_manager *mgr, - int (*get_bridges)(struct fpga_region *)); -void fpga_region_free(struct fpga_region *region); -int fpga_region_register(struct fpga_region *region); -void fpga_region_unregister(struct fpga_region *region); +struct fpga_region * +fpga_region_register_full(struct device *parent, const struct fpga_region_info *info); -struct fpga_region -*devm_fpga_region_create(struct device *dev, struct fpga_manager *mgr, - int (*get_bridges)(struct fpga_region *)); +struct fpga_region * +fpga_region_register(struct device *parent, struct fpga_manager *mgr, + int (*get_bridges)(struct fpga_region *)); +void fpga_region_unregister(struct fpga_region *region); #endif /* _FPGA_REGION_H */ -- cgit v1.2.3-59-g8ed1b From 306456c21c792ac633e660bc45f0854b612a0e98 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Tue, 16 Nov 2021 14:54:35 +0200 Subject: mfd: bd70528: Drop BD70528 support The only known BD70528 use-cases are such that the PMIC is controlled from separate MCU which is not running Linux. I am not aware of any Linux driver users. Furthermore, it seems there is no demand for this IC. Let's ease the maintenance burden and drop the driver. We can always add it back if there is sudden need for it. Signed-off-by: Matti Vaittinen Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/cf7dfd98b3403ad363b2b48b57bdbfd57a6416cb.1637066805.git.matti.vaittinen@fi.rohmeurope.com --- drivers/mfd/Kconfig | 17 -- drivers/mfd/Makefile | 1 - drivers/mfd/rohm-bd70528.c | 314 ------------------------------- include/linux/mfd/rohm-bd70528.h | 389 --------------------------------------- include/linux/mfd/rohm-generic.h | 1 - 5 files changed, 722 deletions(-) delete mode 100644 drivers/mfd/rohm-bd70528.c delete mode 100644 include/linux/mfd/rohm-bd70528.h (limited to 'include/linux') diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 3fb480818599..34c7d9d6b580 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -1945,23 +1945,6 @@ config MFD_ROHM_BD718XX NXP i.MX8. It contains 8 BUCK outputs and 7 LDOs, voltage monitoring and emergency shut down as well as 32,768KHz clock output. -config MFD_ROHM_BD70528 - tristate "ROHM BD70528 Power Management IC" - depends on I2C=y - depends on OF - select REGMAP_I2C - select REGMAP_IRQ - select MFD_CORE - help - Select this option to get support for the ROHM BD70528 Power - Management IC. BD71837 is general purpose single-chip power - management IC for battery-powered portable devices. It contains - 3 ultra-low current consumption buck converters, 3 LDOs and 2 LED - drivers. Also included are 4 GPIOs, a real-time clock (RTC), a 32kHz - crystal oscillator, high-accuracy VREF for use with an external ADC, - 10 bits SAR ADC for battery temperature monitor and 1S battery - charger. - config MFD_ROHM_BD71828 tristate "ROHM BD71828 and BD71815 Power Management IC" depends on I2C=y diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 0b1b629aef3e..4d53e951a92d 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -257,7 +257,6 @@ obj-$(CONFIG_MFD_STM32_TIMERS) += stm32-timers.o obj-$(CONFIG_MFD_MXS_LRADC) += mxs-lradc.o obj-$(CONFIG_MFD_SC27XX_PMIC) += sprd-sc27xx-spi.o obj-$(CONFIG_RAVE_SP_CORE) += rave-sp.o -obj-$(CONFIG_MFD_ROHM_BD70528) += rohm-bd70528.o obj-$(CONFIG_MFD_ROHM_BD71828) += rohm-bd71828.o obj-$(CONFIG_MFD_ROHM_BD718XX) += rohm-bd718x7.o obj-$(CONFIG_MFD_ROHM_BD957XMUF) += rohm-bd9576.o diff --git a/drivers/mfd/rohm-bd70528.c b/drivers/mfd/rohm-bd70528.c deleted file mode 100644 index 5c44d3b77b3e..000000000000 --- a/drivers/mfd/rohm-bd70528.c +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -// -// Copyright (C) 2019 ROHM Semiconductors -// -// ROHM BD70528 PMIC driver - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define BD70528_NUM_OF_GPIOS 4 - -static const struct resource rtc_irqs[] = { - DEFINE_RES_IRQ_NAMED(BD70528_INT_RTC_ALARM, "bd70528-rtc-alm"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_ELPS_TIM, "bd70528-elapsed-timer"), -}; - -static const struct resource charger_irqs[] = { - DEFINE_RES_IRQ_NAMED(BD70528_INT_BAT_OV_RES, "bd70528-bat-ov-res"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BAT_OV_DET, "bd70528-bat-ov-det"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DBAT_DET, "bd70528-bat-dead"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BATTSD_COLD_RES, "bd70528-bat-warmed"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BATTSD_COLD_DET, "bd70528-bat-cold"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BATTSD_HOT_RES, "bd70528-bat-cooled"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BATTSD_HOT_DET, "bd70528-bat-hot"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_CHG_TSD, "bd70528-chg-tshd"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BAT_RMV, "bd70528-bat-removed"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_BAT_DET, "bd70528-bat-detected"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN2_OV_RES, "bd70528-dcin2-ov-res"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN2_OV_DET, "bd70528-dcin2-ov-det"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN2_RMV, "bd70528-dcin2-removed"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN2_DET, "bd70528-dcin2-detected"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN1_RMV, "bd70528-dcin1-removed"), - DEFINE_RES_IRQ_NAMED(BD70528_INT_DCIN1_DET, "bd70528-dcin1-detected"), -}; - -static struct mfd_cell bd70528_mfd_cells[] = { - { .name = "bd70528-pmic", }, - { .name = "bd70528-gpio", }, - /* - * We use BD71837 driver to drive the clock block. Only differences to - * BD70528 clock gate are the register address and mask. - */ - { .name = "bd70528-clk", }, - { .name = "bd70528-wdt", }, - { - .name = "bd70528-power", - .resources = charger_irqs, - .num_resources = ARRAY_SIZE(charger_irqs), - }, { - .name = "bd70528-rtc", - .resources = rtc_irqs, - .num_resources = ARRAY_SIZE(rtc_irqs), - }, -}; - -static const struct regmap_range volatile_ranges[] = { - { - .range_min = BD70528_REG_INT_MAIN, - .range_max = BD70528_REG_INT_OP_FAIL, - }, { - .range_min = BD70528_REG_RTC_COUNT_H, - .range_max = BD70528_REG_RTC_ALM_REPEAT, - }, { - /* - * WDT control reg is special. Magic values must be written to - * it in order to change the control. Should not be cached. - */ - .range_min = BD70528_REG_WDT_CTRL, - .range_max = BD70528_REG_WDT_CTRL, - }, { - /* - * BD70528 also contains a few other registers which require - * magic sequences to be written in order to update the value. - * At least SHIPMODE, HWRESET, WARMRESET,and STANDBY - */ - .range_min = BD70528_REG_SHIPMODE, - .range_max = BD70528_REG_STANDBY, - }, -}; - -static const struct regmap_access_table volatile_regs = { - .yes_ranges = &volatile_ranges[0], - .n_yes_ranges = ARRAY_SIZE(volatile_ranges), -}; - -static struct regmap_config bd70528_regmap = { - .reg_bits = 8, - .val_bits = 8, - .volatile_table = &volatile_regs, - .max_register = BD70528_MAX_REGISTER, - .cache_type = REGCACHE_RBTREE, -}; - -/* - * Mapping of main IRQ register bits to sub-IRQ register offsets so that we can - * access corect sub-IRQ registers based on bits that are set in main IRQ - * register. - */ - -static unsigned int bit0_offsets[] = {0}; /* Shutdown */ -static unsigned int bit1_offsets[] = {1}; /* Power failure */ -static unsigned int bit2_offsets[] = {2}; /* VR FAULT */ -static unsigned int bit3_offsets[] = {3}; /* PMU interrupts */ -static unsigned int bit4_offsets[] = {4, 5}; /* Charger 1 and Charger 2 */ -static unsigned int bit5_offsets[] = {6}; /* RTC */ -static unsigned int bit6_offsets[] = {7}; /* GPIO */ -static unsigned int bit7_offsets[] = {8}; /* Invalid operation */ - -static struct regmap_irq_sub_irq_map bd70528_sub_irq_offsets[] = { - REGMAP_IRQ_MAIN_REG_OFFSET(bit0_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit1_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit2_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit3_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit4_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit5_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit6_offsets), - REGMAP_IRQ_MAIN_REG_OFFSET(bit7_offsets), -}; - -static struct regmap_irq bd70528_irqs[] = { - REGMAP_IRQ_REG(BD70528_INT_LONGPUSH, 0, BD70528_INT_LONGPUSH_MASK), - REGMAP_IRQ_REG(BD70528_INT_WDT, 0, BD70528_INT_WDT_MASK), - REGMAP_IRQ_REG(BD70528_INT_HWRESET, 0, BD70528_INT_HWRESET_MASK), - REGMAP_IRQ_REG(BD70528_INT_RSTB_FAULT, 0, BD70528_INT_RSTB_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_VBAT_UVLO, 0, BD70528_INT_VBAT_UVLO_MASK), - REGMAP_IRQ_REG(BD70528_INT_TSD, 0, BD70528_INT_TSD_MASK), - REGMAP_IRQ_REG(BD70528_INT_RSTIN, 0, BD70528_INT_RSTIN_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK1_FAULT, 1, - BD70528_INT_BUCK1_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK2_FAULT, 1, - BD70528_INT_BUCK2_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK3_FAULT, 1, - BD70528_INT_BUCK3_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_LDO1_FAULT, 1, BD70528_INT_LDO1_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_LDO2_FAULT, 1, BD70528_INT_LDO2_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_LDO3_FAULT, 1, BD70528_INT_LDO3_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED1_FAULT, 1, BD70528_INT_LED1_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED2_FAULT, 1, BD70528_INT_LED2_FAULT_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK1_OCP, 2, BD70528_INT_BUCK1_OCP_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK2_OCP, 2, BD70528_INT_BUCK2_OCP_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK3_OCP, 2, BD70528_INT_BUCK3_OCP_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED1_OCP, 2, BD70528_INT_LED1_OCP_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED2_OCP, 2, BD70528_INT_LED2_OCP_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK1_FULLON, 2, - BD70528_INT_BUCK1_FULLON_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK2_FULLON, 2, - BD70528_INT_BUCK2_FULLON_MASK), - REGMAP_IRQ_REG(BD70528_INT_SHORTPUSH, 3, BD70528_INT_SHORTPUSH_MASK), - REGMAP_IRQ_REG(BD70528_INT_AUTO_WAKEUP, 3, - BD70528_INT_AUTO_WAKEUP_MASK), - REGMAP_IRQ_REG(BD70528_INT_STATE_CHANGE, 3, - BD70528_INT_STATE_CHANGE_MASK), - REGMAP_IRQ_REG(BD70528_INT_BAT_OV_RES, 4, BD70528_INT_BAT_OV_RES_MASK), - REGMAP_IRQ_REG(BD70528_INT_BAT_OV_DET, 4, BD70528_INT_BAT_OV_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_DBAT_DET, 4, BD70528_INT_DBAT_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_BATTSD_COLD_RES, 4, - BD70528_INT_BATTSD_COLD_RES_MASK), - REGMAP_IRQ_REG(BD70528_INT_BATTSD_COLD_DET, 4, - BD70528_INT_BATTSD_COLD_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_BATTSD_HOT_RES, 4, - BD70528_INT_BATTSD_HOT_RES_MASK), - REGMAP_IRQ_REG(BD70528_INT_BATTSD_HOT_DET, 4, - BD70528_INT_BATTSD_HOT_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_CHG_TSD, 4, BD70528_INT_CHG_TSD_MASK), - REGMAP_IRQ_REG(BD70528_INT_BAT_RMV, 5, BD70528_INT_BAT_RMV_MASK), - REGMAP_IRQ_REG(BD70528_INT_BAT_DET, 5, BD70528_INT_BAT_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN2_OV_RES, 5, - BD70528_INT_DCIN2_OV_RES_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN2_OV_DET, 5, - BD70528_INT_DCIN2_OV_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN2_RMV, 5, BD70528_INT_DCIN2_RMV_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN2_DET, 5, BD70528_INT_DCIN2_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN1_RMV, 5, BD70528_INT_DCIN1_RMV_MASK), - REGMAP_IRQ_REG(BD70528_INT_DCIN1_DET, 5, BD70528_INT_DCIN1_DET_MASK), - REGMAP_IRQ_REG(BD70528_INT_RTC_ALARM, 6, BD70528_INT_RTC_ALARM_MASK), - REGMAP_IRQ_REG(BD70528_INT_ELPS_TIM, 6, BD70528_INT_ELPS_TIM_MASK), - REGMAP_IRQ_REG(BD70528_INT_GPIO0, 7, BD70528_INT_GPIO0_MASK), - REGMAP_IRQ_REG(BD70528_INT_GPIO1, 7, BD70528_INT_GPIO1_MASK), - REGMAP_IRQ_REG(BD70528_INT_GPIO2, 7, BD70528_INT_GPIO2_MASK), - REGMAP_IRQ_REG(BD70528_INT_GPIO3, 7, BD70528_INT_GPIO3_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK1_DVS_OPFAIL, 8, - BD70528_INT_BUCK1_DVS_OPFAIL_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK2_DVS_OPFAIL, 8, - BD70528_INT_BUCK2_DVS_OPFAIL_MASK), - REGMAP_IRQ_REG(BD70528_INT_BUCK3_DVS_OPFAIL, 8, - BD70528_INT_BUCK3_DVS_OPFAIL_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED1_VOLT_OPFAIL, 8, - BD70528_INT_LED1_VOLT_OPFAIL_MASK), - REGMAP_IRQ_REG(BD70528_INT_LED2_VOLT_OPFAIL, 8, - BD70528_INT_LED2_VOLT_OPFAIL_MASK), -}; - -static struct regmap_irq_chip bd70528_irq_chip = { - .name = "bd70528_irq", - .main_status = BD70528_REG_INT_MAIN, - .irqs = &bd70528_irqs[0], - .num_irqs = ARRAY_SIZE(bd70528_irqs), - .status_base = BD70528_REG_INT_SHDN, - .mask_base = BD70528_REG_INT_SHDN_MASK, - .ack_base = BD70528_REG_INT_SHDN, - .type_base = BD70528_REG_GPIO1_IN, - .init_ack_masked = true, - .num_regs = 9, - .num_main_regs = 1, - .num_type_reg = 4, - .sub_reg_offsets = &bd70528_sub_irq_offsets[0], - .num_main_status_bits = 8, - .irq_reg_stride = 1, -}; - -static int bd70528_i2c_probe(struct i2c_client *i2c, - const struct i2c_device_id *id) -{ - struct bd70528_data *bd70528; - struct regmap_irq_chip_data *irq_data; - int ret, i; - - if (!i2c->irq) { - dev_err(&i2c->dev, "No IRQ configured\n"); - return -EINVAL; - } - - bd70528 = devm_kzalloc(&i2c->dev, sizeof(*bd70528), GFP_KERNEL); - if (!bd70528) - return -ENOMEM; - - mutex_init(&bd70528->rtc_timer_lock); - - dev_set_drvdata(&i2c->dev, &bd70528->chip); - - bd70528->chip.regmap = devm_regmap_init_i2c(i2c, &bd70528_regmap); - if (IS_ERR(bd70528->chip.regmap)) { - dev_err(&i2c->dev, "Failed to initialize Regmap\n"); - return PTR_ERR(bd70528->chip.regmap); - } - - /* - * Disallow type setting for all IRQs by default as most of them do not - * support setting type. - */ - for (i = 0; i < ARRAY_SIZE(bd70528_irqs); i++) - bd70528_irqs[i].type.types_supported = 0; - - /* Set IRQ typesetting information for GPIO pins 0 - 3 */ - for (i = 0; i < BD70528_NUM_OF_GPIOS; i++) { - struct regmap_irq_type *type; - - type = &bd70528_irqs[BD70528_INT_GPIO0 + i].type; - type->type_reg_offset = 2 * i; - type->type_rising_val = 0x20; - type->type_falling_val = 0x10; - type->type_level_high_val = 0x40; - type->type_level_low_val = 0x50; - type->types_supported = (IRQ_TYPE_EDGE_BOTH | - IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW); - } - - ret = devm_regmap_add_irq_chip(&i2c->dev, bd70528->chip.regmap, - i2c->irq, IRQF_ONESHOT, 0, - &bd70528_irq_chip, &irq_data); - if (ret) { - dev_err(&i2c->dev, "Failed to add IRQ chip\n"); - return ret; - } - dev_dbg(&i2c->dev, "Registered %d IRQs for chip\n", - bd70528_irq_chip.num_irqs); - - /* - * BD70528 IRQ controller is not touching the main mask register. - * So enable the GPIO block interrupts at main level. We can just leave - * them enabled as the IRQ controller should disable IRQs from - * sub-registers when IRQ is disabled or freed. - */ - ret = regmap_update_bits(bd70528->chip.regmap, - BD70528_REG_INT_MAIN_MASK, - BD70528_INT_GPIO_MASK, 0); - - ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_AUTO, - bd70528_mfd_cells, - ARRAY_SIZE(bd70528_mfd_cells), NULL, 0, - regmap_irq_get_domain(irq_data)); - if (ret) - dev_err(&i2c->dev, "Failed to create subdevices\n"); - - return ret; -} - -static const struct of_device_id bd70528_of_match[] = { - { .compatible = "rohm,bd70528", }, - { }, -}; -MODULE_DEVICE_TABLE(of, bd70528_of_match); - -static struct i2c_driver bd70528_drv = { - .driver = { - .name = "rohm-bd70528", - .of_match_table = bd70528_of_match, - }, - .probe = &bd70528_i2c_probe, -}; - -module_i2c_driver(bd70528_drv); - -MODULE_AUTHOR("Matti Vaittinen "); -MODULE_DESCRIPTION("ROHM BD70528 Power Management IC driver"); -MODULE_LICENSE("GPL"); diff --git a/include/linux/mfd/rohm-bd70528.h b/include/linux/mfd/rohm-bd70528.h deleted file mode 100644 index 4a5966475a35..000000000000 --- a/include/linux/mfd/rohm-bd70528.h +++ /dev/null @@ -1,389 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Copyright (C) 2018 ROHM Semiconductors */ - -#ifndef __LINUX_MFD_BD70528_H__ -#define __LINUX_MFD_BD70528_H__ - -#include -#include -#include -#include -#include - -enum { - BD70528_BUCK1, - BD70528_BUCK2, - BD70528_BUCK3, - BD70528_LDO1, - BD70528_LDO2, - BD70528_LDO3, - BD70528_LED1, - BD70528_LED2, -}; - -struct bd70528_data { - struct rohm_regmap_dev chip; - struct mutex rtc_timer_lock; -}; - -#define BD70528_BUCK_VOLTS 0x10 -#define BD70528_LDO_VOLTS 0x20 - -#define BD70528_REG_BUCK1_EN 0x0F -#define BD70528_REG_BUCK1_VOLT 0x15 -#define BD70528_REG_BUCK2_EN 0x10 -#define BD70528_REG_BUCK2_VOLT 0x16 -#define BD70528_REG_BUCK3_EN 0x11 -#define BD70528_REG_BUCK3_VOLT 0x17 -#define BD70528_REG_LDO1_EN 0x1b -#define BD70528_REG_LDO1_VOLT 0x1e -#define BD70528_REG_LDO2_EN 0x1c -#define BD70528_REG_LDO2_VOLT 0x1f -#define BD70528_REG_LDO3_EN 0x1d -#define BD70528_REG_LDO3_VOLT 0x20 -#define BD70528_REG_LED_CTRL 0x2b -#define BD70528_REG_LED_VOLT 0x29 -#define BD70528_REG_LED_EN 0x2a - -/* main irq registers */ -#define BD70528_REG_INT_MAIN 0x7E -#define BD70528_REG_INT_MAIN_MASK 0x74 - -/* 'sub irq' registers */ -#define BD70528_REG_INT_SHDN 0x7F -#define BD70528_REG_INT_PWR_FLT 0x80 -#define BD70528_REG_INT_VR_FLT 0x81 -#define BD70528_REG_INT_MISC 0x82 -#define BD70528_REG_INT_BAT1 0x83 -#define BD70528_REG_INT_BAT2 0x84 -#define BD70528_REG_INT_RTC 0x85 -#define BD70528_REG_INT_GPIO 0x86 -#define BD70528_REG_INT_OP_FAIL 0x87 - -#define BD70528_REG_INT_SHDN_MASK 0x75 -#define BD70528_REG_INT_PWR_FLT_MASK 0x76 -#define BD70528_REG_INT_VR_FLT_MASK 0x77 -#define BD70528_REG_INT_MISC_MASK 0x78 -#define BD70528_REG_INT_BAT1_MASK 0x79 -#define BD70528_REG_INT_BAT2_MASK 0x7a -#define BD70528_REG_INT_RTC_MASK 0x7b -#define BD70528_REG_INT_GPIO_MASK 0x7c -#define BD70528_REG_INT_OP_FAIL_MASK 0x7d - -/* Reset related 'magic' registers */ -#define BD70528_REG_SHIPMODE 0x03 -#define BD70528_REG_HWRESET 0x04 -#define BD70528_REG_WARMRESET 0x05 -#define BD70528_REG_STANDBY 0x06 - -/* GPIO registers */ -#define BD70528_REG_GPIO_STATE 0x8F - -#define BD70528_REG_GPIO1_IN 0x4d -#define BD70528_REG_GPIO2_IN 0x4f -#define BD70528_REG_GPIO3_IN 0x51 -#define BD70528_REG_GPIO4_IN 0x53 -#define BD70528_REG_GPIO1_OUT 0x4e -#define BD70528_REG_GPIO2_OUT 0x50 -#define BD70528_REG_GPIO3_OUT 0x52 -#define BD70528_REG_GPIO4_OUT 0x54 - -/* RTC */ - -#define BD70528_REG_RTC_COUNT_H 0x2d -#define BD70528_REG_RTC_COUNT_L 0x2e -#define BD70528_REG_RTC_SEC 0x2f -#define BD70528_REG_RTC_MINUTE 0x30 -#define BD70528_REG_RTC_HOUR 0x31 -#define BD70528_REG_RTC_WEEK 0x32 -#define BD70528_REG_RTC_DAY 0x33 -#define BD70528_REG_RTC_MONTH 0x34 -#define BD70528_REG_RTC_YEAR 0x35 - -#define BD70528_REG_RTC_ALM_SEC 0x36 -#define BD70528_REG_RTC_ALM_START BD70528_REG_RTC_ALM_SEC -#define BD70528_REG_RTC_ALM_MINUTE 0x37 -#define BD70528_REG_RTC_ALM_HOUR 0x38 -#define BD70528_REG_RTC_ALM_WEEK 0x39 -#define BD70528_REG_RTC_ALM_DAY 0x3a -#define BD70528_REG_RTC_ALM_MONTH 0x3b -#define BD70528_REG_RTC_ALM_YEAR 0x3c -#define BD70528_REG_RTC_ALM_MASK 0x3d -#define BD70528_REG_RTC_ALM_REPEAT 0x3e -#define BD70528_REG_RTC_START BD70528_REG_RTC_SEC - -#define BD70528_REG_RTC_WAKE_SEC 0x43 -#define BD70528_REG_RTC_WAKE_START BD70528_REG_RTC_WAKE_SEC -#define BD70528_REG_RTC_WAKE_MIN 0x44 -#define BD70528_REG_RTC_WAKE_HOUR 0x45 -#define BD70528_REG_RTC_WAKE_CTRL 0x46 - -#define BD70528_REG_ELAPSED_TIMER_EN 0x42 -#define BD70528_REG_WAKE_EN 0x46 - -/* WDT registers */ -#define BD70528_REG_WDT_CTRL 0x4A -#define BD70528_REG_WDT_HOUR 0x49 -#define BD70528_REG_WDT_MINUTE 0x48 -#define BD70528_REG_WDT_SEC 0x47 - -/* Charger / Battery */ -#define BD70528_REG_CHG_CURR_STAT 0x59 -#define BD70528_REG_CHG_BAT_STAT 0x57 -#define BD70528_REG_CHG_BAT_TEMP 0x58 -#define BD70528_REG_CHG_IN_STAT 0x56 -#define BD70528_REG_CHG_DCIN_ILIM 0x5d -#define BD70528_REG_CHG_CHG_CURR_WARM 0x61 -#define BD70528_REG_CHG_CHG_CURR_COLD 0x62 - -/* Masks for main IRQ register bits */ -enum { - BD70528_INT_SHDN, -#define BD70528_INT_SHDN_MASK BIT(BD70528_INT_SHDN) - BD70528_INT_PWR_FLT, -#define BD70528_INT_PWR_FLT_MASK BIT(BD70528_INT_PWR_FLT) - BD70528_INT_VR_FLT, -#define BD70528_INT_VR_FLT_MASK BIT(BD70528_INT_VR_FLT) - BD70528_INT_MISC, -#define BD70528_INT_MISC_MASK BIT(BD70528_INT_MISC) - BD70528_INT_BAT1, -#define BD70528_INT_BAT1_MASK BIT(BD70528_INT_BAT1) - BD70528_INT_RTC, -#define BD70528_INT_RTC_MASK BIT(BD70528_INT_RTC) - BD70528_INT_GPIO, -#define BD70528_INT_GPIO_MASK BIT(BD70528_INT_GPIO) - BD70528_INT_OP_FAIL, -#define BD70528_INT_OP_FAIL_MASK BIT(BD70528_INT_OP_FAIL) -}; - -/* IRQs */ -enum { - /* Shutdown register IRQs */ - BD70528_INT_LONGPUSH, - BD70528_INT_WDT, - BD70528_INT_HWRESET, - BD70528_INT_RSTB_FAULT, - BD70528_INT_VBAT_UVLO, - BD70528_INT_TSD, - BD70528_INT_RSTIN, - /* Power failure register IRQs */ - BD70528_INT_BUCK1_FAULT, - BD70528_INT_BUCK2_FAULT, - BD70528_INT_BUCK3_FAULT, - BD70528_INT_LDO1_FAULT, - BD70528_INT_LDO2_FAULT, - BD70528_INT_LDO3_FAULT, - BD70528_INT_LED1_FAULT, - BD70528_INT_LED2_FAULT, - /* VR FAULT register IRQs */ - BD70528_INT_BUCK1_OCP, - BD70528_INT_BUCK2_OCP, - BD70528_INT_BUCK3_OCP, - BD70528_INT_LED1_OCP, - BD70528_INT_LED2_OCP, - BD70528_INT_BUCK1_FULLON, - BD70528_INT_BUCK2_FULLON, - /* PMU register interrupts */ - BD70528_INT_SHORTPUSH, - BD70528_INT_AUTO_WAKEUP, - BD70528_INT_STATE_CHANGE, - /* Charger 1 register IRQs */ - BD70528_INT_BAT_OV_RES, - BD70528_INT_BAT_OV_DET, - BD70528_INT_DBAT_DET, - BD70528_INT_BATTSD_COLD_RES, - BD70528_INT_BATTSD_COLD_DET, - BD70528_INT_BATTSD_HOT_RES, - BD70528_INT_BATTSD_HOT_DET, - BD70528_INT_CHG_TSD, - /* Charger 2 register IRQs */ - BD70528_INT_BAT_RMV, - BD70528_INT_BAT_DET, - BD70528_INT_DCIN2_OV_RES, - BD70528_INT_DCIN2_OV_DET, - BD70528_INT_DCIN2_RMV, - BD70528_INT_DCIN2_DET, - BD70528_INT_DCIN1_RMV, - BD70528_INT_DCIN1_DET, - /* RTC register IRQs */ - BD70528_INT_RTC_ALARM, - BD70528_INT_ELPS_TIM, - /* GPIO register IRQs */ - BD70528_INT_GPIO0, - BD70528_INT_GPIO1, - BD70528_INT_GPIO2, - BD70528_INT_GPIO3, - /* Invalid operation register IRQs */ - BD70528_INT_BUCK1_DVS_OPFAIL, - BD70528_INT_BUCK2_DVS_OPFAIL, - BD70528_INT_BUCK3_DVS_OPFAIL, - BD70528_INT_LED1_VOLT_OPFAIL, - BD70528_INT_LED2_VOLT_OPFAIL, -}; - -/* Masks */ -#define BD70528_INT_LONGPUSH_MASK 0x1 -#define BD70528_INT_WDT_MASK 0x2 -#define BD70528_INT_HWRESET_MASK 0x4 -#define BD70528_INT_RSTB_FAULT_MASK 0x8 -#define BD70528_INT_VBAT_UVLO_MASK 0x10 -#define BD70528_INT_TSD_MASK 0x20 -#define BD70528_INT_RSTIN_MASK 0x40 - -#define BD70528_INT_BUCK1_FAULT_MASK 0x1 -#define BD70528_INT_BUCK2_FAULT_MASK 0x2 -#define BD70528_INT_BUCK3_FAULT_MASK 0x4 -#define BD70528_INT_LDO1_FAULT_MASK 0x8 -#define BD70528_INT_LDO2_FAULT_MASK 0x10 -#define BD70528_INT_LDO3_FAULT_MASK 0x20 -#define BD70528_INT_LED1_FAULT_MASK 0x40 -#define BD70528_INT_LED2_FAULT_MASK 0x80 - -#define BD70528_INT_BUCK1_OCP_MASK 0x1 -#define BD70528_INT_BUCK2_OCP_MASK 0x2 -#define BD70528_INT_BUCK3_OCP_MASK 0x4 -#define BD70528_INT_LED1_OCP_MASK 0x8 -#define BD70528_INT_LED2_OCP_MASK 0x10 -#define BD70528_INT_BUCK1_FULLON_MASK 0x20 -#define BD70528_INT_BUCK2_FULLON_MASK 0x40 - -#define BD70528_INT_SHORTPUSH_MASK 0x1 -#define BD70528_INT_AUTO_WAKEUP_MASK 0x2 -#define BD70528_INT_STATE_CHANGE_MASK 0x10 - -#define BD70528_INT_BAT_OV_RES_MASK 0x1 -#define BD70528_INT_BAT_OV_DET_MASK 0x2 -#define BD70528_INT_DBAT_DET_MASK 0x4 -#define BD70528_INT_BATTSD_COLD_RES_MASK 0x8 -#define BD70528_INT_BATTSD_COLD_DET_MASK 0x10 -#define BD70528_INT_BATTSD_HOT_RES_MASK 0x20 -#define BD70528_INT_BATTSD_HOT_DET_MASK 0x40 -#define BD70528_INT_CHG_TSD_MASK 0x80 - -#define BD70528_INT_BAT_RMV_MASK 0x1 -#define BD70528_INT_BAT_DET_MASK 0x2 -#define BD70528_INT_DCIN2_OV_RES_MASK 0x4 -#define BD70528_INT_DCIN2_OV_DET_MASK 0x8 -#define BD70528_INT_DCIN2_RMV_MASK 0x10 -#define BD70528_INT_DCIN2_DET_MASK 0x20 -#define BD70528_INT_DCIN1_RMV_MASK 0x40 -#define BD70528_INT_DCIN1_DET_MASK 0x80 - -#define BD70528_INT_RTC_ALARM_MASK 0x1 -#define BD70528_INT_ELPS_TIM_MASK 0x2 - -#define BD70528_INT_GPIO0_MASK 0x1 -#define BD70528_INT_GPIO1_MASK 0x2 -#define BD70528_INT_GPIO2_MASK 0x4 -#define BD70528_INT_GPIO3_MASK 0x8 - -#define BD70528_INT_BUCK1_DVS_OPFAIL_MASK 0x1 -#define BD70528_INT_BUCK2_DVS_OPFAIL_MASK 0x2 -#define BD70528_INT_BUCK3_DVS_OPFAIL_MASK 0x4 -#define BD70528_INT_LED1_VOLT_OPFAIL_MASK 0x10 -#define BD70528_INT_LED2_VOLT_OPFAIL_MASK 0x20 - -#define BD70528_DEBOUNCE_MASK 0x3 - -#define BD70528_DEBOUNCE_DISABLE 0 -#define BD70528_DEBOUNCE_15MS 1 -#define BD70528_DEBOUNCE_30MS 2 -#define BD70528_DEBOUNCE_50MS 3 - -#define BD70528_GPIO_DRIVE_MASK 0x2 -#define BD70528_GPIO_PUSH_PULL 0x0 -#define BD70528_GPIO_OPEN_DRAIN 0x2 - -#define BD70528_GPIO_OUT_EN_MASK 0x80 -#define BD70528_GPIO_OUT_ENABLE 0x80 -#define BD70528_GPIO_OUT_DISABLE 0x0 - -#define BD70528_GPIO_OUT_HI 0x1 -#define BD70528_GPIO_OUT_LO 0x0 -#define BD70528_GPIO_OUT_MASK 0x1 - -#define BD70528_GPIO_IN_STATE_BASE 1 - -/* RTC masks to mask out reserved bits */ - -#define BD70528_MASK_ELAPSED_TIMER_EN 0x1 -/* Mask second, min and hour fields - * HW would support ALM irq for over 24h - * (by setting day, month and year too) - * but as we wish to keep this same as for - * wake-up we limit ALM to 24H and only - * unmask sec, min and hour - */ -#define BD70528_MASK_WAKE_EN 0x1 - -/* WDT masks */ -#define BD70528_MASK_WDT_EN 0x1 -#define BD70528_MASK_WDT_HOUR 0x1 -#define BD70528_MASK_WDT_MINUTE 0x7f -#define BD70528_MASK_WDT_SEC 0x7f - -#define BD70528_WDT_STATE_BIT 0x1 -#define BD70528_ELAPSED_STATE_BIT 0x2 -#define BD70528_WAKE_STATE_BIT 0x4 - -/* Charger masks */ -#define BD70528_MASK_CHG_STAT 0x7f -#define BD70528_MASK_CHG_BAT_TIMER 0x20 -#define BD70528_MASK_CHG_BAT_OVERVOLT 0x10 -#define BD70528_MASK_CHG_BAT_DETECT 0x1 -#define BD70528_MASK_CHG_DCIN1_UVLO 0x1 -#define BD70528_MASK_CHG_DCIN_ILIM 0x3f -#define BD70528_MASK_CHG_CHG_CURR 0x1f -#define BD70528_MASK_CHG_TRICKLE_CURR 0x10 - -/* - * Note, external battery register is the lonely rider at - * address 0xc5. See how to stuff that in the regmap - */ -#define BD70528_MAX_REGISTER 0x94 - -/* Buck control masks */ -#define BD70528_MASK_RUN_EN 0x4 -#define BD70528_MASK_STBY_EN 0x2 -#define BD70528_MASK_IDLE_EN 0x1 -#define BD70528_MASK_LED1_EN 0x1 -#define BD70528_MASK_LED2_EN 0x10 - -#define BD70528_MASK_BUCK_VOLT 0xf -#define BD70528_MASK_LDO_VOLT 0x1f -#define BD70528_MASK_LED1_VOLT 0x1 -#define BD70528_MASK_LED2_VOLT 0x10 - -/* Misc irq masks */ -#define BD70528_INT_MASK_SHORT_PUSH 1 -#define BD70528_INT_MASK_AUTO_WAKE 2 -#define BD70528_INT_MASK_POWER_STATE 4 - -#define BD70528_MASK_BUCK_RAMP 0x10 -#define BD70528_SIFT_BUCK_RAMP 4 - -#if IS_ENABLED(CONFIG_BD70528_WATCHDOG) - -int bd70528_wdt_set(struct rohm_regmap_dev *data, int enable, int *old_state); -void bd70528_wdt_lock(struct rohm_regmap_dev *data); -void bd70528_wdt_unlock(struct rohm_regmap_dev *data); - -#else /* CONFIG_BD70528_WATCHDOG */ - -static inline int bd70528_wdt_set(struct rohm_regmap_dev *data, int enable, - int *old_state) -{ - return 0; -} - -static inline void bd70528_wdt_lock(struct rohm_regmap_dev *data) -{ -} - -static inline void bd70528_wdt_unlock(struct rohm_regmap_dev *data) -{ -} - -#endif /* CONFIG_BD70528_WATCHDOG */ - -#endif /* __LINUX_MFD_BD70528_H__ */ diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h index 35b392a0d73a..8fb763a2265a 100644 --- a/include/linux/mfd/rohm-generic.h +++ b/include/linux/mfd/rohm-generic.h @@ -12,7 +12,6 @@ enum rohm_chip_type { ROHM_CHIP_TYPE_BD9573, ROHM_CHIP_TYPE_BD9574, ROHM_CHIP_TYPE_BD9576, - ROHM_CHIP_TYPE_BD70528, ROHM_CHIP_TYPE_BD71815, ROHM_CHIP_TYPE_BD71828, ROHM_CHIP_TYPE_BD71837, -- cgit v1.2.3-59-g8ed1b From a9c8f68ce2c37ced2f7a8667eda71b7753ede398 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Nov 2021 21:27:22 +0200 Subject: spi: pxa2xx: Get rid of unused ->cs_control() Since the last user of the custom ->cs_control() gone, we may get rid of this legacy API completely. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20211123192723.44537-2-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- Documentation/spi/pxa2xx.rst | 29 +++-------------------------- arch/arm/mach-pxa/stargate2.c | 2 +- drivers/spi/spi-pxa2xx.c | 18 ------------------ drivers/spi/spi-pxa2xx.h | 3 --- include/linux/spi/pxa2xx_spi.h | 4 ---- 5 files changed, 4 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/Documentation/spi/pxa2xx.rst b/Documentation/spi/pxa2xx.rst index 6312968acfe9..dfc7673ed15d 100644 --- a/Documentation/spi/pxa2xx.rst +++ b/Documentation/spi/pxa2xx.rst @@ -102,7 +102,7 @@ device. All fields are optional. u8 dma_burst_size; u32 timeout; u8 enable_loopback; - void (*cs_control)(u32 command); + int gpio_cs; }; The "pxa2xx_spi_chip.tx_threshold" and "pxa2xx_spi_chip.rx_threshold" fields are @@ -133,11 +133,6 @@ into internal loopback mode. In this mode the SSP controller internally connects the SSPTX pin to the SSPRX pin. This is useful for initial setup testing. -The "pxa2xx_spi_chip.cs_control" field is used to point to a board specific -function for asserting/deasserting a slave device chip select. If the field is -NULL, the pxa2xx_spi master controller driver assumes that the SSP port is -configured to use GPIO or SSPFRM instead. - NOTE: the SPI driver cannot control the chip select if SSPFRM is used, so the chipselect is dropped after each spi_transfer. Most devices need chip select asserted around the complete message. Use SSPFRM as a GPIO (through a descriptor) @@ -152,30 +147,12 @@ field. Below is a sample configuration using the PXA255 NSSP. :: - /* Chip Select control for the CS8415A SPI slave device */ - static void cs8415a_cs_control(u32 command) - { - if (command & PXA2XX_CS_ASSERT) - GPCR(2) = GPIO_bit(2); - else - GPSR(2) = GPIO_bit(2); - } - - /* Chip Select control for the CS8405A SPI slave device */ - static void cs8405a_cs_control(u32 command) - { - if (command & PXA2XX_CS_ASSERT) - GPCR(3) = GPIO_bit(3); - else - GPSR(3) = GPIO_bit(3); - } - static struct pxa2xx_spi_chip cs8415a_chip_info = { .tx_threshold = 8, /* SSP hardward FIFO threshold */ .rx_threshold = 8, /* SSP hardward FIFO threshold */ .dma_burst_size = 8, /* Byte wide transfers used so 8 byte bursts */ .timeout = 235, /* See Intel documentation */ - .cs_control = cs8415a_cs_control, /* Use external chip select */ + .gpio_cs = 2, /* Use external chip select */ }; static struct pxa2xx_spi_chip cs8405a_chip_info = { @@ -183,7 +160,7 @@ field. Below is a sample configuration using the PXA255 NSSP. .rx_threshold = 8, /* SSP hardward FIFO threshold */ .dma_burst_size = 8, /* Byte wide transfers used so 8 byte bursts */ .timeout = 235, /* See Intel documentation */ - .cs_control = cs8405a_cs_control, /* Use external chip select */ + .gpio_cs = 3, /* Use external chip select */ }; static struct spi_board_info streetracer_spi_board_info[] __initdata = { diff --git a/arch/arm/mach-pxa/stargate2.c b/arch/arm/mach-pxa/stargate2.c index 7ad627465768..8ca02ec1d44c 100644 --- a/arch/arm/mach-pxa/stargate2.c +++ b/arch/arm/mach-pxa/stargate2.c @@ -347,7 +347,7 @@ static struct pxa2xx_spi_controller pxa_ssp_master_2_info = { }; /* An upcoming kernel change will scrap SFRM usage so these - * drivers have been moved to use gpio's via cs_control */ + * drivers have been moved to use GPIOs */ static struct pxa2xx_spi_chip staccel_chip_info = { .tx_threshold = 8, .rx_threshold = 8, diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c index ee3297dd532e..24196156c609 100644 --- a/drivers/spi/spi-pxa2xx.c +++ b/drivers/spi/spi-pxa2xx.c @@ -427,7 +427,6 @@ static void lpss_ssp_cs_control(struct spi_device *spi, bool enable) static void cs_assert(struct spi_device *spi) { - struct chip_data *chip = spi_get_ctldata(spi); struct driver_data *drv_data = spi_controller_get_devdata(spi->controller); @@ -436,18 +435,12 @@ static void cs_assert(struct spi_device *spi) return; } - if (chip->cs_control) { - chip->cs_control(PXA2XX_CS_ASSERT); - return; - } - if (is_lpss_ssp(drv_data)) lpss_ssp_cs_control(spi, true); } static void cs_deassert(struct spi_device *spi) { - struct chip_data *chip = spi_get_ctldata(spi); struct driver_data *drv_data = spi_controller_get_devdata(spi->controller); unsigned long timeout; @@ -461,11 +454,6 @@ static void cs_deassert(struct spi_device *spi) !time_after(jiffies, timeout)) cpu_relax(); - if (chip->cs_control) { - chip->cs_control(PXA2XX_CS_DEASSERT); - return; - } - if (is_lpss_ssp(drv_data)) lpss_ssp_cs_control(spi, false); } @@ -1204,12 +1192,6 @@ static int setup_cs(struct spi_device *spi, struct chip_data *chip, */ cleanup_cs(spi); - /* If ->cs_control() is provided, ignore GPIO chip select */ - if (chip_info->cs_control) { - chip->cs_control = chip_info->cs_control; - return 0; - } - if (gpio_is_valid(chip_info->gpio_cs)) { int gpio = chip_info->gpio_cs; int err; diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h index 4d77f4de6eda..45cdbbc71c4b 100644 --- a/drivers/spi/spi-pxa2xx.h +++ b/drivers/spi/spi-pxa2xx.h @@ -49,7 +49,6 @@ struct driver_data { int (*write)(struct driver_data *drv_data); int (*read)(struct driver_data *drv_data); irqreturn_t (*transfer_handler)(struct driver_data *drv_data); - void (*cs_control)(u32 command); void __iomem *lpss_base; @@ -67,8 +66,6 @@ struct chip_data { u32 threshold; u16 lpss_rx_threshold; u16 lpss_tx_threshold; - - void (*cs_control)(u32 command); }; static inline u32 pxa2xx_spi_read(const struct driver_data *drv_data, u32 reg) diff --git a/include/linux/spi/pxa2xx_spi.h b/include/linux/spi/pxa2xx_spi.h index eaab121ee575..42e06bfbc2a4 100644 --- a/include/linux/spi/pxa2xx_spi.h +++ b/include/linux/spi/pxa2xx_spi.h @@ -9,9 +9,6 @@ #include -#define PXA2XX_CS_ASSERT (0x01) -#define PXA2XX_CS_DEASSERT (0x02) - struct dma_chan; /* @@ -47,7 +44,6 @@ struct pxa2xx_spi_chip { u32 timeout; u8 enable_loopback; int gpio_cs; - void (*cs_control)(u32 command); }; #if defined(CONFIG_ARCH_PXA) || defined(CONFIG_ARCH_MMP) -- cgit v1.2.3-59-g8ed1b From 8393961c53b31078cfc877bc00eb0f67e1474edd Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 23 Nov 2021 21:27:23 +0200 Subject: spi: pxa2xx: Get rid of unused enable_loopback member There is no user of the enable_loopback member in the struct pxa2xx_spi_chip. Remote this legacy member completely. The mentioned in the documentation the testing phase can be performed with spidev_test tool. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20211123192723.44537-3-andriy.shevchenko@linux.intel.com Signed-off-by: Mark Brown --- Documentation/spi/pxa2xx.rst | 6 ------ drivers/spi/spi-pxa2xx.c | 5 ++--- include/linux/spi/pxa2xx_spi.h | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/Documentation/spi/pxa2xx.rst b/Documentation/spi/pxa2xx.rst index dfc7673ed15d..6347580826be 100644 --- a/Documentation/spi/pxa2xx.rst +++ b/Documentation/spi/pxa2xx.rst @@ -101,7 +101,6 @@ device. All fields are optional. u8 rx_threshold; u8 dma_burst_size; u32 timeout; - u8 enable_loopback; int gpio_cs; }; @@ -128,11 +127,6 @@ dependent on the SPI bus speed ("spi_board_info.max_speed_hz") and the specific slave device. Please note that the PXA2xx SSP 1 does not support trailing byte timeouts and must busy-wait any trailing bytes. -The "pxa2xx_spi_chip.enable_loopback" field is used to place the SSP porting -into internal loopback mode. In this mode the SSP controller internally -connects the SSPTX pin to the SSPRX pin. This is useful for initial setup -testing. - NOTE: the SPI driver cannot control the chip select if SSPFRM is used, so the chipselect is dropped after each spi_transfer. Most devices need chip select asserted around the complete message. Use SSPFRM as a GPIO (through a descriptor) diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c index 24196156c609..b3186bd0c2a8 100644 --- a/drivers/spi/spi-pxa2xx.c +++ b/drivers/spi/spi-pxa2xx.c @@ -1289,7 +1289,6 @@ static int setup(struct spi_device *spi) chip_info = spi->controller_data; /* chip_info isn't always needed */ - chip->cr1 = 0; if (chip_info) { if (chip_info->timeout) chip->timeout = chip_info->timeout; @@ -1300,9 +1299,9 @@ static int setup(struct spi_device *spi) if (chip_info->rx_threshold) rx_thres = chip_info->rx_threshold; chip->dma_threshold = 0; - if (chip_info->enable_loopback) - chip->cr1 = SSCR1_LBM; } + + chip->cr1 = 0; if (spi_controller_is_slave(drv_data->controller)) { chip->cr1 |= SSCR1_SCFR; chip->cr1 |= SSCR1_SCLKDIR; diff --git a/include/linux/spi/pxa2xx_spi.h b/include/linux/spi/pxa2xx_spi.h index 42e06bfbc2a4..ca74dce36706 100644 --- a/include/linux/spi/pxa2xx_spi.h +++ b/include/linux/spi/pxa2xx_spi.h @@ -42,7 +42,6 @@ struct pxa2xx_spi_chip { u8 rx_threshold; u8 dma_burst_size; u32 timeout; - u8 enable_loopback; int gpio_cs; }; -- cgit v1.2.3-59-g8ed1b From b99658452355d316debee11079e8f1c6c1029355 Mon Sep 17 00:00:00 2001 From: Colin Foster Date: Sun, 28 Nov 2021 17:57:37 -0800 Subject: net: dsa: ocelot: felix: utilize shared mscc-miim driver for indirect MDIO access Switch to a shared MDIO access implementation by way of the mdio-mscc-miim driver. Signed-off-by: Colin Foster Tested-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/ocelot/Kconfig | 1 + drivers/net/dsa/ocelot/seville_vsc9953.c | 102 +++---------------------------- drivers/net/mdio/mdio-mscc-miim.c | 38 ++++++++---- include/linux/mdio/mdio-mscc-miim.h | 19 ++++++ 4 files changed, 56 insertions(+), 104 deletions(-) create mode 100644 include/linux/mdio/mdio-mscc-miim.h (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/Kconfig b/drivers/net/dsa/ocelot/Kconfig index 9948544ba1c4..220b0b027b55 100644 --- a/drivers/net/dsa/ocelot/Kconfig +++ b/drivers/net/dsa/ocelot/Kconfig @@ -21,6 +21,7 @@ config NET_DSA_MSCC_SEVILLE depends on NET_VENDOR_MICROSEMI depends on HAS_IOMEM depends on PTP_1588_CLOCK_OPTIONAL + select MDIO_MSCC_MIIM select MSCC_OCELOT_SWITCH_LIB select NET_DSA_TAG_OCELOT_8021Q select NET_DSA_TAG_OCELOT diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index db124922c374..b9be889016ce 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -6,19 +6,14 @@ #include #include #include +#include +#include #include #include #include #include -#include #include "felix.h" -#define MSCC_MIIM_CMD_OPR_WRITE BIT(1) -#define MSCC_MIIM_CMD_OPR_READ BIT(2) -#define MSCC_MIIM_CMD_WRDATA_SHIFT 4 -#define MSCC_MIIM_CMD_REGAD_SHIFT 20 -#define MSCC_MIIM_CMD_PHYAD_SHIFT 25 -#define MSCC_MIIM_CMD_VLD BIT(31) #define VSC9953_VCAP_POLICER_BASE 11 #define VSC9953_VCAP_POLICER_MAX 31 #define VSC9953_VCAP_POLICER_BASE2 120 @@ -862,7 +857,6 @@ static struct vcap_props vsc9953_vcap_props[] = { #define VSC9953_INIT_TIMEOUT 50000 #define VSC9953_GCB_RST_SLEEP 100 #define VSC9953_SYS_RAMINIT_SLEEP 80 -#define VCS9953_MII_TIMEOUT 10000 static int vsc9953_gcb_soft_rst_status(struct ocelot *ocelot) { @@ -882,82 +876,6 @@ static int vsc9953_sys_ram_init_status(struct ocelot *ocelot) return val; } -static int vsc9953_gcb_miim_pending_status(struct ocelot *ocelot) -{ - int val; - - ocelot_field_read(ocelot, GCB_MIIM_MII_STATUS_PENDING, &val); - - return val; -} - -static int vsc9953_gcb_miim_busy_status(struct ocelot *ocelot) -{ - int val; - - ocelot_field_read(ocelot, GCB_MIIM_MII_STATUS_BUSY, &val); - - return val; -} - -static int vsc9953_mdio_write(struct mii_bus *bus, int phy_id, int regnum, - u16 value) -{ - struct ocelot *ocelot = bus->priv; - int err, cmd, val; - - /* Wait while MIIM controller becomes idle */ - err = readx_poll_timeout(vsc9953_gcb_miim_pending_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO write: pending timeout\n"); - goto out; - } - - cmd = MSCC_MIIM_CMD_VLD | (phy_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | - (value << MSCC_MIIM_CMD_WRDATA_SHIFT) | - MSCC_MIIM_CMD_OPR_WRITE; - - ocelot_write(ocelot, cmd, GCB_MIIM_MII_CMD); - -out: - return err; -} - -static int vsc9953_mdio_read(struct mii_bus *bus, int phy_id, int regnum) -{ - struct ocelot *ocelot = bus->priv; - int err, cmd, val; - - /* Wait until MIIM controller becomes idle */ - err = readx_poll_timeout(vsc9953_gcb_miim_pending_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO read: pending timeout\n"); - goto out; - } - - /* Write the MIIM COMMAND register */ - cmd = MSCC_MIIM_CMD_VLD | (phy_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | MSCC_MIIM_CMD_OPR_READ; - - ocelot_write(ocelot, cmd, GCB_MIIM_MII_CMD); - - /* Wait while read operation via the MIIM controller is in progress */ - err = readx_poll_timeout(vsc9953_gcb_miim_busy_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO read: busy timeout\n"); - goto out; - } - - val = ocelot_read(ocelot, GCB_MIIM_MII_DATA); - - err = val & 0xFFFF; -out: - return err; -} /* CORE_ENA is in SYS:SYSTEM:RESET_CFG * MEM_INIT is in SYS:SYSTEM:RESET_CFG @@ -1101,16 +1019,14 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) return -ENOMEM; } - bus = devm_mdiobus_alloc(dev); - if (!bus) - return -ENOMEM; + rc = mscc_miim_setup(dev, &bus, "VSC9953 internal MDIO bus", + ocelot->targets[GCB], + ocelot->map[GCB][GCB_MIIM_MII_STATUS & REG_MASK]); - bus->name = "VSC9953 internal MDIO bus"; - bus->read = vsc9953_mdio_read; - bus->write = vsc9953_mdio_write; - bus->parent = dev; - bus->priv = ocelot; - snprintf(bus->id, MII_BUS_ID_SIZE, "%s-imdio", dev_name(dev)); + if (rc) { + dev_err(dev, "failed to setup MDIO bus\n"); + return rc; + } /* Needed in order to initialize the bus mutex lock */ rc = of_mdiobus_register(bus, NULL); diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c index e016b32ad208..2d420c9d7520 100644 --- a/drivers/net/mdio/mdio-mscc-miim.c +++ b/drivers/net/mdio/mdio-mscc-miim.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,9 @@ struct mscc_miim_dev { struct regmap *regs; + int mii_status_offset; struct regmap *phy_regs; + int phy_reset_offset; }; /* When high resolution timers aren't built-in: we can't use usleep_range() as @@ -56,7 +59,8 @@ static int mscc_miim_status(struct mii_bus *bus) struct mscc_miim_dev *miim = bus->priv; int val, ret; - ret = regmap_read(miim->regs, MSCC_MIIM_REG_STATUS, &val); + ret = regmap_read(miim->regs, + MSCC_MIIM_REG_STATUS + miim->mii_status_offset, &val); if (ret < 0) { WARN_ONCE(1, "mscc miim status read error %d\n", ret); return ret; @@ -93,7 +97,9 @@ static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum) if (ret) goto out; - ret = regmap_write(miim->regs, MSCC_MIIM_REG_CMD, MSCC_MIIM_CMD_VLD | + ret = regmap_write(miim->regs, + MSCC_MIIM_REG_CMD + miim->mii_status_offset, + MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | MSCC_MIIM_CMD_OPR_READ); @@ -107,8 +113,8 @@ static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum) if (ret) goto out; - ret = regmap_read(miim->regs, MSCC_MIIM_REG_DATA, &val); - + ret = regmap_read(miim->regs, + MSCC_MIIM_REG_DATA + miim->mii_status_offset, &val); if (ret < 0) { WARN_ONCE(1, "mscc miim read data reg error %d\n", ret); goto out; @@ -134,7 +140,9 @@ static int mscc_miim_write(struct mii_bus *bus, int mii_id, if (ret < 0) goto out; - ret = regmap_write(miim->regs, MSCC_MIIM_REG_CMD, MSCC_MIIM_CMD_VLD | + ret = regmap_write(miim->regs, + MSCC_MIIM_REG_CMD + miim->mii_status_offset, + MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | (value << MSCC_MIIM_CMD_WRDATA_SHIFT) | @@ -149,16 +157,19 @@ out: static int mscc_miim_reset(struct mii_bus *bus) { struct mscc_miim_dev *miim = bus->priv; + int offset = miim->phy_reset_offset; int ret; if (miim->phy_regs) { - ret = regmap_write(miim->phy_regs, MSCC_PHY_REG_PHY_CFG, 0); + ret = regmap_write(miim->phy_regs, + MSCC_PHY_REG_PHY_CFG + offset, 0); if (ret < 0) { WARN_ONCE(1, "mscc reset set error %d\n", ret); return ret; } - ret = regmap_write(miim->phy_regs, MSCC_PHY_REG_PHY_CFG, 0x1ff); + ret = regmap_write(miim->phy_regs, + MSCC_PHY_REG_PHY_CFG + offset, 0x1ff); if (ret < 0) { WARN_ONCE(1, "mscc reset clear error %d\n", ret); return ret; @@ -176,8 +187,8 @@ static const struct regmap_config mscc_miim_regmap_config = { .reg_stride = 4, }; -static int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, - struct regmap *mii_regmap) +int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, const char *name, + struct regmap *mii_regmap, int status_offset) { struct mscc_miim_dev *miim; struct mii_bus *bus; @@ -186,7 +197,7 @@ static int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, if (!bus) return -ENOMEM; - bus->name = "mscc_miim"; + bus->name = name; bus->read = mscc_miim_read; bus->write = mscc_miim_write; bus->reset = mscc_miim_reset; @@ -198,9 +209,13 @@ static int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, *pbus = bus; miim->regs = mii_regmap; + miim->mii_status_offset = status_offset; + + *pbus = bus; return 0; } +EXPORT_SYMBOL(mscc_miim_setup); static int mscc_miim_probe(struct platform_device *pdev) { @@ -237,7 +252,7 @@ static int mscc_miim_probe(struct platform_device *pdev) return PTR_ERR(phy_regmap); } - ret = mscc_miim_setup(&pdev->dev, &bus, mii_regmap); + ret = mscc_miim_setup(&pdev->dev, &bus, "mscc_miim", mii_regmap, 0); if (ret < 0) { dev_err(&pdev->dev, "Unable to setup the MDIO bus\n"); return ret; @@ -245,6 +260,7 @@ static int mscc_miim_probe(struct platform_device *pdev) miim = bus->priv; miim->phy_regs = phy_regmap; + miim->phy_reset_offset = 0; ret = of_mdiobus_register(bus, pdev->dev.of_node); if (ret < 0) { diff --git a/include/linux/mdio/mdio-mscc-miim.h b/include/linux/mdio/mdio-mscc-miim.h new file mode 100644 index 000000000000..5b4ed2c3cbb9 --- /dev/null +++ b/include/linux/mdio/mdio-mscc-miim.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Driver for the MDIO interface of Microsemi network switches. + * + * Author: Colin Foster + * Copyright (C) 2021 Innovative Advantage + */ +#ifndef MDIO_MSCC_MIIM_H +#define MDIO_MSCC_MIIM_H + +#include +#include +#include + +int mscc_miim_setup(struct device *device, struct mii_bus **bus, + const char *name, struct regmap *mii_regmap, + int status_offset); + +#endif -- cgit v1.2.3-59-g8ed1b From 17247821ae9b40ea6df8d771cfca97d91675be93 Mon Sep 17 00:00:00 2001 From: Dario Binacchi Date: Thu, 25 Nov 2021 23:46:42 +0100 Subject: mfd: ti_am335x_tscadc: Drop the CNTRLREG_TSC_8WIRE macro In TI's reference manual description for the `AFE_Pen_Ctrl' bit-field of the TSC's CTRL register, there is no mention of 8-wire touchscreens. Even commit f0933a60d190 ("mfd: ti_am335x_tscadc: Update logic in CTRL register for 5-wire TS") says that the value of this bit-field must be the same for 4-wire and 8-wire touchscreens. So let's remove the CNTRLREG_TSC_8WIRE macro to avoid misunderstandings. Signed-off-by: Dario Binacchi Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20211125224642.21011-5-dariobin@libero.it --- include/linux/mfd/ti_am335x_tscadc.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mfd/ti_am335x_tscadc.h b/include/linux/mfd/ti_am335x_tscadc.h index ba13e043d910..4063b0614d90 100644 --- a/include/linux/mfd/ti_am335x_tscadc.h +++ b/include/linux/mfd/ti_am335x_tscadc.h @@ -103,7 +103,6 @@ #define CNTRLREG_TSC_AFE_CTRL(val) FIELD_PREP(GENMASK(6, 5), (val)) #define CNTRLREG_TSC_4WIRE CNTRLREG_TSC_AFE_CTRL(1) #define CNTRLREG_TSC_5WIRE CNTRLREG_TSC_AFE_CTRL(2) -#define CNTRLREG_TSC_8WIRE CNTRLREG_TSC_AFE_CTRL(3) #define CNTRLREG_TSC_ENB BIT(7) /*Control registers bitfields for MAGADC IP */ -- cgit v1.2.3-59-g8ed1b From 79478bf9ea9fa48d30836afa796ac13d8a0f320b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 17 Nov 2021 07:13:54 +0100 Subject: block: move blk_rq_err_bytes to scsi blk_rq_err_bytes is only used by the scsi midlayer, so move it there. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20211117061404.331732-2-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 41 ----------------------------------------- drivers/scsi/scsi_lib.c | 42 +++++++++++++++++++++++++++++++++++++++++- include/linux/blk-mq.h | 3 --- 3 files changed, 41 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 1378d084c770..682b112f513f 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1176,47 +1176,6 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * } EXPORT_SYMBOL_GPL(blk_insert_cloned_request); -/** - * blk_rq_err_bytes - determine number of bytes till the next failure boundary - * @rq: request to examine - * - * Description: - * A request could be merge of IOs which require different failure - * handling. This function determines the number of bytes which - * can be failed from the beginning of the request without - * crossing into area which need to be retried further. - * - * Return: - * The number of bytes to fail. - */ -unsigned int blk_rq_err_bytes(const struct request *rq) -{ - unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; - unsigned int bytes = 0; - struct bio *bio; - - if (!(rq->rq_flags & RQF_MIXED_MERGE)) - return blk_rq_bytes(rq); - - /* - * Currently the only 'mixing' which can happen is between - * different fastfail types. We can safely fail portions - * which have all the failfast bits that the first one has - - * the ones which are at least as eager to fail as the first - * one. - */ - for (bio = rq->bio; bio; bio = bio->bi_next) { - if ((bio->bi_opf & ff) != ff) - break; - bytes += bio->bi_iter.bi_size; - } - - /* this could lead to infinite loop */ - BUG_ON(blk_rq_bytes(rq) && !bytes); - return bytes; -} -EXPORT_SYMBOL_GPL(blk_rq_err_bytes); - static void update_io_ticks(struct block_device *part, unsigned long now, bool end) { diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 621d841d819a..5e8b5ecb3245 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -617,6 +617,46 @@ static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result) } } +/** + * scsi_rq_err_bytes - determine number of bytes till the next failure boundary + * @rq: request to examine + * + * Description: + * A request could be merge of IOs which require different failure + * handling. This function determines the number of bytes which + * can be failed from the beginning of the request without + * crossing into area which need to be retried further. + * + * Return: + * The number of bytes to fail. + */ +static unsigned int scsi_rq_err_bytes(const struct request *rq) +{ + unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; + unsigned int bytes = 0; + struct bio *bio; + + if (!(rq->rq_flags & RQF_MIXED_MERGE)) + return blk_rq_bytes(rq); + + /* + * Currently the only 'mixing' which can happen is between + * different fastfail types. We can safely fail portions + * which have all the failfast bits that the first one has - + * the ones which are at least as eager to fail as the first + * one. + */ + for (bio = rq->bio; bio; bio = bio->bi_next) { + if ((bio->bi_opf & ff) != ff) + break; + bytes += bio->bi_iter.bi_size; + } + + /* this could lead to infinite loop */ + BUG_ON(blk_rq_bytes(rq) && !bytes); + return bytes; +} + /* Helper for scsi_io_completion() when "reprep" action required. */ static void scsi_io_completion_reprep(struct scsi_cmnd *cmd, struct request_queue *q) @@ -794,7 +834,7 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) scsi_print_command(cmd); } } - if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req))) + if (!scsi_end_request(req, blk_stat, scsi_rq_err_bytes(req))) return; fallthrough; case ACTION_REPREP: diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 2949d9ac7484..a78d9a0f2a1b 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -947,7 +947,6 @@ struct req_iterator { * blk_rq_pos() : the current sector * blk_rq_bytes() : bytes left in the entire request * blk_rq_cur_bytes() : bytes left in the current segment - * blk_rq_err_bytes() : bytes left till the next error boundary * blk_rq_sectors() : sectors left in the entire request * blk_rq_cur_sectors() : sectors left in the current segment * blk_rq_stats_sectors() : sectors of the entire request used for stats @@ -971,8 +970,6 @@ static inline int blk_rq_cur_bytes(const struct request *rq) return bio_iovec(rq->bio).bv_len; } -unsigned int blk_rq_err_bytes(const struct request *rq); - static inline unsigned int blk_rq_sectors(const struct request *rq) { return blk_rq_bytes(rq) >> SECTOR_SHIFT; -- cgit v1.2.3-59-g8ed1b From 786d4e01c550e8bb7c9f9f23bca0596a2a33483c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 17 Nov 2021 07:13:55 +0100 Subject: block: remove rq_flush_dcache_pages This function is trivial, and flush_dcache_page is always defined, so just open code it in the 2.5 callers. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Link: https://lore.kernel.org/r/20211117061404.331732-3-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 19 ------------------- drivers/mtd/mtd_blkdevs.c | 10 ++++++++-- drivers/mtd/ubi/block.c | 6 +++++- include/linux/blk-mq.h | 10 ---------- 4 files changed, 13 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 682b112f513f..039e28509f59 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1300,25 +1300,6 @@ void blk_steal_bios(struct bio_list *list, struct request *rq) } EXPORT_SYMBOL_GPL(blk_steal_bios); -#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE -/** - * rq_flush_dcache_pages - Helper function to flush all pages in a request - * @rq: the request to be flushed - * - * Description: - * Flush all pages in @rq. - */ -void rq_flush_dcache_pages(struct request *rq) -{ - struct req_iterator iter; - struct bio_vec bvec; - - rq_for_each_segment(bvec, rq, iter) - flush_dcache_page(bvec.bv_page); -} -EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); -#endif - /** * blk_lld_busy - Check if underlying low-level drivers of a device are busy * @q : the queue of the device being checked diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c index 4eaba6f4ec68..66f81d42fe77 100644 --- a/drivers/mtd/mtd_blkdevs.c +++ b/drivers/mtd/mtd_blkdevs.c @@ -46,6 +46,8 @@ static blk_status_t do_blktrans_request(struct mtd_blktrans_ops *tr, struct mtd_blktrans_dev *dev, struct request *req) { + struct req_iterator iter; + struct bio_vec bvec; unsigned long block, nsect; char *buf; @@ -76,13 +78,17 @@ static blk_status_t do_blktrans_request(struct mtd_blktrans_ops *tr, } } kunmap(bio_page(req->bio)); - rq_flush_dcache_pages(req); + + rq_for_each_segment(bvec, req, iter) + flush_dcache_page(bvec.bv_page); return BLK_STS_OK; case REQ_OP_WRITE: if (!tr->writesect) return BLK_STS_IOERR; - rq_flush_dcache_pages(req); + rq_for_each_segment(bvec, req, iter) + flush_dcache_page(bvec.bv_page); + buf = kmap(bio_page(req->bio)) + bio_offset(req->bio); for (; nsect > 0; nsect--, block++, buf += tr->blksize) { if (tr->writesect(dev, block, buf)) { diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c index 062e6c2c45f5..302426ab30f8 100644 --- a/drivers/mtd/ubi/block.c +++ b/drivers/mtd/ubi/block.c @@ -294,6 +294,8 @@ static void ubiblock_do_work(struct work_struct *work) int ret; struct ubiblock_pdu *pdu = container_of(work, struct ubiblock_pdu, work); struct request *req = blk_mq_rq_from_pdu(pdu); + struct req_iterator iter; + struct bio_vec bvec; blk_mq_start_request(req); @@ -305,7 +307,9 @@ static void ubiblock_do_work(struct work_struct *work) blk_rq_map_sg(req->q, req, pdu->usgl.sg); ret = ubiblock_read(pdu); - rq_flush_dcache_pages(req); + + rq_for_each_segment(bvec, req, iter) + flush_dcache_page(bvec.bv_page); blk_mq_end_request(req, errno_to_blk_status(ret)); } diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index a78d9a0f2a1b..308edc2a4925 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -1132,14 +1132,4 @@ static inline bool blk_req_can_dispatch_to_zone(struct request *rq) } #endif /* CONFIG_BLK_DEV_ZONED */ -#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE -# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" -#endif -#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE -void rq_flush_dcache_pages(struct request *rq); -#else -static inline void rq_flush_dcache_pages(struct request *rq) -{ -} -#endif /* ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE */ #endif /* BLK_MQ_H */ -- cgit v1.2.3-59-g8ed1b From 86416916466514e4ae0b7296d20133b6427c4c1f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:12 +0100 Subject: block: move GENHD_FL_NATIVE_CAPACITY to disk->state The flag to indicate an unlocked native capacity is dynamic state, not a driver capability flag, so move it to disk->state. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-2-hch@lst.de Signed-off-by: Jens Axboe --- block/partitions/core.c | 15 ++++++--------- include/linux/genhd.h | 8 +------- 2 files changed, 7 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/block/partitions/core.c b/block/partitions/core.c index 334b72ef1d73..520292fee933 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -527,18 +527,15 @@ out_unlock: static bool disk_unlock_native_capacity(struct gendisk *disk) { - const struct block_device_operations *bdops = disk->fops; - - if (bdops->unlock_native_capacity && - !(disk->flags & GENHD_FL_NATIVE_CAPACITY)) { - printk(KERN_CONT "enabling native capacity\n"); - bdops->unlock_native_capacity(disk); - disk->flags |= GENHD_FL_NATIVE_CAPACITY; - return true; - } else { + if (!disk->fops->unlock_native_capacity || + test_and_set_bit(GD_NATIVE_CAPACITY, &disk->state)) { printk(KERN_CONT "truncated\n"); return false; } + + printk(KERN_CONT "enabling native capacity\n"); + disk->fops->unlock_native_capacity(disk); + return true; } void blk_drop_partitions(struct gendisk *disk) diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 74c410263113..e490a71e5e9d 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -60,12 +60,6 @@ struct partition_meta_info { * (``BLOCK_EXT_MAJOR``). * This affects the maximum number of partitions. * - * ``GENHD_FL_NATIVE_CAPACITY`` (0x0080): based on information in the - * partition table, the device's capacity has been extended to its - * native capacity; i.e. the device has hidden capacity used by one - * of the partitions (this is a flag used so that native capacity is - * only ever unlocked once). - * * ``GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE`` (0x0100): event polling is * blocked whenever a writer holds an exclusive lock. * @@ -86,7 +80,6 @@ struct partition_meta_info { #define GENHD_FL_CD 0x0008 #define GENHD_FL_SUPPRESS_PARTITION_INFO 0x0020 #define GENHD_FL_EXT_DEVT 0x0040 -#define GENHD_FL_NATIVE_CAPACITY 0x0080 #define GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE 0x0100 #define GENHD_FL_NO_PART_SCAN 0x0200 #define GENHD_FL_HIDDEN 0x0400 @@ -140,6 +133,7 @@ struct gendisk { #define GD_NEED_PART_SCAN 0 #define GD_READ_ONLY 1 #define GD_DEAD 2 +#define GD_NATIVE_CAPACITY 3 struct mutex open_mutex; /* open/close mutex */ unsigned open_partitions; /* number of open partitions */ -- cgit v1.2.3-59-g8ed1b From 1545e0b419ba1d9b9bee4061d4826340afe6b0aa Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:13 +0100 Subject: block: move GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE to disk->event_flags GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE is all about the event reporting mechanism, so move it to the event_flags field. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-3-hch@lst.de Signed-off-by: Jens Axboe --- block/bdev.c | 2 +- drivers/block/paride/pcd.c | 2 +- drivers/scsi/sr.c | 5 +++-- include/linux/genhd.h | 6 ++---- 4 files changed, 7 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/block/bdev.c b/block/bdev.c index b1d087e5e205..dd84961bed7e 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -837,7 +837,7 @@ struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) * used in blkdev_get/put(). */ if ((mode & FMODE_WRITE) && !bdev->bd_write_holder && - (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) { + (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { bdev->bd_write_holder = true; unblock_events = false; } diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index f6b1d63e96e1..430ee8004a51 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -928,8 +928,8 @@ static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, disk->minors = 1; strcpy(disk->disk_name, cd->name); /* umm... */ disk->fops = &pcd_bdops; - disk->flags = GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; disk->events = DISK_EVENT_MEDIA_CHANGE; + disk->event_flags = DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE; if (!pi_init(cd->pi, autoprobe, port, mode, unit, protocol, delay, pcd_buffer, PI_PCD, verbose, cd->name)) { diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 8e4af111c078..be445dc35f2c 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -684,9 +684,10 @@ static int sr_probe(struct device *dev) disk->minors = 1; sprintf(disk->disk_name, "sr%d", minor); disk->fops = &sr_bdops; - disk->flags = GENHD_FL_CD | GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; + disk->flags = GENHD_FL_CD; disk->events = DISK_EVENT_MEDIA_CHANGE | DISK_EVENT_EJECT_REQUEST; - disk->event_flags = DISK_EVENT_FLAG_POLL | DISK_EVENT_FLAG_UEVENT; + disk->event_flags = DISK_EVENT_FLAG_POLL | DISK_EVENT_FLAG_UEVENT | + DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE; blk_queue_rq_timeout(sdev->request_queue, SR_TIMEOUT); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index e490a71e5e9d..c1136ff3c91f 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -60,9 +60,6 @@ struct partition_meta_info { * (``BLOCK_EXT_MAJOR``). * This affects the maximum number of partitions. * - * ``GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE`` (0x0100): event polling is - * blocked whenever a writer holds an exclusive lock. - * * ``GENHD_FL_NO_PART_SCAN`` (0x0200): partition scanning is disabled. * Used for loop devices in their default settings and some MMC * devices. @@ -80,7 +77,6 @@ struct partition_meta_info { #define GENHD_FL_CD 0x0008 #define GENHD_FL_SUPPRESS_PARTITION_INFO 0x0020 #define GENHD_FL_EXT_DEVT 0x0040 -#define GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE 0x0100 #define GENHD_FL_NO_PART_SCAN 0x0200 #define GENHD_FL_HIDDEN 0x0400 @@ -94,6 +90,8 @@ enum { DISK_EVENT_FLAG_POLL = 1 << 0, /* Forward events to udev */ DISK_EVENT_FLAG_UEVENT = 1 << 1, + /* Block event polling when open for exclusive write */ + DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2, }; struct disk_events; -- cgit v1.2.3-59-g8ed1b From 1a827ce1b9f2c740d2c6a228afd972970c18bc21 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:14 +0100 Subject: block: remove GENHD_FL_CD GENHD_FL_CD marks a gendisk as a vaguely CD-ROM like device. Besides being used internally inside of sunvdc.c an xen-blkfront it is used by xen-blkback as a hint to claim a device exported to a guest is a CD-ROM like device. Just check for disk->cdi instead which is the right indicator for "real" CD-ROM or DVD drivers. This will miss the paravirtualized guest drivers, but those make little sense to report anyway. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-4-hch@lst.de Signed-off-by: Jens Axboe --- drivers/block/sunvdc.c | 17 +++++++++-------- drivers/block/xen-blkback/xenbus.c | 2 +- drivers/block/xen-blkfront.c | 26 +++++++++++--------------- drivers/scsi/sr.c | 1 - include/linux/genhd.h | 5 ----- 5 files changed, 21 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 6f45a53f7cbf..2157936de623 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -143,8 +143,8 @@ static int vdc_getgeo(struct block_device *bdev, struct hd_geometry *geo) static int vdc_ioctl(struct block_device *bdev, fmode_t mode, unsigned command, unsigned long argument) { + struct vdc_port *port = bdev->bd_disk->private_data; int i; - struct gendisk *disk; switch (command) { case CDROMMULTISESSION: @@ -155,12 +155,15 @@ static int vdc_ioctl(struct block_device *bdev, fmode_t mode, return 0; case CDROM_GET_CAPABILITY: - disk = bdev->bd_disk; - - if (bdev->bd_disk && (disk->flags & GENHD_FL_CD)) + if (!vdc_version_supported(port, 1, 1)) + return -EINVAL; + switch (port->vdisk_mtype) { + case VD_MEDIA_TYPE_CD: + case VD_MEDIA_TYPE_DVD: return 0; - return -EINVAL; - + default: + return -EINVAL; + } default: pr_debug(PFX "ioctl %08x not supported\n", command); return -EINVAL; @@ -854,14 +857,12 @@ static int probe_disk(struct vdc_port *port) switch (port->vdisk_mtype) { case VD_MEDIA_TYPE_CD: pr_info(PFX "Virtual CDROM %s\n", port->disk_name); - g->flags |= GENHD_FL_CD; g->flags |= GENHD_FL_REMOVABLE; set_disk_ro(g, 1); break; case VD_MEDIA_TYPE_DVD: pr_info(PFX "Virtual DVD %s\n", port->disk_name); - g->flags |= GENHD_FL_CD; g->flags |= GENHD_FL_REMOVABLE; set_disk_ro(g, 1); break; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index 914587aabca0..62125fd4af4a 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -510,7 +510,7 @@ static int xen_vbd_create(struct xen_blkif *blkif, blkif_vdev_t handle, } vbd->size = vbd_sz(vbd); - if (vbd->bdev->bd_disk->flags & GENHD_FL_CD || cdrom) + if (cdrom || disk_to_cdi(vbd->bdev->bd_disk)) vbd->type |= VDISK_CDROM; if (vbd->bdev->bd_disk->flags & GENHD_FL_REMOVABLE) vbd->type |= VDISK_REMOVABLE; diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 8e3983e456f3..700c765a759a 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -198,6 +198,7 @@ struct blkfront_info struct gendisk *gd; u16 sector_size; unsigned int physical_sector_size; + unsigned long vdisk_info; int vdevice; blkif_vdev_t handle; enum blkif_state connected; @@ -505,6 +506,7 @@ static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg) static int blkif_ioctl(struct block_device *bdev, fmode_t mode, unsigned command, unsigned long argument) { + struct blkfront_info *info = bdev->bd_disk->private_data; int i; switch (command) { @@ -514,9 +516,9 @@ static int blkif_ioctl(struct block_device *bdev, fmode_t mode, return -EFAULT; return 0; case CDROM_GET_CAPABILITY: - if (bdev->bd_disk->flags & GENHD_FL_CD) - return 0; - return -EINVAL; + if (!(info->vdisk_info & VDISK_CDROM)) + return -EINVAL; + return 0; default: return -EINVAL; } @@ -1057,9 +1059,8 @@ static char *encode_disk_name(char *ptr, unsigned int n) } static int xlvbd_alloc_gendisk(blkif_sector_t capacity, - struct blkfront_info *info, - u16 vdisk_info, u16 sector_size, - unsigned int physical_sector_size) + struct blkfront_info *info, u16 sector_size, + unsigned int physical_sector_size) { struct gendisk *gd; int nr_minors = 1; @@ -1157,15 +1158,11 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity, xlvbd_flush(info); - if (vdisk_info & VDISK_READONLY) + if (info->vdisk_info & VDISK_READONLY) set_disk_ro(gd, 1); - - if (vdisk_info & VDISK_REMOVABLE) + if (info->vdisk_info & VDISK_REMOVABLE) gd->flags |= GENHD_FL_REMOVABLE; - if (vdisk_info & VDISK_CDROM) - gd->flags |= GENHD_FL_CD; - return 0; out_free_tag_set: @@ -2304,7 +2301,6 @@ static void blkfront_connect(struct blkfront_info *info) unsigned long long sectors; unsigned long sector_size; unsigned int physical_sector_size; - unsigned int binfo; int err, i; struct blkfront_ring_info *rinfo; @@ -2342,7 +2338,7 @@ static void blkfront_connect(struct blkfront_info *info) err = xenbus_gather(XBT_NIL, info->xbdev->otherend, "sectors", "%llu", §ors, - "info", "%u", &binfo, + "info", "%u", &info->vdisk_info, "sector-size", "%lu", §or_size, NULL); if (err) { @@ -2371,7 +2367,7 @@ static void blkfront_connect(struct blkfront_info *info) } } - err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size, + err = xlvbd_alloc_gendisk(sectors, info, sector_size, physical_sector_size); if (err) { xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s", diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index be445dc35f2c..6646797a7756 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -684,7 +684,6 @@ static int sr_probe(struct device *dev) disk->minors = 1; sprintf(disk->disk_name, "sr%d", minor); disk->fops = &sr_bdops; - disk->flags = GENHD_FL_CD; disk->events = DISK_EVENT_MEDIA_CHANGE | DISK_EVENT_EJECT_REQUEST; disk->event_flags = DISK_EVENT_FLAG_POLL | DISK_EVENT_FLAG_UEVENT | DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE; diff --git a/include/linux/genhd.h b/include/linux/genhd.h index c1136ff3c91f..74518c576fbb 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -46,10 +46,6 @@ struct partition_meta_info { * Must not be set for devices which are removed entirely when the * media is removed. * - * ``GENHD_FL_CD`` (0x0008): the block device is a CD-ROM-style - * device. - * Affects responses to the ``CDROM_GET_CAPABILITY`` ioctl. - * * ``GENHD_FL_SUPPRESS_PARTITION_INFO`` (0x0020): don't include * partition information in ``/proc/partitions`` or in the output of * printk_all_partitions(). @@ -74,7 +70,6 @@ struct partition_meta_info { #define GENHD_FL_REMOVABLE 0x0001 /* 2 is unused (used to be GENHD_FL_DRIVERFS) */ /* 4 is unused (used to be GENHD_FL_MEDIA_CHANGE_NOTIFY) */ -#define GENHD_FL_CD 0x0008 #define GENHD_FL_SUPPRESS_PARTITION_INFO 0x0020 #define GENHD_FL_EXT_DEVT 0x0040 #define GENHD_FL_NO_PART_SCAN 0x0200 -- cgit v1.2.3-59-g8ed1b From 46e7eac647b34ed4106a8262f8bedbb90801fadd Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:17 +0100 Subject: block: rename GENHD_FL_NO_PART_SCAN to GENHD_FL_NO_PART The GENHD_FL_NO_PART_SCAN controls more than just partitions canning, so rename it to GENHD_FL_NO_PART. Signed-off-by: Christoph Hellwig Acked-by: Ulf Hansson Link: https://lore.kernel.org/r/20211122130625.1136848-7-hch@lst.de Signed-off-by: Jens Axboe --- block/genhd.c | 2 +- drivers/block/loop.c | 8 ++++---- drivers/block/n64cart.c | 2 +- drivers/mmc/core/block.c | 4 ++-- include/linux/genhd.h | 13 ++++++------- 5 files changed, 14 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index 94f39c4333b8..685794a2ebb0 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -500,7 +500,7 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, * and don't bother scanning for partitions either. */ disk->flags |= GENHD_FL_SUPPRESS_PARTITION_INFO; - disk->flags |= GENHD_FL_NO_PART_SCAN; + disk->flags |= GENHD_FL_NO_PART; } else { ret = bdi_register(disk->bdi, "%u:%u", disk->major, disk->first_minor); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index a154cab6cd98..7219d98c6fb8 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1061,7 +1061,7 @@ static int loop_configure(struct loop_device *lo, fmode_t mode, lo->lo_flags |= LO_FLAGS_PARTSCAN; partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; if (partscan) - lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; + lo->lo_disk->flags &= ~GENHD_FL_NO_PART; loop_global_unlock(lo, is_loop); if (partscan) @@ -1191,7 +1191,7 @@ out_unlock: mutex_lock(&lo->lo_mutex); lo->lo_flags = 0; if (!part_shift) - lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN; + lo->lo_disk->flags |= GENHD_FL_NO_PART; lo->lo_state = Lo_unbound; mutex_unlock(&lo->lo_mutex); @@ -1301,7 +1301,7 @@ out_unfreeze: if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) && !(prev_lo_flags & LO_FLAGS_PARTSCAN)) { - lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; + lo->lo_disk->flags &= ~GENHD_FL_NO_PART; partscan = true; } out_unlock: @@ -2032,7 +2032,7 @@ static int loop_add(int i) * userspace tools. Parameters like this in general should be avoided. */ if (!part_shift) - disk->flags |= GENHD_FL_NO_PART_SCAN; + disk->flags |= GENHD_FL_NO_PART; disk->flags |= GENHD_FL_EXT_DEVT; atomic_set(&lo->lo_refcnt, 0); mutex_init(&lo->lo_mutex); diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c index 78282f01f581..4db9a8c244af 100644 --- a/drivers/block/n64cart.c +++ b/drivers/block/n64cart.c @@ -136,7 +136,7 @@ static int __init n64cart_probe(struct platform_device *pdev) goto out; disk->first_minor = 0; - disk->flags = GENHD_FL_NO_PART_SCAN; + disk->flags = GENHD_FL_NO_PART; disk->fops = &n64cart_fops; disk->private_data = &pdev->dev; strcpy(disk->disk_name, "n64cart"); diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 90e1bcd03b46..a71b3512c877 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2397,8 +2397,8 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, set_disk_ro(md->disk, md->read_only || default_ro); md->disk->flags = GENHD_FL_EXT_DEVT; if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT)) - md->disk->flags |= GENHD_FL_NO_PART_SCAN - | GENHD_FL_SUPPRESS_PARTITION_INFO; + md->disk->flags |= GENHD_FL_NO_PART | + GENHD_FL_SUPPRESS_PARTITION_INFO; /* * As discussed on lkml, GENHD_FL_REMOVABLE should: diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 74518c576fbb..0b9be3df9489 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -56,15 +56,15 @@ struct partition_meta_info { * (``BLOCK_EXT_MAJOR``). * This affects the maximum number of partitions. * - * ``GENHD_FL_NO_PART_SCAN`` (0x0200): partition scanning is disabled. - * Used for loop devices in their default settings and some MMC - * devices. + * ``GENHD_FL_NO_PART`` (0x0200): partition support is disabled. + * The kernel will not scan for partitions from add_disk, and users + * can't add partitions manually. * * ``GENHD_FL_HIDDEN`` (0x0400): the block device is hidden; it * doesn't produce events, doesn't appear in sysfs, and doesn't have * an associated ``bdev``. * Implies ``GENHD_FL_SUPPRESS_PARTITION_INFO`` and - * ``GENHD_FL_NO_PART_SCAN``. + * ``GENHD_FL_NO_PART``. * Used for multipath devices. */ #define GENHD_FL_REMOVABLE 0x0001 @@ -72,7 +72,7 @@ struct partition_meta_info { /* 4 is unused (used to be GENHD_FL_MEDIA_CHANGE_NOTIFY) */ #define GENHD_FL_SUPPRESS_PARTITION_INFO 0x0020 #define GENHD_FL_EXT_DEVT 0x0040 -#define GENHD_FL_NO_PART_SCAN 0x0200 +#define GENHD_FL_NO_PART 0x0200 #define GENHD_FL_HIDDEN 0x0400 enum { @@ -180,8 +180,7 @@ static inline int disk_max_parts(struct gendisk *disk) static inline bool disk_part_scan_enabled(struct gendisk *disk) { - return disk_max_parts(disk) > 1 && - !(disk->flags & GENHD_FL_NO_PART_SCAN); + return disk_max_parts(disk) > 1 && !(disk->flags & GENHD_FL_NO_PART); } static inline dev_t disk_devt(struct gendisk *disk) -- cgit v1.2.3-59-g8ed1b From 3b5149ac50970669ee0ddb9629ec77ffd5c0622d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:21 +0100 Subject: block: remove GENHD_FL_SUPPRESS_PARTITION_INFO This flag is not set directly anywhere and only inherited from GENHD_FL_HIDDEN. Just check for GENHD_FL_HIDDEN instead. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-11-hch@lst.de Signed-off-by: Jens Axboe --- block/genhd.c | 11 +++-------- include/linux/genhd.h | 9 +-------- 2 files changed, 4 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index 685794a2ebb0..50a843ee18f6 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -496,10 +496,8 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, if (disk->flags & GENHD_FL_HIDDEN) { /* - * Don't let hidden disks show up in /proc/partitions, - * and don't bother scanning for partitions either. + * Don't bother scanning for partitions. */ - disk->flags |= GENHD_FL_SUPPRESS_PARTITION_INFO; disk->flags |= GENHD_FL_NO_PART; } else { ret = bdi_register(disk->bdi, "%u:%u", @@ -725,8 +723,7 @@ void __init printk_all_partitions(void) * Don't show empty devices or things that have been * suppressed */ - if (get_capacity(disk) == 0 || - (disk->flags & GENHD_FL_SUPPRESS_PARTITION_INFO)) + if (get_capacity(disk) == 0 || (disk->flags & GENHD_FL_HIDDEN)) continue; /* @@ -819,9 +816,7 @@ static int show_partition(struct seq_file *seqf, void *v) struct block_device *part; unsigned long idx; - if (!get_capacity(sgp)) - return 0; - if (sgp->flags & GENHD_FL_SUPPRESS_PARTITION_INFO) + if (!get_capacity(sgp) || (sgp->flags & GENHD_FL_HIDDEN)) return 0; rcu_read_lock(); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 0b9be3df9489..64a2f33ae9ea 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -46,11 +46,6 @@ struct partition_meta_info { * Must not be set for devices which are removed entirely when the * media is removed. * - * ``GENHD_FL_SUPPRESS_PARTITION_INFO`` (0x0020): don't include - * partition information in ``/proc/partitions`` or in the output of - * printk_all_partitions(). - * Used for the null block device and some MMC devices. - * * ``GENHD_FL_EXT_DEVT`` (0x0040): the driver supports extended * dynamic ``dev_t``, i.e. it wants extended device numbers * (``BLOCK_EXT_MAJOR``). @@ -63,14 +58,12 @@ struct partition_meta_info { * ``GENHD_FL_HIDDEN`` (0x0400): the block device is hidden; it * doesn't produce events, doesn't appear in sysfs, and doesn't have * an associated ``bdev``. - * Implies ``GENHD_FL_SUPPRESS_PARTITION_INFO`` and - * ``GENHD_FL_NO_PART``. + * Implies ``GENHD_FL_NO_PART``. * Used for multipath devices. */ #define GENHD_FL_REMOVABLE 0x0001 /* 2 is unused (used to be GENHD_FL_DRIVERFS) */ /* 4 is unused (used to be GENHD_FL_MEDIA_CHANGE_NOTIFY) */ -#define GENHD_FL_SUPPRESS_PARTITION_INFO 0x0020 #define GENHD_FL_EXT_DEVT 0x0040 #define GENHD_FL_NO_PART 0x0200 #define GENHD_FL_HIDDEN 0x0400 -- cgit v1.2.3-59-g8ed1b From 1ebe2e5f9d68e94c524aba876f27b945669a7879 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:22 +0100 Subject: block: remove GENHD_FL_EXT_DEVT All modern drivers can support extra partitions using the extended dev_t. In fact except for the ioctl method drivers never even see partitions in normal operation. So remove the GENHD_FL_EXT_DEVT and allow extra partitions for all block devices that do support partitions, and require those that do not support partitions to explicit disallow them using GENHD_FL_NO_PART. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-12-hch@lst.de Signed-off-by: Jens Axboe --- block/genhd.c | 6 +++--- block/partitions/core.c | 9 ++++----- drivers/block/amiflop.c | 1 + drivers/block/ataflop.c | 1 + drivers/block/brd.c | 1 - drivers/block/drbd/drbd_main.c | 1 + drivers/block/floppy.c | 1 + drivers/block/loop.c | 1 - drivers/block/null_blk/main.c | 1 - drivers/block/paride/pcd.c | 1 + drivers/block/paride/pf.c | 1 + drivers/block/pktcdvd.c | 2 +- drivers/block/ps3vram.c | 1 + drivers/block/rbd.c | 6 ++---- drivers/block/swim.c | 1 + drivers/block/swim3.c | 2 +- drivers/block/virtio_blk.c | 1 - drivers/block/z2ram.c | 1 + drivers/block/zram/zram_drv.c | 1 + drivers/cdrom/gdrom.c | 1 + drivers/md/dm.c | 1 + drivers/md/md.c | 5 ----- drivers/mmc/core/block.c | 1 - drivers/mtd/ubi/block.c | 1 + drivers/scsi/sd.c | 1 - drivers/scsi/sr.c | 1 + include/linux/genhd.h | 28 +++++----------------------- 27 files changed, 30 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index 50a843ee18f6..628632537129 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -376,7 +376,7 @@ int disk_scan_partitions(struct gendisk *disk, fmode_t mode) { struct block_device *bdev; - if (!disk_part_scan_enabled(disk)) + if (disk->flags & GENHD_FL_NO_PART) return -EINVAL; if (disk->open_partitions) return -EBUSY; @@ -438,7 +438,6 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, return ret; disk->major = BLOCK_EXT_MAJOR; disk->first_minor = ret; - disk->flags |= GENHD_FL_EXT_DEVT; } ret = disk_alloc_events(disk); @@ -872,7 +871,8 @@ static ssize_t disk_ext_range_show(struct device *dev, { struct gendisk *disk = dev_to_disk(dev); - return sprintf(buf, "%d\n", disk_max_parts(disk)); + return sprintf(buf, "%d\n", + (disk->flags & GENHD_FL_NO_PART) ? 1 : DISK_MAX_PARTS); } static ssize_t disk_removable_show(struct device *dev, diff --git a/block/partitions/core.c b/block/partitions/core.c index 520292fee933..c2a1635922b1 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -98,13 +98,12 @@ static void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) static struct parsed_partitions *allocate_partitions(struct gendisk *hd) { struct parsed_partitions *state; - int nr; + int nr = DISK_MAX_PARTS; state = kzalloc(sizeof(*state), GFP_KERNEL); if (!state) return NULL; - nr = disk_max_parts(hd); state->parts = vzalloc(array_size(nr, sizeof(state->parts[0]))); if (!state->parts) { kfree(state); @@ -326,7 +325,7 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, lockdep_assert_held(&disk->open_mutex); - if (partno >= disk_max_parts(disk)) + if (partno >= DISK_MAX_PARTS) return ERR_PTR(-EINVAL); /* @@ -604,7 +603,7 @@ static int blk_add_partitions(struct gendisk *disk) struct parsed_partitions *state; int ret = -EAGAIN, p; - if (!disk_part_scan_enabled(disk)) + if (disk->flags & GENHD_FL_NO_PART) return 0; state = check_partition(disk); @@ -687,7 +686,7 @@ rescan: * userspace for this particular setup. */ if (invalidate) { - if (disk_part_scan_enabled(disk) || + if (!(disk->flags & GENHD_FL_NO_PART) || !(disk->flags & GENHD_FL_REMOVABLE)) set_capacity(disk, 0); } diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index bf5c124c5452..1eec5113d0b5 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c @@ -1790,6 +1790,7 @@ static int fd_alloc_disk(int drive, int system) disk->first_minor = drive + system; disk->minors = 1; disk->fops = &floppy_fops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE; if (system) sprintf(disk->disk_name, "fd%d_msdos", drive); diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index bf769e6e32fe..f3ff9babdb5c 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2000,6 +2000,7 @@ static int ataflop_alloc_disk(unsigned int drive, unsigned int type) disk->minors = 1; sprintf(disk->disk_name, "fd%d", drive); disk->fops = &floppy_fops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE; disk->private_data = &unit[drive]; set_capacity(disk, MAX_DISK_SIZE * 2); diff --git a/drivers/block/brd.c b/drivers/block/brd.c index a896ee175d86..8fe2e4289dae 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -405,7 +405,6 @@ static int brd_alloc(int i) disk->minors = max_part; disk->fops = &brd_fops; disk->private_data = brd; - disk->flags = GENHD_FL_EXT_DEVT; strlcpy(disk->disk_name, buf, DISK_NAME_LEN); set_capacity(disk, rd_size * 2); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 53ba2dddba6e..07b3c6093e7d 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2734,6 +2734,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig disk->first_minor = minor; disk->minors = 1; disk->fops = &drbd_ops; + disk->flags |= GENHD_FL_NO_PART; sprintf(disk->disk_name, "drbd%d", minor); disk->private_data = device; diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index c4267da716fe..7f0a60c4079f 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4503,6 +4503,7 @@ static int floppy_alloc_disk(unsigned int drive, unsigned int type) disk->first_minor = TOMINOR(drive) | (type << 2); disk->minors = 1; disk->fops = &floppy_fops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE; if (type) sprintf(disk->disk_name, "fd%d_type%d", drive, type); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 7219d98c6fb8..0954ea8cf9e3 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -2033,7 +2033,6 @@ static int loop_add(int i) */ if (!part_shift) disk->flags |= GENHD_FL_NO_PART; - disk->flags |= GENHD_FL_EXT_DEVT; atomic_set(&lo->lo_refcnt, 0); mutex_init(&lo->lo_mutex); lo->lo_number = i; diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index eb17def1f265..54f7d490f8eb 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1850,7 +1850,6 @@ static int null_gendisk_register(struct nullb *nullb) set_capacity(disk, size); - disk->flags |= GENHD_FL_EXT_DEVT; disk->major = null_major; disk->first_minor = nullb->index; disk->minors = 1; diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 430ee8004a51..255fd3d4b8a8 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -928,6 +928,7 @@ static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, disk->minors = 1; strcpy(disk->disk_name, cd->name); /* umm... */ disk->fops = &pcd_bdops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE; disk->event_flags = DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE; diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index bf8d0ef41a0a..b84a6448a4f7 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -942,6 +942,7 @@ static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, disk->minors = 1; strcpy(disk->disk_name, pf->name); disk->fops = &pf_fops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE; disk->private_data = pf; diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index b53f648302c1..3af0499857ec 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2719,7 +2719,7 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) disk->first_minor = idx; disk->minors = 1; disk->fops = &pktcdvd_ops; - disk->flags = GENHD_FL_REMOVABLE; + disk->flags = GENHD_FL_REMOVABLE | GENHD_FL_NO_PART; strcpy(disk->disk_name, pd->name); disk->private_data = pd; diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index c1876646a4cb..4f90819e245e 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -742,6 +742,7 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) priv->gendisk = gendisk; gendisk->major = ps3vram_major; gendisk->minors = 1; + gendisk->flags |= GENHD_FL_NO_PART; gendisk->fops = &ps3vram_fops; gendisk->private_data = dev; strlcpy(gendisk->disk_name, DEVICE_NAME, sizeof(gendisk->disk_name)); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 953fa134cd3d..8f140da1efe3 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -4924,12 +4924,10 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) rbd_dev->dev_id); disk->major = rbd_dev->major; disk->first_minor = rbd_dev->minor; - if (single_major) { + if (single_major) disk->minors = (1 << RBD_SINGLE_MAJOR_PART_SHIFT); - disk->flags |= GENHD_FL_EXT_DEVT; - } else { + else disk->minors = RBD_MINORS_PER_MAJOR; - } disk->fops = &rbd_bd_ops; disk->private_data = rbd_dev; diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 821594cd1315..fef65a18d56f 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -840,6 +840,7 @@ static int swim_floppy_init(struct swim_priv *swd) swd->unit[drive].disk->minors = 1; sprintf(swd->unit[drive].disk->disk_name, "fd%d", drive); swd->unit[drive].disk->fops = &floppy_fops; + swd->unit[drive].disk->flags |= GENHD_FL_NO_PART; swd->unit[drive].disk->events = DISK_EVENT_MEDIA_CHANGE; swd->unit[drive].disk->private_data = &swd->unit[drive]; set_capacity(swd->unit[drive].disk, 2880); diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 4b91c9aa5892..6c39f2c9f806 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -1227,7 +1227,7 @@ static int swim3_attach(struct macio_dev *mdev, disk->fops = &floppy_fops; disk->private_data = fs; disk->events = DISK_EVENT_MEDIA_CHANGE; - disk->flags |= GENHD_FL_REMOVABLE; + disk->flags |= GENHD_FL_REMOVABLE | GENHD_FL_NO_PART; sprintf(disk->disk_name, "fd%d", floppy_count); set_capacity(disk, 2880); rc = add_disk(disk); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 6ae38776e30e..cfa303fa7318 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -843,7 +843,6 @@ static int virtblk_probe(struct virtio_device *vdev) vblk->disk->minors = 1 << PART_BITS; vblk->disk->private_data = vblk; vblk->disk->fops = &virtblk_fops; - vblk->disk->flags |= GENHD_FL_EXT_DEVT; vblk->index = index; /* configure queue flush support */ diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c index ccc52c935faf..7a6ed83481b8 100644 --- a/drivers/block/z2ram.c +++ b/drivers/block/z2ram.c @@ -327,6 +327,7 @@ static int z2ram_register_disk(int minor) disk->major = Z2RAM_MAJOR; disk->first_minor = minor; disk->minors = 1; + disk->flags |= GENHD_FL_NO_PART; disk->fops = &z2_fops; if (minor) sprintf(disk->disk_name, "z2ram%d", minor); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 25071126995b..f6da5293b913 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1947,6 +1947,7 @@ static int zram_add(void) zram->disk->major = zram_major; zram->disk->first_minor = device_id; zram->disk->minors = 1; + zram->disk->flags |= GENHD_FL_NO_PART; zram->disk->fops = &zram_devops; zram->disk->private_data = zram; snprintf(zram->disk->disk_name, 16, "zram%d", device_id); diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index d50cc1fd34d5..faead41709bc 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c @@ -719,6 +719,7 @@ static void probe_gdrom_setupdisk(void) gd.disk->major = gdrom_major; gd.disk->first_minor = 1; gd.disk->minors = 1; + gd.disk->flags |= GENHD_FL_NO_PART; strcpy(gd.disk->disk_name, GDROM_DEV_NAME); } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 662742a310cb..280918cdcabd 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1778,6 +1778,7 @@ static struct mapped_device *alloc_dev(int minor) md->disk->major = _major; md->disk->first_minor = minor; md->disk->minors = 1; + md->disk->flags |= GENHD_FL_NO_PART; md->disk->fops = &dm_blk_dops; md->disk->queue = md->queue; md->disk->private_data = md; diff --git a/drivers/md/md.c b/drivers/md/md.c index 5111ed966947..7fbf6f0ac01b 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5707,11 +5707,6 @@ static int md_alloc(dev_t dev, char *name) mddev->queue = disk->queue; blk_set_stacking_limits(&mddev->queue->limits); blk_queue_write_cache(mddev->queue, true, true); - /* Allow extended partitions. This makes the - * 'mdp' device redundant, but we can't really - * remove it now. - */ - disk->flags |= GENHD_FL_EXT_DEVT; disk->events |= DISK_EVENT_MEDIA_CHANGE; mddev->gendisk = disk; error = add_disk(disk); diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 2dd93d49d822..5e0960560eab 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -2395,7 +2395,6 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, md->disk->private_data = md; md->parent = parent; set_disk_ro(md->disk, md->read_only || default_ro); - md->disk->flags = GENHD_FL_EXT_DEVT; if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT)) md->disk->flags |= GENHD_FL_NO_PART; diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c index 302426ab30f8..a78fdf3b30f7 100644 --- a/drivers/mtd/ubi/block.c +++ b/drivers/mtd/ubi/block.c @@ -430,6 +430,7 @@ int ubiblock_create(struct ubi_volume_info *vi) ret = -ENODEV; goto out_cleanup_disk; } + gd->flags |= GENHD_FL_NO_PART; gd->private_data = dev; sprintf(gd->disk_name, "ubiblock%d_%d", dev->ubi_num, dev->vol_id); set_capacity(gd, disk_capacity); diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 65875a598d62..bba1f5dafd38 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3566,7 +3566,6 @@ static int sd_probe(struct device *dev) sd_revalidate_disk(gd); - gd->flags = GENHD_FL_EXT_DEVT; if (sdp->removable) { gd->flags |= GENHD_FL_REMOVABLE; gd->events |= DISK_EVENT_MEDIA_CHANGE; diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 6646797a7756..cf093387e42a 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -684,6 +684,7 @@ static int sr_probe(struct device *dev) disk->minors = 1; sprintf(disk->disk_name, "sr%d", minor); disk->fops = &sr_bdops; + disk->flags |= GENHD_FL_NO_PART; disk->events = DISK_EVENT_MEDIA_CHANGE | DISK_EVENT_EJECT_REQUEST; disk->event_flags = DISK_EVENT_FLAG_POLL | DISK_EVENT_FLAG_UEVENT | DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE; diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 64a2f33ae9ea..b8ced80178d6 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -46,11 +46,6 @@ struct partition_meta_info { * Must not be set for devices which are removed entirely when the * media is removed. * - * ``GENHD_FL_EXT_DEVT`` (0x0040): the driver supports extended - * dynamic ``dev_t``, i.e. it wants extended device numbers - * (``BLOCK_EXT_MAJOR``). - * This affects the maximum number of partitions. - * * ``GENHD_FL_NO_PART`` (0x0200): partition support is disabled. * The kernel will not scan for partitions from add_disk, and users * can't add partitions manually. @@ -64,7 +59,6 @@ struct partition_meta_info { #define GENHD_FL_REMOVABLE 0x0001 /* 2 is unused (used to be GENHD_FL_DRIVERFS) */ /* 4 is unused (used to be GENHD_FL_MEDIA_CHANGE_NOTIFY) */ -#define GENHD_FL_EXT_DEVT 0x0040 #define GENHD_FL_NO_PART 0x0200 #define GENHD_FL_HIDDEN 0x0400 @@ -94,13 +88,13 @@ struct blk_integrity { }; struct gendisk { - /* major, first_minor and minors are input parameters only, - * don't use directly. Use disk_devt() and disk_max_parts(). + /* + * major/first_minor/minors should not be set by any new driver, the + * block core will take care of allocating them automatically. */ - int major; /* major number of driver */ + int major; int first_minor; - int minors; /* maximum number of minors, =1 for - * disks that can't be partitioned. */ + int minors; char disk_name[DISK_NAME_LEN]; /* name of major driver */ @@ -164,18 +158,6 @@ static inline bool disk_live(struct gendisk *disk) #define disk_to_cdi(disk) NULL #endif -static inline int disk_max_parts(struct gendisk *disk) -{ - if (disk->flags & GENHD_FL_EXT_DEVT) - return DISK_MAX_PARTS; - return disk->minors; -} - -static inline bool disk_part_scan_enabled(struct gendisk *disk) -{ - return disk_max_parts(disk) > 1 && !(disk->flags & GENHD_FL_NO_PART); -} - static inline dev_t disk_devt(struct gendisk *disk) { return MKDEV(disk->major, disk->first_minor); -- cgit v1.2.3-59-g8ed1b From 430cc5d3ab4d0ba0bd011cfbb0035e46ba92920c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Nov 2021 14:06:24 +0100 Subject: block: cleanup the GENHD_FL_* definitions Switch to an enum and tidy up the documentation. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211122130625.1136848-14-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/genhd.h | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/genhd.h b/include/linux/genhd.h index b8ced80178d6..6906a45bc761 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -39,28 +39,24 @@ struct partition_meta_info { /** * DOC: genhd capability flags * - * ``GENHD_FL_REMOVABLE`` (0x0001): indicates that the block device - * gives access to removable media. - * When set, the device remains present even when media is not - * inserted. - * Must not be set for devices which are removed entirely when the + * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to + * removable media. When set, the device remains present even when media is not + * inserted. Shall not be set for devices which are removed entirely when the * media is removed. * - * ``GENHD_FL_NO_PART`` (0x0200): partition support is disabled. - * The kernel will not scan for partitions from add_disk, and users - * can't add partitions manually. + * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events, + * doesn't appear in sysfs, and can't be opened from userspace or using + * blkdev_get*. Used for the underlying components of multipath devices. + * + * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not + * scan for partitions from add_disk, and users can't add partitions manually. * - * ``GENHD_FL_HIDDEN`` (0x0400): the block device is hidden; it - * doesn't produce events, doesn't appear in sysfs, and doesn't have - * an associated ``bdev``. - * Implies ``GENHD_FL_NO_PART``. - * Used for multipath devices. */ -#define GENHD_FL_REMOVABLE 0x0001 -/* 2 is unused (used to be GENHD_FL_DRIVERFS) */ -/* 4 is unused (used to be GENHD_FL_MEDIA_CHANGE_NOTIFY) */ -#define GENHD_FL_NO_PART 0x0200 -#define GENHD_FL_HIDDEN 0x0400 +enum { + GENHD_FL_REMOVABLE = 1 << 0, + GENHD_FL_HIDDEN = 1 << 1, + GENHD_FL_NO_PART = 1 << 2, +}; enum { DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */ -- cgit v1.2.3-59-g8ed1b From 48b5c1fbcd8c5bc6b91a56399a5257b801391dd8 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 13 Nov 2021 14:03:26 -0700 Subject: block: only allocate poll_stats if there's a user of them This is essentially never used, yet it's about 1/3rd of the total queue size. Allocate it when needed, and don't embed it in the queue. Kill the queue flag for this while at it, since we can just check the assigned pointer now. Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq-debugfs.c | 1 - block/blk-mq.c | 10 ++++------ block/blk-stat.c | 18 ++++++++++++++++++ block/blk-stat.h | 1 + block/blk-sysfs.c | 3 ++- include/linux/blkdev.h | 3 +-- 6 files changed, 26 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4f2cf8399f3d..f4022b198580 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -122,7 +122,6 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(FUA), QUEUE_FLAG_NAME(DAX), QUEUE_FLAG_NAME(STATS), - QUEUE_FLAG_NAME(POLL_STATS), QUEUE_FLAG_NAME(REGISTERED), QUEUE_FLAG_NAME(QUIESCED), QUEUE_FLAG_NAME(PCI_P2PDMA), diff --git a/block/blk-mq.c b/block/blk-mq.c index 3af88ffc9e2c..7cd408408a37 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4581,11 +4581,10 @@ EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); /* Enable polling stats and return whether they were already enabled. */ static bool blk_poll_stats_enable(struct request_queue *q) { - if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || - blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q)) + if (q->poll_stat) return true; - blk_stat_add_callback(q, q->poll_cb); - return false; + + return blk_stats_alloc_enable(q); } static void blk_mq_poll_stats_start(struct request_queue *q) @@ -4594,8 +4593,7 @@ static void blk_mq_poll_stats_start(struct request_queue *q) * We don't arm the callback if polling stats are not enabled or the * callback is already active. */ - if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || - blk_stat_is_active(q->poll_cb)) + if (!q->poll_stat || blk_stat_is_active(q->poll_cb)) return; blk_stat_activate_msecs(q->poll_cb, 100); diff --git a/block/blk-stat.c b/block/blk-stat.c index ae3dd1fb8e61..efb2a80db906 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -219,3 +219,21 @@ void blk_free_queue_stats(struct blk_queue_stats *stats) kfree(stats); } + +bool blk_stats_alloc_enable(struct request_queue *q) +{ + struct blk_rq_stat *poll_stat; + + poll_stat = kcalloc(BLK_MQ_POLL_STATS_BKTS, sizeof(*poll_stat), + GFP_ATOMIC); + if (!poll_stat) + return false; + + if (cmpxchg(&q->poll_stat, NULL, poll_stat) != NULL) { + kfree(poll_stat); + return true; + } + + blk_stat_add_callback(q, q->poll_cb); + return false; +} diff --git a/block/blk-stat.h b/block/blk-stat.h index 17b47a86eefb..58f029af49e5 100644 --- a/block/blk-stat.h +++ b/block/blk-stat.h @@ -64,6 +64,7 @@ struct blk_stat_callback { struct blk_queue_stats *blk_alloc_queue_stats(void); void blk_free_queue_stats(struct blk_queue_stats *); +bool blk_stats_alloc_enable(struct request_queue *q); void blk_stat_add(struct request *rq, u64 now); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index cd75b0f73dc6..c079be1c58a3 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -785,11 +785,12 @@ static void blk_release_queue(struct kobject *kobj) might_sleep(); - if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags)) + if (q->poll_stat) blk_stat_remove_callback(q, q->poll_cb); blk_stat_free_callback(q->poll_cb); blk_free_queue_stats(q->stats); + kfree(q->poll_stat); blk_exit_queue(q); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index bd4370baccca..74118e67f649 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -267,7 +267,7 @@ struct request_queue { int poll_nsec; struct blk_stat_callback *poll_cb; - struct blk_rq_stat poll_stat[BLK_MQ_POLL_STATS_BKTS]; + struct blk_rq_stat *poll_stat; struct timer_list timeout; struct work_struct timeout_work; @@ -397,7 +397,6 @@ struct request_queue { #define QUEUE_FLAG_FUA 18 /* device supports FUA writes */ #define QUEUE_FLAG_DAX 19 /* device supports DAX */ #define QUEUE_FLAG_STATS 20 /* track IO start and completion times */ -#define QUEUE_FLAG_POLL_STATS 21 /* collecting stats for hybrid polling */ #define QUEUE_FLAG_REGISTERED 22 /* queue has been registered to a disk */ #define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */ #define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */ -- cgit v1.2.3-59-g8ed1b From 72cd9df2ef788d88c138d51223a01ca6281f232d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 23 Nov 2021 17:37:33 -0800 Subject: blk-crypto: remove blk_crypto_unregister() This function is trivial and is only used in one place. Having this function is misleading because it implies that blk_crypto_register() needs to be paired with blk_crypto_unregister(), which is not the case. Just set disk->queue->crypto_profile to NULL directly. Signed-off-by: Eric Biggers Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211124013733.347612-1-ebiggers@kernel.org Signed-off-by: Jens Axboe --- block/blk-crypto-profile.c | 5 ----- block/blk-integrity.c | 2 +- include/linux/blkdev.h | 4 ---- 3 files changed, 1 insertion(+), 10 deletions(-) (limited to 'include/linux') diff --git a/block/blk-crypto-profile.c b/block/blk-crypto-profile.c index 605ba0626a5c..96c511967386 100644 --- a/block/blk-crypto-profile.c +++ b/block/blk-crypto-profile.c @@ -463,11 +463,6 @@ bool blk_crypto_register(struct blk_crypto_profile *profile, } EXPORT_SYMBOL_GPL(blk_crypto_register); -void blk_crypto_unregister(struct request_queue *q) -{ - q->crypto_profile = NULL; -} - /** * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities * by child device diff --git a/block/blk-integrity.c b/block/blk-integrity.c index d670d54e5f7a..69eed260a823 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -411,7 +411,7 @@ void blk_integrity_register(struct gendisk *disk, struct blk_integrity *template #ifdef CONFIG_BLK_INLINE_ENCRYPTION if (disk->queue->crypto_profile) { pr_warn("blk-integrity: Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n"); - blk_crypto_unregister(disk->queue); + disk->queue->crypto_profile = NULL; } #endif } diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 74118e67f649..0a4416ef4fbf 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1170,8 +1170,6 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned lo bool blk_crypto_register(struct blk_crypto_profile *profile, struct request_queue *q); -void blk_crypto_unregister(struct request_queue *q); - #else /* CONFIG_BLK_INLINE_ENCRYPTION */ static inline bool blk_crypto_register(struct blk_crypto_profile *profile, @@ -1180,8 +1178,6 @@ static inline bool blk_crypto_register(struct blk_crypto_profile *profile, return true; } -static inline void blk_crypto_unregister(struct request_queue *q) { } - #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ enum blk_unique_id { -- cgit v1.2.3-59-g8ed1b From e8dc17e2893b4107366004810ca2a4acf1fc8563 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 25 Oct 2021 09:06:57 +0200 Subject: blk-mq: Add blk_mq_complete_request_direct() Add blk_mq_complete_request_direct() which completes the block request directly instead deferring it to softirq for single queue devices. This is useful for devices which complete the requests in preemptible context and raising softirq from means scheduling ksoftirqd. Signed-off-by: Sebastian Andrzej Siewior Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211025070658.1565848-2-bigeasy@linutronix.de Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 308edc2a4925..d952c3442261 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -752,6 +752,17 @@ static inline void blk_mq_set_request_complete(struct request *rq) WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); } +/* + * Complete the request directly instead of deferring it to softirq or + * completing it another CPU. Useful in preemptible instead of an interrupt. + */ +static inline void blk_mq_complete_request_direct(struct request *rq, + void (*complete)(struct request *rq)) +{ + WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); + complete(rq); +} + void blk_mq_start_request(struct request *rq); void blk_mq_end_request(struct request *rq, blk_status_t error); void __blk_mq_end_request(struct request *rq, blk_status_t error); -- cgit v1.2.3-59-g8ed1b From 88c9a2ce520ba381bb70658c80ec704f4d60f728 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 26 Nov 2021 12:58:05 +0100 Subject: fork: move copy_io to block/blk-ioc.c Move the copying of the I/O context to the block layer as that is where we can use the proper low-level interfaces. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211126115817.2087431-3-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ioc.c | 27 +++++++++++++++++++++++++++ include/linux/iocontext.h | 23 +++++++++++++---------- kernel/fork.c | 26 -------------------------- 3 files changed, 40 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 70c99e85aee5..3b31cfad4b75 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -322,6 +322,33 @@ struct io_context *get_task_io_context(struct task_struct *task, return NULL; } +int __copy_io(unsigned long clone_flags, struct task_struct *tsk) +{ + struct io_context *ioc = current->io_context; + struct io_context *new_ioc; + + /* + * Share io context with parent, if CLONE_IO is set + */ + if (clone_flags & CLONE_IO) { + get_io_context_active(ioc); + + WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0); + atomic_inc(&ioc->nr_tasks); + + tsk->io_context = ioc; + } else if (ioprio_valid(ioc->ioprio)) { + new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); + if (unlikely(!new_ioc)) + return -ENOMEM; + + new_ioc->ioprio = ioc->ioprio; + put_io_context(new_ioc); + } + + return 0; +} + /** * ioc_lookup_icq - lookup io_cq from ioc * @ioc: the associated io_context diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 0a9dc40b7be8..bcd47d104d8e 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -129,14 +129,6 @@ static inline void get_io_context_active(struct io_context *ioc) atomic_inc(&ioc->active_ref); } -static inline void ioc_task_link(struct io_context *ioc) -{ - get_io_context_active(ioc); - - WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0); - atomic_inc(&ioc->nr_tasks); -} - struct task_struct; #ifdef CONFIG_BLOCK void put_io_context(struct io_context *ioc); @@ -144,10 +136,21 @@ void put_io_context_active(struct io_context *ioc); void exit_io_context(struct task_struct *task); struct io_context *get_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node); +int __copy_io(unsigned long clone_flags, struct task_struct *tsk); +static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk) +{ + if (!current->io_context) + return 0; + return __copy_io(clone_flags, tsk); +} #else struct io_context; static inline void put_io_context(struct io_context *ioc) { } static inline void exit_io_context(struct task_struct *task) { } -#endif +static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk) +{ + return 0; +} +#endif /* CONFIG_BLOCK */ -#endif +#endif /* IOCONTEXT_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 3244cc56b697..3161d7980155 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1556,32 +1556,6 @@ out: return error; } -static int copy_io(unsigned long clone_flags, struct task_struct *tsk) -{ -#ifdef CONFIG_BLOCK - struct io_context *ioc = current->io_context; - struct io_context *new_ioc; - - if (!ioc) - return 0; - /* - * Share io context with parent, if CLONE_IO is set - */ - if (clone_flags & CLONE_IO) { - ioc_task_link(ioc); - tsk->io_context = ioc; - } else if (ioprio_valid(ioc->ioprio)) { - new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); - if (unlikely(!new_ioc)) - return -ENOMEM; - - new_ioc->ioprio = ioc->ioprio; - put_io_context(new_ioc); - } -#endif - return 0; -} - static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) { struct sighand_struct *sig; -- cgit v1.2.3-59-g8ed1b From 3304742562d27fb87a6d8291cc48824dd20f6964 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 26 Nov 2021 12:58:09 +0100 Subject: block: mark put_io_context_active static Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211126115817.2087431-7-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ioc.c | 2 +- include/linux/iocontext.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 3b31cfad4b75..f3ff495756cb 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -175,7 +175,7 @@ void put_io_context(struct io_context *ioc) * Undo get_io_context_active(). If active reference reaches zero after * put, @ioc can never issue further IOs and ioscheds are notified. */ -void put_io_context_active(struct io_context *ioc) +static void put_io_context_active(struct io_context *ioc) { struct io_cq *icq; diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index bcd47d104d8e..3ba45953d522 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -132,7 +132,6 @@ static inline void get_io_context_active(struct io_context *ioc) struct task_struct; #ifdef CONFIG_BLOCK void put_io_context(struct io_context *ioc); -void put_io_context_active(struct io_context *ioc); void exit_io_context(struct task_struct *task); struct io_context *get_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node); -- cgit v1.2.3-59-g8ed1b From 50569c24be61eafb3efa06e2a3ccd447f75ae1b0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 26 Nov 2021 12:58:12 +0100 Subject: block: remove get_io_context_active Fold it into it's only caller, and remove a lof of the debug checks that are not needed. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211126115817.2087431-10-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ioc.c | 8 +++----- include/linux/iocontext.h | 16 ---------------- 2 files changed, 3 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 3ba15c867dfa..cc4eb2ba87f7 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -173,7 +173,7 @@ EXPORT_SYMBOL_GPL(put_io_context); * put_io_context_active - put active reference on ioc * @ioc: ioc of interest * - * Undo get_io_context_active(). If active reference reaches zero after + * Put an active reference to an ioc. If active reference reaches zero after * put, @ioc can never issue further IOs and ioscheds are notified. */ static void put_io_context_active(struct io_context *ioc) @@ -333,11 +333,9 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) * Share io context with parent, if CLONE_IO is set */ if (clone_flags & CLONE_IO) { - get_io_context_active(ioc); - - WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0); + atomic_long_inc(&ioc->refcount); + atomic_inc(&ioc->active_ref); atomic_inc(&ioc->nr_tasks); - tsk->io_context = ioc; } else if (ioprio_valid(ioc->ioprio)) { new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 3ba45953d522..c1229fbd6691 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -113,22 +113,6 @@ struct io_context { struct work_struct release_work; }; -/** - * get_io_context_active - get active reference on ioc - * @ioc: ioc of interest - * - * Only iocs with active reference can issue new IOs. This function - * acquires an active reference on @ioc. The caller must already have an - * active reference on @ioc. - */ -static inline void get_io_context_active(struct io_context *ioc) -{ - WARN_ON_ONCE(atomic_long_read(&ioc->refcount) <= 0); - WARN_ON_ONCE(atomic_read(&ioc->active_ref) <= 0); - atomic_long_inc(&ioc->refcount); - atomic_inc(&ioc->active_ref); -} - struct task_struct; #ifdef CONFIG_BLOCK void put_io_context(struct io_context *ioc); -- cgit v1.2.3-59-g8ed1b From f3fa33acca9f0058157214800f68b10d8e71ab7a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 26 Nov 2021 13:18:00 +0100 Subject: block: remove the ->rq_disk field in struct request Just use the disk attached to the request_queue instead. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20211126121802.2090656-4-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-flush.c | 3 +-- block/blk-mq.c | 14 ++++++-------- block/blk.h | 2 +- drivers/block/amiflop.c | 2 +- drivers/block/ataflop.c | 6 +++--- drivers/block/floppy.c | 6 +++--- drivers/block/null_blk/trace.h | 2 +- drivers/block/paride/pcd.c | 2 +- drivers/block/paride/pd.c | 4 ++-- drivers/block/paride/pf.c | 4 ++-- drivers/block/rnbd/rnbd-clt.c | 4 ++-- drivers/block/sunvdc.c | 2 +- drivers/md/dm-mpath.c | 1 - drivers/mmc/core/block.c | 2 +- drivers/nvme/host/fault_inject.c | 2 +- drivers/nvme/host/trace.h | 6 +++--- drivers/scsi/scsi_lib.c | 3 ++- drivers/scsi/scsi_logging.c | 4 +++- drivers/scsi/sd.c | 24 ++++++++++++------------ drivers/scsi/sd_zbc.c | 8 ++++---- drivers/scsi/sr.c | 4 ++-- drivers/scsi/virtio_scsi.c | 2 +- drivers/usb/storage/transport.c | 2 +- include/linux/blk-mq.h | 4 ---- include/scsi/scsi_cmnd.h | 2 +- include/scsi/scsi_device.h | 4 ++-- include/trace/events/block.h | 8 ++++---- kernel/trace/blktrace.c | 2 +- 28 files changed, 62 insertions(+), 67 deletions(-) (limited to 'include/linux') diff --git a/block/blk-flush.c b/block/blk-flush.c index 902e80e48e4a..fd5187a0898d 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -145,7 +145,7 @@ static void blk_flush_queue_rq(struct request *rq, bool add_front) static void blk_account_io_flush(struct request *rq) { - struct block_device *part = rq->rq_disk->part0; + struct block_device *part = rq->q->disk->part0; part_stat_lock(); part_stat_inc(part, ios[STAT_FLUSH]); @@ -339,7 +339,6 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH; flush_rq->cmd_flags |= (flags & REQ_DRV) | (flags & REQ_FAILFAST_MASK); flush_rq->rq_flags |= RQF_FLUSH_SEQ; - flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io; /* * Order WRITE ->end_io and WRITE rq->ref, and its pair is the one diff --git a/block/blk-mq.c b/block/blk-mq.c index 3e67662f7801..f1abfd2e24f7 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -377,7 +377,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, rq->start_time_ns = ktime_get_ns(); else rq->start_time_ns = 0; - rq->rq_disk = NULL; rq->part = NULL; #ifdef CONFIG_BLK_RQ_ALLOC_TIME rq->alloc_time_ns = alloc_time_ns; @@ -659,7 +658,7 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug) void blk_dump_rq_flags(struct request *rq, char *msg) { printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, - rq->rq_disk ? rq->rq_disk->disk_name : "?", + rq->q->disk ? rq->q->disk->disk_name : "?", (unsigned long long) rq->cmd_flags); printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", @@ -712,7 +711,7 @@ static void blk_print_req_error(struct request *req, blk_status_t status) "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x " "phys_seg %u prio class %u\n", blk_status_to_str(status), - req->rq_disk ? req->rq_disk->disk_name : "?", + req->q->disk ? req->q->disk->disk_name : "?", blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)), req->cmd_flags & ~REQ_OP_MASK, req->nr_phys_segments, @@ -853,8 +852,8 @@ static void __blk_account_io_start(struct request *rq) /* passthrough requests can hold bios that do not have ->bi_bdev set */ if (rq->bio && rq->bio->bi_bdev) rq->part = rq->bio->bi_bdev; - else - rq->part = rq->rq_disk->part0; + else if (rq->q->disk) + rq->part = rq->q->disk->part0; part_stat_lock(); update_io_ticks(rq->part, jiffies, false); @@ -1172,7 +1171,6 @@ void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq, WARN_ON(irqs_disabled()); WARN_ON(!blk_rq_is_passthrough(rq)); - rq->rq_disk = bd_disk; rq->end_io = done; blk_account_io_start(rq); @@ -2902,8 +2900,8 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * if (ret != BLK_STS_OK) return ret; - if (rq->rq_disk && - should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq))) + if (rq->q->disk && + should_fail_request(rq->q->disk->part0, blk_rq_bytes(rq))) return BLK_STS_IOERR; if (blk_crypto_insert_cloned_request(rq)) diff --git a/block/blk.h b/block/blk.h index 3be0fdf76c9a..a55d82c3d1c2 100644 --- a/block/blk.h +++ b/block/blk.h @@ -324,7 +324,7 @@ int blk_dev_init(void); */ static inline bool blk_do_io_stat(struct request *rq) { - return (rq->rq_flags & RQF_IO_STAT) && rq->rq_disk; + return (rq->rq_flags & RQF_IO_STAT) && rq->q->disk; } void update_io_ticks(struct block_device *part, unsigned long now, bool end); diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index 1eec5113d0b5..5a566f2fd533 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c @@ -1505,7 +1505,7 @@ static blk_status_t amiflop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; - struct amiga_floppy_struct *floppy = rq->rq_disk->private_data; + struct amiga_floppy_struct *floppy = rq->q->disk->private_data; blk_status_t err; if (!spin_trylock_irq(&amiflop_lock)) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index f3ff9babdb5c..5d819a466e2f 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -1502,7 +1502,7 @@ static void setup_req_params( int drive ) static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { - struct atari_floppy_struct *floppy = bd->rq->rq_disk->private_data; + struct atari_floppy_struct *floppy = bd->rq->q->disk->private_data; int drive = floppy - unit; int type = floppy->type; @@ -1538,7 +1538,7 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, if (!UDT) { Probing = 1; UDT = atari_disk_type + StartDiskType[DriveType]; - set_capacity(bd->rq->rq_disk, UDT->blocks); + set_capacity(bd->rq->q->disk, UDT->blocks); UD.autoprobe = 1; } } @@ -1558,7 +1558,7 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, } type = minor2disktype[type].index; UDT = &atari_disk_type[type]; - set_capacity(bd->rq->rq_disk, UDT->blocks); + set_capacity(bd->rq->q->disk, UDT->blocks); UD.autoprobe = 0; } diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 7f0a60c4079f..0c638de25023 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -2259,7 +2259,7 @@ static int do_format(int drive, struct format_descr *tmp_format_req) static void floppy_end_request(struct request *req, blk_status_t error) { unsigned int nr_sectors = current_count_sectors; - unsigned int drive = (unsigned long)req->rq_disk->private_data; + unsigned int drive = (unsigned long)req->q->disk->private_data; /* current_count_sectors can be zero if transfer failed */ if (error) @@ -2550,7 +2550,7 @@ static int make_raw_rw_request(void) if (WARN(max_buffer_sectors == 0, "VFS: Block I/O scheduled on unopened device\n")) return 0; - set_fdc((long)current_req->rq_disk->private_data); + set_fdc((long)current_req->q->disk->private_data); raw_cmd = &default_raw_cmd; raw_cmd->flags = FD_RAW_SPIN | FD_RAW_NEED_DISK | FD_RAW_NEED_SEEK; @@ -2792,7 +2792,7 @@ do_request: return; } } - drive = (long)current_req->rq_disk->private_data; + drive = (long)current_req->q->disk->private_data; set_fdc(drive); reschedule_timeout(current_drive, "redo fd request"); diff --git a/drivers/block/null_blk/trace.h b/drivers/block/null_blk/trace.h index ce3b430e88c5..86d6c12c603c 100644 --- a/drivers/block/null_blk/trace.h +++ b/drivers/block/null_blk/trace.h @@ -44,7 +44,7 @@ TRACE_EVENT(nullb_zone_op, __entry->op = req_op(cmd->rq); __entry->zone_no = zone_no; __entry->zone_cond = zone_cond; - __assign_disk_name(__entry->disk, cmd->rq->rq_disk); + __assign_disk_name(__entry->disk, cmd->rq->q->disk); ), TP_printk("%s req=%-15s zone_no=%u zone_cond=%-10s", __print_disk_name(__entry->disk), diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 255fd3d4b8a8..f462ad67931a 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -690,7 +690,7 @@ static void pcd_request(void) if (!pcd_req && !set_next_request()) return; - cd = pcd_req->rq_disk->private_data; + cd = pcd_req->q->disk->private_data; if (cd != pcd_current) pcd_bufblk = -1; pcd_current = cd; diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index fba865058a17..4f8cce510562 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -430,7 +430,7 @@ static void run_fsm(void) int stop = 0; if (!phase) { - pd_current = pd_req->rq_disk->private_data; + pd_current = pd_req->q->disk->private_data; pi_current = pd_current->pi; phase = do_pd_io_start; } @@ -492,7 +492,7 @@ static enum action do_pd_io_start(void) case REQ_OP_WRITE: pd_block = blk_rq_pos(pd_req); pd_count = blk_rq_cur_sectors(pd_req); - if (pd_block + pd_count > get_capacity(pd_req->rq_disk)) + if (pd_block + pd_count > get_capacity(pd_req->q->disk)) return Fail; pd_run = blk_rq_sectors(pd_req); pd_buf = bio_data(pd_req->bio); diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index b84a6448a4f7..292e9a4ce1b9 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -746,12 +746,12 @@ repeat: if (!pf_req && !set_next_request()) return; - pf_current = pf_req->rq_disk->private_data; + pf_current = pf_req->q->disk->private_data; pf_block = blk_rq_pos(pf_req); pf_run = blk_rq_sectors(pf_req); pf_count = blk_rq_cur_sectors(pf_req); - if (pf_block + pf_count > get_capacity(pf_req->rq_disk)) { + if (pf_block + pf_count > get_capacity(pf_req->q->disk)) { pf_end_request(BLK_STS_IOERR); goto repeat; } diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 2df0657cdf00..67a8edbaa1fd 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -393,7 +393,7 @@ static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu) static void rnbd_softirq_done_fn(struct request *rq) { - struct rnbd_clt_dev *dev = rq->rq_disk->private_data; + struct rnbd_clt_dev *dev = rq->q->disk->private_data; struct rnbd_clt_session *sess = dev->sess; struct rnbd_iu *iu; @@ -1133,7 +1133,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; - struct rnbd_clt_dev *dev = rq->rq_disk->private_data; + struct rnbd_clt_dev *dev = rq->q->disk->private_data; struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq); int err; blk_status_t ret = BLK_STS_IOERR; diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 2157936de623..146d85d80e0e 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -462,7 +462,7 @@ static int __vdc_tx_trigger(struct vdc_port *port) static int __send_request(struct request *req) { - struct vdc_port *port = req->rq_disk->private_data; + struct vdc_port *port = req->q->disk->private_data; struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; struct scatterlist sg[MAX_RING_COOKIES]; struct vdc_req_entry *rqe; diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 90dc9cc48881..f4719b65e5e3 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -550,7 +550,6 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq, return DM_MAPIO_REQUEUE; } clone->bio = clone->biotail = NULL; - clone->rq_disk = bdev->bd_disk; clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; *__clone = clone; diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 635b79899b9f..dc094f73d335 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -1837,7 +1837,7 @@ static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req) /* Reset if the card is in a bad state */ if (!mmc_host_is_spi(mq->card->host) && err && mmc_blk_reset(md, card->host, type)) { - pr_err("%s: recovery failed!\n", req->rq_disk->disk_name); + pr_err("%s: recovery failed!\n", req->q->disk->disk_name); mqrq->retries = MMC_NO_RETRIES; return; } diff --git a/drivers/nvme/host/fault_inject.c b/drivers/nvme/host/fault_inject.c index 1352159733b0..83d2e6860d38 100644 --- a/drivers/nvme/host/fault_inject.c +++ b/drivers/nvme/host/fault_inject.c @@ -56,7 +56,7 @@ void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject) void nvme_should_fail(struct request *req) { - struct gendisk *disk = req->rq_disk; + struct gendisk *disk = req->q->disk; struct nvme_fault_inject *fault_inject = NULL; u16 status; diff --git a/drivers/nvme/host/trace.h b/drivers/nvme/host/trace.h index 35bac7a25422..b5f85259461a 100644 --- a/drivers/nvme/host/trace.h +++ b/drivers/nvme/host/trace.h @@ -68,7 +68,7 @@ TRACE_EVENT(nvme_setup_cmd, __entry->nsid = le32_to_cpu(cmd->common.nsid); __entry->metadata = !!blk_integrity_rq(req); __entry->fctype = cmd->fabrics.fctype; - __assign_disk_name(__entry->disk, req->rq_disk); + __assign_disk_name(__entry->disk, req->q->disk); memcpy(__entry->cdw10, &cmd->common.cdw10, sizeof(__entry->cdw10)); ), @@ -103,7 +103,7 @@ TRACE_EVENT(nvme_complete_rq, __entry->retries = nvme_req(req)->retries; __entry->flags = nvme_req(req)->flags; __entry->status = nvme_req(req)->status; - __assign_disk_name(__entry->disk, req->rq_disk); + __assign_disk_name(__entry->disk, req->q->disk); ), TP_printk("nvme%d: %sqid=%d, cmdid=%u, res=%#llx, retries=%u, flags=0x%x, status=%#x", __entry->ctrl_id, __print_disk_name(__entry->disk), @@ -153,7 +153,7 @@ TRACE_EVENT(nvme_sq, ), TP_fast_assign( __entry->ctrl_id = nvme_req(req)->ctrl->instance; - __assign_disk_name(__entry->disk, req->rq_disk); + __assign_disk_name(__entry->disk, req->q->disk); __entry->qid = nvme_req_qid(req); __entry->sq_head = le16_to_cpu(sq_head); __entry->sq_tail = sq_tail; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 5e8b5ecb3245..c23cf8e7b3c3 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -543,8 +543,9 @@ static bool scsi_end_request(struct request *req, blk_status_t error, if (blk_update_request(req, error, bytes)) return true; + // XXX: if (blk_queue_add_random(q)) - add_disk_randomness(req->rq_disk); + add_disk_randomness(req->q->disk); if (!blk_rq_is_passthrough(req)) { WARN_ON_ONCE(!(cmd->flags & SCMD_INITIALIZED)); diff --git a/drivers/scsi/scsi_logging.c b/drivers/scsi/scsi_logging.c index ed9572252a42..1f8f80b2dbfc 100644 --- a/drivers/scsi/scsi_logging.c +++ b/drivers/scsi/scsi_logging.c @@ -30,7 +30,9 @@ static inline const char *scmd_name(const struct scsi_cmnd *scmd) { struct request *rq = scsi_cmd_to_rq((struct scsi_cmnd *)scmd); - return rq->rq_disk ? rq->rq_disk->disk_name : NULL; + if (!rq->q->disk) + return NULL; + return rq->q->disk->disk_name; } static size_t sdev_format_header(char *logbuf, size_t logbuf_len, diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index bba1f5dafd38..8181857ddf53 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -872,7 +872,7 @@ static blk_status_t sd_setup_unmap_cmnd(struct scsi_cmnd *cmd) { struct scsi_device *sdp = cmd->device; struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); unsigned int data_len = 24; @@ -908,7 +908,7 @@ static blk_status_t sd_setup_write_same16_cmnd(struct scsi_cmnd *cmd, { struct scsi_device *sdp = cmd->device; struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; @@ -940,7 +940,7 @@ static blk_status_t sd_setup_write_same10_cmnd(struct scsi_cmnd *cmd, { struct scsi_device *sdp = cmd->device; struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); u32 data_len = sdp->sector_size; @@ -971,7 +971,7 @@ static blk_status_t sd_setup_write_zeroes_cmnd(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); struct scsi_device *sdp = cmd->device; - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); @@ -1068,7 +1068,7 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); struct scsi_device *sdp = cmd->device; - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); struct bio *bio = rq->bio; u64 lba = sectors_to_logical(sdp, blk_rq_pos(rq)); u32 nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); @@ -1116,7 +1116,7 @@ static blk_status_t sd_setup_write_same_cmnd(struct scsi_cmnd *cmd) static blk_status_t sd_setup_flush_cmnd(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); /* flush requests don't perform I/O, zero the S/G table */ memset(&cmd->sdb, 0, sizeof(cmd->sdb)); @@ -1215,7 +1215,7 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); struct scsi_device *sdp = cmd->device; - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); sector_t lba = sectors_to_logical(sdp, blk_rq_pos(rq)); sector_t threshold; unsigned int nr_blocks = sectors_to_logical(sdp, blk_rq_sectors(rq)); @@ -1236,7 +1236,7 @@ static blk_status_t sd_setup_read_write_cmnd(struct scsi_cmnd *cmd) goto fail; } - if (blk_rq_pos(rq) + blk_rq_sectors(rq) > get_capacity(rq->rq_disk)) { + if (blk_rq_pos(rq) + blk_rq_sectors(rq) > get_capacity(rq->q->disk)) { scmd_printk(KERN_ERR, cmd, "access beyond end of device\n"); goto fail; } @@ -1331,7 +1331,7 @@ static blk_status_t sd_init_command(struct scsi_cmnd *cmd) switch (req_op(rq)) { case REQ_OP_DISCARD: - switch (scsi_disk(rq->rq_disk)->provisioning_mode) { + switch (scsi_disk(rq->q->disk)->provisioning_mode) { case SD_LBP_UNMAP: return sd_setup_unmap_cmnd(cmd); case SD_LBP_WS16: @@ -1917,7 +1917,7 @@ static const struct block_device_operations sd_fops = { **/ static void sd_eh_reset(struct scsi_cmnd *scmd) { - struct scsi_disk *sdkp = scsi_disk(scsi_cmd_to_rq(scmd)->rq_disk); + struct scsi_disk *sdkp = scsi_disk(scsi_cmd_to_rq(scmd)->q->disk); /* New SCSI EH run, reset gate variable */ sdkp->ignore_medium_access_errors = false; @@ -1937,7 +1937,7 @@ static void sd_eh_reset(struct scsi_cmnd *scmd) **/ static int sd_eh_action(struct scsi_cmnd *scmd, int eh_disp) { - struct scsi_disk *sdkp = scsi_disk(scsi_cmd_to_rq(scmd)->rq_disk); + struct scsi_disk *sdkp = scsi_disk(scsi_cmd_to_rq(scmd)->q->disk); struct scsi_device *sdev = scmd->device; if (!scsi_device_online(sdev) || @@ -2034,7 +2034,7 @@ static int sd_done(struct scsi_cmnd *SCpnt) unsigned int resid; struct scsi_sense_hdr sshdr; struct request *req = scsi_cmd_to_rq(SCpnt); - struct scsi_disk *sdkp = scsi_disk(req->rq_disk); + struct scsi_disk *sdkp = scsi_disk(req->q->disk); int sense_valid = 0; int sense_deferred = 0; diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index ed06798983f8..65bfd1e170da 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -244,7 +244,7 @@ out: static blk_status_t sd_zbc_cmnd_checks(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); sector_t sector = blk_rq_pos(rq); if (!sd_is_zoned(sdkp)) @@ -322,7 +322,7 @@ blk_status_t sd_zbc_prepare_zone_append(struct scsi_cmnd *cmd, sector_t *lba, unsigned int nr_blocks) { struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); unsigned int wp_offset, zno = blk_rq_zone_no(rq); unsigned long flags; blk_status_t ret; @@ -388,7 +388,7 @@ blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd, { struct request *rq = scsi_cmd_to_rq(cmd); sector_t sector = blk_rq_pos(rq); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); sector_t block = sectors_to_logical(sdkp->device, sector); blk_status_t ret; @@ -443,7 +443,7 @@ static unsigned int sd_zbc_zone_wp_update(struct scsi_cmnd *cmd, { int result = cmd->result; struct request *rq = scsi_cmd_to_rq(cmd); - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); + struct scsi_disk *sdkp = scsi_disk(rq->q->disk); unsigned int zno = blk_rq_zone_no(rq); enum req_opf op = req_op(rq); unsigned long flags; diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 411e2b01966e..7db595c08b20 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -335,7 +335,7 @@ static int sr_done(struct scsi_cmnd *SCpnt) int block_sectors = 0; long error_sector; struct request *rq = scsi_cmd_to_rq(SCpnt); - struct scsi_cd *cd = scsi_cd(rq->rq_disk); + struct scsi_cd *cd = scsi_cd(rq->q->disk); #ifdef DEBUG scmd_printk(KERN_INFO, SCpnt, "done: %x\n", result); @@ -402,7 +402,7 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) ret = scsi_alloc_sgtables(SCpnt); if (ret != BLK_STS_OK) return ret; - cd = scsi_cd(rq->rq_disk); + cd = scsi_cd(rq->q->disk); SCSI_LOG_HLQUEUE(1, scmd_printk(KERN_INFO, SCpnt, "Doing sr request, block = %d\n", block)); diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 28e1d98ae102..65c642b24ecf 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -528,7 +528,7 @@ static void virtio_scsi_init_hdr_pi(struct virtio_device *vdev, if (!rq || !scsi_prot_sg_count(sc)) return; - bi = blk_get_integrity(rq->rq_disk); + bi = blk_get_integrity(rq->q->disk); if (sc->sc_data_direction == DMA_TO_DEVICE) cmd_pi->pi_bytesout = cpu_to_virtio32(vdev, diff --git a/drivers/usb/storage/transport.c b/drivers/usb/storage/transport.c index 4c5a0a49035f..1928b3918242 100644 --- a/drivers/usb/storage/transport.c +++ b/drivers/usb/storage/transport.c @@ -551,7 +551,7 @@ static void last_sector_hacks(struct us_data *us, struct scsi_cmnd *srb) /* Did this command access the last sector? */ sector = (srb->cmnd[2] << 24) | (srb->cmnd[3] << 16) | (srb->cmnd[4] << 8) | (srb->cmnd[5]); - disk = scsi_cmd_to_rq(srb)->rq_disk; + disk = scsi_cmd_to_rq(srb)->q->disk; if (!disk) goto done; sdkp = scsi_disk(disk); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d952c3442261..ede7bef8880a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -100,7 +100,6 @@ struct request { struct request *rq_next; }; - struct gendisk *rq_disk; struct block_device *part; #ifdef CONFIG_BLK_RQ_ALLOC_TIME /* Time that the first bio started allocating this request. */ @@ -890,9 +889,6 @@ static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, rq->__data_len = bio->bi_iter.bi_size; rq->bio = rq->biotail = bio; rq->ioprio = bio_prio(bio); - - if (bio->bi_bdev) - rq->rq_disk = bio->bi_bdev->bd_disk; } void blk_mq_hctx_set_fq_lock_class(struct blk_mq_hw_ctx *hctx, diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index 477a800a9543..6794d7322cbd 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -164,7 +164,7 @@ static inline struct scsi_driver *scsi_cmd_to_driver(struct scsi_cmnd *cmd) { struct request *rq = scsi_cmd_to_rq(cmd); - return *(struct scsi_driver **)rq->rq_disk->private_data; + return *(struct scsi_driver **)rq->q->disk->private_data; } void scsi_done(struct scsi_cmnd *cmd); diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index d1c6fc83b1e3..ab7557d84f75 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -275,9 +275,9 @@ scmd_printk(const char *, const struct scsi_cmnd *, const char *, ...); do { \ struct request *__rq = scsi_cmd_to_rq((scmd)); \ \ - if (__rq->rq_disk) \ + if (__rq->q->disk) \ sdev_dbg((scmd)->device, "[%s] " fmt, \ - __rq->rq_disk->disk_name, ##a); \ + __rq->q->disk->disk_name, ##a); \ else \ sdev_dbg((scmd)->device, fmt, ##a); \ } while (0) diff --git a/include/trace/events/block.h b/include/trace/events/block.h index a95daa4d4caa..27170e40e8c9 100644 --- a/include/trace/events/block.h +++ b/include/trace/events/block.h @@ -85,7 +85,7 @@ TRACE_EVENT(block_rq_requeue, ), TP_fast_assign( - __entry->dev = rq->rq_disk ? disk_devt(rq->rq_disk) : 0; + __entry->dev = rq->q->disk ? disk_devt(rq->q->disk) : 0; __entry->sector = blk_rq_trace_sector(rq); __entry->nr_sector = blk_rq_trace_nr_sectors(rq); @@ -128,7 +128,7 @@ TRACE_EVENT(block_rq_complete, ), TP_fast_assign( - __entry->dev = rq->rq_disk ? disk_devt(rq->rq_disk) : 0; + __entry->dev = rq->q->disk ? disk_devt(rq->q->disk) : 0; __entry->sector = blk_rq_pos(rq); __entry->nr_sector = nr_bytes >> 9; __entry->error = blk_status_to_errno(error); @@ -161,7 +161,7 @@ DECLARE_EVENT_CLASS(block_rq, ), TP_fast_assign( - __entry->dev = rq->rq_disk ? disk_devt(rq->rq_disk) : 0; + __entry->dev = rq->q->disk ? disk_devt(rq->q->disk) : 0; __entry->sector = blk_rq_trace_sector(rq); __entry->nr_sector = blk_rq_trace_nr_sectors(rq); __entry->bytes = blk_rq_bytes(rq); @@ -512,7 +512,7 @@ TRACE_EVENT(block_rq_remap, ), TP_fast_assign( - __entry->dev = disk_devt(rq->rq_disk); + __entry->dev = disk_devt(rq->q->disk); __entry->sector = blk_rq_pos(rq); __entry->nr_sector = blk_rq_sectors(rq); __entry->old_dev = dev; diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index 1183c88634aa..431e41bc4c23 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -1045,7 +1045,7 @@ static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, } r.device_from = cpu_to_be32(dev); - r.device_to = cpu_to_be32(disk_devt(rq->rq_disk)); + r.device_to = cpu_to_be32(disk_devt(rq->q->disk)); r.sector_from = cpu_to_be64(from); __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), -- cgit v1.2.3-59-g8ed1b From b84ba30b6c7a75babdf73b83bc3c7b59b944501a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 26 Nov 2021 13:18:01 +0100 Subject: block: remove the gendisk argument to blk_execute_rq Remove the gendisk aregument to blk_execute_rq and blk_execute_rq_nowait given that it is unused now. Also convert the boolean at_head parameter to actually use the bool type while touching the prototype. Signed-off-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20211126121802.2090656-5-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-mq.c | 10 +++------- block/bsg-lib.c | 2 +- drivers/block/mtip32xx/mtip32xx.c | 2 +- drivers/block/paride/pd.c | 2 +- drivers/block/pktcdvd.c | 2 +- drivers/block/sx8.c | 4 ++-- drivers/block/virtio_blk.c | 2 +- drivers/mmc/core/block.c | 10 +++++----- drivers/nvme/host/core.c | 4 ++-- drivers/nvme/host/pci.c | 7 +++---- drivers/nvme/target/passthru.c | 3 +-- drivers/scsi/scsi_bsg.c | 2 +- drivers/scsi/scsi_error.c | 2 +- drivers/scsi/scsi_ioctl.c | 4 ++-- drivers/scsi/scsi_lib.c | 2 +- drivers/scsi/sg.c | 2 +- drivers/scsi/sr.c | 2 +- drivers/scsi/st.c | 2 +- drivers/scsi/ufs/ufshpb.c | 4 ++-- drivers/target/target_core_pscsi.c | 2 +- include/linux/blk-mq.h | 7 +++---- 21 files changed, 35 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index f1abfd2e24f7..ecfc47fad236 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1153,7 +1153,6 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) /** * blk_execute_rq_nowait - insert a request to I/O scheduler for execution - * @bd_disk: matching gendisk * @rq: request to insert * @at_head: insert request at head or tail of queue * @done: I/O completion handler @@ -1165,8 +1164,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) * Note: * This function will invoke @done directly if the queue is dead. */ -void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq, - int at_head, rq_end_io_fn *done) +void blk_execute_rq_nowait(struct request *rq, bool at_head, rq_end_io_fn *done) { WARN_ON(irqs_disabled()); WARN_ON(!blk_rq_is_passthrough(rq)); @@ -1204,7 +1202,6 @@ static void blk_rq_poll_completion(struct request *rq, struct completion *wait) /** * blk_execute_rq - insert a request into queue for execution - * @bd_disk: matching gendisk * @rq: request to insert * @at_head: insert request at head or tail of queue * @@ -1213,14 +1210,13 @@ static void blk_rq_poll_completion(struct request *rq, struct completion *wait) * for execution and wait for completion. * Return: The blk_status_t result provided to blk_mq_end_request(). */ -blk_status_t blk_execute_rq(struct gendisk *bd_disk, struct request *rq, - int at_head) +blk_status_t blk_execute_rq(struct request *rq, bool at_head) { DECLARE_COMPLETION_ONSTACK(wait); unsigned long hang_check; rq->end_io_data = &wait; - blk_execute_rq_nowait(bd_disk, rq, at_head, blk_end_sync_rq); + blk_execute_rq_nowait(rq, at_head, blk_end_sync_rq); /* Prevent hang_check timer from firing at us during very long I/O */ hang_check = sysctl_hung_task_timeout_secs; diff --git a/block/bsg-lib.c b/block/bsg-lib.c index 10aa378702fa..acfe1357bf6c 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -92,7 +92,7 @@ static int bsg_transport_sg_io_fn(struct request_queue *q, struct sg_io_v4 *hdr, goto out_unmap_bidi_rq; bio = rq->bio; - blk_execute_rq(NULL, rq, !(hdr->flags & BSG_FLAG_Q_AT_TAIL)); + blk_execute_rq(rq, !(hdr->flags & BSG_FLAG_Q_AT_TAIL)); /* * The assignments below don't make much sense, but are kept for diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index c91b9010c1a6..30f471021a40 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -1015,7 +1015,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, rq->timeout = timeout; /* insert request and run queue */ - blk_execute_rq(NULL, rq, true); + blk_execute_rq(rq, true); if (int_cmd->status) { dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index 4f8cce510562..3637c38c72f9 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -781,7 +781,7 @@ static int pd_special_command(struct pd_unit *disk, req = blk_mq_rq_to_pdu(rq); req->func = func; - blk_execute_rq(disk->gd, rq, 0); + blk_execute_rq(rq, false); blk_mq_free_request(rq); return 0; } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 3af0499857ec..887c98d61684 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -722,7 +722,7 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * if (cgc->quiet) rq->rq_flags |= RQF_QUIET; - blk_execute_rq(pd->bdev->bd_disk, rq, 0); + blk_execute_rq(rq, false); if (scsi_req(rq)->result) ret = -EIO; out: diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index d1676fe0da1a..b361583944b9 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -540,7 +540,7 @@ static int carm_array_info (struct carm_host *host, unsigned int array_idx) spin_unlock_irq(&host->lock); DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); - blk_execute_rq_nowait(NULL, rq, true, NULL); + blk_execute_rq_nowait(rq, true, NULL); return 0; @@ -579,7 +579,7 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func) crq->msg_bucket = (u32) rc; DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); - blk_execute_rq_nowait(NULL, rq, true, NULL); + blk_execute_rq_nowait(rq, true, NULL); return 0; } diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index cfa303fa7318..c3dc3cd7a779 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -384,7 +384,7 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str) if (err) goto out; - blk_execute_rq(vblk->disk, req, false); + blk_execute_rq(req, false); err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); out: blk_mq_free_request(req); diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index dc094f73d335..ef8f45fa2cee 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -264,7 +264,7 @@ static ssize_t power_ro_lock_store(struct device *dev, goto out_put; } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP; - blk_execute_rq(NULL, req, 0); + blk_execute_rq(req, false); ret = req_to_mmc_queue_req(req)->drv_op_result; blk_mq_free_request(req); @@ -657,7 +657,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idatas; req_to_mmc_queue_req(req)->ioc_count = 1; - blk_execute_rq(NULL, req, 0); + blk_execute_rq(req, false); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata); blk_mq_free_request(req); @@ -726,7 +726,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idata; req_to_mmc_queue_req(req)->ioc_count = num_of_cmds; - blk_execute_rq(NULL, req, 0); + blk_execute_rq(req, false); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; /* copy to user if data and response */ @@ -2743,7 +2743,7 @@ static int mmc_dbg_card_status_get(void *data, u64 *val) if (IS_ERR(req)) return PTR_ERR(req); req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS; - blk_execute_rq(NULL, req, 0); + blk_execute_rq(req, false); ret = req_to_mmc_queue_req(req)->drv_op_result; if (ret >= 0) { *val = ret; @@ -2782,7 +2782,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp) } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD; req_to_mmc_queue_req(req)->drv_op_data = &ext_csd; - blk_execute_rq(NULL, req, 0); + blk_execute_rq(req, false); err = req_to_mmc_queue_req(req)->drv_op_result; blk_mq_free_request(req); if (err) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 4c63564adeaa..f82c098b1a61 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1056,7 +1056,7 @@ static int nvme_execute_rq(struct gendisk *disk, struct request *rq, { blk_status_t status; - status = blk_execute_rq(disk, rq, at_head); + status = blk_execute_rq(rq, at_head); if (nvme_req(rq)->flags & NVME_REQ_CANCELLED) return -EINTR; if (nvme_req(rq)->status) @@ -1283,7 +1283,7 @@ static void nvme_keep_alive_work(struct work_struct *work) rq->timeout = ctrl->kato * HZ; rq->end_io_data = ctrl; - blk_execute_rq_nowait(NULL, rq, 0, nvme_keep_alive_end_io); + blk_execute_rq_nowait(rq, false, nvme_keep_alive_end_io); } static void nvme_start_keep_alive(struct nvme_ctrl *ctrl) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index ca2ee806d74b..8637538f3fd5 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1371,7 +1371,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) } abort_req->end_io_data = NULL; - blk_execute_rq_nowait(NULL, abort_req, 0, abort_endio); + blk_execute_rq_nowait(abort_req, false, abort_endio); /* * The aborted req will be completed on receiving the abort req. @@ -2416,9 +2416,8 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) req->end_io_data = nvmeq; init_completion(&nvmeq->delete_done); - blk_execute_rq_nowait(NULL, req, false, - opcode == nvme_admin_delete_cq ? - nvme_del_cq_end : nvme_del_queue_end); + blk_execute_rq_nowait(req, false, opcode == nvme_admin_delete_cq ? + nvme_del_cq_end : nvme_del_queue_end); return 0; } diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index f0efb3537989..9e5b89ae29df 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -284,8 +284,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req) schedule_work(&req->p.work); } else { rq->end_io_data = req; - blk_execute_rq_nowait(ns ? ns->disk : NULL, rq, 0, - nvmet_passthru_req_done); + blk_execute_rq_nowait(rq, false, nvmet_passthru_req_done); } if (ns) diff --git a/drivers/scsi/scsi_bsg.c b/drivers/scsi/scsi_bsg.c index 081b84bb7985..b7a464383cc0 100644 --- a/drivers/scsi/scsi_bsg.c +++ b/drivers/scsi/scsi_bsg.c @@ -60,7 +60,7 @@ static int scsi_bsg_sg_io_fn(struct request_queue *q, struct sg_io_v4 *hdr, goto out_free_cmd; bio = rq->bio; - blk_execute_rq(NULL, rq, !(hdr->flags & BSG_FLAG_Q_AT_TAIL)); + blk_execute_rq(rq, !(hdr->flags & BSG_FLAG_Q_AT_TAIL)); /* * fill in all the output members diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 2371edbc3af4..3eae2392ef15 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -2040,7 +2040,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) req->timeout = 10 * HZ; rq->retries = 5; - blk_execute_rq_nowait(NULL, req, 1, eh_lock_door_done); + blk_execute_rq_nowait(req, true, eh_lock_door_done); } /** diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c index 400df3354cd6..340ba0ad6e70 100644 --- a/drivers/scsi/scsi_ioctl.c +++ b/drivers/scsi/scsi_ioctl.c @@ -483,7 +483,7 @@ static int sg_io(struct scsi_device *sdev, struct gendisk *disk, start_time = jiffies; - blk_execute_rq(disk, rq, at_head); + blk_execute_rq(rq, at_head); hdr->duration = jiffies_to_msecs(jiffies - start_time); @@ -620,7 +620,7 @@ static int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, goto error; } - blk_execute_rq(disk, rq, 0); + blk_execute_rq(rq, false); err = req->result & 0xff; /* only 8 bit SCSI status */ if (err) { diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index c23cf8e7b3c3..35e381f6d371 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -241,7 +241,7 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, /* * head injection *required* here otherwise quiesce won't work */ - blk_execute_rq(NULL, req, 1); + blk_execute_rq(req, true); /* * Some devices (USB mass-storage in particular) may transfer diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 141099ab9092..6c8ffad88f9e 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -833,7 +833,7 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, srp->rq->timeout = timeout; kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */ - blk_execute_rq_nowait(NULL, srp->rq, at_head, sg_rq_end_io); + blk_execute_rq_nowait(srp->rq, at_head, sg_rq_end_io); return 0; } diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 7db595c08b20..c29589815468 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -994,7 +994,7 @@ static int sr_read_cdda_bpc(struct cdrom_device_info *cdi, void __user *ubuf, rq->timeout = 60 * HZ; bio = rq->bio; - blk_execute_rq(disk, rq, 0); + blk_execute_rq(rq, false); if (scsi_req(rq)->result) { struct scsi_sense_hdr sshdr; diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index c2d5608f6b1a..06acc7db7755 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -581,7 +581,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd, rq->retries = retries; req->end_io_data = SRpnt; - blk_execute_rq_nowait(NULL, req, 1, st_scsi_execute_end); + blk_execute_rq_nowait(req, true, st_scsi_execute_end); return 0; } diff --git a/drivers/scsi/ufs/ufshpb.c b/drivers/scsi/ufs/ufshpb.c index ded5ba9b1466..13cd21204bf9 100644 --- a/drivers/scsi/ufs/ufshpb.c +++ b/drivers/scsi/ufs/ufshpb.c @@ -677,7 +677,7 @@ static void ufshpb_execute_umap_req(struct ufshpb_lu *hpb, ufshpb_set_unmap_cmd(rq->cmd, rgn); rq->cmd_len = HPB_WRITE_BUFFER_CMD_LENGTH; - blk_execute_rq_nowait(NULL, req, 1, ufshpb_umap_req_compl_fn); + blk_execute_rq_nowait(req, true, ufshpb_umap_req_compl_fn); hpb->stats.umap_req_cnt++; } @@ -719,7 +719,7 @@ static int ufshpb_execute_map_req(struct ufshpb_lu *hpb, map_req->rb.srgn_idx, mem_size); rq->cmd_len = HPB_READ_BUFFER_CMD_LENGTH; - blk_execute_rq_nowait(NULL, req, 1, ufshpb_map_req_compl_fn); + blk_execute_rq_nowait(req, true, ufshpb_map_req_compl_fn); hpb->stats.map_req_cnt++; return 0; diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 7fa57fb57bf2..807d06ecadee 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -1005,7 +1005,7 @@ pscsi_execute_cmd(struct se_cmd *cmd) req->timeout = PS_TIMEOUT_OTHER; scsi_req(req)->retries = PS_RETRY; - blk_execute_rq_nowait(NULL, req, (cmd->sam_task_attr == TCM_HEAD_TAG), + blk_execute_rq_nowait(req, cmd->sam_task_attr == TCM_HEAD_TAG, pscsi_req_done); return 0; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index ede7bef8880a..1b87b7c8bbff 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -924,10 +924,9 @@ int blk_rq_unmap_user(struct bio *); int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t); int blk_rq_append_bio(struct request *rq, struct bio *bio); -void blk_execute_rq_nowait(struct gendisk *, struct request *, int, - rq_end_io_fn *); -blk_status_t blk_execute_rq(struct gendisk *bd_disk, struct request *rq, - int at_head); +void blk_execute_rq_nowait(struct request *rq, bool at_head, + rq_end_io_fn *end_io); +blk_status_t blk_execute_rq(struct request *rq, bool at_head); struct req_iterator { struct bvec_iter iter; -- cgit v1.2.3-59-g8ed1b From 4047b9db1aa7512a10ba3560a3f63821c8c40235 Mon Sep 17 00:00:00 2001 From: Bhupesh Sharma Date: Mon, 29 Nov 2021 01:28:54 +0530 Subject: net: stmmac: Add platform level debug register dump feature dwmac-qcom-ethqos currently exposes a mechanism to dump rgmii registers after the 'stmmac_dvr_probe()' returns. However with commit 5ec55823438e ("net: stmmac: add clocks management for gmac driver"), we now let 'pm_runtime_put()' disable the clocks before returning from 'stmmac_dvr_probe()'. This causes a crash when 'rgmii_dump()' register dumps are enabled, as the clocks are already off. Since other dwmac drivers (possible future users as well) might require a similar register dump feature, introduce a platform level callback to allow the same. This fixes the crash noticed while enabling rgmii_dump() dumps in dwmac-qcom-ethqos driver as well. It also allows future changes to keep a invoking the register dump callback from the correct place inside 'stmmac_dvr_probe()'. Fixes: 5ec55823438e ("net: stmmac: add clocks management for gmac driver") Cc: Joakim Zhang Cc: David S. Miller Signed-off-by: Bhupesh Sharma Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c | 7 ++++--- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 3 +++ include/linux/stmmac.h | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index 8fea48e477e6..2ffa0a11eea5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -113,8 +113,10 @@ static void rgmii_updatel(struct qcom_ethqos *ethqos, rgmii_writel(ethqos, temp, offset); } -static void rgmii_dump(struct qcom_ethqos *ethqos) +static void rgmii_dump(void *priv) { + struct qcom_ethqos *ethqos = priv; + dev_dbg(ðqos->pdev->dev, "Rgmii register dump\n"); dev_dbg(ðqos->pdev->dev, "RGMII_IO_MACRO_CONFIG: %x\n", rgmii_readl(ethqos, RGMII_IO_MACRO_CONFIG)); @@ -519,6 +521,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->bsp_priv = ethqos; plat_dat->fix_mac_speed = ethqos_fix_mac_speed; + plat_dat->dump_debug_regs = rgmii_dump; plat_dat->has_gmac4 = 1; plat_dat->pmt = 1; plat_dat->tso_en = of_property_read_bool(np, "snps,tso"); @@ -527,8 +530,6 @@ static int qcom_ethqos_probe(struct platform_device *pdev) if (ret) goto err_clk; - rgmii_dump(ethqos); - return ret; err_clk: diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 89a6c35e2546..cc1075c08996 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -7211,6 +7211,9 @@ int stmmac_dvr_probe(struct device *device, stmmac_init_fs(ndev); #endif + if (priv->plat->dump_debug_regs) + priv->plat->dump_debug_regs(priv->plat->bsp_priv); + /* Let pm_runtime_put() disable the clocks. * If CONFIG_PM is not enabled, the clocks will stay powered. */ diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 89b8e208cd7b..24eea1b05ca2 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -233,6 +233,7 @@ struct plat_stmmacenet_data { int (*clks_config)(void *priv, bool enabled); int (*crosststamp)(ktime_t *device, struct system_counterval_t *system, void *ctx); + void (*dump_debug_regs)(void *priv); void *bsp_priv; struct clk *stmmac_clk; struct clk *pclk; -- cgit v1.2.3-59-g8ed1b From e6f2dd0f80674e9d5960337b3e9c2a242441b326 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Mon, 29 Nov 2021 19:06:19 -0800 Subject: bpf: Add bpf_loop helper This patch adds the kernel-side and API changes for a new helper function, bpf_loop: long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags); where long (*callback_fn)(u32 index, void *ctx); bpf_loop invokes the "callback_fn" **nr_loops** times or until the callback_fn returns 1. The callback_fn can only return 0 or 1, and this is enforced by the verifier. The callback_fn index is zero-indexed. A few things to please note: ~ The "u64 flags" parameter is currently unused but is included in case a future use case for it arises. ~ In the kernel-side implementation of bpf_loop (kernel/bpf/bpf_iter.c), bpf_callback_t is used as the callback function cast. ~ A program can have nested bpf_loop calls but the program must still adhere to the verifier constraint of its stack depth (the stack depth cannot exceed MAX_BPF_STACK)) ~ Recursive callback_fns do not pass the verifier, due to the call stack for these being too deep. ~ The next patch will include the tests and benchmark Signed-off-by: Joanne Koong Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211130030622.4131246-2-joannekoong@fb.com --- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 25 ++++++++++++ kernel/bpf/bpf_iter.c | 35 +++++++++++++++++ kernel/bpf/helpers.c | 2 + kernel/bpf/verifier.c | 88 ++++++++++++++++++++++++++---------------- tools/include/uapi/linux/bpf.h | 25 ++++++++++++ 6 files changed, 142 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index cc7a0c36e7df..cad0829710be 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2164,6 +2164,7 @@ extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto; extern const struct bpf_func_proto bpf_find_vma_proto; +extern const struct bpf_func_proto bpf_loop_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a69e4b04ffeb..211b43afd0fb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4957,6 +4957,30 @@ union bpf_attr { * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. * **-EBUSY** if failed to try lock mmap_lock. * **-EINVAL** for invalid **flags**. + * + * long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * For **nr_loops**, call **callback_fn** function + * with **callback_ctx** as the context parameter. + * The **callback_fn** should be a static function and + * the **callback_ctx** should be a pointer to the stack. + * The **flags** is used to control certain aspects of the helper. + * Currently, the **flags** must be 0. Currently, nr_loops is + * limited to 1 << 23 (~8 million) loops. + * + * long (\*callback_fn)(u32 index, void \*ctx); + * + * where **index** is the current index in the loop. The index + * is zero-indexed. + * + * If **callback_fn** returns 0, the helper will continue to the next + * loop. If return value is 1, the helper will skip the rest of + * the loops and return. Other return values are not used now, + * and will be rejected by the verifier. + * + * Return + * The number of loops performed, **-EINVAL** for invalid **flags**, + * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5140,6 +5164,7 @@ union bpf_attr { FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ FN(find_vma), \ + FN(loop), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index b2ee45064e06..b7aef5b3416d 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -714,3 +714,38 @@ const struct bpf_func_proto bpf_for_each_map_elem_proto = { .arg3_type = ARG_PTR_TO_STACK_OR_NULL, .arg4_type = ARG_ANYTHING, }; + +/* maximum number of loops */ +#define MAX_LOOPS BIT(23) + +BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, + u64, flags) +{ + bpf_callback_t callback = (bpf_callback_t)callback_fn; + u64 ret; + u32 i; + + if (flags) + return -EINVAL; + if (nr_loops > MAX_LOOPS) + return -E2BIG; + + for (i = 0; i < nr_loops; i++) { + ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); + /* return value: 0 - continue, 1 - stop and return */ + if (ret) + return i + 1; + } + + return i; +} + +const struct bpf_func_proto bpf_loop_proto = { + .func = bpf_loop, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, + .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_STACK_OR_NULL, + .arg4_type = ARG_ANYTHING, +}; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 1ffd469c217f..52188004a9c3 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1378,6 +1378,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) return &bpf_ringbuf_query_proto; case BPF_FUNC_for_each_map_elem: return &bpf_for_each_map_elem_proto; + case BPF_FUNC_loop: + return &bpf_loop_proto; default: break; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0763cca139a7..d7678d8a925c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6085,6 +6085,27 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_loop_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, + * u64 flags); + * callback_fn(u32 index, void *callback_ctx); + */ + callee->regs[BPF_REG_1].type = SCALAR_VALUE; + callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + + callee->in_callback_fn = true; + return 0; +} + static int set_timer_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, @@ -6458,13 +6479,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return err; } - if (func_id == BPF_FUNC_tail_call) { - err = check_reference_leak(env); - if (err) { - verbose(env, "tail_call would lead to reference leak\n"); - return err; - } - } else if (is_release_function(func_id)) { + if (is_release_function(func_id)) { err = release_reference(env, meta.ref_obj_id); if (err) { verbose(env, "func %s#%d reference has not been acquired before\n", @@ -6475,42 +6490,47 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs = cur_regs(env); - /* check that flags argument in get_local_storage(map, flags) is 0, - * this is required because get_local_storage() can't return an error. - */ - if (func_id == BPF_FUNC_get_local_storage && - !register_is_null(®s[BPF_REG_2])) { - verbose(env, "get_local_storage() doesn't support non-zero flags\n"); - return -EINVAL; - } - - if (func_id == BPF_FUNC_for_each_map_elem) { + switch (func_id) { + case BPF_FUNC_tail_call: + err = check_reference_leak(env); + if (err) { + verbose(env, "tail_call would lead to reference leak\n"); + return err; + } + break; + case BPF_FUNC_get_local_storage: + /* check that flags argument in get_local_storage(map, flags) is 0, + * this is required because get_local_storage() can't return an error. + */ + if (!register_is_null(®s[BPF_REG_2])) { + verbose(env, "get_local_storage() doesn't support non-zero flags\n"); + return -EINVAL; + } + break; + case BPF_FUNC_for_each_map_elem: err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, set_map_elem_callback_state); - if (err < 0) - return -EINVAL; - } - - if (func_id == BPF_FUNC_timer_set_callback) { + break; + case BPF_FUNC_timer_set_callback: err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, set_timer_callback_state); - if (err < 0) - return -EINVAL; - } - - if (func_id == BPF_FUNC_find_vma) { + break; + case BPF_FUNC_find_vma: err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, set_find_vma_callback_state); - if (err < 0) - return -EINVAL; - } - - if (func_id == BPF_FUNC_snprintf) { + break; + case BPF_FUNC_snprintf: err = check_bpf_snprintf_call(env, regs); - if (err < 0) - return err; + break; + case BPF_FUNC_loop: + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_loop_callback_state); + break; } + if (err) + return err; + /* reset caller saved regs */ for (i = 0; i < CALLER_SAVED_REGS; i++) { mark_reg_not_init(env, regs, caller_saved[i]); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index a69e4b04ffeb..211b43afd0fb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4957,6 +4957,30 @@ union bpf_attr { * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. * **-EBUSY** if failed to try lock mmap_lock. * **-EINVAL** for invalid **flags**. + * + * long bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * For **nr_loops**, call **callback_fn** function + * with **callback_ctx** as the context parameter. + * The **callback_fn** should be a static function and + * the **callback_ctx** should be a pointer to the stack. + * The **flags** is used to control certain aspects of the helper. + * Currently, the **flags** must be 0. Currently, nr_loops is + * limited to 1 << 23 (~8 million) loops. + * + * long (\*callback_fn)(u32 index, void \*ctx); + * + * where **index** is the current index in the loop. The index + * is zero-indexed. + * + * If **callback_fn** returns 0, the helper will continue to the next + * loop. If return value is 1, the helper will skip the rest of + * the loops and return. Other return values are not used now, + * and will be rejected by the verifier. + * + * Return + * The number of loops performed, **-EINVAL** for invalid **flags**, + * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5140,6 +5164,7 @@ union bpf_attr { FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ FN(find_vma), \ + FN(loop), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3-59-g8ed1b From 7ad639840acf2800b5f387c495795f995a67a329 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 29 Nov 2021 13:06:43 +0000 Subject: thread_info: Add helpers to snapshot thread flags In there are helpers to manipulate individual thread flags, but where code wants to check several flags at once, it must open code reading current_thread_info()->flags and operating on a snapshot. As some flags can be set remotely it's necessary to use READ_ONCE() to get a consistent snapshot even when IRQs are disabled, but some code forgets to do this. Generally this is unlike to cause a problem in practice, but it is somewhat unsound, and KCSAN will legitimately warn that there is a data race. To make it easier to do the right thing, and to highlight that concurrent modification is possible, add new helpers to snapshot the flags, which should be used in preference to plain reads. Subsequent patches will move existing code to use the new helpers. Signed-off-by: Mark Rutland Signed-off-by: Thomas Gleixner Reviewed-by: Thomas Gleixner Acked-by: Marco Elver Acked-by: Paul E. McKenney Cc: Boqun Feng Cc: Dmitry Vyukov Cc: Peter Zijlstra Cc: Will Deacon Link: https://lore.kernel.org/r/20211129130653.2037928-2-mark.rutland@arm.com --- include/linux/thread_info.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index ad0c4e041030..73a6f34b3847 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -118,6 +118,15 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag) return test_bit(flag, (unsigned long *)&ti->flags); } +/* + * This may be used in noinstr code, and needs to be __always_inline to prevent + * inadvertent instrumentation. + */ +static __always_inline unsigned long read_ti_thread_flags(struct thread_info *ti) +{ + return READ_ONCE(ti->flags); +} + #define set_thread_flag(flag) \ set_ti_thread_flag(current_thread_info(), flag) #define clear_thread_flag(flag) \ @@ -130,6 +139,11 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag) test_and_clear_ti_thread_flag(current_thread_info(), flag) #define test_thread_flag(flag) \ test_ti_thread_flag(current_thread_info(), flag) +#define read_thread_flags() \ + read_ti_thread_flags(current_thread_info()) + +#define read_task_thread_flags(t) \ + read_ti_thread_flags(task_thread_info(t)) #ifdef CONFIG_GENERIC_ENTRY #define set_syscall_work(fl) \ -- cgit v1.2.3-59-g8ed1b From 6ce895128b3bff738fe8d9dd74747a03e319e466 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 29 Nov 2021 13:06:44 +0000 Subject: entry: Snapshot thread flags Some thread flags can be set remotely, and so even when IRQs are disabled, the flags can change under our feet. Generally this is unlikely to cause a problem in practice, but it is somewhat unsound, and KCSAN will legitimately warn that there is a data race. To avoid such issues, a snapshot of the flags has to be taken prior to using them. Some places already use READ_ONCE() for that, others do not. Convert them all to the new flag accessor helpers. Signed-off-by: Mark Rutland Signed-off-by: Thomas Gleixner Acked-by: Paul E. McKenney Link: https://lore.kernel.org/r/20211129130653.2037928-3-mark.rutland@arm.com --- include/linux/entry-kvm.h | 2 +- kernel/entry/common.c | 4 ++-- kernel/entry/kvm.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/entry-kvm.h b/include/linux/entry-kvm.h index 0d7865a0731c..07c878d6e323 100644 --- a/include/linux/entry-kvm.h +++ b/include/linux/entry-kvm.h @@ -75,7 +75,7 @@ static inline void xfer_to_guest_mode_prepare(void) */ static inline bool __xfer_to_guest_mode_work_pending(void) { - unsigned long ti_work = READ_ONCE(current_thread_info()->flags); + unsigned long ti_work = read_thread_flags(); return !!(ti_work & XFER_TO_GUEST_MODE_WORK); } diff --git a/kernel/entry/common.c b/kernel/entry/common.c index d5a61d565ad5..bad713684c2e 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -187,7 +187,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, /* Check if any of the above work has queued a deferred wakeup */ tick_nohz_user_enter_prepare(); - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); } /* Return the latest work state for arch_exit_to_user_mode() */ @@ -196,7 +196,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, static void exit_to_user_mode_prepare(struct pt_regs *regs) { - unsigned long ti_work = READ_ONCE(current_thread_info()->flags); + unsigned long ti_work = read_thread_flags(); lockdep_assert_irqs_disabled(); diff --git a/kernel/entry/kvm.c b/kernel/entry/kvm.c index 49972ee99aff..96d476e06c77 100644 --- a/kernel/entry/kvm.c +++ b/kernel/entry/kvm.c @@ -26,7 +26,7 @@ static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) if (ret) return ret; - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); } while (ti_work & XFER_TO_GUEST_MODE_WORK || need_resched()); return 0; } @@ -43,7 +43,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu) * disabled in the inner loop before going into guest mode. No need * to disable interrupts here. */ - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); if (!(ti_work & XFER_TO_GUEST_MODE_WORK)) return 0; -- cgit v1.2.3-59-g8ed1b From 4946f15e8c334840bf277a0bf924371eae120fcd Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Tue, 30 Nov 2021 22:40:43 +0100 Subject: genirq/generic_chip: Constify irq_generic_chip_ops The only usage of irq_generic_chip_ops is to pass its address to irq_domain_add_linear() which takes a pointer to const struct irq_domain_ops. Make it const to allow the compiler to put it in read-only memory. [ tglx: Fixed subject prefix ] Signed-off-by: Rikard Falkeborn Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20211130214043.1257585-1-rikard.falkeborn@gmail.com --- include/linux/irqdomain.h | 2 +- kernel/irq/generic-chip.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 553da4899f55..d476405802e9 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -131,7 +131,7 @@ struct irq_domain_ops { #endif }; -extern struct irq_domain_ops irq_generic_chip_ops; +extern const struct irq_domain_ops irq_generic_chip_ops; struct irq_domain_chip_generic; diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c index 6f29bf4c8515..f0862eb6b506 100644 --- a/kernel/irq/generic-chip.c +++ b/kernel/irq/generic-chip.c @@ -451,7 +451,7 @@ static void irq_unmap_generic_chip(struct irq_domain *d, unsigned int virq) } -struct irq_domain_ops irq_generic_chip_ops = { +const struct irq_domain_ops irq_generic_chip_ops = { .map = irq_map_generic_chip, .unmap = irq_unmap_generic_chip, .xlate = irq_domain_xlate_onetwocell, -- cgit v1.2.3-59-g8ed1b From 24ba53017e188e031f9cb8b290286fad52d2af00 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Wed, 15 Sep 2021 17:02:18 +0800 Subject: rcu: Replace ________p1 and _________p1 with __UNIQUE_ID(rcu) This commit replaces both ________p1 and _________p1 with __UNIQUE_ID(rcu), and also adjusts the callers of the affected macros. __UNIQUE_ID(rcu) will generate unique variable names during compilation, which eliminates the need of ________p1 and _________p1 (both having 4 occurrences prior to the code change). This also avoids the variable name shadowing issue, or at least makes those wishing to cause shadowing problems work much harder to do so. The same idea is used for the min/max macros (commit 589a978 and commit e9092d0). Signed-off-by: Jim Huang Signed-off-by: Chun-Hung Tseng Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 50 +++++++++++++++++++++++++++--------------------- include/linux/srcu.h | 3 ++- 2 files changed, 30 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e0beb5c5659..88b42eb46406 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -364,6 +364,12 @@ static inline void rcu_preempt_sleep_check(void) { } #define rcu_check_sparse(p, space) #endif /* #else #ifdef __CHECKER__ */ +#define __unrcu_pointer(p, local) \ +({ \ + typeof(*p) *local = (typeof(*p) *__force)(p); \ + rcu_check_sparse(p, __rcu); \ + ((typeof(*p) __force __kernel *)(local)); \ +}) /** * unrcu_pointer - mark a pointer as not being RCU protected * @p: pointer needing to lose its __rcu property @@ -371,39 +377,35 @@ static inline void rcu_preempt_sleep_check(void) { } * Converts @p from an __rcu pointer to a __kernel pointer. * This allows an __rcu pointer to be used with xchg() and friends. */ -#define unrcu_pointer(p) \ -({ \ - typeof(*p) *_________p1 = (typeof(*p) *__force)(p); \ - rcu_check_sparse(p, __rcu); \ - ((typeof(*p) __force __kernel *)(_________p1)); \ -}) +#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, space) \ +#define __rcu_access_pointer(p, local, space) \ ({ \ - typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \ + typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(_________p1)); \ + ((typeof(*p) __force __kernel *)(local)); \ }) -#define __rcu_dereference_check(p, c, space) \ +#define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ - typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \ + typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(________p1)); \ + ((typeof(*p) __force __kernel *)(local)); \ }) -#define __rcu_dereference_protected(p, c, space) \ +#define __rcu_dereference_protected(p, local, c, space) \ ({ \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(p)); \ }) -#define rcu_dereference_raw(p) \ +#define __rcu_dereference_raw(p, local) \ ({ \ /* Dependency order vs. p above. */ \ - typeof(p) ________p1 = READ_ONCE(p); \ - ((typeof(*p) __force __kernel *)(________p1)); \ + typeof(p) local = READ_ONCE(p); \ + ((typeof(*p) __force __kernel *)(local)); \ }) +#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu)) /** * RCU_INITIALIZER() - statically initialize an RCU-protected global variable @@ -490,7 +492,7 @@ do { \ * when tearing down multi-linked structures after a grace period * has elapsed. */ -#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu) +#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu) /** * rcu_dereference_check() - rcu_dereference with debug checking @@ -526,7 +528,8 @@ do { \ * annotated as __rcu. */ #define rcu_dereference_check(p, c) \ - __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu) + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ + (c) || rcu_read_lock_held(), __rcu) /** * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking @@ -541,7 +544,8 @@ do { \ * rcu_read_lock() but also rcu_read_lock_bh() into account. */ #define rcu_dereference_bh_check(p, c) \ - __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu) + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ + (c) || rcu_read_lock_bh_held(), __rcu) /** * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking @@ -556,7 +560,8 @@ do { \ * only rcu_read_lock() but also rcu_read_lock_sched() into account. */ #define rcu_dereference_sched_check(p, c) \ - __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \ + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ + (c) || rcu_read_lock_sched_held(), \ __rcu) /* @@ -566,7 +571,8 @@ do { \ * The no-tracing version of rcu_dereference_raw() must not call * rcu_read_lock_held(). */ -#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu) +#define rcu_dereference_raw_check(p) \ + __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu) /** * rcu_dereference_protected() - fetch RCU pointer when updates prevented @@ -585,7 +591,7 @@ do { \ * but very ugly failures. */ #define rcu_dereference_protected(p, c) \ - __rcu_dereference_protected((p), (c), __rcu) + __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu) /** diff --git a/include/linux/srcu.h b/include/linux/srcu.h index e6011a9975af..01226e4d960a 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -117,7 +117,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) * lockdep_is_held() calls. */ #define srcu_dereference_check(p, ssp, c) \ - __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu) + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ + (c) || srcu_read_lock_held(ssp), __rcu) /** * srcu_dereference - fetch SRCU-protected pointer for later dereferencing -- cgit v1.2.3-59-g8ed1b From 2407a64f8045552203ee5cb9904ce75ce2fceef4 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Tue, 28 Sep 2021 08:21:28 +0800 Subject: rcu: in_irq() cleanup This commit replaces the obsolete and ambiguous macro in_irq() with its shiny new in_hardirq() equivalent. Signed-off-by: Changbin Du Signed-off-by: Paul E. McKenney --- include/linux/rcutiny.h | 2 +- kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 9be015305f9f..858f4d429946 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -85,7 +85,7 @@ static inline void rcu_irq_enter_irqson(void) { } static inline void rcu_irq_exit(void) { } static inline void rcu_irq_exit_check_preempt(void) { } #define rcu_is_idle_cpu(cpu) \ - (is_idle_task(current) && !in_nmi() && !in_irq() && !in_serving_softirq()) + (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) static inline void exit_rcu(void) { } static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) { diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ef8d36f580fc..f0f19dc7f19e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1467,7 +1467,7 @@ static void rcu_gp_kthread_wake(void) { struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); - if ((current == t && !in_irq() && !in_serving_softirq()) || + if ((current == t && !in_hardirq() && !in_serving_softirq()) || !READ_ONCE(rcu_state.gp_flags) || !t) return; WRITE_ONCE(rcu_state.gp_wake_time, jiffies); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 5199559fbbf0..599084c4c21f 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -642,7 +642,7 @@ static void rcu_read_unlock_special(struct task_struct *t) (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled && t->rcu_blocked_node); // Need to defer quiescent state until everything is enabled. - if (use_softirq && (in_irq() || (expboost && !irqs_were_disabled))) { + if (use_softirq && (in_hardirq() || (expboost && !irqs_were_disabled))) { // Using softirq, safe to awaken, and either the // wakeup is free or there is either an expedited // GP in flight or a potential need to deboost. -- cgit v1.2.3-59-g8ed1b From af3bf054661fb11497a7f712ece8b838521227a4 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 1 Dec 2021 01:17:36 +0000 Subject: cgroup: fix a typo in comment In commit 8699b7762a62 ("cgroup: s/child_subsys_mask/subtree_ss_mask/"), we rename child_subsys_mask to subtree_ss_mask. While it missed to rename this in comment. Signed-off-by: Wei Yang Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index db2e147e069f..bb1e79791ed5 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -413,7 +413,7 @@ struct cgroup { /* * The bitmask of subsystems enabled on the child cgroups. * ->subtree_control is the one configured through - * "cgroup.subtree_control" while ->child_ss_mask is the effective + * "cgroup.subtree_control" while ->subtree_ss_mask is the effective * one which may have more subsystems enabled. Controller knobs * are made available iff it's enabled in ->subtree_control. */ -- cgit v1.2.3-59-g8ed1b From 57b2b72ac1fc5d55cf3b13207942c109f1a65cb5 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 29 Nov 2021 09:57:59 -0700 Subject: mm, slab: Remove compiler check in __kmalloc_index The minimum supported version of LLVM has been raised to 11.0.0, meaning this check is always true, so it can be dropped. Signed-off-by: Nathan Chancellor Reviewed-by: Miguel Ojeda Reviewed-by: Mark Brown Reviewed-by: Nick Desaulniers Reviewed-by: Kees Cook Signed-off-by: Masahiro Yamada --- include/linux/slab.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 181045148b06..d3fb5ac71c24 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -411,8 +411,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size, if (size <= 16 * 1024 * 1024) return 24; if (size <= 32 * 1024 * 1024) return 25; - if ((IS_ENABLED(CONFIG_CC_IS_GCC) || CONFIG_CLANG_VERSION >= 110000) - && !IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) + if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); else BUG(); -- cgit v1.2.3-59-g8ed1b From 9e3562080950b6e3fe38a5e34ddd5b1c618f2019 Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Thu, 2 Dec 2021 10:53:33 +0100 Subject: HID: add suspend/resume helpers There is a lot of duplication of code in the HID low level drivers. Better have everything in one place so we can eventually extend it in a generic way. Signed-off-by: Benjamin Tissoires Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20211202095334.14399-4-benjamin.tissoires@redhat.com --- drivers/hid/hid-core.c | 29 +++++++++++++++++++++++++++++ drivers/hid/i2c-hid/i2c-hid-core.c | 15 ++++----------- drivers/hid/surface-hid/surface_hid_core.c | 25 +++++-------------------- drivers/hid/usbhid/hid-core.c | 19 ++++++++----------- include/linux/hid.h | 10 ++++++++++ 5 files changed, 56 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index dbed2524fd47..5402329d6eca 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2126,6 +2126,35 @@ void hid_hw_close(struct hid_device *hdev) } EXPORT_SYMBOL_GPL(hid_hw_close); +#ifdef CONFIG_PM +int hid_driver_suspend(struct hid_device *hdev, pm_message_t state) +{ + if (hdev->driver && hdev->driver->suspend) + return hdev->driver->suspend(hdev, state); + + return 0; +} +EXPORT_SYMBOL_GPL(hid_driver_suspend); + +int hid_driver_reset_resume(struct hid_device *hdev) +{ + if (hdev->driver && hdev->driver->reset_resume) + return hdev->driver->reset_resume(hdev); + + return 0; +} +EXPORT_SYMBOL_GPL(hid_driver_reset_resume); + +int hid_driver_resume(struct hid_device *hdev) +{ + if (hdev->driver && hdev->driver->resume) + return hdev->driver->resume(hdev); + + return 0; +} +EXPORT_SYMBOL_GPL(hid_driver_resume); +#endif /* CONFIG_PM */ + struct hid_dynid { struct list_head list; struct hid_device_id id; diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 517141138b00..4cdd862ca1cb 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -1063,11 +1063,9 @@ static int i2c_hid_core_suspend(struct device *dev) int ret; int wake_status; - if (hid->driver && hid->driver->suspend) { - ret = hid->driver->suspend(hid, PMSG_SUSPEND); - if (ret < 0) - return ret; - } + ret = hid_driver_suspend(hid, PMSG_SUSPEND); + if (ret < 0) + return ret; /* Save some power */ i2c_hid_set_power(client, I2C_HID_PWR_SLEEP); @@ -1125,12 +1123,7 @@ static int i2c_hid_core_resume(struct device *dev) if (ret) return ret; - if (hid->driver && hid->driver->reset_resume) { - ret = hid->driver->reset_resume(hid); - return ret; - } - - return 0; + return hid_driver_reset_resume(hid); } #endif diff --git a/drivers/hid/surface-hid/surface_hid_core.c b/drivers/hid/surface-hid/surface_hid_core.c index 5571e74abe91..e46330b2e561 100644 --- a/drivers/hid/surface-hid/surface_hid_core.c +++ b/drivers/hid/surface-hid/surface_hid_core.c @@ -204,50 +204,35 @@ static int surface_hid_suspend(struct device *dev) { struct surface_hid_device *d = dev_get_drvdata(dev); - if (d->hid->driver && d->hid->driver->suspend) - return d->hid->driver->suspend(d->hid, PMSG_SUSPEND); - - return 0; + return hid_driver_suspend(d->hid, PMSG_SUSPEND); } static int surface_hid_resume(struct device *dev) { struct surface_hid_device *d = dev_get_drvdata(dev); - if (d->hid->driver && d->hid->driver->resume) - return d->hid->driver->resume(d->hid); - - return 0; + return hid_driver_resume(d->hid); } static int surface_hid_freeze(struct device *dev) { struct surface_hid_device *d = dev_get_drvdata(dev); - if (d->hid->driver && d->hid->driver->suspend) - return d->hid->driver->suspend(d->hid, PMSG_FREEZE); - - return 0; + return hid_driver_suspend(d->hid, PMSG_FREEZE); } static int surface_hid_poweroff(struct device *dev) { struct surface_hid_device *d = dev_get_drvdata(dev); - if (d->hid->driver && d->hid->driver->suspend) - return d->hid->driver->suspend(d->hid, PMSG_HIBERNATE); - - return 0; + return hid_driver_suspend(d->hid, PMSG_HIBERNATE); } static int surface_hid_restore(struct device *dev) { struct surface_hid_device *d = dev_get_drvdata(dev); - if (d->hid->driver && d->hid->driver->reset_resume) - return d->hid->driver->reset_resume(d->hid); - - return 0; + return hid_driver_reset_resume(d->hid); } const struct dev_pm_ops surface_hid_pm_ops = { diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c index 2dcaf31eb9cd..54752c85604b 100644 --- a/drivers/hid/usbhid/hid-core.c +++ b/drivers/hid/usbhid/hid-core.c @@ -1563,8 +1563,8 @@ static int hid_resume_common(struct hid_device *hid, bool driver_suspended) int status = 0; hid_restart_io(hid); - if (driver_suspended && hid->driver && hid->driver->resume) - status = hid->driver->resume(hid); + if (driver_suspended) + status = hid_driver_resume(hid); return status; } @@ -1588,11 +1588,9 @@ static int hid_suspend(struct usb_interface *intf, pm_message_t message) { set_bit(HID_SUSPENDED, &usbhid->iofl); spin_unlock_irq(&usbhid->lock); - if (hid->driver && hid->driver->suspend) { - status = hid->driver->suspend(hid, message); - if (status < 0) - goto failed; - } + status = hid_driver_suspend(hid, message); + if (status < 0) + goto failed; driver_suspended = true; } else { usbhid_mark_busy(usbhid); @@ -1602,8 +1600,7 @@ static int hid_suspend(struct usb_interface *intf, pm_message_t message) } else { /* TODO: resume() might need to handle suspend failure */ - if (hid->driver && hid->driver->suspend) - status = hid->driver->suspend(hid, message); + status = hid_driver_suspend(hid, message); driver_suspended = true; spin_lock_irq(&usbhid->lock); set_bit(HID_SUSPENDED, &usbhid->iofl); @@ -1644,8 +1641,8 @@ static int hid_reset_resume(struct usb_interface *intf) int status; status = hid_post_reset(intf); - if (status >= 0 && hid->driver && hid->driver->reset_resume) { - int ret = hid->driver->reset_resume(hid); + if (status >= 0) { + int ret = hid_driver_reset_resume(hid); if (ret < 0) status = ret; } diff --git a/include/linux/hid.h b/include/linux/hid.h index 9e067f937dbc..ebe3ec98db6b 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -923,6 +923,16 @@ s32 hid_snto32(__u32 value, unsigned n); __u32 hid_field_extract(const struct hid_device *hid, __u8 *report, unsigned offset, unsigned n); +#ifdef CONFIG_PM +int hid_driver_suspend(struct hid_device *hdev, pm_message_t state); +int hid_driver_reset_resume(struct hid_device *hdev); +int hid_driver_resume(struct hid_device *hdev); +#else +static inline int hid_driver_suspend(struct hid_device *hdev, pm_message_t state) { return 0; } +static inline int hid_driver_reset_resume(struct hid_device *hdev) { return 0; } +static inline int hid_driver_resume(struct hid_device *hdev) { return 0; } +#endif + /** * hid_device_io_start - enable HID input during probe, remove * -- cgit v1.2.3-59-g8ed1b From f65a0b1f3e79444bba9ac56435eeb32db85ab2c9 Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Thu, 2 Dec 2021 10:53:34 +0100 Subject: HID: do not inline some hid_hw_ functions We don't gain much by having them as inline, and it actually prevents us to attach a probe to those helpers. Signed-off-by: Benjamin Tissoires Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20211202095334.14399-5-benjamin.tissoires@redhat.com --- drivers/hid/hid-core.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/hid.h | 68 +++++--------------------------------------------- 2 files changed, 70 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 5402329d6eca..f1aed5bbd000 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2126,6 +2126,70 @@ void hid_hw_close(struct hid_device *hdev) } EXPORT_SYMBOL_GPL(hid_hw_close); +/** + * hid_hw_request - send report request to device + * + * @hdev: hid device + * @report: report to send + * @reqtype: hid request type + */ +void hid_hw_request(struct hid_device *hdev, + struct hid_report *report, int reqtype) +{ + if (hdev->ll_driver->request) + return hdev->ll_driver->request(hdev, report, reqtype); + + __hid_request(hdev, report, reqtype); +} +EXPORT_SYMBOL_GPL(hid_hw_request); + +/** + * hid_hw_raw_request - send report request to device + * + * @hdev: hid device + * @reportnum: report ID + * @buf: in/out data to transfer + * @len: length of buf + * @rtype: HID report type + * @reqtype: HID_REQ_GET_REPORT or HID_REQ_SET_REPORT + * + * Return: count of data transferred, negative if error + * + * Same behavior as hid_hw_request, but with raw buffers instead. + */ +int hid_hw_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, + size_t len, unsigned char rtype, int reqtype) +{ + if (len < 1 || len > HID_MAX_BUFFER_SIZE || !buf) + return -EINVAL; + + return hdev->ll_driver->raw_request(hdev, reportnum, buf, len, + rtype, reqtype); +} +EXPORT_SYMBOL_GPL(hid_hw_raw_request); + +/** + * hid_hw_output_report - send output report to device + * + * @hdev: hid device + * @buf: raw data to transfer + * @len: length of buf + * + * Return: count of data transferred, negative if error + */ +int hid_hw_output_report(struct hid_device *hdev, __u8 *buf, size_t len) +{ + if (len < 1 || len > HID_MAX_BUFFER_SIZE || !buf) + return -EINVAL; + + if (hdev->ll_driver->output_report) + return hdev->ll_driver->output_report(hdev, buf, len); + + return -ENOSYS; +} +EXPORT_SYMBOL_GPL(hid_hw_output_report); + #ifdef CONFIG_PM int hid_driver_suspend(struct hid_device *hdev, pm_message_t state) { diff --git a/include/linux/hid.h b/include/linux/hid.h index ebe3ec98db6b..b2fea7fc54a1 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -1066,6 +1066,12 @@ int __must_check hid_hw_start(struct hid_device *hdev, void hid_hw_stop(struct hid_device *hdev); int __must_check hid_hw_open(struct hid_device *hdev); void hid_hw_close(struct hid_device *hdev); +void hid_hw_request(struct hid_device *hdev, + struct hid_report *report, int reqtype); +int hid_hw_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, + size_t len, unsigned char rtype, int reqtype); +int hid_hw_output_report(struct hid_device *hdev, __u8 *buf, size_t len); /** * hid_hw_power - requests underlying HW to go into given power mode @@ -1083,68 +1089,6 @@ static inline int hid_hw_power(struct hid_device *hdev, int level) } -/** - * hid_hw_request - send report request to device - * - * @hdev: hid device - * @report: report to send - * @reqtype: hid request type - */ -static inline void hid_hw_request(struct hid_device *hdev, - struct hid_report *report, int reqtype) -{ - if (hdev->ll_driver->request) - return hdev->ll_driver->request(hdev, report, reqtype); - - __hid_request(hdev, report, reqtype); -} - -/** - * hid_hw_raw_request - send report request to device - * - * @hdev: hid device - * @reportnum: report ID - * @buf: in/out data to transfer - * @len: length of buf - * @rtype: HID report type - * @reqtype: HID_REQ_GET_REPORT or HID_REQ_SET_REPORT - * - * Return: count of data transferred, negative if error - * - * Same behavior as hid_hw_request, but with raw buffers instead. - */ -static inline int hid_hw_raw_request(struct hid_device *hdev, - unsigned char reportnum, __u8 *buf, - size_t len, unsigned char rtype, int reqtype) -{ - if (len < 1 || len > HID_MAX_BUFFER_SIZE || !buf) - return -EINVAL; - - return hdev->ll_driver->raw_request(hdev, reportnum, buf, len, - rtype, reqtype); -} - -/** - * hid_hw_output_report - send output report to device - * - * @hdev: hid device - * @buf: raw data to transfer - * @len: length of buf - * - * Return: count of data transferred, negative if error - */ -static inline int hid_hw_output_report(struct hid_device *hdev, __u8 *buf, - size_t len) -{ - if (len < 1 || len > HID_MAX_BUFFER_SIZE || !buf) - return -EINVAL; - - if (hdev->ll_driver->output_report) - return hdev->ll_driver->output_report(hdev, buf, len); - - return -ENOSYS; -} - /** * hid_hw_idle - send idle request to device * -- cgit v1.2.3-59-g8ed1b From 8293eb995f349aed28006792cad4cb48091919dd Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:25 -0800 Subject: bpf: Rename btf_member accessors. Rename btf_member_bit_offset() and btf_member_bitfield_size() to avoid conflicts with similarly named helpers in libbpf's btf.h. Rename the kernel helpers, since libbpf helpers are part of uapi. Suggested-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-3-alexei.starovoitov@gmail.com --- include/linux/btf.h | 8 ++++---- kernel/bpf/bpf_struct_ops.c | 6 +++--- kernel/bpf/btf.c | 18 +++++++++--------- net/ipv4/bpf_tcp_ca.c | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 203eef993d76..956f70388f69 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -194,15 +194,15 @@ static inline bool btf_type_kflag(const struct btf_type *t) return BTF_INFO_KFLAG(t->info); } -static inline u32 btf_member_bit_offset(const struct btf_type *struct_type, - const struct btf_member *member) +static inline u32 __btf_member_bit_offset(const struct btf_type *struct_type, + const struct btf_member *member) { return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset) : member->offset; } -static inline u32 btf_member_bitfield_size(const struct btf_type *struct_type, - const struct btf_member *member) +static inline u32 __btf_member_bitfield_size(const struct btf_type *struct_type, + const struct btf_member *member) { return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset) : 0; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 8ecfe4752769..21069dbe9138 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -165,7 +165,7 @@ void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log) break; } - if (btf_member_bitfield_size(t, member)) { + if (__btf_member_bitfield_size(t, member)) { pr_warn("bit field member %s in struct %s is not supported\n", mname, st_ops->name); break; @@ -296,7 +296,7 @@ static int check_zero_holes(const struct btf_type *t, void *data) const struct btf_type *mtype; for_each_member(i, t, member) { - moff = btf_member_bit_offset(t, member) / 8; + moff = __btf_member_bit_offset(t, member) / 8; if (moff > prev_mend && memchr_inv(data + prev_mend, 0, moff - prev_mend)) return -EINVAL; @@ -387,7 +387,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, struct bpf_prog *prog; u32 moff; - moff = btf_member_bit_offset(t, member) / 8; + moff = __btf_member_bit_offset(t, member) / 8; ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL); if (ptype == module_type) { if (*(void **)(udata + moff)) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 6b9d23be1e99..f4119a99da7b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2969,7 +2969,7 @@ static s32 btf_struct_check_meta(struct btf_verifier_env *env, return -EINVAL; } - offset = btf_member_bit_offset(t, member); + offset = __btf_member_bit_offset(t, member); if (is_union && offset) { btf_verifier_log_member(env, t, member, "Invalid member bits_offset"); @@ -3094,7 +3094,7 @@ static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t if (off != -ENOENT) /* only one such field is allowed */ return -E2BIG; - off = btf_member_bit_offset(t, member); + off = __btf_member_bit_offset(t, member); if (off % 8) /* valid C code cannot generate such BTF */ return -EINVAL; @@ -3184,8 +3184,8 @@ static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, btf_show_start_member(show, member); - member_offset = btf_member_bit_offset(t, member); - bitfield_size = btf_member_bitfield_size(t, member); + member_offset = __btf_member_bit_offset(t, member); + bitfield_size = __btf_member_bitfield_size(t, member); bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); bits8_offset = BITS_PER_BYTE_MASKED(member_offset); if (bitfield_size) { @@ -5060,7 +5060,7 @@ again: if (array_elem->nelems != 0) goto error; - moff = btf_member_bit_offset(t, member) / 8; + moff = __btf_member_bit_offset(t, member) / 8; if (off < moff) goto error; @@ -5083,14 +5083,14 @@ error: for_each_member(i, t, member) { /* offset of the field in bytes */ - moff = btf_member_bit_offset(t, member) / 8; + moff = __btf_member_bit_offset(t, member) / 8; if (off + size <= moff) /* won't find anything, field is already too far */ break; - if (btf_member_bitfield_size(t, member)) { - u32 end_bit = btf_member_bit_offset(t, member) + - btf_member_bitfield_size(t, member); + if (__btf_member_bitfield_size(t, member)) { + u32 end_bit = __btf_member_bit_offset(t, member) + + __btf_member_bitfield_size(t, member); /* off <= moff instead of off == moff because clang * does not generate a BTF member for anonymous diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 2cf02b4d77fb..67466dbff152 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -169,7 +169,7 @@ static u32 prog_ops_moff(const struct bpf_prog *prog) t = bpf_tcp_congestion_ops.type; m = &btf_type_member(t)[midx]; - return btf_member_bit_offset(t, m) / 8; + return __btf_member_bit_offset(t, m) / 8; } static const struct bpf_func_proto * @@ -244,7 +244,7 @@ static int bpf_tcp_ca_init_member(const struct btf_type *t, utcp_ca = (const struct tcp_congestion_ops *)udata; tcp_ca = (struct tcp_congestion_ops *)kdata; - moff = btf_member_bit_offset(t, member) / 8; + moff = __btf_member_bit_offset(t, member) / 8; switch (moff) { case offsetof(struct tcp_congestion_ops, flags): if (utcp_ca->flags & ~TCP_CONG_MASK) @@ -274,7 +274,7 @@ static int bpf_tcp_ca_init_member(const struct btf_type *t, static int bpf_tcp_ca_check_member(const struct btf_type *t, const struct btf_member *member) { - if (is_unsupported(btf_member_bit_offset(t, member) / 8)) + if (is_unsupported(__btf_member_bit_offset(t, member) / 8)) return -ENOTSUPP; return 0; } -- cgit v1.2.3-59-g8ed1b From 29db4bea1d10b73749d7992c1fc9ac13499e8871 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:26 -0800 Subject: bpf: Prepare relo_core.c for kernel duty. Make relo_core.c to be compiled for the kernel and for user space libbpf. Note the patch is reducing BPF_CORE_SPEC_MAX_LEN from 64 to 32. This is the maximum number of nested structs and arrays. For example: struct sample { int a; struct { int b[10]; }; }; struct sample *s = ...; int *y = &s->b[5]; This field access is encoded as "0:1:0:5" and spec len is 4. The follow up patch might bump it back to 64. Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-4-alexei.starovoitov@gmail.com --- include/linux/btf.h | 81 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/Makefile | 4 +++ kernel/bpf/btf.c | 26 +++++++++++++++ tools/lib/bpf/relo_core.c | 76 +++++++++++++++++++++++++++++++++++++------- 4 files changed, 176 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 956f70388f69..acef6ef28768 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -144,6 +144,53 @@ static inline bool btf_type_is_enum(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM; } +static inline bool str_is_empty(const char *s) +{ + return !s || !s[0]; +} + +static inline u16 btf_kind(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info); +} + +static inline bool btf_is_enum(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_ENUM; +} + +static inline bool btf_is_composite(const struct btf_type *t) +{ + u16 kind = btf_kind(t); + + return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; +} + +static inline bool btf_is_array(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_ARRAY; +} + +static inline bool btf_is_int(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_INT; +} + +static inline bool btf_is_ptr(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_PTR; +} + +static inline u8 btf_int_offset(const struct btf_type *t) +{ + return BTF_INT_OFFSET(*(u32 *)(t + 1)); +} + +static inline u8 btf_int_encoding(const struct btf_type *t) +{ + return BTF_INT_ENCODING(*(u32 *)(t + 1)); +} + static inline bool btf_type_is_scalar(const struct btf_type *t) { return btf_type_is_int(t) || btf_type_is_enum(t); @@ -184,6 +231,11 @@ static inline u16 btf_type_vlen(const struct btf_type *t) return BTF_INFO_VLEN(t->info); } +static inline u16 btf_vlen(const struct btf_type *t) +{ + return btf_type_vlen(t); +} + static inline u16 btf_func_linkage(const struct btf_type *t) { return BTF_INFO_VLEN(t->info); @@ -208,11 +260,40 @@ static inline u32 __btf_member_bitfield_size(const struct btf_type *struct_type, : 0; } +static inline struct btf_member *btf_members(const struct btf_type *t) +{ + return (struct btf_member *)(t + 1); +} + +static inline u32 btf_member_bit_offset(const struct btf_type *t, u32 member_idx) +{ + const struct btf_member *m = btf_members(t) + member_idx; + + return __btf_member_bit_offset(t, m); +} + +static inline u32 btf_member_bitfield_size(const struct btf_type *t, u32 member_idx) +{ + const struct btf_member *m = btf_members(t) + member_idx; + + return __btf_member_bitfield_size(t, m); +} + static inline const struct btf_member *btf_type_member(const struct btf_type *t) { return (const struct btf_member *)(t + 1); } +static inline struct btf_array *btf_array(const struct btf_type *t) +{ + return (struct btf_array *)(t + 1); +} + +static inline struct btf_enum *btf_enum(const struct btf_type *t) +{ + return (struct btf_enum *)(t + 1); +} + static inline const struct btf_var_secinfo *btf_type_var_secinfo( const struct btf_type *t) { diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index cf6ca339f3cd..c1a9be6a4b9f 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -36,3 +36,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o obj-${CONFIG_BPF_LSM} += bpf_lsm.o endif obj-$(CONFIG_BPF_PRELOAD) += preload/ + +obj-$(CONFIG_BPF_SYSCALL) += relo_core.o +$(obj)/relo_core.o: $(srctree)/tools/lib/bpf/relo_core.c FORCE + $(call if_changed_rule,cc_o_c) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index f4119a99da7b..c79595aad55b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6413,3 +6413,29 @@ bool bpf_check_mod_kfunc_call(struct kfunc_btf_id_list *klist, u32 kfunc_id, DEFINE_KFUNC_BTF_ID_LIST(bpf_tcp_ca_kfunc_list); DEFINE_KFUNC_BTF_ID_LIST(prog_test_kfunc_list); + +int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, + const struct btf *targ_btf, __u32 targ_id) +{ + return -EOPNOTSUPP; +} + +static bool bpf_core_is_flavor_sep(const char *s) +{ + /* check X___Y name pattern, where X and Y are not underscores */ + return s[0] != '_' && /* X */ + s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ + s[4] != '_'; /* Y */ +} + +size_t bpf_core_essential_name_len(const char *name) +{ + size_t n = strlen(name); + int i; + + for (i = n - 5; i >= 0; i--) { + if (bpf_core_is_flavor_sep(name + i)) + return i + 1; + } + return n; +} diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index c0904f4cb514..56dbe6d16664 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -1,6 +1,60 @@ // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* Copyright (c) 2019 Facebook */ +#ifdef __KERNEL__ +#include +#include +#include +#include +#include "relo_core.h" + +static const char *btf_kind_str(const struct btf_type *t) +{ + return btf_type_str(t); +} + +static bool is_ldimm64_insn(struct bpf_insn *insn) +{ + return insn->code == (BPF_LD | BPF_IMM | BPF_DW); +} + +static const struct btf_type * +skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id) +{ + return btf_type_skip_modifiers(btf, id, res_id); +} + +static const char *btf__name_by_offset(const struct btf *btf, u32 offset) +{ + return btf_name_by_offset(btf, offset); +} + +static s64 btf__resolve_size(const struct btf *btf, u32 type_id) +{ + const struct btf_type *t; + int size; + + t = btf_type_by_id(btf, type_id); + t = btf_resolve_size(btf, t, &size); + if (IS_ERR(t)) + return PTR_ERR(t); + return size; +} + +enum libbpf_print_level { + LIBBPF_WARN, + LIBBPF_INFO, + LIBBPF_DEBUG, +}; + +#undef pr_warn +#undef pr_info +#undef pr_debug +#define pr_warn(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define pr_info(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define pr_debug(fmt, log, ...) bpf_log((void *)log, fmt, "", ##__VA_ARGS__) +#define libbpf_print(level, fmt, ...) bpf_log((void *)prog_name, fmt, ##__VA_ARGS__) +#else #include #include #include @@ -12,8 +66,9 @@ #include "btf.h" #include "str_error.h" #include "libbpf_internal.h" +#endif -#define BPF_CORE_SPEC_MAX_LEN 64 +#define BPF_CORE_SPEC_MAX_LEN 32 /* represents BPF CO-RE field or array element accessor */ struct bpf_core_accessor { @@ -150,7 +205,7 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access * string to specify enumerator's value index that need to be relocated. */ -static int bpf_core_parse_spec(const struct btf *btf, +static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, __u32 type_id, const char *spec_str, enum bpf_core_relo_kind relo_kind, @@ -272,8 +327,8 @@ static int bpf_core_parse_spec(const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", - type_id, spec_str, i, id, btf_kind_str(t)); + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + prog_name, type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } } @@ -346,8 +401,6 @@ recur: targ_id = btf_array(targ_type)->type; goto recur; default: - pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", - btf_kind(local_type), local_id, targ_id); return 0; } } @@ -1045,7 +1098,7 @@ poison: * [] () + => @, * where is a C-syntax view of recorded field access, e.g.: x.a[3].b */ -static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) +static void bpf_core_dump_spec(const char *prog_name, int level, const struct bpf_core_spec *spec) { const struct btf_type *t; const struct btf_enum *e; @@ -1167,7 +1220,8 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, if (str_is_empty(spec_str)) return -EINVAL; - err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec); + err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str, + relo->kind, &local_spec); if (err) { pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), @@ -1178,7 +1232,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); - bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &local_spec); libbpf_print(LIBBPF_DEBUG, "\n"); /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ @@ -1204,14 +1258,14 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, if (err < 0) { pr_warn("prog '%s': relo #%d: error matching candidate #%d ", prog_name, relo_idx, i); - bpf_core_dump_spec(LIBBPF_WARN, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_WARN, &cand_spec); libbpf_print(LIBBPF_WARN, ": %d\n", err); return err; } pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name, relo_idx, err == 0 ? "non-matching" : "matching", i); - bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); + bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &cand_spec); libbpf_print(LIBBPF_DEBUG, "\n"); if (err == 0) -- cgit v1.2.3-59-g8ed1b From fbd94c7afcf99c9f3b1ba1168657ecc428eb2c8d Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 1 Dec 2021 10:10:28 -0800 Subject: bpf: Pass a set of bpf_core_relo-s to prog_load command. struct bpf_core_relo is generated by llvm and processed by libbpf. It's a de-facto uapi. With CO-RE in the kernel the struct bpf_core_relo becomes uapi de-jure. Add an ability to pass a set of 'struct bpf_core_relo' to prog_load command and let the kernel perform CO-RE relocations. Note the struct bpf_line_info and struct bpf_func_info have the same layout when passed from LLVM to libbpf and from libbpf to the kernel except "insn_off" fields means "byte offset" when LLVM generates it. Then libbpf converts it to "insn index" to pass to the kernel. The struct bpf_core_relo's "insn_off" field is always "byte offset". Signed-off-by: Alexei Starovoitov Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211201181040.23337-6-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 8 +++++ include/uapi/linux/bpf.h | 59 +++++++++++++++++++++++++++++++- kernel/bpf/btf.c | 6 ++++ kernel/bpf/syscall.c | 2 +- kernel/bpf/verifier.c | 76 ++++++++++++++++++++++++++++++++++++++++++ tools/include/uapi/linux/bpf.h | 59 +++++++++++++++++++++++++++++++- tools/lib/bpf/relo_core.h | 53 ----------------------------- 7 files changed, 207 insertions(+), 56 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index cad0829710be..8bbf08fbab66 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1732,6 +1732,14 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); +struct bpf_core_ctx { + struct bpf_verifier_log *log; + const struct btf *btf; +}; + +int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, + int relo_idx, void *insn); + #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 9e66b1880020..c26871263f1f 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1342,8 +1342,10 @@ union bpf_attr { /* or valid module BTF object fd or 0 to attach to vmlinux */ __u32 attach_btf_obj_fd; }; - __u32 :32; /* pad */ + __u32 core_relo_cnt; /* number of bpf_core_relo */ __aligned_u64 fd_array; /* array of FDs */ + __aligned_u64 core_relos; + __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -6393,4 +6395,59 @@ enum bpf_core_relo_kind { BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ }; +/* + * "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf + * and from libbpf to the kernel. + * + * CO-RE relocation captures the following data: + * - insn_off - instruction offset (in bytes) within a BPF program that needs + * its insn->imm field to be relocated with actual field info; + * - type_id - BTF type ID of the "root" (containing) entity of a relocatable + * type or field; + * - access_str_off - offset into corresponding .BTF string section. String + * interpretation depends on specific relocation kind: + * - for field-based relocations, string encodes an accessed field using + * a sequence of field and array indices, separated by colon (:). It's + * conceptually very close to LLVM's getelementptr ([0]) instruction's + * arguments for identifying offset to a field. + * - for type-based relocations, strings is expected to be just "0"; + * - for enum value-based relocations, string contains an index of enum + * value within its enum type; + * - kind - one of enum bpf_core_relo_kind; + * + * Example: + * struct sample { + * int a; + * struct { + * int b[10]; + * }; + * }; + * + * struct sample *s = ...; + * int *x = &s->a; // encoded as "0:0" (a is field #0) + * int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, + * // b is field #0 inside anon struct, accessing elem #5) + * int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) + * + * type_id for all relocs in this example will capture BTF type id of + * `struct sample`. + * + * Such relocation is emitted when using __builtin_preserve_access_index() + * Clang built-in, passing expression that captures field address, e.g.: + * + * bpf_probe_read(&dst, sizeof(dst), + * __builtin_preserve_access_index(&src->a.b.c)); + * + * In this case Clang will emit field relocation recording necessary data to + * be able to find offset of embedded `a.b.c` field within `src` struct. + * + * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction + */ +struct bpf_core_relo { + __u32 insn_off; + __u32 type_id; + __u32 access_str_off; + enum bpf_core_relo_kind kind; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c79595aad55b..0d070461e2b8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6439,3 +6439,9 @@ size_t bpf_core_essential_name_len(const char *name) } return n; } + +int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, + int relo_idx, void *insn) +{ + return -EOPNOTSUPP; +} diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 47089d1d67a4..b3ada4085f85 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2184,7 +2184,7 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD fd_array +#define BPF_PROG_LOAD_LAST_FIELD core_relo_rec_size static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6c9c0d9a04a0..6522ffdea487 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10273,6 +10273,78 @@ err_free: return err; } +#define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) +#define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE + +static int check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) +{ + u32 i, nr_core_relo, ncopy, expected_size, rec_size; + struct bpf_core_relo core_relo = {}; + struct bpf_prog *prog = env->prog; + const struct btf *btf = prog->aux->btf; + struct bpf_core_ctx ctx = { + .log = &env->log, + .btf = btf, + }; + bpfptr_t u_core_relo; + int err; + + nr_core_relo = attr->core_relo_cnt; + if (!nr_core_relo) + return 0; + if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) + return -EINVAL; + + rec_size = attr->core_relo_rec_size; + if (rec_size < MIN_CORE_RELO_SIZE || + rec_size > MAX_CORE_RELO_SIZE || + rec_size % sizeof(u32)) + return -EINVAL; + + u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); + expected_size = sizeof(struct bpf_core_relo); + ncopy = min_t(u32, expected_size, rec_size); + + /* Unlike func_info and line_info, copy and apply each CO-RE + * relocation record one at a time. + */ + for (i = 0; i < nr_core_relo; i++) { + /* future proofing when sizeof(bpf_core_relo) changes */ + err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); + if (err) { + if (err == -E2BIG) { + verbose(env, "nonzero tailing record in core_relo"); + if (copy_to_bpfptr_offset(uattr, + offsetof(union bpf_attr, core_relo_rec_size), + &expected_size, sizeof(expected_size))) + err = -EFAULT; + } + break; + } + + if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { + err = -EFAULT; + break; + } + + if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { + verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", + i, core_relo.insn_off, prog->len); + err = -EINVAL; + break; + } + + err = bpf_core_apply(&ctx, &core_relo, i, + &prog->insnsi[core_relo.insn_off / 8]); + if (err) + break; + bpfptr_add(&u_core_relo, rec_size); + } + return err; +} + static int check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr) @@ -10303,6 +10375,10 @@ static int check_btf_info(struct bpf_verifier_env *env, if (err) return err; + err = check_core_relo(env, attr, uattr); + if (err) + return err; + return 0; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 9e66b1880020..c26871263f1f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1342,8 +1342,10 @@ union bpf_attr { /* or valid module BTF object fd or 0 to attach to vmlinux */ __u32 attach_btf_obj_fd; }; - __u32 :32; /* pad */ + __u32 core_relo_cnt; /* number of bpf_core_relo */ __aligned_u64 fd_array; /* array of FDs */ + __aligned_u64 core_relos; + __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -6393,4 +6395,59 @@ enum bpf_core_relo_kind { BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ }; +/* + * "struct bpf_core_relo" is used to pass relocation data form LLVM to libbpf + * and from libbpf to the kernel. + * + * CO-RE relocation captures the following data: + * - insn_off - instruction offset (in bytes) within a BPF program that needs + * its insn->imm field to be relocated with actual field info; + * - type_id - BTF type ID of the "root" (containing) entity of a relocatable + * type or field; + * - access_str_off - offset into corresponding .BTF string section. String + * interpretation depends on specific relocation kind: + * - for field-based relocations, string encodes an accessed field using + * a sequence of field and array indices, separated by colon (:). It's + * conceptually very close to LLVM's getelementptr ([0]) instruction's + * arguments for identifying offset to a field. + * - for type-based relocations, strings is expected to be just "0"; + * - for enum value-based relocations, string contains an index of enum + * value within its enum type; + * - kind - one of enum bpf_core_relo_kind; + * + * Example: + * struct sample { + * int a; + * struct { + * int b[10]; + * }; + * }; + * + * struct sample *s = ...; + * int *x = &s->a; // encoded as "0:0" (a is field #0) + * int *y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, + * // b is field #0 inside anon struct, accessing elem #5) + * int *z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) + * + * type_id for all relocs in this example will capture BTF type id of + * `struct sample`. + * + * Such relocation is emitted when using __builtin_preserve_access_index() + * Clang built-in, passing expression that captures field address, e.g.: + * + * bpf_probe_read(&dst, sizeof(dst), + * __builtin_preserve_access_index(&src->a.b.c)); + * + * In this case Clang will emit field relocation recording necessary data to + * be able to find offset of embedded `a.b.c` field within `src` struct. + * + * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction + */ +struct bpf_core_relo { + __u32 insn_off; + __u32 type_id; + __u32 access_str_off; + enum bpf_core_relo_kind kind; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/tools/lib/bpf/relo_core.h b/tools/lib/bpf/relo_core.h index 3d0b86e7f439..f410691cc4e5 100644 --- a/tools/lib/bpf/relo_core.h +++ b/tools/lib/bpf/relo_core.h @@ -6,59 +6,6 @@ #include -/* The minimum bpf_core_relo checked by the loader - * - * CO-RE relocation captures the following data: - * - insn_off - instruction offset (in bytes) within a BPF program that needs - * its insn->imm field to be relocated with actual field info; - * - type_id - BTF type ID of the "root" (containing) entity of a relocatable - * type or field; - * - access_str_off - offset into corresponding .BTF string section. String - * interpretation depends on specific relocation kind: - * - for field-based relocations, string encodes an accessed field using - * a sequence of field and array indices, separated by colon (:). It's - * conceptually very close to LLVM's getelementptr ([0]) instruction's - * arguments for identifying offset to a field. - * - for type-based relocations, strings is expected to be just "0"; - * - for enum value-based relocations, string contains an index of enum - * value within its enum type; - * - * Example to provide a better feel. - * - * struct sample { - * int a; - * struct { - * int b[10]; - * }; - * }; - * - * struct sample *s = ...; - * int x = &s->a; // encoded as "0:0" (a is field #0) - * int y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, - * // b is field #0 inside anon struct, accessing elem #5) - * int z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) - * - * type_id for all relocs in this example will capture BTF type id of - * `struct sample`. - * - * Such relocation is emitted when using __builtin_preserve_access_index() - * Clang built-in, passing expression that captures field address, e.g.: - * - * bpf_probe_read(&dst, sizeof(dst), - * __builtin_preserve_access_index(&src->a.b.c)); - * - * In this case Clang will emit field relocation recording necessary data to - * be able to find offset of embedded `a.b.c` field within `src` struct. - * - * [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction - */ -struct bpf_core_relo { - __u32 insn_off; - __u32 type_id; - __u32 access_str_off; - enum bpf_core_relo_kind kind; -}; - struct bpf_core_cand { const struct btf *btf; const struct btf_type *t; -- cgit v1.2.3-59-g8ed1b From b247f32aecad09e6cf7edff7739e6f2c9dc5fca9 Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Thu, 28 Oct 2021 16:03:06 +0300 Subject: net/mlx5: Dynamically resize flow counters query buffer The flow counters bulk query buffer is allocated once during mlx5_fc_init_stats(). For PFs and VFs this buffer usually takes a little more than 512KB of memory, which is aligned to the next power of 2, to 1MB. For SFs, this buffer is reduced and takes around 128 Bytes. The buffer size determines the maximum number of flow counters that can be queried at a time. Thus, having a bigger buffer can improve performance for users that need to query many flow counters. There are cases that don't use many flow counters and don't need a big buffer (e.g. SFs, VFs). Since this size is critical with large scale, in these cases the buffer size should be reduced. In order to reduce memory consumption while maintaining query performance, change the query buffer's allocation scheme to the following: - First allocate the buffer with small initial size. - If the number of counters surpasses the initial size, resize the buffer to the maximum size. The buffer only grows and isn't shrank, because users with many flow counters don't care about the buffer size and we don't want to add resize overhead if the current number of counters drops. This solution is preferable to the current one, which is less accurate and only addresses SFs. Signed-off-by: Avihai Horon Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/fs_counters.c | 74 ++++++++++++++++++---- include/linux/mlx5/driver.h | 4 ++ 2 files changed, 64 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c index 7e0e04cf26f8..b406e0367af6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c @@ -38,9 +38,10 @@ #include "fs_cmd.h" #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000) +#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000) /* Max number of counters to query in bulk read is 32K */ #define MLX5_SW_MAX_COUNTERS_BULK BIT(15) -#define MLX5_SF_NUM_COUNTERS_BULK 8 +#define MLX5_INIT_COUNTERS_BULK 8 #define MLX5_FC_POOL_MAX_THRESHOLD BIT(18) #define MLX5_FC_POOL_USED_BUFF_RATIO 10 @@ -145,13 +146,15 @@ static void mlx5_fc_stats_remove(struct mlx5_core_dev *dev, spin_unlock(&fc_stats->counters_idr_lock); } -static int get_max_bulk_query_len(struct mlx5_core_dev *dev) +static int get_init_bulk_query_len(struct mlx5_core_dev *dev) { - int num_counters_bulk = mlx5_core_is_sf(dev) ? - MLX5_SF_NUM_COUNTERS_BULK : - MLX5_SW_MAX_COUNTERS_BULK; + return min_t(int, MLX5_INIT_COUNTERS_BULK, + (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk))); +} - return min_t(int, num_counters_bulk, +static int get_max_bulk_query_len(struct mlx5_core_dev *dev) +{ + return min_t(int, MLX5_SW_MAX_COUNTERS_BULK, (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk))); } @@ -177,7 +180,7 @@ static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, { struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; bool query_more_counters = (first->id <= last_id); - int max_bulk_len = get_max_bulk_query_len(dev); + int cur_bulk_len = fc_stats->bulk_query_len; u32 *data = fc_stats->bulk_query_out; struct mlx5_fc *counter = first; u32 bulk_base_id; @@ -189,7 +192,7 @@ static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, bulk_base_id = counter->id & ~0x3; /* number of counters to query inc. the last counter */ - bulk_len = min_t(int, max_bulk_len, + bulk_len = min_t(int, cur_bulk_len, ALIGN(last_id - bulk_base_id + 1, 4)); err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, @@ -230,6 +233,41 @@ static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) mlx5_fc_free(dev, counter); } +static void mlx5_fc_stats_bulk_query_size_increase(struct mlx5_core_dev *dev) +{ + struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; + int max_bulk_len = get_max_bulk_query_len(dev); + unsigned long now = jiffies; + u32 *bulk_query_out_tmp; + int max_out_len; + + if (fc_stats->bulk_query_alloc_failed && + time_before(now, fc_stats->next_bulk_query_alloc)) + return; + + max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); + bulk_query_out_tmp = kzalloc(max_out_len, GFP_KERNEL); + if (!bulk_query_out_tmp) { + mlx5_core_warn_once(dev, + "Can't increase flow counters bulk query buffer size, insufficient memory, bulk_size(%d)\n", + max_bulk_len); + fc_stats->bulk_query_alloc_failed = true; + fc_stats->next_bulk_query_alloc = + now + MLX5_FC_BULK_QUERY_ALLOC_PERIOD; + return; + } + + kfree(fc_stats->bulk_query_out); + fc_stats->bulk_query_out = bulk_query_out_tmp; + fc_stats->bulk_query_len = max_bulk_len; + if (fc_stats->bulk_query_alloc_failed) { + mlx5_core_info(dev, + "Flow counters bulk query buffer size increased, bulk_size(%d)\n", + max_bulk_len); + fc_stats->bulk_query_alloc_failed = false; + } +} + static void mlx5_fc_stats_work(struct work_struct *work) { struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev, @@ -247,15 +285,22 @@ static void mlx5_fc_stats_work(struct work_struct *work) queue_delayed_work(fc_stats->wq, &fc_stats->work, fc_stats->sampling_interval); - llist_for_each_entry(counter, addlist, addlist) + llist_for_each_entry(counter, addlist, addlist) { mlx5_fc_stats_insert(dev, counter); + fc_stats->num_counters++; + } llist_for_each_entry_safe(counter, tmp, dellist, dellist) { mlx5_fc_stats_remove(dev, counter); mlx5_fc_release(dev, counter); + fc_stats->num_counters--; } + if (fc_stats->bulk_query_len < get_max_bulk_query_len(dev) && + fc_stats->num_counters > get_init_bulk_query_len(dev)) + mlx5_fc_stats_bulk_query_size_increase(dev); + if (time_before(now, fc_stats->next_query) || list_empty(&fc_stats->counters)) return; @@ -378,8 +423,8 @@ EXPORT_SYMBOL(mlx5_fc_destroy); int mlx5_init_fc_stats(struct mlx5_core_dev *dev) { struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; - int max_bulk_len; - int max_out_len; + int init_bulk_len; + int init_out_len; spin_lock_init(&fc_stats->counters_idr_lock); idr_init(&fc_stats->counters_idr); @@ -387,11 +432,12 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev) init_llist_head(&fc_stats->addlist); init_llist_head(&fc_stats->dellist); - max_bulk_len = get_max_bulk_query_len(dev); - max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); - fc_stats->bulk_query_out = kzalloc(max_out_len, GFP_KERNEL); + init_bulk_len = get_init_bulk_query_len(dev); + init_out_len = mlx5_cmd_fc_get_bulk_query_out_len(init_bulk_len); + fc_stats->bulk_query_out = kzalloc(init_out_len, GFP_KERNEL); if (!fc_stats->bulk_query_out) return -ENOMEM; + fc_stats->bulk_query_len = init_bulk_len; fc_stats->wq = create_singlethread_workqueue("mlx5_fc"); if (!fc_stats->wq) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index a623ec635947..78655d8d13a7 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -478,6 +478,10 @@ struct mlx5_fc_stats { unsigned long next_query; unsigned long sampling_interval; /* jiffies */ u32 *bulk_query_out; + int bulk_query_len; + size_t num_counters; + bool bulk_query_alloc_failed; + unsigned long next_bulk_query_alloc; struct mlx5_fc_pool fc_pool; }; -- cgit v1.2.3-59-g8ed1b From e2748ad5257754a47376e28c0f9dda4f5c1e5ca3 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 2 Nov 2021 16:02:00 -0600 Subject: mtd: remove unused header file Commit d24dbd7541ff ("mtd: maps: Get rid of the latch-addr-flash driver") removed the last user of but left the header file behind. Nothing uses this file, delete it now. Cc: Boris Brezillon Cc: Miquel Raynal Cc: Richard Weinberger Cc: Vignesh Raghavendra Cc: linux-mtd@lists.infradead.org Signed-off-by: Jonathan Corbet Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20211102220203.940290-7-corbet@lwn.net --- include/linux/mtd/latch-addr-flash.h | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 include/linux/mtd/latch-addr-flash.h (limited to 'include/linux') diff --git a/include/linux/mtd/latch-addr-flash.h b/include/linux/mtd/latch-addr-flash.h deleted file mode 100644 index e94b8e128074..000000000000 --- a/include/linux/mtd/latch-addr-flash.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Interface for NOR flash driver whose high address lines are latched - * - * Copyright © 2008 MontaVista Software, Inc. - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. - */ -#ifndef __LATCH_ADDR_FLASH__ -#define __LATCH_ADDR_FLASH__ - -struct map_info; -struct mtd_partition; - -struct latch_addr_flash_data { - unsigned int width; - unsigned int size; - - int (*init)(void *data, int cs); - void (*done)(void *data); - void (*set_window)(unsigned long offset, void *data); - void *data; - - unsigned int nr_parts; - struct mtd_partition *parts; -}; - -#endif -- cgit v1.2.3-59-g8ed1b From 2c4dcd7fd57b20a21b65da04d89c38a7217d79cf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 29 Nov 2021 14:03:07 +0100 Subject: topology/sysfs: export die attributes only if an architectures has support The die_id and die_cpus topology sysfs attributes have been added with commit 0e344d8c709f ("cpu/topology: Export die_id") and commit 2e4c54dac7b3 ("topology: Create core_cpus and die_cpus sysfs attributes"). While they are currently only used and useful for x86 they are still present with bogus default values for all architectures. Instead of enforcing such new sysfs attributes to all architectures, make them only optional visible if an architecture opts in by defining both the topology_die_id and topology_die_cpumask attributes. This is similar to what was done when the book and drawer topology levels were introduced: avoid useless and therefore confusing sysfs attributes for architectures which cannot make use of them. This should not break any existing applications, since this is a rather new interface and applications should be able to handle also older kernel versions without such attributes - besides that they contain only useful information for x86. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Heiko Carstens Link: https://lore.kernel.org/r/20211129130309.3256168-2-hca@linux.ibm.com Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/cputopology.rst | 3 +++ drivers/base/topology.c | 8 ++++++++ include/linux/topology.h | 4 ++++ 3 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cputopology.rst b/Documentation/admin-guide/cputopology.rst index 6b62e182baf4..c68d07533c45 100644 --- a/Documentation/admin-guide/cputopology.rst +++ b/Documentation/admin-guide/cputopology.rst @@ -11,6 +11,9 @@ Architecture-neutral, drivers/base/topology.c, exports these attributes. However, the book and drawer related sysfs files will only be created if CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are selected, respectively. +The die hierarchy related sysfs files will only be created if an architecture +provides the related macros as described below. + CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are currently only used on s390, where they reflect the cpu and cache hierarchy. diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 8f2b641d0b8c..f079a55793ec 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -45,8 +45,10 @@ static ssize_t name##_list_read(struct file *file, struct kobject *kobj, \ define_id_show_func(physical_package_id); static DEVICE_ATTR_RO(physical_package_id); +#ifdef TOPOLOGY_DIE_SYSFS define_id_show_func(die_id); static DEVICE_ATTR_RO(die_id); +#endif define_id_show_func(cluster_id); static DEVICE_ATTR_RO(cluster_id); @@ -70,9 +72,11 @@ define_siblings_read_func(cluster_cpus, cluster_cpumask); static BIN_ATTR_RO(cluster_cpus, 0); static BIN_ATTR_RO(cluster_cpus_list, 0); +#ifdef TOPOLOGY_DIE_SYSFS define_siblings_read_func(die_cpus, die_cpumask); static BIN_ATTR_RO(die_cpus, 0); static BIN_ATTR_RO(die_cpus_list, 0); +#endif define_siblings_read_func(package_cpus, core_cpumask); static BIN_ATTR_RO(package_cpus, 0); @@ -103,8 +107,10 @@ static struct bin_attribute *bin_attrs[] = { &bin_attr_core_siblings_list, &bin_attr_cluster_cpus, &bin_attr_cluster_cpus_list, +#ifdef TOPOLOGY_DIE_SYSFS &bin_attr_die_cpus, &bin_attr_die_cpus_list, +#endif &bin_attr_package_cpus, &bin_attr_package_cpus_list, #ifdef CONFIG_SCHED_BOOK @@ -120,7 +126,9 @@ static struct bin_attribute *bin_attrs[] = { static struct attribute *default_attrs[] = { &dev_attr_physical_package_id.attr, +#ifdef TOPOLOGY_DIE_SYSFS &dev_attr_die_id.attr, +#endif &dev_attr_cluster_id.attr, &dev_attr_core_id.attr, #ifdef CONFIG_SCHED_BOOK diff --git a/include/linux/topology.h b/include/linux/topology.h index 0b3704ad13c8..8d1bdae76230 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -180,6 +180,10 @@ static inline int cpu_to_mem(int cpu) #endif /* [!]CONFIG_HAVE_MEMORYLESS_NODES */ +#if defined(topology_die_id) && defined(topology_die_cpumask) +#define TOPOLOGY_DIE_SYSFS +#endif + #ifndef topology_physical_package_id #define topology_physical_package_id(cpu) ((void)(cpu), -1) #endif -- cgit v1.2.3-59-g8ed1b From e795707703b32fecdd7467afcc33ff1e92416c05 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 29 Nov 2021 14:03:08 +0100 Subject: topology/sysfs: export cluster attributes only if an architectures has support The cluster_id and cluster_cpus topology sysfs attributes have been added with commit c5e22feffdd7 ("topology: Represent clusters of CPUs within a die"). They are currently only used for x86, arm64, and riscv (via generic arch topology), however they are still present with bogus default values for all other architectures. Instead of enforcing such new sysfs attributes to all architectures, make them only optional visible if an architecture opts in by defining both the topology_cluster_id and topology_cluster_cpumask attributes. This is similar to what was done when the book and drawer topology levels were introduced: avoid useless and therefore confusing sysfs attributes for architectures which cannot make use of them. This should not break any existing applications, since this is a new interface introduced with the v5.16 merge window. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Heiko Carstens Link: https://lore.kernel.org/r/20211129130309.3256168-3-hca@linux.ibm.com Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/cputopology.rst | 4 ++-- drivers/base/topology.c | 8 ++++++++ include/linux/topology.h | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cputopology.rst b/Documentation/admin-guide/cputopology.rst index c68d07533c45..ad2238b41439 100644 --- a/Documentation/admin-guide/cputopology.rst +++ b/Documentation/admin-guide/cputopology.rst @@ -11,8 +11,8 @@ Architecture-neutral, drivers/base/topology.c, exports these attributes. However, the book and drawer related sysfs files will only be created if CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are selected, respectively. -The die hierarchy related sysfs files will only be created if an architecture -provides the related macros as described below. +The die and cluster hierarchy related sysfs files will only be created if an +architecture provides the related macros as described below. CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are currently only used on s390, where they reflect the cpu and cache hierarchy. diff --git a/drivers/base/topology.c b/drivers/base/topology.c index f079a55793ec..9d049724e4b4 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -50,8 +50,10 @@ define_id_show_func(die_id); static DEVICE_ATTR_RO(die_id); #endif +#ifdef TOPOLOGY_CLUSTER_SYSFS define_id_show_func(cluster_id); static DEVICE_ATTR_RO(cluster_id); +#endif define_id_show_func(core_id); static DEVICE_ATTR_RO(core_id); @@ -68,9 +70,11 @@ define_siblings_read_func(core_siblings, core_cpumask); static BIN_ATTR_RO(core_siblings, 0); static BIN_ATTR_RO(core_siblings_list, 0); +#ifdef TOPOLOGY_CLUSTER_SYSFS define_siblings_read_func(cluster_cpus, cluster_cpumask); static BIN_ATTR_RO(cluster_cpus, 0); static BIN_ATTR_RO(cluster_cpus_list, 0); +#endif #ifdef TOPOLOGY_DIE_SYSFS define_siblings_read_func(die_cpus, die_cpumask); @@ -105,8 +109,10 @@ static struct bin_attribute *bin_attrs[] = { &bin_attr_thread_siblings_list, &bin_attr_core_siblings, &bin_attr_core_siblings_list, +#ifdef TOPOLOGY_CLUSTER_SYSFS &bin_attr_cluster_cpus, &bin_attr_cluster_cpus_list, +#endif #ifdef TOPOLOGY_DIE_SYSFS &bin_attr_die_cpus, &bin_attr_die_cpus_list, @@ -129,7 +135,9 @@ static struct attribute *default_attrs[] = { #ifdef TOPOLOGY_DIE_SYSFS &dev_attr_die_id.attr, #endif +#ifdef TOPOLOGY_CLUSTER_SYSFS &dev_attr_cluster_id.attr, +#endif &dev_attr_core_id.attr, #ifdef CONFIG_SCHED_BOOK &dev_attr_book_id.attr, diff --git a/include/linux/topology.h b/include/linux/topology.h index 8d1bdae76230..d52be69037db 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -183,6 +183,9 @@ static inline int cpu_to_mem(int cpu) #if defined(topology_die_id) && defined(topology_die_cpumask) #define TOPOLOGY_DIE_SYSFS #endif +#if defined(topology_cluster_id) && defined(topology_cluster_cpumask) +#define TOPOLOGY_CLUSTER_SYSFS +#endif #ifndef topology_physical_package_id #define topology_physical_package_id(cpu) ((void)(cpu), -1) -- cgit v1.2.3-59-g8ed1b From f1045056c726440469d89d23c13734bcd6c0d15b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 29 Nov 2021 14:03:09 +0100 Subject: topology/sysfs: rework book and drawer topology ifdefery Provide default defines for the topology_book_[id|cpumask] and topology_drawer_[id|cpumask] macros just like for each other topology level. This way all topology levels are handled in a similar way. Still the the book and drawer levels are only used on s390, and also the sysfs attributes are only created on s390. However other architectures may opt in if wanted. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Heiko Carstens Link: https://lore.kernel.org/r/20211129130309.3256168-4-hca@linux.ibm.com Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/cputopology.rst | 36 +++++++++++++------------------ drivers/base/topology.c | 12 +++++------ include/linux/topology.h | 18 ++++++++++++++++ 3 files changed, 39 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cputopology.rst b/Documentation/admin-guide/cputopology.rst index ad2238b41439..677ba1c2c820 100644 --- a/Documentation/admin-guide/cputopology.rst +++ b/Documentation/admin-guide/cputopology.rst @@ -8,14 +8,9 @@ to /proc/cpuinfo output of some architectures. They reside in Documentation/ABI/stable/sysfs-devices-system-cpu. Architecture-neutral, drivers/base/topology.c, exports these attributes. -However, the book and drawer related sysfs files will only be created if -CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are selected, respectively. - -The die and cluster hierarchy related sysfs files will only be created if an -architecture provides the related macros as described below. - -CONFIG_SCHED_BOOK and CONFIG_SCHED_DRAWER are currently only used on s390, -where they reflect the cpu and cache hierarchy. +However the die, cluster, book, and drawer hierarchy related sysfs files will +only be created if an architecture provides the related macros as described +below. For an architecture to support this feature, it must define some of these macros in include/asm-XXX/topology.h:: @@ -42,19 +37,18 @@ To be consistent on all architectures, include/linux/topology.h provides default definitions for any of the above macros that are not defined by include/asm-XXX/topology.h: -1) topology_physical_package_id: -1 -2) topology_die_id: -1 -3) topology_cluster_id: -1 -4) topology_core_id: 0 -5) topology_sibling_cpumask: just the given CPU -6) topology_core_cpumask: just the given CPU -7) topology_cluster_cpumask: just the given CPU -8) topology_die_cpumask: just the given CPU - -For architectures that don't support books (CONFIG_SCHED_BOOK) there are no -default definitions for topology_book_id() and topology_book_cpumask(). -For architectures that don't support drawers (CONFIG_SCHED_DRAWER) there are -no default definitions for topology_drawer_id() and topology_drawer_cpumask(). + 1) topology_physical_package_id: -1 + 2) topology_die_id: -1 + 3) topology_cluster_id: -1 + 4) topology_core_id: 0 + 5) topology_book_id: -1 + 6) topology_drawer_id: -1 + 7) topology_sibling_cpumask: just the given CPU + 8) topology_core_cpumask: just the given CPU + 9) topology_cluster_cpumask: just the given CPU +10) topology_die_cpumask: just the given CPU +11) topology_book_cpumask: just the given CPU +12) topology_drawer_cpumask: just the given CPU Additionally, CPU topology information is provided under /sys/devices/system/cpu and includes these files. The internal diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 9d049724e4b4..fc24e89f9592 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -86,7 +86,7 @@ define_siblings_read_func(package_cpus, core_cpumask); static BIN_ATTR_RO(package_cpus, 0); static BIN_ATTR_RO(package_cpus_list, 0); -#ifdef CONFIG_SCHED_BOOK +#ifdef TOPOLOGY_BOOK_SYSFS define_id_show_func(book_id); static DEVICE_ATTR_RO(book_id); define_siblings_read_func(book_siblings, book_cpumask); @@ -94,7 +94,7 @@ static BIN_ATTR_RO(book_siblings, 0); static BIN_ATTR_RO(book_siblings_list, 0); #endif -#ifdef CONFIG_SCHED_DRAWER +#ifdef TOPOLOGY_DRAWER_SYSFS define_id_show_func(drawer_id); static DEVICE_ATTR_RO(drawer_id); define_siblings_read_func(drawer_siblings, drawer_cpumask); @@ -119,11 +119,11 @@ static struct bin_attribute *bin_attrs[] = { #endif &bin_attr_package_cpus, &bin_attr_package_cpus_list, -#ifdef CONFIG_SCHED_BOOK +#ifdef TOPOLOGY_BOOK_SYSFS &bin_attr_book_siblings, &bin_attr_book_siblings_list, #endif -#ifdef CONFIG_SCHED_DRAWER +#ifdef TOPOLOGY_DRAWER_SYSFS &bin_attr_drawer_siblings, &bin_attr_drawer_siblings_list, #endif @@ -139,10 +139,10 @@ static struct attribute *default_attrs[] = { &dev_attr_cluster_id.attr, #endif &dev_attr_core_id.attr, -#ifdef CONFIG_SCHED_BOOK +#ifdef TOPOLOGY_BOOK_SYSFS &dev_attr_book_id.attr, #endif -#ifdef CONFIG_SCHED_DRAWER +#ifdef TOPOLOGY_DRAWER_SYSFS &dev_attr_drawer_id.attr, #endif NULL diff --git a/include/linux/topology.h b/include/linux/topology.h index d52be69037db..a6e201758ae9 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -186,6 +186,12 @@ static inline int cpu_to_mem(int cpu) #if defined(topology_cluster_id) && defined(topology_cluster_cpumask) #define TOPOLOGY_CLUSTER_SYSFS #endif +#if defined(topology_book_id) && defined(topology_book_cpumask) +#define TOPOLOGY_BOOK_SYSFS +#endif +#if defined(topology_drawer_id) && defined(topology_drawer_cpumask) +#define TOPOLOGY_DRAWER_SYSFS +#endif #ifndef topology_physical_package_id #define topology_physical_package_id(cpu) ((void)(cpu), -1) @@ -199,6 +205,12 @@ static inline int cpu_to_mem(int cpu) #ifndef topology_core_id #define topology_core_id(cpu) ((void)(cpu), 0) #endif +#ifndef topology_book_id +#define topology_book_id(cpu) ((void)(cpu), -1) +#endif +#ifndef topology_drawer_id +#define topology_drawer_id(cpu) ((void)(cpu), -1) +#endif #ifndef topology_sibling_cpumask #define topology_sibling_cpumask(cpu) cpumask_of(cpu) #endif @@ -211,6 +223,12 @@ static inline int cpu_to_mem(int cpu) #ifndef topology_die_cpumask #define topology_die_cpumask(cpu) cpumask_of(cpu) #endif +#ifndef topology_book_cpumask +#define topology_book_cpumask(cpu) cpumask_of(cpu) +#endif +#ifndef topology_drawer_cpumask +#define topology_drawer_cpumask(cpu) cpumask_of(cpu) +#endif #if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask) static inline const struct cpumask *cpu_smt_mask(int cpu) -- cgit v1.2.3-59-g8ed1b From c7fdb2404f66131bc9c22e06f712717288826487 Mon Sep 17 00:00:00 2001 From: Abhyuday Godhasara Date: Sun, 28 Nov 2021 23:02:14 -0800 Subject: drivers: soc: xilinx: add xilinx event management driver Xilinx event management driver provides an interface to subscribe or unsubscribe for the event/callback supported by firmware. An agent can use this driver to register for Error Event, Device Event and Suspend callback. This driver only allows one agent per event to do registration. Driver will return an error in case of multiple registration for the same event. This driver gets notification from firmware through TF-A as SGI. During initialization, event manager driver register handler for SGI used for notification. It also provides SGI number info to TF-A by using IOCTL_REGISTER_SGI call to TF-A. After receiving notification from firmware, the driver makes an SMC call to TF-A to get IPI data. From the IPI data provided by TF-A, event manager identified the cause of event and forward that event/callback notification to the respective subscribed driver. After this, in case of Error Event, driver performs unregistration as firmware expecting from agent to do re-registration if the agent wants to get notified on the second occurrence of an error event. Add new IOCTL id IOCTL_REGISTER_SGI = 25 which is used to register SGI on TF-A. Older firmware doesn't have all required support for event handling which is required by the event manager driver. So add check for the register notifier version in the event manager driver. Xilinx event management driver provides support to subscribe for multiple error events with the use of Event Mask in a single call of xlnx_register_event(). Agent driver can provide 'Event' parameter value as ORed of multiple event masks to register single callback for multiple events. For example, to register callback for event=0x1 and event=0x2 for the given node, agent can provide event=0x3 (0x1 | 0x2). It is not possible to register multiple events for different nodes in a single registration call. Also provide support to receive multiple error events as in single notification from firmware and then forward it to subscribed drivers via registered callback one by one. Acked-by: Michal Simek Signed-off-by: Tejas Patel Signed-off-by: Rajan Vaja Signed-off-by: Abhyuday Godhasara Link: https://lore.kernel.org/r/20211129070216.30253-2-abhyuday.godhasara@xilinx.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 6 + drivers/soc/xilinx/Kconfig | 10 + drivers/soc/xilinx/Makefile | 1 + drivers/soc/xilinx/xlnx_event_manager.c | 600 ++++++++++++++++++++++++++++ include/linux/firmware/xlnx-event-manager.h | 36 ++ include/linux/firmware/xlnx-zynqmp.h | 2 + 6 files changed, 655 insertions(+) create mode 100644 drivers/soc/xilinx/xlnx_event_manager.c create mode 100644 include/linux/firmware/xlnx-event-manager.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 0533c00325d6..3ef68211cc6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20946,6 +20946,12 @@ T: git https://github.com/Xilinx/linux-xlnx.git F: Documentation/devicetree/bindings/phy/xlnx,zynqmp-psgtr.yaml F: drivers/phy/xilinx/phy-zynqmp.c +XILINX EVENT MANAGEMENT DRIVER +M: Abhyuday Godhasara +S: Maintained +F: drivers/soc/xilinx/xlnx_event_manager.c +F: include/linux/firmware/xlnx-event-manager.h + XILLYBUS DRIVER M: Eli Billauer L: linux-kernel@vger.kernel.org diff --git a/drivers/soc/xilinx/Kconfig b/drivers/soc/xilinx/Kconfig index 53af9115dc31..8a755a5c8836 100644 --- a/drivers/soc/xilinx/Kconfig +++ b/drivers/soc/xilinx/Kconfig @@ -25,4 +25,14 @@ config ZYNQMP_PM_DOMAINS Say yes to enable device power management through PM domains If in doubt, say N. +config XLNX_EVENT_MANAGER + bool "Enable Xilinx Event Management Driver" + depends on ZYNQMP_FIRMWARE + default ZYNQMP_FIRMWARE + help + Say yes to enable event management support for Xilinx. + This driver uses firmware driver as an interface for event/power + management request to firmware. + + If in doubt, say N. endmenu diff --git a/drivers/soc/xilinx/Makefile b/drivers/soc/xilinx/Makefile index 9854e6f6086b..41e585bc9c67 100644 --- a/drivers/soc/xilinx/Makefile +++ b/drivers/soc/xilinx/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ZYNQMP_POWER) += zynqmp_power.o obj-$(CONFIG_ZYNQMP_PM_DOMAINS) += zynqmp_pm_domains.o +obj-$(CONFIG_XLNX_EVENT_MANAGER) += xlnx_event_manager.o diff --git a/drivers/soc/xilinx/xlnx_event_manager.c b/drivers/soc/xilinx/xlnx_event_manager.c new file mode 100644 index 000000000000..b27f8853508e --- /dev/null +++ b/drivers/soc/xilinx/xlnx_event_manager.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx Event Management Driver + * + * Copyright (C) 2021 Xilinx, Inc. + * + * Abhyuday Godhasara + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number1); + +static int virq_sgi; +static int event_manager_availability = -EACCES; + +/* SGI number used for Event management driver */ +#define XLNX_EVENT_SGI_NUM (15) + +/* Max number of driver can register for same event */ +#define MAX_DRIVER_PER_EVENT (10U) + +/* Max HashMap Order for PM API feature check (1<<7 = 128) */ +#define REGISTERED_DRIVER_MAX_ORDER (7) + +#define MAX_BITS (32U) /* Number of bits available for error mask */ + +#define FIRMWARE_VERSION_MASK (0xFFFFU) +#define REGISTER_NOTIFIER_FIRMWARE_VERSION (2U) + +static DEFINE_HASHTABLE(reg_driver_map, REGISTERED_DRIVER_MAX_ORDER); +static int sgi_num = XLNX_EVENT_SGI_NUM; + +/** + * struct registered_event_data - Registered Event Data. + * @key: key is the combine id(Node-Id | Event-Id) of type u64 + * where upper u32 for Node-Id and lower u32 for Event-Id, + * And this used as key to index into hashmap. + * @agent_data: Data passed back to handler function. + * @cb_type: Type of Api callback, like PM_NOTIFY_CB, etc. + * @eve_cb: Function pointer to store the callback function. + * @wake: If this flag set, firmware will wakeup processor if is + * in sleep or power down state. + * @hentry: hlist_node that hooks this entry into hashtable. + */ +struct registered_event_data { + u64 key; + enum pm_api_cb_id cb_type; + void *agent_data; + + event_cb_func_t eve_cb; + bool wake; + struct hlist_node hentry; +}; + +static bool xlnx_is_error_event(const u32 node_id) +{ + if (node_id == EVENT_ERROR_PMC_ERR1 || + node_id == EVENT_ERROR_PMC_ERR2 || + node_id == EVENT_ERROR_PSM_ERR1 || + node_id == EVENT_ERROR_PSM_ERR2) + return true; + + return false; +} + +static int xlnx_add_cb_for_notify_event(const u32 node_id, const u32 event, const bool wake, + event_cb_func_t cb_fun, void *data) +{ + u64 key = 0; + struct registered_event_data *eve_data; + + key = ((u64)node_id << 32U) | (u64)event; + /* Check for existing entry in hash table for given key id */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, key) { + if (eve_data->key == key) { + pr_err("Found as already registered\n"); + return -EINVAL; + } + } + + /* Add new entry if not present */ + eve_data = kmalloc(sizeof(*eve_data), GFP_KERNEL); + if (!eve_data) + return -ENOMEM; + + eve_data->key = key; + eve_data->cb_type = PM_NOTIFY_CB; + eve_data->eve_cb = cb_fun; + eve_data->wake = wake; + eve_data->agent_data = data; + + hash_add(reg_driver_map, &eve_data->hentry, key); + + return 0; +} + +static int xlnx_add_cb_for_suspend(event_cb_func_t cb_fun, void *data) +{ + struct registered_event_data *eve_data; + + /* Check for existing entry in hash table for given cb_type */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, PM_INIT_SUSPEND_CB) { + if (eve_data->cb_type == PM_INIT_SUSPEND_CB) { + pr_err("Found as already registered\n"); + return -EINVAL; + } + } + + /* Add new entry if not present */ + eve_data = kmalloc(sizeof(*eve_data), GFP_KERNEL); + if (!eve_data) + return -ENOMEM; + + eve_data->key = 0; + eve_data->cb_type = PM_INIT_SUSPEND_CB; + eve_data->eve_cb = cb_fun; + eve_data->agent_data = data; + + hash_add(reg_driver_map, &eve_data->hentry, PM_INIT_SUSPEND_CB); + + return 0; +} + +static int xlnx_remove_cb_for_suspend(event_cb_func_t cb_fun) +{ + bool is_callback_found = false; + struct registered_event_data *eve_data; + + /* Check for existing entry in hash table for given cb_type */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, PM_INIT_SUSPEND_CB) { + if (eve_data->cb_type == PM_INIT_SUSPEND_CB && + eve_data->eve_cb == cb_fun) { + is_callback_found = true; + /* remove an object from a hashtable */ + hash_del(&eve_data->hentry); + kfree(eve_data); + } + } + if (!is_callback_found) { + pr_warn("Didn't find any registered callback for suspend event\n"); + return -EINVAL; + } + + return 0; +} + +static int xlnx_remove_cb_for_notify_event(const u32 node_id, const u32 event, + event_cb_func_t cb_fun) +{ + bool is_callback_found = false; + struct registered_event_data *eve_data; + u64 key = ((u64)node_id << 32U) | (u64)event; + + /* Check for existing entry in hash table for given key id */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, key) { + if (eve_data->key == key && + eve_data->eve_cb == cb_fun) { + is_callback_found = true; + /* remove an object from a hashtable */ + hash_del(&eve_data->hentry); + kfree(eve_data); + } + } + if (!is_callback_found) { + pr_warn("Didn't find any registered callback for 0x%x 0x%x\n", + node_id, event); + return -EINVAL; + } + + return 0; +} + +/** + * xlnx_register_event() - Register for the event. + * @cb_type: Type of callback from pm_api_cb_id, + * PM_NOTIFY_CB - for Error Events, + * PM_INIT_SUSPEND_CB - for suspend callback. + * @node_id: Node-Id related to event. + * @event: Event Mask for the Error Event. + * @wake: Flag specifying whether the subsystem should be woken upon + * event notification. + * @cb_fun: Function pointer to store the callback function. + * @data: Pointer for the driver instance. + * + * Return: Returns 0 on successful registration else error code. + */ +int xlnx_register_event(const enum pm_api_cb_id cb_type, const u32 node_id, const u32 event, + const bool wake, event_cb_func_t cb_fun, void *data) +{ + int ret = 0; + u32 eve; + int pos; + + if (event_manager_availability) + return event_manager_availability; + + if (cb_type != PM_NOTIFY_CB && cb_type != PM_INIT_SUSPEND_CB) { + pr_err("%s() Unsupported Callback 0x%x\n", __func__, cb_type); + return -EINVAL; + } + + if (!cb_fun) + return -EFAULT; + + if (cb_type == PM_INIT_SUSPEND_CB) { + ret = xlnx_add_cb_for_suspend(cb_fun, data); + } else { + if (!xlnx_is_error_event(node_id)) { + /* Add entry for Node-Id/Event in hash table */ + ret = xlnx_add_cb_for_notify_event(node_id, event, wake, cb_fun, data); + } else { + /* Add into Hash table */ + for (pos = 0; pos < MAX_BITS; pos++) { + eve = event & (1 << pos); + if (!eve) + continue; + + /* Add entry for Node-Id/Eve in hash table */ + ret = xlnx_add_cb_for_notify_event(node_id, eve, wake, cb_fun, + data); + /* Break the loop if got error */ + if (ret) + break; + } + if (ret) { + /* Skip the Event for which got the error */ + pos--; + /* Remove registered(during this call) event from hash table */ + for ( ; pos >= 0; pos--) { + eve = event & (1 << pos); + if (!eve) + continue; + xlnx_remove_cb_for_notify_event(node_id, eve, cb_fun); + } + } + } + + if (ret) { + pr_err("%s() failed for 0x%x and 0x%x: %d\r\n", __func__, node_id, + event, ret); + return ret; + } + + /* Register for Node-Id/Event combination in firmware */ + ret = zynqmp_pm_register_notifier(node_id, event, wake, true); + if (ret) { + pr_err("%s() failed for 0x%x and 0x%x: %d\r\n", __func__, node_id, + event, ret); + /* Remove already registered event from hash table */ + if (xlnx_is_error_event(node_id)) { + for (pos = 0; pos < MAX_BITS; pos++) { + eve = event & (1 << pos); + if (!eve) + continue; + xlnx_remove_cb_for_notify_event(node_id, eve, cb_fun); + } + } else { + xlnx_remove_cb_for_notify_event(node_id, event, cb_fun); + } + return ret; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(xlnx_register_event); + +/** + * xlnx_unregister_event() - Unregister for the event. + * @cb_type: Type of callback from pm_api_cb_id, + * PM_NOTIFY_CB - for Error Events, + * PM_INIT_SUSPEND_CB - for suspend callback. + * @node_id: Node-Id related to event. + * @event: Event Mask for the Error Event. + * @cb_fun: Function pointer of callback function. + * + * Return: Returns 0 on successful unregistration else error code. + */ +int xlnx_unregister_event(const enum pm_api_cb_id cb_type, const u32 node_id, const u32 event, + event_cb_func_t cb_fun) +{ + int ret; + u32 eve, pos; + + if (event_manager_availability) + return event_manager_availability; + + if (cb_type != PM_NOTIFY_CB && cb_type != PM_INIT_SUSPEND_CB) { + pr_err("%s() Unsupported Callback 0x%x\n", __func__, cb_type); + return -EINVAL; + } + + if (!cb_fun) + return -EFAULT; + + if (cb_type == PM_INIT_SUSPEND_CB) { + ret = xlnx_remove_cb_for_suspend(cb_fun); + } else { + /* Remove Node-Id/Event from hash table */ + if (!xlnx_is_error_event(node_id)) { + xlnx_remove_cb_for_notify_event(node_id, event, cb_fun); + } else { + for (pos = 0; pos < MAX_BITS; pos++) { + eve = event & (1 << pos); + if (!eve) + continue; + + xlnx_remove_cb_for_notify_event(node_id, eve, cb_fun); + } + } + + /* Un-register for Node-Id/Event combination */ + ret = zynqmp_pm_register_notifier(node_id, event, false, false); + if (ret) { + pr_err("%s() failed for 0x%x and 0x%x: %d\n", + __func__, node_id, event, ret); + return ret; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(xlnx_unregister_event); + +static void xlnx_call_suspend_cb_handler(const u32 *payload) +{ + bool is_callback_found = false; + struct registered_event_data *eve_data; + u32 cb_type = payload[0]; + + /* Check for existing entry in hash table for given cb_type */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, cb_type) { + if (eve_data->cb_type == cb_type) { + eve_data->eve_cb(&payload[0], eve_data->agent_data); + is_callback_found = true; + } + } + if (!is_callback_found) + pr_warn("Didn't find any registered callback for suspend event\n"); +} + +static void xlnx_call_notify_cb_handler(const u32 *payload) +{ + bool is_callback_found = false; + struct registered_event_data *eve_data; + u64 key = ((u64)payload[1] << 32U) | (u64)payload[2]; + int ret; + + /* Check for existing entry in hash table for given key id */ + hash_for_each_possible(reg_driver_map, eve_data, hentry, key) { + if (eve_data->key == key) { + eve_data->eve_cb(&payload[0], eve_data->agent_data); + is_callback_found = true; + + /* re register with firmware to get future events */ + ret = zynqmp_pm_register_notifier(payload[1], payload[2], + eve_data->wake, true); + if (ret) { + pr_err("%s() failed for 0x%x and 0x%x: %d\r\n", __func__, + payload[1], payload[2], ret); + /* Remove already registered event from hash table */ + xlnx_remove_cb_for_notify_event(payload[1], payload[2], + eve_data->eve_cb); + } + } + } + if (!is_callback_found) + pr_warn("Didn't find any registered callback for 0x%x 0x%x\n", + payload[1], payload[2]); +} + +static void xlnx_get_event_callback_data(u32 *buf) +{ + zynqmp_pm_invoke_fn(GET_CALLBACK_DATA, 0, 0, 0, 0, buf); +} + +static irqreturn_t xlnx_event_handler(int irq, void *dev_id) +{ + u32 cb_type, node_id, event, pos; + u32 payload[CB_MAX_PAYLOAD_SIZE] = {0}; + u32 event_data[CB_MAX_PAYLOAD_SIZE] = {0}; + + /* Get event data */ + xlnx_get_event_callback_data(payload); + + /* First element is callback type, others are callback arguments */ + cb_type = payload[0]; + + if (cb_type == PM_NOTIFY_CB) { + node_id = payload[1]; + event = payload[2]; + if (!xlnx_is_error_event(node_id)) { + xlnx_call_notify_cb_handler(payload); + } else { + /* + * Each call back function expecting payload as an input arguments. + * We can get multiple error events as in one call back through error + * mask. So payload[2] may can contain multiple error events. + * In reg_driver_map database we store data in the combination of single + * node_id-error combination. + * So coping the payload message into event_data and update the + * event_data[2] with Error Mask for single error event and use + * event_data as input argument for registered call back function. + * + */ + memcpy(event_data, payload, (4 * CB_MAX_PAYLOAD_SIZE)); + /* Support Multiple Error Event */ + for (pos = 0; pos < MAX_BITS; pos++) { + if ((0 == (event & (1 << pos)))) + continue; + event_data[2] = (event & (1 << pos)); + xlnx_call_notify_cb_handler(event_data); + } + } + } else if (cb_type == PM_INIT_SUSPEND_CB) { + xlnx_call_suspend_cb_handler(payload); + } else { + pr_err("%s() Unsupported Callback %d\n", __func__, cb_type); + } + + return IRQ_HANDLED; +} + +static int xlnx_event_cpuhp_start(unsigned int cpu) +{ + enable_percpu_irq(virq_sgi, IRQ_TYPE_NONE); + + return 0; +} + +static int xlnx_event_cpuhp_down(unsigned int cpu) +{ + disable_percpu_irq(virq_sgi); + + return 0; +} + +static void xlnx_disable_percpu_irq(void *data) +{ + disable_percpu_irq(virq_sgi); +} + +static int xlnx_event_init_sgi(struct platform_device *pdev) +{ + int ret = 0; + int cpu = smp_processor_id(); + /* + * IRQ related structures are used for the following: + * for each SGI interrupt ensure its mapped by GIC IRQ domain + * and that each corresponding linux IRQ for the HW IRQ has + * a handler for when receiving an interrupt from the remote + * processor. + */ + struct irq_domain *domain; + struct irq_fwspec sgi_fwspec; + struct device_node *interrupt_parent = NULL; + struct device *parent = pdev->dev.parent; + + /* Find GIC controller to map SGIs. */ + interrupt_parent = of_irq_find_parent(parent->of_node); + if (!interrupt_parent) { + dev_err(&pdev->dev, "Failed to find property for Interrupt parent\n"); + return -EINVAL; + } + + /* Each SGI needs to be associated with GIC's IRQ domain. */ + domain = irq_find_host(interrupt_parent); + of_node_put(interrupt_parent); + + /* Each mapping needs GIC domain when finding IRQ mapping. */ + sgi_fwspec.fwnode = domain->fwnode; + + /* + * When irq domain looks at mapping each arg is as follows: + * 3 args for: interrupt type (SGI), interrupt # (set later), type + */ + sgi_fwspec.param_count = 1; + + /* Set SGI's hwirq */ + sgi_fwspec.param[0] = sgi_num; + virq_sgi = irq_create_fwspec_mapping(&sgi_fwspec); + + per_cpu(cpu_number1, cpu) = cpu; + ret = request_percpu_irq(virq_sgi, xlnx_event_handler, "xlnx_event_mgmt", + &cpu_number1); + WARN_ON(ret); + if (ret) { + irq_dispose_mapping(virq_sgi); + return ret; + } + + irq_to_desc(virq_sgi); + irq_set_status_flags(virq_sgi, IRQ_PER_CPU); + + return ret; +} + +static void xlnx_event_cleanup_sgi(struct platform_device *pdev) +{ + int cpu = smp_processor_id(); + + per_cpu(cpu_number1, cpu) = cpu; + + cpuhp_remove_state(CPUHP_AP_ONLINE_DYN); + + on_each_cpu(xlnx_disable_percpu_irq, NULL, 1); + + irq_clear_status_flags(virq_sgi, IRQ_PER_CPU); + free_percpu_irq(virq_sgi, &cpu_number1); + irq_dispose_mapping(virq_sgi); +} + +static int xlnx_event_manager_probe(struct platform_device *pdev) +{ + int ret; + + ret = zynqmp_pm_feature(PM_REGISTER_NOTIFIER); + if (ret < 0) { + dev_err(&pdev->dev, "Feature check failed with %d\n", ret); + return ret; + } + + if ((ret & FIRMWARE_VERSION_MASK) < + REGISTER_NOTIFIER_FIRMWARE_VERSION) { + dev_err(&pdev->dev, "Register notifier version error. Expected Firmware: v%d - Found: v%d\n", + REGISTER_NOTIFIER_FIRMWARE_VERSION, + ret & FIRMWARE_VERSION_MASK); + return -EOPNOTSUPP; + } + + /* Initialize the SGI */ + ret = xlnx_event_init_sgi(pdev); + if (ret) { + dev_err(&pdev->dev, "SGI Init has been failed with %d\n", ret); + return ret; + } + + /* Setup function for the CPU hot-plug cases */ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "soc/event:starting", + xlnx_event_cpuhp_start, xlnx_event_cpuhp_down); + + ret = zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_REGISTER_SGI, sgi_num, + 0, NULL); + if (ret) { + dev_err(&pdev->dev, "SGI %d Registration over TF-A failed with %d\n", sgi_num, ret); + xlnx_event_cleanup_sgi(pdev); + return ret; + } + + event_manager_availability = 0; + + dev_info(&pdev->dev, "SGI %d Registered over TF-A\n", sgi_num); + dev_info(&pdev->dev, "Xilinx Event Management driver probed\n"); + + return ret; +} + +static int xlnx_event_manager_remove(struct platform_device *pdev) +{ + int i; + struct registered_event_data *eve_data; + struct hlist_node *tmp; + int ret; + + hash_for_each_safe(reg_driver_map, i, tmp, eve_data, hentry) { + hash_del(&eve_data->hentry); + kfree(eve_data); + } + + ret = zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_REGISTER_SGI, 0, 1, NULL); + if (ret) + dev_err(&pdev->dev, "SGI unregistration over TF-A failed with %d\n", ret); + + xlnx_event_cleanup_sgi(pdev); + + event_manager_availability = -EACCES; + + return ret; +} + +static struct platform_driver xlnx_event_manager_driver = { + .probe = xlnx_event_manager_probe, + .remove = xlnx_event_manager_remove, + .driver = { + .name = "xlnx_event_manager", + }, +}; +module_param(sgi_num, uint, 0); +module_platform_driver(xlnx_event_manager_driver); diff --git a/include/linux/firmware/xlnx-event-manager.h b/include/linux/firmware/xlnx-event-manager.h new file mode 100644 index 000000000000..3f87c4929d21 --- /dev/null +++ b/include/linux/firmware/xlnx-event-manager.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FIRMWARE_XLNX_EVENT_MANAGER_H_ +#define _FIRMWARE_XLNX_EVENT_MANAGER_H_ + +#include + +#define CB_MAX_PAYLOAD_SIZE (4U) /*In payload maximum 32bytes */ + +/************************** Exported Function *****************************/ + +typedef void (*event_cb_func_t)(const u32 *payload, void *data); + +#if IS_REACHABLE(CONFIG_XLNX_EVENT_MANAGER) +int xlnx_register_event(const enum pm_api_cb_id cb_type, const u32 node_id, + const u32 event, const bool wake, + event_cb_func_t cb_fun, void *data); + +int xlnx_unregister_event(const enum pm_api_cb_id cb_type, const u32 node_id, + const u32 event, event_cb_func_t cb_fun); +#else +static inline int xlnx_register_event(const enum pm_api_cb_id cb_type, const u32 node_id, + const u32 event, const bool wake, + event_cb_func_t cb_fun, void *data) +{ + return -ENODEV; +} + +static inline int xlnx_unregister_event(const enum pm_api_cb_id cb_type, const u32 node_id, + const u32 event, event_cb_func_t cb_fun) +{ + return -ENODEV; +} +#endif + +#endif /* _FIRMWARE_XLNX_EVENT_MANAGER_H_ */ diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 077e894bb340..907cb01890cf 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -141,6 +141,8 @@ enum pm_ioctl_id { /* Set healthy bit value */ IOCTL_SET_BOOT_HEALTH_STATUS = 17, IOCTL_OSPI_MUX_SELECT = 21, + /* Register SGI to ATF */ + IOCTL_REGISTER_SGI = 25, }; enum pm_query_id { -- cgit v1.2.3-59-g8ed1b From 14866a7db8da1f61fb6135c461b733694eea9580 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 1 Dec 2021 20:43:03 -0800 Subject: Documentation/auxiliary_bus: Add example code for module_auxiliary_driver() Add an example code snipit to the module_auxiliary_driver() documentation which is consistent with the other example code in the elsewhere in the documentation. Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20211202044305.4006853-6-ira.weiny@intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/auxiliary_bus.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index fc51d45f106b..605b27aab693 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -66,6 +66,10 @@ void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv); * Helper macro for auxiliary drivers which do not do anything special in * module init/exit. This eliminates a lot of boilerplate. Each module may only * use this macro once, and calling it replaces module_init() and module_exit() + * + * .. code-block:: c + * + * module_auxiliary_driver(my_drv); */ #define module_auxiliary_driver(__auxiliary_driver) \ module_driver(__auxiliary_driver, auxiliary_driver_register, auxiliary_driver_unregister) -- cgit v1.2.3-59-g8ed1b From e1b5186810cc7d4ec60447032636b8e6772dbbc6 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 1 Dec 2021 20:43:05 -0800 Subject: Documentation/auxiliary_bus: Move the text into the code The code and documentation are more difficult to maintain when kept separately. This is further compounded when the standard structure documentation infrastructure is not used. Move the documentation into the code, use the standard documentation infrastructure, add current documented functions, and reference the text in the rst file. Suggested-by: Greg Kroah-Hartman Signed-off-by: Ira Weiny Link: https://lore.kernel.org/r/20211202044305.4006853-8-ira.weiny@intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/auxiliary_bus.rst | 298 ++--------------------------- drivers/base/auxiliary.c | 141 ++++++++++++++ include/linux/auxiliary_bus.h | 160 ++++++++++++++++ 3 files changed, 318 insertions(+), 281 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/auxiliary_bus.rst b/Documentation/driver-api/auxiliary_bus.rst index 3786e4664a1e..cec84908fbc0 100644 --- a/Documentation/driver-api/auxiliary_bus.rst +++ b/Documentation/driver-api/auxiliary_bus.rst @@ -6,309 +6,45 @@ Auxiliary Bus ============= -In some subsystems, the functionality of the core device (PCI/ACPI/other) is -too complex for a single device to be managed by a monolithic driver -(e.g. Sound Open Firmware), multiple devices might implement a common -intersection of functionality (e.g. NICs + RDMA), or a driver may want to -export an interface for another subsystem to drive (e.g. SIOV Physical Function -export Virtual Function management). A split of the functionality into child- -devices representing sub-domains of functionality makes it possible to -compartmentalize, layer, and distribute domain-specific concerns via a Linux -device-driver model. - -An example for this kind of requirement is the audio subsystem where a single -IP is handling multiple entities such as HDMI, Soundwire, local devices such as -mics/speakers etc. The split for the core's functionality can be arbitrary or -be defined by the DSP firmware topology and include hooks for test/debug. This -allows for the audio core device to be minimal and focused on hardware-specific -control and communication. - -Each auxiliary_device represents a part of its parent functionality. The -generic behavior can be extended and specialized as needed by encapsulating an -auxiliary_device within other domain-specific structures and the use of .ops -callbacks. Devices on the auxiliary bus do not share any structures and the use -of a communication channel with the parent is domain-specific. - -Note that ops are intended as a way to augment instance behavior within a class -of auxiliary devices, it is not the mechanism for exporting common -infrastructure from the parent. Consider EXPORT_SYMBOL_NS() to convey -infrastructure from the parent module to the auxiliary module(s). - +.. kernel-doc:: drivers/base/auxiliary.c + :doc: PURPOSE When Should the Auxiliary Bus Be Used ===================================== -The auxiliary bus is to be used when a driver and one or more kernel modules, -who share a common header file with the driver, need a mechanism to connect and -provide access to a shared object allocated by the auxiliary_device's -registering driver. The registering driver for the auxiliary_device(s) and the -kernel module(s) registering auxiliary_drivers can be from the same subsystem, -or from multiple subsystems. - -The emphasis here is on a common generic interface that keeps subsystem -customization out of the bus infrastructure. - -One example is a PCI network device that is RDMA-capable and exports a child -device to be driven by an auxiliary_driver in the RDMA subsystem. The PCI -driver allocates and registers an auxiliary_device for each physical -function on the NIC. The RDMA driver registers an auxiliary_driver that claims -each of these auxiliary_devices. This conveys data/ops published by the parent -PCI device/driver to the RDMA auxiliary_driver. - -Another use case is for the PCI device to be split out into multiple sub -functions. For each sub function an auxiliary_device is created. A PCI sub -function driver binds to such devices that creates its own one or more class -devices. A PCI sub function auxiliary device is likely to be contained in a -struct with additional attributes such as user defined sub function number and -optional attributes such as resources and a link to the parent device. These -attributes could be used by systemd/udev; and hence should be initialized -before a driver binds to an auxiliary_device. +.. kernel-doc:: drivers/base/auxiliary.c + :doc: USAGE -A key requirement for utilizing the auxiliary bus is that there is no -dependency on a physical bus, device, register accesses or regmap support. -These individual devices split from the core cannot live on the platform bus as -they are not physical devices that are controlled by DT/ACPI. The same -argument applies for not using MFD in this scenario as MFD relies on individual -function devices being physical devices. Auxiliary Device Creation ========================= -An auxiliary_device represents a part of its parent device's functionality. It -is given a name that, combined with the registering drivers KBUILD_MODNAME, -creates a match_name that is used for driver binding, and an id that combined -with the match_name provide a unique name to register with the bus subsystem. -For example, a driver registering an auxiliary device is named 'foo_mod.ko' and -the subdevice is named 'foo_dev'. The match name is therefore -'foo_mod.foo_dev'. - -.. code-block:: c - - struct auxiliary_device { - struct device dev; - const char *name; - u32 id; - }; - -Registering an auxiliary_device is a three-step process. - -First, a 'struct auxiliary_device' needs to be defined or allocated for each -sub-device desired. The name, id, dev.release, and dev.parent fields of this -structure must be filled in as follows. - -The 'name' field is to be given a name that is recognized by the auxiliary -driver. If two auxiliary_devices with the same match_name, eg -"foo_mod.foo_dev", are registered onto the bus, they must have unique id -values (e.g. "x" and "y") so that the registered devices names are "foo_mod.foo_dev.x" -and "foo_mod.foo_dev.y". If match_name + id are not unique, then the device_add fails -and generates an error message. - -The auxiliary_device.dev.type.release or auxiliary_device.dev.release must be -populated with a non-NULL pointer to successfully register the -auxiliary_device. This release call is where resources associated with the -auxiliary device must be free'ed. Because once the device is placed on the bus -the parent driver can not tell what other code may have a reference to this -data. - -The auxiliary_device.dev.parent should be set. Typically to the registering -drivers device. - -Second, call auxiliary_device_init(), which checks several aspects of the -auxiliary_device struct and performs a device_initialize(). After this step -completes, any error state must have a call to auxiliary_device_uninit() in its -resolution path. - -The third and final step in registering an auxiliary_device is to perform a -call to auxiliary_device_add(), which sets the name of the device and adds the -device to the bus. - -.. code-block:: c - - #define MY_DEVICE_NAME "foo_dev" - - ... - - struct auxiliary_device *my_aux_dev = my_aux_dev_alloc(xxx); - - /* Step 1: */ - my_aux_dev->name = MY_DEVICE_NAME; - my_aux_dev->id = my_unique_id_alloc(xxx); - my_aux_dev->dev.release = my_aux_dev_release; - my_aux_dev->dev.parent = my_dev; - - /* Step 2: */ - if (auxiliary_device_init(my_aux_dev)) - goto fail; - - /* Step 3: */ - if (auxiliary_device_add(my_aux_dev)) { - auxiliary_device_uninit(my_aux_dev); - goto fail; - } - - ... - - -Unregistering an auxiliary_device is a two-step process to mirror the register -process. First call auxiliary_device_delete(), then call -auxiliary_device_uninit(). - - -.. code-block:: c - - auxiliary_device_delete(my_dev->my_aux_dev); - auxiliary_device_uninit(my_dev->my_aux_dev); +.. kernel-doc:: include/linux/auxiliary_bus.h + :identifiers: auxiliary_device +.. kernel-doc:: drivers/base/auxiliary.c + :identifiers: auxiliary_device_init __auxiliary_device_add + auxiliary_find_device Auxiliary Device Memory Model and Lifespan ------------------------------------------ -The registering driver is the entity that allocates memory for the -auxiliary_device and registers it on the auxiliary bus. It is important to note -that, as opposed to the platform bus, the registering driver is wholly -responsible for the management of the memory used for the device object. - -To be clear the memory for the auxiliary_device is freed in the release() -callback defined by the registering driver. The registering driver should only -call auxiliary_device_delete() and then auxiliary_device_uninit() when it is -done with the device. The release() function is then automatically called if -and when other code releases their reference to the devices. - -A parent object, defined in the shared header file, contains the -auxiliary_device. It also contains a pointer to the shared object(s), which -also is defined in the shared header. Both the parent object and the shared -object(s) are allocated by the registering driver. This layout allows the -auxiliary_driver's registering module to perform a container_of() call to go -from the pointer to the auxiliary_device, that is passed during the call to the -auxiliary_driver's probe function, up to the parent object, and then have -access to the shared object(s). - -The memory for the shared object(s) must have a lifespan equal to, or greater -than, the lifespan of the memory for the auxiliary_device. The -auxiliary_driver should only consider that the shared object is valid as long -as the auxiliary_device is still registered on the auxiliary bus. It is up to -the registering driver to manage (e.g. free or keep available) the memory for -the shared object beyond the life of the auxiliary_device. - -The registering driver must unregister all auxiliary devices before its own -driver.remove() is completed. An easy way to ensure this is to use the -devm_add_action_or_reset() call to register a function against the parent device -which unregisters the auxiliary device object(s). - -Finally, any operations which operate on the auxiliary devices must continue to -function (if only to return an error) after the registering driver unregisters -the auxiliary device. +.. kernel-doc:: include/linux/auxiliary_bus.h + :doc: DEVICE_LIFESPAN Auxiliary Drivers ================= -Auxiliary drivers follow the standard driver model convention, where -discovery/enumeration is handled by the core, and drivers -provide probe() and remove() methods. They support power management -and shutdown notifications using the standard conventions. - -.. code-block:: c - - struct auxiliary_driver { - int (*probe)(struct auxiliary_device *, - const struct auxiliary_device_id *id); - void (*remove)(struct auxiliary_device *); - void (*shutdown)(struct auxiliary_device *); - int (*suspend)(struct auxiliary_device *, pm_message_t); - int (*resume)(struct auxiliary_device *); - struct device_driver driver; - const struct auxiliary_device_id *id_table; - }; - -Auxiliary drivers register themselves with the bus by calling -auxiliary_driver_register(). The id_table contains the match_names of auxiliary -devices that a driver can bind with. - -.. code-block:: c - - static const struct auxiliary_device_id my_auxiliary_id_table[] = { - { .name = "foo_mod.foo_dev" }, - {}, - }; - - MODULE_DEVICE_TABLE(auxiliary, my_auxiliary_id_table); - - struct auxiliary_driver my_drv = { - .name = "myauxiliarydrv", - .id_table = my_auxiliary_id_table, - .probe = my_drv_probe, - .remove = my_drv_remove - }; +.. kernel-doc:: include/linux/auxiliary_bus.h + :identifiers: auxiliary_driver module_auxiliary_driver +.. kernel-doc:: drivers/base/auxiliary.c + :identifiers: __auxiliary_driver_register auxiliary_driver_unregister Example Usage ============= -Auxiliary devices are created and registered by a subsystem-level core device -that needs to break up its functionality into smaller fragments. One way to -extend the scope of an auxiliary_device is to encapsulate it within a domain- -pecific structure defined by the parent device. This structure contains the -auxiliary_device and any associated shared data/callbacks needed to establish -the connection with the parent. - -An example is: - -.. code-block:: c - - struct foo { - struct auxiliary_device auxdev; - void (*connect)(struct auxiliary_device *auxdev); - void (*disconnect)(struct auxiliary_device *auxdev); - void *data; - }; - -The parent device then registers the auxiliary_device by calling -auxiliary_device_init(), and then auxiliary_device_add(), with the pointer to -the auxdev member of the above structure. The parent provides a name for the -auxiliary_device that, combined with the parent's KBUILD_MODNAME, creates a -match_name that is be used for matching and binding with a driver. - -Whenever an auxiliary_driver is registered, based on the match_name, the -auxiliary_driver's probe() is invoked for the matching devices. The -auxiliary_driver can also be encapsulated inside custom drivers that make the -core device's functionality extensible by adding additional domain-specific ops -as follows: - -.. code-block:: c - - struct my_ops { - void (*send)(struct auxiliary_device *auxdev); - void (*receive)(struct auxiliary_device *auxdev); - }; - - - struct my_driver { - struct auxiliary_driver auxiliary_drv; - const struct my_ops ops; - }; - -An example of this type of usage is: - -.. code-block:: c - - const struct auxiliary_device_id my_auxiliary_id_table[] = { - { .name = "foo_mod.foo_dev" }, - { }, - }; - - const struct my_ops my_custom_ops = { - .send = my_tx, - .receive = my_rx, - }; +.. kernel-doc:: drivers/base/auxiliary.c + :doc: EXAMPLE - const struct my_driver my_drv = { - .auxiliary_drv = { - .name = "myauxiliarydrv", - .id_table = my_auxiliary_id_table, - .probe = my_probe, - .remove = my_remove, - .shutdown = my_shutdown, - }, - .ops = my_custom_ops, - }; diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index ab5315681a42..8c5e65930617 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -17,6 +17,147 @@ #include #include "base.h" +/** + * DOC: PURPOSE + * + * In some subsystems, the functionality of the core device (PCI/ACPI/other) is + * too complex for a single device to be managed by a monolithic driver (e.g. + * Sound Open Firmware), multiple devices might implement a common intersection + * of functionality (e.g. NICs + RDMA), or a driver may want to export an + * interface for another subsystem to drive (e.g. SIOV Physical Function export + * Virtual Function management). A split of the functionality into child- + * devices representing sub-domains of functionality makes it possible to + * compartmentalize, layer, and distribute domain-specific concerns via a Linux + * device-driver model. + * + * An example for this kind of requirement is the audio subsystem where a + * single IP is handling multiple entities such as HDMI, Soundwire, local + * devices such as mics/speakers etc. The split for the core's functionality + * can be arbitrary or be defined by the DSP firmware topology and include + * hooks for test/debug. This allows for the audio core device to be minimal + * and focused on hardware-specific control and communication. + * + * Each auxiliary_device represents a part of its parent functionality. The + * generic behavior can be extended and specialized as needed by encapsulating + * an auxiliary_device within other domain-specific structures and the use of + * .ops callbacks. Devices on the auxiliary bus do not share any structures and + * the use of a communication channel with the parent is domain-specific. + * + * Note that ops are intended as a way to augment instance behavior within a + * class of auxiliary devices, it is not the mechanism for exporting common + * infrastructure from the parent. Consider EXPORT_SYMBOL_NS() to convey + * infrastructure from the parent module to the auxiliary module(s). + */ + +/** + * DOC: USAGE + * + * The auxiliary bus is to be used when a driver and one or more kernel + * modules, who share a common header file with the driver, need a mechanism to + * connect and provide access to a shared object allocated by the + * auxiliary_device's registering driver. The registering driver for the + * auxiliary_device(s) and the kernel module(s) registering auxiliary_drivers + * can be from the same subsystem, or from multiple subsystems. + * + * The emphasis here is on a common generic interface that keeps subsystem + * customization out of the bus infrastructure. + * + * One example is a PCI network device that is RDMA-capable and exports a child + * device to be driven by an auxiliary_driver in the RDMA subsystem. The PCI + * driver allocates and registers an auxiliary_device for each physical + * function on the NIC. The RDMA driver registers an auxiliary_driver that + * claims each of these auxiliary_devices. This conveys data/ops published by + * the parent PCI device/driver to the RDMA auxiliary_driver. + * + * Another use case is for the PCI device to be split out into multiple sub + * functions. For each sub function an auxiliary_device is created. A PCI sub + * function driver binds to such devices that creates its own one or more class + * devices. A PCI sub function auxiliary device is likely to be contained in a + * struct with additional attributes such as user defined sub function number + * and optional attributes such as resources and a link to the parent device. + * These attributes could be used by systemd/udev; and hence should be + * initialized before a driver binds to an auxiliary_device. + * + * A key requirement for utilizing the auxiliary bus is that there is no + * dependency on a physical bus, device, register accesses or regmap support. + * These individual devices split from the core cannot live on the platform bus + * as they are not physical devices that are controlled by DT/ACPI. The same + * argument applies for not using MFD in this scenario as MFD relies on + * individual function devices being physical devices. + */ + +/** + * DOC: EXAMPLE + * + * Auxiliary devices are created and registered by a subsystem-level core + * device that needs to break up its functionality into smaller fragments. One + * way to extend the scope of an auxiliary_device is to encapsulate it within a + * domain- pecific structure defined by the parent device. This structure + * contains the auxiliary_device and any associated shared data/callbacks + * needed to establish the connection with the parent. + * + * An example is: + * + * .. code-block:: c + * + * struct foo { + * struct auxiliary_device auxdev; + * void (*connect)(struct auxiliary_device *auxdev); + * void (*disconnect)(struct auxiliary_device *auxdev); + * void *data; + * }; + * + * The parent device then registers the auxiliary_device by calling + * auxiliary_device_init(), and then auxiliary_device_add(), with the pointer + * to the auxdev member of the above structure. The parent provides a name for + * the auxiliary_device that, combined with the parent's KBUILD_MODNAME, + * creates a match_name that is be used for matching and binding with a driver. + * + * Whenever an auxiliary_driver is registered, based on the match_name, the + * auxiliary_driver's probe() is invoked for the matching devices. The + * auxiliary_driver can also be encapsulated inside custom drivers that make + * the core device's functionality extensible by adding additional + * domain-specific ops as follows: + * + * .. code-block:: c + * + * struct my_ops { + * void (*send)(struct auxiliary_device *auxdev); + * void (*receive)(struct auxiliary_device *auxdev); + * }; + * + * + * struct my_driver { + * struct auxiliary_driver auxiliary_drv; + * const struct my_ops ops; + * }; + * + * An example of this type of usage is: + * + * .. code-block:: c + * + * const struct auxiliary_device_id my_auxiliary_id_table[] = { + * { .name = "foo_mod.foo_dev" }, + * { }, + * }; + * + * const struct my_ops my_custom_ops = { + * .send = my_tx, + * .receive = my_rx, + * }; + * + * const struct my_driver my_drv = { + * .auxiliary_drv = { + * .name = "myauxiliarydrv", + * .id_table = my_auxiliary_id_table, + * .probe = my_probe, + * .remove = my_remove, + * .shutdown = my_shutdown, + * }, + * .ops = my_custom_ops, + * }; + */ + static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, const struct auxiliary_device *auxdev) { diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index 605b27aab693..e6d8b5c16226 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -11,12 +11,172 @@ #include #include +/** + * DOC: DEVICE_LIFESPAN + * + * The registering driver is the entity that allocates memory for the + * auxiliary_device and registers it on the auxiliary bus. It is important to + * note that, as opposed to the platform bus, the registering driver is wholly + * responsible for the management of the memory used for the device object. + * + * To be clear the memory for the auxiliary_device is freed in the release() + * callback defined by the registering driver. The registering driver should + * only call auxiliary_device_delete() and then auxiliary_device_uninit() when + * it is done with the device. The release() function is then automatically + * called if and when other code releases their reference to the devices. + * + * A parent object, defined in the shared header file, contains the + * auxiliary_device. It also contains a pointer to the shared object(s), which + * also is defined in the shared header. Both the parent object and the shared + * object(s) are allocated by the registering driver. This layout allows the + * auxiliary_driver's registering module to perform a container_of() call to go + * from the pointer to the auxiliary_device, that is passed during the call to + * the auxiliary_driver's probe function, up to the parent object, and then + * have access to the shared object(s). + * + * The memory for the shared object(s) must have a lifespan equal to, or + * greater than, the lifespan of the memory for the auxiliary_device. The + * auxiliary_driver should only consider that the shared object is valid as + * long as the auxiliary_device is still registered on the auxiliary bus. It + * is up to the registering driver to manage (e.g. free or keep available) the + * memory for the shared object beyond the life of the auxiliary_device. + * + * The registering driver must unregister all auxiliary devices before its own + * driver.remove() is completed. An easy way to ensure this is to use the + * devm_add_action_or_reset() call to register a function against the parent + * device which unregisters the auxiliary device object(s). + * + * Finally, any operations which operate on the auxiliary devices must continue + * to function (if only to return an error) after the registering driver + * unregisters the auxiliary device. + */ + +/** + * struct auxiliary_device - auxiliary device object. + * @dev: Device, + * The release and parent fields of the device structure must be filled + * in + * @name: Match name found by the auxiliary device driver, + * @id: unique identitier if multiple devices of the same name are exported, + * + * An auxiliary_device represents a part of its parent device's functionality. + * It is given a name that, combined with the registering drivers + * KBUILD_MODNAME, creates a match_name that is used for driver binding, and an + * id that combined with the match_name provide a unique name to register with + * the bus subsystem. For example, a driver registering an auxiliary device is + * named 'foo_mod.ko' and the subdevice is named 'foo_dev'. The match name is + * therefore 'foo_mod.foo_dev'. + * + * Registering an auxiliary_device is a three-step process. + * + * First, a 'struct auxiliary_device' needs to be defined or allocated for each + * sub-device desired. The name, id, dev.release, and dev.parent fields of + * this structure must be filled in as follows. + * + * The 'name' field is to be given a name that is recognized by the auxiliary + * driver. If two auxiliary_devices with the same match_name, eg + * "foo_mod.foo_dev", are registered onto the bus, they must have unique id + * values (e.g. "x" and "y") so that the registered devices names are + * "foo_mod.foo_dev.x" and "foo_mod.foo_dev.y". If match_name + id are not + * unique, then the device_add fails and generates an error message. + * + * The auxiliary_device.dev.type.release or auxiliary_device.dev.release must + * be populated with a non-NULL pointer to successfully register the + * auxiliary_device. This release call is where resources associated with the + * auxiliary device must be free'ed. Because once the device is placed on the + * bus the parent driver can not tell what other code may have a reference to + * this data. + * + * The auxiliary_device.dev.parent should be set. Typically to the registering + * drivers device. + * + * Second, call auxiliary_device_init(), which checks several aspects of the + * auxiliary_device struct and performs a device_initialize(). After this step + * completes, any error state must have a call to auxiliary_device_uninit() in + * its resolution path. + * + * The third and final step in registering an auxiliary_device is to perform a + * call to auxiliary_device_add(), which sets the name of the device and adds + * the device to the bus. + * + * .. code-block:: c + * + * #define MY_DEVICE_NAME "foo_dev" + * + * ... + * + * struct auxiliary_device *my_aux_dev = my_aux_dev_alloc(xxx); + * + * // Step 1: + * my_aux_dev->name = MY_DEVICE_NAME; + * my_aux_dev->id = my_unique_id_alloc(xxx); + * my_aux_dev->dev.release = my_aux_dev_release; + * my_aux_dev->dev.parent = my_dev; + * + * // Step 2: + * if (auxiliary_device_init(my_aux_dev)) + * goto fail; + * + * // Step 3: + * if (auxiliary_device_add(my_aux_dev)) { + * auxiliary_device_uninit(my_aux_dev); + * goto fail; + * } + * + * ... + * + * + * Unregistering an auxiliary_device is a two-step process to mirror the + * register process. First call auxiliary_device_delete(), then call + * auxiliary_device_uninit(). + * + * .. code-block:: c + * + * auxiliary_device_delete(my_dev->my_aux_dev); + * auxiliary_device_uninit(my_dev->my_aux_dev); + */ struct auxiliary_device { struct device dev; const char *name; u32 id; }; +/** + * struct auxiliary_driver - Definition of an auxiliary bus driver + * @probe: Called when a matching device is added to the bus. + * @remove: Called when device is removed from the bus. + * @shutdown: Called at shut-down time to quiesce the device. + * @suspend: Called to put the device to sleep mode. Usually to a power state. + * @resume: Called to bring a device from sleep mode. + * @name: Driver name. + * @driver: Core driver structure. + * @id_table: Table of devices this driver should match on the bus. + * + * Auxiliary drivers follow the standard driver model convention, where + * discovery/enumeration is handled by the core, and drivers provide probe() + * and remove() methods. They support power management and shutdown + * notifications using the standard conventions. + * + * Auxiliary drivers register themselves with the bus by calling + * auxiliary_driver_register(). The id_table contains the match_names of + * auxiliary devices that a driver can bind with. + * + * .. code-block:: c + * + * static const struct auxiliary_device_id my_auxiliary_id_table[] = { + * { .name = "foo_mod.foo_dev" }, + * {}, + * }; + * + * MODULE_DEVICE_TABLE(auxiliary, my_auxiliary_id_table); + * + * struct auxiliary_driver my_drv = { + * .name = "myauxiliarydrv", + * .id_table = my_auxiliary_id_table, + * .probe = my_drv_probe, + * .remove = my_drv_remove + * }; + */ struct auxiliary_driver { int (*probe)(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id); void (*remove)(struct auxiliary_device *auxdev); -- cgit v1.2.3-59-g8ed1b From bb49e9e730c2906a958eee273a7819f401543d6c Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:16:58 +0100 Subject: fs: add is_idmapped_mnt() helper Multiple places open-code the same check to determine whether a given mount is idmapped. Introduce a simple helper function that can be used instead. This allows us to get rid of the fragile open-coding. We will later change the check that is used to determine whether a given mount is idmapped. Introducing a helper allows us to do this in a single place instead of doing it for multiple places. Link: https://lore.kernel.org/r/20211123114227.3124056-2-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-2-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-2-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- fs/cachefiles/bind.c | 2 +- fs/ecryptfs/main.c | 2 +- fs/namespace.c | 2 +- fs/nfsd/export.c | 2 +- fs/overlayfs/super.c | 2 +- fs/proc_namespace.c | 2 +- include/linux/fs.h | 14 ++++++++++++++ 7 files changed, 20 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/cachefiles/bind.c b/fs/cachefiles/bind.c index d463d89f5db8..146291be6263 100644 --- a/fs/cachefiles/bind.c +++ b/fs/cachefiles/bind.c @@ -117,7 +117,7 @@ static int cachefiles_daemon_add_cache(struct cachefiles_cache *cache) root = path.dentry; ret = -EINVAL; - if (mnt_user_ns(path.mnt) != &init_user_ns) { + if (is_idmapped_mnt(path.mnt)) { pr_warn("File cache on idmapped mounts not supported"); goto error_unsupported; } diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index d66bbd2df191..2dd23a82e0de 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -537,7 +537,7 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags goto out_free; } - if (mnt_user_ns(path.mnt) != &init_user_ns) { + if (is_idmapped_mnt(path.mnt)) { rc = -EINVAL; printk(KERN_ERR "Mounting on idmapped mounts currently disallowed\n"); goto out_free; diff --git a/fs/namespace.c b/fs/namespace.c index 659a8f39c61a..4994b816a74c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -3936,7 +3936,7 @@ static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) * mapping. It makes things simpler and callers can just create * another bind-mount they can idmap if they want to. */ - if (mnt_user_ns(m) != &init_user_ns) + if (is_idmapped_mnt(m)) return -EPERM; /* The underlying filesystem doesn't support idmapped mounts yet. */ diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 9421dae22737..668c7527b17e 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -427,7 +427,7 @@ static int check_export(struct path *path, int *flags, unsigned char *uuid) return -EINVAL; } - if (mnt_user_ns(path->mnt) != &init_user_ns) { + if (is_idmapped_mnt(path->mnt)) { dprintk("exp_export: export of idmapped mounts not yet supported.\n"); return -EINVAL; } diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 265181c110ae..7bb0a47cb615 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -873,7 +873,7 @@ static int ovl_mount_dir_noesc(const char *name, struct path *path) pr_err("filesystem on '%s' not supported\n", name); goto out_put; } - if (mnt_user_ns(path->mnt) != &init_user_ns) { + if (is_idmapped_mnt(path->mnt)) { pr_err("idmapped layers are currently not supported\n"); goto out_put; } diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c index 392ef5162655..49650e54d2f8 100644 --- a/fs/proc_namespace.c +++ b/fs/proc_namespace.c @@ -80,7 +80,7 @@ static void show_mnt_opts(struct seq_file *m, struct vfsmount *mnt) seq_puts(m, fs_infop->str); } - if (mnt_user_ns(mnt) != &init_user_ns) + if (is_idmapped_mnt(mnt)) seq_puts(m, ",idmapped"); } diff --git a/include/linux/fs.h b/include/linux/fs.h index bbf812ce89a8..06cbefd76de7 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2724,6 +2724,20 @@ static inline struct user_namespace *file_mnt_user_ns(struct file *file) { return mnt_user_ns(file->f_path.mnt); } + +/** + * is_idmapped_mnt - check whether a mount is mapped + * @mnt: the mount to check + * + * If @mnt has an idmapping attached to it @mnt is mapped. + * + * Return: true if mount is mapped, false if not. + */ +static inline bool is_idmapped_mnt(const struct vfsmount *mnt) +{ + return mnt_user_ns(mnt) != &init_user_ns; +} + extern long vfs_truncate(const struct path *, loff_t); int do_truncate(struct user_namespace *, struct dentry *, loff_t start, unsigned int time_attrs, struct file *filp); -- cgit v1.2.3-59-g8ed1b From a793d79ea3e041081cd7cbd8ee43d0b5e4914a2b Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:16:59 +0100 Subject: fs: move mapping helpers The low-level mapping helpers were so far crammed into fs.h. They are out of place there. The fs.h header should just contain the higher-level mapping helpers that interact directly with vfs objects such as struct super_block or struct inode and not the bare mapping helpers. Similarly, only vfs and specific fs code shall interact with low-level mapping helpers. And so they won't be made accessible automatically through regular {g,u}id helpers. Link: https://lore.kernel.org/r/20211123114227.3124056-3-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-3-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-3-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- fs/ksmbd/smbacl.c | 1 + fs/ksmbd/smbacl.h | 1 + fs/open.c | 1 + fs/posix_acl.c | 1 + fs/xfs/xfs_linux.h | 1 + include/linux/fs.h | 91 +------------------------------------ include/linux/mnt_idmapping.h | 101 ++++++++++++++++++++++++++++++++++++++++++ security/commoncap.c | 1 + 8 files changed, 108 insertions(+), 90 deletions(-) create mode 100644 include/linux/mnt_idmapping.h (limited to 'include/linux') diff --git a/fs/ksmbd/smbacl.c b/fs/ksmbd/smbacl.c index bd792db32623..ab8099e0fd7f 100644 --- a/fs/ksmbd/smbacl.c +++ b/fs/ksmbd/smbacl.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "smbacl.h" #include "smb_common.h" diff --git a/fs/ksmbd/smbacl.h b/fs/ksmbd/smbacl.h index 73e08cad412b..eba1ebb9e92e 100644 --- a/fs/ksmbd/smbacl.h +++ b/fs/ksmbd/smbacl.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "mgmt/tree_connect.h" diff --git a/fs/open.c b/fs/open.c index f732fb94600c..2450cc1a2f64 100644 --- a/fs/open.c +++ b/fs/open.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "internal.h" diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 9323a854a60a..632bfdcf7cc0 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -23,6 +23,7 @@ #include #include #include +#include static struct posix_acl **acl_by_type(struct inode *inode, int type) { diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h index c174262a074e..09a8fba84ff9 100644 --- a/fs/xfs/xfs_linux.h +++ b/fs/xfs/xfs_linux.h @@ -61,6 +61,7 @@ typedef __u32 xfs_nlink_t; #include #include #include +#include #include #include diff --git a/include/linux/fs.h b/include/linux/fs.h index 06cbefd76de7..b3bcb2129699 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -1624,34 +1625,6 @@ static inline void i_gid_write(struct inode *inode, gid_t gid) inode->i_gid = make_kgid(inode->i_sb->s_user_ns, gid); } -/** - * kuid_into_mnt - map a kuid down into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kuid: kuid to be mapped - * - * Return: @kuid mapped according to @mnt_userns. - * If @kuid has no mapping INVALID_UID is returned. - */ -static inline kuid_t kuid_into_mnt(struct user_namespace *mnt_userns, - kuid_t kuid) -{ - return make_kuid(mnt_userns, __kuid_val(kuid)); -} - -/** - * kgid_into_mnt - map a kgid down into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kgid: kgid to be mapped - * - * Return: @kgid mapped according to @mnt_userns. - * If @kgid has no mapping INVALID_GID is returned. - */ -static inline kgid_t kgid_into_mnt(struct user_namespace *mnt_userns, - kgid_t kgid) -{ - return make_kgid(mnt_userns, __kgid_val(kgid)); -} - /** * i_uid_into_mnt - map an inode's i_uid down into a mnt_userns * @mnt_userns: user namespace of the mount the inode was found from @@ -1680,68 +1653,6 @@ static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns, return kgid_into_mnt(mnt_userns, inode->i_gid); } -/** - * kuid_from_mnt - map a kuid up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kuid: kuid to be mapped - * - * Return: @kuid mapped up according to @mnt_userns. - * If @kuid has no mapping INVALID_UID is returned. - */ -static inline kuid_t kuid_from_mnt(struct user_namespace *mnt_userns, - kuid_t kuid) -{ - return KUIDT_INIT(from_kuid(mnt_userns, kuid)); -} - -/** - * kgid_from_mnt - map a kgid up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kgid: kgid to be mapped - * - * Return: @kgid mapped up according to @mnt_userns. - * If @kgid has no mapping INVALID_GID is returned. - */ -static inline kgid_t kgid_from_mnt(struct user_namespace *mnt_userns, - kgid_t kgid) -{ - return KGIDT_INIT(from_kgid(mnt_userns, kgid)); -} - -/** - * mapped_fsuid - return caller's fsuid mapped up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * - * Use this helper to initialize a new vfs or filesystem object based on - * the caller's fsuid. A common example is initializing the i_uid field of - * a newly allocated inode triggered by a creation event such as mkdir or - * O_CREAT. Other examples include the allocation of quotas for a specific - * user. - * - * Return: the caller's current fsuid mapped up according to @mnt_userns. - */ -static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) -{ - return kuid_from_mnt(mnt_userns, current_fsuid()); -} - -/** - * mapped_fsgid - return caller's fsgid mapped up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * - * Use this helper to initialize a new vfs or filesystem object based on - * the caller's fsgid. A common example is initializing the i_gid field of - * a newly allocated inode triggered by a creation event such as mkdir or - * O_CREAT. Other examples include the allocation of quotas for a specific - * user. - * - * Return: the caller's current fsgid mapped up according to @mnt_userns. - */ -static inline kgid_t mapped_fsgid(struct user_namespace *mnt_userns) -{ - return kgid_from_mnt(mnt_userns, current_fsgid()); -} - /** * inode_fsuid_set - initialize inode's i_uid field with callers fsuid * @inode: inode to initialize diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h new file mode 100644 index 000000000000..47c7811fadfe --- /dev/null +++ b/include/linux/mnt_idmapping.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_MNT_IDMAPPING_H +#define _LINUX_MNT_IDMAPPING_H + +#include +#include + +struct user_namespace; +extern struct user_namespace init_user_ns; + +/** + * kuid_into_mnt - map a kuid down into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * @kuid: kuid to be mapped + * + * Return: @kuid mapped according to @mnt_userns. + * If @kuid has no mapping INVALID_UID is returned. + */ +static inline kuid_t kuid_into_mnt(struct user_namespace *mnt_userns, + kuid_t kuid) +{ + return make_kuid(mnt_userns, __kuid_val(kuid)); +} + +/** + * kgid_into_mnt - map a kgid down into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * @kgid: kgid to be mapped + * + * Return: @kgid mapped according to @mnt_userns. + * If @kgid has no mapping INVALID_GID is returned. + */ +static inline kgid_t kgid_into_mnt(struct user_namespace *mnt_userns, + kgid_t kgid) +{ + return make_kgid(mnt_userns, __kgid_val(kgid)); +} + +/** + * kuid_from_mnt - map a kuid up into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * @kuid: kuid to be mapped + * + * Return: @kuid mapped up according to @mnt_userns. + * If @kuid has no mapping INVALID_UID is returned. + */ +static inline kuid_t kuid_from_mnt(struct user_namespace *mnt_userns, + kuid_t kuid) +{ + return KUIDT_INIT(from_kuid(mnt_userns, kuid)); +} + +/** + * kgid_from_mnt - map a kgid up into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * @kgid: kgid to be mapped + * + * Return: @kgid mapped up according to @mnt_userns. + * If @kgid has no mapping INVALID_GID is returned. + */ +static inline kgid_t kgid_from_mnt(struct user_namespace *mnt_userns, + kgid_t kgid) +{ + return KGIDT_INIT(from_kgid(mnt_userns, kgid)); +} + +/** + * mapped_fsuid - return caller's fsuid mapped up into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * + * Use this helper to initialize a new vfs or filesystem object based on + * the caller's fsuid. A common example is initializing the i_uid field of + * a newly allocated inode triggered by a creation event such as mkdir or + * O_CREAT. Other examples include the allocation of quotas for a specific + * user. + * + * Return: the caller's current fsuid mapped up according to @mnt_userns. + */ +static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) +{ + return kuid_from_mnt(mnt_userns, current_fsuid()); +} + +/** + * mapped_fsgid - return caller's fsgid mapped up into a mnt_userns + * @mnt_userns: user namespace of the relevant mount + * + * Use this helper to initialize a new vfs or filesystem object based on + * the caller's fsgid. A common example is initializing the i_gid field of + * a newly allocated inode triggered by a creation event such as mkdir or + * O_CREAT. Other examples include the allocation of quotas for a specific + * user. + * + * Return: the caller's current fsgid mapped up according to @mnt_userns. + */ +static inline kgid_t mapped_fsgid(struct user_namespace *mnt_userns) +{ + return kgid_from_mnt(mnt_userns, current_fsgid()); +} + +#endif /* _LINUX_MNT_IDMAPPING_H */ diff --git a/security/commoncap.c b/security/commoncap.c index 3f810d37b71b..09479f71ee2e 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -24,6 +24,7 @@ #include #include #include +#include /* * If a non-root user executes a setuid-root binary in -- cgit v1.2.3-59-g8ed1b From 476860b3eb4a50958243158861d5340066df5af2 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:00 +0100 Subject: fs: tweak fsuidgid_has_mapping() If the caller's fs{g,u}id aren't mapped in the mount's idmapping we can return early and skip the check whether the mapped fs{g,u}id also have a mapping in the filesystem's idmapping. If the fs{g,u}id aren't mapped in the mount's idmapping they consequently can't be mapped in the filesystem's idmapping. So there's no point in checking that. Link: https://lore.kernel.org/r/20211123114227.3124056-4-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-4-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-4-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- include/linux/fs.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index b3bcb2129699..db5ee15e36b1 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1695,10 +1695,18 @@ static inline void inode_fsgid_set(struct inode *inode, static inline bool fsuidgid_has_mapping(struct super_block *sb, struct user_namespace *mnt_userns) { - struct user_namespace *s_user_ns = sb->s_user_ns; + struct user_namespace *fs_userns = sb->s_user_ns; + kuid_t kuid; + kgid_t kgid; - return kuid_has_mapping(s_user_ns, mapped_fsuid(mnt_userns)) && - kgid_has_mapping(s_user_ns, mapped_fsgid(mnt_userns)); + kuid = mapped_fsuid(mnt_userns); + if (!uid_valid(kuid)) + return false; + kgid = mapped_fsgid(mnt_userns); + if (!gid_valid(kgid)) + return false; + return kuid_has_mapping(fs_userns, kuid) && + kgid_has_mapping(fs_userns, kgid); } extern struct timespec64 current_time(struct inode *inode); -- cgit v1.2.3-59-g8ed1b From 1ac2a4104968e0a60b4b3572216a92aab5c1b025 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:01 +0100 Subject: fs: account for filesystem mappings Currently we only support idmapped mounts for filesystems mounted without an idmapping. This was a conscious decision mentioned in multiple places (cf. e.g. [1]). As explained at length in [3] it is perfectly fine to extend support for idmapped mounts to filesystem's mounted with an idmapping should the need arise. The need has been there for some time now. Various container projects in userspace need this to run unprivileged and nested unprivileged containers (cf. [2]). Before we can port any filesystem that is mountable with an idmapping to support idmapped mounts we need to first extend the mapping helpers to account for the filesystem's idmapping. This again, is explained at length in our documentation at [3] but I'll give an overview here again. Currently, the low-level mapping helpers implement the remapping algorithms described in [3] in a simplified manner. Because we could rely on the fact that all filesystems supporting idmapped mounts are mounted without an idmapping the translation step from or into the filesystem idmapping could be skipped. In order to support idmapped mounts of filesystem's mountable with an idmapping the translation step we were able to skip before cannot be skipped anymore. A filesystem mounted with an idmapping is very likely to not use an identity mapping and will instead use a non-identity mapping. So the translation step from or into the filesystem's idmapping in the remapping algorithm cannot be skipped for such filesystems. More details with examples can be found in [3]. This patch adds a few new and prepares some already existing low-level mapping helpers to perform the full translation algorithm explained in [3]. The low-level helpers can be written in a way that they only perform the additional translation step when the filesystem is indeed mounted with an idmapping. If the low-level helpers detect that they are not dealing with an idmapped mount they can simply return the relevant k{g,u}id unchanged; no remapping needs to be performed at all. The no_idmapping() helper detects whether the shortcut can be used. If the low-level helpers detected that they are dealing with an idmapped mount but the underlying filesystem is mounted without an idmapping we can rely on the previous shorcut and can continue to skip the translation step from or into the filesystem's idmapping. These checks guarantee that only the minimal amount of work is performed. As before, if idmapped mounts aren't used the low-level helpers are idempotent and no work is performed at all. This patch adds the helpers mapped_k{g,u}id_fs() and mapped_k{g,u}id_user(). Following patches will port all places to replace the old k{g,u}id_into_mnt() and k{g,u}id_from_mnt() with these two new helpers. After the conversion is done k{g,u}id_into_mnt() and k{g,u}id_from_mnt() will be removed. This also concludes the renaming of the mapping helpers we started in [4]. Now, all mapping helpers will started with the "mapped_" prefix making everything nice and consistent. The mapped_k{g,u}id_fs() helpers replace the k{g,u}id_into_mnt() helpers. They are to be used when k{g,u}ids are to be mapped from the vfs, e.g. from from struct inode's i_{g,u}id. Conversely, the mapped_k{g,u}id_user() helpers replace the k{g,u}id_from_mnt() helpers. They are to be used when k{g,u}ids are to be written to disk, e.g. when entering from a system call to change ownership of a file. This patch only introduces the helpers. It doesn't yet convert the relevant places to account for filesystem mounted with an idmapping. [1]: commit 2ca4dcc4909d ("fs/mount_setattr: tighten permission checks") [2]: https://github.com/containers/podman/issues/10374 [3]: Documentations/filesystems/idmappings.rst [4]: commit a65e58e791a1 ("fs: document and rename fsid helpers") Link: https://lore.kernel.org/r/20211123114227.3124056-5-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-5-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-5-brauner@kernel.org Cc: Seth Forshee Cc: Amir Goldstein Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- include/linux/fs.h | 4 +- include/linux/mnt_idmapping.h | 193 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index db5ee15e36b1..57aee6ebba72 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1636,7 +1636,7 @@ static inline void i_gid_write(struct inode *inode, gid_t gid) static inline kuid_t i_uid_into_mnt(struct user_namespace *mnt_userns, const struct inode *inode) { - return kuid_into_mnt(mnt_userns, inode->i_uid); + return mapped_kuid_fs(mnt_userns, &init_user_ns, inode->i_uid); } /** @@ -1650,7 +1650,7 @@ static inline kuid_t i_uid_into_mnt(struct user_namespace *mnt_userns, static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns, const struct inode *inode) { - return kgid_into_mnt(mnt_userns, inode->i_gid); + return mapped_kgid_fs(mnt_userns, &init_user_ns, inode->i_gid); } /** diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h index 47c7811fadfe..60341cd33ccc 100644 --- a/include/linux/mnt_idmapping.h +++ b/include/linux/mnt_idmapping.h @@ -6,6 +6,11 @@ #include struct user_namespace; +/* + * Carries the initial idmapping of 0:0:4294967295 which is an identity + * mapping. This means that {g,u}id 0 is mapped to {g,u}id 0, {g,u}id 1 is + * mapped to {g,u}id 1, [...], {g,u}id 1000 to {g,u}id 1000, [...]. + */ extern struct user_namespace init_user_ns; /** @@ -64,9 +69,189 @@ static inline kgid_t kgid_from_mnt(struct user_namespace *mnt_userns, return KGIDT_INIT(from_kgid(mnt_userns, kgid)); } +/** + * initial_idmapping - check whether this is the initial mapping + * @ns: idmapping to check + * + * Check whether this is the initial mapping, mapping 0 to 0, 1 to 1, + * [...], 1000 to 1000 [...]. + * + * Return: true if this is the initial mapping, false if not. + */ +static inline bool initial_idmapping(const struct user_namespace *ns) +{ + return ns == &init_user_ns; +} + +/** + * no_idmapping - check whether we can skip remapping a kuid/gid + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * + * This function can be used to check whether a remapping between two + * idmappings is required. + * An idmapped mount is a mount that has an idmapping attached to it that + * is different from the filsystem's idmapping and the initial idmapping. + * If the initial mapping is used or the idmapping of the mount and the + * filesystem are identical no remapping is required. + * + * Return: true if remapping can be skipped, false if not. + */ +static inline bool no_idmapping(const struct user_namespace *mnt_userns, + const struct user_namespace *fs_userns) +{ + return initial_idmapping(mnt_userns) || mnt_userns == fs_userns; +} + +/** + * mapped_kuid_fs - map a filesystem kuid into a mnt_userns + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kuid : kuid to be mapped + * + * Take a @kuid and remap it from @fs_userns into @mnt_userns. Use this + * function when preparing a @kuid to be reported to userspace. + * + * If no_idmapping() determines that this is not an idmapped mount we can + * simply return @kuid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kuid won't change when calling + * from_kuid() so we can simply retrieve the value via __kuid_val() + * directly. + * + * Return: @kuid mapped according to @mnt_userns. + * If @kuid has no mapping in either @mnt_userns or @fs_userns INVALID_UID is + * returned. + */ +static inline kuid_t mapped_kuid_fs(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + kuid_t kuid) +{ + uid_t uid; + + if (no_idmapping(mnt_userns, fs_userns)) + return kuid; + if (initial_idmapping(fs_userns)) + uid = __kuid_val(kuid); + else + uid = from_kuid(fs_userns, kuid); + if (uid == (uid_t)-1) + return INVALID_UID; + return make_kuid(mnt_userns, uid); +} + +/** + * mapped_kgid_fs - map a filesystem kgid into a mnt_userns + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kgid : kgid to be mapped + * + * Take a @kgid and remap it from @fs_userns into @mnt_userns. Use this + * function when preparing a @kgid to be reported to userspace. + * + * If no_idmapping() determines that this is not an idmapped mount we can + * simply return @kgid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kgid won't change when calling + * from_kgid() so we can simply retrieve the value via __kgid_val() + * directly. + * + * Return: @kgid mapped according to @mnt_userns. + * If @kgid has no mapping in either @mnt_userns or @fs_userns INVALID_GID is + * returned. + */ +static inline kgid_t mapped_kgid_fs(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + kgid_t kgid) +{ + gid_t gid; + + if (no_idmapping(mnt_userns, fs_userns)) + return kgid; + if (initial_idmapping(fs_userns)) + gid = __kgid_val(kgid); + else + gid = from_kgid(fs_userns, kgid); + if (gid == (gid_t)-1) + return INVALID_GID; + return make_kgid(mnt_userns, gid); +} + +/** + * mapped_kuid_user - map a user kuid into a mnt_userns + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kuid : kuid to be mapped + * + * Use the idmapping of @mnt_userns to remap a @kuid into @fs_userns. Use this + * function when preparing a @kuid to be written to disk or inode. + * + * If no_idmapping() determines that this is not an idmapped mount we can + * simply return @kuid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kuid won't change when calling + * make_kuid() so we can simply retrieve the value via KUIDT_INIT() + * directly. + * + * Return: @kuid mapped according to @mnt_userns. + * If @kuid has no mapping in either @mnt_userns or @fs_userns INVALID_UID is + * returned. + */ +static inline kuid_t mapped_kuid_user(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + kuid_t kuid) +{ + uid_t uid; + + if (no_idmapping(mnt_userns, fs_userns)) + return kuid; + uid = from_kuid(mnt_userns, kuid); + if (uid == (uid_t)-1) + return INVALID_UID; + if (initial_idmapping(fs_userns)) + return KUIDT_INIT(uid); + return make_kuid(fs_userns, uid); +} + +/** + * mapped_kgid_user - map a user kgid into a mnt_userns + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kgid : kgid to be mapped + * + * Use the idmapping of @mnt_userns to remap a @kgid into @fs_userns. Use this + * function when preparing a @kgid to be written to disk or inode. + * + * If no_idmapping() determines that this is not an idmapped mount we can + * simply return @kgid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kgid won't change when calling + * make_kgid() so we can simply retrieve the value via KGIDT_INIT() + * directly. + * + * Return: @kgid mapped according to @mnt_userns. + * If @kgid has no mapping in either @mnt_userns or @fs_userns INVALID_GID is + * returned. + */ +static inline kgid_t mapped_kgid_user(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + kgid_t kgid) +{ + gid_t gid; + + if (no_idmapping(mnt_userns, fs_userns)) + return kgid; + gid = from_kgid(mnt_userns, kgid); + if (gid == (gid_t)-1) + return INVALID_GID; + if (initial_idmapping(fs_userns)) + return KGIDT_INIT(gid); + return make_kgid(fs_userns, gid); +} + /** * mapped_fsuid - return caller's fsuid mapped up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount + * @mnt_userns: the mount's idmapping * * Use this helper to initialize a new vfs or filesystem object based on * the caller's fsuid. A common example is initializing the i_uid field of @@ -78,12 +263,12 @@ static inline kgid_t kgid_from_mnt(struct user_namespace *mnt_userns, */ static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) { - return kuid_from_mnt(mnt_userns, current_fsuid()); + return mapped_kuid_user(mnt_userns, &init_user_ns, current_fsuid()); } /** * mapped_fsgid - return caller's fsgid mapped up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount + * @mnt_userns: the mount's idmapping * * Use this helper to initialize a new vfs or filesystem object based on * the caller's fsgid. A common example is initializing the i_gid field of @@ -95,7 +280,7 @@ static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) */ static inline kgid_t mapped_fsgid(struct user_namespace *mnt_userns) { - return kgid_from_mnt(mnt_userns, current_fsgid()); + return mapped_kgid_user(mnt_userns, &init_user_ns, current_fsgid()); } #endif /* _LINUX_MNT_IDMAPPING_H */ -- cgit v1.2.3-59-g8ed1b From 4bdcd1dd4d2f973b1a89fb20ba720d879e9e506b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 28 Oct 2021 08:47:05 -0600 Subject: mm: move filemap_range_needs_writeback() into header No functional changes in this patch, just in preparation for efficiently calling this light function from the block O_DIRECT handling. Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Jens Axboe --- fs/iomap/direct-io.c | 1 + include/linux/fs.h | 2 -- include/linux/pagemap.h | 29 +++++++++++++++++++++++++++++ mm/filemap.c | 32 +++----------------------------- 4 files changed, 33 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b4dc51063d36..03ea367df19a 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/fs.h b/include/linux/fs.h index bbf812ce89a8..6b8dc1a78df6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2847,8 +2847,6 @@ static inline int filemap_fdatawait(struct address_space *mapping) extern bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend); -extern bool filemap_range_needs_writeback(struct address_space *, - loff_t lstart, loff_t lend); extern int filemap_write_and_wait_range(struct address_space *mapping, loff_t lstart, loff_t lend); extern int __filemap_fdatawrite_range(struct address_space *mapping, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 605246452305..274a0710f2c5 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -963,6 +963,35 @@ static inline int add_to_page_cache(struct page *page, int __filemap_add_folio(struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp); +bool filemap_range_has_writeback(struct address_space *mapping, + loff_t start_byte, loff_t end_byte); + +/** + * filemap_range_needs_writeback - check if range potentially needs writeback + * @mapping: address space within which to check + * @start_byte: offset in bytes where the range starts + * @end_byte: offset in bytes where the range ends (inclusive) + * + * Find at least one page in the range supplied, usually used to check if + * direct writing in this range will trigger a writeback. Used by O_DIRECT + * read/write with IOCB_NOWAIT, to see if the caller needs to do + * filemap_write_and_wait_range() before proceeding. + * + * Return: %true if the caller should do filemap_write_and_wait_range() before + * doing O_DIRECT to a page in this range, %false otherwise. + */ +static inline bool filemap_range_needs_writeback(struct address_space *mapping, + loff_t start_byte, + loff_t end_byte) +{ + if (!mapping->nrpages) + return false; + if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && + !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) + return false; + return filemap_range_has_writeback(mapping, start_byte, end_byte); +} + /** * struct readahead_control - Describes a readahead request. * diff --git a/mm/filemap.c b/mm/filemap.c index daa0e23a6ee6..655c9eec06b3 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -646,8 +646,8 @@ static bool mapping_needs_writeback(struct address_space *mapping) return mapping->nrpages; } -static bool filemap_range_has_writeback(struct address_space *mapping, - loff_t start_byte, loff_t end_byte) +bool filemap_range_has_writeback(struct address_space *mapping, + loff_t start_byte, loff_t end_byte) { XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); pgoff_t max = end_byte >> PAGE_SHIFT; @@ -667,34 +667,8 @@ static bool filemap_range_has_writeback(struct address_space *mapping, } rcu_read_unlock(); return page != NULL; - -} - -/** - * filemap_range_needs_writeback - check if range potentially needs writeback - * @mapping: address space within which to check - * @start_byte: offset in bytes where the range starts - * @end_byte: offset in bytes where the range ends (inclusive) - * - * Find at least one page in the range supplied, usually used to check if - * direct writing in this range will trigger a writeback. Used by O_DIRECT - * read/write with IOCB_NOWAIT, to see if the caller needs to do - * filemap_write_and_wait_range() before proceeding. - * - * Return: %true if the caller should do filemap_write_and_wait_range() before - * doing O_DIRECT to a page in this range, %false otherwise. - */ -bool filemap_range_needs_writeback(struct address_space *mapping, - loff_t start_byte, loff_t end_byte) -{ - if (!mapping_needs_writeback(mapping)) - return false; - if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && - !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) - return false; - return filemap_range_has_writeback(mapping, start_byte, end_byte); } -EXPORT_SYMBOL_GPL(filemap_range_needs_writeback); +EXPORT_SYMBOL_GPL(filemap_range_has_writeback); /** * filemap_write_and_wait_range - write out & wait on a file range -- cgit v1.2.3-59-g8ed1b From 0a467d0fdd9594fbb449ebc93852533332c528fd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 14 Oct 2021 14:39:59 -0600 Subject: block: switch to atomic_t for request references refcount_t is not as expensive as it used to be, but it's still more expensive than the io_uring method of using atomic_t and just checking for potential over/underflow. This borrows that same implementation, which in turn is based on the mm implementation from Linus. Reviewed-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-flush.c | 4 ++-- block/blk-mq-tag.c | 2 +- block/blk-mq.c | 12 ++++++------ block/blk.h | 31 +++++++++++++++++++++++++++++++ include/linux/blk-mq.h | 2 +- 5 files changed, 41 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/block/blk-flush.c b/block/blk-flush.c index f78bb39e589e..e4df894189ce 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -229,7 +229,7 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error) /* release the tag's ownership to the req cloned from */ spin_lock_irqsave(&fq->mq_flush_lock, flags); - if (!refcount_dec_and_test(&flush_rq->ref)) { + if (!req_ref_put_and_test(flush_rq)) { fq->rq_status = error; spin_unlock_irqrestore(&fq->mq_flush_lock, flags); return; @@ -349,7 +349,7 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, * and READ flush_rq->end_io */ smp_wmb(); - refcount_set(&flush_rq->ref, 1); + req_ref_set(flush_rq, 1); blk_flush_queue_rq(flush_rq, false); } diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 995336abee33..380e2dd31bfc 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -228,7 +228,7 @@ static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags, spin_lock_irqsave(&tags->lock, flags); rq = tags->rqs[bitnr]; - if (!rq || rq->tag != bitnr || !refcount_inc_not_zero(&rq->ref)) + if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq)) rq = NULL; spin_unlock_irqrestore(&tags->lock, flags); return rq; diff --git a/block/blk-mq.c b/block/blk-mq.c index fc4520e992b1..8c7cab75229e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -394,7 +394,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, INIT_LIST_HEAD(&rq->queuelist); /* tag was already set */ WRITE_ONCE(rq->deadline, 0); - refcount_set(&rq->ref, 1); + req_ref_set(rq, 1); if (rq->rq_flags & RQF_ELV) { struct elevator_queue *e = data->q->elevator; @@ -642,7 +642,7 @@ void blk_mq_free_request(struct request *rq) rq_qos_done(q, rq); WRITE_ONCE(rq->state, MQ_RQ_IDLE); - if (refcount_dec_and_test(&rq->ref)) + if (req_ref_put_and_test(rq)) __blk_mq_free_request(rq); } EXPORT_SYMBOL_GPL(blk_mq_free_request); @@ -938,7 +938,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) rq_qos_done(rq->q, rq); WRITE_ONCE(rq->state, MQ_RQ_IDLE); - if (!refcount_dec_and_test(&rq->ref)) + if (!req_ref_put_and_test(rq)) continue; blk_crypto_free_request(rq); @@ -1401,7 +1401,7 @@ void blk_mq_put_rq_ref(struct request *rq) { if (is_flush_rq(rq)) rq->end_io(rq, 0); - else if (refcount_dec_and_test(&rq->ref)) + else if (req_ref_put_and_test(rq)) __blk_mq_free_request(rq); } @@ -3049,7 +3049,7 @@ static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, unsigned long rq_addr = (unsigned long)rq; if (rq_addr >= start && rq_addr < end) { - WARN_ON_ONCE(refcount_read(&rq->ref) != 0); + WARN_ON_ONCE(req_ref_read(rq) != 0); cmpxchg(&drv_tags->rqs[i], rq, NULL); } } @@ -3383,7 +3383,7 @@ static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, if (!tags) return; - WARN_ON_ONCE(refcount_read(&flush_rq->ref) != 0); + WARN_ON_ONCE(req_ref_read(flush_rq) != 0); for (i = 0; i < queue_depth; i++) cmpxchg(&tags->rqs[i], flush_rq, NULL); diff --git a/block/blk.h b/block/blk.h index a55d82c3d1c2..24d8b333bb03 100644 --- a/block/blk.h +++ b/block/blk.h @@ -461,4 +461,35 @@ static inline bool should_fail_request(struct block_device *part, } #endif /* CONFIG_FAIL_MAKE_REQUEST */ +/* + * Optimized request reference counting. Ideally we'd make timeouts be more + * clever, as that's the only reason we need references at all... But until + * this happens, this is faster than using refcount_t. Also see: + * + * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count") + */ +#define req_ref_zero_or_close_to_overflow(req) \ + ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u) + +static inline bool req_ref_inc_not_zero(struct request *req) +{ + return atomic_inc_not_zero(&req->ref); +} + +static inline bool req_ref_put_and_test(struct request *req) +{ + WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req)); + return atomic_dec_and_test(&req->ref); +} + +static inline void req_ref_set(struct request *req, int value) +{ + atomic_set(&req->ref, value); +} + +static inline int req_ref_read(struct request *req) +{ + return atomic_read(&req->ref); +} + #endif /* BLK_INTERNAL_H */ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 1b87b7c8bbff..561beb5be7ec 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -139,7 +139,7 @@ struct request { unsigned short ioprio; enum mq_rq_state state; - refcount_t ref; + atomic_t ref; unsigned long deadline; -- cgit v1.2.3-59-g8ed1b From 704b914f15fb7daaf517e3acc4bed472b50ca19e Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 3 Dec 2021 21:15:32 +0800 Subject: blk-mq: move srcu from blk_mq_hw_ctx to request_queue In case of BLK_MQ_F_BLOCKING, per-hctx srcu is used to protect dispatch critical area. However, this srcu instance stays at the end of hctx, and it often takes standalone cacheline, often cold. Inside srcu_read_lock() and srcu_read_unlock(), WRITE is always done on the indirect percpu variable which is allocated from heap instead of being embedded, srcu->srcu_idx is read only in srcu_read_lock(). It doesn't matter if srcu structure stays in hctx or request queue. So switch to per-request-queue srcu for protecting dispatch, and this way simplifies quiesce a lot, not mention quiesce is always done on the request queue wide. Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20211203131534.3668411-3-ming.lei@redhat.com Signed-off-by: Jens Axboe --- block/blk-core.c | 27 ++++++++++++++++++++++----- block/blk-mq-sysfs.c | 2 -- block/blk-mq.c | 37 ++++++++----------------------------- block/blk-mq.h | 4 ++-- block/blk-sysfs.c | 3 ++- block/blk.h | 10 +++++++++- block/genhd.c | 2 +- include/linux/blk-mq.h | 8 -------- include/linux/blkdev.h | 9 +++++++++ 9 files changed, 53 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index b0660c9df852..10619fd83c1b 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -66,6 +66,7 @@ DEFINE_IDA(blk_queue_ida); * For queue allocation */ struct kmem_cache *blk_requestq_cachep; +struct kmem_cache *blk_requestq_srcu_cachep; /* * Controlling structure to kblockd @@ -437,21 +438,27 @@ static void blk_timeout_work(struct work_struct *work) { } -struct request_queue *blk_alloc_queue(int node_id) +struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu) { struct request_queue *q; int ret; - q = kmem_cache_alloc_node(blk_requestq_cachep, - GFP_KERNEL | __GFP_ZERO, node_id); + q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu), + GFP_KERNEL | __GFP_ZERO, node_id); if (!q) return NULL; + if (alloc_srcu) { + blk_queue_flag_set(QUEUE_FLAG_HAS_SRCU, q); + if (init_srcu_struct(q->srcu) != 0) + goto fail_q; + } + q->last_merge = NULL; q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL); if (q->id < 0) - goto fail_q; + goto fail_srcu; ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0); if (ret) @@ -508,8 +515,11 @@ fail_split: bioset_exit(&q->bio_split); fail_id: ida_simple_remove(&blk_queue_ida, q->id); +fail_srcu: + if (alloc_srcu) + cleanup_srcu_struct(q->srcu); fail_q: - kmem_cache_free(blk_requestq_cachep, q); + kmem_cache_free(blk_get_queue_kmem_cache(alloc_srcu), q); return NULL; } @@ -1301,6 +1311,9 @@ int __init blk_dev_init(void) sizeof_field(struct request, cmd_flags)); BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * sizeof_field(struct bio, bi_opf)); + BUILD_BUG_ON(ALIGN(offsetof(struct request_queue, srcu), + __alignof__(struct request_queue)) != + sizeof(struct request_queue)); /* used for unplugging and affects IO latency/throughput - HIGHPRI */ kblockd_workqueue = alloc_workqueue("kblockd", @@ -1311,6 +1324,10 @@ int __init blk_dev_init(void) blk_requestq_cachep = kmem_cache_create("request_queue", sizeof(struct request_queue), 0, SLAB_PANIC, NULL); + blk_requestq_srcu_cachep = kmem_cache_create("request_queue_srcu", + sizeof(struct request_queue) + + sizeof(struct srcu_struct), 0, SLAB_PANIC, NULL); + blk_debugfs_root = debugfs_create_dir("block", NULL); return 0; diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index 253c857cba47..674786574075 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -36,8 +36,6 @@ static void blk_mq_hw_sysfs_release(struct kobject *kobj) struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); - if (hctx->flags & BLK_MQ_F_BLOCKING) - cleanup_srcu_struct(hctx->srcu); blk_free_flush_queue(hctx->fq); sbitmap_free(&hctx->ctx_map); free_cpumask_var(hctx->cpumask); diff --git a/block/blk-mq.c b/block/blk-mq.c index 494da31dc1a5..6a2c2704454e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -260,17 +260,9 @@ EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); */ void blk_mq_wait_quiesce_done(struct request_queue *q) { - struct blk_mq_hw_ctx *hctx; - unsigned int i; - bool rcu = false; - - queue_for_each_hw_ctx(q, hctx, i) { - if (hctx->flags & BLK_MQ_F_BLOCKING) - synchronize_srcu(hctx->srcu); - else - rcu = true; - } - if (rcu) + if (blk_queue_has_srcu(q)) + synchronize_srcu(q->srcu); + else synchronize_rcu(); } EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done); @@ -3400,20 +3392,6 @@ static void blk_mq_exit_hw_queues(struct request_queue *q, } } -static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) -{ - int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); - - BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu), - __alignof__(struct blk_mq_hw_ctx)) != - sizeof(struct blk_mq_hw_ctx)); - - if (tag_set->flags & BLK_MQ_F_BLOCKING) - hw_ctx_size += sizeof(struct srcu_struct); - - return hw_ctx_size; -} - static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) @@ -3451,7 +3429,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx; gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; - hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node); + hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node); if (!hctx) goto fail_alloc_hctx; @@ -3493,8 +3471,6 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, if (!hctx->fq) goto free_bitmap; - if (hctx->flags & BLK_MQ_F_BLOCKING) - init_srcu_struct(hctx->srcu); blk_mq_hctx_kobj_init(hctx); return hctx; @@ -3830,7 +3806,7 @@ static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, struct request_queue *q; int ret; - q = blk_alloc_queue(set->numa_node); + q = blk_alloc_queue(set->numa_node, set->flags & BLK_MQ_F_BLOCKING); if (!q) return ERR_PTR(-ENOMEM); q->queuedata = queuedata; @@ -3979,6 +3955,9 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q) { + WARN_ON_ONCE(blk_queue_has_srcu(q) != + !!(set->flags & BLK_MQ_F_BLOCKING)); + /* mark the queue as mq asap */ q->mq_ops = set->ops; diff --git a/block/blk-mq.h b/block/blk-mq.h index e4c396204928..792f0b29c6eb 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -385,9 +385,9 @@ do { \ int srcu_idx; \ \ might_sleep(); \ - srcu_idx = srcu_read_lock((hctx)->srcu); \ + srcu_idx = srcu_read_lock((hctx)->queue->srcu); \ (dispatch_ops); \ - srcu_read_unlock((hctx)->srcu, srcu_idx); \ + srcu_read_unlock((hctx)->queue->srcu, srcu_idx); \ } \ } while (0) diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 4622da4bb992..3e6357321225 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -735,7 +735,8 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head) { struct request_queue *q = container_of(rcu_head, struct request_queue, rcu_head); - kmem_cache_free(blk_requestq_cachep, q); + + kmem_cache_free(blk_get_queue_kmem_cache(blk_queue_has_srcu(q)), q); } /* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */ diff --git a/block/blk.h b/block/blk.h index 24d8b333bb03..7ccb7c7d86b3 100644 --- a/block/blk.h +++ b/block/blk.h @@ -27,6 +27,7 @@ struct blk_flush_queue { }; extern struct kmem_cache *blk_requestq_cachep; +extern struct kmem_cache *blk_requestq_srcu_cachep; extern struct kobj_type blk_queue_ktype; extern struct ida blk_queue_ida; @@ -424,7 +425,14 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page); -struct request_queue *blk_alloc_queue(int node_id); +static inline struct kmem_cache *blk_get_queue_kmem_cache(bool srcu) +{ + if (srcu) + return blk_requestq_srcu_cachep; + return blk_requestq_cachep; +} +struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu); + int disk_scan_partitions(struct gendisk *disk, fmode_t mode); int disk_alloc_events(struct gendisk *disk); diff --git a/block/genhd.c b/block/genhd.c index 5179a4f00fba..3c139a1b6f04 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1338,7 +1338,7 @@ struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass) struct request_queue *q; struct gendisk *disk; - q = blk_alloc_queue(node); + q = blk_alloc_queue(node, false); if (!q) return NULL; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 561beb5be7ec..ecdc049b52fa 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -375,13 +374,6 @@ struct blk_mq_hw_ctx { * q->unused_hctx_list. */ struct list_head hctx_list; - - /** - * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is - * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also - * blk_mq_hw_ctx_size(). - */ - struct srcu_struct srcu[]; }; /** diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0a4416ef4fbf..c80cfaefc0a8 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -16,6 +16,7 @@ #include #include #include +#include struct module; struct request_queue; @@ -373,11 +374,18 @@ struct request_queue { * devices that do not have multiple independent access ranges. */ struct blk_independent_access_ranges *ia_ranges; + + /** + * @srcu: Sleepable RCU. Use as lock when type of the request queue + * is blocking (BLK_MQ_F_BLOCKING). Must be the last member + */ + struct srcu_struct srcu[]; }; /* Keep blk_queue_flag_name[] in sync with the definitions below */ #define QUEUE_FLAG_STOPPED 0 /* queue is stopped */ #define QUEUE_FLAG_DYING 1 /* queue being torn down */ +#define QUEUE_FLAG_HAS_SRCU 2 /* SRCU is allocated */ #define QUEUE_FLAG_NOMERGES 3 /* disable merge attempts */ #define QUEUE_FLAG_SAME_COMP 4 /* complete on same CPU-group */ #define QUEUE_FLAG_FAIL_IO 5 /* fake timeout */ @@ -415,6 +423,7 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) +#define blk_queue_has_srcu(q) test_bit(QUEUE_FLAG_HAS_SRCU, &(q)->queue_flags) #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) -- cgit v1.2.3-59-g8ed1b From 0cc3a8017900f856f9bf4fdc41c2b5cb1670aabe Mon Sep 17 00:00:00 2001 From: Manish Chopra Date: Thu, 2 Dec 2021 13:01:56 -0800 Subject: qed*: enhance tx timeout debug info This patch add some new qed APIs to query status block info and report various data to MFW on tx timeout event Along with that it enhances qede to dump more debug logs (not just specific to the queue which was reported by stack) on tx timeout which includes various other basic metadata about all tx queues and other info (like status block etc.) Signed-off-by: Manish Chopra Signed-off-by: Prabhakar Kushwaha Signed-off-by: Alok Prasad Signed-off-by: Ariel Elior Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/qlogic/qed/qed_int.c | 22 +++++++ drivers/net/ethernet/qlogic/qed/qed_int.h | 13 ++++ drivers/net/ethernet/qlogic/qed/qed_main.c | 47 +++++++++++++ drivers/net/ethernet/qlogic/qed/qed_mcp.h | 2 + drivers/net/ethernet/qlogic/qed/qed_reg_addr.h | 2 + drivers/net/ethernet/qlogic/qede/qede_main.c | 91 +++++++++++++++++++++----- include/linux/qed/qed_if.h | 11 ++++ 7 files changed, 173 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c index 6958adeca86d..82e74f62b677 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.c +++ b/drivers/net/ethernet/qlogic/qed/qed_int.c @@ -2399,3 +2399,25 @@ int qed_int_set_timer_res(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, return rc; } + +int qed_int_get_sb_dbg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, + struct qed_sb_info *p_sb, struct qed_sb_info_dbg *p_info) +{ + u16 sbid = p_sb->igu_sb_id; + u32 i; + + if (IS_VF(p_hwfn->cdev)) + return -EINVAL; + + if (sbid >= NUM_OF_SBS(p_hwfn->cdev)) + return -EINVAL; + + p_info->igu_prod = qed_rd(p_hwfn, p_ptt, IGU_REG_PRODUCER_MEMORY + sbid * 4); + p_info->igu_cons = qed_rd(p_hwfn, p_ptt, IGU_REG_CONSUMER_MEM + sbid * 4); + + for (i = 0; i < PIS_PER_SB; i++) + p_info->pi[i] = (u16)qed_rd(p_hwfn, p_ptt, + CAU_REG_PI_MEMORY + sbid * 4 * PIS_PER_SB + i * 4); + + return 0; +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.h b/drivers/net/ethernet/qlogic/qed/qed_int.h index 84c17e97f569..7e5127f61744 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.h +++ b/drivers/net/ethernet/qlogic/qed/qed_int.h @@ -185,6 +185,19 @@ void qed_int_disable_post_isr_release(struct qed_dev *cdev); */ void qed_int_attn_clr_enable(struct qed_dev *cdev, bool clr_enable); +/** + * qed_int_get_sb_dbg: Read debug information regarding a given SB + * + * @p_hwfn: hw function pointer + * @p_ptt: ptt resource + * @p_sb: pointer to status block for which we want to get info + * @p_info: pointer to struct to fill with information regarding SB + * + * Return: 0 with status block info filled on success, otherwise return error + */ +int qed_int_get_sb_dbg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, + struct qed_sb_info *p_sb, struct qed_sb_info_dbg *p_info); + /** * qed_db_rec_handler(): Doorbell Recovery handler. * Run doorbell recovery in case of PF overflow (and flush DORQ if diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c index 7673b3e07736..a18d2ea96b26 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_main.c +++ b/drivers/net/ethernet/qlogic/qed/qed_main.c @@ -2936,6 +2936,30 @@ out: return status; } +static int +qed_get_sb_info(struct qed_dev *cdev, struct qed_sb_info *sb, + u16 qid, struct qed_sb_info_dbg *sb_dbg) +{ + struct qed_hwfn *hwfn = &cdev->hwfns[qid % cdev->num_hwfns]; + struct qed_ptt *ptt; + int rc; + + if (IS_VF(cdev)) + return -EINVAL; + + ptt = qed_ptt_acquire(hwfn); + if (!ptt) { + DP_NOTICE(hwfn, "Can't acquire PTT\n"); + return -EAGAIN; + } + + memset(sb_dbg, 0, sizeof(*sb_dbg)); + rc = qed_int_get_sb_dbg(hwfn, ptt, sb, sb_dbg); + + qed_ptt_release(hwfn, ptt); + return rc; +} + static int qed_read_module_eeprom(struct qed_dev *cdev, char *buf, u8 dev_addr, u32 offset, u32 len) { @@ -2978,6 +3002,27 @@ static int qed_set_grc_config(struct qed_dev *cdev, u32 cfg_id, u32 val) return rc; } +static __printf(2, 3) void qed_mfw_report(struct qed_dev *cdev, char *fmt, ...) +{ + char buf[QED_MFW_REPORT_STR_SIZE]; + struct qed_hwfn *p_hwfn; + struct qed_ptt *p_ptt; + va_list vl; + + va_start(vl, fmt); + vsnprintf(buf, QED_MFW_REPORT_STR_SIZE, fmt, vl); + va_end(vl); + + if (IS_PF(cdev)) { + p_hwfn = QED_LEADING_HWFN(cdev); + p_ptt = qed_ptt_acquire(p_hwfn); + if (p_ptt) { + qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, strlen(buf)); + qed_ptt_release(p_hwfn, p_ptt); + } + } +} + static u8 qed_get_affin_hwfn_idx(struct qed_dev *cdev) { return QED_AFFIN_HWFN_IDX(cdev); @@ -3038,6 +3083,8 @@ const struct qed_common_ops qed_common_ops_pass = { .read_nvm_cfg = &qed_nvm_flash_cfg_read, .read_nvm_cfg_len = &qed_nvm_flash_cfg_len, .set_grc_config = &qed_set_grc_config, + .mfw_report = &qed_mfw_report, + .get_sb_info = &qed_get_sb_info, }; void qed_get_protocol_stats(struct qed_dev *cdev, diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h index 564723800d15..2c28d5f86497 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h @@ -15,6 +15,8 @@ #include "qed_hsi.h" #include "qed_dev_api.h" +#define QED_MFW_REPORT_STR_SIZE 256 + struct qed_mcp_link_speed_params { bool autoneg; diff --git a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h index 6f1a52e6beb2..b5e35f433a20 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h +++ b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h @@ -550,6 +550,8 @@ 0x1 << 1) #define IGU_REG_BLOCK_CONFIGURATION_PXP_TPH_INTERFACE_EN ( \ 0x1 << 0) +#define IGU_REG_PRODUCER_MEMORY 0x182000UL +#define IGU_REG_CONSUMER_MEM 0x183000UL #define IGU_REG_MAPPING_MEMORY \ 0x184000UL #define IGU_REG_STATISTIC_NUM_VF_MSG_SENT \ diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 06c6a5813606..b4e5a15e308b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -509,34 +509,95 @@ static int qede_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) return 0; } -static void qede_tx_log_print(struct qede_dev *edev, struct qede_tx_queue *txq) +static void qede_fp_sb_dump(struct qede_dev *edev, struct qede_fastpath *fp) { + char *p_sb = (char *)fp->sb_info->sb_virt; + u32 sb_size, i; + + sb_size = sizeof(struct status_block); + + for (i = 0; i < sb_size; i += 8) + DP_NOTICE(edev, + "%02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX\n", + p_sb[i], p_sb[i + 1], p_sb[i + 2], p_sb[i + 3], + p_sb[i + 4], p_sb[i + 5], p_sb[i + 6], p_sb[i + 7]); +} + +static void +qede_txq_fp_log_metadata(struct qede_dev *edev, + struct qede_fastpath *fp, struct qede_tx_queue *txq) +{ + struct qed_chain *p_chain = &txq->tx_pbl; + + /* Dump txq/fp/sb ids etc. other metadata */ DP_NOTICE(edev, - "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n", - txq->index, le16_to_cpu(*txq->hw_cons_ptr), - qed_chain_get_cons_idx(&txq->tx_pbl), - qed_chain_get_prod_idx(&txq->tx_pbl), - jiffies); + "fpid 0x%x sbid 0x%x txqid [0x%x] ndev_qid [0x%x] cos [0x%x] p_chain %p cap %d size %d jiffies %lu HZ 0x%x\n", + fp->id, fp->sb_info->igu_sb_id, txq->index, txq->ndev_txq_id, txq->cos, + p_chain, p_chain->capacity, p_chain->size, jiffies, HZ); + + /* Dump all the relevant prod/cons indexes */ + DP_NOTICE(edev, + "hw cons %04x sw_tx_prod=0x%x, sw_tx_cons=0x%x, bd_prod 0x%x bd_cons 0x%x\n", + le16_to_cpu(*txq->hw_cons_ptr), txq->sw_tx_prod, txq->sw_tx_cons, + qed_chain_get_prod_idx(p_chain), qed_chain_get_cons_idx(p_chain)); +} + +static void +qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp, struct qede_tx_queue *txq) +{ + struct qed_sb_info_dbg sb_dbg; + int rc; + + /* sb info */ + qede_fp_sb_dump(edev, fp); + + memset(&sb_dbg, 0, sizeof(sb_dbg)); + rc = edev->ops->common->get_sb_info(edev->cdev, fp->sb_info, (u16)fp->id, &sb_dbg); + + DP_NOTICE(edev, "IGU: prod %08x cons %08x CAU Tx %04x\n", + sb_dbg.igu_prod, sb_dbg.igu_cons, sb_dbg.pi[TX_PI(txq->cos)]); + + /* report to mfw */ + edev->ops->common->mfw_report(edev->cdev, + "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n", + txq->index, le16_to_cpu(*txq->hw_cons_ptr), + qed_chain_get_cons_idx(&txq->tx_pbl), + qed_chain_get_prod_idx(&txq->tx_pbl), jiffies); + if (!rc) + edev->ops->common->mfw_report(edev->cdev, + "Txq[%d]: SB[0x%04x] - IGU: prod %08x cons %08x CAU Tx %04x\n", + txq->index, fp->sb_info->igu_sb_id, + sb_dbg.igu_prod, sb_dbg.igu_cons, + sb_dbg.pi[TX_PI(txq->cos)]); } static void qede_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct qede_dev *edev = netdev_priv(dev); - struct qede_tx_queue *txq; - int cos; + int i; netif_carrier_off(dev); DP_NOTICE(edev, "TX timeout on queue %u!\n", txqueue); - if (!(edev->fp_array[txqueue].type & QEDE_FASTPATH_TX)) - return; + for_each_queue(i) { + struct qede_tx_queue *txq; + struct qede_fastpath *fp; + int cos; - for_each_cos_in_txq(edev, cos) { - txq = &edev->fp_array[txqueue].txq[cos]; + fp = &edev->fp_array[i]; + if (!(fp->type & QEDE_FASTPATH_TX)) + continue; - if (qed_chain_get_cons_idx(&txq->tx_pbl) != - qed_chain_get_prod_idx(&txq->tx_pbl)) - qede_tx_log_print(edev, txq); + for_each_cos_in_txq(edev, cos) { + txq = &fp->txq[cos]; + + /* Dump basic metadata for all queues */ + qede_txq_fp_log_metadata(edev, fp, txq); + + if (qed_chain_get_cons_idx(&txq->tx_pbl) != + qed_chain_get_prod_idx(&txq->tx_pbl)) + qede_tx_log_print(edev, fp, txq); + } } if (IS_VF(edev)) diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h index 0dae7fcc5ef2..9f4bfa2a4829 100644 --- a/include/linux/qed/qed_if.h +++ b/include/linux/qed/qed_if.h @@ -807,6 +807,12 @@ struct qed_devlink { struct devlink_health_reporter *fw_reporter; }; +struct qed_sb_info_dbg { + u32 igu_prod; + u32 igu_cons; + u16 pi[PIS_PER_SB]; +}; + struct qed_common_cb_ops { void (*arfs_filter_op)(void *dev, void *fltr, u8 fw_rc); void (*link_update)(void *dev, struct qed_link_output *link); @@ -1194,6 +1200,11 @@ struct qed_common_ops { struct devlink* (*devlink_register)(struct qed_dev *cdev); void (*devlink_unregister)(struct devlink *devlink); + + __printf(2, 3) void (*mfw_report)(struct qed_dev *cdev, char *fmt, ...); + + int (*get_sb_info)(struct qed_dev *cdev, struct qed_sb_info *sb, + u16 qid, struct qed_sb_info_dbg *sb_dbg); }; #define MASK_FIELD(_name, _value) \ -- cgit v1.2.3-59-g8ed1b From 823163ba6e52e644be5df4539a19e3df8d0988dd Mon Sep 17 00:00:00 2001 From: Manish Chopra Date: Thu, 2 Dec 2021 13:01:57 -0800 Subject: qed*: esl priv flag support through ethtool ESL(Enhanced System Lockdown) was designed to lock PCI adapter firmware images and prevent changes to critical non-volatile configuration data so that uncontrolled, malicious or unintentional modification to the adapters are avoided, ensuring it's operational state. Once this feature is enabled, the device is locked, rejecting any modification to non-volatile images. Once unlocked, the protection is off such that firmware and non-volatile configurations may be altered. Driver just reflects the capability and status of this through the ethtool private flag. Signed-off-by: Manish Chopra Signed-off-by: Prabhakar Kushwaha Signed-off-by: Alok Prasad Signed-off-by: Ariel Elior Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/qlogic/qed/qed_main.c | 25 ++++++++++++++++++++++++- drivers/net/ethernet/qlogic/qed/qed_mcp.c | 22 ++++++++++++++++++++++ drivers/net/ethernet/qlogic/qed/qed_mcp.h | 20 ++++++++++++++++++++ drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h | 1 + drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 13 +++++++++++++ include/linux/qed/qed_if.h | 3 +++ 6 files changed, 83 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c index a18d2ea96b26..46d4207f22a3 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_main.c +++ b/drivers/net/ethernet/qlogic/qed/qed_main.c @@ -447,7 +447,7 @@ int qed_fill_dev_info(struct qed_dev *cdev, dev_info->wol_support = true; dev_info->smart_an = qed_mcp_is_smart_an_supported(p_hwfn); - + dev_info->esl = qed_mcp_is_esl_supported(p_hwfn); dev_info->abs_pf_id = QED_LEADING_HWFN(cdev)->abs_pf_id; } else { qed_vf_get_fw_version(&cdev->hwfns[0], &dev_info->fw_major, @@ -3028,6 +3028,28 @@ static u8 qed_get_affin_hwfn_idx(struct qed_dev *cdev) return QED_AFFIN_HWFN_IDX(cdev); } +static int qed_get_esl_status(struct qed_dev *cdev, bool *esl_active) +{ + struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); + struct qed_ptt *ptt; + int rc = 0; + + *esl_active = false; + + if (IS_VF(cdev)) + return 0; + + ptt = qed_ptt_acquire(hwfn); + if (!ptt) + return -EAGAIN; + + rc = qed_mcp_get_esl_status(hwfn, ptt, esl_active); + + qed_ptt_release(hwfn, ptt); + + return rc; +} + static struct qed_selftest_ops qed_selftest_ops_pass = { .selftest_memory = &qed_selftest_memory, .selftest_interrupt = &qed_selftest_interrupt, @@ -3085,6 +3107,7 @@ const struct qed_common_ops qed_common_ops_pass = { .set_grc_config = &qed_set_grc_config, .mfw_report = &qed_mfw_report, .get_sb_info = &qed_get_sb_info, + .get_esl_status = &qed_get_esl_status, }; void qed_get_protocol_stats(struct qed_dev *cdev, diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c index 64678a256f3b..da1eadabcb41 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c @@ -4158,3 +4158,25 @@ qed_mcp_send_raw_debug_data(struct qed_hwfn *p_hwfn, return qed_mcp_send_debug_data(p_hwfn, p_ptt, QED_MCP_DBG_DATA_TYPE_RAW, p_buf, size); } + +bool qed_mcp_is_esl_supported(struct qed_hwfn *p_hwfn) +{ + return !!(p_hwfn->mcp_info->capabilities & + FW_MB_PARAM_FEATURE_SUPPORT_ENHANCED_SYS_LCK); +} + +int qed_mcp_get_esl_status(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool *active) +{ + u32 resp = 0, param = 0; + int rc; + + rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_MANAGEMENT_STATUS, 0, &resp, ¶m); + if (rc) { + DP_NOTICE(p_hwfn, "Failed to send ESL command, rc = %d\n", rc); + return rc; + } + + *active = !!(param & FW_MB_PARAM_MANAGEMENT_STATUS_LOCKDOWN_ENABLED); + + return 0; +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h index 2c28d5f86497..369e1892450a 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h @@ -1339,4 +1339,24 @@ int qed_mcp_nvm_get_cfg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, int qed_mcp_nvm_set_cfg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 option_id, u8 entity_id, u16 flags, u8 *p_buf, u32 len); + +/** + * qed_mcp_is_esl_supported(): Return whether management firmware support ESL or not. + * + * @p_hwfn: hw function pointer + * + * Return: true if esl is supported, otherwise return false + */ +bool qed_mcp_is_esl_supported(struct qed_hwfn *p_hwfn); + +/** + * qed_mcp_get_esl_status(): Get enhanced system lockdown status + * + * @p_hwfn: hw function pointer + * @p_ptt: ptt resource pointer + * @active: ESL active status data pointer + * + * Return: 0 with esl status info on success, otherwise return error + */ +int qed_mcp_get_esl_status(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool *active); #endif diff --git a/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h index 8a0e3c5d4bda..b70ee8200e15 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h +++ b/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h @@ -1191,6 +1191,7 @@ enum drv_msg_code_enum { DRV_MSG_CODE_CFG_VF_MSIX = DRV_MSG_CODE(0xc001), DRV_MSG_CODE_CFG_PF_VFS_MSIX = DRV_MSG_CODE(0xc002), DRV_MSG_CODE_DEBUG_DATA_SEND = DRV_MSG_CODE(0xc004), + DRV_MSG_CODE_GET_MANAGEMENT_STATUS = DRV_MSG_CODE(0xc007), }; #define DRV_MSG_CODE_VMAC_TYPE_SHIFT 4 diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 100c9c52c20b..97a7ab0826ed 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -168,6 +168,8 @@ enum { QEDE_PRI_FLAG_CMT, QEDE_PRI_FLAG_SMART_AN_SUPPORT, /* MFW supports SmartAN */ QEDE_PRI_FLAG_RECOVER_ON_ERROR, + QEDE_PRI_FLAG_ESL_SUPPORT, /* MFW supports Enhanced System Lockdown */ + QEDE_PRI_FLAG_ESL_ACTIVE, /* Enhanced System Lockdown Active status */ QEDE_PRI_FLAG_LEN, }; @@ -175,6 +177,8 @@ static const char qede_private_arr[QEDE_PRI_FLAG_LEN][ETH_GSTRING_LEN] = { "Coupled-Function", "SmartAN capable", "Recover on error", + "ESL capable", + "ESL active", }; enum qede_ethtool_tests { @@ -478,6 +482,7 @@ static int qede_get_sset_count(struct net_device *dev, int stringset) static u32 qede_get_priv_flags(struct net_device *dev) { struct qede_dev *edev = netdev_priv(dev); + bool esl_active; u32 flags = 0; if (edev->dev_info.common.num_hwfns > 1) @@ -489,6 +494,14 @@ static u32 qede_get_priv_flags(struct net_device *dev) if (edev->err_flags & BIT(QEDE_ERR_IS_RECOVERABLE)) flags |= BIT(QEDE_PRI_FLAG_RECOVER_ON_ERROR); + if (edev->dev_info.common.esl) + flags |= BIT(QEDE_PRI_FLAG_ESL_SUPPORT); + + edev->ops->common->get_esl_status(edev->cdev, &esl_active); + + if (esl_active) + flags |= BIT(QEDE_PRI_FLAG_ESL_ACTIVE); + return flags; } diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h index 9f4bfa2a4829..6dc4943d8aec 100644 --- a/include/linux/qed/qed_if.h +++ b/include/linux/qed/qed_if.h @@ -652,6 +652,7 @@ struct qed_dev_info { bool wol_support; bool smart_an; + bool esl; /* MBI version */ u32 mbi_version; @@ -1205,6 +1206,8 @@ struct qed_common_ops { int (*get_sb_info)(struct qed_dev *cdev, struct qed_sb_info *sb, u16 qid, struct qed_sb_info_dbg *sb_dbg); + + int (*get_esl_status)(struct qed_dev *cdev, bool *esl_active); }; #define MASK_FIELD(_name, _value) \ -- cgit v1.2.3-59-g8ed1b From a3642021923b26d86bb27d88c826494827612c06 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 29 Nov 2021 18:46:47 +0100 Subject: locking/rtmutex: Add rt_mutex_lock_nest_lock() and rt_mutex_lock_killable(). The locking selftest for ww-mutex expects to operate directly on the base-mutex which becomes a rtmutex on PREEMPT_RT. Add a rtmutex based implementation of mutex_lock_nest_lock() and mutex_lock_killable() named rt_mutex_lock_nest_lock() abd rt_mutex_lock_killable(). Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20211129174654.668506-5-bigeasy@linutronix.de --- include/linux/rtmutex.h | 9 +++++++++ kernel/locking/rtmutex_api.c | 30 ++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index 9deedfeec2b1..7d049883a08a 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -99,13 +99,22 @@ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock #ifdef CONFIG_DEBUG_LOCK_ALLOC extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass); +extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock); #define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0) +#define rt_mutex_lock_nest_lock(lock, nest_lock) \ + do { \ + typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \ + _rt_mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \ + } while (0) + #else extern void rt_mutex_lock(struct rt_mutex *lock); #define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock) +#define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock) #endif extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); +extern int rt_mutex_lock_killable(struct rt_mutex *lock); extern int rt_mutex_trylock(struct rt_mutex *lock); extern void rt_mutex_unlock(struct rt_mutex *lock); diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 5c9299aaabae..900220941caa 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -21,12 +21,13 @@ int max_lock_depth = 1024; */ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, unsigned int state, + struct lockdep_map *nest_lock, unsigned int subclass) { int ret; might_sleep(); - mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); + mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_); ret = __rt_mutex_lock(&lock->rtmutex, state); if (ret) mutex_release(&lock->dep_map, _RET_IP_); @@ -48,10 +49,16 @@ EXPORT_SYMBOL(rt_mutex_base_init); */ void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass); + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass); } EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); +void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) +{ + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0); +} +EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); + #else /* !CONFIG_DEBUG_LOCK_ALLOC */ /** @@ -61,7 +68,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); */ void __sched rt_mutex_lock(struct rt_mutex *lock) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0); + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0); } EXPORT_SYMBOL_GPL(rt_mutex_lock); #endif @@ -77,10 +84,25 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock); */ int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) { - return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0); + return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0); } EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); +/** + * rt_mutex_lock_killable - lock a rt_mutex killable + * + * @lock: the rt_mutex to be locked + * + * Returns: + * 0 on success + * -EINTR when interrupted by a signal + */ +int __sched rt_mutex_lock_killable(struct rt_mutex *lock) +{ + return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0); +} +EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); + /** * rt_mutex_trylock - try to lock a rt_mutex * -- cgit v1.2.3-59-g8ed1b From 0c1d7a2c2d32fac7ff4a644724b2d52a64184645 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 29 Nov 2021 18:46:48 +0100 Subject: lockdep: Remove softirq accounting on PREEMPT_RT. There is not really a softirq context on PREEMPT_RT. Softirqs on PREEMPT_RT are always invoked within the context of a threaded interrupt handler or within ksoftirqd. The "in-softirq" context is preemptible and is protected by a per-CPU lock to ensure mutual exclusion. There is no difference on PREEMPT_RT between spin_lock_irq() and spin_lock() because the former does not disable interrupts. Therefore if a lock is used in_softirq() and locked once with spin_lock_irq() then lockdep will report this with "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage". Teach lockdep that we don't really do softirqs on -RT. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20211129174654.668506-6-bigeasy@linutronix.de --- include/linux/irqflags.h | 23 +++++++++++++++-------- kernel/locking/lockdep.c | 2 ++ 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index 600c10da321a..4b140938b03e 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -71,14 +71,6 @@ do { \ do { \ __this_cpu_dec(hardirq_context); \ } while (0) -# define lockdep_softirq_enter() \ -do { \ - current->softirq_context++; \ -} while (0) -# define lockdep_softirq_exit() \ -do { \ - current->softirq_context--; \ -} while (0) # define lockdep_hrtimer_enter(__hrtimer) \ ({ \ @@ -140,6 +132,21 @@ do { \ # define lockdep_irq_work_exit(__work) do { } while (0) #endif +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT) +# define lockdep_softirq_enter() \ +do { \ + current->softirq_context++; \ +} while (0) +# define lockdep_softirq_exit() \ +do { \ + current->softirq_context--; \ +} while (0) + +#else +# define lockdep_softirq_enter() do { } while (0) +# define lockdep_softirq_exit() do { } while (0) +#endif + #if defined(CONFIG_IRQSOFF_TRACER) || \ defined(CONFIG_PREEMPT_TRACER) extern void stop_critical_timings(void); diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2270ec68f10a..4a882f83aeb9 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -5485,6 +5485,7 @@ static noinstr void check_flags(unsigned long flags) } } +#ifndef CONFIG_PREEMPT_RT /* * We dont accurately track softirq state in e.g. * hardirq contexts (such as on 4KSTACKS), so only @@ -5499,6 +5500,7 @@ static noinstr void check_flags(unsigned long flags) DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); } } +#endif if (!debug_locks) print_irqtrace_events(current); -- cgit v1.2.3-59-g8ed1b From c0bed69daf4b67809b58cc7cd81a8fa4f45bc161 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 3 Dec 2021 15:59:34 +0800 Subject: locking: Make owner_on_cpu() into Move the owner_on_cpu() from kernel/locking/rwsem.c into include/linux/sched.h with under CONFIG_SMP, then use it in the mutex/rwsem/rtmutex to simplify the code. Signed-off-by: Kefeng Wang Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20211203075935.136808-2-wangkefeng.wang@huawei.com --- include/linux/sched.h | 9 +++++++++ kernel/locking/mutex.c | 11 ++--------- kernel/locking/rtmutex.c | 5 ++--- kernel/locking/rwsem.c | 9 --------- 4 files changed, 13 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 78c351e35fec..ff609d9c2f21 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2171,6 +2171,15 @@ extern long sched_getaffinity(pid_t pid, struct cpumask *mask); #endif #ifdef CONFIG_SMP +static inline bool owner_on_cpu(struct task_struct *owner) +{ + /* + * As lock holder preemption issue, we both skip spinning if + * task is not on cpu or its cpu is preempted + */ + return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); +} + /* Returns effective CPU energy utilization, as seen by the scheduler */ unsigned long sched_cpu_util(int cpu, unsigned long max); #endif /* CONFIG_SMP */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index db1913611192..5e3585950ec8 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -367,8 +367,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, /* * Use vcpu_is_preempted to detect lock holder preemption issue. */ - if (!owner->on_cpu || need_resched() || - vcpu_is_preempted(task_cpu(owner))) { + if (!owner_on_cpu(owner) || need_resched()) { ret = false; break; } @@ -403,14 +402,8 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) * structure won't go away during the spinning period. */ owner = __mutex_owner(lock); - - /* - * As lock holder preemption issue, we both skip spinning if task is not - * on cpu or its cpu is preempted - */ - if (owner) - retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); + retval = owner_on_cpu(owner); /* * If lock->owner is not set, the mutex has been released. Return true diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index f89620852774..0c1f2e3f019a 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1382,9 +1382,8 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, * for CONFIG_PREEMPT_RCU=y) * - the VCPU on which owner runs is preempted */ - if (!owner->on_cpu || need_resched() || - rt_mutex_waiter_is_top_waiter(lock, waiter) || - vcpu_is_preempted(task_cpu(owner))) { + if (!owner_on_cpu(owner) || need_resched() || + rt_mutex_waiter_is_top_waiter(lock, waiter)) { res = false; break; } diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index c51387a43265..b92d0a830568 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -613,15 +613,6 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) return false; } -static inline bool owner_on_cpu(struct task_struct *owner) -{ - /* - * As lock holder preemption issue, we both skip spinning if - * task is not on cpu or its cpu is preempted - */ - return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); -} - static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) { struct task_struct *owner; -- cgit v1.2.3-59-g8ed1b From 4cf75fd4a2545ca4deea992f929602c9fdbe8058 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 3 Dec 2021 15:59:35 +0800 Subject: locking: Mark racy reads of owner->on_cpu One of the more frequent data races reported by KCSAN is the racy read in mutex_spin_on_owner(), which is usually reported as "race of unknown origin" without showing the writer. This is due to the racing write occurring in kernel/sched. Locally enabling KCSAN in kernel/sched shows: | write (marked) to 0xffff97f205079934 of 4 bytes by task 316 on cpu 6: | finish_task kernel/sched/core.c:4632 [inline] | finish_task_switch kernel/sched/core.c:4848 | context_switch kernel/sched/core.c:4975 [inline] | __schedule kernel/sched/core.c:6253 | schedule kernel/sched/core.c:6326 | schedule_preempt_disabled kernel/sched/core.c:6385 | __mutex_lock_common kernel/locking/mutex.c:680 | __mutex_lock kernel/locking/mutex.c:740 [inline] | __mutex_lock_slowpath kernel/locking/mutex.c:1028 | mutex_lock kernel/locking/mutex.c:283 | tty_open_by_driver drivers/tty/tty_io.c:2062 [inline] | ... | | read to 0xffff97f205079934 of 4 bytes by task 322 on cpu 3: | mutex_spin_on_owner kernel/locking/mutex.c:370 | mutex_optimistic_spin kernel/locking/mutex.c:480 | __mutex_lock_common kernel/locking/mutex.c:610 | __mutex_lock kernel/locking/mutex.c:740 [inline] | __mutex_lock_slowpath kernel/locking/mutex.c:1028 | mutex_lock kernel/locking/mutex.c:283 | tty_open_by_driver drivers/tty/tty_io.c:2062 [inline] | ... | | value changed: 0x00000001 -> 0x00000000 This race is clearly intentional, and the potential for miscompilation is slim due to surrounding barrier() and cpu_relax(), and the value being used as a boolean. Nevertheless, marking this reader would more clearly denote intent and make it obvious that concurrency is expected. Use READ_ONCE() to avoid having to reason about compiler optimizations now and in future. With previous refactor, mark the read to owner->on_cpu in owner_on_cpu(), which immediately precedes the loop executing mutex_spin_on_owner(). Signed-off-by: Marco Elver Signed-off-by: Kefeng Wang Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20211203075935.136808-3-wangkefeng.wang@huawei.com --- include/linux/sched.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index ff609d9c2f21..0b9b0e3f4791 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2177,7 +2177,7 @@ static inline bool owner_on_cpu(struct task_struct *owner) * As lock holder preemption issue, we both skip spinning if * task is not on cpu or its cpu is preempted */ - return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); + return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner)); } /* Returns effective CPU energy utilization, as seen by the scheduler */ -- cgit v1.2.3-59-g8ed1b From fb08a1908cb119a4585611d91461ab6d27756b14 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:38 +0100 Subject: dax: simplify the dax_device <-> gendisk association Replace the dax_host_hash with an xarray indexed by the pointer value of the gendisk, and require explicitly calls from the block drivers that want to associate their gendisk with a dax_device. Signed-off-by: Christoph Hellwig Acked-by: Mike Snitzer Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-5-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/bus.c | 6 +-- drivers/dax/super.c | 109 +++++++++++-------------------------------- drivers/md/dm.c | 6 ++- drivers/nvdimm/pmem.c | 10 +++- drivers/s390/block/dcssblk.c | 11 +++-- fs/fuse/virtio_fs.c | 2 +- include/linux/dax.h | 19 ++++++-- 7 files changed, 66 insertions(+), 97 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 452cf7860926..6683d42c32c5 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -1321,10 +1321,10 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) } /* - * No 'host' or dax_operations since there is no access to this - * device outside of mmap of the resulting character device. + * No dax_operations since there is no access to this device outside of + * mmap of the resulting character device. */ - dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC); + dax_dev = alloc_dax(dev_dax, NULL, DAXDEV_F_SYNC); if (IS_ERR(dax_dev)) { rc = PTR_ERR(dax_dev); goto err_alloc_dax; diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e20d0cef10a1..bf77c3da5d56 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -7,10 +7,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -21,15 +19,12 @@ * struct dax_device - anchor object for dax services * @inode: core vfs * @cdev: optional character interface for "device dax" - * @host: optional name for lookups where the device path is not available * @private: dax driver private data * @flags: state and boolean properties */ struct dax_device { - struct hlist_node list; struct inode inode; struct cdev cdev; - const char *host; void *private; unsigned long flags; const struct dax_operations *ops; @@ -42,10 +37,6 @@ static DEFINE_IDA(dax_minor_ida); static struct kmem_cache *dax_cache __read_mostly; static struct super_block *dax_superblock __read_mostly; -#define DAX_HASH_SIZE (PAGE_SIZE / sizeof(struct hlist_head)) -static struct hlist_head dax_host_list[DAX_HASH_SIZE]; -static DEFINE_SPINLOCK(dax_host_lock); - int dax_read_lock(void) { return srcu_read_lock(&dax_srcu); @@ -58,13 +49,22 @@ void dax_read_unlock(int id) } EXPORT_SYMBOL_GPL(dax_read_unlock); -static int dax_host_hash(const char *host) +#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) +#include + +static DEFINE_XARRAY(dax_hosts); + +int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) { - return hashlen_hash(hashlen_string("DAX", host)) % DAX_HASH_SIZE; + return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL); } +EXPORT_SYMBOL_GPL(dax_add_host); -#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) -#include +void dax_remove_host(struct gendisk *disk) +{ + xa_erase(&dax_hosts, (unsigned long)disk); +} +EXPORT_SYMBOL_GPL(dax_remove_host); int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size, pgoff_t *pgoff) @@ -81,41 +81,24 @@ int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size, EXPORT_SYMBOL(bdev_dax_pgoff); /** - * dax_get_by_host() - temporary lookup mechanism for filesystem-dax - * @host: alternate name for the device registered by a dax driver + * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax + * @bdev: block device to find a dax_device for */ -static struct dax_device *dax_get_by_host(const char *host) +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev) { - struct dax_device *dax_dev, *found = NULL; - int hash, id; + struct dax_device *dax_dev; + int id; - if (!host) + if (!blk_queue_dax(bdev->bd_disk->queue)) return NULL; - hash = dax_host_hash(host); - id = dax_read_lock(); - spin_lock(&dax_host_lock); - hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) { - if (!dax_alive(dax_dev) - || strcmp(host, dax_dev->host) != 0) - continue; - - if (igrab(&dax_dev->inode)) - found = dax_dev; - break; - } - spin_unlock(&dax_host_lock); + dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk); + if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) + dax_dev = NULL; dax_read_unlock(id); - return found; -} - -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev) -{ - if (!blk_queue_dax(bdev->bd_disk->queue)) - return NULL; - return dax_get_by_host(bdev->bd_disk->disk_name); + return dax_dev; } EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); @@ -361,12 +344,7 @@ void kill_dax(struct dax_device *dax_dev) return; clear_bit(DAXDEV_ALIVE, &dax_dev->flags); - synchronize_srcu(&dax_srcu); - - spin_lock(&dax_host_lock); - hlist_del_init(&dax_dev->list); - spin_unlock(&dax_host_lock); } EXPORT_SYMBOL_GPL(kill_dax); @@ -398,8 +376,6 @@ static struct dax_device *to_dax_dev(struct inode *inode) static void dax_free_inode(struct inode *inode) { struct dax_device *dax_dev = to_dax_dev(inode); - kfree(dax_dev->host); - dax_dev->host = NULL; if (inode->i_rdev) ida_simple_remove(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); @@ -474,54 +450,25 @@ static struct dax_device *dax_dev_get(dev_t devt) return dax_dev; } -static void dax_add_host(struct dax_device *dax_dev, const char *host) -{ - int hash; - - /* - * Unconditionally init dax_dev since it's coming from a - * non-zeroed slab cache - */ - INIT_HLIST_NODE(&dax_dev->list); - dax_dev->host = host; - if (!host) - return; - - hash = dax_host_hash(host); - spin_lock(&dax_host_lock); - hlist_add_head(&dax_dev->list, &dax_host_list[hash]); - spin_unlock(&dax_host_lock); -} - -struct dax_device *alloc_dax(void *private, const char *__host, - const struct dax_operations *ops, unsigned long flags) +struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, + unsigned long flags) { struct dax_device *dax_dev; - const char *host; dev_t devt; int minor; - if (ops && !ops->zero_page_range) { - pr_debug("%s: error: device does not provide dax" - " operation zero_page_range()\n", - __host ? __host : "Unknown"); + if (WARN_ON_ONCE(ops && !ops->zero_page_range)) return ERR_PTR(-EINVAL); - } - - host = kstrdup(__host, GFP_KERNEL); - if (__host && !host) - return ERR_PTR(-ENOMEM); minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL); if (minor < 0) - goto err_minor; + return ERR_PTR(-ENOMEM); devt = MKDEV(MAJOR(dax_devt), minor); dax_dev = dax_dev_get(devt); if (!dax_dev) goto err_dev; - dax_add_host(dax_dev, host); dax_dev->ops = ops; dax_dev->private = private; if (flags & DAXDEV_F_SYNC) @@ -531,8 +478,6 @@ struct dax_device *alloc_dax(void *private, const char *__host, err_dev: ida_simple_remove(&dax_minor_ida, minor); - err_minor: - kfree(host); return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(alloc_dax); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index b93fcc91176e..a8c650276b32 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1683,6 +1683,7 @@ static void cleanup_mapped_device(struct mapped_device *md) bioset_exit(&md->io_bs); if (md->dax_dev) { + dax_remove_host(md->disk); kill_dax(md->dax_dev); put_dax(md->dax_dev); md->dax_dev = NULL; @@ -1784,12 +1785,13 @@ static struct mapped_device *alloc_dev(int minor) sprintf(md->disk->disk_name, "dm-%d", minor); if (IS_ENABLED(CONFIG_FS_DAX)) { - md->dax_dev = alloc_dax(md, md->disk->disk_name, - &dm_dax_ops, 0); + md->dax_dev = alloc_dax(md, &dm_dax_ops, 0); if (IS_ERR(md->dax_dev)) { md->dax_dev = NULL; goto bad; } + if (dax_add_host(md->dax_dev, md->disk)) + goto bad; } format_dev_t(md->name, MKDEV(_major, minor)); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index fe7ece1534e1..1018f0d44acb 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -379,6 +379,7 @@ static void pmem_release_disk(void *__pmem) { struct pmem_device *pmem = __pmem; + dax_remove_host(pmem->disk); kill_dax(pmem->dax_dev); put_dax(pmem->dax_dev); del_gendisk(pmem->disk); @@ -497,17 +498,20 @@ static int pmem_attach_disk(struct device *dev, if (is_nvdimm_sync(nd_region)) flags = DAXDEV_F_SYNC; - dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags); + dax_dev = alloc_dax(pmem, &pmem_dax_ops, flags); if (IS_ERR(dax_dev)) { rc = PTR_ERR(dax_dev); goto out; } + rc = dax_add_host(dax_dev, disk); + if (rc) + goto out_cleanup_dax; dax_write_cache(dax_dev, nvdimm_has_cache(nd_region)); pmem->dax_dev = dax_dev; rc = device_add_disk(dev, disk, pmem_attribute_groups); if (rc) - goto out_cleanup_dax; + goto out_remove_host; if (devm_add_action_or_reset(dev, pmem_release_disk, pmem)) return -ENOMEM; @@ -519,6 +523,8 @@ static int pmem_attach_disk(struct device *dev, dev_warn(dev, "'badblocks' notification disabled\n"); return 0; +out_remove_host: + dax_remove_host(pmem->disk); out_cleanup_dax: kill_dax(pmem->dax_dev); put_dax(pmem->dax_dev); diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 27ab888b44d0..657e492f2bc2 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -687,18 +687,21 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char if (rc) goto put_dev; - dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name, - &dcssblk_dax_ops, DAXDEV_F_SYNC); + dev_info->dax_dev = alloc_dax(dev_info, &dcssblk_dax_ops, + DAXDEV_F_SYNC); if (IS_ERR(dev_info->dax_dev)) { rc = PTR_ERR(dev_info->dax_dev); dev_info->dax_dev = NULL; goto put_dev; } + rc = dax_add_host(dev_info->dax_dev, dev_info->gd); + if (rc) + goto out_dax; get_device(&dev_info->dev); rc = device_add_disk(&dev_info->dev, dev_info->gd, NULL); if (rc) - goto out_dax; + goto out_dax_host; switch (dev_info->segment_type) { case SEG_TYPE_SR: @@ -714,6 +717,8 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char rc = count; goto out; +out_dax_host: + dax_remove_host(dev_info->gd); out_dax: put_device(&dev_info->dev); kill_dax(dev_info->dax_dev); diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 4cfa4bc1f579..242cc1c0d7ed 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -850,7 +850,7 @@ static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); - fs->dax_dev = alloc_dax(fs, NULL, &virtio_fs_dax_ops, 0); + fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops, 0); if (IS_ERR(fs->dax_dev)) return PTR_ERR(fs->dax_dev); diff --git a/include/linux/dax.h b/include/linux/dax.h index 8623caa67388..e2e9a67004cb 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -11,9 +11,11 @@ typedef unsigned long dax_entry_t; +struct dax_device; +struct gendisk; struct iomap_ops; struct iomap; -struct dax_device; + struct dax_operations { /* * direct_access: translate a device-relative @@ -39,8 +41,8 @@ struct dax_operations { }; #if IS_ENABLED(CONFIG_DAX) -struct dax_device *alloc_dax(void *private, const char *host, - const struct dax_operations *ops, unsigned long flags); +struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, + unsigned long flags); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); @@ -68,7 +70,7 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, return dax_synchronous(dax_dev); } #else -static inline struct dax_device *alloc_dax(void *private, const char *host, +static inline struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, unsigned long flags) { /* @@ -107,6 +109,8 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, struct writeback_control; int bdev_dax_pgoff(struct block_device *, sector_t, size_t, pgoff_t *pgoff); #if IS_ENABLED(CONFIG_FS_DAX) +int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); +void dax_remove_host(struct gendisk *disk); bool generic_fsdax_supported(struct dax_device *dax_dev, struct block_device *bdev, int blocksize, sector_t start, sector_t sectors); @@ -128,6 +132,13 @@ struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t st dax_entry_t dax_lock_page(struct page *page); void dax_unlock_page(struct page *page, dax_entry_t cookie); #else +static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) +{ + return 0; +} +static inline void dax_remove_host(struct gendisk *disk) +{ +} #define generic_fsdax_supported NULL static inline bool dax_supported(struct dax_device *dax_dev, -- cgit v1.2.3-59-g8ed1b From 7b0800d00dae8c897398abaf61e82db0d67d7afc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:42 +0100 Subject: dax: remove dax_capable Just open code the block size and dax_dev == NULL checks in the callers. Signed-off-by: Christoph Hellwig Acked-by: Mike Snitzer Reviewed-by: Gao Xiang [erofs] Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-9-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/super.c | 36 ------------------------------------ drivers/md/dm-table.c | 22 +++++++++++----------- drivers/md/dm.c | 21 --------------------- drivers/md/dm.h | 4 ---- drivers/nvdimm/pmem.c | 1 - drivers/s390/block/dcssblk.c | 1 - fs/erofs/super.c | 11 +++++++---- fs/ext2/super.c | 6 ++++-- fs/ext4/super.c | 9 ++++++--- fs/xfs/xfs_super.c | 21 ++++++++------------- include/linux/dax.h | 14 -------------- 11 files changed, 36 insertions(+), 110 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index f2cef47bdeaf..90b5733f5a70 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -107,42 +107,6 @@ struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev) return dax_dev; } EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); - -bool generic_fsdax_supported(struct dax_device *dax_dev, - struct block_device *bdev, int blocksize, sector_t start, - sector_t sectors) -{ - if (blocksize != PAGE_SIZE) { - pr_info("%pg: error: unsupported blocksize for dax\n", bdev); - return false; - } - - if (!dax_dev) { - pr_debug("%pg: error: dax unsupported by block device\n", bdev); - return false; - } - - return true; -} -EXPORT_SYMBOL_GPL(generic_fsdax_supported); - -bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev, - int blocksize, sector_t start, sector_t len) -{ - bool ret = false; - int id; - - if (!dax_dev) - return false; - - id = dax_read_lock(); - if (dax_alive(dax_dev) && dax_dev->ops->dax_supported) - ret = dax_dev->ops->dax_supported(dax_dev, bdev, blocksize, - start, len); - dax_read_unlock(id); - return ret; -} -EXPORT_SYMBOL_GPL(dax_supported); #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ enum dax_device_flags { diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index aa173f5bdc3d..e43096cfe9e2 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -806,12 +806,14 @@ void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type) EXPORT_SYMBOL_GPL(dm_table_set_type); /* validate the dax capability of the target device span */ -int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev, +static int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { - int blocksize = *(int *) data; + if (dev->dax_dev) + return false; - return !dax_supported(dev->dax_dev, dev->bdev, blocksize, start, len); + DMDEBUG("%pg: error: dax unsupported by block device", dev->bdev); + return true; } /* Check devices support synchronous DAX */ @@ -821,8 +823,8 @@ static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_de return !dev->dax_dev || !dax_synchronous(dev->dax_dev); } -bool dm_table_supports_dax(struct dm_table *t, - iterate_devices_callout_fn iterate_fn, int *blocksize) +static bool dm_table_supports_dax(struct dm_table *t, + iterate_devices_callout_fn iterate_fn) { struct dm_target *ti; unsigned i; @@ -835,7 +837,7 @@ bool dm_table_supports_dax(struct dm_table *t, return false; if (!ti->type->iterate_devices || - ti->type->iterate_devices(ti, iterate_fn, blocksize)) + ti->type->iterate_devices(ti, iterate_fn, NULL)) return false; } @@ -862,7 +864,6 @@ static int dm_table_determine_type(struct dm_table *t) struct dm_target *tgt; struct list_head *devices = dm_table_get_devices(t); enum dm_queue_mode live_md_type = dm_get_md_type(t->md); - int page_size = PAGE_SIZE; if (t->type != DM_TYPE_NONE) { /* target already set the table's type */ @@ -906,7 +907,7 @@ static int dm_table_determine_type(struct dm_table *t) verify_bio_based: /* We must use this table as bio-based */ t->type = DM_TYPE_BIO_BASED; - if (dm_table_supports_dax(t, device_not_dax_capable, &page_size) || + if (dm_table_supports_dax(t, device_not_dax_capable) || (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) { t->type = DM_TYPE_DAX_BIO_BASED; } @@ -1976,7 +1977,6 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, struct queue_limits *limits) { bool wc = false, fua = false; - int page_size = PAGE_SIZE; int r; /* @@ -2010,9 +2010,9 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, } blk_queue_write_cache(q, wc, fua); - if (dm_table_supports_dax(t, device_not_dax_capable, &page_size)) { + if (dm_table_supports_dax(t, device_not_dax_capable)) { blk_queue_flag_set(QUEUE_FLAG_DAX, q); - if (dm_table_supports_dax(t, device_not_dax_synchronous_capable, NULL)) + if (dm_table_supports_dax(t, device_not_dax_synchronous_capable)) set_dax_synchronous(t->md->dax_dev); } else diff --git a/drivers/md/dm.c b/drivers/md/dm.c index a8c650276b32..4eba27e75c23 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1027,26 +1027,6 @@ static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, return ret; } -static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev, - int blocksize, sector_t start, sector_t len) -{ - struct mapped_device *md = dax_get_private(dax_dev); - struct dm_table *map; - bool ret = false; - int srcu_idx; - - map = dm_get_live_table(md, &srcu_idx); - if (!map) - goto out; - - ret = dm_table_supports_dax(map, device_not_dax_capable, &blocksize); - -out: - dm_put_live_table(md, srcu_idx); - - return ret; -} - static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) { @@ -3044,7 +3024,6 @@ static const struct block_device_operations dm_rq_blk_dops = { static const struct dax_operations dm_dax_ops = { .direct_access = dm_dax_direct_access, - .dax_supported = dm_dax_supported, .copy_from_iter = dm_dax_copy_from_iter, .copy_to_iter = dm_dax_copy_to_iter, .zero_page_range = dm_dax_zero_page_range, diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 742d9c80efe1..9013dc1a7b00 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -73,10 +73,6 @@ bool dm_table_bio_based(struct dm_table *t); bool dm_table_request_based(struct dm_table *t); void dm_table_free_md_mempools(struct dm_table *t); struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); -bool dm_table_supports_dax(struct dm_table *t, iterate_devices_callout_fn fn, - int *blocksize); -int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev, - sector_t start, sector_t len, void *data); void dm_lock_md_type(struct mapped_device *md); void dm_unlock_md_type(struct mapped_device *md); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 1018f0d44acb..4190c8c46ca8 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -321,7 +321,6 @@ static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, static const struct dax_operations pmem_dax_ops = { .direct_access = pmem_dax_direct_access, - .dax_supported = generic_fsdax_supported, .copy_from_iter = pmem_copy_from_iter, .copy_to_iter = pmem_copy_to_iter, .zero_page_range = pmem_dax_zero_page_range, diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 657e492f2bc2..e65e83764d1c 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -72,7 +72,6 @@ static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev, static const struct dax_operations dcssblk_dax_ops = { .direct_access = dcssblk_dax_direct_access, - .dax_supported = generic_fsdax_supported, .copy_from_iter = dcssblk_dax_copy_from_iter, .copy_to_iter = dcssblk_dax_copy_to_iter, .zero_page_range = dcssblk_dax_zero_page_range, diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 6a969b1e0ee6..0aed886473c8 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -652,10 +652,13 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) if (err) return err; - if (test_opt(&sbi->opt, DAX_ALWAYS) && - !dax_supported(sbi->dax_dev, sb->s_bdev, EROFS_BLKSIZ, 0, bdev_nr_sectors(sb->s_bdev))) { - errorfc(fc, "DAX unsupported by block device. Turning off DAX."); - clear_opt(&sbi->opt, DAX_ALWAYS); + if (test_opt(&sbi->opt, DAX_ALWAYS)) { + BUILD_BUG_ON(EROFS_BLKSIZ != PAGE_SIZE); + + if (!sbi->dax_dev) { + errorfc(fc, "DAX unsupported by block device. Turning off DAX."); + clear_opt(&sbi->opt, DAX_ALWAYS); + } } sb->s_flags |= SB_RDONLY | SB_NOATIME; sb->s_maxbytes = MAX_LFS_FILESIZE; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index d8d580b609ba..a964066a80aa 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -946,11 +946,13 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) blocksize = BLOCK_SIZE << le32_to_cpu(sbi->s_es->s_log_block_size); if (test_opt(sb, DAX)) { - if (!dax_supported(dax_dev, sb->s_bdev, blocksize, 0, - bdev_nr_sectors(sb->s_bdev))) { + if (!dax_dev) { ext2_msg(sb, KERN_ERR, "DAX unsupported by block device. Turning off DAX."); clear_opt(sbi->s_mount_opt, DAX); + } else if (blocksize != PAGE_SIZE) { + ext2_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); + clear_opt(sbi->s_mount_opt, DAX); } } diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 4e33b5eca694..fd3d68f10ee5 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -4299,9 +4299,12 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) goto failed_mount; } - if (dax_supported(dax_dev, sb->s_bdev, blocksize, 0, - bdev_nr_sectors(sb->s_bdev))) - set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); + if (dax_dev) { + if (blocksize == PAGE_SIZE) + set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); + else + ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); + } if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { if (ext4_has_feature_inline_data(sb)) { diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 875fd3151d6c..c4297206f483 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -331,28 +331,23 @@ xfs_set_inode_alloc( return xfs_is_inode32(mp) ? maxagi : agcount; } -static bool -xfs_buftarg_is_dax( - struct super_block *sb, - struct xfs_buftarg *bt) -{ - return dax_supported(bt->bt_daxdev, bt->bt_bdev, sb->s_blocksize, 0, - bdev_nr_sectors(bt->bt_bdev)); -} - static int xfs_setup_dax_always( struct xfs_mount *mp) { - struct super_block *sb = mp->m_super; - - if (!xfs_buftarg_is_dax(sb, mp->m_ddev_targp) && - (!mp->m_rtdev_targp || !xfs_buftarg_is_dax(sb, mp->m_rtdev_targp))) { + if (!mp->m_ddev_targp->bt_daxdev && + (!mp->m_rtdev_targp || !mp->m_rtdev_targp->bt_daxdev)) { xfs_alert(mp, "DAX unsupported by block device. Turning off DAX."); goto disable_dax; } + if (mp->m_super->s_blocksize != PAGE_SIZE) { + xfs_alert(mp, + "DAX not supported for blocksize. Turning off DAX."); + goto disable_dax; + } + if (xfs_has_reflink(mp)) { xfs_alert(mp, "DAX and reflink cannot be used together!"); return -EINVAL; diff --git a/include/linux/dax.h b/include/linux/dax.h index e2e9a67004cb..439c3c70e347 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -111,12 +111,6 @@ int bdev_dax_pgoff(struct block_device *, sector_t, size_t, pgoff_t *pgoff); #if IS_ENABLED(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); void dax_remove_host(struct gendisk *disk); -bool generic_fsdax_supported(struct dax_device *dax_dev, - struct block_device *bdev, int blocksize, sector_t start, - sector_t sectors); - -bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev, - int blocksize, sector_t start, sector_t len); static inline void fs_put_dax(struct dax_device *dax_dev) { @@ -139,14 +133,6 @@ static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) static inline void dax_remove_host(struct gendisk *disk) { } -#define generic_fsdax_supported NULL - -static inline bool dax_supported(struct dax_device *dax_dev, - struct block_device *bdev, int blocksize, sector_t start, - sector_t len) -{ - return false; -} static inline void fs_put_dax(struct dax_device *dax_dev) { -- cgit v1.2.3-59-g8ed1b From 60696eb26a37ab0199f7833ddbc1b75138c36d16 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:48 +0100 Subject: fsdax: simplify the pgoff calculation Replace the two steps of dax_iomap_sector and bdev_dax_pgoff with a single dax_iomap_pgoff helper that avoids lots of cumbersome sector conversions. Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-15-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/super.c | 14 -------------- fs/dax.c | 35 ++++++++++------------------------- include/linux/dax.h | 1 - 3 files changed, 10 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 90b5733f5a70..45d931aefd06 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -66,20 +66,6 @@ void dax_remove_host(struct gendisk *disk) } EXPORT_SYMBOL_GPL(dax_remove_host); -int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size, - pgoff_t *pgoff) -{ - sector_t start_sect = bdev ? get_start_sect(bdev) : 0; - phys_addr_t phys_off = (start_sect + sector) * 512; - - if (pgoff) - *pgoff = PHYS_PFN(phys_off); - if (phys_off % PAGE_SIZE || size % PAGE_SIZE) - return -EINVAL; - return 0; -} -EXPORT_SYMBOL(bdev_dax_pgoff); - /** * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax * @bdev: block device to find a dax_device for diff --git a/fs/dax.c b/fs/dax.c index e51b4129d1b6..5364549d67a4 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -709,23 +709,22 @@ int dax_invalidate_mapping_entry_sync(struct address_space *mapping, return __dax_invalidate_entry(mapping, index, false); } -static sector_t dax_iomap_sector(const struct iomap *iomap, loff_t pos) +static pgoff_t dax_iomap_pgoff(const struct iomap *iomap, loff_t pos) { - return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9; + phys_addr_t paddr = iomap->addr + (pos & PAGE_MASK) - iomap->offset; + + if (iomap->bdev) + paddr += (get_start_sect(iomap->bdev) << SECTOR_SHIFT); + return PHYS_PFN(paddr); } static int copy_cow_page_dax(struct vm_fault *vmf, const struct iomap_iter *iter) { - sector_t sector = dax_iomap_sector(&iter->iomap, iter->pos); + pgoff_t pgoff = dax_iomap_pgoff(&iter->iomap, iter->pos); void *vto, *kaddr; - pgoff_t pgoff; long rc; int id; - rc = bdev_dax_pgoff(iter->iomap.bdev, sector, PAGE_SIZE, &pgoff); - if (rc) - return rc; - id = dax_read_lock(); rc = dax_direct_access(iter->iomap.dax_dev, pgoff, 1, &kaddr, NULL); if (rc < 0) { @@ -1013,14 +1012,10 @@ EXPORT_SYMBOL_GPL(dax_writeback_mapping_range); static int dax_iomap_pfn(const struct iomap *iomap, loff_t pos, size_t size, pfn_t *pfnp) { - const sector_t sector = dax_iomap_sector(iomap, pos); - pgoff_t pgoff; + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); int id, rc; long length; - rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff); - if (rc) - return rc; id = dax_read_lock(); length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size), NULL, pfnp); @@ -1129,7 +1124,7 @@ static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf, s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap) { sector_t sector = iomap_sector(iomap, pos & PAGE_MASK); - pgoff_t pgoff; + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); long rc, id; void *kaddr; bool page_aligned = false; @@ -1140,10 +1135,6 @@ s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap) (size == PAGE_SIZE)) page_aligned = true; - rc = bdev_dax_pgoff(iomap->bdev, sector, PAGE_SIZE, &pgoff); - if (rc) - return rc; - id = dax_read_lock(); if (page_aligned) @@ -1169,7 +1160,6 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi, const struct iomap *iomap = &iomi->iomap; loff_t length = iomap_length(iomi); loff_t pos = iomi->pos; - struct block_device *bdev = iomap->bdev; struct dax_device *dax_dev = iomap->dax_dev; loff_t end = pos + length, done = 0; ssize_t ret = 0; @@ -1203,9 +1193,8 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi, while (pos < end) { unsigned offset = pos & (PAGE_SIZE - 1); const size_t size = ALIGN(length + offset, PAGE_SIZE); - const sector_t sector = dax_iomap_sector(iomap, pos); + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); ssize_t map_len; - pgoff_t pgoff; void *kaddr; if (fatal_signal_pending(current)) { @@ -1213,10 +1202,6 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi, break; } - ret = bdev_dax_pgoff(bdev, sector, size, &pgoff); - if (ret) - break; - map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL); if (map_len < 0) { diff --git a/include/linux/dax.h b/include/linux/dax.h index 439c3c70e347..324363b798ec 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -107,7 +107,6 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, #endif struct writeback_control; -int bdev_dax_pgoff(struct block_device *, sector_t, size_t, pgoff_t *pgoff); #if IS_ENABLED(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); void dax_remove_host(struct gendisk *disk); -- cgit v1.2.3-59-g8ed1b From c6f40468657d16e4010ef84bf32a761feb3469ea Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:52 +0100 Subject: fsdax: decouple zeroing from the iomap buffered I/O code Unshare the DAX and iomap buffered I/O page zeroing code. This code previously did a IS_DAX check deep inside the iomap code, which in fact was the only DAX check in the code. Instead move these checks into the callers. Most callers already have DAX special casing anyway and XFS will need it for reflink support as well. Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-19-hch@lst.de Signed-off-by: Dan Williams --- fs/dax.c | 77 +++++++++++++++++++++++++++++++++++++++++--------- fs/ext2/inode.c | 7 +++-- fs/ext4/inode.c | 5 ++-- fs/iomap/buffered-io.c | 35 ++++++++--------------- fs/xfs/xfs_iomap.c | 7 ++++- include/linux/dax.h | 7 ++++- 6 files changed, 94 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index d5db1297a0bb..43d58b4219fd 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1135,24 +1135,73 @@ static int dax_memzero(struct dax_device *dax_dev, pgoff_t pgoff, return ret; } -s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap) +static s64 dax_zero_iter(struct iomap_iter *iter, bool *did_zero) { - pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); - long rc, id; - unsigned offset = offset_in_page(pos); - unsigned size = min_t(u64, PAGE_SIZE - offset, length); + const struct iomap *iomap = &iter->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iter); + loff_t pos = iter->pos; + u64 length = iomap_length(iter); + s64 written = 0; + + /* already zeroed? we're done. */ + if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) + return length; + + do { + unsigned offset = offset_in_page(pos); + unsigned size = min_t(u64, PAGE_SIZE - offset, length); + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); + long rc; + int id; - id = dax_read_lock(); - if (IS_ALIGNED(pos, PAGE_SIZE) && size == PAGE_SIZE) - rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1); - else - rc = dax_memzero(iomap->dax_dev, pgoff, offset, size); - dax_read_unlock(id); + id = dax_read_lock(); + if (IS_ALIGNED(pos, PAGE_SIZE) && size == PAGE_SIZE) + rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1); + else + rc = dax_memzero(iomap->dax_dev, pgoff, offset, size); + dax_read_unlock(id); - if (rc < 0) - return rc; - return size; + if (rc < 0) + return rc; + pos += size; + length -= size; + written += size; + if (did_zero) + *did_zero = true; + } while (length > 0); + + return written; +} + +int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, + const struct iomap_ops *ops) +{ + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .len = len, + .flags = IOMAP_ZERO, + }; + int ret; + + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.processed = dax_zero_iter(&iter, did_zero); + return ret; +} +EXPORT_SYMBOL_GPL(dax_zero_range); + +int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, + const struct iomap_ops *ops) +{ + unsigned int blocksize = i_blocksize(inode); + unsigned int off = pos & (blocksize - 1); + + /* Block boundary? Nothing to do */ + if (!off) + return 0; + return dax_zero_range(inode, pos, blocksize - off, did_zero, ops); } +EXPORT_SYMBOL_GPL(dax_truncate_page); static loff_t dax_iomap_iter(const struct iomap_iter *iomi, struct iov_iter *iter) diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 333fa62661d5..01d69618277d 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "ext2.h" #include "acl.h" #include "xattr.h" @@ -1297,9 +1298,9 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) inode_dio_wait(inode); if (IS_DAX(inode)) { - error = iomap_zero_range(inode, newsize, - PAGE_ALIGN(newsize) - newsize, NULL, - &ext2_iomap_ops); + error = dax_zero_range(inode, newsize, + PAGE_ALIGN(newsize) - newsize, NULL, + &ext2_iomap_ops); } else if (test_opt(inode->i_sb, NOBH)) error = nobh_truncate_page(inode->i_mapping, newsize, ext2_get_block); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index bfd3545f1e5d..d316a2009489 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "ext4_jbd2.h" #include "xattr.h" @@ -3780,8 +3781,8 @@ static int ext4_block_zero_page_range(handle_t *handle, length = max; if (IS_DAX(inode)) { - return iomap_zero_range(inode, from, length, NULL, - &ext4_iomap_ops); + return dax_zero_range(inode, from, length, NULL, + &ext4_iomap_ops); } return __ext4_block_zero_page_range(handle, mapping, from, length); } diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 1753c26c8e76..b1511255b4df 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -870,26 +870,8 @@ iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, } EXPORT_SYMBOL_GPL(iomap_file_unshare); -static s64 __iomap_zero_iter(struct iomap_iter *iter, loff_t pos, u64 length) -{ - struct page *page; - int status; - unsigned offset = offset_in_page(pos); - unsigned bytes = min_t(u64, PAGE_SIZE - offset, length); - - status = iomap_write_begin(iter, pos, bytes, &page); - if (status) - return status; - - zero_user(page, offset, bytes); - mark_page_accessed(page); - - return iomap_write_end(iter, pos, bytes, bytes, page); -} - static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) { - struct iomap *iomap = &iter->iomap; const struct iomap *srcmap = iomap_iter_srcmap(iter); loff_t pos = iter->pos; loff_t length = iomap_length(iter); @@ -900,12 +882,19 @@ static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) return length; do { - s64 bytes; + unsigned offset = offset_in_page(pos); + size_t bytes = min_t(u64, PAGE_SIZE - offset, length); + struct page *page; + int status; - if (IS_DAX(iter->inode)) - bytes = dax_iomap_zero(pos, length, iomap); - else - bytes = __iomap_zero_iter(iter, pos, length); + status = iomap_write_begin(iter, pos, bytes, &page); + if (status) + return status; + + zero_user(page, offset, bytes); + mark_page_accessed(page); + + bytes = iomap_write_end(iter, pos, bytes, bytes, page); if (bytes < 0) return bytes; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index d6d71ae9f2ae..6a0c3b307bd7 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -28,7 +28,6 @@ #include "xfs_dquot.h" #include "xfs_reflink.h" - #define XFS_ALLOC_ALIGN(mp, off) \ (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) @@ -1321,6 +1320,9 @@ xfs_zero_range( { struct inode *inode = VFS_I(ip); + if (IS_DAX(inode)) + return dax_zero_range(inode, pos, len, did_zero, + &xfs_buffered_write_iomap_ops); return iomap_zero_range(inode, pos, len, did_zero, &xfs_buffered_write_iomap_ops); } @@ -1333,6 +1335,9 @@ xfs_truncate_page( { struct inode *inode = VFS_I(ip); + if (IS_DAX(inode)) + return dax_truncate_page(inode, pos, did_zero, + &xfs_buffered_write_iomap_ops); return iomap_truncate_page(inode, pos, did_zero, &xfs_buffered_write_iomap_ops); } diff --git a/include/linux/dax.h b/include/linux/dax.h index 324363b798ec..b79036743e7f 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -14,6 +14,7 @@ typedef unsigned long dax_entry_t; struct dax_device; struct gendisk; struct iomap_ops; +struct iomap_iter; struct iomap; struct dax_operations { @@ -170,6 +171,11 @@ static inline void dax_unlock_page(struct page *page, dax_entry_t cookie) } #endif +int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, + const struct iomap_ops *ops); +int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, + const struct iomap_ops *ops); + #if IS_ENABLED(CONFIG_DAX) int dax_read_lock(void); void dax_read_unlock(int id); @@ -204,7 +210,6 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); int dax_invalidate_mapping_entry_sync(struct address_space *mapping, pgoff_t index); -s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap); static inline bool dax_mapping(struct address_space *mapping) { return mapping->host && IS_DAX(mapping->host); -- cgit v1.2.3-59-g8ed1b From 952da06375c8f3aa58474fff718d9ae8442531b9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:58 +0100 Subject: iomap: add a IOMAP_DAX flag Add a flag so that the file system can easily detect DAX operations based just on the iomap operation requested instead of looking at inode state using IS_DAX. This will be needed to apply the to be added partition offset only for operations that actually use DAX, but not things like fiemap that are based on the block device. In the long run it should also allow turning the bdev, dax_dev and inline_data into a union. Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-25-hch@lst.de Signed-off-by: Dan Williams --- fs/dax.c | 7 ++++--- fs/ext4/inode.c | 4 ++-- fs/xfs/xfs_iomap.c | 7 ++++--- fs/xfs/xfs_iomap.h | 3 ++- fs/xfs/xfs_pnfs.c | 2 +- include/linux/iomap.h | 5 +++++ 6 files changed, 18 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 43d58b4219fd..148e8b0967f3 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1180,7 +1180,7 @@ int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, .inode = inode, .pos = pos, .len = len, - .flags = IOMAP_ZERO, + .flags = IOMAP_DAX | IOMAP_ZERO, }; int ret; @@ -1308,6 +1308,7 @@ dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, .inode = iocb->ki_filp->f_mapping->host, .pos = iocb->ki_pos, .len = iov_iter_count(iter), + .flags = IOMAP_DAX, }; loff_t done = 0; int ret; @@ -1461,7 +1462,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, .inode = mapping->host, .pos = (loff_t)vmf->pgoff << PAGE_SHIFT, .len = PAGE_SIZE, - .flags = IOMAP_FAULT, + .flags = IOMAP_DAX | IOMAP_FAULT, }; vm_fault_t ret = 0; void *entry; @@ -1570,7 +1571,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, struct iomap_iter iter = { .inode = mapping->host, .len = PMD_SIZE, - .flags = IOMAP_FAULT, + .flags = IOMAP_DAX | IOMAP_FAULT, }; vm_fault_t ret = VM_FAULT_FALLBACK; pgoff_t max_pgoff; diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index d316a2009489..89c4a174bd39 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3349,8 +3349,8 @@ retry: * DAX and direct I/O are the only two operations that are currently * supported with IOMAP_WRITE. */ - WARN_ON(!IS_DAX(inode) && !(flags & IOMAP_DIRECT)); - if (IS_DAX(inode)) + WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT))); + if (flags & IOMAP_DAX) m_flags = EXT4_GET_BLOCKS_CREATE_ZERO; /* * We use i_size instead of i_disksize here because delalloc writeback diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index d6beb1502f8b..0ed3e7674353 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -188,6 +188,7 @@ xfs_iomap_write_direct( struct xfs_inode *ip, xfs_fileoff_t offset_fsb, xfs_fileoff_t count_fsb, + unsigned int flags, struct xfs_bmbt_irec *imap) { struct xfs_mount *mp = ip->i_mount; @@ -229,7 +230,7 @@ xfs_iomap_write_direct( * the reserve block pool for bmbt block allocation if there is no space * left but we need to do unwritten extent conversion. */ - if (IS_DAX(VFS_I(ip))) { + if (flags & IOMAP_DAX) { bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; if (imap->br_state == XFS_EXT_UNWRITTEN) { force = true; @@ -620,7 +621,7 @@ imap_needs_alloc( imap->br_startblock == DELAYSTARTBLOCK) return true; /* we convert unwritten extents before copying the data for DAX */ - if (IS_DAX(inode) && imap->br_state == XFS_EXT_UNWRITTEN) + if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN) return true; return false; } @@ -826,7 +827,7 @@ allocate_blocks: xfs_iunlock(ip, lockmode); error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb, - &imap); + flags, &imap); if (error) return error; diff --git a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h index 657cc02290f2..e88dc162c785 100644 --- a/fs/xfs/xfs_iomap.h +++ b/fs/xfs/xfs_iomap.h @@ -12,7 +12,8 @@ struct xfs_inode; struct xfs_bmbt_irec; int xfs_iomap_write_direct(struct xfs_inode *ip, xfs_fileoff_t offset_fsb, - xfs_fileoff_t count_fsb, struct xfs_bmbt_irec *imap); + xfs_fileoff_t count_fsb, unsigned int flags, + struct xfs_bmbt_irec *imap); int xfs_iomap_write_unwritten(struct xfs_inode *, xfs_off_t, xfs_off_t, bool); xfs_fileoff_t xfs_iomap_eof_align_last_fsb(struct xfs_inode *ip, xfs_fileoff_t end_fsb); diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c index 7ce1ea11fc3f..d6334abbc0b3 100644 --- a/fs/xfs/xfs_pnfs.c +++ b/fs/xfs/xfs_pnfs.c @@ -155,7 +155,7 @@ xfs_fs_map_blocks( xfs_iunlock(ip, lock_flags); error = xfs_iomap_write_direct(ip, offset_fsb, - end_fsb - offset_fsb, &imap); + end_fsb - offset_fsb, 0, &imap); if (error) goto out_unlock; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 6d1b08d0ae93..5b9432f9f79e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -141,6 +141,11 @@ struct iomap_page_ops { #define IOMAP_NOWAIT (1 << 5) /* do not block */ #define IOMAP_OVERWRITE_ONLY (1 << 6) /* only pure overwrites allowed */ #define IOMAP_UNSHARE (1 << 7) /* unshare_file_range */ +#ifdef CONFIG_FS_DAX +#define IOMAP_DAX (1 << 8) /* DAX mapping */ +#else +#define IOMAP_DAX 0 +#endif /* CONFIG_FS_DAX */ struct iomap_ops { /* -- cgit v1.2.3-59-g8ed1b From cd913c76f489def1a388e3a5b10df94948ede3f5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:21:59 +0100 Subject: dax: return the partition offset from fs_dax_get_by_bdev Prepare for the removal of the block_device from the DAX I/O path by returning the partition offset from fs_dax_get_by_bdev so that the file systems have it at hand for use during I/O. Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-26-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/super.c | 9 ++++++--- drivers/md/dm.c | 4 ++-- fs/erofs/internal.h | 2 ++ fs/erofs/super.c | 4 ++-- fs/ext2/ext2.h | 1 + fs/ext2/super.c | 2 +- fs/ext4/ext4.h | 1 + fs/ext4/super.c | 2 +- fs/xfs/xfs_buf.c | 2 +- fs/xfs/xfs_buf.h | 1 + include/linux/dax.h | 6 ++++-- 11 files changed, 22 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 45d931aefd06..e7152a6c4cc4 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -69,17 +69,20 @@ EXPORT_SYMBOL_GPL(dax_remove_host); /** * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax * @bdev: block device to find a dax_device for + * @start_off: returns the byte offset into the dax_device that @bdev starts */ -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev) +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off) { struct dax_device *dax_dev; + u64 part_size; int id; if (!blk_queue_dax(bdev->bd_disk->queue)) return NULL; - if ((get_start_sect(bdev) * SECTOR_SIZE) % PAGE_SIZE || - (bdev_nr_sectors(bdev) * SECTOR_SIZE) % PAGE_SIZE) { + *start_off = get_start_sect(bdev) * SECTOR_SIZE; + part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE; + if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) { pr_info("%pg: error: unaligned partition for dax\n", bdev); return NULL; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 4eba27e75c23..4e997c02bb0a 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -637,7 +637,7 @@ static int open_table_device(struct table_device *td, dev_t dev, struct mapped_device *md) { struct block_device *bdev; - + u64 part_off; int r; BUG_ON(td->dm_dev.bdev); @@ -653,7 +653,7 @@ static int open_table_device(struct table_device *td, dev_t dev, } td->dm_dev.bdev = bdev; - td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev); + td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off); return 0; } diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 3265688af7f9..c1e65346e9f1 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -51,6 +51,7 @@ struct erofs_device_info { char *path; struct block_device *bdev; struct dax_device *dax_dev; + u64 dax_part_off; u32 blocks; u32 mapped_blkaddr; @@ -109,6 +110,7 @@ struct erofs_sb_info { #endif /* CONFIG_EROFS_FS_ZIP */ struct erofs_dev_context *devs; struct dax_device *dax_dev; + u64 dax_part_off; u64 total_blocks; u32 primarydevice_blocks; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 0aed886473c8..71efce16024d 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -312,7 +312,7 @@ static int erofs_init_devices(struct super_block *sb, goto err_out; } dif->bdev = bdev; - dif->dax_dev = fs_dax_get_by_bdev(bdev); + dif->dax_dev = fs_dax_get_by_bdev(bdev, &dif->dax_part_off); dif->blocks = le32_to_cpu(dis->blocks); dif->mapped_blkaddr = le32_to_cpu(dis->mapped_blkaddr); sbi->total_blocks += dif->blocks; @@ -644,7 +644,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_fs_info = sbi; sbi->opt = ctx->opt; - sbi->dax_dev = fs_dax_get_by_bdev(sb->s_bdev); + sbi->dax_dev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->dax_part_off); sbi->devs = ctx->devs; ctx->devs = NULL; diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 3be9dd6412b7..d4f306aa5ace 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -118,6 +118,7 @@ struct ext2_sb_info { spinlock_t s_lock; struct mb_cache *s_ea_block_cache; struct dax_device *s_daxdev; + u64 s_dax_part_off; }; static inline spinlock_t * diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 7e23482862e6..94f1fbd7d3ac 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -831,7 +831,7 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) } sb->s_fs_info = sbi; sbi->s_sb_block = sb_block; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev); + sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off); spin_lock_init(&sbi->s_lock); ret = -EINVAL; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 404dd50856e5..9cc55bcda6ba 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1697,6 +1697,7 @@ struct ext4_sb_info { */ struct percpu_rw_semaphore s_writepages_rwsem; struct dax_device *s_daxdev; + u64 s_dax_part_off; #ifdef CONFIG_EXT4_DEBUG unsigned long s_simulate_fail; #endif diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 8d7e3449c647..56228e33e52a 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -3913,7 +3913,7 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) if (!sbi->s_blockgroup_lock) goto out_free_base; - sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev); + sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off); sb->s_fs_info = sbi; sbi->s_sb = sb; sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 4d4553ffa705..bbb0fbd34e64 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1945,7 +1945,7 @@ xfs_alloc_buftarg( btp->bt_mount = mp; btp->bt_dev = bdev->bd_dev; btp->bt_bdev = bdev; - btp->bt_daxdev = fs_dax_get_by_bdev(bdev); + btp->bt_daxdev = fs_dax_get_by_bdev(bdev, &btp->bt_dax_part_off); /* * Buffer IO error rate limiting. Limit it to no more than 10 messages diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index bd7f709f0d23..edcb6254fa6a 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -89,6 +89,7 @@ typedef struct xfs_buftarg { dev_t bt_dev; struct block_device *bt_bdev; struct dax_device *bt_daxdev; + u64 bt_dax_part_off; struct xfs_mount *bt_mount; unsigned int bt_meta_sectorsize; size_t bt_meta_sectormask; diff --git a/include/linux/dax.h b/include/linux/dax.h index b79036743e7f..f6f353382cc9 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -117,7 +117,8 @@ static inline void fs_put_dax(struct dax_device *dax_dev) put_dax(dax_dev); } -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev); +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, + u64 *start_off); int dax_writeback_mapping_range(struct address_space *mapping, struct dax_device *dax_dev, struct writeback_control *wbc); @@ -138,7 +139,8 @@ static inline void fs_put_dax(struct dax_device *dax_dev) { } -static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev) +static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, + u64 *start_off) { return NULL; } -- cgit v1.2.3-59-g8ed1b From 2ede892342b3c628991ff1b9060108a7edd92d94 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Nov 2021 11:22:01 +0100 Subject: dax: fix up some of the block device related ifdefs The DAX device <-> block device association is only enabled if CONFIG_BLOCK is enabled. Update dax.h to account for that and use the right conditions for the fs_put_dax stub as well. Signed-off-by: Christoph Hellwig Reviewed-by: Dan Williams Reviewed-by: Darrick J. Wong Link: https://lore.kernel.org/r/20211129102203.2243509-28-hch@lst.de Signed-off-by: Dan Williams --- include/linux/dax.h | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dax.h b/include/linux/dax.h index f6f353382cc9..87ae4c9b1d65 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -108,24 +108,15 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, #endif struct writeback_control; -#if IS_ENABLED(CONFIG_FS_DAX) +#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); void dax_remove_host(struct gendisk *disk); - +struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, + u64 *start_off); static inline void fs_put_dax(struct dax_device *dax_dev) { put_dax(dax_dev); } - -struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, - u64 *start_off); -int dax_writeback_mapping_range(struct address_space *mapping, - struct dax_device *dax_dev, struct writeback_control *wbc); - -struct page *dax_layout_busy_page(struct address_space *mapping); -struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); -dax_entry_t dax_lock_page(struct page *page); -void dax_unlock_page(struct page *page, dax_entry_t cookie); #else static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) { @@ -134,17 +125,25 @@ static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) static inline void dax_remove_host(struct gendisk *disk) { } - -static inline void fs_put_dax(struct dax_device *dax_dev) -{ -} - static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off) { return NULL; } +static inline void fs_put_dax(struct dax_device *dax_dev) +{ +} +#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ +#if IS_ENABLED(CONFIG_FS_DAX) +int dax_writeback_mapping_range(struct address_space *mapping, + struct dax_device *dax_dev, struct writeback_control *wbc); + +struct page *dax_layout_busy_page(struct address_space *mapping); +struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); +dax_entry_t dax_lock_page(struct page *page); +void dax_unlock_page(struct page *page, dax_entry_t cookie); +#else static inline struct page *dax_layout_busy_page(struct address_space *mapping) { return NULL; -- cgit v1.2.3-59-g8ed1b From 866de407444398bc8140ea70de1dba5f91cc34ac Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 3 Dec 2021 13:30:01 +0800 Subject: bpf: Disallow BPF_LOG_KERNEL log level for bpf(BPF_BTF_LOAD) BPF_LOG_KERNEL is only used internally, so disallow bpf_btf_load() to set log level as BPF_LOG_KERNEL. The same checking has already been done in bpf_check(), so factor out a helper to check the validity of log attributes and use it in both places. Fixes: 8580ac9404f6 ("bpf: Process in-kernel BTF") Signed-off-by: Hou Tao Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20211203053001.740945-1-houtao1@huawei.com --- include/linux/bpf_verifier.h | 7 +++++++ kernel/bpf/btf.c | 3 +-- kernel/bpf/verifier.c | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c8a78e830fca..182b16a91084 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -396,6 +396,13 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) log->level == BPF_LOG_KERNEL); } +static inline bool +bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log) +{ + return log->len_total >= 128 && log->len_total <= UINT_MAX >> 2 && + log->level && log->ubuf && !(log->level & ~BPF_LOG_MASK); +} + #define BPF_MAX_SUBPROGS 256 struct bpf_subprog_info { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 2a902a946f70..36a5cc0f53c6 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4473,8 +4473,7 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size, log->len_total = log_size; /* log attributes have to be sane */ - if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || - !log->level || !log->ubuf) { + if (!bpf_verifier_log_attr_valid(log)) { err = -EINVAL; goto errout; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6522ffdea487..1126b75fe650 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14050,11 +14050,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) log->ubuf = (char __user *) (unsigned long) attr->log_buf; log->len_total = attr->log_size; - ret = -EINVAL; /* log attributes have to be sane */ - if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || - !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) + if (!bpf_verifier_log_attr_valid(log)) { + ret = -EINVAL; goto err_unlock; + } } if (IS_ERR(btf_vmlinux)) { -- cgit v1.2.3-59-g8ed1b From b80892ca022e9eb484771a66eb68e12364695a2a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 28 Oct 2021 17:10:17 +0200 Subject: memremap: remove support for external pgmap refcounts No driver is left using the external pgmap refcount, so remove the code to support it. Signed-off-by: Christoph Hellwig Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211028151017.50234-1-hch@lst.de Signed-off-by: Dan Williams --- drivers/pci/p2pdma.c | 2 +- include/linux/memremap.h | 18 ++---------- mm/memremap.c | 59 ++++++++------------------------------- tools/testing/nvdimm/test/iomap.c | 43 ++++++++-------------------- 4 files changed, 28 insertions(+), 94 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 8d47cb7218d1..454d5f6f16ff 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -219,7 +219,7 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, error = gen_pool_add_owner(p2pdma->pool, (unsigned long)addr, pci_bus_address(pdev, bar) + offset, range_len(&pgmap->range), dev_to_node(&pdev->dev), - pgmap->ref); + &pgmap->ref); if (error) goto pages_free; diff --git a/include/linux/memremap.h b/include/linux/memremap.h index c0e9d35889e8..a8bc588fe7aa 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -72,16 +72,6 @@ struct dev_pagemap_ops { */ void (*page_free)(struct page *page); - /* - * Transition the refcount in struct dev_pagemap to the dead state. - */ - void (*kill)(struct dev_pagemap *pgmap); - - /* - * Wait for refcount in struct dev_pagemap to be idle and reap it. - */ - void (*cleanup)(struct dev_pagemap *pgmap); - /* * Used for private (un-addressable) device memory only. Must migrate * the page back to a CPU accessible page. @@ -95,8 +85,7 @@ struct dev_pagemap_ops { * struct dev_pagemap - metadata for ZONE_DEVICE mappings * @altmap: pre-allocated/reserved memory for vmemmap allocations * @ref: reference count that pins the devm_memremap_pages() mapping - * @internal_ref: internal reference if @ref is not provided by the caller - * @done: completion for @internal_ref + * @done: completion for @ref * @type: memory type: see MEMORY_* in memory_hotplug.h * @flags: PGMAP_* flags to specify defailed behavior * @ops: method table @@ -109,8 +98,7 @@ struct dev_pagemap_ops { */ struct dev_pagemap { struct vmem_altmap altmap; - struct percpu_ref *ref; - struct percpu_ref internal_ref; + struct percpu_ref ref; struct completion done; enum memory_type type; unsigned int flags; @@ -191,7 +179,7 @@ static inline unsigned long memremap_compat_align(void) static inline void put_dev_pagemap(struct dev_pagemap *pgmap) { if (pgmap) - percpu_ref_put(pgmap->ref); + percpu_ref_put(&pgmap->ref); } #endif /* _LINUX_MEMREMAP_H_ */ diff --git a/mm/memremap.c b/mm/memremap.c index 5a66a71ab591..643965da13a6 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -112,30 +112,6 @@ static unsigned long pfn_next(unsigned long pfn) #define for_each_device_pfn(pfn, map, i) \ for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); pfn = pfn_next(pfn)) -static void dev_pagemap_kill(struct dev_pagemap *pgmap) -{ - if (pgmap->ops && pgmap->ops->kill) - pgmap->ops->kill(pgmap); - else - percpu_ref_kill(pgmap->ref); -} - -static void dev_pagemap_cleanup(struct dev_pagemap *pgmap) -{ - if (pgmap->ops && pgmap->ops->cleanup) { - pgmap->ops->cleanup(pgmap); - } else { - wait_for_completion(&pgmap->done); - percpu_ref_exit(pgmap->ref); - } - /* - * Undo the pgmap ref assignment for the internal case as the - * caller may re-enable the same pgmap. - */ - if (pgmap->ref == &pgmap->internal_ref) - pgmap->ref = NULL; -} - static void pageunmap_range(struct dev_pagemap *pgmap, int range_id) { struct range *range = &pgmap->ranges[range_id]; @@ -167,11 +143,12 @@ void memunmap_pages(struct dev_pagemap *pgmap) unsigned long pfn; int i; - dev_pagemap_kill(pgmap); + percpu_ref_kill(&pgmap->ref); for (i = 0; i < pgmap->nr_range; i++) for_each_device_pfn(pfn, pgmap, i) put_page(pfn_to_page(pfn)); - dev_pagemap_cleanup(pgmap); + wait_for_completion(&pgmap->done); + percpu_ref_exit(&pgmap->ref); for (i = 0; i < pgmap->nr_range; i++) pageunmap_range(pgmap, i); @@ -188,8 +165,7 @@ static void devm_memremap_pages_release(void *data) static void dev_pagemap_percpu_release(struct percpu_ref *ref) { - struct dev_pagemap *pgmap = - container_of(ref, struct dev_pagemap, internal_ref); + struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref); complete(&pgmap->done); } @@ -295,8 +271,8 @@ static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params, memmap_init_zone_device(&NODE_DATA(nid)->node_zones[ZONE_DEVICE], PHYS_PFN(range->start), PHYS_PFN(range_len(range)), pgmap); - percpu_ref_get_many(pgmap->ref, pfn_end(pgmap, range_id) - - pfn_first(pgmap, range_id)); + percpu_ref_get_many(&pgmap->ref, + pfn_end(pgmap, range_id) - pfn_first(pgmap, range_id)); return 0; err_add_memory: @@ -362,22 +338,11 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid) break; } - if (!pgmap->ref) { - if (pgmap->ops && (pgmap->ops->kill || pgmap->ops->cleanup)) - return ERR_PTR(-EINVAL); - - init_completion(&pgmap->done); - error = percpu_ref_init(&pgmap->internal_ref, - dev_pagemap_percpu_release, 0, GFP_KERNEL); - if (error) - return ERR_PTR(error); - pgmap->ref = &pgmap->internal_ref; - } else { - if (!pgmap->ops || !pgmap->ops->kill || !pgmap->ops->cleanup) { - WARN(1, "Missing reference count teardown definition\n"); - return ERR_PTR(-EINVAL); - } - } + init_completion(&pgmap->done); + error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0, + GFP_KERNEL); + if (error) + return ERR_PTR(error); devmap_managed_enable_get(pgmap); @@ -486,7 +451,7 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, /* fall back to slow path lookup */ rcu_read_lock(); pgmap = xa_load(&pgmap_array, PHYS_PFN(phys)); - if (pgmap && !percpu_ref_tryget_live(pgmap->ref)) + if (pgmap && !percpu_ref_tryget_live(&pgmap->ref)) pgmap = NULL; rcu_read_unlock(); diff --git a/tools/testing/nvdimm/test/iomap.c b/tools/testing/nvdimm/test/iomap.c index ed563bdd88f3..b752ce47ead3 100644 --- a/tools/testing/nvdimm/test/iomap.c +++ b/tools/testing/nvdimm/test/iomap.c @@ -100,25 +100,17 @@ static void nfit_test_kill(void *_pgmap) { struct dev_pagemap *pgmap = _pgmap; - WARN_ON(!pgmap || !pgmap->ref); - - if (pgmap->ops && pgmap->ops->kill) - pgmap->ops->kill(pgmap); - else - percpu_ref_kill(pgmap->ref); - - if (pgmap->ops && pgmap->ops->cleanup) { - pgmap->ops->cleanup(pgmap); - } else { - wait_for_completion(&pgmap->done); - percpu_ref_exit(pgmap->ref); - } + WARN_ON(!pgmap); + + percpu_ref_kill(&pgmap->ref); + + wait_for_completion(&pgmap->done); + percpu_ref_exit(&pgmap->ref); } static void dev_pagemap_percpu_release(struct percpu_ref *ref) { - struct dev_pagemap *pgmap = - container_of(ref, struct dev_pagemap, internal_ref); + struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref); complete(&pgmap->done); } @@ -132,22 +124,11 @@ void *__wrap_devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) if (!nfit_res) return devm_memremap_pages(dev, pgmap); - if (!pgmap->ref) { - if (pgmap->ops && (pgmap->ops->kill || pgmap->ops->cleanup)) - return ERR_PTR(-EINVAL); - - init_completion(&pgmap->done); - error = percpu_ref_init(&pgmap->internal_ref, - dev_pagemap_percpu_release, 0, GFP_KERNEL); - if (error) - return ERR_PTR(error); - pgmap->ref = &pgmap->internal_ref; - } else { - if (!pgmap->ops || !pgmap->ops->kill || !pgmap->ops->cleanup) { - WARN(1, "Missing reference count teardown definition\n"); - return ERR_PTR(-EINVAL); - } - } + init_completion(&pgmap->done); + error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0, + GFP_KERNEL); + if (error) + return ERR_PTR(error); error = devm_add_action_or_reset(dev, nfit_test_kill, pgmap); if (error) -- cgit v1.2.3-59-g8ed1b From 02e4079913500f24ceb082d8d87d8665f044b298 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:04 +0100 Subject: fs: remove unused low-level mapping helpers Now that we ported all places to use the new low-level mapping helpers that are able to support filesystems mounted with an idmapping we can remove the old low-level mapping helpers. With the removal of these old helpers we also conclude the renaming of the mapping helpers we started in commit a65e58e791a1 ("fs: document and rename fsid helpers"). Link: https://lore.kernel.org/r/20211123114227.3124056-8-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-8-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-8-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- include/linux/mnt_idmapping.h | 56 ------------------------------------------- 1 file changed, 56 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h index 60341cd33ccc..0c6ab3f4c952 100644 --- a/include/linux/mnt_idmapping.h +++ b/include/linux/mnt_idmapping.h @@ -13,62 +13,6 @@ struct user_namespace; */ extern struct user_namespace init_user_ns; -/** - * kuid_into_mnt - map a kuid down into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kuid: kuid to be mapped - * - * Return: @kuid mapped according to @mnt_userns. - * If @kuid has no mapping INVALID_UID is returned. - */ -static inline kuid_t kuid_into_mnt(struct user_namespace *mnt_userns, - kuid_t kuid) -{ - return make_kuid(mnt_userns, __kuid_val(kuid)); -} - -/** - * kgid_into_mnt - map a kgid down into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kgid: kgid to be mapped - * - * Return: @kgid mapped according to @mnt_userns. - * If @kgid has no mapping INVALID_GID is returned. - */ -static inline kgid_t kgid_into_mnt(struct user_namespace *mnt_userns, - kgid_t kgid) -{ - return make_kgid(mnt_userns, __kgid_val(kgid)); -} - -/** - * kuid_from_mnt - map a kuid up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kuid: kuid to be mapped - * - * Return: @kuid mapped up according to @mnt_userns. - * If @kuid has no mapping INVALID_UID is returned. - */ -static inline kuid_t kuid_from_mnt(struct user_namespace *mnt_userns, - kuid_t kuid) -{ - return KUIDT_INIT(from_kuid(mnt_userns, kuid)); -} - -/** - * kgid_from_mnt - map a kgid up into a mnt_userns - * @mnt_userns: user namespace of the relevant mount - * @kgid: kgid to be mapped - * - * Return: @kgid mapped up according to @mnt_userns. - * If @kgid has no mapping INVALID_GID is returned. - */ -static inline kgid_t kgid_from_mnt(struct user_namespace *mnt_userns, - kgid_t kgid) -{ - return KGIDT_INIT(from_kgid(mnt_userns, kgid)); -} - /** * initial_idmapping - check whether this is the initial mapping * @ns: idmapping to check -- cgit v1.2.3-59-g8ed1b From 209188ce75d0d357c292f6bb81d712acdd4e7db7 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:05 +0100 Subject: fs: port higher-level mapping helpers Enable the mapped_fs{g,u}id() helpers to support filesystems mounted with an idmapping. Apart from core mapping helpers that use mapped_fs{g,u}id() to initialize struct inode's i_{g,u}id fields xfs is the only place that uses these low-level helpers directly. The patch only extends the helpers to be able to take the filesystem idmapping into account. Since we don't actually yet pass the filesystem's idmapping in no functional changes happen. This will happen in a final patch. Link: https://lore.kernel.org/r/20211123114227.3124056-9-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-9-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-9-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- fs/xfs/xfs_inode.c | 8 ++++---- fs/xfs/xfs_symlink.c | 4 ++-- include/linux/fs.h | 8 ++++---- include/linux/mnt_idmapping.h | 12 ++++++++---- 4 files changed, 18 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 64b9bf334806..5ca689459bed 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -988,8 +988,8 @@ xfs_create( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns), - mapped_fsgid(mnt_userns), prid, + error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns, &init_user_ns), + mapped_fsgid(mnt_userns, &init_user_ns), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) @@ -1142,8 +1142,8 @@ xfs_create_tmpfile( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns), - mapped_fsgid(mnt_userns), prid, + error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns, &init_user_ns), + mapped_fsgid(mnt_userns, &init_user_ns), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index fc2c6a404647..a31d2e5d0321 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -184,8 +184,8 @@ xfs_symlink( /* * Make sure that we have allocated dquot(s) on disk. */ - error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns), - mapped_fsgid(mnt_userns), prid, + error = xfs_qm_vop_dqalloc(dp, mapped_fsuid(mnt_userns, &init_user_ns), + mapped_fsgid(mnt_userns, &init_user_ns), prid, XFS_QMOPT_QUOTALL | XFS_QMOPT_INHERIT, &udqp, &gdqp, &pdqp); if (error) diff --git a/include/linux/fs.h b/include/linux/fs.h index 57aee6ebba72..e1f28f757f1b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1664,7 +1664,7 @@ static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns, static inline void inode_fsuid_set(struct inode *inode, struct user_namespace *mnt_userns) { - inode->i_uid = mapped_fsuid(mnt_userns); + inode->i_uid = mapped_fsuid(mnt_userns, &init_user_ns); } /** @@ -1678,7 +1678,7 @@ static inline void inode_fsuid_set(struct inode *inode, static inline void inode_fsgid_set(struct inode *inode, struct user_namespace *mnt_userns) { - inode->i_gid = mapped_fsgid(mnt_userns); + inode->i_gid = mapped_fsgid(mnt_userns, &init_user_ns); } /** @@ -1699,10 +1699,10 @@ static inline bool fsuidgid_has_mapping(struct super_block *sb, kuid_t kuid; kgid_t kgid; - kuid = mapped_fsuid(mnt_userns); + kuid = mapped_fsuid(mnt_userns, &init_user_ns); if (!uid_valid(kuid)) return false; - kgid = mapped_fsgid(mnt_userns); + kgid = mapped_fsgid(mnt_userns, &init_user_ns); if (!gid_valid(kgid)) return false; return kuid_has_mapping(fs_userns, kuid) && diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h index 0c6ab3f4c952..ee5a217de2a8 100644 --- a/include/linux/mnt_idmapping.h +++ b/include/linux/mnt_idmapping.h @@ -196,6 +196,7 @@ static inline kgid_t mapped_kgid_user(struct user_namespace *mnt_userns, /** * mapped_fsuid - return caller's fsuid mapped up into a mnt_userns * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping * * Use this helper to initialize a new vfs or filesystem object based on * the caller's fsuid. A common example is initializing the i_uid field of @@ -205,14 +206,16 @@ static inline kgid_t mapped_kgid_user(struct user_namespace *mnt_userns, * * Return: the caller's current fsuid mapped up according to @mnt_userns. */ -static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) +static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns) { - return mapped_kuid_user(mnt_userns, &init_user_ns, current_fsuid()); + return mapped_kuid_user(mnt_userns, fs_userns, current_fsuid()); } /** * mapped_fsgid - return caller's fsgid mapped up into a mnt_userns * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping * * Use this helper to initialize a new vfs or filesystem object based on * the caller's fsgid. A common example is initializing the i_gid field of @@ -222,9 +225,10 @@ static inline kuid_t mapped_fsuid(struct user_namespace *mnt_userns) * * Return: the caller's current fsgid mapped up according to @mnt_userns. */ -static inline kgid_t mapped_fsgid(struct user_namespace *mnt_userns) +static inline kgid_t mapped_fsgid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns) { - return mapped_kgid_user(mnt_userns, &init_user_ns, current_fsgid()); + return mapped_kgid_user(mnt_userns, fs_userns, current_fsgid()); } #endif /* _LINUX_MNT_IDMAPPING_H */ -- cgit v1.2.3-59-g8ed1b From a1ec9040a2a9122605ac26e5725c6de019184419 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:06 +0100 Subject: fs: add i_user_ns() helper Since we'll be passing the filesystem's idmapping in even more places in the following patches and we do already dereference struct inode to get to the filesystem's idmapping multiple times add a tiny helper. Link: https://lore.kernel.org/r/20211123114227.3124056-10-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-10-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-10-brauner@kernel.org Cc: Seth Forshee Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Amir Goldstein Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- include/linux/fs.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index e1f28f757f1b..3d6d514943ab 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1600,6 +1600,11 @@ struct super_block { struct list_head s_inodes_wb; /* writeback inodes */ } __randomize_layout; +static inline struct user_namespace *i_user_ns(const struct inode *inode) +{ + return inode->i_sb->s_user_ns; +} + /* Helper functions so that in most cases filesystems will * not need to deal directly with kuid_t and kgid_t and can * instead deal with the raw numeric values that are stored @@ -1607,22 +1612,22 @@ struct super_block { */ static inline uid_t i_uid_read(const struct inode *inode) { - return from_kuid(inode->i_sb->s_user_ns, inode->i_uid); + return from_kuid(i_user_ns(inode), inode->i_uid); } static inline gid_t i_gid_read(const struct inode *inode) { - return from_kgid(inode->i_sb->s_user_ns, inode->i_gid); + return from_kgid(i_user_ns(inode), inode->i_gid); } static inline void i_uid_write(struct inode *inode, uid_t uid) { - inode->i_uid = make_kuid(inode->i_sb->s_user_ns, uid); + inode->i_uid = make_kuid(i_user_ns(inode), uid); } static inline void i_gid_write(struct inode *inode, gid_t gid) { - inode->i_gid = make_kgid(inode->i_sb->s_user_ns, gid); + inode->i_gid = make_kgid(i_user_ns(inode), gid); } /** -- cgit v1.2.3-59-g8ed1b From bd303368b776eead1c29e6cdda82bde7128b82a7 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Fri, 3 Dec 2021 12:17:07 +0100 Subject: fs: support mapped mounts of mapped filesystems In previous patches we added new and modified existing helpers to handle idmapped mounts of filesystems mounted with an idmapping. In this final patch we convert all relevant places in the vfs to actually pass the filesystem's idmapping into these helpers. With this the vfs is in shape to handle idmapped mounts of filesystems mounted with an idmapping. Note that this is just the generic infrastructure. Actually adding support for idmapped mounts to a filesystem mountable with an idmapping is follow-up work. In this patch we extend the definition of an idmapped mount from a mount that that has the initial idmapping attached to it to a mount that has an idmapping attached to it which is not the same as the idmapping the filesystem was mounted with. As before we do not allow the initial idmapping to be attached to a mount. In addition this patch prevents that the idmapping the filesystem was mounted with can be attached to a mount created based on this filesystem. This has multiple reasons and advantages. First, attaching the initial idmapping or the filesystem's idmapping doesn't make much sense as in both cases the values of the i_{g,u}id and other places where k{g,u}ids are used do not change. Second, a user that really wants to do this for whatever reason can just create a separate dedicated identical idmapping to attach to the mount. Third, we can continue to use the initial idmapping as an indicator that a mount is not idmapped allowing us to continue to keep passing the initial idmapping into the mapping helpers to tell them that something isn't an idmapped mount even if the filesystem is mounted with an idmapping. Link: https://lore.kernel.org/r/20211123114227.3124056-11-brauner@kernel.org (v1) Link: https://lore.kernel.org/r/20211130121032.3753852-11-brauner@kernel.org (v2) Link: https://lore.kernel.org/r/20211203111707.3901969-11-brauner@kernel.org Cc: Seth Forshee Cc: Amir Goldstein Cc: Christoph Hellwig Cc: Al Viro CC: linux-fsdevel@vger.kernel.org Reviewed-by: Seth Forshee Signed-off-by: Christian Brauner --- fs/namespace.c | 51 ++++++++++++++++++++++++++++++++++++++------------- fs/open.c | 7 ++++--- fs/posix_acl.c | 8 ++++---- include/linux/fs.h | 17 +++++++++-------- security/commoncap.c | 9 ++++----- 5 files changed, 59 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/fs/namespace.c b/fs/namespace.c index 4994b816a74c..08266a35c0c1 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "pnode.h" #include "internal.h" @@ -561,7 +562,7 @@ static void free_vfsmnt(struct mount *mnt) struct user_namespace *mnt_userns; mnt_userns = mnt_user_ns(&mnt->mnt); - if (mnt_userns != &init_user_ns) + if (!initial_idmapping(mnt_userns)) put_user_ns(mnt_userns); kfree_const(mnt->mnt_devname); #ifdef CONFIG_SMP @@ -965,6 +966,7 @@ static struct mount *skip_mnt_tree(struct mount *p) struct vfsmount *vfs_create_mount(struct fs_context *fc) { struct mount *mnt; + struct user_namespace *fs_userns; if (!fc->root) return ERR_PTR(-EINVAL); @@ -982,6 +984,10 @@ struct vfsmount *vfs_create_mount(struct fs_context *fc) mnt->mnt_mountpoint = mnt->mnt.mnt_root; mnt->mnt_parent = mnt; + fs_userns = mnt->mnt.mnt_sb->s_user_ns; + if (!initial_idmapping(fs_userns)) + mnt->mnt.mnt_userns = get_user_ns(fs_userns); + lock_mount_hash(); list_add_tail(&mnt->mnt_instance, &mnt->mnt.mnt_sb->s_mounts); unlock_mount_hash(); @@ -1072,7 +1078,7 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, atomic_inc(&sb->s_active); mnt->mnt.mnt_userns = mnt_user_ns(&old->mnt); - if (mnt->mnt.mnt_userns != &init_user_ns) + if (!initial_idmapping(mnt->mnt.mnt_userns)) mnt->mnt.mnt_userns = get_user_ns(mnt->mnt.mnt_userns); mnt->mnt.mnt_sb = sb; mnt->mnt.mnt_root = dget(root); @@ -3927,10 +3933,18 @@ static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt) static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) { struct vfsmount *m = &mnt->mnt; + struct user_namespace *fs_userns = m->mnt_sb->s_user_ns; if (!kattr->mnt_userns) return 0; + /* + * Creating an idmapped mount with the filesystem wide idmapping + * doesn't make sense so block that. We don't allow mushy semantics. + */ + if (kattr->mnt_userns == fs_userns) + return -EINVAL; + /* * Once a mount has been idmapped we don't allow it to change its * mapping. It makes things simpler and callers can just create @@ -3943,12 +3957,8 @@ static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) if (!(m->mnt_sb->s_type->fs_flags & FS_ALLOW_IDMAP)) return -EINVAL; - /* Don't yet support filesystem mountable in user namespaces. */ - if (m->mnt_sb->s_user_ns != &init_user_ns) - return -EINVAL; - /* We're not controlling the superblock. */ - if (!capable(CAP_SYS_ADMIN)) + if (!ns_capable(fs_userns, CAP_SYS_ADMIN)) return -EPERM; /* Mount has already been visible in the filesystem hierarchy. */ @@ -4002,14 +4012,27 @@ out: static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) { - struct user_namespace *mnt_userns; + struct user_namespace *mnt_userns, *old_mnt_userns; if (!kattr->mnt_userns) return; + /* + * We're the only ones able to change the mount's idmapping. So + * mnt->mnt.mnt_userns is stable and we can retrieve it directly. + */ + old_mnt_userns = mnt->mnt.mnt_userns; + mnt_userns = get_user_ns(kattr->mnt_userns); /* Pairs with smp_load_acquire() in mnt_user_ns(). */ smp_store_release(&mnt->mnt.mnt_userns, mnt_userns); + + /* + * If this is an idmapped filesystem drop the reference we've taken + * in vfs_create_mount() before. + */ + if (!initial_idmapping(old_mnt_userns)) + put_user_ns(old_mnt_userns); } static void mount_setattr_commit(struct mount_kattr *kattr, @@ -4133,13 +4156,15 @@ static int build_mount_idmapped(const struct mount_attr *attr, size_t usize, } /* - * The init_user_ns is used to indicate that a vfsmount is not idmapped. - * This is simpler than just having to treat NULL as unmapped. Users - * wanting to idmap a mount to init_user_ns can just use a namespace - * with an identity mapping. + * The initial idmapping cannot be used to create an idmapped + * mount. We use the initial idmapping as an indicator of a mount + * that is not idmapped. It can simply be passed into helpers that + * are aware of idmapped mounts as a convenient shortcut. A user + * can just create a dedicated identity mapping to achieve the same + * result. */ mnt_userns = container_of(ns, struct user_namespace, ns); - if (mnt_userns == &init_user_ns) { + if (initial_idmapping(mnt_userns)) { err = -EPERM; goto out_fput; } diff --git a/fs/open.c b/fs/open.c index 40a00e71865b..9ff2f621b760 100644 --- a/fs/open.c +++ b/fs/open.c @@ -641,7 +641,7 @@ SYSCALL_DEFINE2(chmod, const char __user *, filename, umode_t, mode) int chown_common(const struct path *path, uid_t user, gid_t group) { - struct user_namespace *mnt_userns; + struct user_namespace *mnt_userns, *fs_userns; struct inode *inode = path->dentry->d_inode; struct inode *delegated_inode = NULL; int error; @@ -653,8 +653,9 @@ int chown_common(const struct path *path, uid_t user, gid_t group) gid = make_kgid(current_user_ns(), group); mnt_userns = mnt_user_ns(path->mnt); - uid = mapped_kuid_user(mnt_userns, &init_user_ns, uid); - gid = mapped_kgid_user(mnt_userns, &init_user_ns, gid); + fs_userns = i_user_ns(inode); + uid = mapped_kuid_user(mnt_userns, fs_userns, uid); + gid = mapped_kgid_user(mnt_userns, fs_userns, gid); retry_deleg: newattrs.ia_valid = ATTR_CTIME; diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 4b5fb9a9b90f..80acb6885cf9 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -376,8 +376,8 @@ posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, break; case ACL_USER: uid = mapped_kuid_fs(mnt_userns, - &init_user_ns, - pa->e_uid); + i_user_ns(inode), + pa->e_uid); if (uid_eq(uid, current_fsuid())) goto mask; break; @@ -391,8 +391,8 @@ posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, break; case ACL_GROUP: gid = mapped_kgid_fs(mnt_userns, - &init_user_ns, - pa->e_gid); + i_user_ns(inode), + pa->e_gid); if (in_group_p(gid)) { found = 1; if ((pa->e_perm & want) == want) diff --git a/include/linux/fs.h b/include/linux/fs.h index 3d6d514943ab..493b87e3616b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1641,7 +1641,7 @@ static inline void i_gid_write(struct inode *inode, gid_t gid) static inline kuid_t i_uid_into_mnt(struct user_namespace *mnt_userns, const struct inode *inode) { - return mapped_kuid_fs(mnt_userns, &init_user_ns, inode->i_uid); + return mapped_kuid_fs(mnt_userns, i_user_ns(inode), inode->i_uid); } /** @@ -1655,7 +1655,7 @@ static inline kuid_t i_uid_into_mnt(struct user_namespace *mnt_userns, static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns, const struct inode *inode) { - return mapped_kgid_fs(mnt_userns, &init_user_ns, inode->i_gid); + return mapped_kgid_fs(mnt_userns, i_user_ns(inode), inode->i_gid); } /** @@ -1669,7 +1669,7 @@ static inline kgid_t i_gid_into_mnt(struct user_namespace *mnt_userns, static inline void inode_fsuid_set(struct inode *inode, struct user_namespace *mnt_userns) { - inode->i_uid = mapped_fsuid(mnt_userns, &init_user_ns); + inode->i_uid = mapped_fsuid(mnt_userns, i_user_ns(inode)); } /** @@ -1683,7 +1683,7 @@ static inline void inode_fsuid_set(struct inode *inode, static inline void inode_fsgid_set(struct inode *inode, struct user_namespace *mnt_userns) { - inode->i_gid = mapped_fsgid(mnt_userns, &init_user_ns); + inode->i_gid = mapped_fsgid(mnt_userns, i_user_ns(inode)); } /** @@ -1704,10 +1704,10 @@ static inline bool fsuidgid_has_mapping(struct super_block *sb, kuid_t kuid; kgid_t kgid; - kuid = mapped_fsuid(mnt_userns, &init_user_ns); + kuid = mapped_fsuid(mnt_userns, fs_userns); if (!uid_valid(kuid)) return false; - kgid = mapped_fsgid(mnt_userns, &init_user_ns); + kgid = mapped_fsgid(mnt_userns, fs_userns); if (!gid_valid(kgid)) return false; return kuid_has_mapping(fs_userns, kuid) && @@ -2653,13 +2653,14 @@ static inline struct user_namespace *file_mnt_user_ns(struct file *file) * is_idmapped_mnt - check whether a mount is mapped * @mnt: the mount to check * - * If @mnt has an idmapping attached to it @mnt is mapped. + * If @mnt has an idmapping attached different from the + * filesystem's idmapping then @mnt is mapped. * * Return: true if mount is mapped, false if not. */ static inline bool is_idmapped_mnt(const struct vfsmount *mnt) { - return mnt_user_ns(mnt) != &init_user_ns; + return mnt_user_ns(mnt) != mnt->mnt_sb->s_user_ns; } extern long vfs_truncate(const struct path *, loff_t); diff --git a/security/commoncap.c b/security/commoncap.c index d288a62e2999..5fc8986c3c77 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -419,7 +419,7 @@ int cap_inode_getsecurity(struct user_namespace *mnt_userns, kroot = make_kuid(fs_ns, root); /* If this is an idmapped mount shift the kuid. */ - kroot = mapped_kuid_fs(mnt_userns, &init_user_ns, kroot); + kroot = mapped_kuid_fs(mnt_userns, fs_ns, kroot); /* If the root kuid maps to a valid uid in current ns, then return * this as a nscap. */ @@ -556,13 +556,12 @@ int cap_convert_nscap(struct user_namespace *mnt_userns, struct dentry *dentry, return -EINVAL; if (!capable_wrt_inode_uidgid(mnt_userns, inode, CAP_SETFCAP)) return -EPERM; - if (size == XATTR_CAPS_SZ_2 && (mnt_userns == &init_user_ns)) + if (size == XATTR_CAPS_SZ_2 && (mnt_userns == fs_ns)) if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP)) /* user is privileged, just write the v2 */ return size; - rootid = rootid_from_xattr(*ivalue, size, task_ns, mnt_userns, - &init_user_ns); + rootid = rootid_from_xattr(*ivalue, size, task_ns, mnt_userns, fs_ns); if (!uid_valid(rootid)) return -EINVAL; @@ -703,7 +702,7 @@ int get_vfs_caps_from_disk(struct user_namespace *mnt_userns, /* Limit the caps to the mounter of the filesystem * or the more limited uid specified in the xattr. */ - rootkuid = mapped_kuid_fs(mnt_userns, &init_user_ns, rootkuid); + rootkuid = mapped_kuid_fs(mnt_userns, fs_ns, rootkuid); if (!rootid_owns_currentns(rootkuid)) return -ENODATA; -- cgit v1.2.3-59-g8ed1b From 019cd8a9e3bcbbf6bac8036a9ae545f7858e0c08 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 2 Nov 2021 16:02:01 -0600 Subject: ARM: ixp4xx: remove unused header file pata_ixp4xx_cf.h Commit b00ced38e317 ("ARM: ixp4xx: Delete Avila boardfiles") removed the last use of but left the header file in place. Nothing uses this file, delete it now. Cc: Linus Walleij Cc: Arnd Bergmann Signed-off-by: Jonathan Corbet Acked-by: Arnd Bergmann Signed-off-by: Linus Walleij --- include/linux/platform_data/pata_ixp4xx_cf.h | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 include/linux/platform_data/pata_ixp4xx_cf.h (limited to 'include/linux') diff --git a/include/linux/platform_data/pata_ixp4xx_cf.h b/include/linux/platform_data/pata_ixp4xx_cf.h deleted file mode 100644 index e60fa41da4a5..000000000000 --- a/include/linux/platform_data/pata_ixp4xx_cf.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __PLATFORM_DATA_PATA_IXP4XX_H -#define __PLATFORM_DATA_PATA_IXP4XX_H - -#include - -/* - * This structure provide a means for the board setup code - * to give information to th pata_ixp4xx driver. It is - * passed as platform_data. - */ -struct ixp4xx_pata_data { - volatile u32 *cs0_cfg; - volatile u32 *cs1_cfg; - unsigned long cs0_bits; - unsigned long cs1_bits; - void __iomem *cmd; - void __iomem *ctl; -}; - -#endif -- cgit v1.2.3-59-g8ed1b From 52f982f00b220d097a71a23c149a1d18efc08e63 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Mon, 6 Dec 2021 14:24:06 +0100 Subject: security,selinux: remove security_add_mnt_opt() Its last user has been removed in commit f2aedb713c28 ("NFS: Add fs_context support."). Signed-off-by: Ondrej Mosnacek Reviewed-by: Casey Schaufler Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 2 -- include/linux/lsm_hooks.h | 2 -- include/linux/security.h | 8 -------- security/security.c | 8 -------- security/selinux/hooks.c | 39 --------------------------------------- 5 files changed, 59 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index ae2228f0711d..a5a724c308d8 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -78,8 +78,6 @@ LSM_HOOK(int, 0, sb_set_mnt_opts, struct super_block *sb, void *mnt_opts, LSM_HOOK(int, 0, sb_clone_mnt_opts, const struct super_block *oldsb, struct super_block *newsb, unsigned long kern_flags, unsigned long *set_kern_flags) -LSM_HOOK(int, 0, sb_add_mnt_opt, const char *option, const char *val, - int len, void **mnt_opts) LSM_HOOK(int, 0, move_mount, const struct path *from_path, const struct path *to_path) LSM_HOOK(int, 0, dentry_init_security, struct dentry *dentry, diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 52c1990644b9..3bf5c658bc44 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -180,8 +180,6 @@ * Copy all security options from a given superblock to another * @oldsb old superblock which contain information to clone * @newsb new superblock which needs filled in - * @sb_add_mnt_opt: - * Add one mount @option to @mnt_opts. * @sb_parse_opts_str: * Parse a string of security data filling in the opts structure * @options string containing all mount options known by the LSM diff --git a/include/linux/security.h b/include/linux/security.h index bb301963e333..6d72772182c8 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -313,8 +313,6 @@ int security_sb_clone_mnt_opts(const struct super_block *oldsb, struct super_block *newsb, unsigned long kern_flags, unsigned long *set_kern_flags); -int security_add_mnt_opt(const char *option, const char *val, - int len, void **mnt_opts); int security_move_mount(const struct path *from_path, const struct path *to_path); int security_dentry_init_security(struct dentry *dentry, int mode, const struct qstr *name, @@ -711,12 +709,6 @@ static inline int security_sb_clone_mnt_opts(const struct super_block *oldsb, return 0; } -static inline int security_add_mnt_opt(const char *option, const char *val, - int len, void **mnt_opts) -{ - return 0; -} - static inline int security_move_mount(const struct path *from_path, const struct path *to_path) { diff --git a/security/security.c b/security/security.c index edb922b8bf4a..3d4eb474f35b 100644 --- a/security/security.c +++ b/security/security.c @@ -994,14 +994,6 @@ int security_sb_clone_mnt_opts(const struct super_block *oldsb, } EXPORT_SYMBOL(security_sb_clone_mnt_opts); -int security_add_mnt_opt(const char *option, const char *val, int len, - void **mnt_opts) -{ - return call_int_hook(sb_add_mnt_opt, -EINVAL, - option, val, len, mnt_opts); -} -EXPORT_SYMBOL(security_add_mnt_opt); - int security_move_mount(const struct path *from_path, const struct path *to_path) { return call_int_hook(move_mount, 0, from_path, to_path); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 726175254f60..818ce976ff6c 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1010,44 +1010,6 @@ Einval: return -EINVAL; } -static int selinux_add_mnt_opt(const char *option, const char *val, int len, - void **mnt_opts) -{ - int token = Opt_error; - int rc, i; - - for (i = 0; i < ARRAY_SIZE(tokens); i++) { - if (strcmp(option, tokens[i].name) == 0) { - token = tokens[i].opt; - break; - } - } - - if (token == Opt_error) - return -EINVAL; - - if (token != Opt_seclabel) { - val = kmemdup_nul(val, len, GFP_KERNEL); - if (!val) { - rc = -ENOMEM; - goto free_opt; - } - } - rc = selinux_add_opt(token, val, mnt_opts); - if (unlikely(rc)) { - kfree(val); - goto free_opt; - } - return rc; - -free_opt: - if (*mnt_opts) { - selinux_free_mnt_opts(*mnt_opts); - *mnt_opts = NULL; - } - return rc; -} - static int show_sid(struct seq_file *m, u32 sid) { char *context = NULL; @@ -7285,7 +7247,6 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(fs_context_dup, selinux_fs_context_dup), LSM_HOOK_INIT(fs_context_parse_param, selinux_fs_context_parse_param), LSM_HOOK_INIT(sb_eat_lsm_opts, selinux_sb_eat_lsm_opts), - LSM_HOOK_INIT(sb_add_mnt_opt, selinux_add_mnt_opt), #ifdef CONFIG_SECURITY_NETWORK_XFRM LSM_HOOK_INIT(xfrm_policy_clone_security, selinux_xfrm_policy_clone), #endif -- cgit v1.2.3-59-g8ed1b From 8ab30a331946c34e4ba022c44df8624acea1c74e Mon Sep 17 00:00:00 2001 From: John Garry Date: Mon, 6 Dec 2021 20:49:48 +0800 Subject: blk-mq: Drop busy_iter_fn blk_mq_hw_ctx argument The only user of blk_mq_hw_ctx blk_mq_hw_ctx argument is blk_mq_rq_inflight(). Function blk_mq_rq_inflight() uses the hctx to find the associated request queue to match against the request. However this same check is already done in caller bt_iter(), so drop this check. With that change there are no more users of busy_iter_fn blk_mq_hw_ctx argument, so drop the argument. Reviewed-by Hannes Reinecke Signed-off-by: John Garry Reviewed-by: Ming Lei Tested-by: Kashyap Desai Link: https://lore.kernel.org/r/1638794990-137490-2-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 2 +- block/blk-mq.c | 17 ++++++++--------- include/linux/blk-mq.h | 3 +-- 3 files changed, 10 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 380e2dd31bfc..d3cf91d764d5 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -254,7 +254,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) return true; if (rq->q == hctx->queue && rq->mq_hctx == hctx) - ret = iter_data->fn(hctx, rq, iter_data->data, reserved); + ret = iter_data->fn(rq, iter_data->data, reserved); blk_mq_put_rq_ref(rq); return ret; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 0bf3523dd1f5..103c0f58853c 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -127,8 +127,7 @@ struct mq_inflight { unsigned int inflight[2]; }; -static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, - struct request *rq, void *priv, +static bool blk_mq_check_inflight(struct request *rq, void *priv, bool reserved) { struct mq_inflight *mi = priv; @@ -1308,14 +1307,15 @@ void blk_mq_delay_kick_requeue_list(struct request_queue *q, } EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); -static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq, - void *priv, bool reserved) +static bool blk_mq_rq_inflight(struct request *rq, void *priv, + bool reserved) { /* - * If we find a request that isn't idle and the queue matches, - * we know the queue is busy. Return false to stop the iteration. + * If we find a request that isn't idle we know the queue is busy + * as it's checked in the iter. + * Return false to stop the iteration. */ - if (blk_mq_request_started(rq) && rq->q == hctx->queue) { + if (blk_mq_request_started(rq)) { bool *busy = priv; *busy = true; @@ -1377,8 +1377,7 @@ void blk_mq_put_rq_ref(struct request *rq) __blk_mq_free_request(rq); } -static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, - struct request *rq, void *priv, bool reserved) +static bool blk_mq_check_expired(struct request *rq, void *priv, bool reserved) { unsigned long *next = priv; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index ecdc049b52fa..17ebf29e42d8 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -470,8 +470,7 @@ struct blk_mq_queue_data { bool last; }; -typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, - bool); +typedef bool (busy_iter_fn)(struct request *, void *, bool); typedef bool (busy_tag_iter_fn)(struct request *, void *, bool); /** -- cgit v1.2.3-59-g8ed1b From fc39f8d2d1c10ac04976b0a247865bb0cec4dd88 Mon Sep 17 00:00:00 2001 From: John Garry Date: Mon, 6 Dec 2021 20:49:49 +0800 Subject: blk-mq: Delete busy_iter_fn Typedefs busy_iter_fn and busy_tag_iter_fn are now identical, so delete busy_iter_fn to reduce duplication. It would be nicer to delete busy_tag_iter_fn, as the name busy_iter_fn is less specific. However busy_tag_iter_fn is used in many different parts of the tree, unlike busy_iter_fn which is just use in block/, so just take the straightforward path now, so that we could rename later treewide. Signed-off-by: John Garry Reviewed-by: Ming Lei Reviewed-by: Hannes Reinecke Tested-by: Kashyap Desai Link: https://lore.kernel.org/r/1638794990-137490-3-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 6 +++--- block/blk-mq-tag.h | 2 +- include/linux/blk-mq.h | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d3cf91d764d5..58b80d4b7a07 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -215,7 +215,7 @@ void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags) struct bt_iter_data { struct blk_mq_hw_ctx *hctx; - busy_iter_fn *fn; + busy_tag_iter_fn *fn; void *data; bool reserved; }; @@ -274,7 +274,7 @@ static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) * bitmap_tags member of struct blk_mq_tags. */ static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt, - busy_iter_fn *fn, void *data, bool reserved) + busy_tag_iter_fn *fn, void *data, bool reserved) { struct bt_iter_data iter_data = { .hctx = hctx, @@ -457,7 +457,7 @@ EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request); * called for all requests on all queues that share that tag set and not only * for requests associated with @q. */ -void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, +void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, void *priv) { struct blk_mq_hw_ctx *hctx; diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index df787b5a23bd..5668e28be0b7 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -28,7 +28,7 @@ extern void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, extern void blk_mq_tag_update_sched_shared_tags(struct request_queue *q); extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool); -void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, +void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, void *priv); void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, void *priv); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 17ebf29e42d8..772f8f921526 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -470,7 +470,6 @@ struct blk_mq_queue_data { bool last; }; -typedef bool (busy_iter_fn)(struct request *, void *, bool); typedef bool (busy_tag_iter_fn)(struct request *, void *, bool); /** -- cgit v1.2.3-59-g8ed1b From 05770dd0ad110854c7157d95700d7c89979cdb3e Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Mon, 22 Nov 2021 18:30:12 +0900 Subject: tracing: Support __rel_loc relative dynamic data location attribute Add '__rel_loc' new dynamic data location attribute which encodes the data location from the next to the field itself. The '__data_loc' is used for encoding the dynamic data location on the trace event record. But '__data_loc' is not useful if the writer doesn't know the event header (e.g. user event), because it records the dynamic data offset from the entry of the record, not the field itself. This new '__rel_loc' attribute encodes the data location relatively from the next of the field. For example, when there is a record like below (the number in the parentheses is the size of fields) |header(N)|common(M)|fields(K)|__data_loc(4)|fields(L)|data(G)| In this case, '__data_loc' field will be __data_loc = (G << 16) | (N+M+K+4+L) If '__rel_loc' is used, this will be |header(N)|common(M)|fields(K)|__rel_loc(4)|fields(L)|data(G)| where __rel_loc = (G << 16) | (L) This case shows L bytes after the '__rel_loc' attribute field, if there is no fields after the __rel_loc field, L must be 0. This is relatively easy (and no need to consider the kernel header change) when the event data fields are composed by user who doesn't know header and common fields. Link: https://lkml.kernel.org/r/163757341258.510314.4214431827833229956.stgit@devnote2 Cc: Beau Belgrave Cc: Namhyung Kim Cc: Tom Zanussi Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- include/linux/trace_events.h | 1 + kernel/trace/trace.h | 1 + kernel/trace/trace_events_filter.c | 32 ++++++++++++++++++++++++++++++-- kernel/trace/trace_events_hist.c | 21 +++++++++++++++++++-- kernel/trace/trace_events_inject.c | 11 +++++++++-- 5 files changed, 60 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 2d167ac3452c..3900404aa063 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -782,6 +782,7 @@ enum { FILTER_OTHER = 0, FILTER_STATIC_STRING, FILTER_DYN_STRING, + FILTER_RDYN_STRING, FILTER_PTR_STRING, FILTER_TRACE_FN, FILTER_COMM, diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 38715aa6cfdf..5db2bec8ca7e 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1465,6 +1465,7 @@ struct filter_pred { static inline bool is_string_field(struct ftrace_event_field *field) { return field->filter_type == FILTER_DYN_STRING || + field->filter_type == FILTER_RDYN_STRING || field->filter_type == FILTER_STATIC_STRING || field->filter_type == FILTER_PTR_STRING || field->filter_type == FILTER_COMM; diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index c9124038b140..996920ed1812 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -706,6 +706,29 @@ static int filter_pred_strloc(struct filter_pred *pred, void *event) return match; } +/* + * Filter predicate for relative dynamic sized arrays of characters. + * These are implemented through a list of strings at the end + * of the entry as same as dynamic string. + * The difference is that the relative one records the location offset + * from the field itself, not the event entry. + */ +static int filter_pred_strrelloc(struct filter_pred *pred, void *event) +{ + u32 *item = (u32 *)(event + pred->offset); + u32 str_item = *item; + int str_loc = str_item & 0xffff; + int str_len = str_item >> 16; + char *addr = (char *)(&item[1]) + str_loc; + int cmp, match; + + cmp = pred->regex.match(addr, &pred->regex, str_len); + + match = cmp ^ pred->not; + + return match; +} + /* Filter predicate for CPUs. */ static int filter_pred_cpu(struct filter_pred *pred, void *event) { @@ -756,7 +779,7 @@ static int filter_pred_none(struct filter_pred *pred, void *event) * * Note: * - @str might not be NULL-terminated if it's of type DYN_STRING - * or STATIC_STRING, unless @len is zero. + * RDYN_STRING, or STATIC_STRING, unless @len is zero. */ static int regex_match_full(char *str, struct regex *r, int len) @@ -1083,6 +1106,9 @@ int filter_assign_type(const char *type) if (strstr(type, "__data_loc") && strstr(type, "char")) return FILTER_DYN_STRING; + if (strstr(type, "__rel_loc") && strstr(type, "char")) + return FILTER_RDYN_STRING; + if (strchr(type, '[') && strstr(type, "char")) return FILTER_STATIC_STRING; @@ -1318,8 +1344,10 @@ static int parse_pred(const char *str, void *data, pred->fn = filter_pred_string; pred->regex.field_len = field->size; - } else if (field->filter_type == FILTER_DYN_STRING) + } else if (field->filter_type == FILTER_DYN_STRING) { pred->fn = filter_pred_strloc; + } else if (field->filter_type == FILTER_RDYN_STRING) + pred->fn = filter_pred_strrelloc; else pred->fn = filter_pred_pchar; /* go past the last quote */ diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 319f9c8ca7e7..9b8da439149c 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -217,6 +217,20 @@ static u64 hist_field_dynstring(struct hist_field *hist_field, return (u64)(unsigned long)addr; } +static u64 hist_field_reldynstring(struct hist_field *hist_field, + struct tracing_map_elt *elt, + struct trace_buffer *buffer, + struct ring_buffer_event *rbe, + void *event) +{ + u32 *item = event + hist_field->field->offset; + u32 str_item = *item; + int str_loc = str_item & 0xffff; + char *addr = (char *)&item[1] + str_loc; + + return (u64)(unsigned long)addr; +} + static u64 hist_field_pstring(struct hist_field *hist_field, struct tracing_map_elt *elt, struct trace_buffer *buffer, @@ -1956,8 +1970,10 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, if (field->filter_type == FILTER_STATIC_STRING) { hist_field->fn = hist_field_string; hist_field->size = field->size; - } else if (field->filter_type == FILTER_DYN_STRING) + } else if (field->filter_type == FILTER_DYN_STRING) { hist_field->fn = hist_field_dynstring; + } else if (field->filter_type == FILTER_RDYN_STRING) + hist_field->fn = hist_field_reldynstring; else hist_field->fn = hist_field_pstring; } else { @@ -4961,7 +4977,8 @@ static inline void add_to_key(char *compound_key, void *key, struct ftrace_event_field *field; field = key_field->field; - if (field->filter_type == FILTER_DYN_STRING) + if (field->filter_type == FILTER_DYN_STRING || + field->filter_type == FILTER_RDYN_STRING) size = *(u32 *)(rec + field->offset) >> 16; else if (field->filter_type == FILTER_STATIC_STRING) size = field->size; diff --git a/kernel/trace/trace_events_inject.c b/kernel/trace/trace_events_inject.c index c188045c5f97..d6b4935a78c0 100644 --- a/kernel/trace/trace_events_inject.c +++ b/kernel/trace/trace_events_inject.c @@ -168,10 +168,14 @@ static void *trace_alloc_entry(struct trace_event_call *call, int *size) continue; if (field->filter_type == FILTER_STATIC_STRING) continue; - if (field->filter_type == FILTER_DYN_STRING) { + if (field->filter_type == FILTER_DYN_STRING || + field->filter_type == FILTER_RDYN_STRING) { u32 *str_item; int str_loc = entry_size & 0xffff; + if (field->filter_type == FILTER_RDYN_STRING) + str_loc -= field->offset + field->size; + str_item = (u32 *)(entry + field->offset); *str_item = str_loc; /* string length is 0. */ } else { @@ -214,7 +218,8 @@ static int parse_entry(char *str, struct trace_event_call *call, void **pentry) if (field->filter_type == FILTER_STATIC_STRING) { strlcpy(entry + field->offset, addr, field->size); - } else if (field->filter_type == FILTER_DYN_STRING) { + } else if (field->filter_type == FILTER_DYN_STRING || + field->filter_type == FILTER_RDYN_STRING) { int str_len = strlen(addr) + 1; int str_loc = entry_size & 0xffff; u32 *str_item; @@ -229,6 +234,8 @@ static int parse_entry(char *str, struct trace_event_call *call, void **pentry) strlcpy(entry + (entry_size - str_len), addr, str_len); str_item = (u32 *)(entry + field->offset); + if (field->filter_type == FILTER_RDYN_STRING) + str_loc -= field->offset + field->size; *str_item = (str_len << 16) | str_loc; } else { char **paddr; -- cgit v1.2.3-59-g8ed1b From 8c33915d77a565b8b5d44e6368e22b6ea300b7a8 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 28 Nov 2021 20:00:29 +0100 Subject: platform/x86: wmi: Add no_notify_data flag to struct wmi_driver Some WMI implementations do notifies on WMI objects without a _WED method allow WMI drivers to indicate that _WED should not be called for notifies on the WMI objects the driver is bound to. Instead the driver's notify callback will simply be called with a NULL data argument. Reported-by: Yauhen Kharuzhy Reviewed-by: Andy Shevchenko Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20211128190031.405620-3-hdegoede@redhat.com --- drivers/platform/x86/wmi.c | 10 ++++++---- include/linux/wmi.h | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c index 02aba274c4bc..58a23a9adbef 100644 --- a/drivers/platform/x86/wmi.c +++ b/drivers/platform/x86/wmi.c @@ -1331,10 +1331,12 @@ static void acpi_wmi_notify_handler(acpi_handle handle, u32 event, struct acpi_buffer evdata = { ACPI_ALLOCATE_BUFFER, NULL }; acpi_status status; - status = get_event_data(wblock, &evdata); - if (ACPI_FAILURE(status)) { - dev_warn(&wblock->dev.dev, "failed to get event data\n"); - return; + if (!driver->no_notify_data) { + status = get_event_data(wblock, &evdata); + if (ACPI_FAILURE(status)) { + dev_warn(&wblock->dev.dev, "failed to get event data\n"); + return; + } } if (driver->notify) diff --git a/include/linux/wmi.h b/include/linux/wmi.h index 2cb3913c1f50..b88d7b58e61e 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -35,6 +35,7 @@ extern int set_required_buffer_size(struct wmi_device *wdev, u64 length); struct wmi_driver { struct device_driver driver; const struct wmi_device_id *id_table; + bool no_notify_data; int (*probe)(struct wmi_device *wdev, const void *context); void (*remove)(struct wmi_device *wdev); -- cgit v1.2.3-59-g8ed1b From 4e66934eaadc83b27ada8d42b60894018f3bfabf Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:21:55 -0800 Subject: lib: add reference counting tracking infrastructure It can be hard to track where references are taken and released. In networking, we have annoying issues at device or netns dismantles, and we had various proposals to ease root causing them. This patch adds new infrastructure pairing refcount increases and decreases. This will self document code, because programmers will have to associate increments/decrements. This is controled by CONFIG_REF_TRACKER which can be selected by users of this feature. This adds both cpu and memory costs, and thus should probably be used with care. Signed-off-by: Eric Dumazet Reviewed-by: Dmitry Vyukov Signed-off-by: Jakub Kicinski --- include/linux/ref_tracker.h | 73 +++++++++++++++++++++++ lib/Kconfig | 5 ++ lib/Makefile | 2 + lib/ref_tracker.c | 140 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 220 insertions(+) create mode 100644 include/linux/ref_tracker.h create mode 100644 lib/ref_tracker.c (limited to 'include/linux') diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h new file mode 100644 index 000000000000..c11c9db5825c --- /dev/null +++ b/include/linux/ref_tracker.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#ifndef _LINUX_REF_TRACKER_H +#define _LINUX_REF_TRACKER_H +#include +#include +#include + +struct ref_tracker; + +struct ref_tracker_dir { +#ifdef CONFIG_REF_TRACKER + spinlock_t lock; + unsigned int quarantine_avail; + refcount_t untracked; + struct list_head list; /* List of active trackers */ + struct list_head quarantine; /* List of dead trackers */ +#endif +}; + +#ifdef CONFIG_REF_TRACKER +static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, + unsigned int quarantine_count) +{ + INIT_LIST_HEAD(&dir->list); + INIT_LIST_HEAD(&dir->quarantine); + spin_lock_init(&dir->lock); + dir->quarantine_avail = quarantine_count; + refcount_set(&dir->untracked, 1); +} + +void ref_tracker_dir_exit(struct ref_tracker_dir *dir); + +void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit); + +int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, gfp_t gfp); + +int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp); + +#else /* CONFIG_REF_TRACKER */ + +static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, + unsigned int quarantine_count) +{ +} + +static inline void ref_tracker_dir_exit(struct ref_tracker_dir *dir) +{ +} + +static inline void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit) +{ +} + +static inline int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, + gfp_t gfp) +{ + return 0; +} + +static inline int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp) +{ + return 0; +} + +#endif + +#endif /* _LINUX_REF_TRACKER_H */ diff --git a/lib/Kconfig b/lib/Kconfig index 5e7165e6a346..655b0e43f260 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -680,6 +680,11 @@ config STACK_HASH_ORDER Select the hash size as a power of 2 for the stackdepot hash table. Choose a lower value to reduce the memory impact. +config REF_TRACKER + bool + depends on STACKTRACE_SUPPORT + select STACKDEPOT + config SBITMAP bool diff --git a/lib/Makefile b/lib/Makefile index 364c23f15578..c1fd9243ddb9 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -270,6 +270,8 @@ obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n +obj-$(CONFIG_REF_TRACKER) += ref_tracker.o + libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ fdt_empty_tree.o fdt_addresses.o $(foreach file, $(libfdt_files), \ diff --git a/lib/ref_tracker.c b/lib/ref_tracker.c new file mode 100644 index 000000000000..0ae2e66dcf0f --- /dev/null +++ b/lib/ref_tracker.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include +#include +#include +#include + +#define REF_TRACKER_STACK_ENTRIES 16 + +struct ref_tracker { + struct list_head head; /* anchor into dir->list or dir->quarantine */ + bool dead; + depot_stack_handle_t alloc_stack_handle; + depot_stack_handle_t free_stack_handle; +}; + +void ref_tracker_dir_exit(struct ref_tracker_dir *dir) +{ + struct ref_tracker *tracker, *n; + unsigned long flags; + bool leak = false; + + spin_lock_irqsave(&dir->lock, flags); + list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { + list_del(&tracker->head); + kfree(tracker); + dir->quarantine_avail++; + } + list_for_each_entry_safe(tracker, n, &dir->list, head) { + pr_err("leaked reference.\n"); + if (tracker->alloc_stack_handle) + stack_depot_print(tracker->alloc_stack_handle); + leak = true; + list_del(&tracker->head); + kfree(tracker); + } + spin_unlock_irqrestore(&dir->lock, flags); + WARN_ON_ONCE(leak); + WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); +} +EXPORT_SYMBOL(ref_tracker_dir_exit); + +void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit) +{ + struct ref_tracker *tracker; + unsigned long flags; + unsigned int i = 0; + + spin_lock_irqsave(&dir->lock, flags); + list_for_each_entry(tracker, &dir->list, head) { + if (i < display_limit) { + pr_err("leaked reference.\n"); + if (tracker->alloc_stack_handle) + stack_depot_print(tracker->alloc_stack_handle); + i++; + } else { + break; + } + } + spin_unlock_irqrestore(&dir->lock, flags); +} +EXPORT_SYMBOL(ref_tracker_dir_print); + +int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, + gfp_t gfp) +{ + unsigned long entries[REF_TRACKER_STACK_ENTRIES]; + struct ref_tracker *tracker; + unsigned int nr_entries; + unsigned long flags; + + *trackerp = tracker = kzalloc(sizeof(*tracker), gfp | __GFP_NOFAIL); + if (unlikely(!tracker)) { + pr_err_once("memory allocation failure, unreliable refcount tracker.\n"); + refcount_inc(&dir->untracked); + return -ENOMEM; + } + nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + nr_entries = filter_irq_stacks(entries, nr_entries); + tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); + + spin_lock_irqsave(&dir->lock, flags); + list_add(&tracker->head, &dir->list); + spin_unlock_irqrestore(&dir->lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(ref_tracker_alloc); + +int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp) +{ + unsigned long entries[REF_TRACKER_STACK_ENTRIES]; + struct ref_tracker *tracker = *trackerp; + depot_stack_handle_t stack_handle; + unsigned int nr_entries; + unsigned long flags; + + if (!tracker) { + refcount_dec(&dir->untracked); + return -EEXIST; + } + nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + nr_entries = filter_irq_stacks(entries, nr_entries); + stack_handle = stack_depot_save(entries, nr_entries, GFP_ATOMIC); + + spin_lock_irqsave(&dir->lock, flags); + if (tracker->dead) { + pr_err("reference already released.\n"); + if (tracker->alloc_stack_handle) { + pr_err("allocated in:\n"); + stack_depot_print(tracker->alloc_stack_handle); + } + if (tracker->free_stack_handle) { + pr_err("freed in:\n"); + stack_depot_print(tracker->free_stack_handle); + } + spin_unlock_irqrestore(&dir->lock, flags); + WARN_ON_ONCE(1); + return -EINVAL; + } + tracker->dead = true; + + tracker->free_stack_handle = stack_handle; + + list_move_tail(&tracker->head, &dir->quarantine); + if (!dir->quarantine_avail) { + tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); + list_del(&tracker->head); + } else { + dir->quarantine_avail--; + tracker = NULL; + } + spin_unlock_irqrestore(&dir->lock, flags); + + kfree(tracker); + return 0; +} +EXPORT_SYMBOL_GPL(ref_tracker_free); -- cgit v1.2.3-59-g8ed1b From 4d92b95ff2f95f13df9bad0b5a25a9f60e72758d Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:21:57 -0800 Subject: net: add net device refcount tracker infrastructure net device are refcounted. Over the years we had numerous bugs caused by imbalanced dev_hold() and dev_put() calls. The general idea is to be able to precisely pair each decrement with a corresponding prior increment. Both share a cookie, basically a pointer to private data storing stack traces. This patch adds dev_hold_track() and dev_put_track(). To use these helpers, each data structure owning a refcount should also use a "netdevice_tracker" to pair the hold and put. netdevice_tracker dev_tracker; ... dev_hold_track(dev, &dev_tracker, GFP_ATOMIC); ... dev_put_track(dev, &dev_tracker); Whenever a leak happens, we will get precise stack traces of the point dev_hold_track() happened, at device dismantle phase. We will also get a stack trace if too many dev_put_track() for the same netdevice_tracker are attempted. This is guarded by CONFIG_NET_DEV_REFCNT_TRACKER option. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ lib/Kconfig.debug | 5 +++++ net/Kconfig.debug | 10 ++++++++++ net/core/dev.c | 3 +++ 4 files changed, 63 insertions(+) create mode 100644 net/Kconfig.debug (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 65117f01d5f2..143d60ed0047 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -48,6 +48,7 @@ #include #include #include +#include struct netpoll_info; struct device; @@ -300,6 +301,12 @@ enum netdev_state_t { }; +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +typedef struct ref_tracker *netdevice_tracker; +#else +typedef struct {} netdevice_tracker; +#endif + struct gro_list { struct list_head list; int count; @@ -1865,6 +1872,7 @@ enum netdev_ml_priv_type { * @proto_down_reason: reason a netdev interface is held down * @pcpu_refcnt: Number of references to this device * @dev_refcnt: Number of references to this device + * @refcnt_tracker: Tracker directory for tracked references to this device * @todo_list: Delayed register/unregister * @link_watch_list: XXX: need comments on this one * @@ -2178,6 +2186,7 @@ struct net_device { #else refcount_t dev_refcnt; #endif + struct ref_tracker_dir refcnt_tracker; struct list_head link_watch_list; @@ -3805,6 +3814,7 @@ void netdev_run_todo(void); * @dev: network device * * Release reference to device to allow it to be freed. + * Try using dev_put_track() instead. */ static inline void dev_put(struct net_device *dev) { @@ -3822,6 +3832,7 @@ static inline void dev_put(struct net_device *dev) * @dev: network device * * Hold reference to device to keep it from being freed. + * Try using dev_hold_track() instead. */ static inline void dev_hold(struct net_device *dev) { @@ -3834,6 +3845,40 @@ static inline void dev_hold(struct net_device *dev) } } +static inline void netdev_tracker_alloc(struct net_device *dev, + netdevice_tracker *tracker, gfp_t gfp) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp); +#endif +} + +static inline void netdev_tracker_free(struct net_device *dev, + netdevice_tracker *tracker) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + ref_tracker_free(&dev->refcnt_tracker, tracker); +#endif +} + +static inline void dev_hold_track(struct net_device *dev, + netdevice_tracker *tracker, gfp_t gfp) +{ + if (dev) { + dev_hold(dev); + netdev_tracker_alloc(dev, tracker, gfp); + } +} + +static inline void dev_put_track(struct net_device *dev, + netdevice_tracker *tracker) +{ + if (dev) { + netdev_tracker_free(dev, tracker); + dev_put(dev); + } +} + /* Carrier loss detection, dial on demand. The functions netif_carrier_on * and _off may be called from IRQ context, but it is caller * who is responsible for serialization of these calls. diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 633c2c5cb45b..6504b97f8dfd 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -598,6 +598,11 @@ config DEBUG_MISC Say Y here if you need to enable miscellaneous debug code that should be under a more specific debug option but isn't. +menu "Networking Debugging" + +source "net/Kconfig.debug" + +endmenu # "Networking Debugging" menu "Memory Debugging" diff --git a/net/Kconfig.debug b/net/Kconfig.debug new file mode 100644 index 000000000000..fb5c70e01cb3 --- /dev/null +++ b/net/Kconfig.debug @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NET_DEV_REFCNT_TRACKER + bool "Enable net device refcount tracking" + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + select REF_TRACKER + default n + help + Enable debugging feature to track device references. + This adds memory and cpu costs. diff --git a/net/core/dev.c b/net/core/dev.c index aba8acc1238c..1740d6cfe86b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -9864,6 +9864,7 @@ static void netdev_wait_allrefs(struct net_device *dev) netdev_unregister_timeout_secs * HZ)) { pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", dev->name, refcnt); + ref_tracker_dir_print(&dev->refcnt_tracker, 10); warning_time = jiffies; } } @@ -10154,6 +10155,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, dev = PTR_ALIGN(p, NETDEV_ALIGN); dev->padded = (char *)dev - (char *)p; + ref_tracker_dir_init(&dev->refcnt_tracker, 128); #ifdef CONFIG_PCPU_DEV_REFCNT dev->pcpu_refcnt = alloc_percpu(int); if (!dev->pcpu_refcnt) @@ -10270,6 +10272,7 @@ void free_netdev(struct net_device *dev) list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) netif_napi_del(p); + ref_tracker_dir_exit(&dev->refcnt_tracker); #ifdef CONFIG_PCPU_DEV_REFCNT free_percpu(dev->pcpu_refcnt); dev->pcpu_refcnt = NULL; -- cgit v1.2.3-59-g8ed1b From 80e8921b2b72c300ca56a01729004d30bedb82cd Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:21:58 -0800 Subject: net: add net device refcount tracker to struct netdev_rx_queue This helps debugging net device refcount leaks. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 ++ net/core/net-sysfs.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 143d60ed0047..3d691fadd569 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -741,6 +741,8 @@ struct netdev_rx_queue { #endif struct kobject kobj; struct net_device *dev; + netdevice_tracker dev_tracker; + #ifdef CONFIG_XDP_SOCKETS struct xsk_buff_pool *pool; #endif diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index affe34d71d31..27a7ac2e516f 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1004,7 +1004,7 @@ static void rx_queue_release(struct kobject *kobj) #endif memset(kobj, 0, sizeof(*kobj)); - dev_put(queue->dev); + dev_put_track(queue->dev, &queue->dev_tracker); } static const void *rx_queue_namespace(struct kobject *kobj) @@ -1044,7 +1044,7 @@ static int rx_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger rx_queue_release call which * decreases dev refcount: Take that reference here */ - dev_hold(queue->dev); + dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &rx_queue_ktype, NULL, -- cgit v1.2.3-59-g8ed1b From 0b688f24b7d611db3a02f3d4ab562d049c78a17d Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:21:59 -0800 Subject: net: add net device refcount tracker to struct netdev_queue This will help debugging pesky netdev reference leaks. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 ++ net/core/net-sysfs.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3d691fadd569..b4f704337f65 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -586,6 +586,8 @@ struct netdev_queue { * read-mostly part */ struct net_device *dev; + netdevice_tracker dev_tracker; + struct Qdisc __rcu *qdisc; struct Qdisc *qdisc_sleeping; #ifdef CONFIG_SYSFS diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 27a7ac2e516f..3b2cdbbdc858 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1607,7 +1607,7 @@ static void netdev_queue_release(struct kobject *kobj) struct netdev_queue *queue = to_netdev_queue(kobj); memset(kobj, 0, sizeof(*kobj)); - dev_put(queue->dev); + dev_put_track(queue->dev, &queue->dev_tracker); } static const void *netdev_queue_namespace(struct kobject *kobj) @@ -1647,7 +1647,7 @@ static int netdev_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger netdev_queue_release call * which decreases dev refcount: Take that reference here */ - dev_hold(queue->dev); + dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &netdev_queue_ktype, NULL, -- cgit v1.2.3-59-g8ed1b From 9038c320001dd07f60736018edf608ac5baca0ab Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:22:03 -0800 Subject: net: dst: add net device refcount tracking to dst_entry We want to track all dev_hold()/dev_put() to ease leak hunting. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 17 +++++++++++++++++ include/net/dst.h | 1 + net/core/dst.c | 8 ++++---- net/ipv4/route.c | 7 ++++--- net/ipv6/route.c | 5 +++-- 5 files changed, 29 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index b4f704337f65..afed3b10491b 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3883,6 +3883,23 @@ static inline void dev_put_track(struct net_device *dev, } } +static inline void dev_replace_track(struct net_device *odev, + struct net_device *ndev, + netdevice_tracker *tracker, + gfp_t gfp) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + if (odev) + ref_tracker_free(&odev->refcnt_tracker, tracker); +#endif + dev_hold(ndev); + dev_put(odev); +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + if (ndev) + ref_tracker_alloc(&ndev->refcnt_tracker, tracker, gfp); +#endif +} + /* Carrier loss detection, dial on demand. The functions netif_carrier_on * and _off may be called from IRQ context, but it is caller * who is responsible for serialization of these calls. diff --git a/include/net/dst.h b/include/net/dst.h index a057319aabef..6aa252c3fc55 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -77,6 +77,7 @@ struct dst_entry { #ifndef CONFIG_64BIT atomic_t __refcnt; /* 32-bit offset 64 */ #endif + netdevice_tracker dev_tracker; }; struct dst_metrics { diff --git a/net/core/dst.c b/net/core/dst.c index 497ef9b3fc6a..d16c2c9bfebd 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -49,7 +49,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops, unsigned short flags) { dst->dev = dev; - dev_hold(dev); + dev_hold_track(dev, &dst->dev_tracker, GFP_ATOMIC); dst->ops = ops; dst_init_metrics(dst, dst_default_metrics.metrics, true); dst->expires = 0UL; @@ -117,7 +117,7 @@ struct dst_entry *dst_destroy(struct dst_entry * dst) if (dst->ops->destroy) dst->ops->destroy(dst); - dev_put(dst->dev); + dev_put_track(dst->dev, &dst->dev_tracker); lwtstate_put(dst->lwtstate); @@ -159,8 +159,8 @@ void dst_dev_put(struct dst_entry *dst) dst->input = dst_discard; dst->output = dst_discard_out; dst->dev = blackhole_netdev; - dev_hold(dst->dev); - dev_put(dev); + dev_replace_track(dev, blackhole_netdev, &dst->dev_tracker, + GFP_ATOMIC); } EXPORT_SYMBOL(dst_dev_put); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 243a0c52be42..843a7a3699fe 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1531,8 +1531,9 @@ void rt_flush_dev(struct net_device *dev) if (rt->dst.dev != dev) continue; rt->dst.dev = blackhole_netdev; - dev_hold(rt->dst.dev); - dev_put(dev); + dev_replace_track(dev, blackhole_netdev, + &rt->dst.dev_tracker, + GFP_ATOMIC); } spin_unlock_bh(&ul->lock); } @@ -2819,7 +2820,7 @@ struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_or new->output = dst_discard_out; new->dev = net->loopback_dev; - dev_hold(new->dev); + dev_hold_track(new->dev, &new->dev_tracker, GFP_ATOMIC); rt->rt_is_input = ort->rt_is_input; rt->rt_iif = ort->rt_iif; diff --git a/net/ipv6/route.c b/net/ipv6/route.c index f0d29fcb2094..ba4dc94d76d6 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -182,8 +182,9 @@ static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev) if (rt_dev == dev) { rt->dst.dev = blackhole_netdev; - dev_hold(rt->dst.dev); - dev_put(rt_dev); + dev_replace_track(rt_dev, blackhole_netdev, + &rt->dst.dev_tracker, + GFP_ATOMIC); } } spin_unlock_bh(&ul->lock); -- cgit v1.2.3-59-g8ed1b From c04438f58d140723e58050fcb9d33d84cb39e9e9 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:22:12 -0800 Subject: ipv4: add net device refcount tracker to struct in_device Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/inetdevice.h | 2 ++ net/ipv4/devinet.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index 518b484a7f07..674aeead6260 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -24,6 +24,8 @@ struct ipv4_devconf { struct in_device { struct net_device *dev; + netdevice_tracker dev_tracker; + refcount_t refcnt; int dead; struct in_ifaddr __rcu *ifa_list;/* IP ifaddr chain */ diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index 323e622ff9b7..fba2bffd65f7 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -243,7 +243,7 @@ void in_dev_finish_destroy(struct in_device *idev) #ifdef NET_REFCNT_DEBUG pr_debug("%s: %p=%s\n", __func__, idev, dev ? dev->name : "NIL"); #endif - dev_put(dev); + dev_put_track(dev, &idev->dev_tracker); if (!idev->dead) pr_err("Freeing alive in_device %p\n", idev); else @@ -271,7 +271,7 @@ static struct in_device *inetdev_init(struct net_device *dev) if (IPV4_DEVCONF(in_dev->cnf, FORWARDING)) dev_disable_lro(dev); /* Reference in_dev->dev */ - dev_hold(dev); + dev_hold_track(dev, &in_dev->dev_tracker, GFP_KERNEL); /* Account for reference dev->ip_ptr (below) */ refcount_set(&in_dev->refcnt, 1); -- cgit v1.2.3-59-g8ed1b From 63f13937cbe9b00982dfc8e578b1aec8e5037333 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:22:14 -0800 Subject: net: linkwatch: add net device refcount tracker Add a netdevice_tracker inside struct net_device, to track the self reference when a device is in lweventlist. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 ++ net/core/link_watch.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index afed3b10491b..69dca1edd5a6 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1950,6 +1950,7 @@ enum netdev_ml_priv_type { * keep a list of interfaces to be deleted. * * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. + * @linkwatch_dev_tracker: refcount tracker used by linkwatch. * * FIXME: cleanup struct net_device such that network protocol info * moves out. @@ -2280,6 +2281,7 @@ struct net_device { struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; u8 dev_addr_shadow[MAX_ADDR_LEN]; + netdevice_tracker linkwatch_dev_tracker; }; #define to_net_dev(d) container_of(d, struct net_device, dev) diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 9599afd0862d..d7d089963b1d 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -109,7 +109,7 @@ static void linkwatch_add_event(struct net_device *dev) spin_lock_irqsave(&lweventlist_lock, flags); if (list_empty(&dev->link_watch_list)) { list_add_tail(&dev->link_watch_list, &lweventlist); - dev_hold(dev); + dev_hold_track(dev, &dev->linkwatch_dev_tracker, GFP_ATOMIC); } spin_unlock_irqrestore(&lweventlist_lock, flags); } @@ -166,7 +166,7 @@ static void linkwatch_do_dev(struct net_device *dev) netdev_state_change(dev); } - dev_put(dev); + dev_put_track(dev, &dev->linkwatch_dev_tracker); } static void __linkwatch_run_queue(int urgent_only) -- cgit v1.2.3-59-g8ed1b From 42120a86438379eb77424831ae3d696c2d5cb622 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:22:16 -0800 Subject: ipmr, ip6mr: add net device refcount tracker to struct vif_device Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/mroute_base.h | 2 ++ net/ipv4/ipmr.c | 3 ++- net/ipv6/ip6mr.c | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index 8071148f29a6..e05ee9f001ff 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -12,6 +12,7 @@ /** * struct vif_device - interface representor for multicast routing * @dev: network device being used + * @dev_tracker: refcount tracker for @dev reference * @bytes_in: statistic; bytes ingressing * @bytes_out: statistic; bytes egresing * @pkt_in: statistic; packets ingressing @@ -26,6 +27,7 @@ */ struct vif_device { struct net_device *dev; + netdevice_tracker dev_tracker; unsigned long bytes_in, bytes_out; unsigned long pkt_in, pkt_out; unsigned long rate_limit; diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 2dda856ca260..4c7aca884fa9 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -696,7 +696,7 @@ static int vif_delete(struct mr_table *mrt, int vifi, int notify, if (v->flags & (VIFF_TUNNEL | VIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put(dev); + dev_put_track(dev, &v->dev_tracker); return 0; } @@ -896,6 +896,7 @@ static int vif_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); v->dev = dev; + netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); if (v->flags & VIFF_REGISTER) mrt->mroute_reg_vif_num = vifi; if (vifi+1 > mrt->maxvif) diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 36ed9efb8825..a77a15a7f3dc 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -746,7 +746,7 @@ static int mif6_delete(struct mr_table *mrt, int vifi, int notify, if ((v->flags & MIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put(dev); + dev_put_track(dev, &v->dev_tracker); return 0; } @@ -919,6 +919,7 @@ static int mif6_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); v->dev = dev; + netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); #ifdef CONFIG_IPV6_PIMSM_V2 if (v->flags & MIFF_REGISTER) mrt->mroute_reg_vif_num = vifi; -- cgit v1.2.3-59-g8ed1b From 5fa5ae605821e0e10ee489d9a6e331fd287ccc57 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 4 Dec 2021 20:22:17 -0800 Subject: netpoll: add net device refcount tracker to struct netpoll Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- drivers/net/netconsole.c | 2 +- include/linux/netpoll.h | 1 + net/core/netpoll.c | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index ccecba908ded..ab8cd5551020 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -721,7 +721,7 @@ restart: __netpoll_cleanup(&nt->np); spin_lock_irqsave(&target_list_lock, flags); - dev_put(nt->np.dev); + dev_put_track(nt->np.dev, &nt->np.dev_tracker); nt->np.dev = NULL; nt->enabled = false; stopped = true; diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index e6a2d72e0dc7..bd19c4b91e31 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -24,6 +24,7 @@ union inet_addr { struct netpoll { struct net_device *dev; + netdevice_tracker dev_tracker; char dev_name[IFNAMSIZ]; const char *name; diff --git a/net/core/netpoll.c b/net/core/netpoll.c index edfc0f8011f8..db724463e7cd 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -776,7 +776,7 @@ put_noaddr: err = __netpoll_setup(np, ndev); if (err) goto put; - + netdev_tracker_alloc(ndev, &np->dev_tracker, GFP_KERNEL); rtnl_unlock(); return 0; @@ -853,7 +853,7 @@ void netpoll_cleanup(struct netpoll *np) if (!np->dev) goto out; __netpoll_cleanup(np); - dev_put(np->dev); + dev_put_track(np->dev, &np->dev_tracker); np->dev = NULL; out: rtnl_unlock(); -- cgit v1.2.3-59-g8ed1b From 45cac6754529ae17345d8f5b632d9e602a091a20 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 3 Dec 2021 20:53:56 -0800 Subject: net: fix recent csum changes Vladimir reported csum issues after my recent change in skb_postpull_rcsum() Issue here is the following: initial skb->csum is the csum of [part to be pulled][rest of packet] Old code: skb->csum = csum_sub(skb->csum, csum_partial(pull, pull_length, 0)); New code: skb->csum = ~csum_partial(pull, pull_length, ~skb->csum); This is broken if the csum of [pulled part] happens to be equal to skb->csum, because end result of skb->csum is 0 in new code, instead of being 0xffffffff David Laight suggested to use skb->csum = -csum_partial(pull, pull_length, -skb->csum); I based my patches on existing code present in include/net/seg6.h, update_csum_diff4() and update_csum_diff16() which might need a similar fix. I guess that my tests, mostly pulling 40 bytes of IPv6 header were not providing enough entropy to hit this bug. v2: added wsum_negate() to make sparse happy. Fixes: 29c3002644bd ("net: optimize skb_postpull_rcsum()") Fixes: 0bd28476f636 ("gro: optimize skb_gro_postpull_rcsum()") Signed-off-by: Eric Dumazet Reported-by: Vladimir Oltean Suggested-by: David Laight Cc: David Lebrun Tested-by: Vladimir Oltean Link: https://lore.kernel.org/r/20211204045356.3659278-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 3 ++- include/net/checksum.h | 4 ++++ include/net/gro.h | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index eae4bd3237a4..dd262bd8ddbe 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3486,7 +3486,8 @@ static inline void skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len) { if (skb->ip_summed == CHECKSUM_COMPLETE) - skb->csum = ~csum_partial(start, len, ~skb->csum); + skb->csum = wsum_negate(csum_partial(start, len, + wsum_negate(skb->csum))); else if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_start_offset(skb) < 0) skb->ip_summed = CHECKSUM_NONE; diff --git a/include/net/checksum.h b/include/net/checksum.h index 5b96d5bd6e54..5218041e5c8f 100644 --- a/include/net/checksum.h +++ b/include/net/checksum.h @@ -180,4 +180,8 @@ static inline void remcsum_unadjust(__sum16 *psum, __wsum delta) *psum = csum_fold(csum_sub(delta, (__force __wsum)*psum)); } +static inline __wsum wsum_negate(__wsum val) +{ + return (__force __wsum)-((__force u32)val); +} #endif diff --git a/include/net/gro.h b/include/net/gro.h index b1139fca7c43..8f75802d50fd 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -173,8 +173,8 @@ static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len) { if (NAPI_GRO_CB(skb)->csum_valid) - NAPI_GRO_CB(skb)->csum = ~csum_partial(start, len, - ~NAPI_GRO_CB(skb)->csum); + NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, + wsum_negate(NAPI_GRO_CB(skb)->csum))); } /* GRO checksum functions. These are logical equivalents of the normal -- cgit v1.2.3-59-g8ed1b From 77993b595ada5731e513eb06a0f4bf4b9f1e9532 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 29 Nov 2021 18:46:54 +0100 Subject: locking: Allow to include asm/spinlock_types.h from linux/spinlock_types_raw.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The printk header file includes ratelimit_types.h for its __ratelimit() based usage. It is required for the static initializer used in printk_ratelimited(). It uses a raw_spinlock_t and includes the spinlock_types.h. PREEMPT_RT substitutes spinlock_t with a rtmutex based implementation and so its spinlock_t implmentation (provided by spinlock_rt.h) includes rtmutex.h and atomic.h which leads to recursive includes where defines are missing. By including only the raw_spinlock_t defines it avoids the atomic.h related includes at this stage. An example on powerpc: | CALL scripts/atomic/check-atomics.sh |In file included from include/linux/bug.h:5, | from include/linux/page-flags.h:10, | from kernel/bounds.c:10: |arch/powerpc/include/asm/page_32.h: In function ‘clear_page’: |arch/powerpc/include/asm/bug.h:87:4: error: implicit declaration of function â=80=98__WARNâ=80=99 [-Werror=3Dimplicit-function-declaration] | 87 | __WARN(); \ | | ^~~~~~ |arch/powerpc/include/asm/page_32.h:48:2: note: in expansion of macro ‘WARN_ONâ€=99 | 48 | WARN_ON((unsigned long)addr & (L1_CACHE_BYTES - 1)); | | ^~~~~~~ |arch/powerpc/include/asm/bug.h:58:17: error: invalid application of ‘sizeofâ€=99 to incomplete type ‘struct bug_entryâ€=99 | 58 | "i" (sizeof(struct bug_entry)), \ | | ^~~~~~ |arch/powerpc/include/asm/bug.h:89:3: note: in expansion of macro ‘BUG_ENTRYâ€=99 | 89 | BUG_ENTRY(PPC_TLNEI " %4, 0", \ | | ^~~~~~~~~ |arch/powerpc/include/asm/page_32.h:48:2: note: in expansion of macro ‘WARN_ONâ€=99 | 48 | WARN_ON((unsigned long)addr & (L1_CACHE_BYTES - 1)); | | ^~~~~~~ |In file included from arch/powerpc/include/asm/ptrace.h:298, | from arch/powerpc/include/asm/hw_irq.h:12, | from arch/powerpc/include/asm/irqflags.h:12, | from include/linux/irqflags.h:16, | from include/asm-generic/cmpxchg-local.h:6, | from arch/powerpc/include/asm/cmpxchg.h:526, | from arch/powerpc/include/asm/atomic.h:11, | from include/linux/atomic.h:7, | from include/linux/rwbase_rt.h:6, | from include/linux/rwlock_types.h:55, | from include/linux/spinlock_types.h:74, | from include/linux/ratelimit_types.h:7, | from include/linux/printk.h:10, | from include/asm-generic/bug.h:22, | from arch/powerpc/include/asm/bug.h:109, | from include/linux/bug.h:5, | from include/linux/page-flags.h:10, | from kernel/bounds.c:10: |include/linux/thread_info.h: In function â=80=98copy_overflowâ=80=99: |include/linux/thread_info.h:210:2: error: implicit declaration of function â=80=98WARNâ=80=99 [-Werror=3Dimplicit-function-declaration] | 210 | WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); | | ^~~~ The WARN / BUG include pulls in printk.h and then ptrace.h expects WARN (from bug.h) which is not yet complete. Even hw_irq.h has WARN_ON() statements. On POWERPC64 there are missing atomic64 defines while building 32bit VDSO: | VDSO32C arch/powerpc/kernel/vdso32/vgettimeofday.o |In file included from include/linux/atomic.h:80, | from include/linux/rwbase_rt.h:6, | from include/linux/rwlock_types.h:55, | from include/linux/spinlock_types.h:74, | from include/linux/ratelimit_types.h:7, | from include/linux/printk.h:10, | from include/linux/kernel.h:19, | from arch/powerpc/include/asm/page.h:11, | from arch/powerpc/include/asm/vdso/gettimeofday.h:5, | from include/vdso/datapage.h:137, | from lib/vdso/gettimeofday.c:5, | from : |include/linux/atomic-arch-fallback.h: In function ‘arch_atomic64_incâ€=99: |include/linux/atomic-arch-fallback.h:1447:2: error: implicit declaration of function ‘arch_atomic64_add’; did you mean ‘arch_atomic_add’? [-Werror=3Dimpl |icit-function-declaration] | 1447 | arch_atomic64_add(1, v); | | ^~~~~~~~~~~~~~~~~ | | arch_atomic_add The generic fallback is not included, atomics itself are not used. If kernel.h does not include printk.h then it comes later from the bug.h include. Allow asm/spinlock_types.h to be included from linux/spinlock_types_raw.h. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20211129174654.668506-12-bigeasy@linutronix.de --- arch/alpha/include/asm/spinlock_types.h | 2 +- arch/arm/include/asm/spinlock_types.h | 2 +- arch/arm64/include/asm/spinlock_types.h | 2 +- arch/csky/include/asm/spinlock_types.h | 2 +- arch/hexagon/include/asm/spinlock_types.h | 2 +- arch/ia64/include/asm/spinlock_types.h | 2 +- arch/powerpc/include/asm/simple_spinlock_types.h | 2 +- arch/powerpc/include/asm/spinlock_types.h | 2 +- arch/riscv/include/asm/spinlock_types.h | 2 +- arch/s390/include/asm/spinlock_types.h | 2 +- arch/sh/include/asm/spinlock_types.h | 2 +- arch/xtensa/include/asm/spinlock_types.h | 2 +- include/linux/ratelimit_types.h | 2 +- include/linux/spinlock_types_up.h | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/include/asm/spinlock_types.h b/arch/alpha/include/asm/spinlock_types.h index 1d5716bc060b..2526fd3be5fd 100644 --- a/arch/alpha/include/asm/spinlock_types.h +++ b/arch/alpha/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ALPHA_SPINLOCK_TYPES_H #define _ALPHA_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/arm/include/asm/spinlock_types.h b/arch/arm/include/asm/spinlock_types.h index 5976958647fe..0c14b36ef101 100644 --- a/arch/arm/include/asm/spinlock_types.h +++ b/arch/arm/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef __ASM_SPINLOCK_TYPES_H #define __ASM_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/arm64/include/asm/spinlock_types.h b/arch/arm64/include/asm/spinlock_types.h index 18782f0c4721..11ab1c077697 100644 --- a/arch/arm64/include/asm/spinlock_types.h +++ b/arch/arm64/include/asm/spinlock_types.h @@ -5,7 +5,7 @@ #ifndef __ASM_SPINLOCK_TYPES_H #define __ASM_SPINLOCK_TYPES_H -#if !defined(__LINUX_SPINLOCK_TYPES_H) && !defined(__ASM_SPINLOCK_H) +#if !defined(__LINUX_SPINLOCK_TYPES_RAW_H) && !defined(__ASM_SPINLOCK_H) # error "please don't include this file directly" #endif diff --git a/arch/csky/include/asm/spinlock_types.h b/arch/csky/include/asm/spinlock_types.h index 8ff0f6ff3a00..db87a12c3827 100644 --- a/arch/csky/include/asm/spinlock_types.h +++ b/arch/csky/include/asm/spinlock_types.h @@ -3,7 +3,7 @@ #ifndef __ASM_CSKY_SPINLOCK_TYPES_H #define __ASM_CSKY_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/hexagon/include/asm/spinlock_types.h b/arch/hexagon/include/asm/spinlock_types.h index 19d233497ba5..d5f66495b670 100644 --- a/arch/hexagon/include/asm/spinlock_types.h +++ b/arch/hexagon/include/asm/spinlock_types.h @@ -8,7 +8,7 @@ #ifndef _ASM_SPINLOCK_TYPES_H #define _ASM_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/ia64/include/asm/spinlock_types.h b/arch/ia64/include/asm/spinlock_types.h index 6e345fefcdca..14b8a161c165 100644 --- a/arch/ia64/include/asm/spinlock_types.h +++ b/arch/ia64/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ASM_IA64_SPINLOCK_TYPES_H #define _ASM_IA64_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/powerpc/include/asm/simple_spinlock_types.h b/arch/powerpc/include/asm/simple_spinlock_types.h index 0f3cdd8faa95..08243338069d 100644 --- a/arch/powerpc/include/asm/simple_spinlock_types.h +++ b/arch/powerpc/include/asm/simple_spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H #define _ASM_POWERPC_SIMPLE_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/powerpc/include/asm/spinlock_types.h b/arch/powerpc/include/asm/spinlock_types.h index c5d742f18021..d5f8a74ed2e8 100644 --- a/arch/powerpc/include/asm/spinlock_types.h +++ b/arch/powerpc/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef _ASM_POWERPC_SPINLOCK_TYPES_H #define _ASM_POWERPC_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/riscv/include/asm/spinlock_types.h b/arch/riscv/include/asm/spinlock_types.h index f398e7638dd6..5a35a49505da 100644 --- a/arch/riscv/include/asm/spinlock_types.h +++ b/arch/riscv/include/asm/spinlock_types.h @@ -6,7 +6,7 @@ #ifndef _ASM_RISCV_SPINLOCK_TYPES_H #define _ASM_RISCV_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/s390/include/asm/spinlock_types.h b/arch/s390/include/asm/spinlock_types.h index a2bbfd7df85f..b69695e39957 100644 --- a/arch/s390/include/asm/spinlock_types.h +++ b/arch/s390/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef __ASM_SPINLOCK_TYPES_H #define __ASM_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/sh/include/asm/spinlock_types.h b/arch/sh/include/asm/spinlock_types.h index e82369f286a2..907bda4b1619 100644 --- a/arch/sh/include/asm/spinlock_types.h +++ b/arch/sh/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef __ASM_SH_SPINLOCK_TYPES_H #define __ASM_SH_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif diff --git a/arch/xtensa/include/asm/spinlock_types.h b/arch/xtensa/include/asm/spinlock_types.h index 64c9389254f1..797aed7df3dd 100644 --- a/arch/xtensa/include/asm/spinlock_types.h +++ b/arch/xtensa/include/asm/spinlock_types.h @@ -2,7 +2,7 @@ #ifndef __ASM_SPINLOCK_TYPES_H #define __ASM_SPINLOCK_TYPES_H -#if !defined(__LINUX_SPINLOCK_TYPES_H) && !defined(__ASM_SPINLOCK_H) +#if !defined(__LINUX_SPINLOCK_TYPES_RAW_H) && !defined(__ASM_SPINLOCK_H) # error "please don't include this file directly" #endif diff --git a/include/linux/ratelimit_types.h b/include/linux/ratelimit_types.h index b676aa419eef..c21c7f8103e2 100644 --- a/include/linux/ratelimit_types.h +++ b/include/linux/ratelimit_types.h @@ -4,7 +4,7 @@ #include #include -#include +#include #define DEFAULT_RATELIMIT_INTERVAL (5 * HZ) #define DEFAULT_RATELIMIT_BURST 10 diff --git a/include/linux/spinlock_types_up.h b/include/linux/spinlock_types_up.h index c09b6407ae1b..7f86a2016ac5 100644 --- a/include/linux/spinlock_types_up.h +++ b/include/linux/spinlock_types_up.h @@ -1,7 +1,7 @@ #ifndef __LINUX_SPINLOCK_TYPES_UP_H #define __LINUX_SPINLOCK_TYPES_UP_H -#ifndef __LINUX_SPINLOCK_TYPES_H +#ifndef __LINUX_SPINLOCK_TYPES_RAW_H # error "please don't include this file directly" #endif -- cgit v1.2.3-59-g8ed1b From 8d9f738f16a3ee9f2341578873c542ddd9802fe4 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Tue, 7 Dec 2021 20:32:30 +0800 Subject: regulator: fix bullet lists of regulator_ops comment Since 89a6a5e56c82("regulator: add property parsing and callbacks to set protection limits") which introduced a warning: Documentation/driver-api/regulator:166: ./include/linux/regulator/driver.h:96: WARNING: Unexpected indentation. Documentation/driver-api/regulator:166: ./include/linux/regulator/driver.h:98: WARNING: Block quote ends without a blank line; unexpected unindent. Let's fix them. Signed-off-by: Yanteng Si Link: https://lore.kernel.org/r/20211207123230.2262047-1-siyanteng@loongson.cn Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index 4078c7776453..720684995a77 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -90,15 +90,19 @@ enum regulator_detection_severity { * @set_over_current_protection: Support enabling of and setting limits for over * current situation detection. Detection can be configured for three * levels of severity. - * REGULATOR_SEVERITY_PROT should automatically shut down the regulator(s). - * REGULATOR_SEVERITY_ERR should indicate that over-current situation is - * caused by an unrecoverable error but HW does not perform - * automatic shut down. - * REGULATOR_SEVERITY_WARN should indicate situation where hardware is - * still believed to not be damaged but that a board sepcific - * recovery action is needed. If lim_uA is 0 the limit should not - * be changed but the detection should just be enabled/disabled as - * is requested. + * + * - REGULATOR_SEVERITY_PROT should automatically shut down the regulator(s). + * + * - REGULATOR_SEVERITY_ERR should indicate that over-current situation is + * caused by an unrecoverable error but HW does not perform + * automatic shut down. + * + * - REGULATOR_SEVERITY_WARN should indicate situation where hardware is + * still believed to not be damaged but that a board sepcific + * recovery action is needed. If lim_uA is 0 the limit should not + * be changed but the detection should just be enabled/disabled as + * is requested. + * * @set_over_voltage_protection: Support enabling of and setting limits for over * voltage situation detection. Detection can be configured for same * severities as over current protection. Units of uV. -- cgit v1.2.3-59-g8ed1b From 13244cccc2b61ec715f0ac583d3037497004d4a5 Mon Sep 17 00:00:00 2001 From: Luiz Augusto von Dentz Date: Wed, 1 Dec 2021 10:54:52 -0800 Subject: skbuff: introduce skb_pull_data Like skb_pull but returns the original data pointer before pulling the data after performing a check against sbk->len. This allows to change code that does "struct foo *p = (void *)skb->data;" which is hard to audit and error prone, to: p = skb_pull_data(skb, sizeof(*p)); if (!p) return; Which is both safer and cleaner. Acked-by: Jakub Kicinski Signed-off-by: Luiz Augusto von Dentz Signed-off-by: Dan Carpenter Signed-off-by: Marcel Holtmann --- include/linux/skbuff.h | 2 ++ net/core/skbuff.c | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index eba256af64a5..877dda38684a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2373,6 +2373,8 @@ static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len) return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); } +void *skb_pull_data(struct sk_buff *skb, size_t len); + void *__pskb_pull_tail(struct sk_buff *skb, int delta); static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a33247fdb8f5..dd3ef966efdb 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -2023,6 +2023,30 @@ void *skb_pull(struct sk_buff *skb, unsigned int len) } EXPORT_SYMBOL(skb_pull); +/** + * skb_pull_data - remove data from the start of a buffer returning its + * original position. + * @skb: buffer to use + * @len: amount of data to remove + * + * This function removes data from the start of a buffer, returning + * the memory to the headroom. A pointer to the original data in the buffer + * is returned after checking if there is enough data to pull. Once the + * data has been pulled future pushes will overwrite the old data. + */ +void *skb_pull_data(struct sk_buff *skb, size_t len) +{ + void *data = skb->data; + + if (skb->len < len) + return NULL; + + skb_pull(skb, len); + + return data; +} +EXPORT_SYMBOL(skb_pull_data); + /** * skb_trim - remove end from a buffer * @skb: buffer to alter -- cgit v1.2.3-59-g8ed1b From 6fadaa565882cd7afc501de5921db6f5e45c784b Mon Sep 17 00:00:00 2001 From: Maxim Galaganov Date: Fri, 3 Dec 2021 14:35:39 -0800 Subject: tcp: expose __tcp_sock_set_cork and __tcp_sock_set_nodelay Expose __tcp_sock_set_cork() and __tcp_sock_set_nodelay() for use in MPTCP setsockopt code -- namely for syncing MPTCP socket options with subflows inside sync_socket_options() while already holding the subflow socket lock. Acked-by: Paolo Abeni Acked-by: Matthieu Baerts Signed-off-by: Maxim Galaganov Signed-off-by: Mat Martineau Signed-off-by: Jakub Kicinski --- include/linux/tcp.h | 2 ++ net/ipv4/tcp.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 48d8a363319e..78b91bb92f0d 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -512,11 +512,13 @@ static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss) int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from, int pcount, int shiftlen); +void __tcp_sock_set_cork(struct sock *sk, bool on); void tcp_sock_set_cork(struct sock *sk, bool on); int tcp_sock_set_keepcnt(struct sock *sk, int val); int tcp_sock_set_keepidle_locked(struct sock *sk, int val); int tcp_sock_set_keepidle(struct sock *sk, int val); int tcp_sock_set_keepintvl(struct sock *sk, int val); +void __tcp_sock_set_nodelay(struct sock *sk, bool on); void tcp_sock_set_nodelay(struct sock *sk); void tcp_sock_set_quickack(struct sock *sk, int val); int tcp_sock_set_syncnt(struct sock *sk, int val); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 6ab82e1a1d41..20054618c87e 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3207,7 +3207,7 @@ static void tcp_enable_tx_delay(void) * TCP_CORK can be set together with TCP_NODELAY and it is stronger than * TCP_NODELAY. */ -static void __tcp_sock_set_cork(struct sock *sk, bool on) +void __tcp_sock_set_cork(struct sock *sk, bool on) { struct tcp_sock *tp = tcp_sk(sk); @@ -3235,7 +3235,7 @@ EXPORT_SYMBOL(tcp_sock_set_cork); * However, when TCP_NODELAY is set we make an explicit push, which overrides * even TCP_CORK for currently queued segments. */ -static void __tcp_sock_set_nodelay(struct sock *sk, bool on) +void __tcp_sock_set_nodelay(struct sock *sk, bool on) { if (on) { tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; -- cgit v1.2.3-59-g8ed1b From 213d56bf33bdda835bac04046f09256a75c5ca8e Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 19 Oct 2021 02:08:07 +0200 Subject: rcu/nocb: Prepare state machine for a new step Currently SEGCBLIST_SOFTIRQ_ONLY is a bit of an exception among the segcblist flags because it is an exclusive state that doesn't mix up with the other flags. Remove it in favour of: _ A flag specifying that rcu_core() needs to perform callbacks execution and acceleration and _ A flag specifying we want the nocb lock to be held in any needed circumstances This clarifies the code and is more flexible: It allows to have a state where rcu_core() runs with locking while offloading hasn't started yet. This is a necessary step to prepare for triggering rcu_core() at the very beginning of the de-offloading process so that rcu_core() won't dismiss work while being preempted by the de-offloading process, at least not without a pending subsequent rcu_core() that will quickly catch up. Reviewed-by: Valentin Schneider Tested-by: Valentin Schneider Tested-by: Sebastian Andrzej Siewior Signed-off-by: Frederic Weisbecker Cc: Peter Zijlstra Cc: Sebastian Andrzej Siewior Cc: Josh Triplett Cc: Joel Fernandes Cc: Boqun Feng Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Thomas Gleixner Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 37 +++++++++++++++++++++++-------------- kernel/rcu/rcu_segcblist.c | 6 +++--- kernel/rcu/rcu_segcblist.h | 12 +++++++----- kernel/rcu/tree.c | 2 +- kernel/rcu/tree_nocb.h | 24 ++++++++++++++++-------- 5 files changed, 50 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 3db96c4f45fd..812961b1d064 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -69,7 +69,7 @@ struct rcu_cblist { * * * ---------------------------------------------------------------------------- - * | SEGCBLIST_SOFTIRQ_ONLY | + * | SEGCBLIST_RCU_CORE | * | | * | Callbacks processed by rcu_core() from softirqs or local | * | rcuc kthread, without holding nocb_lock. | @@ -77,7 +77,7 @@ struct rcu_cblist { * | * v * ---------------------------------------------------------------------------- - * | SEGCBLIST_OFFLOADED | + * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED | * | | * | Callbacks processed by rcu_core() from softirqs or local | * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | @@ -89,7 +89,9 @@ struct rcu_cblist { * | | * v v * --------------------------------------- ----------------------------------| - * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | | + * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | | + * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | * | | | | * | | | | @@ -104,9 +106,10 @@ struct rcu_cblist { * | * v * |--------------------------------------------------------------------------| - * | SEGCBLIST_OFFLOADED | | - * | SEGCBLIST_KTHREAD_CB | | - * | SEGCBLIST_KTHREAD_GP | + * | SEGCBLIST_LOCKING | | + * | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_KTHREAD_GP | | + * | SEGCBLIST_KTHREAD_CB | * | | * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | * | handling callbacks. Enable bypass queueing. | @@ -120,7 +123,8 @@ struct rcu_cblist { * * * |--------------------------------------------------------------------------| - * | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_LOCKING | | + * | SEGCBLIST_OFFLOADED | | * | SEGCBLIST_KTHREAD_CB | | * | SEGCBLIST_KTHREAD_GP | * | | @@ -130,6 +134,8 @@ struct rcu_cblist { * | * v * |--------------------------------------------------------------------------| + * | SEGCBLIST_RCU_CORE | | + * | SEGCBLIST_LOCKING | | * | SEGCBLIST_KTHREAD_CB | | * | SEGCBLIST_KTHREAD_GP | * | | @@ -143,7 +149,9 @@ struct rcu_cblist { * | | * v v * ---------------------------------------------------------------------------| - * | | + * | | | + * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | | + * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | | * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | * | | | * | GP kthread woke up and | CB kthread woke up and | @@ -159,7 +167,7 @@ struct rcu_cblist { * | * v * ---------------------------------------------------------------------------- - * | 0 | + * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | * | | * | Callbacks processed by rcu_core() from softirqs or local | * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | @@ -168,17 +176,18 @@ struct rcu_cblist { * | * v * ---------------------------------------------------------------------------- - * | SEGCBLIST_SOFTIRQ_ONLY | + * | SEGCBLIST_RCU_CORE | * | | * | Callbacks processed by rcu_core() from softirqs or local | * | rcuc kthread, without holding nocb_lock. | * ---------------------------------------------------------------------------- */ #define SEGCBLIST_ENABLED BIT(0) -#define SEGCBLIST_SOFTIRQ_ONLY BIT(1) -#define SEGCBLIST_KTHREAD_CB BIT(2) -#define SEGCBLIST_KTHREAD_GP BIT(3) -#define SEGCBLIST_OFFLOADED BIT(4) +#define SEGCBLIST_RCU_CORE BIT(1) +#define SEGCBLIST_LOCKING BIT(2) +#define SEGCBLIST_KTHREAD_CB BIT(3) +#define SEGCBLIST_KTHREAD_GP BIT(4) +#define SEGCBLIST_OFFLOADED BIT(5) struct rcu_segcblist { struct rcu_head *head; diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index aaa111237b60..c07aab6e39ef 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -261,14 +261,14 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) } /* - * Mark the specified rcu_segcblist structure as offloaded. + * Mark the specified rcu_segcblist structure as offloaded (or not) */ void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload) { if (offload) { - rcu_segcblist_clear_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY); - rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED); + rcu_segcblist_set_flags(rsclp, SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED); } else { + rcu_segcblist_set_flags(rsclp, SEGCBLIST_RCU_CORE); rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED); } } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 9a19328ff251..e373fbe44da5 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -80,11 +80,14 @@ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); } -/* Is the specified rcu_segcblist offloaded, or is SEGCBLIST_SOFTIRQ_ONLY set? */ +/* + * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the + * [de]offloading process)? + */ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) { if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && - !rcu_segcblist_test_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY)) + rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING)) return true; return false; @@ -92,9 +95,8 @@ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp) { - int flags = SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | SEGCBLIST_OFFLOADED; - - if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && (rsclp->flags & flags) == flags) + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && + !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE)) return true; return false; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 8706b30c2ac8..e905d7e4ddb9 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -79,7 +79,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(1), #ifdef CONFIG_RCU_NOCB_CPU - .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY, + .cblist.flags = SEGCBLIST_RCU_CORE, #endif }; static struct rcu_state rcu_state = { diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 368ef7b9af4f..b3e07d0bfbbf 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1000,12 +1000,12 @@ static long rcu_nocb_rdp_deoffload(void *arg) */ rcu_nocb_lock_irqsave(rdp, flags); /* - * Theoretically we could set SEGCBLIST_SOFTIRQ_ONLY after the nocb + * Theoretically we could clear SEGCBLIST_LOCKING after the nocb * lock is released but how about being paranoid for once? */ - rcu_segcblist_set_flags(cblist, SEGCBLIST_SOFTIRQ_ONLY); + rcu_segcblist_clear_flags(cblist, SEGCBLIST_LOCKING); /* - * With SEGCBLIST_SOFTIRQ_ONLY, we can't use + * Without SEGCBLIST_LOCKING, we can't use * rcu_nocb_unlock_irqrestore() anymore. */ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); @@ -1058,14 +1058,14 @@ static long rcu_nocb_rdp_offload(void *arg) pr_info("Offloading %d\n", rdp->cpu); /* - * Can't use rcu_nocb_lock_irqsave() while we are in - * SEGCBLIST_SOFTIRQ_ONLY mode. + * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING + * is set. */ raw_spin_lock_irqsave(&rdp->nocb_lock, flags); /* * We didn't take the nocb lock while working on the - * rdp->cblist in SEGCBLIST_SOFTIRQ_ONLY mode. + * rdp->cblist with SEGCBLIST_LOCKING cleared (pure softirq/rcuc mode). * Every modifications that have been done previously on * rdp->cblist must be visible remotely by the nocb kthreads * upon wake up after reading the cblist flags. @@ -1084,6 +1084,14 @@ static long rcu_nocb_rdp_offload(void *arg) rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); + /* + * All kthreads are ready to work, we can finally relieve rcu_core() and + * enable nocb bypass. + */ + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE); + rcu_nocb_unlock_irqrestore(rdp, flags); + return ret; } @@ -1154,8 +1162,8 @@ void __init rcu_init_nohz(void) if (rcu_segcblist_empty(&rdp->cblist)) rcu_segcblist_init(&rdp->cblist); rcu_segcblist_offload(&rdp->cblist, true); - rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); - rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP); + rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP); + rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE); } rcu_organize_nocb_kthreads(); } -- cgit v1.2.3-59-g8ed1b From fbb94cbd70d41c7511460896dfc7f9ea5da704b3 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 19 Oct 2021 02:08:08 +0200 Subject: rcu/nocb: Invoke rcu_core() at the start of deoffloading On PREEMPT_RT, if rcu_core() is preempted by the de-offloading process, some work, such as callbacks acceleration and invocation, may be left unattended due to the volatile checks on the offloaded state. In the worst case this work is postponed until the next rcu_pending() check that can take a jiffy to reach, which can be a problem in case of callbacks flooding. Solve that with invoking rcu_core() early in the de-offloading process. This way any work dismissed by an ongoing rcu_core() call fooled by a preempting deoffloading process will be caught up by a nearby future recall to rcu_core(), this time fully aware of the de-offloading state. Tested-by: Valentin Schneider Tested-by: Sebastian Andrzej Siewior Signed-off-by: Frederic Weisbecker Cc: Valentin Schneider Cc: Peter Zijlstra Cc: Sebastian Andrzej Siewior Cc: Josh Triplett Cc: Joel Fernandes Cc: Boqun Feng Cc: Neeraj Upadhyay Cc: Uladzislau Rezki Cc: Thomas Gleixner Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 14 ++++++++++++++ kernel/rcu/rcu_segcblist.c | 6 ++---- kernel/rcu/tree.c | 17 +++++++++++++++++ kernel/rcu/tree_nocb.h | 9 +++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 812961b1d064..659d13a7ddaa 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -136,6 +136,20 @@ struct rcu_cblist { * |--------------------------------------------------------------------------| * | SEGCBLIST_RCU_CORE | | * | SEGCBLIST_LOCKING | | + * | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_KTHREAD_CB | | + * | SEGCBLIST_KTHREAD_GP | + * | | + * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | + * | handles callbacks concurrently. Bypass enqueue is enabled. | + * | Invoke RCU core so we make sure not to preempt it in the middle with | + * | leaving some urgent work unattended within a jiffy. | + * ---------------------------------------------------------------------------- + * | + * v + * |--------------------------------------------------------------------------| + * | SEGCBLIST_RCU_CORE | | + * | SEGCBLIST_LOCKING | | * | SEGCBLIST_KTHREAD_CB | | * | SEGCBLIST_KTHREAD_GP | * | | diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index c07aab6e39ef..81145c3ece25 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -265,12 +265,10 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) */ void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload) { - if (offload) { + if (offload) rcu_segcblist_set_flags(rsclp, SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED); - } else { - rcu_segcblist_set_flags(rsclp, SEGCBLIST_RCU_CORE); + else rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED); - } } /* diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e905d7e4ddb9..a329adfece86 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2707,6 +2707,23 @@ static __latent_entropy void rcu_core(void) unsigned long flags; struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; + /* + * On RT rcu_core() can be preempted when IRQs aren't disabled. + * Therefore this function can race with concurrent NOCB (de-)offloading + * on this CPU and the below condition must be considered volatile. + * However if we race with: + * + * _ Offloading: In the worst case we accelerate or process callbacks + * concurrently with NOCB kthreads. We are guaranteed to + * call rcu_nocb_lock() if that happens. + * + * _ Deoffloading: In the worst case we miss callbacks acceleration or + * processing. This is fine because the early stage + * of deoffloading invokes rcu_core() after setting + * SEGCBLIST_RCU_CORE. So we guarantee that we'll process + * what could have been dismissed without the need to wait + * for the next rcu_pending() check in the next jiffy. + */ const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); if (cpu_is_offline(smp_processor_id())) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index b3e07d0bfbbf..2461fe8d0c23 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -990,6 +990,15 @@ static long rcu_nocb_rdp_deoffload(void *arg) * will refuse to put anything into the bypass. */ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + /* + * Start with invoking rcu_core() early. This way if the current thread + * happens to preempt an ongoing call to rcu_core() in the middle, + * leaving some work dismissed because rcu_core() still thinks the rdp is + * completely offloaded, we are guaranteed a nearby future instance of + * rcu_core() to catch up. + */ + rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE); + invoke_rcu_core(); ret = rdp_offload_toggle(rdp, false, flags); swait_event_exclusive(rdp->nocb_state_wq, !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | -- cgit v1.2.3-59-g8ed1b From 81faa4f6fba429334ff72bb5ba7696818509b5b5 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Wed, 3 Nov 2021 16:30:28 +0800 Subject: locktorture,rcutorture,torture: Always log error message Unconditionally log messages corresponding to errors. Acked-by: Davidlohr Bueso Signed-off-by: Li Zhijian Signed-off-by: Paul E. McKenney --- include/linux/torture.h | 9 ++------- kernel/locking/locktorture.c | 4 ++-- kernel/rcu/rcutorture.c | 8 ++++---- kernel/torture.c | 4 ++-- 4 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/torture.h b/include/linux/torture.h index 24f58e50a94b..63fa4196e51c 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -38,13 +38,8 @@ do { \ pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \ } \ } while (0) -#define VERBOSE_TOROUT_ERRSTRING(s) \ -do { \ - if (verbose) { \ - verbose_torout_sleep(); \ - pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \ - } \ -} while (0) +#define TOROUT_ERRSTRING(s) \ + pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s) void verbose_torout_sleep(void); #define torture_init_error(firsterr) \ diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 397ac13d2ef7..9c2fb613a55d 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -1047,7 +1047,7 @@ static int __init lock_torture_init(void) sizeof(writer_tasks[0]), GFP_KERNEL); if (writer_tasks == NULL) { - VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory"); + TOROUT_ERRSTRING("writer_tasks: Out of memory"); firsterr = -ENOMEM; goto unwind; } @@ -1058,7 +1058,7 @@ static int __init lock_torture_init(void) sizeof(reader_tasks[0]), GFP_KERNEL); if (reader_tasks == NULL) { - VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory"); + TOROUT_ERRSTRING("reader_tasks: Out of memory"); kfree(writer_tasks); writer_tasks = NULL; firsterr = -ENOMEM; diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 503e14e62e8f..36a273589a35 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2774,7 +2774,7 @@ static int rcu_torture_read_exit(void *unused) &trs, "%s", "rcu_torture_read_exit_child"); if (IS_ERR(tsp)) { - VERBOSE_TOROUT_ERRSTRING("out of memory"); + TOROUT_ERRSTRING("out of memory"); errexit = true; tsp = NULL; break; @@ -3101,7 +3101,7 @@ rcu_torture_init(void) sizeof(fakewriter_tasks[0]), GFP_KERNEL); if (fakewriter_tasks == NULL) { - VERBOSE_TOROUT_ERRSTRING("out of memory"); + TOROUT_ERRSTRING("out of memory"); firsterr = -ENOMEM; goto unwind; } @@ -3117,7 +3117,7 @@ rcu_torture_init(void) rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), GFP_KERNEL); if (!reader_tasks || !rcu_torture_reader_mbchk) { - VERBOSE_TOROUT_ERRSTRING("out of memory"); + TOROUT_ERRSTRING("out of memory"); firsterr = -ENOMEM; goto unwind; } @@ -3136,7 +3136,7 @@ rcu_torture_init(void) if (nrealnocbers > 0) { nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); if (nocb_tasks == NULL) { - VERBOSE_TOROUT_ERRSTRING("out of memory"); + TOROUT_ERRSTRING("out of memory"); firsterr = -ENOMEM; goto unwind; } diff --git a/kernel/torture.c b/kernel/torture.c index bb8f411c974b..ef27a6c82451 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -570,7 +570,7 @@ int torture_shuffle_init(long shuffint) shuffle_idle_cpu = -1; if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { - VERBOSE_TOROUT_ERRSTRING("Failed to alloc mask"); + TOROUT_ERRSTRING("Failed to alloc mask"); return -ENOMEM; } @@ -934,7 +934,7 @@ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, *tp = kthread_run(fn, arg, "%s", s); if (IS_ERR(*tp)) { ret = PTR_ERR(*tp); - VERBOSE_TOROUT_ERRSTRING(f); + TOROUT_ERRSTRING(f); *tp = NULL; } torture_shuffle_task_register(*tp); -- cgit v1.2.3-59-g8ed1b From 08f0b22d731fa86957749c649d6ef6ebc07e8ad2 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 6 Dec 2021 17:30:27 -0800 Subject: net: eql: add net device refcount tracker Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- drivers/net/eql.c | 4 ++-- include/linux/if_eql.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/eql.c b/drivers/net/eql.c index 8ef34901c2d8..1111d1f33865 100644 --- a/drivers/net/eql.c +++ b/drivers/net/eql.c @@ -225,7 +225,7 @@ static void eql_kill_one_slave(slave_queue_t *queue, slave_t *slave) list_del(&slave->list); queue->num_slaves--; slave->dev->flags &= ~IFF_SLAVE; - dev_put(slave->dev); + dev_put_track(slave->dev, &slave->dev_tracker); kfree(slave); } @@ -399,7 +399,7 @@ static int __eql_insert_slave(slave_queue_t *queue, slave_t *slave) if (duplicate_slave) eql_kill_one_slave(queue, duplicate_slave); - dev_hold(slave->dev); + dev_hold_track(slave->dev, &slave->dev_tracker, GFP_ATOMIC); list_add(&slave->list, &queue->all_slaves); queue->num_slaves++; slave->dev->flags |= IFF_SLAVE; diff --git a/include/linux/if_eql.h b/include/linux/if_eql.h index d984694c384d..d75601d613cc 100644 --- a/include/linux/if_eql.h +++ b/include/linux/if_eql.h @@ -26,6 +26,7 @@ typedef struct slave { struct list_head list; struct net_device *dev; + netdevice_tracker dev_tracker; long priority; long priority_bps; long priority_Bps; -- cgit v1.2.3-59-g8ed1b From 19c9ebf6ed70856385296a65e78c1699081b152f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 6 Dec 2021 17:30:28 -0800 Subject: vlan: add net device refcount tracker Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/if_vlan.h | 3 +++ net/8021q/vlan_dev.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 41a518336673..8420fe504927 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -162,6 +162,7 @@ struct netpoll; * @vlan_id: VLAN identifier * @flags: device flags * @real_dev: underlying netdevice + * @dev_tracker: refcount tracker for @real_dev reference * @real_dev_addr: address of underlying netdevice * @dent: proc dir entry * @vlan_pcpu_stats: ptr to percpu rx stats @@ -177,6 +178,8 @@ struct vlan_dev_priv { u16 flags; struct net_device *real_dev; + netdevice_tracker dev_tracker; + unsigned char real_dev_addr[ETH_ALEN]; struct proc_dir_entry *dent; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 866556e041b7..26d031a43cc1 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -616,7 +616,7 @@ static int vlan_dev_init(struct net_device *dev) return -ENOMEM; /* Get vlan's reference to real_dev */ - dev_hold(real_dev); + dev_hold_track(real_dev, &vlan->dev_tracker, GFP_KERNEL); return 0; } @@ -848,7 +848,7 @@ static void vlan_dev_free(struct net_device *dev) vlan->vlan_pcpu_stats = NULL; /* Get rid of the vlan's reference to real_dev */ - dev_put(vlan->real_dev); + dev_put_track(vlan->real_dev, &vlan->dev_tracker); } void vlan_setup(struct net_device *dev) -- cgit v1.2.3-59-g8ed1b From f12bf6f3f942b37de65eeea8be25903587fec930 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 6 Dec 2021 17:30:30 -0800 Subject: net: watchdog: add net device refcount tracker Add a netdevice_tracker inside struct net_device, to track the self reference when a device has an active watchdog timer. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 2 ++ net/sched/sch_generic.c | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 69dca1edd5a6..1a748ee9a421 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1951,6 +1951,7 @@ enum netdev_ml_priv_type { * * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. * @linkwatch_dev_tracker: refcount tracker used by linkwatch. + * @watchdog_dev_tracker: refcount tracker used by watchdog. * * FIXME: cleanup struct net_device such that network protocol info * moves out. @@ -2282,6 +2283,7 @@ struct net_device { u8 dev_addr_shadow[MAX_ADDR_LEN]; netdevice_tracker linkwatch_dev_tracker; + netdevice_tracker watchdog_dev_tracker; }; #define to_net_dev(d) container_of(d, struct net_device, dev) diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 8c8fbf2e3856..b07bd1c7330f 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -499,6 +499,7 @@ EXPORT_SYMBOL(netif_tx_unlock); static void dev_watchdog(struct timer_list *t) { struct net_device *dev = from_timer(dev, t, watchdog_timer); + bool release = true; spin_lock(&dev->tx_global_lock); if (!qdisc_tx_is_noop(dev)) { @@ -534,12 +535,13 @@ static void dev_watchdog(struct timer_list *t) if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) - dev_hold(dev); + release = false; } } spin_unlock(&dev->tx_global_lock); - dev_put(dev); + if (release) + dev_put_track(dev, &dev->watchdog_dev_tracker); } void __netdev_watchdog_up(struct net_device *dev) @@ -549,7 +551,7 @@ void __netdev_watchdog_up(struct net_device *dev) dev->watchdog_timeo = 5*HZ; if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) - dev_hold(dev); + dev_hold_track(dev, &dev->watchdog_dev_tracker, GFP_ATOMIC); } } EXPORT_SYMBOL_GPL(__netdev_watchdog_up); @@ -563,7 +565,7 @@ static void dev_watchdog_down(struct net_device *dev) { netif_tx_lock_bh(dev); if (del_timer(&dev->watchdog_timer)) - dev_put(dev); + dev_put_track(dev, &dev->watchdog_dev_tracker); netif_tx_unlock_bh(dev); } -- cgit v1.2.3-59-g8ed1b From 330c6d3bfa268794bf692165d0f781f1c2d4d83e Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Wed, 24 Nov 2021 10:45:36 +0900 Subject: can: bittiming: replace CAN units with the generic ones from linux/units.h In [1], we introduced a set of units in linux/can/bittiming.h. Since then, generic SI prefixes were added to linux/units.h in [2]. Those new prefixes can perfectly replace CAN specific ones. This patch replaces all occurrences of the CAN units with their corresponding prefix (from linux/units) and the unit (as a comment) according to below table. CAN units SI metric prefix (from linux/units) + unit (as a comment) ------------------------------------------------------------------------ CAN_KBPS KILO /* BPS */ CAN_MBPS MEGA /* BPS */ CAM_MHZ MEGA /* Hz */ The definition are then removed from linux/can/bittiming.h [1] commit 1d7750760b70 ("can: bittiming: add CAN_KBPS, CAN_MBPS and CAN_MHZ macros") [2] commit 26471d4a6cf8 ("units: Add SI metric prefix definitions") Link: https://lore.kernel.org/all/20211124014536.782550-1-mailhol.vincent@wanadoo.fr Suggested-by: Jimmy Assarsson Suggested-by: Oliver Hartkopp Signed-off-by: Vincent Mailhol Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/bittiming.c | 5 +++-- drivers/net/can/usb/etas_es58x/es581_4.c | 5 +++-- drivers/net/can/usb/etas_es58x/es58x_fd.c | 5 +++-- include/linux/can/bittiming.h | 7 ------- 4 files changed, 9 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/bittiming.c b/drivers/net/can/dev/bittiming.c index 0509625c3082..d5fca3bfaf9a 100644 --- a/drivers/net/can/dev/bittiming.c +++ b/drivers/net/can/dev/bittiming.c @@ -4,6 +4,7 @@ * Copyright (C) 2008-2009 Wolfgang Grandegger */ +#include #include #ifdef CONFIG_CAN_CALC_BITTIMING @@ -81,9 +82,9 @@ int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, if (bt->sample_point) { sample_point_nominal = bt->sample_point; } else { - if (bt->bitrate > 800 * CAN_KBPS) + if (bt->bitrate > 800 * KILO /* BPS */) sample_point_nominal = 750; - else if (bt->bitrate > 500 * CAN_KBPS) + else if (bt->bitrate > 500 * KILO /* BPS */) sample_point_nominal = 800; else sample_point_nominal = 875; diff --git a/drivers/net/can/usb/etas_es58x/es581_4.c b/drivers/net/can/usb/etas_es58x/es581_4.c index 14e360c9f2c9..1bcdcece5ec7 100644 --- a/drivers/net/can/usb/etas_es58x/es581_4.c +++ b/drivers/net/can/usb/etas_es58x/es581_4.c @@ -10,6 +10,7 @@ */ #include +#include #include #include "es58x_core.h" @@ -469,8 +470,8 @@ const struct es58x_parameters es581_4_param = { .bittiming_const = &es581_4_bittiming_const, .data_bittiming_const = NULL, .tdc_const = NULL, - .bitrate_max = 1 * CAN_MBPS, - .clock = {.freq = 50 * CAN_MHZ}, + .bitrate_max = 1 * MEGA /* BPS */, + .clock = {.freq = 50 * MEGA /* Hz */}, .ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC, .tx_start_of_frame = 0xAFAF, .rx_start_of_frame = 0xFAFA, diff --git a/drivers/net/can/usb/etas_es58x/es58x_fd.c b/drivers/net/can/usb/etas_es58x/es58x_fd.c index 4f0cae29f4d8..ec87126e1a7d 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_fd.c +++ b/drivers/net/can/usb/etas_es58x/es58x_fd.c @@ -12,6 +12,7 @@ */ #include +#include #include #include "es58x_core.h" @@ -522,8 +523,8 @@ const struct es58x_parameters es58x_fd_param = { * Mbps work in an optimal environment but are not recommended * for production environment. */ - .bitrate_max = 8 * CAN_MBPS, - .clock = {.freq = 80 * CAN_MHZ}, + .bitrate_max = 8 * MEGA /* BPS */, + .clock = {.freq = 80 * MEGA /* Hz */}, .ctrlmode_supported = CAN_CTRLMODE_LOOPBACK | CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_3_SAMPLES | CAN_CTRLMODE_FD | CAN_CTRLMODE_FD_NON_ISO | CAN_CTRLMODE_CC_LEN8_DLC | CAN_CTRLMODE_TDC_AUTO, diff --git a/include/linux/can/bittiming.h b/include/linux/can/bittiming.h index 20b50baf3a02..a81652d1c6f3 100644 --- a/include/linux/can/bittiming.h +++ b/include/linux/can/bittiming.h @@ -12,13 +12,6 @@ #define CAN_SYNC_SEG 1 -/* Kilobits and Megabits per second */ -#define CAN_KBPS 1000UL -#define CAN_MBPS 1000000UL - -/* Megahertz */ -#define CAN_MHZ 1000000UL - #define CAN_CTRLMODE_TDC_MASK \ (CAN_CTRLMODE_TDC_AUTO | CAN_CTRLMODE_TDC_MANUAL) -- cgit v1.2.3-59-g8ed1b From 815f0e738a8d5663a02350e2580706829144a722 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Wed, 3 Nov 2021 09:50:59 +0100 Subject: clk: gate: Add devm_clk_hw_register_gate() Add devm_clk_hw_register_gate() - devres-managed version of clk_hw_register_gate() Suggested-by: Stephen Boyd Signed-off-by: Horatiu Vultur Acked-by: Nicolas Ferre Signed-off-by: Nicolas Ferre Link: https://lore.kernel.org/r/20211103085102.1656081-2-horatiu.vultur@microchip.com --- drivers/clk/clk-gate.c | 35 +++++++++++++++++++++++++++++++++++ include/linux/clk-provider.h | 23 +++++++++++++++++++++++ 2 files changed, 58 insertions(+) (limited to 'include/linux') diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c index 070dc47e95a1..64283807600b 100644 --- a/drivers/clk/clk-gate.c +++ b/drivers/clk/clk-gate.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -222,3 +223,37 @@ void clk_hw_unregister_gate(struct clk_hw *hw) kfree(gate); } EXPORT_SYMBOL_GPL(clk_hw_unregister_gate); + +static void devm_clk_hw_release_gate(struct device *dev, void *res) +{ + clk_hw_unregister_gate(*(struct clk_hw **)res); +} + +struct clk_hw *__devm_clk_hw_register_gate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, + unsigned long flags, + void __iomem *reg, u8 bit_idx, + u8 clk_gate_flags, spinlock_t *lock) +{ + struct clk_hw **ptr, *hw; + + ptr = devres_alloc(devm_clk_hw_release_gate, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + hw = __clk_hw_register_gate(dev, np, name, parent_name, parent_hw, + parent_data, flags, reg, bit_idx, + clk_gate_flags, lock); + + if (!IS_ERR(hw)) { + *ptr = hw; + devres_add(dev, ptr); + } else { + devres_free(ptr); + } + + return hw; +} +EXPORT_SYMBOL_GPL(__devm_clk_hw_register_gate); diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index f59c875271a0..2faa6f7aa8a8 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -490,6 +490,13 @@ struct clk_hw *__clk_hw_register_gate(struct device *dev, unsigned long flags, void __iomem *reg, u8 bit_idx, u8 clk_gate_flags, spinlock_t *lock); +struct clk_hw *__devm_clk_hw_register_gate(struct device *dev, + struct device_node *np, const char *name, + const char *parent_name, const struct clk_hw *parent_hw, + const struct clk_parent_data *parent_data, + unsigned long flags, + void __iomem *reg, u8 bit_idx, + u8 clk_gate_flags, spinlock_t *lock); struct clk *clk_register_gate(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 bit_idx, @@ -544,6 +551,22 @@ struct clk *clk_register_gate(struct device *dev, const char *name, __clk_hw_register_gate((dev), NULL, (name), NULL, NULL, (parent_data), \ (flags), (reg), (bit_idx), \ (clk_gate_flags), (lock)) +/** + * devm_clk_hw_register_gate - register a gate clock with the clock framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_name: name of this clock's parent + * @flags: framework-specific flags for this clock + * @reg: register address to control gating of this clock + * @bit_idx: which bit in the register controls gating of this clock + * @clk_gate_flags: gate-specific flags for this clock + * @lock: shared register lock for this clock + */ +#define devm_clk_hw_register_gate(dev, name, parent_name, flags, reg, bit_idx,\ + clk_gate_flags, lock) \ + __devm_clk_hw_register_gate((dev), NULL, (name), (parent_name), NULL, \ + NULL, (flags), (reg), (bit_idx), \ + (clk_gate_flags), (lock)) void clk_unregister_gate(struct clk *clk); void clk_hw_unregister_gate(struct clk_hw *hw); int clk_gate_is_enabled(struct clk_hw *hw); -- cgit v1.2.3-59-g8ed1b From 74d9555580c48a04b2c3b742dfb0c80777aa0b26 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 8 Nov 2021 16:09:41 +0000 Subject: PM: hibernate: Allow ACPI hardware signature to be honoured Theoretically, when the hardware signature in FACS changes, the OS is supposed to gracefully decline to attempt to resume from S4: "If the signature has changed, OSPM will not restore the system context and can boot from scratch" In practice, Windows doesn't do this and many laptop vendors do allow the signature to change especially when docking/undocking, so it would be a bad idea to simply comply with the specification by default in the general case. However, there are use cases where we do want the compliant behaviour and we know it's safe. Specifically, when resuming virtual machines where we know the hypervisor has changed sufficiently that resume will fail. We really want to be able to *tell* the guest kernel not to try, so it boots cleanly and doesn't just crash. This patch provides a way to opt in to the spec-compliant behaviour on the command line. A follow-up patch may do this automatically for certain "known good" machines based on a DMI match, or perhaps just for all hypervisor guests since there's no good reason a hypervisor would change the hardware_signature that it exposes to guests *unless* it wants them to obey the ACPI specification. Signed-off-by: David Woodhouse Signed-off-by: Rafael J. Wysocki --- Documentation/admin-guide/kernel-parameters.txt | 15 +++++++++++--- arch/x86/kernel/acpi/sleep.c | 4 +++- drivers/acpi/sleep.c | 26 ++++++++++++++++++++----- include/linux/acpi.h | 2 +- include/linux/suspend.h | 1 + kernel/power/power.h | 1 + kernel/power/swap.c | 16 +++++++++++++-- 7 files changed, 53 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9725c546a0d4..10e0a3a27516 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -225,14 +225,23 @@ For broken nForce2 BIOS resulting in XT-PIC timer. acpi_sleep= [HW,ACPI] Sleep options - Format: { s3_bios, s3_mode, s3_beep, s4_nohwsig, - old_ordering, nonvs, sci_force_enable, nobl } + Format: { s3_bios, s3_mode, s3_beep, s4_hwsig, + s4_nohwsig, old_ordering, nonvs, + sci_force_enable, nobl } See Documentation/power/video.rst for information on s3_bios and s3_mode. s3_beep is for debugging; it makes the PC's speaker beep as soon as the kernel's real-mode entry point is called. + s4_hwsig causes the kernel to check the ACPI hardware + signature during resume from hibernation, and gracefully + refuse to resume if it has changed. This complies with + the ACPI specification but not with reality, since + Windows does not do this and many laptops do change it + on docking. So the default behaviour is to allow resume + and simply warn when the signature changes, unless the + s4_hwsig option is enabled. s4_nohwsig prevents ACPI hardware signature from being - used during resume from hibernation. + used (or even warned about) during resume. old_ordering causes the ACPI 1.0 ordering of the _PTS control method, with respect to putting devices into low power states, to be enforced (the ACPI 2.0 ordering diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 3f85fcae450c..1e97f944b47d 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -139,8 +139,10 @@ static int __init acpi_sleep_setup(char *str) if (strncmp(str, "s3_beep", 7) == 0) acpi_realmode_flags |= 4; #ifdef CONFIG_HIBERNATION + if (strncmp(str, "s4_hwsig", 8) == 0) + acpi_check_s4_hw_signature(1); if (strncmp(str, "s4_nohwsig", 10) == 0) - acpi_no_s4_hw_signature(); + acpi_check_s4_hw_signature(0); #endif if (strncmp(str, "nonvs", 5) == 0) acpi_nvs_nosave(); diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index eaa47753b758..4dfbfc69db34 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -877,11 +877,11 @@ static inline void acpi_sleep_syscore_init(void) {} #ifdef CONFIG_HIBERNATION static unsigned long s4_hardware_signature; static struct acpi_table_facs *facs; -static bool nosigcheck; +static int sigcheck = -1; /* Default behaviour is just to warn */ -void __init acpi_no_s4_hw_signature(void) +void __init acpi_check_s4_hw_signature(int check) { - nosigcheck = true; + sigcheck = check; } static int acpi_hibernation_begin(pm_message_t stage) @@ -1009,12 +1009,28 @@ static void acpi_sleep_hibernate_setup(void) hibernation_set_ops(old_suspend_ordering ? &acpi_hibernation_ops_old : &acpi_hibernation_ops); sleep_states[ACPI_STATE_S4] = 1; - if (nosigcheck) + if (!sigcheck) return; acpi_get_table(ACPI_SIG_FACS, 1, (struct acpi_table_header **)&facs); - if (facs) + if (facs) { + /* + * s4_hardware_signature is the local variable which is just + * used to warn about mismatch after we're attempting to + * resume (in violation of the ACPI specification.) + */ s4_hardware_signature = facs->hardware_signature; + + if (sigcheck > 0) { + /* + * If we're actually obeying the ACPI specification + * then the signature is written out as part of the + * swsusp header, in order to allow the boot kernel + * to gracefully decline to resume. + */ + swsusp_hardware_signature = facs->hardware_signature; + } + } } #else /* !CONFIG_HIBERNATION */ static inline void acpi_sleep_hibernate_setup(void) {} diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b28f8790192a..6c0798db6bde 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -506,7 +506,7 @@ acpi_status acpi_release_memory(acpi_handle handle, struct resource *res, int acpi_resources_are_enforced(void); #ifdef CONFIG_HIBERNATION -void __init acpi_no_s4_hw_signature(void); +void __init acpi_check_s4_hw_signature(int check); #endif #ifdef CONFIG_PM_SLEEP diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 8af13ba60c7e..5785d909c321 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -446,6 +446,7 @@ extern unsigned long get_safe_page(gfp_t gfp_mask); extern asmlinkage int swsusp_arch_suspend(void); extern asmlinkage int swsusp_arch_resume(void); +extern u32 swsusp_hardware_signature; extern void hibernation_set_ops(const struct platform_hibernation_ops *ops); extern int hibernate(void); extern bool system_entering_hibernation(void); diff --git a/kernel/power/power.h b/kernel/power/power.h index 326f8d032eb5..b4f433943209 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -170,6 +170,7 @@ extern int swsusp_swap_in_use(void); #define SF_PLATFORM_MODE 1 #define SF_NOCOMPRESS_MODE 2 #define SF_CRC32_MODE 4 +#define SF_HW_SIG 8 /* kernel/power/hibernate.c */ extern int swsusp_check(void); diff --git a/kernel/power/swap.c b/kernel/power/swap.c index ff326c2cb77b..ad10359030a4 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -36,6 +36,8 @@ #define HIBERNATE_SIG "S1SUSPEND" +u32 swsusp_hardware_signature; + /* * When reading an {un,}compressed image, we may restore pages in place, * in which case some architectures need these pages cleaning before they @@ -104,7 +106,8 @@ struct swap_map_handle { struct swsusp_header { char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) - - sizeof(u32)]; + sizeof(u32) - sizeof(u32)]; + u32 hw_sig; u32 crc32; sector_t image; unsigned int flags; /* Flags to pass to the "boot" kernel */ @@ -312,7 +315,6 @@ static int hib_wait_io(struct hib_bio_batch *hb) /* * Saving part */ - static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) { int error; @@ -324,6 +326,10 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10); memcpy(swsusp_header->sig, HIBERNATE_SIG, 10); swsusp_header->image = handle->first_sector; + if (swsusp_hardware_signature) { + swsusp_header->hw_sig = swsusp_hardware_signature; + flags |= SF_HW_SIG; + } swsusp_header->flags = flags; if (flags & SF_CRC32_MODE) swsusp_header->crc32 = handle->crc32; @@ -1537,6 +1543,12 @@ int swsusp_check(void) } else { error = -EINVAL; } + if (!error && swsusp_header->flags & SF_HW_SIG && + swsusp_header->hw_sig != swsusp_hardware_signature) { + pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n", + swsusp_header->hw_sig, swsusp_hardware_signature); + error = -EINVAL; + } put: if (error) -- cgit v1.2.3-59-g8ed1b From 8260b9820f7050461b8969305bbd8cb5654f0c74 Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Mon, 6 Dec 2021 16:55:03 +0300 Subject: x86/sev: Use CC_ATTR attribute to generalize string I/O unroll INS/OUTS are not supported in TDX guests and cause #UD. Kernel has to avoid them when running in TDX guest. To support existing usage, string I/O operations are unrolled using IN/OUT instructions. AMD SEV platform implements this support by adding unroll logic in ins#bwl()/outs#bwl() macros with SEV-specific checks. Since TDX VM guests will also need similar support, use CC_ATTR_GUEST_UNROLL_STRING_IO and generic cc_platform_has() API to implement it. String I/O helpers were the last users of sev_key_active() interface and sev_enable_key static key. Remove them. [ bp: Move comment too and do not delete it. ] Suggested-by: Tom Lendacky Signed-off-by: Kuppuswamy Sathyanarayanan Signed-off-by: Kirill A. Shutemov Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Reviewed-by: Tom Lendacky Tested-by: Tom Lendacky Link: https://lkml.kernel.org/r/20211206135505.75045-2-kirill.shutemov@linux.intel.com --- arch/x86/include/asm/io.h | 20 +++----------------- arch/x86/kernel/cc_platform.c | 8 ++++++++ arch/x86/mm/mem_encrypt.c | 10 ---------- include/linux/cc_platform.h | 11 +++++++++++ 4 files changed, 22 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 5c6a4af0b911..f6d91ecb8026 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -256,21 +257,6 @@ static inline void slow_down_io(void) #endif -#ifdef CONFIG_AMD_MEM_ENCRYPT -#include - -extern struct static_key_false sev_enable_key; -static inline bool sev_key_active(void) -{ - return static_branch_unlikely(&sev_enable_key); -} - -#else /* !CONFIG_AMD_MEM_ENCRYPT */ - -static inline bool sev_key_active(void) { return false; } - -#endif /* CONFIG_AMD_MEM_ENCRYPT */ - #define BUILDIO(bwl, bw, type) \ static inline void out##bwl(unsigned type value, int port) \ { \ @@ -301,7 +287,7 @@ static inline unsigned type in##bwl##_p(int port) \ \ static inline void outs##bwl(int port, const void *addr, unsigned long count) \ { \ - if (sev_key_active()) { \ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { \ unsigned type *value = (unsigned type *)addr; \ while (count) { \ out##bwl(*value, port); \ @@ -317,7 +303,7 @@ static inline void outs##bwl(int port, const void *addr, unsigned long count) \ \ static inline void ins##bwl(int port, void *addr, unsigned long count) \ { \ - if (sev_key_active()) { \ + if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { \ unsigned type *value = (unsigned type *)addr; \ while (count) { \ *value = in##bwl(port); \ diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/kernel/cc_platform.c index 03bb2f343ddb..8a25b1c0d480 100644 --- a/arch/x86/kernel/cc_platform.c +++ b/arch/x86/kernel/cc_platform.c @@ -50,6 +50,14 @@ static bool amd_cc_platform_has(enum cc_attr attr) case CC_ATTR_GUEST_STATE_ENCRYPT: return sev_status & MSR_AMD64_SEV_ES_ENABLED; + /* + * With SEV, the rep string I/O instructions need to be unrolled + * but SEV-ES supports them through the #VC handler. + */ + case CC_ATTR_GUEST_UNROLL_STRING_IO: + return (sev_status & MSR_AMD64_SEV_ENABLED) && + !(sev_status & MSR_AMD64_SEV_ES_ENABLED); + default: return false; } diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 35487305d8af..b520021a7e7b 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -43,8 +43,6 @@ u64 sme_me_mask __section(".data") = 0; u64 sev_status __section(".data") = 0; u64 sev_check_data __section(".data") = 0; EXPORT_SYMBOL(sme_me_mask); -DEFINE_STATIC_KEY_FALSE(sev_enable_key); -EXPORT_SYMBOL_GPL(sev_enable_key); /* Buffer used for early in-place encryption by BSP, no locking needed */ static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE); @@ -499,14 +497,6 @@ void __init mem_encrypt_init(void) /* Call into SWIOTLB to update the SWIOTLB DMA buffers */ swiotlb_update_mem_attributes(); - /* - * With SEV, we need to unroll the rep string I/O instructions, - * but SEV-ES supports them through the #VC handler. - */ - if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) && - !cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) - static_branch_enable(&sev_enable_key); - print_mem_encrypt_feature_info(); } diff --git a/include/linux/cc_platform.h b/include/linux/cc_platform.h index a075b70b9a70..efd8205282da 100644 --- a/include/linux/cc_platform.h +++ b/include/linux/cc_platform.h @@ -61,6 +61,17 @@ enum cc_attr { * Examples include SEV-ES. */ CC_ATTR_GUEST_STATE_ENCRYPT, + + /** + * @CC_ATTR_GUEST_UNROLL_STRING_IO: String I/O is implemented with + * IN/OUT instructions + * + * The platform/OS is running as a guest/virtual machine and uses + * IN/OUT instructions in place of string I/O. + * + * Examples include TDX guest & SEV. + */ + CC_ATTR_GUEST_UNROLL_STRING_IO, }; #ifdef CONFIG_ARCH_HAS_CC_PLATFORM -- cgit v1.2.3-59-g8ed1b From 3f9bb0301d50ce27421eff4b710c2bbe58111a83 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Mon, 6 Dec 2021 18:57:47 +0200 Subject: net: dsa: make dp->bridge_num one-based MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I have seen too many bugs already due to the fact that we must encode an invalid dp->bridge_num as a negative value, because the natural tendency is to check that invalid value using (!dp->bridge_num). Latest example can be seen in commit 1bec0f05062c ("net: dsa: fix bridge_num not getting cleared after ports leaving the bridge"). Convert the existing users to assume that dp->bridge_num == 0 is the encoding for invalid, and valid bridge numbers start from 1. Signed-off-by: Vladimir Oltean Reviewed-by: Alvin Šipraga Signed-off-by: Jakub Kicinski --- drivers/net/dsa/mv88e6xxx/chip.c | 12 ++++++------ include/linux/dsa/8021q.h | 6 +++--- include/net/dsa.h | 6 +++--- net/dsa/dsa2.c | 24 ++++++++++++------------ net/dsa/dsa_priv.h | 5 +++-- net/dsa/port.c | 11 ++++++----- net/dsa/tag_8021q.c | 12 +++++++----- net/dsa/tag_dsa.c | 2 +- 8 files changed, 41 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index f00cbf5753b9..de3401b2c86c 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -1250,10 +1250,10 @@ static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port) /* dev is a virtual bridge */ } else { list_for_each_entry(dp, &dst->ports, list) { - if (dp->bridge_num < 0) + if (!dp->bridge_num) continue; - if (dp->bridge_num + 1 + dst->last_switch != dev) + if (dp->bridge_num + dst->last_switch != dev) continue; br = dp->bridge_dev; @@ -2527,9 +2527,9 @@ static void mv88e6xxx_crosschip_bridge_leave(struct dsa_switch *ds, * physical switches, so start from beyond that range. */ static int mv88e6xxx_map_virtual_bridge_to_pvt(struct dsa_switch *ds, - int bridge_num) + unsigned int bridge_num) { - u8 dev = bridge_num + ds->dst->last_switch + 1; + u8 dev = bridge_num + ds->dst->last_switch; struct mv88e6xxx_chip *chip = ds->priv; int err; @@ -2542,14 +2542,14 @@ static int mv88e6xxx_map_virtual_bridge_to_pvt(struct dsa_switch *ds, static int mv88e6xxx_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num) + unsigned int bridge_num) { return mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge_num); } static void mv88e6xxx_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num) + unsigned int bridge_num) { int err; diff --git a/include/linux/dsa/8021q.h b/include/linux/dsa/8021q.h index 254b165f2b44..0af4371fbebb 100644 --- a/include/linux/dsa/8021q.h +++ b/include/linux/dsa/8021q.h @@ -38,13 +38,13 @@ void dsa_8021q_rcv(struct sk_buff *skb, int *source_port, int *switch_id); int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num); + unsigned int bridge_num); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num); + unsigned int bridge_num); -u16 dsa_8021q_bridge_tx_fwd_offload_vid(int bridge_num); +u16 dsa_8021q_bridge_tx_fwd_offload_vid(unsigned int bridge_num); u16 dsa_tag_8021q_tx_vid(const struct dsa_port *dp); diff --git a/include/net/dsa.h b/include/net/dsa.h index 8ca9d50cbbc2..a23cfbaa09d6 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -257,7 +257,7 @@ struct dsa_port { bool learning; u8 stp_state; struct net_device *bridge_dev; - int bridge_num; + unsigned int bridge_num; struct devlink_port devlink_port; bool devlink_port_setup; struct phylink *pl; @@ -754,11 +754,11 @@ struct dsa_switch_ops { /* Called right after .port_bridge_join() */ int (*port_bridge_tx_fwd_offload)(struct dsa_switch *ds, int port, struct net_device *bridge, - int bridge_num); + unsigned int bridge_num); /* Called right before .port_bridge_leave() */ void (*port_bridge_tx_fwd_unoffload)(struct dsa_switch *ds, int port, struct net_device *bridge, - int bridge_num); + unsigned int bridge_num); void (*port_stp_state_set)(struct dsa_switch *ds, int port, u8 state); void (*port_fast_age)(struct dsa_switch *ds, int port); diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index 826957b6442b..9606e56710a5 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -141,23 +141,23 @@ static int dsa_bridge_num_find(const struct net_device *bridge_dev) */ list_for_each_entry(dst, &dsa_tree_list, list) list_for_each_entry(dp, &dst->ports, list) - if (dp->bridge_dev == bridge_dev && - dp->bridge_num != -1) + if (dp->bridge_dev == bridge_dev && dp->bridge_num) return dp->bridge_num; - return -1; + return 0; } -int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) +unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) { - int bridge_num = dsa_bridge_num_find(bridge_dev); + unsigned int bridge_num = dsa_bridge_num_find(bridge_dev); - if (bridge_num < 0) { + if (!bridge_num) { /* First port that offloads TX forwarding for this bridge */ - bridge_num = find_first_zero_bit(&dsa_fwd_offloading_bridges, - DSA_MAX_NUM_OFFLOADING_BRIDGES); + bridge_num = find_next_zero_bit(&dsa_fwd_offloading_bridges, + DSA_MAX_NUM_OFFLOADING_BRIDGES, + 1); if (bridge_num >= max) - return -1; + return 0; set_bit(bridge_num, &dsa_fwd_offloading_bridges); } @@ -165,12 +165,13 @@ int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) return bridge_num; } -void dsa_bridge_num_put(const struct net_device *bridge_dev, int bridge_num) +void dsa_bridge_num_put(const struct net_device *bridge_dev, + unsigned int bridge_num) { /* Check if the bridge is still in use, otherwise it is time * to clean it up so we can reuse this bridge_num later. */ - if (dsa_bridge_num_find(bridge_dev) < 0) + if (!dsa_bridge_num_find(bridge_dev)) clear_bit(bridge_num, &dsa_fwd_offloading_bridges); } @@ -1184,7 +1185,6 @@ static struct dsa_port *dsa_port_touch(struct dsa_switch *ds, int index) dp->ds = ds; dp->index = index; - dp->bridge_num = -1; INIT_LIST_HEAD(&dp->list); list_add_tail(&dp->list, &dst->ports); diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index 3fb2c37c9b88..70c4a5b36a8b 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -546,8 +546,9 @@ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, struct net_device *master, const struct dsa_device_ops *tag_ops, const struct dsa_device_ops *old_tag_ops); -int dsa_bridge_num_get(const struct net_device *bridge_dev, int max); -void dsa_bridge_num_put(const struct net_device *bridge_dev, int bridge_num); +unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max); +void dsa_bridge_num_put(const struct net_device *bridge_dev, + unsigned int bridge_num); /* tag_8021q.c */ int dsa_tag_8021q_bridge_join(struct dsa_switch *ds, diff --git a/net/dsa/port.c b/net/dsa/port.c index 6d5ebe61280b..9a77bd1373e2 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -273,14 +273,14 @@ static void dsa_port_switchdev_unsync_attrs(struct dsa_port *dp) static void dsa_port_bridge_tx_fwd_unoffload(struct dsa_port *dp, struct net_device *bridge_dev) { - int bridge_num = dp->bridge_num; + unsigned int bridge_num = dp->bridge_num; struct dsa_switch *ds = dp->ds; /* No bridge TX forwarding offload => do nothing */ - if (!ds->ops->port_bridge_tx_fwd_unoffload || dp->bridge_num == -1) + if (!ds->ops->port_bridge_tx_fwd_unoffload || !dp->bridge_num) return; - dp->bridge_num = -1; + dp->bridge_num = 0; dsa_bridge_num_put(bridge_dev, bridge_num); @@ -295,14 +295,15 @@ static bool dsa_port_bridge_tx_fwd_offload(struct dsa_port *dp, struct net_device *bridge_dev) { struct dsa_switch *ds = dp->ds; - int bridge_num, err; + unsigned int bridge_num; + int err; if (!ds->ops->port_bridge_tx_fwd_offload) return false; bridge_num = dsa_bridge_num_get(bridge_dev, ds->num_fwd_offloading_bridges); - if (bridge_num < 0) + if (!bridge_num) return false; dp->bridge_num = bridge_num; diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c index 72cac2c0af7b..df59f16436a5 100644 --- a/net/dsa/tag_8021q.c +++ b/net/dsa/tag_8021q.c @@ -67,10 +67,12 @@ #define DSA_8021Q_PORT(x) (((x) << DSA_8021Q_PORT_SHIFT) & \ DSA_8021Q_PORT_MASK) -u16 dsa_8021q_bridge_tx_fwd_offload_vid(int bridge_num) +u16 dsa_8021q_bridge_tx_fwd_offload_vid(unsigned int bridge_num) { - /* The VBID value of 0 is reserved for precise TX */ - return DSA_8021Q_DIR_TX | DSA_8021Q_VBID(bridge_num + 1); + /* The VBID value of 0 is reserved for precise TX, but it is also + * reserved/invalid for the bridge_num, so all is well. + */ + return DSA_8021Q_DIR_TX | DSA_8021Q_VBID(bridge_num); } EXPORT_SYMBOL_GPL(dsa_8021q_bridge_tx_fwd_offload_vid); @@ -409,7 +411,7 @@ int dsa_tag_8021q_bridge_leave(struct dsa_switch *ds, int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num) + unsigned int bridge_num) { u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); @@ -420,7 +422,7 @@ EXPORT_SYMBOL_GPL(dsa_tag_8021q_bridge_tx_fwd_offload); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, struct net_device *br, - int bridge_num) + unsigned int bridge_num) { u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); diff --git a/net/dsa/tag_dsa.c b/net/dsa/tag_dsa.c index b3da4b2ea11c..a7d70ae7cc97 100644 --- a/net/dsa/tag_dsa.c +++ b/net/dsa/tag_dsa.c @@ -140,7 +140,7 @@ static struct sk_buff *dsa_xmit_ll(struct sk_buff *skb, struct net_device *dev, * packets on behalf of a virtual switch device with an index * past the physical switches. */ - tag_dev = dst->last_switch + 1 + dp->bridge_num; + tag_dev = dst->last_switch + dp->bridge_num; tag_port = 0; } else { cmd = DSA_CMD_FROM_CPU; -- cgit v1.2.3-59-g8ed1b From d3eed0e57d5d1bcbf1bd60f83a4adfe7d7b8dd9c Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Mon, 6 Dec 2021 18:57:56 +0200 Subject: net: dsa: keep the bridge_dev and bridge_num as part of the same structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main desire behind this is to provide coherent bridge information to the fast path without locking. For example, right now we set dp->bridge_dev and dp->bridge_num from separate code paths, it is theoretically possible for a packet transmission to read these two port properties consecutively and find a bridge number which does not correspond with the bridge device. Another desire is to start passing more complex bridge information to dsa_switch_ops functions. For example, with FDB isolation, it is expected that drivers will need to be passed the bridge which requested an FDB/MDB entry to be offloaded, and along with that bridge_dev, the associated bridge_num should be passed too, in case the driver might want to implement an isolation scheme based on that number. We already pass the {bridge_dev, bridge_num} pair to the TX forwarding offload switch API, however we'd like to remove that and squash it into the basic bridge join/leave API. So that means we need to pass this pair to the bridge join/leave API. During dsa_port_bridge_leave, first we unset dp->bridge_dev, then we call the driver's .port_bridge_leave with what used to be our dp->bridge_dev, but provided as an argument. When bridge_dev and bridge_num get folded into a single structure, we need to preserve this behavior in dsa_port_bridge_leave: we need a copy of what used to be in dp->bridge. Switch drivers check bridge membership by comparing dp->bridge_dev with the provided bridge_dev, but now, if we provide the struct dsa_bridge as a pointer, they cannot keep comparing dp->bridge to the provided pointer, since this only points to an on-stack copy. To make this obvious and prevent driver writers from forgetting and doing stupid things, in this new API, the struct dsa_bridge is provided as a full structure (not very large, contains an int and a pointer) instead of a pointer. An explicit comparison function needs to be used to determine bridge membership: dsa_port_offloads_bridge(). Signed-off-by: Vladimir Oltean Reviewed-by: Alvin Šipraga Signed-off-by: Jakub Kicinski --- drivers/net/dsa/b53/b53_common.c | 8 ++-- drivers/net/dsa/b53/b53_priv.h | 4 +- drivers/net/dsa/dsa_loop.c | 8 ++-- drivers/net/dsa/hirschmann/hellcreek.c | 4 +- drivers/net/dsa/lan9303-core.c | 4 +- drivers/net/dsa/lantiq_gswip.c | 14 ++++--- drivers/net/dsa/microchip/ksz_common.c | 4 +- drivers/net/dsa/microchip/ksz_common.h | 4 +- drivers/net/dsa/mt7530.c | 8 ++-- drivers/net/dsa/mv88e6xxx/chip.c | 26 ++++++------- drivers/net/dsa/ocelot/felix.c | 8 ++-- drivers/net/dsa/qca8k.c | 12 +++--- drivers/net/dsa/rtl8366rb.c | 8 ++-- drivers/net/dsa/sja1105/sja1105_main.c | 12 +++--- drivers/net/dsa/xrs700x/xrs700x.c | 10 ++--- include/linux/dsa/8021q.h | 7 ++-- include/net/dsa.h | 34 ++++++++++------- net/dsa/dsa2.c | 43 ++++++++++++++------- net/dsa/dsa_priv.h | 10 +++-- net/dsa/port.c | 70 ++++++++++++++++++---------------- net/dsa/slave.c | 2 +- net/dsa/switch.c | 13 ++++--- net/dsa/tag_8021q.c | 12 +++--- 23 files changed, 177 insertions(+), 148 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index d5e78f51f42d..4e41b1a63108 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -1860,7 +1860,7 @@ int b53_mdb_del(struct dsa_switch *ds, int port, } EXPORT_SYMBOL(b53_mdb_del); -int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) +int b53_br_join(struct dsa_switch *ds, int port, struct dsa_bridge bridge) { struct b53_device *dev = ds->priv; s8 cpu_port = dsa_to_port(ds, port)->cpu_dp->index; @@ -1887,7 +1887,7 @@ int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan); b53_for_each_port(dev, i) { - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Add this local port to the remote port VLAN control @@ -1911,7 +1911,7 @@ int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) } EXPORT_SYMBOL(b53_br_join); -void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *br) +void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge) { struct b53_device *dev = ds->priv; struct b53_vlan *vl = &dev->vlans[0]; @@ -1923,7 +1923,7 @@ void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *br) b53_for_each_port(dev, i) { /* Don't touch the remaining ports */ - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), ®); diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h index 579da74ada64..ee17f8b516ca 100644 --- a/drivers/net/dsa/b53/b53_priv.h +++ b/drivers/net/dsa/b53/b53_priv.h @@ -324,8 +324,8 @@ void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset, void b53_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data); int b53_get_sset_count(struct dsa_switch *ds, int port, int sset); void b53_get_ethtool_phy_stats(struct dsa_switch *ds, int port, uint64_t *data); -int b53_br_join(struct dsa_switch *ds, int port, struct net_device *bridge); -void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *bridge); +int b53_br_join(struct dsa_switch *ds, int port, struct dsa_bridge bridge); +void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge); void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state); void b53_br_fast_age(struct dsa_switch *ds, int port); int b53_br_flags_pre(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c index e638e3eea911..70db3a9aa355 100644 --- a/drivers/net/dsa/dsa_loop.c +++ b/drivers/net/dsa/dsa_loop.c @@ -167,19 +167,19 @@ static int dsa_loop_phy_write(struct dsa_switch *ds, int port, } static int dsa_loop_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n", - __func__, port, bridge->name); + __func__, port, bridge.dev->name); return 0; } static void dsa_loop_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n", - __func__, port, bridge->name); + __func__, port, bridge.dev->name); } static void dsa_loop_port_stp_state_set(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c index 86839b43011b..c8dc83c69147 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.c +++ b/drivers/net/dsa/hirschmann/hellcreek.c @@ -674,7 +674,7 @@ static int hellcreek_bridge_flags(struct dsa_switch *ds, int port, } static int hellcreek_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct hellcreek *hellcreek = ds->priv; @@ -691,7 +691,7 @@ static int hellcreek_port_bridge_join(struct dsa_switch *ds, int port, } static void hellcreek_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct hellcreek *hellcreek = ds->priv; diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c index 1c2bdcde6979..29d909484275 100644 --- a/drivers/net/dsa/lan9303-core.c +++ b/drivers/net/dsa/lan9303-core.c @@ -1103,7 +1103,7 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port) } static int lan9303_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct lan9303 *chip = ds->priv; @@ -1117,7 +1117,7 @@ static int lan9303_port_bridge_join(struct dsa_switch *ds, int port, } static void lan9303_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct lan9303 *chip = ds->priv; diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c index 6317d0ae42d0..1f59fefc29c1 100644 --- a/drivers/net/dsa/lantiq_gswip.c +++ b/drivers/net/dsa/lantiq_gswip.c @@ -1146,16 +1146,17 @@ static int gswip_vlan_remove(struct gswip_priv *priv, } static int gswip_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { + struct net_device *br = bridge.dev; struct gswip_priv *priv = ds->priv; int err; /* When the bridge uses VLAN filtering we have to configure VLAN * specific bridges. No bridge is configured here. */ - if (!br_vlan_enabled(bridge)) { - err = gswip_vlan_add_unaware(priv, bridge, port); + if (!br_vlan_enabled(br)) { + err = gswip_vlan_add_unaware(priv, br, port); if (err) return err; priv->port_vlan_filter &= ~BIT(port); @@ -1166,8 +1167,9 @@ static int gswip_port_bridge_join(struct dsa_switch *ds, int port, } static void gswip_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { + struct net_device *br = bridge.dev; struct gswip_priv *priv = ds->priv; gswip_add_single_port_br(priv, port, true); @@ -1175,8 +1177,8 @@ static void gswip_port_bridge_leave(struct dsa_switch *ds, int port, /* When the bridge uses VLAN filtering we have to configure VLAN * specific bridges. No bridge is configured here. */ - if (!br_vlan_enabled(bridge)) - gswip_vlan_remove(priv, bridge, port, 0, true, false); + if (!br_vlan_enabled(br)) + gswip_vlan_remove(priv, br, port, 0, true, false); } static int gswip_port_vlan_prepare(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index cebcb73cda76..40d6e3f4deb5 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -192,7 +192,7 @@ void ksz_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *buf) EXPORT_SYMBOL_GPL(ksz_get_ethtool_stats); int ksz_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { /* port_stp_state_set() will be called after to put the port in * appropriate state so there is no need to do anything. @@ -203,7 +203,7 @@ int ksz_port_bridge_join(struct dsa_switch *ds, int port, EXPORT_SYMBOL_GPL(ksz_port_bridge_join); void ksz_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { /* port_stp_state_set() will be called after to put the port in * forwarding state so there is no need to do anything. diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index 54b456bc8972..88e5a5d56219 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -155,9 +155,9 @@ void ksz_mac_link_down(struct dsa_switch *ds, int port, unsigned int mode, int ksz_sset_count(struct dsa_switch *ds, int port, int sset); void ksz_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *buf); int ksz_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br); + struct dsa_bridge bridge); void ksz_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br); + struct dsa_bridge bridge); void ksz_port_fast_age(struct dsa_switch *ds, int port); int ksz_port_fdb_dump(struct dsa_switch *ds, int port, dsa_fdb_dump_cb_t *cb, void *data); diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 73c9f79f9e9f..5b74c542b1e6 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -1186,7 +1186,7 @@ mt7530_port_bridge_flags(struct dsa_switch *ds, int port, static int mt7530_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; u32 port_bitmap = BIT(MT7530_CPU_PORT); @@ -1204,7 +1204,7 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port, * same bridge. If the port is disabled, port matrix is kept * and not being setup until the port becomes enabled. */ - if (dsa_port_bridge_dev_get(other_dp) != bridge) + if (!dsa_port_offloads_bridge(other_dp, &bridge)) continue; if (priv->ports[other_port].enable) @@ -1303,7 +1303,7 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port) static void mt7530_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; struct mt7530_priv *priv = ds->priv; @@ -1320,7 +1320,7 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port, * in the same bridge. If the port is disabled, port matrix * is kept and not being setup until the port becomes enabled. */ - if (dsa_port_bridge_dev_get(other_dp) != bridge) + if (!dsa_port_offloads_bridge(other_dp, &bridge)) continue; if (priv->ports[other_port].enable) diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 5afc7a1c0dbb..aa5c5d4950d8 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -2410,7 +2410,7 @@ static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port, } static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, - struct net_device *br) + struct dsa_bridge bridge) { struct dsa_switch *ds = chip->ds; struct dsa_switch_tree *dst = ds->dst; @@ -2418,7 +2418,7 @@ static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, int err; list_for_each_entry(dp, &dst->ports, list) { - if (dsa_port_bridge_dev_get(dp) == br) { + if (dsa_port_offloads_bridge(dp, &bridge)) { if (dp->ds == ds) { /* This is a local bridge group member, * remap its Port VLAN Map. @@ -2442,14 +2442,14 @@ static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, } static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; int err; mv88e6xxx_reg_lock(chip); - err = mv88e6xxx_bridge_map(chip, br); + err = mv88e6xxx_bridge_map(chip, bridge); if (err) goto unlock; @@ -2464,14 +2464,14 @@ unlock: } static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; int err; mv88e6xxx_reg_lock(chip); - if (mv88e6xxx_bridge_map(chip, br) || + if (mv88e6xxx_bridge_map(chip, bridge) || mv88e6xxx_port_vlan_map(chip, port)) dev_err(ds->dev, "failed to remap in-chip Port VLAN\n"); @@ -2486,7 +2486,7 @@ static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port, static int mv88e6xxx_crosschip_bridge_join(struct dsa_switch *ds, int tree_index, int sw_index, - int port, struct net_device *br) + int port, struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; int err; @@ -2503,7 +2503,7 @@ static int mv88e6xxx_crosschip_bridge_join(struct dsa_switch *ds, static void mv88e6xxx_crosschip_bridge_leave(struct dsa_switch *ds, int tree_index, int sw_index, - int port, struct net_device *br) + int port, struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; @@ -2535,19 +2535,17 @@ static int mv88e6xxx_map_virtual_bridge_to_pvt(struct dsa_switch *ds, } static int mv88e6xxx_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num) + struct dsa_bridge bridge) { - return mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge_num); + return mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge.num); } static void mv88e6xxx_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num) + struct dsa_bridge bridge) { int err; - err = mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge_num); + err = mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge.num); if (err) { dev_err(ds->dev, "failed to remap cross-chip Port VLAN: %pe\n", ERR_PTR(err)); diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index 57beab3d3ff3..b7e5a5c611dd 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -706,21 +706,21 @@ static int felix_bridge_flags(struct dsa_switch *ds, int port, } static int felix_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct ocelot *ocelot = ds->priv; - ocelot_port_bridge_join(ocelot, port, br); + ocelot_port_bridge_join(ocelot, port, bridge.dev); return 0; } static void felix_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct ocelot *ocelot = ds->priv; - ocelot_port_bridge_leave(ocelot, port, br); + ocelot_port_bridge_leave(ocelot, port, bridge.dev); } static int felix_lag_join(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c index 7053a3510d71..dc983f79f0d6 100644 --- a/drivers/net/dsa/qca8k.c +++ b/drivers/net/dsa/qca8k.c @@ -1810,8 +1810,8 @@ qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) QCA8K_PORT_LOOKUP_STATE_MASK, stp_state); } -static int -qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) +static int qca8k_port_bridge_join(struct dsa_switch *ds, int port, + struct dsa_bridge bridge) { struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv; int port_mask, cpu_port; @@ -1823,7 +1823,7 @@ qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) for (i = 0; i < QCA8K_NUM_PORTS; i++) { if (dsa_is_cpu_port(ds, i)) continue; - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Add this port to the portvlan mask of the other ports * in the bridge @@ -1844,8 +1844,8 @@ qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) return ret; } -static void -qca8k_port_bridge_leave(struct dsa_switch *ds, int port, struct net_device *br) +static void qca8k_port_bridge_leave(struct dsa_switch *ds, int port, + struct dsa_bridge bridge) { struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv; int cpu_port, i; @@ -1855,7 +1855,7 @@ qca8k_port_bridge_leave(struct dsa_switch *ds, int port, struct net_device *br) for (i = 0; i < QCA8K_NUM_PORTS; i++) { if (dsa_is_cpu_port(ds, i)) continue; - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Remove this port to the portvlan mask of the other ports * in the bridge diff --git a/drivers/net/dsa/rtl8366rb.c b/drivers/net/dsa/rtl8366rb.c index b6f277a04989..fac2333a3f5e 100644 --- a/drivers/net/dsa/rtl8366rb.c +++ b/drivers/net/dsa/rtl8366rb.c @@ -1186,7 +1186,7 @@ rtl8366rb_port_disable(struct dsa_switch *ds, int port) static int rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { struct realtek_smi *smi = ds->priv; unsigned int port_bitmap = 0; @@ -1198,7 +1198,7 @@ rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, if (i == port) continue; /* Not on this bridge */ - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Join this port to each other port on the bridge */ ret = regmap_update_bits(smi->map, RTL8366RB_PORT_ISO(i), @@ -1218,7 +1218,7 @@ rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, static void rtl8366rb_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { struct realtek_smi *smi = ds->priv; unsigned int port_bitmap = 0; @@ -1230,7 +1230,7 @@ rtl8366rb_port_bridge_leave(struct dsa_switch *ds, int port, if (i == port) continue; /* Not on this bridge */ - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Remove this port from any other port on the bridge */ ret = regmap_update_bits(smi->map, RTL8366RB_PORT_ISO(i), diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 5e03eda4c16f..24584fe2e760 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -1980,7 +1980,7 @@ static int sja1105_manage_flood_domains(struct sja1105_private *priv) } static int sja1105_bridge_member(struct dsa_switch *ds, int port, - struct net_device *br, bool member) + struct dsa_bridge bridge, bool member) { struct sja1105_l2_forwarding_entry *l2_fwd; struct sja1105_private *priv = ds->priv; @@ -2005,7 +2005,7 @@ static int sja1105_bridge_member(struct dsa_switch *ds, int port, */ if (i == port) continue; - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; sja1105_port_allow_traffic(l2_fwd, i, port, member); sja1105_port_allow_traffic(l2_fwd, port, i, member); @@ -2074,15 +2074,15 @@ static void sja1105_bridge_stp_state_set(struct dsa_switch *ds, int port, } static int sja1105_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { - return sja1105_bridge_member(ds, port, br, true); + return sja1105_bridge_member(ds, port, bridge, true); } static void sja1105_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { - sja1105_bridge_member(ds, port, br, false); + sja1105_bridge_member(ds, port, bridge, false); } #define BYTES_PER_KBIT (1000LL / 8) diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index 7c2b6c32242d..ebb55dfd9c4e 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -501,7 +501,7 @@ static void xrs700x_mac_link_up(struct dsa_switch *ds, int port, } static int xrs700x_bridge_common(struct dsa_switch *ds, int port, - struct net_device *bridge, bool join) + struct dsa_bridge bridge, bool join) { unsigned int i, cpu_mask = 0, mask = 0; struct xrs700x *priv = ds->priv; @@ -513,14 +513,14 @@ static int xrs700x_bridge_common(struct dsa_switch *ds, int port, cpu_mask |= BIT(i); - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) == bridge) + if (dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; mask |= BIT(i); } for (i = 0; i < ds->num_ports; i++) { - if (dsa_port_bridge_dev_get(dsa_to_port(ds, i)) != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* 1 = Disable forwarding to the port */ @@ -540,13 +540,13 @@ static int xrs700x_bridge_common(struct dsa_switch *ds, int port, } static int xrs700x_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { return xrs700x_bridge_common(ds, port, bridge, true); } static void xrs700x_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { xrs700x_bridge_common(ds, port, bridge, false); } diff --git a/include/linux/dsa/8021q.h b/include/linux/dsa/8021q.h index 0af4371fbebb..939a1beaddf7 100644 --- a/include/linux/dsa/8021q.h +++ b/include/linux/dsa/8021q.h @@ -7,6 +7,7 @@ #include #include +#include struct dsa_switch; struct dsa_port; @@ -37,12 +38,10 @@ struct sk_buff *dsa_8021q_xmit(struct sk_buff *skb, struct net_device *netdev, void dsa_8021q_rcv(struct sk_buff *skb, int *source_port, int *switch_id); int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num); + struct dsa_bridge bridge); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num); + struct dsa_bridge bridge); u16 dsa_8021q_bridge_tx_fwd_offload_vid(unsigned int bridge_num); diff --git a/include/net/dsa.h b/include/net/dsa.h index 899e13d56fc2..b9789c0cd5e3 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -219,6 +219,11 @@ struct dsa_mall_tc_entry { }; }; +struct dsa_bridge { + struct net_device *dev; + unsigned int num; + refcount_t refcount; +}; struct dsa_port { /* A CPU port is physically connected to a master device. @@ -256,8 +261,7 @@ struct dsa_port { /* Managed by DSA on user ports and by drivers on CPU and DSA ports */ bool learning; u8 stp_state; - struct net_device *bridge_dev; - unsigned int bridge_num; + struct dsa_bridge *bridge; struct devlink_port devlink_port; bool devlink_port_setup; struct phylink *pl; @@ -588,7 +592,7 @@ static inline bool dsa_port_is_vlan_filtering(const struct dsa_port *dp) static inline struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp) { - if (!dp->bridge_dev) + if (!dp->bridge) return NULL; if (dp->lag_dev) @@ -602,12 +606,12 @@ struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp) static inline struct net_device * dsa_port_bridge_dev_get(const struct dsa_port *dp) { - return dp->bridge_dev; + return dp->bridge ? dp->bridge->dev : NULL; } static inline unsigned int dsa_port_bridge_num_get(struct dsa_port *dp) { - return dp->bridge_num; + return dp->bridge ? dp->bridge->num : 0; } static inline bool dsa_port_bridge_same(const struct dsa_port *a, @@ -636,6 +640,12 @@ dsa_port_offloads_bridge_dev(struct dsa_port *dp, return dsa_port_bridge_dev_get(dp) == bridge_dev; } +static inline bool dsa_port_offloads_bridge(struct dsa_port *dp, + const struct dsa_bridge *bridge) +{ + return dsa_port_bridge_dev_get(dp) == bridge->dev; +} + /* Returns true if any port of this tree offloads the given net_device */ static inline bool dsa_tree_offloads_bridge_port(struct dsa_switch_tree *dst, const struct net_device *dev) @@ -812,17 +822,15 @@ struct dsa_switch_ops { */ int (*set_ageing_time)(struct dsa_switch *ds, unsigned int msecs); int (*port_bridge_join)(struct dsa_switch *ds, int port, - struct net_device *bridge); + struct dsa_bridge bridge); void (*port_bridge_leave)(struct dsa_switch *ds, int port, - struct net_device *bridge); + struct dsa_bridge bridge); /* Called right after .port_bridge_join() */ int (*port_bridge_tx_fwd_offload)(struct dsa_switch *ds, int port, - struct net_device *bridge, - unsigned int bridge_num); + struct dsa_bridge bridge); /* Called right before .port_bridge_leave() */ void (*port_bridge_tx_fwd_unoffload)(struct dsa_switch *ds, int port, - struct net_device *bridge, - unsigned int bridge_num); + struct dsa_bridge bridge); void (*port_stp_state_set)(struct dsa_switch *ds, int port, u8 state); void (*port_fast_age)(struct dsa_switch *ds, int port); @@ -894,10 +902,10 @@ struct dsa_switch_ops { */ int (*crosschip_bridge_join)(struct dsa_switch *ds, int tree_index, int sw_index, int port, - struct net_device *br); + struct dsa_bridge bridge); void (*crosschip_bridge_leave)(struct dsa_switch *ds, int tree_index, int sw_index, int port, - struct net_device *br); + struct dsa_bridge bridge); int (*crosschip_lag_change)(struct dsa_switch *ds, int sw_index, int port); int (*crosschip_lag_join)(struct dsa_switch *ds, int sw_index, diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index 4901cdc264ee..8814fa0e44c8 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -129,20 +129,29 @@ void dsa_lag_unmap(struct dsa_switch_tree *dst, struct net_device *lag) } } +struct dsa_bridge *dsa_tree_bridge_find(struct dsa_switch_tree *dst, + const struct net_device *br) +{ + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) + if (dsa_port_bridge_dev_get(dp) == br) + return dp->bridge; + + return NULL; +} + static int dsa_bridge_num_find(const struct net_device *bridge_dev) { struct dsa_switch_tree *dst; - struct dsa_port *dp; - /* When preparing the offload for a port, it will have a valid - * dp->bridge_dev pointer but a not yet valid dp->bridge_num. - * However there might be other ports having the same dp->bridge_dev - * and a valid dp->bridge_num, so just ignore this port. - */ - list_for_each_entry(dst, &dsa_tree_list, list) - list_for_each_entry(dp, &dst->ports, list) - if (dp->bridge_dev == bridge_dev && dp->bridge_num) - return dp->bridge_num; + list_for_each_entry(dst, &dsa_tree_list, list) { + struct dsa_bridge *bridge; + + bridge = dsa_tree_bridge_find(dst, bridge_dev); + if (bridge) + return bridge->num; + } return 0; } @@ -151,6 +160,12 @@ unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) { unsigned int bridge_num = dsa_bridge_num_find(bridge_dev); + /* Switches without FDB isolation support don't get unique + * bridge numbering + */ + if (!max) + return 0; + if (!bridge_num) { /* First port that requests FDB isolation or TX forwarding * offload for this bridge @@ -170,11 +185,11 @@ unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) void dsa_bridge_num_put(const struct net_device *bridge_dev, unsigned int bridge_num) { - /* Check if the bridge is still in use, otherwise it is time - * to clean it up so we can reuse this bridge_num later. + /* Since we refcount bridges, we know that when we call this function + * it is no longer in use, so we can just go ahead and remove it from + * the bit mask. */ - if (!dsa_bridge_num_find(bridge_dev)) - clear_bit(bridge_num, &dsa_fwd_offloading_bridges); + clear_bit(bridge_num, &dsa_fwd_offloading_bridges); } struct dsa_switch *dsa_switch_find(int tree_index, int sw_index) diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index b4f9df4e38b2..da6ff99ba5ed 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -52,7 +52,7 @@ struct dsa_notifier_ageing_time_info { /* DSA_NOTIFIER_BRIDGE_* */ struct dsa_notifier_bridge_info { - struct net_device *br; + struct dsa_bridge bridge; int tree_index; int sw_index; int port; @@ -374,7 +374,7 @@ dsa_find_designated_bridge_port_by_vid(struct net_device *master, u16 vid) if (dp->type != DSA_PORT_TYPE_USER) continue; - if (!dp->bridge_dev) + if (!dp->bridge) continue; if (dp->stp_state != BR_STATE_LEARNING && @@ -403,7 +403,7 @@ dsa_find_designated_bridge_port_by_vid(struct net_device *master, u16 vid) /* If the ingress port offloads the bridge, we mark the frame as autonomously * forwarded by hardware, so the software bridge doesn't forward in twice, back * to us, because we already did. However, if we're in fallback mode and we do - * software bridging, we are not offloading it, therefore the dp->bridge_dev + * software bridging, we are not offloading it, therefore the dp->bridge * pointer is not populated, and flooding needs to be done by software (we are * effectively operating in standalone ports mode). */ @@ -411,7 +411,7 @@ static inline void dsa_default_offload_fwd_mark(struct sk_buff *skb) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - skb->offload_fwd_mark = !!(dp->bridge_dev); + skb->offload_fwd_mark = !!(dp->bridge); } /* Helper for removing DSA header tags from packets in the RX path. @@ -508,6 +508,8 @@ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max); void dsa_bridge_num_put(const struct net_device *bridge_dev, unsigned int bridge_num); +struct dsa_bridge *dsa_tree_bridge_find(struct dsa_switch_tree *dst, + const struct net_device *br); /* tag_8021q.c */ int dsa_tag_8021q_bridge_join(struct dsa_switch *ds, diff --git a/net/dsa/port.c b/net/dsa/port.c index f6ea41cbcdd5..fbf2d7fc5c91 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -130,7 +130,7 @@ int dsa_port_enable_rt(struct dsa_port *dp, struct phy_device *phy) return err; } - if (!dp->bridge_dev) + if (!dp->bridge) dsa_port_set_state_now(dp, BR_STATE_FORWARDING, false); if (dp->pl) @@ -158,7 +158,7 @@ void dsa_port_disable_rt(struct dsa_port *dp) if (dp->pl) phylink_stop(dp->pl); - if (!dp->bridge_dev) + if (!dp->bridge) dsa_port_set_state_now(dp, BR_STATE_DISABLED, false); if (ds->ops->port_disable) @@ -271,36 +271,32 @@ static void dsa_port_switchdev_unsync_attrs(struct dsa_port *dp) } static void dsa_port_bridge_tx_fwd_unoffload(struct dsa_port *dp, - struct net_device *bridge_dev, - unsigned int bridge_num) + struct dsa_bridge bridge) { struct dsa_switch *ds = dp->ds; /* No bridge TX forwarding offload => do nothing */ - if (!ds->ops->port_bridge_tx_fwd_unoffload || !bridge_num) + if (!ds->ops->port_bridge_tx_fwd_unoffload || !bridge.num) return; /* Notify the chips only once the offload has been deactivated, so * that they can update their configuration accordingly. */ - ds->ops->port_bridge_tx_fwd_unoffload(ds, dp->index, bridge_dev, - bridge_num); + ds->ops->port_bridge_tx_fwd_unoffload(ds, dp->index, bridge); } static bool dsa_port_bridge_tx_fwd_offload(struct dsa_port *dp, - struct net_device *bridge_dev, - unsigned int bridge_num) + struct dsa_bridge bridge) { struct dsa_switch *ds = dp->ds; int err; /* FDB isolation is required for TX forwarding offload */ - if (!ds->ops->port_bridge_tx_fwd_offload || !bridge_num) + if (!ds->ops->port_bridge_tx_fwd_offload || !bridge.num) return false; /* Notify the driver */ - err = ds->ops->port_bridge_tx_fwd_offload(ds, dp->index, bridge_dev, - bridge_num); + err = ds->ops->port_bridge_tx_fwd_offload(ds, dp->index, bridge); return err ? false : true; } @@ -310,21 +306,32 @@ static int dsa_port_bridge_create(struct dsa_port *dp, struct netlink_ext_ack *extack) { struct dsa_switch *ds = dp->ds; - unsigned int bridge_num; + struct dsa_bridge *bridge; - dp->bridge_dev = br; - - if (!ds->max_num_bridges) + bridge = dsa_tree_bridge_find(ds->dst, br); + if (bridge) { + refcount_inc(&bridge->refcount); + dp->bridge = bridge; return 0; + } + + bridge = kzalloc(sizeof(*bridge), GFP_KERNEL); + if (!bridge) + return -ENOMEM; + + refcount_set(&bridge->refcount, 1); + + bridge->dev = br; - bridge_num = dsa_bridge_num_get(br, ds->max_num_bridges); - if (!bridge_num) { + bridge->num = dsa_bridge_num_get(br, ds->max_num_bridges); + if (ds->max_num_bridges && !bridge->num) { NL_SET_ERR_MSG_MOD(extack, "Range of offloadable bridges exceeded"); + kfree(bridge); return -EOPNOTSUPP; } - dp->bridge_num = bridge_num; + dp->bridge = bridge; return 0; } @@ -332,16 +339,17 @@ static int dsa_port_bridge_create(struct dsa_port *dp, static void dsa_port_bridge_destroy(struct dsa_port *dp, const struct net_device *br) { - struct dsa_switch *ds = dp->ds; + struct dsa_bridge *bridge = dp->bridge; + + dp->bridge = NULL; - dp->bridge_dev = NULL; + if (!refcount_dec_and_test(&bridge->refcount)) + return; - if (ds->max_num_bridges) { - int bridge_num = dp->bridge_num; + if (bridge->num) + dsa_bridge_num_put(br, bridge->num); - dp->bridge_num = 0; - dsa_bridge_num_put(br, bridge_num); - } + kfree(bridge); } int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br, @@ -351,7 +359,6 @@ int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br, .tree_index = dp->ds->dst->index, .sw_index = dp->ds->index, .port = dp->index, - .br = br, }; struct net_device *dev = dp->slave; struct net_device *brport_dev; @@ -367,12 +374,12 @@ int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br, brport_dev = dsa_port_to_bridge_port(dp); + info.bridge = *dp->bridge; err = dsa_broadcast(DSA_NOTIFIER_BRIDGE_JOIN, &info); if (err) goto out_rollback; - tx_fwd_offload = dsa_port_bridge_tx_fwd_offload(dp, br, - dsa_port_bridge_num_get(dp)); + tx_fwd_offload = dsa_port_bridge_tx_fwd_offload(dp, info.bridge); err = switchdev_bridge_port_offload(brport_dev, dev, dp, &dsa_slave_switchdev_notifier, @@ -415,12 +422,11 @@ void dsa_port_pre_bridge_leave(struct dsa_port *dp, struct net_device *br) void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br) { - unsigned int bridge_num = dsa_port_bridge_num_get(dp); struct dsa_notifier_bridge_info info = { .tree_index = dp->ds->dst->index, .sw_index = dp->ds->index, .port = dp->index, - .br = br, + .bridge = *dp->bridge, }; int err; @@ -429,7 +435,7 @@ void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br) */ dsa_port_bridge_destroy(dp, br); - dsa_port_bridge_tx_fwd_unoffload(dp, br, bridge_num); + dsa_port_bridge_tx_fwd_unoffload(dp, info.bridge); err = dsa_broadcast(DSA_NOTIFIER_BRIDGE_LEAVE, &info); if (err) diff --git a/net/dsa/slave.c b/net/dsa/slave.c index 4a4c31cce80d..88f7b8686dac 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -1564,7 +1564,7 @@ static void dsa_bridge_mtu_normalization(struct dsa_port *dp) if (!dp->ds->mtu_enforcement_ingress) return; - if (!dp->bridge_dev) + if (!dp->bridge) return; INIT_LIST_HEAD(&hw_port_list); diff --git a/net/dsa/switch.c b/net/dsa/switch.c index 7993192fe769..cd0630dd5417 100644 --- a/net/dsa/switch.c +++ b/net/dsa/switch.c @@ -95,7 +95,7 @@ static int dsa_switch_bridge_join(struct dsa_switch *ds, if (!ds->ops->port_bridge_join) return -EOPNOTSUPP; - err = ds->ops->port_bridge_join(ds, info->port, info->br); + err = ds->ops->port_bridge_join(ds, info->port, info->bridge); if (err) return err; } @@ -104,7 +104,7 @@ static int dsa_switch_bridge_join(struct dsa_switch *ds, ds->ops->crosschip_bridge_join) { err = ds->ops->crosschip_bridge_join(ds, info->tree_index, info->sw_index, - info->port, info->br); + info->port, info->bridge); if (err) return err; } @@ -124,19 +124,20 @@ static int dsa_switch_bridge_leave(struct dsa_switch *ds, if (dst->index == info->tree_index && ds->index == info->sw_index && ds->ops->port_bridge_leave) - ds->ops->port_bridge_leave(ds, info->port, info->br); + ds->ops->port_bridge_leave(ds, info->port, info->bridge); if ((dst->index != info->tree_index || ds->index != info->sw_index) && ds->ops->crosschip_bridge_leave) ds->ops->crosschip_bridge_leave(ds, info->tree_index, info->sw_index, info->port, - info->br); + info->bridge); - if (ds->needs_standalone_vlan_filtering && !br_vlan_enabled(info->br)) { + if (ds->needs_standalone_vlan_filtering && + !br_vlan_enabled(info->bridge.dev)) { change_vlan_filtering = true; vlan_filtering = true; } else if (!ds->needs_standalone_vlan_filtering && - br_vlan_enabled(info->br)) { + br_vlan_enabled(info->bridge.dev)) { change_vlan_filtering = true; vlan_filtering = false; } diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c index e9d5e566973c..27712a81c967 100644 --- a/net/dsa/tag_8021q.c +++ b/net/dsa/tag_8021q.c @@ -337,7 +337,7 @@ dsa_port_tag_8021q_bridge_match(struct dsa_port *dp, return false; if (dsa_port_is_user(dp)) - return dsa_port_bridge_dev_get(dp) == info->br; + return dsa_port_offloads_bridge(dp, &info->bridge); return false; } @@ -410,10 +410,9 @@ int dsa_tag_8021q_bridge_leave(struct dsa_switch *ds, } int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num) + struct dsa_bridge bridge) { - u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); + u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge.num); return dsa_port_tag_8021q_vlan_add(dsa_to_port(ds, port), tx_vid, true); @@ -421,10 +420,9 @@ int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, EXPORT_SYMBOL_GPL(dsa_tag_8021q_bridge_tx_fwd_offload); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - unsigned int bridge_num) + struct dsa_bridge bridge) { - u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); + u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge.num); dsa_port_tag_8021q_vlan_del(dsa_to_port(ds, port), tx_vid, true); } -- cgit v1.2.3-59-g8ed1b From 283e6f5a8166f075eba78da6b867d76cc5d47e77 Mon Sep 17 00:00:00 2001 From: Sergey Ryazanov Date: Tue, 7 Dec 2021 12:21:40 +0300 Subject: net: wwan: make debugfs optional Debugfs interface is optional for the regular modem use. Some distros and users will want to disable this feature for security or kernel size reasons. So add a configuration option that allows to completely disable the debugfs interface of the WWAN devices. A primary considered use case for this option was embedded firmwares. For example, in OpenWrt, you can not completely disable debugfs, as a lot of wireless stuff can only be configured and monitored with the debugfs knobs. At the same time, reducing the size of a kernel and modules is an essential task in the world of embedded software. Disabling the WWAN and IOSM debugfs interfaces allows us to save 50K (x86-64 build) of space for module storage. Not much, but already considerable when you only have 16MB of storage. So it is hard to just disable whole debugfs. Users need some fine grained set of options to control which debugfs interface is important and should be available and which is not. The new configuration symbol is enabled by default and is hidden under the EXPERT option. So a regular user would not be bothered by another one configuration question. While an embedded distro maintainer will be able to a little more reduce the final image size. Signed-off-by: Sergey Ryazanov Reviewed-by: Leon Romanovsky Reviewed-by: Loic Poulain Acked-by: M Chetan Kumar Signed-off-by: Jakub Kicinski --- drivers/net/wwan/Kconfig | 13 ++++++++++++- drivers/net/wwan/iosm/Makefile | 4 +++- drivers/net/wwan/iosm/iosm_ipc_debugfs.h | 5 +++++ drivers/net/wwan/iosm/iosm_ipc_imem.c | 2 +- drivers/net/wwan/iosm/iosm_ipc_imem.h | 4 ++++ drivers/net/wwan/iosm/iosm_ipc_trace.c | 6 ++++-- drivers/net/wwan/iosm/iosm_ipc_trace.h | 20 +++++++++++++++++++- drivers/net/wwan/wwan_core.c | 17 +++++++++++++---- include/linux/wwan.h | 7 +++++++ 9 files changed, 68 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wwan/Kconfig b/drivers/net/wwan/Kconfig index 9f5111a77da9..609fd4a2c865 100644 --- a/drivers/net/wwan/Kconfig +++ b/drivers/net/wwan/Kconfig @@ -16,6 +16,17 @@ config WWAN if WWAN +config WWAN_DEBUGFS + bool "WWAN devices debugfs interface" if EXPERT + depends on DEBUG_FS + default y + help + Enables debugfs infrastructure for the WWAN core and device drivers. + + If this option is selected, then you can find the debug interface + elements for each WWAN device in a directory that is corresponding to + the device name: debugfs/wwan/wwanX. + config WWAN_HWSIM tristate "Simulated WWAN device" help @@ -85,7 +96,7 @@ config IOSM tristate "IOSM Driver for Intel M.2 WWAN Device" depends on INTEL_IOMMU select NET_DEVLINK - select RELAY + select RELAY if WWAN_DEBUGFS help This driver enables Intel M.2 WWAN Device communication. diff --git a/drivers/net/wwan/iosm/Makefile b/drivers/net/wwan/iosm/Makefile index 5091f664af0d..fa8d6afd18e1 100644 --- a/drivers/net/wwan/iosm/Makefile +++ b/drivers/net/wwan/iosm/Makefile @@ -21,7 +21,9 @@ iosm-y = \ iosm_ipc_mux_codec.o \ iosm_ipc_devlink.o \ iosm_ipc_flash.o \ - iosm_ipc_coredump.o \ + iosm_ipc_coredump.o + +iosm-$(CONFIG_WWAN_DEBUGFS) += \ iosm_ipc_debugfs.o \ iosm_ipc_trace.o diff --git a/drivers/net/wwan/iosm/iosm_ipc_debugfs.h b/drivers/net/wwan/iosm/iosm_ipc_debugfs.h index 35788039f13f..8a84bfa2c14a 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_debugfs.h +++ b/drivers/net/wwan/iosm/iosm_ipc_debugfs.h @@ -6,7 +6,12 @@ #ifndef IOSM_IPC_DEBUGFS_H #define IOSM_IPC_DEBUGFS_H +#ifdef CONFIG_WWAN_DEBUGFS void ipc_debugfs_init(struct iosm_imem *ipc_imem); void ipc_debugfs_deinit(struct iosm_imem *ipc_imem); +#else +static inline void ipc_debugfs_init(struct iosm_imem *ipc_imem) {} +static inline void ipc_debugfs_deinit(struct iosm_imem *ipc_imem) {} +#endif #endif diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem.c b/drivers/net/wwan/iosm/iosm_ipc_imem.c index 25b889922912..2a6ddd7c6c88 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem.c +++ b/drivers/net/wwan/iosm/iosm_ipc_imem.c @@ -274,7 +274,7 @@ static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem, ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb); else if (ipc_is_trace_channel(ipc_imem, port_id)) - ipc_trace_port_rx(ipc_imem->trace, skb); + ipc_trace_port_rx(ipc_imem, skb); else wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port, skb); diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem.h b/drivers/net/wwan/iosm/iosm_ipc_imem.h index 1b8c7b8959c6..86a1ffe61729 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem.h +++ b/drivers/net/wwan/iosm/iosm_ipc_imem.h @@ -351,7 +351,9 @@ struct iosm_imem { struct iosm_mux *mux; struct iosm_cdev *ipc_port[IPC_MEM_MAX_CHANNELS]; struct iosm_pcie *pcie; +#ifdef CONFIG_WWAN_DEBUGFS struct iosm_trace *trace; +#endif struct device *dev; enum ipc_mem_device_ipc_state ipc_requested_state; struct ipc_mem_channel channels[IPC_MEM_MAX_CHANNELS]; @@ -381,7 +383,9 @@ struct iosm_imem { ev_mux_net_transmit_pending:1, reset_det_n:1, pcie_wake_n:1; +#ifdef CONFIG_WWAN_DEBUGFS struct dentry *debugfs_dir; +#endif }; /** diff --git a/drivers/net/wwan/iosm/iosm_ipc_trace.c b/drivers/net/wwan/iosm/iosm_ipc_trace.c index 5243ead90b5f..eeecfa3d10c5 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_trace.c +++ b/drivers/net/wwan/iosm/iosm_ipc_trace.c @@ -17,11 +17,13 @@ /** * ipc_trace_port_rx - Receive trace packet from cp and write to relay buffer - * @ipc_trace: Pointer to the ipc trace data-struct + * @ipc_imem: Pointer to iosm_imem structure * @skb: Pointer to struct sk_buff */ -void ipc_trace_port_rx(struct iosm_trace *ipc_trace, struct sk_buff *skb) +void ipc_trace_port_rx(struct iosm_imem *ipc_imem, struct sk_buff *skb) { + struct iosm_trace *ipc_trace = ipc_imem->trace; + if (ipc_trace->ipc_rchan) relay_write(ipc_trace->ipc_rchan, skb->data, skb->len); diff --git a/drivers/net/wwan/iosm/iosm_ipc_trace.h b/drivers/net/wwan/iosm/iosm_ipc_trace.h index 419540c91219..5ebe7790585c 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_trace.h +++ b/drivers/net/wwan/iosm/iosm_ipc_trace.h @@ -45,6 +45,8 @@ struct iosm_trace { enum trace_ctrl_mode mode; }; +#ifdef CONFIG_WWAN_DEBUGFS + static inline bool ipc_is_trace_channel(struct iosm_imem *ipc_mem, u16 chl_id) { return ipc_mem->trace && ipc_mem->trace->chl_id == chl_id; @@ -52,5 +54,21 @@ static inline bool ipc_is_trace_channel(struct iosm_imem *ipc_mem, u16 chl_id) struct iosm_trace *ipc_trace_init(struct iosm_imem *ipc_imem); void ipc_trace_deinit(struct iosm_trace *ipc_trace); -void ipc_trace_port_rx(struct iosm_trace *ipc_trace, struct sk_buff *skb); +void ipc_trace_port_rx(struct iosm_imem *ipc_imem, struct sk_buff *skb); + +#else + +static inline bool ipc_is_trace_channel(struct iosm_imem *ipc_mem, u16 chl_id) +{ + return false; +} + +static inline void ipc_trace_port_rx(struct iosm_imem *ipc_imem, + struct sk_buff *skb) +{ + dev_kfree_skb(skb); +} + +#endif + #endif diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index 5bf62dc35ac7..1508dc2a497b 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -50,7 +50,9 @@ struct wwan_device { atomic_t port_id; const struct wwan_ops *ops; void *ops_ctxt; +#ifdef CONFIG_WWAN_DEBUGFS struct dentry *debugfs_dir; +#endif }; /** @@ -146,6 +148,7 @@ static struct wwan_device *wwan_dev_get_by_name(const char *name) return to_wwan_dev(dev); } +#ifdef CONFIG_WWAN_DEBUGFS struct dentry *wwan_get_debugfs_dir(struct device *parent) { struct wwan_device *wwandev; @@ -157,6 +160,7 @@ struct dentry *wwan_get_debugfs_dir(struct device *parent) return wwandev->debugfs_dir; } EXPORT_SYMBOL_GPL(wwan_get_debugfs_dir); +#endif /* This function allocates and registers a new WWAN device OR if a WWAN device * already exist for the given parent, it gets a reference and return it. @@ -166,7 +170,6 @@ EXPORT_SYMBOL_GPL(wwan_get_debugfs_dir); static struct wwan_device *wwan_create_dev(struct device *parent) { struct wwan_device *wwandev; - const char *wwandev_name; int err, id; /* The 'find-alloc-register' operation must be protected against @@ -206,9 +209,11 @@ static struct wwan_device *wwan_create_dev(struct device *parent) goto done_unlock; } - wwandev_name = kobject_name(&wwandev->dev.kobj); - wwandev->debugfs_dir = debugfs_create_dir(wwandev_name, - wwan_debugfs_dir); +#ifdef CONFIG_WWAN_DEBUGFS + wwandev->debugfs_dir = + debugfs_create_dir(kobject_name(&wwandev->dev.kobj), + wwan_debugfs_dir); +#endif done_unlock: mutex_unlock(&wwan_register_lock); @@ -240,7 +245,9 @@ static void wwan_remove_dev(struct wwan_device *wwandev) ret = device_for_each_child(&wwandev->dev, NULL, is_wwan_child); if (!ret) { +#ifdef CONFIG_WWAN_DEBUGFS debugfs_remove_recursive(wwandev->debugfs_dir); +#endif device_unregister(&wwandev->dev); } else { put_device(&wwandev->dev); @@ -1140,7 +1147,9 @@ static int __init wwan_init(void) goto destroy; } +#ifdef CONFIG_WWAN_DEBUGFS wwan_debugfs_dir = debugfs_create_dir("wwan", NULL); +#endif return 0; diff --git a/include/linux/wwan.h b/include/linux/wwan.h index 1646aa3e6779..e143c88bf4b0 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -171,6 +171,13 @@ int wwan_register_ops(struct device *parent, const struct wwan_ops *ops, void wwan_unregister_ops(struct device *parent); +#ifdef CONFIG_WWAN_DEBUGFS struct dentry *wwan_get_debugfs_dir(struct device *parent); +#else +static inline struct dentry *wwan_get_debugfs_dir(struct device *parent) +{ + return ERR_PTR(-ENODEV); +} +#endif #endif /* __WWAN_H */ -- cgit v1.2.3-59-g8ed1b From 1197528aaea79ed4909aba695d18fdecc5387a36 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:28 +0100 Subject: genirq/msi: Guard sysfs code No point in building unused code when CONFIG_SYSFS=n. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20211206210223.985907940@linutronix.de --- include/linux/msi.h | 10 ++++++++++ kernel/irq/msi.c | 2 ++ 2 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index e616f94c7c58..d43b9469c88b 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -239,9 +239,19 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void pci_msi_mask_irq(struct irq_data *data); void pci_msi_unmask_irq(struct irq_data *data); +#ifdef CONFIG_SYSFS const struct attribute_group **msi_populate_sysfs(struct device *dev); void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups); +#else +static inline const struct attribute_group **msi_populate_sysfs(struct device *dev) +{ + return NULL; +} +static inline void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups) +{ +} +#endif /* * The arch hooks to setup up msi irqs. Default functions are implemented diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 7f350ae59c5f..a8a0daeb22f5 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -72,6 +72,7 @@ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) } EXPORT_SYMBOL_GPL(get_cached_msi_msg); +#ifdef CONFIG_SYSFS static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -204,6 +205,7 @@ void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_ir kfree(msi_irq_groups); } } +#endif #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN static inline void irq_chip_write_msi_msg(struct irq_data *data, -- cgit v1.2.3-59-g8ed1b From 1dd2c6a0817fd08f80dee75d7d3bd99a0c4b828d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:29 +0100 Subject: genirq/msi: Remove unused domain callbacks No users and there is no need to grow them. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211126223824.322987915@linutronix.de Link: https://lore.kernel.org/r/20211206210224.041777889@linutronix.de --- include/linux/msi.h | 11 ++++------- kernel/irq/msi.c | 5 ----- 2 files changed, 4 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index d43b9469c88b..4b962f73f84a 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -304,7 +304,6 @@ struct msi_domain_info; * @msi_free: Domain specific function to free a MSI interrupts * @msi_check: Callback for verification of the domain/info/dev data * @msi_prepare: Prepare the allocation of the interrupts in the domain - * @msi_finish: Optional callback to finalize the allocation * @set_desc: Set the msi descriptor for an interrupt * @handle_error: Optional error handler if the allocation fails * @domain_alloc_irqs: Optional function to override the default allocation @@ -312,12 +311,11 @@ struct msi_domain_info; * @domain_free_irqs: Optional function to override the default free * function. * - * @get_hwirq, @msi_init and @msi_free are callbacks used by - * msi_create_irq_domain() and related interfaces + * @get_hwirq, @msi_init and @msi_free are callbacks used by the underlying + * irqdomain. * - * @msi_check, @msi_prepare, @msi_finish, @set_desc and @handle_error - * are callbacks used by msi_domain_alloc_irqs() and related - * interfaces which are based on msi_desc. + * @msi_check, @msi_prepare, @handle_error and @set_desc are callbacks used by + * msi_domain_alloc/free_irqs(). * * @domain_alloc_irqs, @domain_free_irqs can be used to override the * default allocation/free functions (__msi_domain_alloc/free_irqs). This @@ -351,7 +349,6 @@ struct msi_domain_ops { int (*msi_prepare)(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg); - void (*msi_finish)(msi_alloc_info_t *arg, int retval); void (*set_desc)(msi_alloc_info_t *arg, struct msi_desc *desc); int (*handle_error)(struct irq_domain *domain, diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index a8a0daeb22f5..cd4fa264c7c6 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -562,8 +562,6 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, ret = -ENOSPC; if (ops->handle_error) ret = ops->handle_error(domain, desc, ret); - if (ops->msi_finish) - ops->msi_finish(&arg, ret); return ret; } @@ -573,9 +571,6 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, } } - if (ops->msi_finish) - ops->msi_finish(&arg, 0); - can_reserve = msi_check_reservation_mode(domain, info, dev); /* -- cgit v1.2.3-59-g8ed1b From 3ba1f050c91d5ce3672dbf3a55dc2451c0b342e2 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:31 +0100 Subject: genirq/msi: Fixup includes Remove the kobject.h include from msi.h as it's not required and add a sysfs.h include to the core code instead. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20211206210224.103502021@linutronix.de --- include/linux/msi.h | 2 +- kernel/irq/msi.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 4b962f73f84a..5c627750f269 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -2,7 +2,7 @@ #ifndef LINUX_MSI_H #define LINUX_MSI_H -#include +#include #include #include diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index cd4fa264c7c6..6718bab1bde3 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "internals.h" -- cgit v1.2.3-59-g8ed1b From 9e8688c5f2990dadcf83728cd00a7e8497fc6aa9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:33 +0100 Subject: PCI/MSI: Make pci_msi_domain_write_msg() static There is no point to have this function public as it is set by the PCI core anyway when a PCI/MSI irqdomain is created. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas # PCI Link: https://lore.kernel.org/r/20211206210224.157070464@linutronix.de --- drivers/irqchip/irq-gic-v2m.c | 1 - drivers/irqchip/irq-gic-v3-its-pci-msi.c | 1 - drivers/irqchip/irq-gic-v3-mbi.c | 1 - drivers/pci/msi.c | 2 +- include/linux/msi.h | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index 9349fc68b81a..62cc78e0da78 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -88,7 +88,6 @@ static struct irq_chip gicv2m_msi_irq_chip = { .irq_mask = gicv2m_mask_msi_irq, .irq_unmask = gicv2m_unmask_msi_irq, .irq_eoi = irq_chip_eoi_parent, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static struct msi_domain_info gicv2m_msi_domain_info = { diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c index ad2810c017ed..93f77a8196da 100644 --- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c +++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c @@ -28,7 +28,6 @@ static struct irq_chip its_msi_irq_chip = { .irq_unmask = its_unmask_msi_irq, .irq_mask = its_mask_msi_irq, .irq_eoi = irq_chip_eoi_parent, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data) diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index b84c9c2eccdc..a2163d32f17d 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -171,7 +171,6 @@ static struct irq_chip mbi_msi_irq_chip = { .irq_unmask = mbi_unmask_msi_irq, .irq_eoi = irq_chip_eoi_parent, .irq_compose_msi_msg = mbi_compose_msi_msg, - .irq_write_msi_msg = pci_msi_domain_write_msg, }; static struct msi_domain_info mbi_msi_domain_info = { diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 00ed45fea482..afa8ba28904b 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -1281,7 +1281,7 @@ EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata); * @irq_data: Pointer to interrupt data of the MSI interrupt * @msg: Pointer to the message */ -void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) +static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) { struct msi_desc *desc = irq_data_get_msi_desc(irq_data); diff --git a/include/linux/msi.h b/include/linux/msi.h index 5c627750f269..d7b143a79cb4 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -455,7 +455,6 @@ void *platform_msi_get_host_data(struct irq_domain *domain); #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg); struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, struct msi_domain_info *info, struct irq_domain *parent); -- cgit v1.2.3-59-g8ed1b From ade044a3d0f0389e4f916337c505550acc3fd011 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:34 +0100 Subject: PCI/MSI: Remove msi_desc_to_pci_sysdata() Last user is gone long ago. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211206210224.210768199@linutronix.de --- drivers/pci/msi.c | 8 -------- include/linux/msi.h | 5 ----- 2 files changed, 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index afa8ba28904b..e1aecd9b82d9 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -1267,14 +1267,6 @@ struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) } EXPORT_SYMBOL(msi_desc_to_pci_dev); -void *msi_desc_to_pci_sysdata(struct msi_desc *desc) -{ - struct pci_dev *dev = msi_desc_to_pci_dev(desc); - - return dev->bus->sysdata; -} -EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata); - #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN /** * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space diff --git a/include/linux/msi.h b/include/linux/msi.h index d7b143a79cb4..ac6fec105edc 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -218,13 +218,8 @@ static inline void msi_desc_set_iommu_cookie(struct msi_desc *desc, for_each_msi_entry((desc), &(pdev)->dev) struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc); -void *msi_desc_to_pci_sysdata(struct msi_desc *desc); void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg); #else /* CONFIG_PCI_MSI */ -static inline void *msi_desc_to_pci_sysdata(struct msi_desc *desc) -{ - return NULL; -} static inline void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) { } -- cgit v1.2.3-59-g8ed1b From e58f2259b91c02974c20db7b28d39d810a21249b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:39 +0100 Subject: genirq/msi, treewide: Use a named struct for PCI/MSI attributes The unnamed struct sucks and is in the way of further cleanups. Stick the PCI related MSI data into a real data structure and cleanup all users. No functional change. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Acked-by: Kalle Valo Link: https://lore.kernel.org/r/20211206210224.374863119@linutronix.de --- arch/powerpc/platforms/cell/axon_msi.c | 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 4 +- arch/powerpc/platforms/pseries/msi.c | 6 +- arch/sparc/kernel/pci_msi.c | 4 +- arch/x86/kernel/apic/msi.c | 2 +- arch/x86/pci/xen.c | 6 +- drivers/net/wireless/ath/ath11k/pci.c | 2 +- drivers/pci/msi.c | 116 +++++++++++++++--------------- drivers/pci/xen-pcifront.c | 2 +- include/linux/msi.h | 84 +++++++++++----------- kernel/irq/msi.c | 4 +- 11 files changed, 115 insertions(+), 117 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c index 82335e364c44..79e21128518c 100644 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ b/arch/powerpc/platforms/cell/axon_msi.c @@ -212,7 +212,7 @@ static int setup_msi_msg_address(struct pci_dev *dev, struct msi_msg *msg) entry = first_pci_msi_entry(dev); for (; dn; dn = of_get_next_parent(dn)) { - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { prop = of_get_property(dn, "msi-address-64", &len); if (prop) break; diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 004cd6a96c8a..8913c86009d9 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -2154,10 +2154,10 @@ static void pnv_msi_compose_msg(struct irq_data *d, struct msi_msg *msg) int rc; rc = __pnv_pci_ioda_msi_setup(phb, pdev, d->hwirq, - entry->msi_attrib.is_64, msg); + entry->pci.msi_attrib.is_64, msg); if (rc) dev_err(&pdev->dev, "Failed to setup %s-bit MSI #%ld : %d\n", - entry->msi_attrib.is_64 ? "64" : "32", d->hwirq, rc); + entry->pci.msi_attrib.is_64 ? "64" : "32", d->hwirq, rc); } /* diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index 8627362f613e..8e287204eeae 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -332,7 +332,7 @@ static int check_msix_entries(struct pci_dev *pdev) expected = 0; for_each_pci_msi_entry(entry, pdev) { - if (entry->msi_attrib.entry_nr != expected) { + if (entry->pci.msi_attrib.entry_nr != expected) { pr_debug("rtas_msi: bad MSI-X entries.\n"); return -EINVAL; } @@ -449,7 +449,7 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev { struct pci_dev *pdev = to_pci_dev(dev); struct msi_desc *desc = first_pci_msi_entry(pdev); - int type = desc->msi_attrib.is_msix ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int type = desc->pci.msi_attrib.is_msix ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; return rtas_prepare_msi_irqs(pdev, nvec, type, arg); } @@ -580,7 +580,7 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq int hwirq; int i, ret; - hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_attrib.entry_nr); + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->pci.msi_attrib.entry_nr); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); return hwirq; diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index fb5899cbfa51..9ed11985768e 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -146,13 +146,13 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msiqid = pick_msiq(pbm); err = ops->msi_setup(pbm, msiqid, msi, - (entry->msi_attrib.is_64 ? 1 : 0)); + (entry->pci.msi_attrib.is_64 ? 1 : 0)); if (err) goto out_msi_free; pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { msg.address_hi = pbm->msi64_start >> 32; msg.address_lo = pbm->msi64_start & 0xffffffff; } else { diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index dbacb9ec8843..1656477e4169 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -163,7 +163,7 @@ int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, struct msi_desc *desc = first_pci_msi_entry(pdev); init_irq_alloc_info(arg, NULL); - if (desc->msi_attrib.is_msix) { + if (desc->pci.msi_attrib.is_msix) { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSIX; } else { arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSI; diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 12da00558631..2dace884d6f7 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -306,7 +306,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; map_irq.table_base = pci_resource_start(dev, bir); - map_irq.entry_nr = msidesc->msi_attrib.entry_nr; + map_irq.entry_nr = msidesc->pci.msi_attrib.entry_nr; } ret = -EINVAL; @@ -398,7 +398,7 @@ static void xen_pv_teardown_msi_irqs(struct pci_dev *dev) { struct msi_desc *msidesc = first_pci_msi_entry(dev); - if (msidesc->msi_attrib.is_msix) + if (msidesc->pci.msi_attrib.is_msix) xen_pci_frontend_disable_msix(dev); else xen_pci_frontend_disable_msi(dev); @@ -414,7 +414,7 @@ static int xen_msi_domain_alloc_irqs(struct irq_domain *domain, if (WARN_ON_ONCE(!dev_is_pci(dev))) return -EINVAL; - if (first_msi_entry(dev)->msi_attrib.is_msix) + if (first_msi_entry(dev)->pci.msi_attrib.is_msix) type = PCI_CAP_ID_MSIX; else type = PCI_CAP_ID_MSI; diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index 3d353e7c9d5c..d9d00e499174 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -911,7 +911,7 @@ static int ath11k_pci_alloc_msi(struct ath11k_pci *ab_pci) } ab_pci->msi_ep_base_data = msi_desc->msg.data; - if (msi_desc->msi_attrib.is_64) + if (msi_desc->pci.msi_attrib.is_64) set_bit(ATH11K_PCI_FLAG_IS_MSI_64, &ab_pci->flags); ath11k_dbg(ab, ATH11K_DBG_PCI, "msi base data is %d\n", ab_pci->msi_ep_base_data); diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index e1aecd9b82d9..b6cd8b337210 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -138,9 +138,9 @@ void __weak arch_restore_msi_irqs(struct pci_dev *dev) static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) { /* Don't shift by >= width of type */ - if (desc->msi_attrib.multi_cap >= 5) + if (desc->pci.msi_attrib.multi_cap >= 5) return 0xffffffff; - return (1 << (1 << desc->msi_attrib.multi_cap)) - 1; + return (1 << (1 << desc->pci.msi_attrib.multi_cap)) - 1; } static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) @@ -148,14 +148,14 @@ static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 s raw_spinlock_t *lock = &desc->dev->msi_lock; unsigned long flags; - if (!desc->msi_attrib.can_mask) + if (!desc->pci.msi_attrib.can_mask) return; raw_spin_lock_irqsave(lock, flags); - desc->msi_mask &= ~clear; - desc->msi_mask |= set; - pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, - desc->msi_mask); + desc->pci.msi_mask &= ~clear; + desc->pci.msi_mask |= set; + pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos, + desc->pci.msi_mask); raw_spin_unlock_irqrestore(lock, flags); } @@ -171,7 +171,7 @@ static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask) static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) { - return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; + return desc->pci.mask_base + desc->pci.msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; } /* @@ -184,27 +184,27 @@ static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { void __iomem *desc_addr = pci_msix_desc_addr(desc); - if (desc->msi_attrib.can_mask) + if (desc->pci.msi_attrib.can_mask) writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } static inline void pci_msix_mask(struct msi_desc *desc) { - desc->msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); + desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); /* Flush write to device */ - readl(desc->mask_base); + readl(desc->pci.mask_base); } static inline void pci_msix_unmask(struct msi_desc *desc) { - desc->msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); + desc->pci.msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); } static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) { - if (desc->msi_attrib.is_msix) + if (desc->pci.msi_attrib.is_msix) pci_msix_mask(desc); else pci_msi_mask(desc, mask); @@ -212,7 +212,7 @@ static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask) { - if (desc->msi_attrib.is_msix) + if (desc->pci.msi_attrib.is_msix) pci_msix_unmask(desc); else pci_msi_unmask(desc, mask); @@ -256,10 +256,10 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) BUG_ON(dev->current_state != PCI_D0); - if (entry->msi_attrib.is_msix) { + if (entry->pci.msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); - if (WARN_ON_ONCE(entry->msi_attrib.is_virtual)) + if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return; msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); @@ -271,7 +271,7 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &msg->address_lo); - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &msg->address_hi); pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data); @@ -289,12 +289,12 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ - } else if (entry->msi_attrib.is_msix) { + } else if (entry->pci.msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); - u32 ctrl = entry->msix_ctrl; + u32 ctrl = entry->pci.msix_ctrl; bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - if (entry->msi_attrib.is_virtual) + if (entry->pci.msi_attrib.is_virtual) goto skip; /* @@ -323,12 +323,12 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); msgctl &= ~PCI_MSI_FLAGS_QSIZE; - msgctl |= entry->msi_attrib.multiple << 4; + msgctl |= entry->pci.msi_attrib.multiple << 4; pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo); - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, msg->address_hi); pci_write_config_word(dev, pos + PCI_MSI_DATA_64, @@ -376,9 +376,9 @@ static void free_msi_irqs(struct pci_dev *dev) pci_msi_teardown_msi_irqs(dev); list_for_each_entry_safe(entry, tmp, msi_list, list) { - if (entry->msi_attrib.is_msix) { + if (entry->pci.msi_attrib.is_msix) { if (list_is_last(&entry->list, msi_list)) - iounmap(entry->mask_base); + iounmap(entry->pci.mask_base); } list_del(&entry->list); @@ -420,7 +420,7 @@ static void __pci_restore_msi_state(struct pci_dev *dev) pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); pci_msi_update_mask(entry, 0, 0); control &= ~PCI_MSI_FLAGS_QSIZE; - control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; + control |= (entry->pci.msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } @@ -449,7 +449,7 @@ static void __pci_restore_msix_state(struct pci_dev *dev) arch_restore_msi_irqs(dev); for_each_pci_msi_entry(entry, dev) - pci_msix_write_vector_ctrl(entry, entry->msix_ctrl); + pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } @@ -481,24 +481,24 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING) control |= PCI_MSI_FLAGS_MASKBIT; - entry->msi_attrib.is_msix = 0; - entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); - entry->msi_attrib.is_virtual = 0; - entry->msi_attrib.entry_nr = 0; - entry->msi_attrib.can_mask = !pci_msi_ignore_mask && + entry->pci.msi_attrib.is_msix = 0; + entry->pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); + entry->pci.msi_attrib.is_virtual = 0; + entry->pci.msi_attrib.entry_nr = 0; + entry->pci.msi_attrib.can_mask = !pci_msi_ignore_mask && !!(control & PCI_MSI_FLAGS_MASKBIT); - entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ - entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; - entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); + entry->pci.msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ + entry->pci.msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; + entry->pci.msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); if (control & PCI_MSI_FLAGS_64BIT) - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; + entry->pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_64; else - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; + entry->pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_32; /* Save the initial mask status */ - if (entry->msi_attrib.can_mask) - pci_read_config_dword(dev, entry->mask_pos, &entry->msi_mask); + if (entry->pci.msi_attrib.can_mask) + pci_read_config_dword(dev, entry->pci.mask_pos, &entry->pci.msi_mask); out: kfree(masks); @@ -630,26 +630,26 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, goto out; } - entry->msi_attrib.is_msix = 1; - entry->msi_attrib.is_64 = 1; + entry->pci.msi_attrib.is_msix = 1; + entry->pci.msi_attrib.is_64 = 1; if (entries) - entry->msi_attrib.entry_nr = entries[i].entry; + entry->pci.msi_attrib.entry_nr = entries[i].entry; else - entry->msi_attrib.entry_nr = i; + entry->pci.msi_attrib.entry_nr = i; - entry->msi_attrib.is_virtual = - entry->msi_attrib.entry_nr >= vec_count; + entry->pci.msi_attrib.is_virtual = + entry->pci.msi_attrib.entry_nr >= vec_count; - entry->msi_attrib.can_mask = !pci_msi_ignore_mask && - !entry->msi_attrib.is_virtual; + entry->pci.msi_attrib.can_mask = !pci_msi_ignore_mask && + !entry->pci.msi_attrib.is_virtual; - entry->msi_attrib.default_irq = dev->irq; - entry->mask_base = base; + entry->pci.msi_attrib.default_irq = dev->irq; + entry->pci.mask_base = base; - if (entry->msi_attrib.can_mask) { + if (entry->pci.msi_attrib.can_mask) { addr = pci_msix_desc_addr(entry); - entry->msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); + entry->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); @@ -874,7 +874,7 @@ static void pci_msi_shutdown(struct pci_dev *dev) pci_msi_unmask(desc, msi_multi_mask(desc)); /* Restore dev->irq to its default pin-assertion IRQ */ - dev->irq = desc->msi_attrib.default_irq; + dev->irq = desc->pci.msi_attrib.default_irq; pcibios_alloc_irq(dev); } @@ -1203,7 +1203,7 @@ int pci_irq_vector(struct pci_dev *dev, unsigned int nr) struct msi_desc *entry; for_each_pci_msi_entry(entry, dev) { - if (entry->msi_attrib.entry_nr == nr) + if (entry->pci.msi_attrib.entry_nr == nr) return entry->irq; } WARN_ON_ONCE(1); @@ -1242,7 +1242,7 @@ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) struct msi_desc *entry; for_each_pci_msi_entry(entry, dev) { - if (entry->msi_attrib.entry_nr == nr) + if (entry->pci.msi_attrib.entry_nr == nr) return &entry->affinity->mask; } WARN_ON_ONCE(1); @@ -1295,14 +1295,14 @@ static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) { struct pci_dev *dev = msi_desc_to_pci_dev(desc); - return (irq_hw_number_t)desc->msi_attrib.entry_nr | + return (irq_hw_number_t)desc->pci.msi_attrib.entry_nr | pci_dev_id(dev) << 11 | (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; } static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) { - return !desc->msi_attrib.is_msix && desc->nvec_used > 1; + return !desc->pci.msi_attrib.is_msix && desc->nvec_used > 1; } /** @@ -1326,7 +1326,7 @@ int pci_msi_domain_check_cap(struct irq_domain *domain, if (pci_msi_desc_is_multi_msi(desc) && !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) return 1; - else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) + else if (desc->pci.msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) return -ENOTSUPP; return 0; diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index d858d25b6cab..699cc9544424 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -263,7 +263,7 @@ static int pci_frontend_enable_msix(struct pci_dev *dev, i = 0; for_each_pci_msi_entry(entry, dev) { - op.msix_entries[i].entry = entry->msi_attrib.entry_nr; + op.msix_entries[i].entry = entry->pci.msi_attrib.entry_nr; /* Vector is useless at this point. */ op.msix_entries[i].vector = -1; i++; diff --git a/include/linux/msi.h b/include/linux/msi.h index ac6fec105edc..7e5c13f4e41b 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -68,6 +68,42 @@ static inline void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc, struct msi_msg *msg); +/** + * pci_msi_desc - PCI/MSI specific MSI descriptor data + * + * @msi_mask: [PCI MSI] MSI cached mask bits + * @msix_ctrl: [PCI MSI-X] MSI-X cached per vector control bits + * @is_msix: [PCI MSI/X] True if MSI-X + * @multiple: [PCI MSI/X] log2 num of messages allocated + * @multi_cap: [PCI MSI/X] log2 num of messages supported + * @can_mask: [PCI MSI/X] Masking supported? + * @is_64: [PCI MSI/X] Address size: 0=32bit 1=64bit + * @entry_nr: [PCI MSI/X] Entry which is described by this descriptor + * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq + * @mask_pos: [PCI MSI] Mask register position + * @mask_base: [PCI MSI-X] Mask register base address + */ +struct pci_msi_desc { + union { + u32 msi_mask; + u32 msix_ctrl; + }; + struct { + u8 is_msix : 1; + u8 multiple : 3; + u8 multi_cap : 3; + u8 can_mask : 1; + u8 is_64 : 1; + u8 is_virtual : 1; + u16 entry_nr; + unsigned default_irq; + } msi_attrib; + union { + u8 mask_pos; + void __iomem *mask_base; + }; +}; + /** * platform_msi_desc - Platform device specific msi descriptor data * @msi_priv_data: Pointer to platform private data @@ -107,17 +143,7 @@ struct ti_sci_inta_msi_desc { * address or data changes * @write_msi_msg_data: Data parameter for the callback. * - * @msi_mask: [PCI MSI] MSI cached mask bits - * @msix_ctrl: [PCI MSI-X] MSI-X cached per vector control bits - * @is_msix: [PCI MSI/X] True if MSI-X - * @multiple: [PCI MSI/X] log2 num of messages allocated - * @multi_cap: [PCI MSI/X] log2 num of messages supported - * @maskbit: [PCI MSI/X] Mask-Pending bit supported? - * @is_64: [PCI MSI/X] Address size: 0=32bit 1=64bit - * @entry_nr: [PCI MSI/X] Entry which is described by this descriptor - * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq - * @mask_pos: [PCI MSI] Mask register position - * @mask_base: [PCI MSI-X] Mask register base address + * @pci: [PCI] PCI speficic msi descriptor data * @platform: [platform] Platform device specific msi descriptor data * @fsl_mc: [fsl-mc] FSL MC device specific msi descriptor data * @inta: [INTA] TISCI based INTA specific msi descriptor data @@ -138,38 +164,10 @@ struct msi_desc { void *write_msi_msg_data; union { - /* PCI MSI/X specific data */ - struct { - union { - u32 msi_mask; - u32 msix_ctrl; - }; - struct { - u8 is_msix : 1; - u8 multiple : 3; - u8 multi_cap : 3; - u8 can_mask : 1; - u8 is_64 : 1; - u8 is_virtual : 1; - u16 entry_nr; - unsigned default_irq; - } msi_attrib; - union { - u8 mask_pos; - void __iomem *mask_base; - }; - }; - - /* - * Non PCI variants add their data structure here. New - * entries need to use a named structure. We want - * proper name spaces for this. The PCI part is - * anonymous for now as it would require an immediate - * tree wide cleanup. - */ - struct platform_msi_desc platform; - struct fsl_mc_msi_desc fsl_mc; - struct ti_sci_inta_msi_desc inta; + struct pci_msi_desc pci; + struct platform_msi_desc platform; + struct fsl_mc_msi_desc fsl_mc; + struct ti_sci_inta_msi_desc inta; }; }; diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 6718bab1bde3..7d78d8aff076 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -91,7 +91,7 @@ static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, return -ENODEV; if (dev_is_pci(dev)) - is_msix = entry->msi_attrib.is_msix; + is_msix = entry->pci.msi_attrib.is_msix; return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); } @@ -535,7 +535,7 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, * masking and MSI does so when the can_mask attribute is set. */ desc = first_msi_entry(dev); - return desc->msi_attrib.is_msix || desc->msi_attrib.can_mask; + return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; } int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, -- cgit v1.2.3-59-g8ed1b From ae72f3156729541581f526b85883ca53a20df2fa Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:42 +0100 Subject: PCI/MSI: Make arch_restore_msi_irqs() less horrible. Make arch_restore_msi_irqs() return a boolean which indicates whether the core code should restore the MSI message or not. Get rid of the indirection in x86. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas # PCI Link: https://lore.kernel.org/r/20211206210224.485668098@linutronix.de --- arch/s390/pci/pci_irq.c | 4 +-- arch/x86/include/asm/x86_init.h | 6 ---- arch/x86/include/asm/xen/hypervisor.h | 8 ++++++ arch/x86/kernel/apic/msi.c | 6 ++++ arch/x86/kernel/x86_init.c | 12 -------- arch/x86/pci/xen.c | 13 +++++---- drivers/pci/msi.c | 54 ++++++++++++----------------------- include/linux/msi.h | 7 ++--- 8 files changed, 45 insertions(+), 65 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 954bb7a83124..2beb8a082e6f 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -387,13 +387,13 @@ void arch_teardown_msi_irqs(struct pci_dev *pdev) airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); } -void arch_restore_msi_irqs(struct pci_dev *pdev) +bool arch_restore_msi_irqs(struct pci_dev *pdev) { struct zpci_dev *zdev = to_zpci(pdev); if (!zdev->irqs_registered) zpci_set_irq(zdev); - default_restore_msi_irqs(pdev); + return true; } static struct airq_struct zpci_airq = { diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 5c69f7eb5d47..22b7412c08f6 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -289,12 +289,6 @@ struct x86_platform_ops { struct x86_hyper_runtime hyper; }; -struct pci_dev; - -struct x86_msi_ops { - void (*restore_msi_irqs)(struct pci_dev *dev); -}; - struct x86_apic_ops { unsigned int (*io_apic_read) (unsigned int apic, unsigned int reg); void (*restore)(void); diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index 5adab895127e..677f84d0039f 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -57,6 +57,14 @@ static inline bool __init xen_x2apic_para_available(void) } #endif +struct pci_dev; + +#ifdef CONFIG_XEN_DOM0 +bool xen_initdom_restore_msi(struct pci_dev *dev); +#else +static inline bool xen_initdom_restore_msi(struct pci_dev *dev) { return true; } +#endif + #ifdef CONFIG_HOTPLUG_CPU void xen_arch_register_cpu(int num); void xen_arch_unregister_cpu(int num); diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index 1656477e4169..b270b70f41c6 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -19,6 +19,7 @@ #include #include #include +#include struct irq_domain *x86_pci_msi_default_domain __ro_after_init; @@ -345,3 +346,8 @@ void dmar_free_hwirq(int irq) irq_domain_free_irqs(irq, 1); } #endif + +bool arch_restore_msi_irqs(struct pci_dev *dev) +{ + return xen_initdom_restore_msi(dev); +} diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 8b395821cb8d..7d20c1d34a3c 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -145,18 +145,6 @@ struct x86_platform_ops x86_platform __ro_after_init = { EXPORT_SYMBOL_GPL(x86_platform); -#if defined(CONFIG_PCI_MSI) -struct x86_msi_ops x86_msi __ro_after_init = { - .restore_msi_irqs = default_restore_msi_irqs, -}; - -/* MSI arch specific hooks */ -void arch_restore_msi_irqs(struct pci_dev *dev) -{ - x86_msi.restore_msi_irqs(dev); -} -#endif - struct x86_apic_ops x86_apic_ops __ro_after_init = { .io_apic_read = native_io_apic_read, .restore = native_restore_boot_irq_mode, diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 2dace884d6f7..a63d30a52879 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -351,10 +351,13 @@ out: return ret; } -static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) +bool xen_initdom_restore_msi(struct pci_dev *dev) { int ret = 0; + if (!xen_initial_domain()) + return true; + if (pci_seg_supported) { struct physdev_pci_device restore_ext; @@ -375,10 +378,10 @@ static void xen_initdom_restore_msi_irqs(struct pci_dev *dev) ret = HYPERVISOR_physdev_op(PHYSDEVOP_restore_msi, &restore); WARN(ret && ret != -ENOSYS, "restore_msi -> %d\n", ret); } + return false; } #else /* CONFIG_XEN_PV_DOM0 */ #define xen_initdom_setup_msi_irqs NULL -#define xen_initdom_restore_msi_irqs NULL #endif /* !CONFIG_XEN_PV_DOM0 */ static void xen_teardown_msi_irqs(struct pci_dev *dev) @@ -466,12 +469,10 @@ static __init struct irq_domain *xen_create_pci_msi_domain(void) static __init void xen_setup_pci_msi(void) { if (xen_pv_domain()) { - if (xen_initial_domain()) { + if (xen_initial_domain()) xen_msi_ops.setup_msi_irqs = xen_initdom_setup_msi_irqs; - x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs; - } else { + else xen_msi_ops.setup_msi_irqs = xen_setup_msi_irqs; - } xen_msi_ops.teardown_msi_irqs = xen_pv_teardown_msi_irqs; pci_msi_ignore_mask = 1; } else if (xen_hvm_domain()) { diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index b6cd8b337210..8260b06ab583 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -106,29 +106,6 @@ void __weak arch_teardown_msi_irqs(struct pci_dev *dev) } #endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ -static void default_restore_msi_irq(struct pci_dev *dev, int irq) -{ - struct msi_desc *entry; - - entry = NULL; - if (dev->msix_enabled) { - for_each_pci_msi_entry(entry, dev) { - if (irq == entry->irq) - break; - } - } else if (dev->msi_enabled) { - entry = irq_get_msi_desc(irq); - } - - if (entry) - __pci_write_msi_msg(entry, &entry->msg); -} - -void __weak arch_restore_msi_irqs(struct pci_dev *dev) -{ - return default_restore_msi_irqs(dev); -} - /* * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to * mask all MSI interrupts by clearing the MSI enable bit does not work @@ -242,14 +219,6 @@ void pci_msi_unmask_irq(struct irq_data *data) } EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); -void default_restore_msi_irqs(struct pci_dev *dev) -{ - struct msi_desc *entry; - - for_each_pci_msi_entry(entry, dev) - default_restore_msi_irq(dev, entry->irq); -} - void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); @@ -403,10 +372,19 @@ static void pci_msi_set_enable(struct pci_dev *dev, int enable) pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } +/* + * Architecture override returns true when the PCI MSI message should be + * written by the generic restore function. + */ +bool __weak arch_restore_msi_irqs(struct pci_dev *dev) +{ + return true; +} + static void __pci_restore_msi_state(struct pci_dev *dev) { - u16 control; struct msi_desc *entry; + u16 control; if (!dev->msi_enabled) return; @@ -415,7 +393,8 @@ static void __pci_restore_msi_state(struct pci_dev *dev) pci_intx_for_msi(dev, 0); pci_msi_set_enable(dev, 0); - arch_restore_msi_irqs(dev); + if (arch_restore_msi_irqs(dev)) + __pci_write_msi_msg(entry, &entry->msg); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); pci_msi_update_mask(entry, 0, 0); @@ -437,6 +416,7 @@ static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) static void __pci_restore_msix_state(struct pci_dev *dev) { struct msi_desc *entry; + bool write_msg; if (!dev->msix_enabled) return; @@ -447,9 +427,13 @@ static void __pci_restore_msix_state(struct pci_dev *dev) pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); - arch_restore_msi_irqs(dev); - for_each_pci_msi_entry(entry, dev) + write_msg = arch_restore_msi_irqs(dev); + + for_each_pci_msi_entry(entry, dev) { + if (write_msg) + __pci_write_msi_msg(entry, &entry->msg); pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); + } pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } diff --git a/include/linux/msi.h b/include/linux/msi.h index 7e5c13f4e41b..673899fc24f6 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -273,11 +273,10 @@ static inline void arch_teardown_msi_irqs(struct pci_dev *dev) #endif /* - * The restore hooks are still available as they are useful even - * for fully irq domain based setups. Courtesy to XEN/X86. + * The restore hook is still available even for fully irq domain based + * setups. Courtesy to XEN/X86. */ -void arch_restore_msi_irqs(struct pci_dev *dev); -void default_restore_msi_irqs(struct pci_dev *dev); +bool arch_restore_msi_irqs(struct pci_dev *dev); #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN -- cgit v1.2.3-59-g8ed1b From aa423ac4221abdfb8588751e7838ca5f42f56db3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:52 +0100 Subject: PCI/MSI: Split out irqdomain code Move the irqdomain specific code into its own file. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211206210224.817754783@linutronix.de --- drivers/pci/msi/Makefile | 1 + drivers/pci/msi/irqdomain.c | 279 ++++++++++++++++++++++++++++++++++++++ drivers/pci/msi/legacy.c | 13 +- drivers/pci/msi/msi.c | 319 ++------------------------------------------ drivers/pci/msi/msi.h | 39 ++++++ include/linux/msi.h | 11 -- 6 files changed, 340 insertions(+), 322 deletions(-) create mode 100644 drivers/pci/msi/irqdomain.c create mode 100644 drivers/pci/msi/msi.h (limited to 'include/linux') diff --git a/drivers/pci/msi/Makefile b/drivers/pci/msi/Makefile index 64bf14737bde..93ef7b9e404d 100644 --- a/drivers/pci/msi/Makefile +++ b/drivers/pci/msi/Makefile @@ -3,4 +3,5 @@ # Makefile for the PCI/MSI obj-$(CONFIG_PCI) += pcidev_msi.o obj-$(CONFIG_PCI_MSI) += msi.o +obj-$(CONFIG_PCI_MSI_IRQ_DOMAIN) += irqdomain.o obj-$(CONFIG_PCI_MSI_ARCH_FALLBACKS) += legacy.o diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c new file mode 100644 index 000000000000..123450e715cb --- /dev/null +++ b/drivers/pci/msi/irqdomain.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI Message Signaled Interrupt (MSI) - irqdomain support + */ +#include +#include +#include + +#include "msi.h" + +int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct irq_domain *domain; + + domain = dev_get_msi_domain(&dev->dev); + if (domain && irq_domain_is_hierarchy(domain)) + return msi_domain_alloc_irqs(domain, &dev->dev, nvec); + + return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); +} + +void pci_msi_teardown_msi_irqs(struct pci_dev *dev) +{ + struct irq_domain *domain; + + domain = dev_get_msi_domain(&dev->dev); + if (domain && irq_domain_is_hierarchy(domain)) + msi_domain_free_irqs(domain, &dev->dev); + else + pci_msi_legacy_teardown_msi_irqs(dev); +} + +/** + * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space + * @irq_data: Pointer to interrupt data of the MSI interrupt + * @msg: Pointer to the message + */ +static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) +{ + struct msi_desc *desc = irq_data_get_msi_desc(irq_data); + + /* + * For MSI-X desc->irq is always equal to irq_data->irq. For + * MSI only the first interrupt of MULTI MSI passes the test. + */ + if (desc->irq == irq_data->irq) + __pci_write_msi_msg(desc, msg); +} + +/** + * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source + * @desc: Pointer to the MSI descriptor + * + * The ID number is only used within the irqdomain. + */ +static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + + return (irq_hw_number_t)desc->pci.msi_attrib.entry_nr | + pci_dev_id(dev) << 11 | + (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; +} + +static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) +{ + return !desc->pci.msi_attrib.is_msix && desc->nvec_used > 1; +} + +/** + * pci_msi_domain_check_cap - Verify that @domain supports the capabilities + * for @dev + * @domain: The interrupt domain to check + * @info: The domain info for verification + * @dev: The device to check + * + * Returns: + * 0 if the functionality is supported + * 1 if Multi MSI is requested, but the domain does not support it + * -ENOTSUPP otherwise + */ +int pci_msi_domain_check_cap(struct irq_domain *domain, + struct msi_domain_info *info, struct device *dev) +{ + struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); + + /* Special handling to support __pci_enable_msi_range() */ + if (pci_msi_desc_is_multi_msi(desc) && + !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) + return 1; + else if (desc->pci.msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) + return -ENOTSUPP; + + return 0; +} + +static int pci_msi_domain_handle_error(struct irq_domain *domain, + struct msi_desc *desc, int error) +{ + /* Special handling to support __pci_enable_msi_range() */ + if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) + return 1; + + return error; +} + +static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, + struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = pci_msi_domain_calc_hwirq(desc); +} + +static struct msi_domain_ops pci_msi_domain_ops_default = { + .set_desc = pci_msi_domain_set_desc, + .msi_check = pci_msi_domain_check_cap, + .handle_error = pci_msi_domain_handle_error, +}; + +static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) +{ + struct msi_domain_ops *ops = info->ops; + + if (ops == NULL) { + info->ops = &pci_msi_domain_ops_default; + } else { + if (ops->set_desc == NULL) + ops->set_desc = pci_msi_domain_set_desc; + if (ops->msi_check == NULL) + ops->msi_check = pci_msi_domain_check_cap; + if (ops->handle_error == NULL) + ops->handle_error = pci_msi_domain_handle_error; + } +} + +static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + BUG_ON(!chip); + if (!chip->irq_write_msi_msg) + chip->irq_write_msi_msg = pci_msi_domain_write_msg; + if (!chip->irq_mask) + chip->irq_mask = pci_msi_mask_irq; + if (!chip->irq_unmask) + chip->irq_unmask = pci_msi_unmask_irq; +} + +/** + * pci_msi_create_irq_domain - Create a MSI interrupt domain + * @fwnode: Optional fwnode of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + * + * Updates the domain and chip ops and creates a MSI interrupt domain. + * + * Returns: + * A domain pointer or NULL in case of failure. + */ +struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, + struct msi_domain_info *info, + struct irq_domain *parent) +{ + struct irq_domain *domain; + + if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) + info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; + + if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) + pci_msi_domain_update_dom_ops(info); + if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) + pci_msi_domain_update_chip_ops(info); + + info->flags |= MSI_FLAG_ACTIVATE_EARLY; + if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) + info->flags |= MSI_FLAG_MUST_REACTIVATE; + + /* PCI-MSI is oneshot-safe */ + info->chip->flags |= IRQCHIP_ONESHOT_SAFE; + + domain = msi_create_irq_domain(fwnode, info, parent); + if (!domain) + return NULL; + + irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI); + return domain; +} +EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); + +/* + * Users of the generic MSI infrastructure expect a device to have a single ID, + * so with DMA aliases we have to pick the least-worst compromise. Devices with + * DMA phantom functions tend to still emit MSIs from the real function number, + * so we ignore those and only consider topological aliases where either the + * alias device or RID appears on a different bus number. We also make the + * reasonable assumption that bridges are walked in an upstream direction (so + * the last one seen wins), and the much braver assumption that the most likely + * case is that of PCI->PCIe so we should always use the alias RID. This echoes + * the logic from intel_irq_remapping's set_msi_sid(), which presumably works + * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions + * for taking ownership all we can really do is close our eyes and hope... + */ +static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) +{ + u32 *pa = data; + u8 bus = PCI_BUS_NUM(*pa); + + if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) + *pa = alias; + + return 0; +} + +/** + * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) + * @domain: The interrupt domain + * @pdev: The PCI device. + * + * The RID for a device is formed from the alias, with a firmware + * supplied mapping applied + * + * Returns: The RID. + */ +u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) +{ + struct device_node *of_node; + u32 rid = pci_dev_id(pdev); + + pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); + + of_node = irq_domain_get_of_node(domain); + rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : + iort_msi_map_id(&pdev->dev, rid); + + return rid; +} + +/** + * pci_msi_get_device_domain - Get the MSI domain for a given PCI device + * @pdev: The PCI device + * + * Use the firmware data to find a device-specific MSI domain + * (i.e. not one that is set as a default). + * + * Returns: The corresponding MSI domain or NULL if none has been found. + */ +struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) +{ + struct irq_domain *dom; + u32 rid = pci_dev_id(pdev); + + pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); + dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); + if (!dom) + dom = iort_get_device_domain(&pdev->dev, rid, + DOMAIN_BUS_PCI_MSI); + return dom; +} + +/** + * pci_dev_has_special_msi_domain - Check whether the device is handled by + * a non-standard PCI-MSI domain + * @pdev: The PCI device to check. + * + * Returns: True if the device irqdomain or the bus irqdomain is + * non-standard PCI/MSI. + */ +bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) +{ + struct irq_domain *dom = dev_get_msi_domain(&pdev->dev); + + if (!dom) + dom = dev_get_msi_domain(&pdev->bus->dev); + + if (!dom) + return true; + + return dom->bus_token != DOMAIN_BUS_PCI_MSI; +} diff --git a/drivers/pci/msi/legacy.c b/drivers/pci/msi/legacy.c index 4c76c59f563e..023de20ee3a7 100644 --- a/drivers/pci/msi/legacy.c +++ b/drivers/pci/msi/legacy.c @@ -4,8 +4,7 @@ * * Legacy architecture specific setup and teardown mechanism. */ -#include -#include +#include "msi.h" /* Arch hooks */ int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) @@ -50,3 +49,13 @@ void __weak arch_teardown_msi_irqs(struct pci_dev *dev) } } } + +int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + return arch_setup_msi_irqs(dev, nvec, type); +} + +void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev) +{ + arch_teardown_msi_irqs(dev); +} diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 78f10ce6e0cb..8926a6c8b345 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -6,64 +6,16 @@ * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) * Copyright (C) 2016 Christoph Hellwig. */ - -#include #include #include #include -#include -#include -#include -#include #include "../pci.h" +#include "msi.h" static int pci_msi_enable = 1; int pci_msi_ignore_mask; -#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) - -#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct irq_domain *domain; - - domain = dev_get_msi_domain(&dev->dev); - if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs(domain, &dev->dev, nvec); - - return arch_setup_msi_irqs(dev, nvec, type); -} - -static void pci_msi_teardown_msi_irqs(struct pci_dev *dev) -{ - struct irq_domain *domain; - - domain = dev_get_msi_domain(&dev->dev); - if (domain && irq_domain_is_hierarchy(domain)) - msi_domain_free_irqs(domain, &dev->dev); - else - arch_teardown_msi_irqs(dev); -} -#else -#define pci_msi_setup_msi_irqs arch_setup_msi_irqs -#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs -#endif - -/* - * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to - * mask all MSI interrupts by clearing the MSI enable bit does not work - * reliably as devices without an INTx disable bit will then generate a - * level IRQ which will never be cleared. - */ -static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) -{ - /* Don't shift by >= width of type */ - if (desc->pci.msi_attrib.multi_cap >= 5) - return 0xffffffff; - return (1 << (1 << desc->pci.msi_attrib.multi_cap)) - 1; -} - static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) { raw_spinlock_t *lock = &desc->dev->msi_lock; @@ -903,23 +855,6 @@ void pci_disable_msix(struct pci_dev *dev) } EXPORT_SYMBOL(pci_disable_msix); -void pci_no_msi(void) -{ - pci_msi_enable = 0; -} - -/** - * pci_msi_enabled - is MSI enabled? - * - * Returns true if MSI has not been disabled by the command-line option - * pci=nomsi. - **/ -int pci_msi_enabled(void) -{ - return pci_msi_enable; -} -EXPORT_SYMBOL(pci_msi_enabled); - static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, struct irq_affinity *affd) { @@ -1195,253 +1130,19 @@ struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) } EXPORT_SYMBOL(msi_desc_to_pci_dev); -#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN -/** - * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space - * @irq_data: Pointer to interrupt data of the MSI interrupt - * @msg: Pointer to the message - */ -static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) -{ - struct msi_desc *desc = irq_data_get_msi_desc(irq_data); - - /* - * For MSI-X desc->irq is always equal to irq_data->irq. For - * MSI only the first interrupt of MULTI MSI passes the test. - */ - if (desc->irq == irq_data->irq) - __pci_write_msi_msg(desc, msg); -} - -/** - * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source - * @desc: Pointer to the MSI descriptor - * - * The ID number is only used within the irqdomain. - */ -static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) -{ - struct pci_dev *dev = msi_desc_to_pci_dev(desc); - - return (irq_hw_number_t)desc->pci.msi_attrib.entry_nr | - pci_dev_id(dev) << 11 | - (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; -} - -static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) -{ - return !desc->pci.msi_attrib.is_msix && desc->nvec_used > 1; -} - -/** - * pci_msi_domain_check_cap - Verify that @domain supports the capabilities - * for @dev - * @domain: The interrupt domain to check - * @info: The domain info for verification - * @dev: The device to check - * - * Returns: - * 0 if the functionality is supported - * 1 if Multi MSI is requested, but the domain does not support it - * -ENOTSUPP otherwise - */ -int pci_msi_domain_check_cap(struct irq_domain *domain, - struct msi_domain_info *info, struct device *dev) -{ - struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); - - /* Special handling to support __pci_enable_msi_range() */ - if (pci_msi_desc_is_multi_msi(desc) && - !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) - return 1; - else if (desc->pci.msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) - return -ENOTSUPP; - - return 0; -} - -static int pci_msi_domain_handle_error(struct irq_domain *domain, - struct msi_desc *desc, int error) -{ - /* Special handling to support __pci_enable_msi_range() */ - if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) - return 1; - - return error; -} - -static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, - struct msi_desc *desc) -{ - arg->desc = desc; - arg->hwirq = pci_msi_domain_calc_hwirq(desc); -} - -static struct msi_domain_ops pci_msi_domain_ops_default = { - .set_desc = pci_msi_domain_set_desc, - .msi_check = pci_msi_domain_check_cap, - .handle_error = pci_msi_domain_handle_error, -}; - -static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) -{ - struct msi_domain_ops *ops = info->ops; - - if (ops == NULL) { - info->ops = &pci_msi_domain_ops_default; - } else { - if (ops->set_desc == NULL) - ops->set_desc = pci_msi_domain_set_desc; - if (ops->msi_check == NULL) - ops->msi_check = pci_msi_domain_check_cap; - if (ops->handle_error == NULL) - ops->handle_error = pci_msi_domain_handle_error; - } -} - -static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) -{ - struct irq_chip *chip = info->chip; - - BUG_ON(!chip); - if (!chip->irq_write_msi_msg) - chip->irq_write_msi_msg = pci_msi_domain_write_msg; - if (!chip->irq_mask) - chip->irq_mask = pci_msi_mask_irq; - if (!chip->irq_unmask) - chip->irq_unmask = pci_msi_unmask_irq; -} - -/** - * pci_msi_create_irq_domain - Create a MSI interrupt domain - * @fwnode: Optional fwnode of the interrupt controller - * @info: MSI domain info - * @parent: Parent irq domain - * - * Updates the domain and chip ops and creates a MSI interrupt domain. - * - * Returns: - * A domain pointer or NULL in case of failure. - */ -struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, - struct msi_domain_info *info, - struct irq_domain *parent) -{ - struct irq_domain *domain; - - if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) - info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; - - if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) - pci_msi_domain_update_dom_ops(info); - if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) - pci_msi_domain_update_chip_ops(info); - - info->flags |= MSI_FLAG_ACTIVATE_EARLY; - if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) - info->flags |= MSI_FLAG_MUST_REACTIVATE; - - /* PCI-MSI is oneshot-safe */ - info->chip->flags |= IRQCHIP_ONESHOT_SAFE; - - domain = msi_create_irq_domain(fwnode, info, parent); - if (!domain) - return NULL; - - irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI); - return domain; -} -EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); - -/* - * Users of the generic MSI infrastructure expect a device to have a single ID, - * so with DMA aliases we have to pick the least-worst compromise. Devices with - * DMA phantom functions tend to still emit MSIs from the real function number, - * so we ignore those and only consider topological aliases where either the - * alias device or RID appears on a different bus number. We also make the - * reasonable assumption that bridges are walked in an upstream direction (so - * the last one seen wins), and the much braver assumption that the most likely - * case is that of PCI->PCIe so we should always use the alias RID. This echoes - * the logic from intel_irq_remapping's set_msi_sid(), which presumably works - * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions - * for taking ownership all we can really do is close our eyes and hope... - */ -static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) -{ - u32 *pa = data; - u8 bus = PCI_BUS_NUM(*pa); - - if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) - *pa = alias; - - return 0; -} - -/** - * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) - * @domain: The interrupt domain - * @pdev: The PCI device. - * - * The RID for a device is formed from the alias, with a firmware - * supplied mapping applied - * - * Returns: The RID. - */ -u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) -{ - struct device_node *of_node; - u32 rid = pci_dev_id(pdev); - - pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); - - of_node = irq_domain_get_of_node(domain); - rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : - iort_msi_map_id(&pdev->dev, rid); - - return rid; -} - -/** - * pci_msi_get_device_domain - Get the MSI domain for a given PCI device - * @pdev: The PCI device - * - * Use the firmware data to find a device-specific MSI domain - * (i.e. not one that is set as a default). - * - * Returns: The corresponding MSI domain or NULL if none has been found. - */ -struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) +void pci_no_msi(void) { - struct irq_domain *dom; - u32 rid = pci_dev_id(pdev); - - pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); - dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); - if (!dom) - dom = iort_get_device_domain(&pdev->dev, rid, - DOMAIN_BUS_PCI_MSI); - return dom; + pci_msi_enable = 0; } /** - * pci_dev_has_special_msi_domain - Check whether the device is handled by - * a non-standard PCI-MSI domain - * @pdev: The PCI device to check. + * pci_msi_enabled - is MSI enabled? * - * Returns: True if the device irqdomain or the bus irqdomain is - * non-standard PCI/MSI. - */ -bool pci_dev_has_special_msi_domain(struct pci_dev *pdev) + * Returns true if MSI has not been disabled by the command-line option + * pci=nomsi. + **/ +int pci_msi_enabled(void) { - struct irq_domain *dom = dev_get_msi_domain(&pdev->dev); - - if (!dom) - dom = dev_get_msi_domain(&pdev->bus->dev); - - if (!dom) - return true; - - return dom->bus_token != DOMAIN_BUS_PCI_MSI; + return pci_msi_enable; } - -#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +EXPORT_SYMBOL(pci_msi_enabled); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h new file mode 100644 index 000000000000..dbeff066bedd --- /dev/null +++ b/drivers/pci/msi/msi.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include + +#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) + +extern int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern void pci_msi_teardown_msi_irqs(struct pci_dev *dev); + +#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS +extern int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev); +#else +static inline int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + WARN_ON_ONCE(1); + return -ENODEV; +} + +static inline void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev) +{ + WARN_ON_ONCE(1); +} +#endif + +/* + * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to + * mask all MSI interrupts by clearing the MSI enable bit does not work + * reliably as devices without an INTx disable bit will then generate a + * level IRQ which will never be cleared. + */ +static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) +{ + /* Don't shift by >= width of type */ + if (desc->pci.msi_attrib.multi_cap >= 5) + return 0xffffffff; + return (1 << (1 << desc->pci.msi_attrib.multi_cap)) - 1; +} diff --git a/include/linux/msi.h b/include/linux/msi.h index 673899fc24f6..7ff7cf23b78d 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -259,17 +259,6 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc); void arch_teardown_msi_irq(unsigned int irq); int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); void arch_teardown_msi_irqs(struct pci_dev *dev); -#else -static inline int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - WARN_ON_ONCE(1); - return -ENODEV; -} - -static inline void arch_teardown_msi_irqs(struct pci_dev *dev) -{ - WARN_ON_ONCE(1); -} #endif /* -- cgit v1.2.3-59-g8ed1b From 85aa607e79f8343f1ea028b29bdf8b6bc99c729a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:54 +0100 Subject: PCI/MSI: Sanitize MSI-X table map handling Unmapping the MSI-X base mapping in the loops which allocate/free MSI descriptors is daft and in the way of allowing runtime expansion of MSI-X descriptors. Store the mapping in struct pci_dev and free it after freeing the MSI-X descriptors. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211206210224.871651518@linutronix.de --- drivers/pci/msi/msi.c | 18 ++++++++---------- include/linux/pci.h | 1 + 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 8926a6c8b345..465fe9e00294 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -241,14 +241,14 @@ static void free_msi_irqs(struct pci_dev *dev) pci_msi_teardown_msi_irqs(dev); list_for_each_entry_safe(entry, tmp, msi_list, list) { - if (entry->pci.msi_attrib.is_msix) { - if (list_is_last(&entry->list, msi_list)) - iounmap(entry->pci.mask_base); - } - list_del(&entry->list); free_msi_entry(entry); } + + if (dev->msix_base) { + iounmap(dev->msix_base); + dev->msix_base = NULL; + } } static void pci_intx_for_msi(struct pci_dev *dev, int enable) @@ -501,10 +501,6 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, for (i = 0, curmsk = masks; i < nvec; i++) { entry = alloc_msi_entry(&dev->dev, 1, curmsk); if (!entry) { - if (!i) - iounmap(base); - else - free_msi_irqs(dev); /* No enough memory. Don't try again */ ret = -ENOMEM; goto out; @@ -602,12 +598,14 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, goto out_disable; } + dev->msix_base = base; + /* Ensure that all table entries are masked. */ msix_mask_all(base, tsize); ret = msix_setup_entries(dev, base, entries, nvec, affd); if (ret) - goto out_disable; + goto out_free; ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); if (ret) diff --git a/include/linux/pci.h b/include/linux/pci.h index 18a75c8e615c..8cb103677f5a 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -473,6 +473,7 @@ struct pci_dev { u8 ptm_granularity; #endif #ifdef CONFIG_PCI_MSI + void __iomem *msix_base; const struct attribute_group **msi_irq_groups; #endif struct pci_vpd vpd; -- cgit v1.2.3-59-g8ed1b From cd119b09a87d8beb50356d8c5c6aa42d89c44eb7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:56 +0100 Subject: PCI/MSI: Move msi_lock to struct pci_dev It's only required for PCI/MSI. So no point in having it in every struct device. Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211206210224.925241961@linutronix.de --- drivers/base/core.c | 1 - drivers/pci/msi/msi.c | 2 +- drivers/pci/probe.c | 4 +++- include/linux/device.h | 2 -- include/linux/pci.h | 1 + 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index fd034d742447..f26c668092d6 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2875,7 +2875,6 @@ void device_initialize(struct device *dev) device_pm_init(dev); set_dev_node(dev, NUMA_NO_NODE); #ifdef CONFIG_GENERIC_MSI_IRQ - raw_spin_lock_init(&dev->msi_lock); INIT_LIST_HEAD(&dev->msi_list); #endif INIT_LIST_HEAD(&dev->links.consumers); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 465fe9e00294..443a16c67e1a 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -18,7 +18,7 @@ int pci_msi_ignore_mask; static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) { - raw_spinlock_t *lock = &desc->dev->msi_lock; + raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock; unsigned long flags; if (!desc->pci.msi_attrib.can_mask) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 087d3658f75c..443efb00e219 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2311,7 +2311,9 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus) INIT_LIST_HEAD(&dev->bus_list); dev->dev.type = &pci_dev_type; dev->bus = pci_bus_get(bus); - +#ifdef CONFIG_PCI_MSI + raw_spin_lock_init(&dev->msi_lock); +#endif return dev; } EXPORT_SYMBOL(pci_alloc_dev); diff --git a/include/linux/device.h b/include/linux/device.h index e270cb740b9e..2a22875238a6 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -407,7 +407,6 @@ struct dev_links_info { * @em_pd: device's energy model performance domain * @pins: For device pin management. * See Documentation/driver-api/pin-control.rst for details. - * @msi_lock: Lock to protect MSI mask cache and mask register * @msi_list: Hosts MSI descriptors * @msi_domain: The generic MSI domain this device is using. * @numa_node: NUMA node this device is close to. @@ -508,7 +507,6 @@ struct device { struct dev_pin_info *pins; #endif #ifdef CONFIG_GENERIC_MSI_IRQ - raw_spinlock_t msi_lock; struct list_head msi_list; #endif #ifdef CONFIG_DMA_OPS diff --git a/include/linux/pci.h b/include/linux/pci.h index 8cb103677f5a..5cc46baef519 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -474,6 +474,7 @@ struct pci_dev { #endif #ifdef CONFIG_PCI_MSI void __iomem *msix_base; + raw_spinlock_t msi_lock; const struct attribute_group **msi_irq_groups; #endif struct pci_vpd vpd; -- cgit v1.2.3-59-g8ed1b From 57ce3a3c99b21e9c4f951ef01e0a3603c987c259 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:57 +0100 Subject: PCI/MSI: Make pci_msi_domain_check_cap() static No users outside of that file. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211206210224.980989243@linutronix.de --- drivers/pci/msi/irqdomain.c | 5 +++-- include/linux/msi.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 123450e715cb..6abd8aff2cea 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -79,8 +79,9 @@ static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) * 1 if Multi MSI is requested, but the domain does not support it * -ENOTSUPP otherwise */ -int pci_msi_domain_check_cap(struct irq_domain *domain, - struct msi_domain_info *info, struct device *dev) +static int pci_msi_domain_check_cap(struct irq_domain *domain, + struct msi_domain_info *info, + struct device *dev) { struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); diff --git a/include/linux/msi.h b/include/linux/msi.h index 7ff7cf23b78d..5248678e05d1 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -439,8 +439,6 @@ void *platform_msi_get_host_data(struct irq_domain *domain); struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, struct msi_domain_info *info, struct irq_domain *parent); -int pci_msi_domain_check_cap(struct irq_domain *domain, - struct msi_domain_info *info, struct device *dev); u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev); struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev); bool pci_dev_has_special_msi_domain(struct pci_dev *pdev); -- cgit v1.2.3-59-g8ed1b From 890337624e1fa2da079fc1c036a62d178c985280 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:27:59 +0100 Subject: genirq/msi: Handle PCI/MSI allocation fail in core code Get rid of yet another irqdomain callback and let the core code return the already available information of how many descriptors could be allocated. Signed-off-by: Thomas Gleixner Tested-by: Juergen Gross Reviewed-by: Jason Gunthorpe Reviewed-by: Greg Kroah-Hartman Acked-by: Bjorn Helgaas # PCI Link: https://lore.kernel.org/r/20211206210225.046615302@linutronix.de --- drivers/pci/msi/irqdomain.c | 13 ------------- include/linux/msi.h | 5 +---- kernel/irq/msi.c | 29 +++++++++++++++++++++++++---- 3 files changed, 26 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 6abd8aff2cea..a5546900244d 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -95,16 +95,6 @@ static int pci_msi_domain_check_cap(struct irq_domain *domain, return 0; } -static int pci_msi_domain_handle_error(struct irq_domain *domain, - struct msi_desc *desc, int error) -{ - /* Special handling to support __pci_enable_msi_range() */ - if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) - return 1; - - return error; -} - static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) { @@ -115,7 +105,6 @@ static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, static struct msi_domain_ops pci_msi_domain_ops_default = { .set_desc = pci_msi_domain_set_desc, .msi_check = pci_msi_domain_check_cap, - .handle_error = pci_msi_domain_handle_error, }; static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) @@ -129,8 +118,6 @@ static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) ops->set_desc = pci_msi_domain_set_desc; if (ops->msi_check == NULL) ops->msi_check = pci_msi_domain_check_cap; - if (ops->handle_error == NULL) - ops->handle_error = pci_msi_domain_handle_error; } } diff --git a/include/linux/msi.h b/include/linux/msi.h index 5248678e05d1..ba4a39c430b5 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -286,7 +286,6 @@ struct msi_domain_info; * @msi_check: Callback for verification of the domain/info/dev data * @msi_prepare: Prepare the allocation of the interrupts in the domain * @set_desc: Set the msi descriptor for an interrupt - * @handle_error: Optional error handler if the allocation fails * @domain_alloc_irqs: Optional function to override the default allocation * function. * @domain_free_irqs: Optional function to override the default free @@ -295,7 +294,7 @@ struct msi_domain_info; * @get_hwirq, @msi_init and @msi_free are callbacks used by the underlying * irqdomain. * - * @msi_check, @msi_prepare, @handle_error and @set_desc are callbacks used by + * @msi_check, @msi_prepare and @set_desc are callbacks used by * msi_domain_alloc/free_irqs(). * * @domain_alloc_irqs, @domain_free_irqs can be used to override the @@ -332,8 +331,6 @@ struct msi_domain_ops { msi_alloc_info_t *arg); void (*set_desc)(msi_alloc_info_t *arg, struct msi_desc *desc); - int (*handle_error)(struct irq_domain *domain, - struct msi_desc *desc, int error); int (*domain_alloc_irqs)(struct irq_domain *domain, struct device *dev, int nvec); void (*domain_free_irqs)(struct irq_domain *domain, diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 7d78d8aff076..4a7a7f0f5102 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -538,6 +538,27 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask; } +static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc, + int allocated) +{ + switch(domain->bus_token) { + case DOMAIN_BUS_PCI_MSI: + case DOMAIN_BUS_VMD_MSI: + if (IS_ENABLED(CONFIG_PCI_MSI)) + break; + fallthrough; + default: + return -ENOSPC; + } + + /* Let a failed PCI multi MSI allocation retry */ + if (desc->nvec_used > 1) + return 1; + + /* If there was a successful allocation let the caller know */ + return allocated ? allocated : -ENOSPC; +} + int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec) { @@ -546,6 +567,7 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, struct irq_data *irq_data; struct msi_desc *desc; msi_alloc_info_t arg = { }; + int allocated = 0; int i, ret, virq; bool can_reserve; @@ -560,16 +582,15 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, dev_to_node(dev), &arg, false, desc->affinity); if (virq < 0) { - ret = -ENOSPC; - if (ops->handle_error) - ret = ops->handle_error(domain, desc, ret); - return ret; + ret = msi_handle_pci_fail(domain, desc, allocated); + goto cleanup; } for (i = 0; i < desc->nvec_used; i++) { irq_set_msi_desc_off(virq, i, desc); irq_debugfs_copy_devname(virq + i, dev); } + allocated++; } can_reserve = msi_check_reservation_mode(domain, info, dev); -- cgit v1.2.3-59-g8ed1b From 50468e4313355b161cac8a5155a45832995b7f25 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Tue, 16 Nov 2021 18:21:16 +0200 Subject: x86/sgx: Add an attribute for the amount of SGX memory in a NUMA node == Problem == The amount of SGX memory on a system is determined by the BIOS and it varies wildly between systems. It can be as small as dozens of MB's and as large as many GB's on servers. Just like how applications need to know how much regular RAM is available, enclave builders need to know how much SGX memory an enclave can consume. == Solution == Introduce a new sysfs file: /sys/devices/system/node/nodeX/x86/sgx_total_bytes to enumerate the amount of SGX memory available in each NUMA node. This serves the same function for SGX as /proc/meminfo or /sys/devices/system/node/nodeX/meminfo does for normal RAM. 'sgx_total_bytes' is needed today to help drive the SGX selftests. SGX-specific swap code is exercised by creating overcommitted enclaves which are larger than the physical SGX memory on the system. They currently use a CPUID-based approach which can diverge from the actual amount of SGX memory available. 'sgx_total_bytes' ensures that the selftests can work efficiently and do not attempt stupid things like creating a 100,000 MB enclave on a system with 128 MB of SGX memory. == Implementation Details == Introduce CONFIG_HAVE_ARCH_NODE_DEV_GROUP opt-in flag to expose an arch specific attribute group, and add an attribute for the amount of SGX memory in bytes to each NUMA node: == ABI Design Discussion == As opposed to the per-node ABI, a single, global ABI was considered. However, this would prevent enclaves from being able to size themselves so that they fit on a single NUMA node. Essentially, a single value would rule out NUMA optimizations for enclaves. Create a new "x86/" directory inside each "nodeX/" sysfs directory. 'sgx_total_bytes' is expected to be the first of at least a few sgx-specific files to be placed in the new directory. Just scanning /proc/meminfo, these are the no-brainers that we have for RAM, but we need for SGX: MemTotal: xxxx kB // sgx_total_bytes (implemented here) MemFree: yyyy kB // sgx_free_bytes SwapTotal: zzzz kB // sgx_swapped_bytes So, at *least* three. I think we will eventually end up needing something more along the lines of a dozen. A new directory (as opposed to being in the nodeX/ "root") directory avoids cluttering the root with several "sgx_*" files. Place the new file in a new "nodeX/x86/" directory because SGX is highly x86-specific. It is very unlikely that any other architecture (or even non-Intel x86 vendor) will ever implement SGX. Using "sgx/" as opposed to "x86/" was also considered. But, there is a real chance this can get used for other arch-specific purposes. [ dhansen: rewrite changelog ] Signed-off-by: Jarkko Sakkinen Signed-off-by: Dave Hansen Acked-by: Greg Kroah-Hartman Acked-by: Borislav Petkov Link: https://lkml.kernel.org/r/20211116162116.93081-2-jarkko@kernel.org --- Documentation/ABI/stable/sysfs-devices-node | 6 ++++++ arch/Kconfig | 4 ++++ arch/x86/Kconfig | 1 + arch/x86/kernel/cpu/sgx/main.c | 20 ++++++++++++++++++++ arch/x86/kernel/cpu/sgx/sgx.h | 1 + drivers/base/node.c | 3 +++ include/linux/numa.h | 4 ++++ 7 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/stable/sysfs-devices-node b/Documentation/ABI/stable/sysfs-devices-node index 484fc04bcc25..8db67aa472f1 100644 --- a/Documentation/ABI/stable/sysfs-devices-node +++ b/Documentation/ABI/stable/sysfs-devices-node @@ -176,3 +176,9 @@ Contact: Keith Busch Description: The cache write policy: 0 for write-back, 1 for write-through, other or unknown. + +What: /sys/devices/system/node/nodeX/x86/sgx_total_bytes +Date: November 2021 +Contact: Jarkko Sakkinen +Description: + The total amount of SGX physical memory in bytes. diff --git a/arch/Kconfig b/arch/Kconfig index 26b8ed11639d..0a9dadb00b61 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1302,6 +1302,10 @@ config ARCH_HAS_PARANOID_L1D_FLUSH config DYNAMIC_SIGFRAME bool +# Select, if arch has a named attribute group bound to NUMA device nodes. +config HAVE_ARCH_NODE_DEV_GROUP + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index b9281fab4e3e..f2b699d12eb8 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -269,6 +269,7 @@ config X86 select HAVE_ARCH_KCSAN if X86_64 select X86_FEATURE_NAMES if PROC_FS select PROC_PID_ARCH_STATUS if PROC_FS + select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI config INSTRUCTION_DECODER diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 6036328de255..2857a49f2335 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -825,9 +825,11 @@ static bool __init sgx_page_cache_init(void) INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list); INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list); node_set(nid, sgx_numa_mask); + sgx_numa_nodes[nid].size = 0; } sgx_epc_sections[i].node = &sgx_numa_nodes[nid]; + sgx_numa_nodes[nid].size += size; sgx_nr_epc_sections++; } @@ -901,6 +903,24 @@ int sgx_set_attribute(unsigned long *allowed_attributes, } EXPORT_SYMBOL_GPL(sgx_set_attribute); +#ifdef CONFIG_NUMA +static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size); +} +static DEVICE_ATTR_RO(sgx_total_bytes); + +static struct attribute *arch_node_dev_attrs[] = { + &dev_attr_sgx_total_bytes.attr, + NULL, +}; + +const struct attribute_group arch_node_dev_group = { + .name = "x86", + .attrs = arch_node_dev_attrs, +}; +#endif /* CONFIG_NUMA */ + static int __init sgx_init(void) { int ret; diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index 9ec3136c7800..0f17def9fe6f 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -44,6 +44,7 @@ struct sgx_epc_page { struct sgx_numa_node { struct list_head free_page_list; struct list_head sgx_poison_page_list; + unsigned long size; spinlock_t lock; }; diff --git a/drivers/base/node.c b/drivers/base/node.c index b5a4ba18f9f9..87acc47e8951 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -581,6 +581,9 @@ static const struct attribute_group node_dev_group = { static const struct attribute_group *node_dev_groups[] = { &node_dev_group, +#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP + &arch_node_dev_group, +#endif NULL }; diff --git a/include/linux/numa.h b/include/linux/numa.h index cb44cfe2b725..59df211d051f 100644 --- a/include/linux/numa.h +++ b/include/linux/numa.h @@ -58,4 +58,8 @@ static inline int phys_to_target_node(u64 start) } #endif +#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP +extern const struct attribute_group arch_node_dev_group; +#endif + #endif /* _LINUX_NUMA_H */ -- cgit v1.2.3-59-g8ed1b From 67b967ddd93d0ed57d392a00f6f90060f0910c0e Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 18 Nov 2021 12:46:59 +0100 Subject: mtd: Introduce an expert mode for forensics and debugging purposes When developping NAND controller drivers or when debugging filesystem corruptions, it is quite common to need hacking locally into the MTD/NAND core in order to get access to the content of the bad blocks. Instead of having multiple implementations out there let's provide a simple yet effective specific MTD-wide debugfs entry to fully disable these checks on purpose. A warning is added to inform the user when this mode gets enabled. Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20211118114659.1282855-1-miquel.raynal@bootlin.com --- drivers/mtd/mtdcore.c | 10 ++++++++++ drivers/mtd/nand/core.c | 3 +++ drivers/mtd/nand/raw/nand_base.c | 3 +++ drivers/mtd/nand/raw/nand_bbt.c | 3 +++ include/linux/mtd/mtd.h | 3 +++ 5 files changed, 22 insertions(+) (limited to 'include/linux') diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index a7723a708ea7..70f492dce158 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -2370,6 +2370,14 @@ static struct backing_dev_info * __init mtd_bdi_init(const char *name) return ret ? ERR_PTR(ret) : bdi; } +char *mtd_expert_analysis_warning = + "Bad block checks have been entirely disabled.\n" + "This is only reserved for post-mortem forensics and debug purposes.\n" + "Never enable this mode if you do not know what you are doing!\n"; +EXPORT_SYMBOL_GPL(mtd_expert_analysis_warning); +bool mtd_expert_analysis_mode; +EXPORT_SYMBOL_GPL(mtd_expert_analysis_mode); + static struct proc_dir_entry *proc_mtd; static int __init init_mtd(void) @@ -2393,6 +2401,8 @@ static int __init init_mtd(void) goto out_procfs; dfs_dir_mtd = debugfs_create_dir("mtd", NULL); + debugfs_create_bool("expert_analysis_mode", 0600, dfs_dir_mtd, + &mtd_expert_analysis_mode); return 0; diff --git a/drivers/mtd/nand/core.c b/drivers/mtd/nand/core.c index 5e13a03d2b32..416947f28b67 100644 --- a/drivers/mtd/nand/core.c +++ b/drivers/mtd/nand/core.c @@ -21,6 +21,9 @@ */ bool nanddev_isbad(struct nand_device *nand, const struct nand_pos *pos) { + if (WARN_ONCE(mtd_expert_analysis_mode, mtd_expert_analysis_warning)) + return false; + if (nanddev_bbt_is_initialized(nand)) { unsigned int entry; int status; diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 3d6c6e880520..b3a9bc08b4bb 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -321,6 +321,9 @@ static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) if (nand_region_is_secured(chip, ofs, mtd->erasesize)) return -EIO; + if (WARN_ONCE(mtd_expert_analysis_mode, mtd_expert_analysis_warning)) + return 0; + if (chip->legacy.block_bad) return chip->legacy.block_bad(chip, ofs); diff --git a/drivers/mtd/nand/raw/nand_bbt.c b/drivers/mtd/nand/raw/nand_bbt.c index b7ad030225f8..ab630af3a309 100644 --- a/drivers/mtd/nand/raw/nand_bbt.c +++ b/drivers/mtd/nand/raw/nand_bbt.c @@ -1455,6 +1455,9 @@ int nand_isbad_bbt(struct nand_chip *this, loff_t offs, int allowbbt) pr_debug("nand_isbad_bbt(): bbt info for offs 0x%08x: (block %d) 0x%02x\n", (unsigned int)offs, block, res); + if (WARN_ONCE(mtd_expert_analysis_mode, mtd_expert_analysis_warning)) + return 0; + switch (res) { case BBT_BLOCK_GOOD: return 0; diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h index f5e7dfc2e4e9..1ffa933121f6 100644 --- a/include/linux/mtd/mtd.h +++ b/include/linux/mtd/mtd.h @@ -711,4 +711,7 @@ static inline int mtd_is_bitflip_or_eccerr(int err) { unsigned mtd_mmap_capabilities(struct mtd_info *mtd); +extern char *mtd_expert_analysis_warning; +extern bool mtd_expert_analysis_mode; + #endif /* __MTD_MTD_H__ */ -- cgit v1.2.3-59-g8ed1b From 6abfaaf124a81b7d2ab132cc2c9885baa14171e5 Mon Sep 17 00:00:00 2001 From: Lukas Czerner Date: Wed, 27 Oct 2021 16:18:45 +0200 Subject: fs_parse: allow parameter value to be empty Allow parameter value to be empty by specifying fs_param_can_be_empty flag. Signed-off-by: Lukas Czerner Cc: Al Viro Reviewed-by: Carlos Maiolino Link: https://lore.kernel.org/r/20211027141857.33657-2-lczerner@redhat.com Signed-off-by: Theodore Ts'o --- fs/fs_parser.c | 31 +++++++++++++++++++++++-------- include/linux/fs_parser.h | 2 +- 2 files changed, 24 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/fs_parser.c b/fs/fs_parser.c index 3df07c0e32b3..ed40ce5742fd 100644 --- a/fs/fs_parser.c +++ b/fs/fs_parser.c @@ -199,6 +199,8 @@ int fs_param_is_bool(struct p_log *log, const struct fs_parameter_spec *p, int b; if (param->type != fs_value_is_string) return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; b = lookup_constant(bool_names, param->string, -1); if (b == -1) return fs_param_bad_value(log, param); @@ -211,8 +213,11 @@ int fs_param_is_u32(struct p_log *log, const struct fs_parameter_spec *p, struct fs_parameter *param, struct fs_parse_result *result) { int base = (unsigned long)p->data; - if (param->type != fs_value_is_string || - kstrtouint(param->string, base, &result->uint_32) < 0) + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtouint(param->string, base, &result->uint_32) < 0) return fs_param_bad_value(log, param); return 0; } @@ -221,8 +226,11 @@ EXPORT_SYMBOL(fs_param_is_u32); int fs_param_is_s32(struct p_log *log, const struct fs_parameter_spec *p, struct fs_parameter *param, struct fs_parse_result *result) { - if (param->type != fs_value_is_string || - kstrtoint(param->string, 0, &result->int_32) < 0) + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtoint(param->string, 0, &result->int_32) < 0) return fs_param_bad_value(log, param); return 0; } @@ -231,8 +239,11 @@ EXPORT_SYMBOL(fs_param_is_s32); int fs_param_is_u64(struct p_log *log, const struct fs_parameter_spec *p, struct fs_parameter *param, struct fs_parse_result *result) { - if (param->type != fs_value_is_string || - kstrtoull(param->string, 0, &result->uint_64) < 0) + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtoull(param->string, 0, &result->uint_64) < 0) return fs_param_bad_value(log, param); return 0; } @@ -244,6 +255,8 @@ int fs_param_is_enum(struct p_log *log, const struct fs_parameter_spec *p, const struct constant_table *c; if (param->type != fs_value_is_string) return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; c = __lookup_constant(p->data, param->string); if (!c) return fs_param_bad_value(log, param); @@ -255,7 +268,8 @@ EXPORT_SYMBOL(fs_param_is_enum); int fs_param_is_string(struct p_log *log, const struct fs_parameter_spec *p, struct fs_parameter *param, struct fs_parse_result *result) { - if (param->type != fs_value_is_string || !*param->string) + if (param->type != fs_value_is_string || + (!*param->string && !(p->flags & fs_param_can_be_empty))) return fs_param_bad_value(log, param); return 0; } @@ -275,7 +289,8 @@ int fs_param_is_fd(struct p_log *log, const struct fs_parameter_spec *p, { switch (param->type) { case fs_value_is_string: - if (kstrtouint(param->string, 0, &result->uint_32) < 0) + if ((!*param->string && !(p->flags & fs_param_can_be_empty)) || + kstrtouint(param->string, 0, &result->uint_32) < 0) break; if (result->uint_32 <= INT_MAX) return 0; diff --git a/include/linux/fs_parser.h b/include/linux/fs_parser.h index aab0ffc6bac6..f103c91139d4 100644 --- a/include/linux/fs_parser.h +++ b/include/linux/fs_parser.h @@ -42,7 +42,7 @@ struct fs_parameter_spec { u8 opt; /* Option number (returned by fs_parse()) */ unsigned short flags; #define fs_param_neg_with_no 0x0002 /* "noxxx" is negative param */ -#define fs_param_neg_with_empty 0x0004 /* "xxx=" is negative param */ +#define fs_param_can_be_empty 0x0004 /* "xxx=" is allowed */ #define fs_param_deprecated 0x0008 /* The param is deprecated */ const void *data; }; -- cgit v1.2.3-59-g8ed1b From 3e5b1feccea7db576353ffc302f78d522e4116e6 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 9 Dec 2021 13:11:32 +0000 Subject: net: phylink: add legacy_pre_march2020 indicator Add a boolean to phylink_config to indicate whether a driver has not been updated for the changes in commit 7cceb599d15d ("net: phylink: avoid mac_config calls"), and thus are reliant on the old behaviour. We were currently keying the phylink behaviour on the presence of a PCS, but this is sub-optimal for modern drivers that may not have a PCS. This commit merely introduces the new flag, but does not add any use, since we need all legacy drivers to set this flag before it can be used. Once these legacy drivers have been updated, we can remove this flag. Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/phylink.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 01224235df0f..d005b8e36048 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -84,6 +84,8 @@ enum phylink_op_type { * struct phylink_config - PHYLINK configuration structure * @dev: a pointer to a struct device associated with the MAC * @type: operation type of PHYLINK instance + * @legacy_pre_march2020: driver has not been updated for March 2020 updates + * (See commit 7cceb599d15d ("net: phylink: avoid mac_config calls") * @pcs_poll: MAC PCS cannot provide link change interrupt * @poll_fixed_state: if true, starts link_poll, * if MAC link is at %MLO_AN_FIXED mode. @@ -97,6 +99,7 @@ enum phylink_op_type { struct phylink_config { struct device *dev; enum phylink_op_type type; + bool legacy_pre_march2020; bool pcs_poll; bool poll_fixed_state; bool ovr_an_inband; -- cgit v1.2.3-59-g8ed1b From 001f4261fe4d5ae710cf1f445b6cae6d9d3ae26e Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 9 Dec 2021 13:11:48 +0000 Subject: net: phylink: use legacy_pre_march2020 Use the legacy flag to indicate whether we should operate in legacy mode. This allows us to stop using the presence of a PCS as an indicator to the age of the phylink user, and make PCS presence optional. Legacy mode involves: 1) calling mac_config() whenever the link comes up 2) calling mac_config() whenever the inband advertisement changes, possibly followed by a call to mac_an_restart() 3) making use of mac_an_restart() 4) making use of mac_pcs_get_state() All the above functionality was moved to a seperate "PCS" block of operations in March 2020. Update the documents to indicate that the differences that this flag makes. Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- drivers/net/phy/phylink.c | 12 ++++++------ include/linux/phylink.h | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 8e3861f09b4f..e47f2baf4b07 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -742,7 +742,7 @@ static void phylink_mac_pcs_an_restart(struct phylink *pl) phylink_autoneg_inband(pl->cur_link_an_mode)) { if (pl->pcs_ops) pl->pcs_ops->pcs_an_restart(pl->pcs); - else + else if (pl->config->legacy_pre_march2020) pl->mac_ops->mac_an_restart(pl->config); } } @@ -803,7 +803,7 @@ static int phylink_change_inband_advert(struct phylink *pl) if (test_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state)) return 0; - if (!pl->pcs_ops) { + if (!pl->pcs_ops && pl->config->legacy_pre_march2020) { /* Legacy method */ phylink_mac_config(pl, &pl->link_config); phylink_mac_pcs_an_restart(pl); @@ -854,7 +854,8 @@ static void phylink_mac_pcs_get_state(struct phylink *pl, if (pl->pcs_ops) pl->pcs_ops->pcs_get_state(pl->pcs, state); - else if (pl->mac_ops->mac_pcs_get_state) + else if (pl->mac_ops->mac_pcs_get_state && + pl->config->legacy_pre_march2020) pl->mac_ops->mac_pcs_get_state(pl->config, state); else state->link = 0; @@ -1048,12 +1049,11 @@ static void phylink_resolve(struct work_struct *w) } phylink_major_config(pl, false, &link_state); pl->link_config.interface = link_state.interface; - } else if (!pl->pcs_ops) { + } else if (!pl->pcs_ops && pl->config->legacy_pre_march2020) { /* The interface remains unchanged, only the speed, * duplex or pause settings have changed. Call the * old mac_config() method to configure the MAC/PCS - * only if we do not have a PCS installed (an - * unconverted user.) + * only if we do not have a legacy MAC driver. */ phylink_mac_config(pl, &link_state); } diff --git a/include/linux/phylink.h b/include/linux/phylink.h index d005b8e36048..a2f266cc3442 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -190,6 +190,10 @@ void validate(struct phylink_config *config, unsigned long *supported, * negotiation completion state in @state->an_complete, and link up state * in @state->link. If possible, @state->lp_advertising should also be * populated. + * + * Note: This is a legacy method. This function will not be called unless + * legacy_pre_march2020 is set in &struct phylink_config and there is no + * PCS attached. */ void mac_pcs_get_state(struct phylink_config *config, struct phylink_link_state *state); @@ -230,6 +234,15 @@ int mac_prepare(struct phylink_config *config, unsigned int mode, * guaranteed to be correct, and so any mac_config() implementation must * never reference these fields. * + * Note: For legacy March 2020 drivers (drivers with legacy_pre_march2020 set + * in their &phylnk_config and which don't have a PCS), this function will be + * called on each link up event, and to also change the in-band advert. For + * non-legacy drivers, it will only be called to reconfigure the MAC for a + * "major" change in e.g. interface mode. It will not be called for changes + * in speed, duplex or pause modes or to change the in-band advertisement. + * In any case, it is strongly preferred that speed, duplex and pause settings + * are handled in the mac_link_up() method and not in this method. + * * (this requires a rewrite - please refer to mac_link_up() for situations * where the PCS and MAC are not tightly integrated.) * @@ -314,6 +327,10 @@ int mac_finish(struct phylink_config *config, unsigned int mode, /** * mac_an_restart() - restart 802.3z BaseX autonegotiation * @config: a pointer to a &struct phylink_config. + * + * Note: This is a legacy method. This function will not be called unless + * legacy_pre_march2020 is set in &struct phylink_config and there is no + * PCS attached. */ void mac_an_restart(struct phylink_config *config); -- cgit v1.2.3-59-g8ed1b From 1a2fb220edca98d18f90e3ef5bd6853a6b22b1b8 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 6 Dec 2021 22:27:58 -0800 Subject: skbuff: Extract list pointers to silence compiler warnings Under both -Warray-bounds and the object_size sanitizer, the compiler is upset about accessing prev/next of sk_buff when the object it thinks it is coming from is sk_buff_head. The warning is a false positive due to the compiler taking a conservative approach, opting to warn at casting time rather than access time. However, in support of enabling -Warray-bounds globally (which has found many real bugs), arrange things for sk_buff so that the compiler can unambiguously see that there is no intention to access anything except prev/next. Introduce and cast to a separate struct sk_buff_list, which contains _only_ the first two fields, silencing the warnings: In file included from ./include/net/net_namespace.h:39, from ./include/linux/netdevice.h:37, from net/core/netpoll.c:17: net/core/netpoll.c: In function 'refill_skbs': ./include/linux/skbuff.h:2086:9: warning: array subscript 'struct sk_buff[0]' is partly outside array bounds of 'struct sk_buff_head[1]' [-Warray-bounds] 2086 | __skb_insert(newsk, next->prev, next, list); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ net/core/netpoll.c:49:28: note: while referencing 'skb_pool' 49 | static struct sk_buff_head skb_pool; | ^~~~~~~~ This change results in no executable instruction differences. Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20211207062758.2324338-1-keescook@chromium.org Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index dd262bd8ddbe..6535294f6a48 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -292,9 +292,11 @@ struct tc_skb_ext { #endif struct sk_buff_head { - /* These two members must be first. */ - struct sk_buff *next; - struct sk_buff *prev; + /* These two members must be first to match sk_buff. */ + struct_group_tagged(sk_buff_list, list, + struct sk_buff *next; + struct sk_buff *prev; + ); __u32 qlen; spinlock_t lock; @@ -730,7 +732,7 @@ typedef unsigned char *sk_buff_data_t; struct sk_buff { union { struct { - /* These two members must be first. */ + /* These two members must be first to match sk_buff_head. */ struct sk_buff *next; struct sk_buff *prev; @@ -1976,8 +1978,8 @@ static inline void __skb_insert(struct sk_buff *newsk, */ WRITE_ONCE(newsk->next, next); WRITE_ONCE(newsk->prev, prev); - WRITE_ONCE(next->prev, newsk); - WRITE_ONCE(prev->next, newsk); + WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk); + WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk); WRITE_ONCE(list->qlen, list->qlen + 1); } @@ -2073,7 +2075,7 @@ static inline void __skb_queue_after(struct sk_buff_head *list, struct sk_buff *prev, struct sk_buff *newsk) { - __skb_insert(newsk, prev, prev->next, list); + __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list); } void skb_append(struct sk_buff *old, struct sk_buff *newsk, @@ -2083,7 +2085,7 @@ static inline void __skb_queue_before(struct sk_buff_head *list, struct sk_buff *next, struct sk_buff *newsk) { - __skb_insert(newsk, next->prev, next, list); + __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list); } /** -- cgit v1.2.3-59-g8ed1b From 9756f64c8f2d19c0029a5827bda8ac275302ec22 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 30 Nov 2021 12:44:11 +0100 Subject: kcsan: Avoid checking scoped accesses from nested contexts Avoid checking scoped accesses from nested contexts (such as nested interrupts or in scheduler code) which share the same kcsan_ctx. This is to avoid detecting false positive races of accesses in the same thread with currently scoped accesses: consider setting up a watchpoint for a non-scoped (normal) access that also "conflicts" with a current scoped access. In a nested interrupt (or in the scheduler), which shares the same kcsan_ctx, we cannot check scoped accesses set up in the parent context -- simply ignore them in this case. With the introduction of kcsan_ctx::disable_scoped, we can also clean up kcsan_check_scoped_accesses()'s recursion guard, and do not need to modify the list's prev pointer. Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/kcsan.h | 1 + kernel/kcsan/core.c | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kcsan.h b/include/linux/kcsan.h index fc266ecb2a4d..13cef3458fed 100644 --- a/include/linux/kcsan.h +++ b/include/linux/kcsan.h @@ -21,6 +21,7 @@ */ struct kcsan_ctx { int disable_count; /* disable counter */ + int disable_scoped; /* disable scoped access counter */ int atomic_next; /* number of following atomic ops */ /* diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index e34a1710b7bc..bd359f8ee63a 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -204,15 +204,17 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip); static noinline void kcsan_check_scoped_accesses(void) { struct kcsan_ctx *ctx = get_ctx(); - struct list_head *prev_save = ctx->scoped_accesses.prev; struct kcsan_scoped_access *scoped_access; - ctx->scoped_accesses.prev = NULL; /* Avoid recursion. */ + if (ctx->disable_scoped) + return; + + ctx->disable_scoped++; list_for_each_entry(scoped_access, &ctx->scoped_accesses, list) { check_access(scoped_access->ptr, scoped_access->size, scoped_access->type, scoped_access->ip); } - ctx->scoped_accesses.prev = prev_save; + ctx->disable_scoped--; } /* Rules for generic atomic accesses. Called from fast-path. */ @@ -465,6 +467,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned goto out; } + /* + * Avoid races of scoped accesses from nested interrupts (or scheduler). + * Assume setting up a watchpoint for a non-scoped (normal) access that + * also conflicts with a current scoped access. In a nested interrupt, + * which shares the context, it would check a conflicting scoped access. + * To avoid, disable scoped access checking. + */ + ctx->disable_scoped++; + /* * Save and restore the IRQ state trace touched by KCSAN, since KCSAN's * runtime is entered for every memory access, and potentially useful @@ -578,6 +589,7 @@ out_unlock: if (!kcsan_interrupt_watcher) local_irq_restore(irq_flags); kcsan_restore_irqtrace(current); + ctx->disable_scoped--; out: user_access_restore(ua_flags); } -- cgit v1.2.3-59-g8ed1b From 69562e4983d93e2791c0bf128b07462afbd7f4dc Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Thu, 5 Aug 2021 14:57:45 +0200 Subject: kcsan: Add core support for a subset of weak memory modeling Add support for modeling a subset of weak memory, which will enable detection of a subset of data races due to missing memory barriers. KCSAN's approach to detecting missing memory barriers is based on modeling access reordering, and enabled if `CONFIG_KCSAN_WEAK_MEMORY=y`, which depends on `CONFIG_KCSAN_STRICT=y`. The feature can be enabled or disabled at boot and runtime via the `kcsan.weak_memory` boot parameter. Each memory access for which a watchpoint is set up, is also selected for simulated reordering within the scope of its function (at most 1 in-flight access). We are limited to modeling the effects of "buffering" (delaying the access), since the runtime cannot "prefetch" accesses (therefore no acquire modeling). Once an access has been selected for reordering, it is checked along every other access until the end of the function scope. If an appropriate memory barrier is encountered, the access will no longer be considered for reordering. When the result of a memory operation should be ordered by a barrier, KCSAN can then detect data races where the conflict only occurs as a result of a missing barrier due to reordering accesses. Suggested-by: Dmitry Vyukov Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/kcsan-checks.h | 10 ++- include/linux/kcsan.h | 10 ++- include/linux/sched.h | 3 + kernel/kcsan/core.c | 202 +++++++++++++++++++++++++++++++++++++++---- lib/Kconfig.kcsan | 20 +++++ scripts/Makefile.kcsan | 9 +- 6 files changed, 235 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h index 5f5965246877..a1c6a89fde71 100644 --- a/include/linux/kcsan-checks.h +++ b/include/linux/kcsan-checks.h @@ -99,7 +99,15 @@ void kcsan_set_access_mask(unsigned long mask); /* Scoped access information. */ struct kcsan_scoped_access { - struct list_head list; + union { + struct list_head list; /* scoped_accesses list */ + /* + * Not an entry in scoped_accesses list; stack depth from where + * the access was initialized. + */ + int stack_depth; + }; + /* Access information. */ const volatile void *ptr; size_t size; diff --git a/include/linux/kcsan.h b/include/linux/kcsan.h index 13cef3458fed..c07c71f5ba4f 100644 --- a/include/linux/kcsan.h +++ b/include/linux/kcsan.h @@ -49,8 +49,16 @@ struct kcsan_ctx { */ unsigned long access_mask; - /* List of scoped accesses. */ + /* List of scoped accesses; likely to be empty. */ struct list_head scoped_accesses; + +#ifdef CONFIG_KCSAN_WEAK_MEMORY + /* + * Scoped access for modeling access reordering to detect missing memory + * barriers; only keep 1 to keep fast-path complexity manageable. + */ + struct kcsan_scoped_access reorder_access; +#endif }; /** diff --git a/include/linux/sched.h b/include/linux/sched.h index 78c351e35fec..0cd40b010487 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1339,6 +1339,9 @@ struct task_struct { #ifdef CONFIG_TRACE_IRQFLAGS struct irqtrace_events kcsan_save_irqtrace; #endif +#ifdef CONFIG_KCSAN_WEAK_MEMORY + int kcsan_stack_depth; +#endif #endif #if IS_ENABLED(CONFIG_KUNIT) diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index bd359f8ee63a..481f8a524089 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -40,6 +40,13 @@ module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644); module_param_named(skip_watch, kcsan_skip_watch, long, 0644); module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444); +#ifdef CONFIG_KCSAN_WEAK_MEMORY +static bool kcsan_weak_memory = true; +module_param_named(weak_memory, kcsan_weak_memory, bool, 0644); +#else +#define kcsan_weak_memory false +#endif + bool kcsan_enabled; /* Per-CPU kcsan_ctx for interrupts */ @@ -351,6 +358,67 @@ void kcsan_restore_irqtrace(struct task_struct *task) #endif } +static __always_inline int get_kcsan_stack_depth(void) +{ +#ifdef CONFIG_KCSAN_WEAK_MEMORY + return current->kcsan_stack_depth; +#else + BUILD_BUG(); + return 0; +#endif +} + +static __always_inline void add_kcsan_stack_depth(int val) +{ +#ifdef CONFIG_KCSAN_WEAK_MEMORY + current->kcsan_stack_depth += val; +#else + BUILD_BUG(); +#endif +} + +static __always_inline struct kcsan_scoped_access *get_reorder_access(struct kcsan_ctx *ctx) +{ +#ifdef CONFIG_KCSAN_WEAK_MEMORY + return ctx->disable_scoped ? NULL : &ctx->reorder_access; +#else + return NULL; +#endif +} + +static __always_inline bool +find_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, + int type, unsigned long ip) +{ + struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); + + if (!reorder_access) + return false; + + /* + * Note: If accesses are repeated while reorder_access is identical, + * never matches the new access, because !(type & KCSAN_ACCESS_SCOPED). + */ + return reorder_access->ptr == ptr && reorder_access->size == size && + reorder_access->type == type && reorder_access->ip == ip; +} + +static inline void +set_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, + int type, unsigned long ip) +{ + struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); + + if (!reorder_access || !kcsan_weak_memory) + return; + + reorder_access->ptr = ptr; + reorder_access->size = size; + reorder_access->type = type | KCSAN_ACCESS_SCOPED; + reorder_access->ip = ip; + reorder_access->stack_depth = get_kcsan_stack_depth(); +} + /* * Pull everything together: check_access() below contains the performance * critical operations; the fast-path (including check_access) functions should @@ -389,8 +457,10 @@ static noinline void kcsan_found_watchpoint(const volatile void *ptr, * The access_mask check relies on value-change comparison. To avoid * reporting a race where e.g. the writer set up the watchpoint, but the * reader has access_mask!=0, we have to ignore the found watchpoint. + * + * reorder_access is never created from an access with access_mask set. */ - if (ctx->access_mask) + if (ctx->access_mask && !find_reorder_access(ctx, ptr, size, type, ip)) return; /* @@ -440,11 +510,13 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0; atomic_long_t *watchpoint; u64 old, new, diff; - unsigned long access_mask; enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE; + bool interrupt_watcher = kcsan_interrupt_watcher; unsigned long ua_flags = user_access_save(); struct kcsan_ctx *ctx = get_ctx(); + unsigned long access_mask = ctx->access_mask; unsigned long irq_flags = 0; + bool is_reorder_access; /* * Always reset kcsan_skip counter in slow-path to avoid underflow; see @@ -467,6 +539,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned goto out; } + /* + * The local CPU cannot observe reordering of its own accesses, and + * therefore we need to take care of 2 cases to avoid false positives: + * + * 1. Races of the reordered access with interrupts. To avoid, if + * the current access is reorder_access, disable interrupts. + * 2. Avoid races of scoped accesses from nested interrupts (below). + */ + is_reorder_access = find_reorder_access(ctx, ptr, size, type, ip); + if (is_reorder_access) + interrupt_watcher = false; /* * Avoid races of scoped accesses from nested interrupts (or scheduler). * Assume setting up a watchpoint for a non-scoped (normal) access that @@ -482,7 +565,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * information is lost if dirtied by KCSAN. */ kcsan_save_irqtrace(current); - if (!kcsan_interrupt_watcher) + if (!interrupt_watcher) local_irq_save(irq_flags); watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); @@ -503,7 +586,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * Read the current value, to later check and infer a race if the data * was modified via a non-instrumented access, e.g. from a device. */ - old = read_instrumented_memory(ptr, size); + old = is_reorder_access ? 0 : read_instrumented_memory(ptr, size); /* * Delay this thread, to increase probability of observing a racy @@ -515,8 +598,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * Re-read value, and check if it is as expected; if not, we infer a * racy access. */ - access_mask = ctx->access_mask; - new = read_instrumented_memory(ptr, size); + if (!is_reorder_access) { + new = read_instrumented_memory(ptr, size); + } else { + /* + * Reordered accesses cannot be used for value change detection, + * because the memory location may no longer be accessible and + * could result in a fault. + */ + new = 0; + access_mask = 0; + } diff = old ^ new; if (access_mask) @@ -585,11 +677,20 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned */ remove_watchpoint(watchpoint); atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); + out_unlock: - if (!kcsan_interrupt_watcher) + if (!interrupt_watcher) local_irq_restore(irq_flags); kcsan_restore_irqtrace(current); ctx->disable_scoped--; + + /* + * Reordered accesses cannot be used for value change detection, + * therefore never consider for reordering if access_mask is set. + * ASSERT_EXCLUSIVE are not real accesses, ignore them as well. + */ + if (!access_mask && !is_assert) + set_reorder_access(ctx, ptr, size, type, ip); out: user_access_restore(ua_flags); } @@ -597,7 +698,6 @@ out: static __always_inline void check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) { - const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0; atomic_long_t *watchpoint; long encoded_watchpoint; @@ -608,12 +708,14 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) if (unlikely(size == 0)) return; +again: /* * Avoid user_access_save in fast-path: find_watchpoint is safe without * user_access_save, as the address that ptr points to is only used to * check if a watchpoint exists; ptr is never dereferenced. */ - watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write, + watchpoint = find_watchpoint((unsigned long)ptr, size, + !(type & KCSAN_ACCESS_WRITE), &encoded_watchpoint); /* * It is safe to check kcsan_is_enabled() after find_watchpoint in the @@ -627,9 +729,42 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) else { struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */ - if (unlikely(should_watch(ctx, ptr, size, type))) + if (unlikely(should_watch(ctx, ptr, size, type))) { kcsan_setup_watchpoint(ptr, size, type, ip); - else if (unlikely(ctx->scoped_accesses.prev)) + return; + } + + if (!(type & KCSAN_ACCESS_SCOPED)) { + struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); + + if (reorder_access) { + /* + * reorder_access check: simulates reordering of + * the access after subsequent operations. + */ + ptr = reorder_access->ptr; + type = reorder_access->type; + ip = reorder_access->ip; + /* + * Upon a nested interrupt, this context's + * reorder_access can be modified (shared ctx). + * We know that upon return, reorder_access is + * always invalidated by setting size to 0 via + * __tsan_func_exit(). Therefore we must read + * and check size after the other fields. + */ + barrier(); + size = READ_ONCE(reorder_access->size); + if (size) + goto again; + } + } + + /* + * Always checked last, right before returning from runtime; + * if reorder_access is valid, checked after it was checked. + */ + if (unlikely(ctx->scoped_accesses.prev)) kcsan_check_scoped_accesses(); } } @@ -916,19 +1051,56 @@ DEFINE_TSAN_VOLATILE_READ_WRITE(8); DEFINE_TSAN_VOLATILE_READ_WRITE(16); /* - * The below are not required by KCSAN, but can still be emitted by the - * compiler. + * Function entry and exit are used to determine the validty of reorder_access. + * Reordering of the access ends at the end of the function scope where the + * access happened. This is done for two reasons: + * + * 1. Artificially limits the scope where missing barriers are detected. + * This minimizes false positives due to uninstrumented functions that + * contain the required barriers but were missed. + * + * 2. Simplifies generating the stack trace of the access. */ void __tsan_func_entry(void *call_pc); -void __tsan_func_entry(void *call_pc) +noinline void __tsan_func_entry(void *call_pc) { + if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) + return; + + add_kcsan_stack_depth(1); } EXPORT_SYMBOL(__tsan_func_entry); + void __tsan_func_exit(void); -void __tsan_func_exit(void) +noinline void __tsan_func_exit(void) { + struct kcsan_scoped_access *reorder_access; + + if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) + return; + + reorder_access = get_reorder_access(get_ctx()); + if (!reorder_access) + goto out; + + if (get_kcsan_stack_depth() <= reorder_access->stack_depth) { + /* + * Access check to catch cases where write without a barrier + * (supposed release) was last access in function: because + * instrumentation is inserted before the real access, a data + * race due to the write giving up a c-s would only be caught if + * we do the conflicting access after. + */ + check_access(reorder_access->ptr, reorder_access->size, + reorder_access->type, reorder_access->ip); + reorder_access->size = 0; + reorder_access->stack_depth = INT_MIN; + } +out: + add_kcsan_stack_depth(-1); } EXPORT_SYMBOL(__tsan_func_exit); + void __tsan_init(void); void __tsan_init(void) { diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan index e0a93ffdef30..e4394ea8068b 100644 --- a/lib/Kconfig.kcsan +++ b/lib/Kconfig.kcsan @@ -191,6 +191,26 @@ config KCSAN_STRICT closely aligns with the rules defined by the Linux-kernel memory consistency model (LKMM). +config KCSAN_WEAK_MEMORY + bool "Enable weak memory modeling to detect missing memory barriers" + default y + depends on KCSAN_STRICT + # We can either let objtool nop __tsan_func_{entry,exit}() and builtin + # atomics instrumentation in .noinstr.text, or use a compiler that can + # implement __no_kcsan to really remove all instrumentation. + depends on STACK_VALIDATION || CC_IS_GCC + help + Enable support for modeling a subset of weak memory, which allows + detecting a subset of data races due to missing memory barriers. + + Depends on KCSAN_STRICT, because the options strenghtening certain + plain accesses by default (depending on !KCSAN_STRICT) reduce the + ability to detect any data races invoving reordered accesses, in + particular reordered writes. + + Weak memory modeling relies on additional instrumentation and may + affect performance. + config KCSAN_REPORT_VALUE_CHANGE_ONLY bool "Only report races where watcher observed a data value change" default y diff --git a/scripts/Makefile.kcsan b/scripts/Makefile.kcsan index 37cb504c77e1..4c7f0d282e42 100644 --- a/scripts/Makefile.kcsan +++ b/scripts/Makefile.kcsan @@ -9,7 +9,12 @@ endif # Keep most options here optional, to allow enabling more compilers if absence # of some options does not break KCSAN nor causes false positive reports. -export CFLAGS_KCSAN := -fsanitize=thread \ - $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \ +kcsan-cflags := -fsanitize=thread -fno-optimize-sibling-calls \ $(call cc-option,$(call cc-param,tsan-compound-read-before-write=1),$(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1))) \ $(call cc-param,tsan-distinguish-volatile=1) + +ifndef CONFIG_KCSAN_WEAK_MEMORY +kcsan-cflags += $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0)) +endif + +export CFLAGS_KCSAN := $(kcsan-cflags) -- cgit v1.2.3-59-g8ed1b From 0b8b0830ac1419d7250fde31ea78793a03f3db44 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 30 Nov 2021 12:44:13 +0100 Subject: kcsan: Add core memory barrier instrumentation functions Add the core memory barrier instrumentation functions. These invalidate the current in-flight reordered access based on the rules for the respective barrier types and in-flight access type. To obtain barrier instrumentation that can be disabled via __no_kcsan with appropriate compiler-support (and not just with objtool help), barrier instrumentation repurposes __atomic_signal_fence(), instead of inserting explicit calls. Crucially, __atomic_signal_fence() normally does not map to any real instructions, but is still intercepted by fsanitize=thread. As a result, like any other instrumentation done by the compiler, barrier instrumentation can be disabled with __no_kcsan. Unfortunately Clang and GCC currently differ in their __no_kcsan aka __no_sanitize_thread behaviour with respect to builtin atomics (and __tsan_func_{entry,exit}) instrumentation. This is already reflected in Kconfig.kcsan's dependencies for KCSAN_WEAK_MEMORY. A later change will introduce support for newer versions of Clang that can implement __no_kcsan to also remove the additional instrumentation introduced by KCSAN_WEAK_MEMORY. Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/kcsan-checks.h | 71 ++++++++++++++++++++++++++++++++++++++++++-- kernel/kcsan/core.c | 68 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h index a1c6a89fde71..9d2c869167f2 100644 --- a/include/linux/kcsan-checks.h +++ b/include/linux/kcsan-checks.h @@ -36,6 +36,36 @@ */ void __kcsan_check_access(const volatile void *ptr, size_t size, int type); +/* + * See definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c. + * Note: The mappings are arbitrary, and do not reflect any real mappings of C11 + * memory orders to the LKMM memory orders and vice-versa! + */ +#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_mb __ATOMIC_SEQ_CST +#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_wmb __ATOMIC_ACQ_REL +#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb __ATOMIC_ACQUIRE +#define __KCSAN_BARRIER_TO_SIGNAL_FENCE_release __ATOMIC_RELEASE + +/** + * __kcsan_mb - full memory barrier instrumentation + */ +void __kcsan_mb(void); + +/** + * __kcsan_wmb - write memory barrier instrumentation + */ +void __kcsan_wmb(void); + +/** + * __kcsan_rmb - read memory barrier instrumentation + */ +void __kcsan_rmb(void); + +/** + * __kcsan_release - release barrier instrumentation + */ +void __kcsan_release(void); + /** * kcsan_disable_current - disable KCSAN for the current context * @@ -159,6 +189,10 @@ void kcsan_end_scoped_access(struct kcsan_scoped_access *sa); static inline void __kcsan_check_access(const volatile void *ptr, size_t size, int type) { } +static inline void __kcsan_mb(void) { } +static inline void __kcsan_wmb(void) { } +static inline void __kcsan_rmb(void) { } +static inline void __kcsan_release(void) { } static inline void kcsan_disable_current(void) { } static inline void kcsan_enable_current(void) { } static inline void kcsan_enable_current_nowarn(void) { } @@ -191,12 +225,45 @@ static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { } */ #define __kcsan_disable_current kcsan_disable_current #define __kcsan_enable_current kcsan_enable_current_nowarn -#else +#else /* __SANITIZE_THREAD__ */ static inline void kcsan_check_access(const volatile void *ptr, size_t size, int type) { } static inline void __kcsan_enable_current(void) { } static inline void __kcsan_disable_current(void) { } -#endif +#endif /* __SANITIZE_THREAD__ */ + +#if defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__SANITIZE_THREAD__) +/* + * Normal barrier instrumentation is not done via explicit calls, but by mapping + * to a repurposed __atomic_signal_fence(), which normally does not generate any + * real instructions, but is still intercepted by fsanitize=thread. This means, + * like any other compile-time instrumentation, barrier instrumentation can be + * disabled with the __no_kcsan function attribute. + * + * Also see definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c. + */ +#define __KCSAN_BARRIER_TO_SIGNAL_FENCE(name) \ + static __always_inline void kcsan_##name(void) \ + { \ + barrier(); \ + __atomic_signal_fence(__KCSAN_BARRIER_TO_SIGNAL_FENCE_##name); \ + barrier(); \ + } +__KCSAN_BARRIER_TO_SIGNAL_FENCE(mb) +__KCSAN_BARRIER_TO_SIGNAL_FENCE(wmb) +__KCSAN_BARRIER_TO_SIGNAL_FENCE(rmb) +__KCSAN_BARRIER_TO_SIGNAL_FENCE(release) +#elif defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__KCSAN_INSTRUMENT_BARRIERS__) +#define kcsan_mb __kcsan_mb +#define kcsan_wmb __kcsan_wmb +#define kcsan_rmb __kcsan_rmb +#define kcsan_release __kcsan_release +#else /* CONFIG_KCSAN_WEAK_MEMORY && ... */ +static inline void kcsan_mb(void) { } +static inline void kcsan_wmb(void) { } +static inline void kcsan_rmb(void) { } +static inline void kcsan_release(void) { } +#endif /* CONFIG_KCSAN_WEAK_MEMORY && ... */ /** * __kcsan_check_read - check regular read access for races diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 481f8a524089..916060913966 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -939,6 +939,22 @@ void __kcsan_check_access(const volatile void *ptr, size_t size, int type) } EXPORT_SYMBOL(__kcsan_check_access); +#define DEFINE_MEMORY_BARRIER(name, order_before_cond) \ + void __kcsan_##name(void) \ + { \ + struct kcsan_scoped_access *sa = get_reorder_access(get_ctx()); \ + if (!sa) \ + return; \ + if (order_before_cond) \ + sa->size = 0; \ + } \ + EXPORT_SYMBOL(__kcsan_##name) + +DEFINE_MEMORY_BARRIER(mb, true); +DEFINE_MEMORY_BARRIER(wmb, sa->type & (KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND)); +DEFINE_MEMORY_BARRIER(rmb, !(sa->type & KCSAN_ACCESS_WRITE) || (sa->type & KCSAN_ACCESS_COMPOUND)); +DEFINE_MEMORY_BARRIER(release, true); + /* * KCSAN uses the same instrumentation that is emitted by supported compilers * for ThreadSanitizer (TSAN). @@ -1123,10 +1139,19 @@ EXPORT_SYMBOL(__tsan_init); * functions, whose job is to also execute the operation itself. */ +static __always_inline void kcsan_atomic_builtin_memorder(int memorder) +{ + if (memorder == __ATOMIC_RELEASE || + memorder == __ATOMIC_SEQ_CST || + memorder == __ATOMIC_ACQ_REL) + __kcsan_release(); +} + #define DEFINE_TSAN_ATOMIC_LOAD_STORE(bits) \ u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder); \ u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder) \ { \ + kcsan_atomic_builtin_memorder(memorder); \ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \ check_access(ptr, bits / BITS_PER_BYTE, KCSAN_ACCESS_ATOMIC, _RET_IP_); \ } \ @@ -1136,6 +1161,7 @@ EXPORT_SYMBOL(__tsan_init); void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder); \ void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder) \ { \ + kcsan_atomic_builtin_memorder(memorder); \ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \ check_access(ptr, bits / BITS_PER_BYTE, \ KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC, _RET_IP_); \ @@ -1148,6 +1174,7 @@ EXPORT_SYMBOL(__tsan_init); u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder); \ u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder) \ { \ + kcsan_atomic_builtin_memorder(memorder); \ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \ check_access(ptr, bits / BITS_PER_BYTE, \ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \ @@ -1180,6 +1207,7 @@ EXPORT_SYMBOL(__tsan_init); int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \ u##bits val, int mo, int fail_mo) \ { \ + kcsan_atomic_builtin_memorder(mo); \ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \ check_access(ptr, bits / BITS_PER_BYTE, \ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \ @@ -1195,6 +1223,7 @@ EXPORT_SYMBOL(__tsan_init); u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \ int mo, int fail_mo) \ { \ + kcsan_atomic_builtin_memorder(mo); \ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \ check_access(ptr, bits / BITS_PER_BYTE, \ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \ @@ -1226,10 +1255,47 @@ DEFINE_TSAN_ATOMIC_OPS(64); void __tsan_atomic_thread_fence(int memorder); void __tsan_atomic_thread_fence(int memorder) { + kcsan_atomic_builtin_memorder(memorder); __atomic_thread_fence(memorder); } EXPORT_SYMBOL(__tsan_atomic_thread_fence); +/* + * In instrumented files, we emit instrumentation for barriers by mapping the + * kernel barriers to an __atomic_signal_fence(), which is interpreted specially + * and otherwise has no relation to a real __atomic_signal_fence(). No known + * kernel code uses __atomic_signal_fence(). + * + * Since fsanitize=thread instrumentation handles __atomic_signal_fence(), which + * are turned into calls to __tsan_atomic_signal_fence(), such instrumentation + * can be disabled via the __no_kcsan function attribute (vs. an explicit call + * which could not). When __no_kcsan is requested, __atomic_signal_fence() + * generates no code. + * + * Note: The result of using __atomic_signal_fence() with KCSAN enabled is + * potentially limiting the compiler's ability to reorder operations; however, + * if barriers were instrumented with explicit calls (without LTO), the compiler + * couldn't optimize much anyway. The result of a hypothetical architecture + * using __atomic_signal_fence() in normal code would be KCSAN false negatives. + */ void __tsan_atomic_signal_fence(int memorder); -void __tsan_atomic_signal_fence(int memorder) { } +noinline void __tsan_atomic_signal_fence(int memorder) +{ + switch (memorder) { + case __KCSAN_BARRIER_TO_SIGNAL_FENCE_mb: + __kcsan_mb(); + break; + case __KCSAN_BARRIER_TO_SIGNAL_FENCE_wmb: + __kcsan_wmb(); + break; + case __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb: + __kcsan_rmb(); + break; + case __KCSAN_BARRIER_TO_SIGNAL_FENCE_release: + __kcsan_release(); + break; + default: + break; + } +} EXPORT_SYMBOL(__tsan_atomic_signal_fence); -- cgit v1.2.3-59-g8ed1b From f948666de517cf8ebef7cb2c9b2d669dec4bfe2e Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 30 Nov 2021 12:44:22 +0100 Subject: locking/barriers, kcsan: Add instrumentation for barriers Adds the required KCSAN instrumentation for barriers if CONFIG_SMP. KCSAN supports modeling the effects of: smp_mb() smp_rmb() smp_wmb() smp_store_release() Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/asm-generic/barrier.h | 29 +++++++++++++++-------------- include/linux/spinlock.h | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 640f09479bdf..27a9c9edfef6 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -14,6 +14,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #ifndef nop @@ -62,15 +63,15 @@ #ifdef CONFIG_SMP #ifndef smp_mb -#define smp_mb() __smp_mb() +#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0) #endif #ifndef smp_rmb -#define smp_rmb() __smp_rmb() +#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) #endif #ifndef smp_wmb -#define smp_wmb() __smp_wmb() +#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) #endif #else /* !CONFIG_SMP */ @@ -123,19 +124,19 @@ do { \ #ifdef CONFIG_SMP #ifndef smp_store_mb -#define smp_store_mb(var, value) __smp_store_mb(var, value) +#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) #endif #ifndef smp_mb__before_atomic -#define smp_mb__before_atomic() __smp_mb__before_atomic() +#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) #endif #ifndef smp_mb__after_atomic -#define smp_mb__after_atomic() __smp_mb__after_atomic() +#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) #endif #ifndef smp_store_release -#define smp_store_release(p, v) __smp_store_release(p, v) +#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) #endif #ifndef smp_load_acquire @@ -178,13 +179,13 @@ do { \ #endif /* CONFIG_SMP */ /* Barriers for virtual machine guests when talking to an SMP host */ -#define virt_mb() __smp_mb() -#define virt_rmb() __smp_rmb() -#define virt_wmb() __smp_wmb() -#define virt_store_mb(var, value) __smp_store_mb(var, value) -#define virt_mb__before_atomic() __smp_mb__before_atomic() -#define virt_mb__after_atomic() __smp_mb__after_atomic() -#define virt_store_release(p, v) __smp_store_release(p, v) +#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0) +#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) +#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) +#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) +#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) +#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) +#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0) #define virt_load_acquire(p) __smp_load_acquire(p) /** diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index b4e5ca23f840..5c0c5174155d 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -171,7 +171,7 @@ do { \ * Architectures that can implement ACQUIRE better need to take care. */ #ifndef smp_mb__after_spinlock -#define smp_mb__after_spinlock() do { } while (0) +#define smp_mb__after_spinlock() kcsan_mb() #endif #ifdef CONFIG_DEBUG_SPINLOCK -- cgit v1.2.3-59-g8ed1b From e87c4f6642f49627c3430cb3ee78c73fb51b48e4 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 30 Nov 2021 12:44:24 +0100 Subject: locking/atomics, kcsan: Add instrumentation for barriers Adds the required KCSAN instrumentation for barriers of atomics. Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/atomic/atomic-instrumented.h | 135 ++++++++++++++++++++++++++++- scripts/atomic/gen-atomic-instrumented.sh | 41 +++++++-- 2 files changed, 166 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index a0f654370da3..5d69b143c28e 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -45,6 +45,7 @@ atomic_set(atomic_t *v, int i) static __always_inline void atomic_set_release(atomic_t *v, int i) { + kcsan_release(); instrument_atomic_write(v, sizeof(*v)); arch_atomic_set_release(v, i); } @@ -59,6 +60,7 @@ atomic_add(int i, atomic_t *v) static __always_inline int atomic_add_return(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return(i, v); } @@ -73,6 +75,7 @@ atomic_add_return_acquire(int i, atomic_t *v) static __always_inline int atomic_add_return_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return_release(i, v); } @@ -87,6 +90,7 @@ atomic_add_return_relaxed(int i, atomic_t *v) static __always_inline int atomic_fetch_add(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add(i, v); } @@ -101,6 +105,7 @@ atomic_fetch_add_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_add_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_release(i, v); } @@ -122,6 +127,7 @@ atomic_sub(int i, atomic_t *v) static __always_inline int atomic_sub_return(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return(i, v); } @@ -136,6 +142,7 @@ atomic_sub_return_acquire(int i, atomic_t *v) static __always_inline int atomic_sub_return_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return_release(i, v); } @@ -150,6 +157,7 @@ atomic_sub_return_relaxed(int i, atomic_t *v) static __always_inline int atomic_fetch_sub(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub(i, v); } @@ -164,6 +172,7 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_sub_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub_release(i, v); } @@ -185,6 +194,7 @@ atomic_inc(atomic_t *v) static __always_inline int atomic_inc_return(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return(v); } @@ -199,6 +209,7 @@ atomic_inc_return_acquire(atomic_t *v) static __always_inline int atomic_inc_return_release(atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return_release(v); } @@ -213,6 +224,7 @@ atomic_inc_return_relaxed(atomic_t *v) static __always_inline int atomic_fetch_inc(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc(v); } @@ -227,6 +239,7 @@ atomic_fetch_inc_acquire(atomic_t *v) static __always_inline int atomic_fetch_inc_release(atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc_release(v); } @@ -248,6 +261,7 @@ atomic_dec(atomic_t *v) static __always_inline int atomic_dec_return(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return(v); } @@ -262,6 +276,7 @@ atomic_dec_return_acquire(atomic_t *v) static __always_inline int atomic_dec_return_release(atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return_release(v); } @@ -276,6 +291,7 @@ atomic_dec_return_relaxed(atomic_t *v) static __always_inline int atomic_fetch_dec(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec(v); } @@ -290,6 +306,7 @@ atomic_fetch_dec_acquire(atomic_t *v) static __always_inline int atomic_fetch_dec_release(atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec_release(v); } @@ -311,6 +328,7 @@ atomic_and(int i, atomic_t *v) static __always_inline int atomic_fetch_and(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and(i, v); } @@ -325,6 +343,7 @@ atomic_fetch_and_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_and_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and_release(i, v); } @@ -346,6 +365,7 @@ atomic_andnot(int i, atomic_t *v) static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot(i, v); } @@ -360,6 +380,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_release(i, v); } @@ -381,6 +402,7 @@ atomic_or(int i, atomic_t *v) static __always_inline int atomic_fetch_or(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or(i, v); } @@ -395,6 +417,7 @@ atomic_fetch_or_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_or_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or_release(i, v); } @@ -416,6 +439,7 @@ atomic_xor(int i, atomic_t *v) static __always_inline int atomic_fetch_xor(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor(i, v); } @@ -430,6 +454,7 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) static __always_inline int atomic_fetch_xor_release(int i, atomic_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor_release(i, v); } @@ -444,6 +469,7 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v) static __always_inline int atomic_xchg(atomic_t *v, int i) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg(v, i); } @@ -458,6 +484,7 @@ atomic_xchg_acquire(atomic_t *v, int i) static __always_inline int atomic_xchg_release(atomic_t *v, int i) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg_release(v, i); } @@ -472,6 +499,7 @@ atomic_xchg_relaxed(atomic_t *v, int i) static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg(v, old, new); } @@ -486,6 +514,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) static __always_inline int atomic_cmpxchg_release(atomic_t *v, int old, int new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg_release(v, old, new); } @@ -500,6 +529,7 @@ atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg(v, old, new); @@ -516,6 +546,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_release(v, old, new); @@ -532,6 +563,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_and_test(i, v); } @@ -539,6 +571,7 @@ atomic_sub_and_test(int i, atomic_t *v) static __always_inline bool atomic_dec_and_test(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_and_test(v); } @@ -546,6 +579,7 @@ atomic_dec_and_test(atomic_t *v) static __always_inline bool atomic_inc_and_test(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_and_test(v); } @@ -553,6 +587,7 @@ atomic_inc_and_test(atomic_t *v) static __always_inline bool atomic_add_negative(int i, atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_negative(i, v); } @@ -560,6 +595,7 @@ atomic_add_negative(int i, atomic_t *v) static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_unless(v, a, u); } @@ -567,6 +603,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) static __always_inline bool atomic_add_unless(atomic_t *v, int a, int u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_unless(v, a, u); } @@ -574,6 +611,7 @@ atomic_add_unless(atomic_t *v, int a, int u) static __always_inline bool atomic_inc_not_zero(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_not_zero(v); } @@ -581,6 +619,7 @@ atomic_inc_not_zero(atomic_t *v) static __always_inline bool atomic_inc_unless_negative(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_unless_negative(v); } @@ -588,6 +627,7 @@ atomic_inc_unless_negative(atomic_t *v) static __always_inline bool atomic_dec_unless_positive(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_unless_positive(v); } @@ -595,6 +635,7 @@ atomic_dec_unless_positive(atomic_t *v) static __always_inline int atomic_dec_if_positive(atomic_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_if_positive(v); } @@ -623,6 +664,7 @@ atomic64_set(atomic64_t *v, s64 i) static __always_inline void atomic64_set_release(atomic64_t *v, s64 i) { + kcsan_release(); instrument_atomic_write(v, sizeof(*v)); arch_atomic64_set_release(v, i); } @@ -637,6 +679,7 @@ atomic64_add(s64 i, atomic64_t *v) static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return(i, v); } @@ -651,6 +694,7 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_add_return_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return_release(i, v); } @@ -665,6 +709,7 @@ atomic64_add_return_relaxed(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add(i, v); } @@ -679,6 +724,7 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_add_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_release(i, v); } @@ -700,6 +746,7 @@ atomic64_sub(s64 i, atomic64_t *v) static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return(i, v); } @@ -714,6 +761,7 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return_release(i, v); } @@ -728,6 +776,7 @@ atomic64_sub_return_relaxed(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub(i, v); } @@ -742,6 +791,7 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_sub_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_release(i, v); } @@ -763,6 +813,7 @@ atomic64_inc(atomic64_t *v) static __always_inline s64 atomic64_inc_return(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return(v); } @@ -777,6 +828,7 @@ atomic64_inc_return_acquire(atomic64_t *v) static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return_release(v); } @@ -791,6 +843,7 @@ atomic64_inc_return_relaxed(atomic64_t *v) static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc(v); } @@ -805,6 +858,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_release(v); } @@ -826,6 +880,7 @@ atomic64_dec(atomic64_t *v) static __always_inline s64 atomic64_dec_return(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return(v); } @@ -840,6 +895,7 @@ atomic64_dec_return_acquire(atomic64_t *v) static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return_release(v); } @@ -854,6 +910,7 @@ atomic64_dec_return_relaxed(atomic64_t *v) static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec(v); } @@ -868,6 +925,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_release(v); } @@ -889,6 +947,7 @@ atomic64_and(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and(i, v); } @@ -903,6 +962,7 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_and_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and_release(i, v); } @@ -924,6 +984,7 @@ atomic64_andnot(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot(i, v); } @@ -938,6 +999,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_release(i, v); } @@ -959,6 +1021,7 @@ atomic64_or(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or(i, v); } @@ -973,6 +1036,7 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_or_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or_release(i, v); } @@ -994,6 +1058,7 @@ atomic64_xor(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor(i, v); } @@ -1008,6 +1073,7 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_xor_release(s64 i, atomic64_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_release(i, v); } @@ -1022,6 +1088,7 @@ atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg(v, i); } @@ -1036,6 +1103,7 @@ atomic64_xchg_acquire(atomic64_t *v, s64 i) static __always_inline s64 atomic64_xchg_release(atomic64_t *v, s64 i) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg_release(v, i); } @@ -1050,6 +1118,7 @@ atomic64_xchg_relaxed(atomic64_t *v, s64 i) static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg(v, old, new); } @@ -1064,6 +1133,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) static __always_inline s64 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_release(v, old, new); } @@ -1078,6 +1148,7 @@ atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg(v, old, new); @@ -1094,6 +1165,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_release(v, old, new); @@ -1110,6 +1182,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_and_test(i, v); } @@ -1117,6 +1190,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) static __always_inline bool atomic64_dec_and_test(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_and_test(v); } @@ -1124,6 +1198,7 @@ atomic64_dec_and_test(atomic64_t *v) static __always_inline bool atomic64_inc_and_test(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_and_test(v); } @@ -1131,6 +1206,7 @@ atomic64_inc_and_test(atomic64_t *v) static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_negative(i, v); } @@ -1138,6 +1214,7 @@ atomic64_add_negative(s64 i, atomic64_t *v) static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_unless(v, a, u); } @@ -1145,6 +1222,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_unless(v, a, u); } @@ -1152,6 +1230,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_not_zero(v); } @@ -1159,6 +1238,7 @@ atomic64_inc_not_zero(atomic64_t *v) static __always_inline bool atomic64_inc_unless_negative(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_unless_negative(v); } @@ -1166,6 +1246,7 @@ atomic64_inc_unless_negative(atomic64_t *v) static __always_inline bool atomic64_dec_unless_positive(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_unless_positive(v); } @@ -1173,6 +1254,7 @@ atomic64_dec_unless_positive(atomic64_t *v) static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_if_positive(v); } @@ -1201,6 +1283,7 @@ atomic_long_set(atomic_long_t *v, long i) static __always_inline void atomic_long_set_release(atomic_long_t *v, long i) { + kcsan_release(); instrument_atomic_write(v, sizeof(*v)); arch_atomic_long_set_release(v, i); } @@ -1215,6 +1298,7 @@ atomic_long_add(long i, atomic_long_t *v) static __always_inline long atomic_long_add_return(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_add_return(i, v); } @@ -1229,6 +1313,7 @@ atomic_long_add_return_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_add_return_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_add_return_release(i, v); } @@ -1243,6 +1328,7 @@ atomic_long_add_return_relaxed(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_add(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_add(i, v); } @@ -1257,6 +1343,7 @@ atomic_long_fetch_add_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_add_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_add_release(i, v); } @@ -1278,6 +1365,7 @@ atomic_long_sub(long i, atomic_long_t *v) static __always_inline long atomic_long_sub_return(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_sub_return(i, v); } @@ -1292,6 +1380,7 @@ atomic_long_sub_return_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_sub_return_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_sub_return_release(i, v); } @@ -1306,6 +1395,7 @@ atomic_long_sub_return_relaxed(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_sub(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_sub(i, v); } @@ -1320,6 +1410,7 @@ atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_sub_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_sub_release(i, v); } @@ -1341,6 +1432,7 @@ atomic_long_inc(atomic_long_t *v) static __always_inline long atomic_long_inc_return(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_inc_return(v); } @@ -1355,6 +1447,7 @@ atomic_long_inc_return_acquire(atomic_long_t *v) static __always_inline long atomic_long_inc_return_release(atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_inc_return_release(v); } @@ -1369,6 +1462,7 @@ atomic_long_inc_return_relaxed(atomic_long_t *v) static __always_inline long atomic_long_fetch_inc(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_inc(v); } @@ -1383,6 +1477,7 @@ atomic_long_fetch_inc_acquire(atomic_long_t *v) static __always_inline long atomic_long_fetch_inc_release(atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_inc_release(v); } @@ -1404,6 +1499,7 @@ atomic_long_dec(atomic_long_t *v) static __always_inline long atomic_long_dec_return(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_dec_return(v); } @@ -1418,6 +1514,7 @@ atomic_long_dec_return_acquire(atomic_long_t *v) static __always_inline long atomic_long_dec_return_release(atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_dec_return_release(v); } @@ -1432,6 +1529,7 @@ atomic_long_dec_return_relaxed(atomic_long_t *v) static __always_inline long atomic_long_fetch_dec(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_dec(v); } @@ -1446,6 +1544,7 @@ atomic_long_fetch_dec_acquire(atomic_long_t *v) static __always_inline long atomic_long_fetch_dec_release(atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_dec_release(v); } @@ -1467,6 +1566,7 @@ atomic_long_and(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_and(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_and(i, v); } @@ -1481,6 +1581,7 @@ atomic_long_fetch_and_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_and_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_and_release(i, v); } @@ -1502,6 +1603,7 @@ atomic_long_andnot(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_andnot(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_andnot(i, v); } @@ -1516,6 +1618,7 @@ atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_andnot_release(i, v); } @@ -1537,6 +1640,7 @@ atomic_long_or(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_or(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_or(i, v); } @@ -1551,6 +1655,7 @@ atomic_long_fetch_or_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_or_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_or_release(i, v); } @@ -1572,6 +1677,7 @@ atomic_long_xor(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_xor(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_xor(i, v); } @@ -1586,6 +1692,7 @@ atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_xor_release(long i, atomic_long_t *v) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_xor_release(i, v); } @@ -1600,6 +1707,7 @@ atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) static __always_inline long atomic_long_xchg(atomic_long_t *v, long i) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_xchg(v, i); } @@ -1614,6 +1722,7 @@ atomic_long_xchg_acquire(atomic_long_t *v, long i) static __always_inline long atomic_long_xchg_release(atomic_long_t *v, long i) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_xchg_release(v, i); } @@ -1628,6 +1737,7 @@ atomic_long_xchg_relaxed(atomic_long_t *v, long i) static __always_inline long atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_cmpxchg(v, old, new); } @@ -1642,6 +1752,7 @@ atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) static __always_inline long atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_cmpxchg_release(v, old, new); } @@ -1656,6 +1767,7 @@ atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) static __always_inline bool atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_long_try_cmpxchg(v, old, new); @@ -1672,6 +1784,7 @@ atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) static __always_inline bool atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { + kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_long_try_cmpxchg_release(v, old, new); @@ -1688,6 +1801,7 @@ atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) static __always_inline bool atomic_long_sub_and_test(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_sub_and_test(i, v); } @@ -1695,6 +1809,7 @@ atomic_long_sub_and_test(long i, atomic_long_t *v) static __always_inline bool atomic_long_dec_and_test(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_dec_and_test(v); } @@ -1702,6 +1817,7 @@ atomic_long_dec_and_test(atomic_long_t *v) static __always_inline bool atomic_long_inc_and_test(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_inc_and_test(v); } @@ -1709,6 +1825,7 @@ atomic_long_inc_and_test(atomic_long_t *v) static __always_inline bool atomic_long_add_negative(long i, atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_add_negative(i, v); } @@ -1716,6 +1833,7 @@ atomic_long_add_negative(long i, atomic_long_t *v) static __always_inline long atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_fetch_add_unless(v, a, u); } @@ -1723,6 +1841,7 @@ atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) static __always_inline bool atomic_long_add_unless(atomic_long_t *v, long a, long u) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_add_unless(v, a, u); } @@ -1730,6 +1849,7 @@ atomic_long_add_unless(atomic_long_t *v, long a, long u) static __always_inline bool atomic_long_inc_not_zero(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_inc_not_zero(v); } @@ -1737,6 +1857,7 @@ atomic_long_inc_not_zero(atomic_long_t *v) static __always_inline bool atomic_long_inc_unless_negative(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_inc_unless_negative(v); } @@ -1744,6 +1865,7 @@ atomic_long_inc_unless_negative(atomic_long_t *v) static __always_inline bool atomic_long_dec_unless_positive(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_dec_unless_positive(v); } @@ -1751,6 +1873,7 @@ atomic_long_dec_unless_positive(atomic_long_t *v) static __always_inline long atomic_long_dec_if_positive(atomic_long_t *v) { + kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_long_dec_if_positive(v); } @@ -1758,6 +1881,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define xchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg(__ai_ptr, __VA_ARGS__); \ }) @@ -1772,6 +1896,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define xchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_release(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_release(__ai_ptr, __VA_ARGS__); \ }) @@ -1786,6 +1911,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) @@ -1800,6 +1926,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_release(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ }) @@ -1814,6 +1941,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg64(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) @@ -1828,6 +1956,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg64_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_release(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) @@ -1843,6 +1972,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ @@ -1861,6 +1991,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ + kcsan_release(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ @@ -1892,6 +2023,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define sync_cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) @@ -1899,6 +2031,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg_double(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ }) @@ -1912,4 +2045,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) }) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 2a9553f0a9d5619f19151092df5cabbbf16ce835 +// 87c974b93032afd42143613434d1a7788fa598f9 diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index 035ceb4ee85c..68f902731d01 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -34,6 +34,14 @@ gen_param_check() gen_params_checks() { local meta="$1"; shift + local order="$1"; shift + + if [ "${order}" = "_release" ]; then + printf "\tkcsan_release();\n" + elif [ -z "${order}" ] && ! meta_in "$meta" "slv"; then + # RMW with return value is fully ordered + printf "\tkcsan_mb();\n" + fi while [ "$#" -gt 0 ]; do gen_param_check "$meta" "$1" @@ -56,7 +64,7 @@ gen_proto_order_variant() local ret="$(gen_ret_type "${meta}" "${int}")" local params="$(gen_params "${int}" "${atomic}" "$@")" - local checks="$(gen_params_checks "${meta}" "$@")" + local checks="$(gen_params_checks "${meta}" "${order}" "$@")" local args="$(gen_args "$@")" local retstmt="$(gen_ret_stmt "${meta}")" @@ -75,29 +83,44 @@ EOF gen_xchg() { local xchg="$1"; shift + local order="$1"; shift local mult="$1"; shift + kcsan_barrier="" + if [ "${xchg%_local}" = "${xchg}" ]; then + case "$order" in + _release) kcsan_barrier="kcsan_release()" ;; + "") kcsan_barrier="kcsan_mb()" ;; + esac + fi + if [ "${xchg%${xchg#try_cmpxchg}}" = "try_cmpxchg" ] ; then cat < Date: Tue, 30 Nov 2021 12:44:32 +0100 Subject: compiler_attributes.h: Add __disable_sanitizer_instrumentation The new attribute maps to __attribute__((disable_sanitizer_instrumentation)), which will be supported by Clang >= 14.0. Future support in GCC is also possible. This attribute disables compiler instrumentation for kernel sanitizer tools, making it easier to implement noinstr. It is different from the existing __no_sanitize* attributes, which may still allow certain types of instrumentation to prevent false positives. Signed-off-by: Alexander Potapenko Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/compiler_attributes.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index b9121afd8733..37e260020221 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -308,6 +308,24 @@ # define __compiletime_warning(msg) #endif +/* + * Optional: only supported since clang >= 14.0 + * + * clang: https://clang.llvm.org/docs/AttributeReference.html#disable-sanitizer-instrumentation + * + * disable_sanitizer_instrumentation is not always similar to + * no_sanitize(()): the latter may still let specific sanitizers + * insert code into functions to prevent false positives. Unlike that, + * disable_sanitizer_instrumentation prevents all kinds of instrumentation to + * functions with the attribute. + */ +#if __has_attribute(disable_sanitizer_instrumentation) +# define __disable_sanitizer_instrumentation \ + __attribute__((disable_sanitizer_instrumentation)) +#else +# define __disable_sanitizer_instrumentation +#endif + /* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-weak-function-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-weak-variable-attribute -- cgit v1.2.3-59-g8ed1b From bd3d5bd1a0ad386475ea7a3de8a91e7d8a600536 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 30 Nov 2021 12:44:33 +0100 Subject: kcsan: Support WEAK_MEMORY with Clang where no objtool support exists Clang and GCC behave a little differently when it comes to the __no_sanitize_thread attribute, which has valid reasons, and depending on context either one could be right. Traditionally, user space ThreadSanitizer [1] still expects instrumented builtin atomics (to avoid false positives) and __tsan_func_{entry,exit} (to generate meaningful stack traces), even if the function has the attribute no_sanitize("thread"). [1] https://clang.llvm.org/docs/ThreadSanitizer.html#attribute-no-sanitize-thread GCC doesn't follow the same policy (for better or worse), and removes all kinds of instrumentation if no_sanitize is added. Arguably, since this may be a problem for user space ThreadSanitizer, we expect this may change in future. Since KCSAN != ThreadSanitizer, the likelihood of false positives even without barrier instrumentation everywhere, is much lower by design. At least for Clang, however, to fully remove all sanitizer instrumentation, we must add the disable_sanitizer_instrumentation attribute, which is available since Clang 14.0. Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/compiler_types.h | 13 ++++++++++++- lib/Kconfig.kcsan | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 1d32f4c03c9e..3c1795fdb568 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -198,9 +198,20 @@ struct ftrace_likely_data { # define __no_kasan_or_inline __always_inline #endif -#define __no_kcsan __no_sanitize_thread #ifdef __SANITIZE_THREAD__ +/* + * Clang still emits instrumentation for __tsan_func_{entry,exit}() and builtin + * atomics even with __no_sanitize_thread (to avoid false positives in userspace + * ThreadSanitizer). The kernel's requirements are stricter and we really do not + * want any instrumentation with __no_kcsan. + * + * Therefore we add __disable_sanitizer_instrumentation where available to + * disable all instrumentation. See Kconfig.kcsan where this is mandatory. + */ +# define __no_kcsan __no_sanitize_thread __disable_sanitizer_instrumentation # define __no_sanitize_or_inline __no_kcsan notrace __maybe_unused +#else +# define __no_kcsan #endif #ifndef __no_sanitize_or_inline diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan index e4394ea8068b..63b70b8c5551 100644 --- a/lib/Kconfig.kcsan +++ b/lib/Kconfig.kcsan @@ -198,7 +198,7 @@ config KCSAN_WEAK_MEMORY # We can either let objtool nop __tsan_func_{entry,exit}() and builtin # atomics instrumentation in .noinstr.text, or use a compiler that can # implement __no_kcsan to really remove all instrumentation. - depends on STACK_VALIDATION || CC_IS_GCC + depends on STACK_VALIDATION || CC_IS_GCC || CLANG_VERSION >= 140000 help Enable support for modeling a subset of weak memory, which allows detecting a subset of data races due to missing memory barriers. -- cgit v1.2.3-59-g8ed1b From 80d7476fa20a3cc83f76b3b02a7575891d1e7511 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Sat, 4 Dec 2021 13:57:03 +0100 Subject: kcsan: Turn barrier instrumentation into macros Some architectures use barriers in 'extern inline' functions, from which we should not refer to static inline functions. For example, building Alpha with gcc and W=1 shows: ./include/asm-generic/barrier.h:70:30: warning: 'kcsan_rmb' is static but used in inline function 'pmd_offset' which is not static 70 | #define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) | ^~~~~~~~~ ./arch/alpha/include/asm/pgtable.h:293:9: note: in expansion of macro 'smp_rmb' 293 | smp_rmb(); /* see above */ | ^~~~~~~ Which seems to warn about 6.7.4#3 of the C standard: "An inline definition of a function with external linkage shall not contain a definition of a modifiable object with static or thread storage duration, and shall not contain a reference to an identifier with internal linkage." Fix it by turning barrier instrumentation into macros, which matches definitions in . Perhaps we can revert this change in future, when there are no more 'extern inline' users left. Link: https://lkml.kernel.org/r/202112041334.X44uWZXf-lkp@intel.com Reported-by: kernel test robot Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- include/linux/kcsan-checks.h | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h index 9d2c869167f2..92f3843d9ebb 100644 --- a/include/linux/kcsan-checks.h +++ b/include/linux/kcsan-checks.h @@ -241,28 +241,30 @@ static inline void __kcsan_disable_current(void) { } * disabled with the __no_kcsan function attribute. * * Also see definition of __tsan_atomic_signal_fence() in kernel/kcsan/core.c. + * + * These are all macros, like , since some architectures use them + * in non-static inline functions. */ #define __KCSAN_BARRIER_TO_SIGNAL_FENCE(name) \ - static __always_inline void kcsan_##name(void) \ - { \ + do { \ barrier(); \ __atomic_signal_fence(__KCSAN_BARRIER_TO_SIGNAL_FENCE_##name); \ barrier(); \ - } -__KCSAN_BARRIER_TO_SIGNAL_FENCE(mb) -__KCSAN_BARRIER_TO_SIGNAL_FENCE(wmb) -__KCSAN_BARRIER_TO_SIGNAL_FENCE(rmb) -__KCSAN_BARRIER_TO_SIGNAL_FENCE(release) + } while (0) +#define kcsan_mb() __KCSAN_BARRIER_TO_SIGNAL_FENCE(mb) +#define kcsan_wmb() __KCSAN_BARRIER_TO_SIGNAL_FENCE(wmb) +#define kcsan_rmb() __KCSAN_BARRIER_TO_SIGNAL_FENCE(rmb) +#define kcsan_release() __KCSAN_BARRIER_TO_SIGNAL_FENCE(release) #elif defined(CONFIG_KCSAN_WEAK_MEMORY) && defined(__KCSAN_INSTRUMENT_BARRIERS__) #define kcsan_mb __kcsan_mb #define kcsan_wmb __kcsan_wmb #define kcsan_rmb __kcsan_rmb #define kcsan_release __kcsan_release #else /* CONFIG_KCSAN_WEAK_MEMORY && ... */ -static inline void kcsan_mb(void) { } -static inline void kcsan_wmb(void) { } -static inline void kcsan_rmb(void) { } -static inline void kcsan_release(void) { } +#define kcsan_mb() do { } while (0) +#define kcsan_wmb() do { } while (0) +#define kcsan_rmb() do { } while (0) +#define kcsan_release() do { } while (0) #endif /* CONFIG_KCSAN_WEAK_MEMORY && ... */ /** -- cgit v1.2.3-59-g8ed1b From db67097aa6f2587b44055f2e16db72a11e17faef Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 9 Dec 2021 21:31:56 -0700 Subject: pktdvd: stop using bdi congestion framework. The bdi congestion framework isn't widely used and should be deprecated. pktdvd makes use of it to track congestion, but this can be done entirely internally to pktdvd, so it doesn't need to use the framework. So introduce a "congested" flag. When waiting for bio_queue_size to drop, set this flag and a var_waitqueue() to wait for it. When bio_queue_size does drop and this flag is set, clear the flag and call wake_up_var(). We don't use a wait_var_event macro for the waiting as we need to set the flag and drop the spinlock before calling schedule() and while that is possible with __wait_var_event(), result is not easy to read. Reviewed-by: Christoph Hellwig Signed-off-by: NeilBrown Link: https://lore.kernel.org/r/163910843527.9928.857338663717630212@noble.neil.brown.name Signed-off-by: Jens Axboe --- drivers/block/pktcdvd.c | 31 ++++++++++++++++++++----------- include/linux/pktcdvd.h | 2 ++ 2 files changed, 22 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 887c98d61684..713b7dcf39f9 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -1107,7 +1107,6 @@ static int pkt_handle_queue(struct pktcdvd_device *pd) sector_t zone = 0; /* Suppress gcc warning */ struct pkt_rb_node *node, *first_node; struct rb_node *n; - int wakeup; atomic_set(&pd->scan_queue, 0); @@ -1179,12 +1178,14 @@ try_next_bio: spin_unlock(&pkt->lock); } /* check write congestion marks, and if bio_queue_size is - below, wake up any waiters */ - wakeup = (pd->write_congestion_on > 0 - && pd->bio_queue_size <= pd->write_congestion_off); + * below, wake up any waiters + */ + if (pd->congested && + pd->bio_queue_size <= pd->write_congestion_off) { + pd->congested = false; + wake_up_var(&pd->congested); + } spin_unlock(&pd->lock); - if (wakeup) - clear_bdi_congested(pd->disk->bdi, BLK_RW_ASYNC); pkt->sleep_time = max(PACKET_WAIT_TIME, 1); pkt_set_state(pkt, PACKET_WAITING_STATE); @@ -2356,7 +2357,7 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio) } spin_unlock(&pd->cdrw.active_list_lock); - /* + /* * Test if there is enough room left in the bio work queue * (queue size >= congestion on mark). * If not, wait till the work queue size is below the congestion off mark. @@ -2364,12 +2365,20 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio) spin_lock(&pd->lock); if (pd->write_congestion_on > 0 && pd->bio_queue_size >= pd->write_congestion_on) { - set_bdi_congested(bio->bi_bdev->bd_disk->bdi, BLK_RW_ASYNC); - do { + struct wait_bit_queue_entry wqe; + + init_wait_var_entry(&wqe, &pd->congested, 0); + for (;;) { + prepare_to_wait_event(__var_waitqueue(&pd->congested), + &wqe.wq_entry, + TASK_UNINTERRUPTIBLE); + if (pd->bio_queue_size <= pd->write_congestion_off) + break; + pd->congested = true; spin_unlock(&pd->lock); - congestion_wait(BLK_RW_ASYNC, HZ); + schedule(); spin_lock(&pd->lock); - } while(pd->bio_queue_size > pd->write_congestion_off); + } } spin_unlock(&pd->lock); diff --git a/include/linux/pktcdvd.h b/include/linux/pktcdvd.h index 174601554b06..c391e694aa26 100644 --- a/include/linux/pktcdvd.h +++ b/include/linux/pktcdvd.h @@ -183,6 +183,8 @@ struct pktcdvd_device spinlock_t lock; /* Serialize access to bio_queue */ struct rb_root bio_queue; /* Work queue of bios we need to handle */ int bio_queue_size; /* Number of nodes in bio_queue */ + bool congested; /* Someone is waiting for bio_queue_size + * to drop. */ sector_t current_sector; /* Keep track of where the elevator is */ atomic_t scan_queue; /* Set to non-zero when pkt_handle_queue */ /* needs to be run. */ -- cgit v1.2.3-59-g8ed1b From f95711242390d759f69fd67ad46b31491fe904d6 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Wed, 8 Dec 2021 17:43:53 +0000 Subject: EDAC: Add RDDR5 and LRDDR5 memory types Include Registered-DDR5 and Load-Reduced DDR5 in the list of memory types. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov Link: https://lore.kernel.org/r/20211208174356.1997855-2-yazen.ghannam@amd.com --- drivers/edac/edac_mc.c | 2 ++ include/linux/edac.h | 6 ++++++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 9f82ca295353..9d9aabdec96b 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -162,6 +162,8 @@ const char * const edac_mem_types[] = { [MEM_LPDDR4] = "Low-Power-DDR4-RAM", [MEM_LRDDR4] = "Load-Reduced-DDR4-RAM", [MEM_DDR5] = "Unbuffered-DDR5", + [MEM_RDDR5] = "Registered-DDR5", + [MEM_LRDDR5] = "Load-Reduced-DDR5-RAM", [MEM_NVDIMM] = "Non-volatile-RAM", [MEM_WIO2] = "Wide-IO-2", [MEM_HBM2] = "High-bandwidth-memory-Gen2", diff --git a/include/linux/edac.h b/include/linux/edac.h index 4207d06996a4..e730b3468719 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -182,6 +182,8 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_LRDDR4: Load-Reduced DDR4 memory. * @MEM_LPDDR4: Low-Power DDR4 memory. * @MEM_DDR5: Unbuffered DDR5 RAM + * @MEM_RDDR5: Registered DDR5 RAM + * @MEM_LRDDR5: Load-Reduced DDR5 memory. * @MEM_NVDIMM: Non-volatile RAM * @MEM_WIO2: Wide I/O 2. * @MEM_HBM2: High bandwidth Memory Gen 2. @@ -211,6 +213,8 @@ enum mem_type { MEM_LRDDR4, MEM_LPDDR4, MEM_DDR5, + MEM_RDDR5, + MEM_LRDDR5, MEM_NVDIMM, MEM_WIO2, MEM_HBM2, @@ -239,6 +243,8 @@ enum mem_type { #define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) #define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) #define MEM_FLAG_DDR5 BIT(MEM_DDR5) +#define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) +#define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) #define MEM_FLAG_HBM2 BIT(MEM_HBM2) -- cgit v1.2.3-59-g8ed1b From 1614b2b11fab29dd4ff31ebba9d266961f5af69e Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 29 Nov 2021 14:28:41 +0000 Subject: arch: Make ARCH_STACKWALK independent of STACKTRACE Make arch_stack_walk() available for ARCH_STACKWALK architectures without it being entangled in STACKTRACE. Link: https://lore.kernel.org/lkml/20211022152104.356586621@infradead.org/ Signed-off-by: Peter Zijlstra (Intel) [Mark: rebase, drop unnecessary arm change] Signed-off-by: Mark Rutland Cc: Albert Ou Cc: Borislav Petkov Cc: Christian Borntraeger Cc: Dave Hansen Cc: Heiko Carstens Cc: Ingo Molnar Cc: Michael Ellerman Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Thomas Gleixner Cc: Vasily Gorbik Link: https://lore.kernel.org/r/20211129142849.3056714-2-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/stacktrace.c | 4 ---- arch/powerpc/kernel/Makefile | 3 +-- arch/riscv/kernel/stacktrace.c | 4 ---- arch/s390/kernel/Makefile | 3 +-- arch/x86/kernel/Makefile | 2 +- include/linux/stacktrace.h | 35 ++++++++++++++++++----------------- 6 files changed, 21 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 94f83cd44e50..e6ba6b000564 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -221,8 +221,6 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) barrier(); } -#ifdef CONFIG_STACKTRACE - noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, struct task_struct *task, struct pt_regs *regs) @@ -241,5 +239,3 @@ noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry, walk_stackframe(task, &frame, consume_entry, cookie); } - -#endif diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 5fa68c2ef1f8..b039877c743d 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -47,7 +47,7 @@ obj-y := cputable.o syscalls.o \ udbg.o misc.o io.o misc_$(BITS).o \ of_platform.o prom_parse.o firmware.o \ hw_breakpoint_constraints.o interrupt.o \ - kdebugfs.o + kdebugfs.o stacktrace.o obj-y += ptrace/ obj-$(CONFIG_PPC64) += setup_64.o \ paca.o nvram_64.o note.o @@ -116,7 +116,6 @@ obj-$(CONFIG_OPTPROBES) += optprobes.o optprobes_head.o obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o obj-$(CONFIG_ARCH_HAS_DMA_SET_MASK) += dma-mask.o diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 0fcdc0233fac..201ee206fb57 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -139,12 +139,8 @@ unsigned long __get_wchan(struct task_struct *task) return pc; } -#ifdef CONFIG_STACKTRACE - noinline void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, struct task_struct *task, struct pt_regs *regs) { walk_stackframe(task, regs, consume_entry, cookie); } - -#endif /* CONFIG_STACKTRACE */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 80f500ffb55c..be8007f367aa 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -40,7 +40,7 @@ obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o +obj-y += smp.o text_amode31.o stacktrace.o extra-y += head64.o vmlinux.lds @@ -55,7 +55,6 @@ compat-obj-$(CONFIG_AUDIT) += compat_audit.o obj-$(CONFIG_COMPAT) += compat_linux.o compat_signal.o obj-$(CONFIG_COMPAT) += $(compat-obj-y) obj-$(CONFIG_EARLY_PRINTK) += early_printk.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KPROBES) += kprobes_insn_page.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 2ff3e600f426..6aef9ee28a39 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -84,7 +84,7 @@ obj-$(CONFIG_IA32_EMULATION) += tls.o obj-y += step.o obj-$(CONFIG_INTEL_TXT) += tboot.o obj-$(CONFIG_ISA_DMA_API) += i8237.o -obj-$(CONFIG_STACKTRACE) += stacktrace.o +obj-y += stacktrace.o obj-y += cpu/ obj-y += acpi/ obj-y += reboot.o diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h index bef158815e83..97455880ac41 100644 --- a/include/linux/stacktrace.h +++ b/include/linux/stacktrace.h @@ -8,22 +8,6 @@ struct task_struct; struct pt_regs; -#ifdef CONFIG_STACKTRACE -void stack_trace_print(const unsigned long *trace, unsigned int nr_entries, - int spaces); -int stack_trace_snprint(char *buf, size_t size, const unsigned long *entries, - unsigned int nr_entries, int spaces); -unsigned int stack_trace_save(unsigned long *store, unsigned int size, - unsigned int skipnr); -unsigned int stack_trace_save_tsk(struct task_struct *task, - unsigned long *store, unsigned int size, - unsigned int skipnr); -unsigned int stack_trace_save_regs(struct pt_regs *regs, unsigned long *store, - unsigned int size, unsigned int skipnr); -unsigned int stack_trace_save_user(unsigned long *store, unsigned int size); -unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries); - -/* Internal interfaces. Do not use in generic code */ #ifdef CONFIG_ARCH_STACKWALK /** @@ -76,8 +60,25 @@ int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry, void *cookie, void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie, const struct pt_regs *regs); +#endif /* CONFIG_ARCH_STACKWALK */ -#else /* CONFIG_ARCH_STACKWALK */ +#ifdef CONFIG_STACKTRACE +void stack_trace_print(const unsigned long *trace, unsigned int nr_entries, + int spaces); +int stack_trace_snprint(char *buf, size_t size, const unsigned long *entries, + unsigned int nr_entries, int spaces); +unsigned int stack_trace_save(unsigned long *store, unsigned int size, + unsigned int skipnr); +unsigned int stack_trace_save_tsk(struct task_struct *task, + unsigned long *store, unsigned int size, + unsigned int skipnr); +unsigned int stack_trace_save_regs(struct pt_regs *regs, unsigned long *store, + unsigned int size, unsigned int skipnr); +unsigned int stack_trace_save_user(unsigned long *store, unsigned int size); +unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries); + +#ifndef CONFIG_ARCH_STACKWALK +/* Internal interfaces. Do not use in generic code */ struct stack_trace { unsigned int nr_entries, max_entries; unsigned long *entries; -- cgit v1.2.3-59-g8ed1b From 9ba74e6c9e9d0c5c1e5792a7111fc7d1a0589cb8 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 9 Dec 2021 23:44:21 -0800 Subject: net: add networking namespace refcount tracker We have 100+ syzbot reports about netns being dismantled too soon, still unresolved as of today. We think a missing get_net() or an extra put_net() is the root cause. In order to find the bug(s), and be able to spot future ones, this patch adds CONFIG_NET_NS_REFCNT_TRACKER and new helpers to precisely pair all put_net() with corresponding get_net(). To use these helpers, each data structure owning a refcount should also use a "netns_tracker" to pair the get and put. Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 9 +-------- include/net/net_namespace.h | 34 ++++++++++++++++++++++++++++++++++ include/net/net_trackers.h | 18 ++++++++++++++++++ net/Kconfig.debug | 9 +++++++++ net/core/net_namespace.c | 3 +++ 5 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 include/net/net_trackers.h (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1a748ee9a421..235d5d082f1a 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -48,7 +48,7 @@ #include #include #include -#include +#include struct netpoll_info; struct device; @@ -300,13 +300,6 @@ enum netdev_state_t { __LINK_STATE_TESTING, }; - -#ifdef CONFIG_NET_DEV_REFCNT_TRACKER -typedef struct ref_tracker *netdevice_tracker; -#else -typedef struct {} netdevice_tracker; -#endif - struct gro_list { struct list_head list; int count; diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index bb5fa5914032..5b61c462e534 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,7 @@ struct net { struct idr netns_ids; struct ns_common ns; + struct ref_tracker_dir refcnt_tracker; struct list_head dev_base_head; struct proc_dir_entry *proc_net; @@ -240,6 +242,7 @@ void ipx_unregister_sysctl(void); #ifdef CONFIG_NET_NS void __put_net(struct net *net); +/* Try using get_net_track() instead */ static inline struct net *get_net(struct net *net) { refcount_inc(&net->ns.count); @@ -258,6 +261,7 @@ static inline struct net *maybe_get_net(struct net *net) return net; } +/* Try using put_net_track() instead */ static inline void put_net(struct net *net) { if (refcount_dec_and_test(&net->ns.count)) @@ -308,6 +312,36 @@ static inline int check_net(const struct net *net) #endif +static inline void netns_tracker_alloc(struct net *net, + netns_tracker *tracker, gfp_t gfp) +{ +#ifdef CONFIG_NET_NS_REFCNT_TRACKER + ref_tracker_alloc(&net->refcnt_tracker, tracker, gfp); +#endif +} + +static inline void netns_tracker_free(struct net *net, + netns_tracker *tracker) +{ +#ifdef CONFIG_NET_NS_REFCNT_TRACKER + ref_tracker_free(&net->refcnt_tracker, tracker); +#endif +} + +static inline struct net *get_net_track(struct net *net, + netns_tracker *tracker, gfp_t gfp) +{ + get_net(net); + netns_tracker_alloc(net, tracker, gfp); + return net; +} + +static inline void put_net_track(struct net *net, netns_tracker *tracker) +{ + netns_tracker_free(net, tracker); + put_net(net); +} + typedef struct { #ifdef CONFIG_NET_NS struct net *net; diff --git a/include/net/net_trackers.h b/include/net/net_trackers.h new file mode 100644 index 000000000000..d94c76cf15a9 --- /dev/null +++ b/include/net/net_trackers.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NET_NET_TRACKERS_H +#define __NET_NET_TRACKERS_H +#include + +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +typedef struct ref_tracker *netdevice_tracker; +#else +typedef struct {} netdevice_tracker; +#endif + +#ifdef CONFIG_NET_NS_REFCNT_TRACKER +typedef struct ref_tracker *netns_tracker; +#else +typedef struct {} netns_tracker; +#endif + +#endif /* __NET_NET_TRACKERS_H */ diff --git a/net/Kconfig.debug b/net/Kconfig.debug index fb5c70e01cb3..2f50611df858 100644 --- a/net/Kconfig.debug +++ b/net/Kconfig.debug @@ -8,3 +8,12 @@ config NET_DEV_REFCNT_TRACKER help Enable debugging feature to track device references. This adds memory and cpu costs. + +config NET_NS_REFCNT_TRACKER + bool "Enable networking namespace refcount tracking" + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + select REF_TRACKER + default n + help + Enable debugging feature to track netns references. + This adds memory and cpu costs. diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 202fa5eacd0f..9b7171c40434 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -311,6 +311,8 @@ static __net_init int setup_net(struct net *net, struct user_namespace *user_ns) LIST_HEAD(net_exit_list); refcount_set(&net->ns.count, 1); + ref_tracker_dir_init(&net->refcnt_tracker, 128); + refcount_set(&net->passive, 1); get_random_bytes(&net->hash_mix, sizeof(u32)); preempt_disable(); @@ -635,6 +637,7 @@ static DECLARE_WORK(net_cleanup_work, cleanup_net); void __put_net(struct net *net) { + ref_tracker_dir_exit(&net->refcnt_tracker); /* Cleanup the network namespace in process context */ if (llist_add(&net->cleanup_list, &cleanup_list)) queue_work(netns_wq, &net_cleanup_work); -- cgit v1.2.3-59-g8ed1b From 04a931e58d1944ab3d1e11fdfde1947fbe5b6a37 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 9 Dec 2021 23:44:23 -0800 Subject: net: add netns refcount tracker to struct seq_net_private Signed-off-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- fs/proc/proc_net.c | 19 ++++++++++++++++--- include/linux/seq_file_net.h | 3 ++- 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 15c2e55d2ed2..39b823ab2564 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -61,15 +61,27 @@ static int seq_open_net(struct inode *inode, struct file *file) } #ifdef CONFIG_NET_NS p->net = net; + netns_tracker_alloc(net, &p->ns_tracker, GFP_KERNEL); #endif return 0; } +static void seq_file_net_put_net(struct seq_file *seq) +{ +#ifdef CONFIG_NET_NS + struct seq_net_private *priv = seq->private; + + put_net_track(priv->net, &priv->ns_tracker); +#else + put_net(&init_net); +#endif +} + static int seq_release_net(struct inode *ino, struct file *f) { struct seq_file *seq = f->private_data; - put_net(seq_file_net(seq)); + seq_file_net_put_net(seq); seq_release_private(ino, f); return 0; } @@ -87,7 +99,8 @@ int bpf_iter_init_seq_net(void *priv_data, struct bpf_iter_aux_info *aux) #ifdef CONFIG_NET_NS struct seq_net_private *p = priv_data; - p->net = get_net(current->nsproxy->net_ns); + p->net = get_net_track(current->nsproxy->net_ns, &p->ns_tracker, + GFP_KERNEL); #endif return 0; } @@ -97,7 +110,7 @@ void bpf_iter_fini_seq_net(void *priv_data) #ifdef CONFIG_NET_NS struct seq_net_private *p = priv_data; - put_net(p->net); + put_net_track(p->net, &p->ns_tracker); #endif } diff --git a/include/linux/seq_file_net.h b/include/linux/seq_file_net.h index 0fdbe1ddd8d1..b97912fdbae7 100644 --- a/include/linux/seq_file_net.h +++ b/include/linux/seq_file_net.h @@ -9,7 +9,8 @@ extern struct net init_net; struct seq_net_private { #ifdef CONFIG_NET_NS - struct net *net; + struct net *net; + netns_tracker ns_tracker; #endif }; -- cgit v1.2.3-59-g8ed1b From 65c7cdedeb3026fabcc967a7aae2f755ad4d0783 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 3 Sep 2021 11:24:17 -0400 Subject: genirq: Provide new interfaces for affinity hints The discussion about removing the side effect of irq_set_affinity_hint() of actually applying the cpumask (if not NULL) as affinity to the interrupt, unearthed a few unpleasantries: 1) The modular perf drivers rely on the current behaviour for the very wrong reasons. 2) While none of the other drivers prevents user space from changing the affinity, a cursorily inspection shows that there are at least expectations in some drivers. #1 needs to be cleaned up anyway, so that's not a problem #2 might result in subtle regressions especially when irqbalanced (which nowadays ignores the affinity hint) is disabled. Provide new interfaces: irq_update_affinity_hint() - Only sets the affinity hint pointer irq_set_affinity_and_hint() - Set the pointer and apply the affinity to the interrupt Make irq_set_affinity_hint() a wrapper around irq_apply_affinity_hint() and document it to be phased out. Signed-off-by: Thomas Gleixner Signed-off-by: Nitesh Narayan Lal Signed-off-by: Thomas Gleixner Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20210501021832.743094-1-jesse.brandeburg@intel.com Link: https://lore.kernel.org/r/20210903152430.244937-2-nitesh@redhat.com --- include/linux/interrupt.h | 53 ++++++++++++++++++++++++++++++++++++++++++++++- kernel/irq/manage.c | 8 +++---- 2 files changed, 56 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 1f22a30c0963..9367f1cb2e3c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -329,7 +329,46 @@ extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask); extern int irq_can_set_affinity(unsigned int irq); extern int irq_select_affinity(unsigned int irq); -extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m); +extern int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m, + bool setaffinity); + +/** + * irq_update_affinity_hint - Update the affinity hint + * @irq: Interrupt to update + * @m: cpumask pointer (NULL to clear the hint) + * + * Updates the affinity hint, but does not change the affinity of the interrupt. + */ +static inline int +irq_update_affinity_hint(unsigned int irq, const struct cpumask *m) +{ + return __irq_apply_affinity_hint(irq, m, false); +} + +/** + * irq_set_affinity_and_hint - Update the affinity hint and apply the provided + * cpumask to the interrupt + * @irq: Interrupt to update + * @m: cpumask pointer (NULL to clear the hint) + * + * Updates the affinity hint and if @m is not NULL it applies it as the + * affinity of that interrupt. + */ +static inline int +irq_set_affinity_and_hint(unsigned int irq, const struct cpumask *m) +{ + return __irq_apply_affinity_hint(irq, m, true); +} + +/* + * Deprecated. Use irq_update_affinity_hint() or irq_set_affinity_and_hint() + * instead. + */ +static inline int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) +{ + return irq_set_affinity_and_hint(irq, m); +} + extern int irq_update_affinity_desc(unsigned int irq, struct irq_affinity_desc *affinity); @@ -361,6 +400,18 @@ static inline int irq_can_set_affinity(unsigned int irq) static inline int irq_select_affinity(unsigned int irq) { return 0; } +static inline int irq_update_affinity_hint(unsigned int irq, + const struct cpumask *m) +{ + return -EINVAL; +} + +static inline int irq_set_affinity_and_hint(unsigned int irq, + const struct cpumask *m) +{ + return -EINVAL; +} + static inline int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) { diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 7405e384e5ed..f23ffd30385b 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -486,7 +486,8 @@ int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) } EXPORT_SYMBOL_GPL(irq_force_affinity); -int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) +int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m, + bool setaffinity) { unsigned long flags; struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL); @@ -495,12 +496,11 @@ int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) return -EINVAL; desc->affinity_hint = m; irq_put_desc_unlock(desc, flags); - /* set the initial affinity to prevent every interrupt being on CPU0 */ - if (m) + if (m && setaffinity) __irq_set_affinity(irq, m, false); return 0; } -EXPORT_SYMBOL_GPL(irq_set_affinity_hint); +EXPORT_SYMBOL_GPL(__irq_apply_affinity_hint); static void irq_affinity_notify(struct work_struct *work) { -- cgit v1.2.3-59-g8ed1b From bff8c3848e071d387d8b0784dc91fa49cd563774 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Nov 2021 11:01:03 +0100 Subject: bitfield.h: Fix "type of reg too small for mask" test The test: 'mask > (typeof(_reg))~0ull' only works correctly when both sides are unsigned, consider: - 0xff000000 vs (int)~0ull - 0x000000ff vs (int)~0ull Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/r/20211110101324.950210584@infradead.org --- include/linux/bitfield.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 4e035aca6f7e..6093fa6db260 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -41,6 +41,22 @@ #define __bf_shf(x) (__builtin_ffsll(x) - 1) +#define __scalar_type_to_unsigned_cases(type) \ + unsigned type: (unsigned type)0, \ + signed type: (unsigned type)0 + +#define __unsigned_scalar_typeof(x) typeof( \ + _Generic((x), \ + char: (unsigned char)0, \ + __scalar_type_to_unsigned_cases(char), \ + __scalar_type_to_unsigned_cases(short), \ + __scalar_type_to_unsigned_cases(int), \ + __scalar_type_to_unsigned_cases(long), \ + __scalar_type_to_unsigned_cases(long long), \ + default: (x))) + +#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x)) + #define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \ ({ \ BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \ @@ -49,7 +65,8 @@ BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \ ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \ _pfx "value too large for the field"); \ - BUILD_BUG_ON_MSG((_mask) > (typeof(_reg))~0ull, \ + BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \ + __bf_cast_unsigned(_reg, ~0ull), \ _pfx "type of reg too small for mask"); \ __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \ (1ULL << __bf_shf(_mask))); \ -- cgit v1.2.3-59-g8ed1b From 4121485d271bd730537f613ce041e7ea659606a7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 9 Dec 2021 21:52:31 +0200 Subject: PCI: Sort Intel Device IDs by value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sort Intel Device IDs by value. [bhelgaas: lower-case Intel section since we're touching it anyway] Link: https://lore.kernel.org/r/20211209195231.2785-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Signed-off-by: Bjorn Helgaas Reviewed-by: Krzysztof Wilczyński --- include/linux/pci_ids.h | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 011f2f1ea5bb..0d26ab7eb7dc 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2635,8 +2635,8 @@ #define PCI_DEVICE_ID_INTEL_PXHD_0 0x0320 #define PCI_DEVICE_ID_INTEL_PXHD_1 0x0321 #define PCI_DEVICE_ID_INTEL_PXH_0 0x0329 -#define PCI_DEVICE_ID_INTEL_PXH_1 0x032A -#define PCI_DEVICE_ID_INTEL_PXHV 0x032C +#define PCI_DEVICE_ID_INTEL_PXH_1 0x032a +#define PCI_DEVICE_ID_INTEL_PXHV 0x032c #define PCI_DEVICE_ID_INTEL_80332_0 0x0330 #define PCI_DEVICE_ID_INTEL_80332_1 0x0332 #define PCI_DEVICE_ID_INTEL_80333_0 0x0370 @@ -2654,14 +2654,14 @@ #define PCI_DEVICE_ID_INTEL_MFD_SDIO2 0x0822 #define PCI_DEVICE_ID_INTEL_MFD_EMMC0 0x0823 #define PCI_DEVICE_ID_INTEL_MFD_EMMC1 0x0824 -#define PCI_DEVICE_ID_INTEL_MRST_SD2 0x084F -#define PCI_DEVICE_ID_INTEL_QUARK_X1000_ILB 0x095E +#define PCI_DEVICE_ID_INTEL_MRST_SD2 0x084f +#define PCI_DEVICE_ID_INTEL_QUARK_X1000_ILB 0x095e #define PCI_DEVICE_ID_INTEL_I960 0x0960 #define PCI_DEVICE_ID_INTEL_I960RM 0x0962 #define PCI_DEVICE_ID_INTEL_CENTERTON_ILB 0x0c60 #define PCI_DEVICE_ID_INTEL_8257X_SOL 0x1062 #define PCI_DEVICE_ID_INTEL_82573E_SOL 0x1085 -#define PCI_DEVICE_ID_INTEL_82573L_SOL 0x108F +#define PCI_DEVICE_ID_INTEL_82573L_SOL 0x108f #define PCI_DEVICE_ID_INTEL_82815_MC 0x1130 #define PCI_DEVICE_ID_INTEL_82815_CGC 0x1132 #define PCI_DEVICE_ID_INTEL_82092AA_0 0x1221 @@ -2755,12 +2755,6 @@ #define PCI_DEVICE_ID_INTEL_82801EB_11 0x24db #define PCI_DEVICE_ID_INTEL_82801EB_12 0x24dc #define PCI_DEVICE_ID_INTEL_82801EB_13 0x24dd -#define PCI_DEVICE_ID_INTEL_ESB_1 0x25a1 -#define PCI_DEVICE_ID_INTEL_ESB_2 0x25a2 -#define PCI_DEVICE_ID_INTEL_ESB_4 0x25a4 -#define PCI_DEVICE_ID_INTEL_ESB_5 0x25a6 -#define PCI_DEVICE_ID_INTEL_ESB_9 0x25ab -#define PCI_DEVICE_ID_INTEL_ESB_10 0x25ac #define PCI_DEVICE_ID_INTEL_82820_HB 0x2500 #define PCI_DEVICE_ID_INTEL_82820_UP_HB 0x2501 #define PCI_DEVICE_ID_INTEL_82850_HB 0x2530 @@ -2775,14 +2769,15 @@ #define PCI_DEVICE_ID_INTEL_82915G_IG 0x2582 #define PCI_DEVICE_ID_INTEL_82915GM_HB 0x2590 #define PCI_DEVICE_ID_INTEL_82915GM_IG 0x2592 -#define PCI_DEVICE_ID_INTEL_5000_ERR 0x25F0 -#define PCI_DEVICE_ID_INTEL_5000_FBD0 0x25F5 -#define PCI_DEVICE_ID_INTEL_5000_FBD1 0x25F6 -#define PCI_DEVICE_ID_INTEL_82945G_HB 0x2770 -#define PCI_DEVICE_ID_INTEL_82945G_IG 0x2772 -#define PCI_DEVICE_ID_INTEL_3000_HB 0x2778 -#define PCI_DEVICE_ID_INTEL_82945GM_HB 0x27A0 -#define PCI_DEVICE_ID_INTEL_82945GM_IG 0x27A2 +#define PCI_DEVICE_ID_INTEL_ESB_1 0x25a1 +#define PCI_DEVICE_ID_INTEL_ESB_2 0x25a2 +#define PCI_DEVICE_ID_INTEL_ESB_4 0x25a4 +#define PCI_DEVICE_ID_INTEL_ESB_5 0x25a6 +#define PCI_DEVICE_ID_INTEL_ESB_9 0x25ab +#define PCI_DEVICE_ID_INTEL_ESB_10 0x25ac +#define PCI_DEVICE_ID_INTEL_5000_ERR 0x25f0 +#define PCI_DEVICE_ID_INTEL_5000_FBD0 0x25f5 +#define PCI_DEVICE_ID_INTEL_5000_FBD1 0x25f6 #define PCI_DEVICE_ID_INTEL_ICH6_0 0x2640 #define PCI_DEVICE_ID_INTEL_ICH6_1 0x2641 #define PCI_DEVICE_ID_INTEL_ICH6_2 0x2642 @@ -2794,6 +2789,11 @@ #define PCI_DEVICE_ID_INTEL_ESB2_14 0x2698 #define PCI_DEVICE_ID_INTEL_ESB2_17 0x269b #define PCI_DEVICE_ID_INTEL_ESB2_18 0x269e +#define PCI_DEVICE_ID_INTEL_82945G_HB 0x2770 +#define PCI_DEVICE_ID_INTEL_82945G_IG 0x2772 +#define PCI_DEVICE_ID_INTEL_3000_HB 0x2778 +#define PCI_DEVICE_ID_INTEL_82945GM_HB 0x27a0 +#define PCI_DEVICE_ID_INTEL_82945GM_IG 0x27a2 #define PCI_DEVICE_ID_INTEL_ICH7_0 0x27b8 #define PCI_DEVICE_ID_INTEL_ICH7_1 0x27b9 #define PCI_DEVICE_ID_INTEL_ICH7_30 0x27b0 @@ -2846,7 +2846,7 @@ #define PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_PHY0 0x2c91 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR 0x2c98 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD 0x2c99 -#define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST 0x2c9C +#define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST 0x2c9c #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL 0x2ca0 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR 0x2ca1 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK 0x2ca2 @@ -2958,16 +2958,16 @@ #define PCI_DEVICE_ID_INTEL_SBRIDGE_BR 0x3cf5 /* 13.6 */ #define PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1 0x3cf6 /* 12.7 */ #define PCI_DEVICE_ID_INTEL_IOAT_SNB 0x402f -#define PCI_DEVICE_ID_INTEL_5100_16 0x65f0 -#define PCI_DEVICE_ID_INTEL_5100_19 0x65f3 -#define PCI_DEVICE_ID_INTEL_5100_21 0x65f5 -#define PCI_DEVICE_ID_INTEL_5100_22 0x65f6 #define PCI_DEVICE_ID_INTEL_5400_ERR 0x4030 #define PCI_DEVICE_ID_INTEL_5400_FBD0 0x4035 #define PCI_DEVICE_ID_INTEL_5400_FBD1 0x4036 -#define PCI_DEVICE_ID_INTEL_IOAT_SCNB 0x65ff #define PCI_DEVICE_ID_INTEL_EP80579_0 0x5031 #define PCI_DEVICE_ID_INTEL_EP80579_1 0x5032 +#define PCI_DEVICE_ID_INTEL_5100_16 0x65f0 +#define PCI_DEVICE_ID_INTEL_5100_19 0x65f3 +#define PCI_DEVICE_ID_INTEL_5100_21 0x65f5 +#define PCI_DEVICE_ID_INTEL_5100_22 0x65f6 +#define PCI_DEVICE_ID_INTEL_IOAT_SCNB 0x65ff #define PCI_DEVICE_ID_INTEL_82371SB_0 0x7000 #define PCI_DEVICE_ID_INTEL_82371SB_1 0x7010 #define PCI_DEVICE_ID_INTEL_82371SB_2 0x7020 -- cgit v1.2.3-59-g8ed1b From c5fb19937455095573a19ddcbff32e993ed10e35 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 10 Dec 2021 22:16:49 +0800 Subject: bpf: Add bpf_strncmp helper The helper compares two strings: one string is a null-terminated read-only string, and another string has const max storage size but doesn't need to be null-terminated. It can be used to compare file name in tracing or LSM program. Signed-off-by: Hou Tao Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211210141652.877186-2-houtao1@huawei.com --- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/helpers.c | 16 ++++++++++++++++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 4 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0ceb54c6342f..7a40022e3d00 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2163,6 +2163,7 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto; extern const struct bpf_func_proto bpf_find_vma_proto; extern const struct bpf_func_proto bpf_loop_proto; +extern const struct bpf_func_proto bpf_strncmp_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c26871263f1f..2820c77e4846 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4983,6 +4983,16 @@ union bpf_attr { * Return * The number of loops performed, **-EINVAL** for invalid **flags**, * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. + * + * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2) + * Description + * Do strncmp() between **s1** and **s2**. **s1** doesn't need + * to be null-terminated and **s1_sz** is the maximum storage + * size of **s1**. **s2** must be a read-only string. + * Return + * An integer less than, equal to, or greater than zero + * if the first **s1_sz** bytes of **s1** is found to be + * less than, to match, or be greater than **s2**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5167,6 +5177,7 @@ union bpf_attr { FN(kallsyms_lookup_name), \ FN(find_vma), \ FN(loop), \ + FN(strncmp), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 19fecfeaa9c2..8babae03d30a 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -565,6 +565,20 @@ const struct bpf_func_proto bpf_strtoul_proto = { }; #endif +BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2) +{ + return strncmp(s1, s2, s1_sz); +} + +const struct bpf_func_proto bpf_strncmp_proto = { + .func = bpf_strncmp, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_CONST_SIZE, + .arg3_type = ARG_PTR_TO_CONST_STR, +}; + BPF_CALL_4(bpf_get_ns_current_pid_tgid, u64, dev, u64, ino, struct bpf_pidns_info *, nsdata, u32, size) { @@ -1378,6 +1392,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) return &bpf_for_each_map_elem_proto; case BPF_FUNC_loop: return &bpf_loop_proto; + case BPF_FUNC_strncmp: + return &bpf_strncmp_proto; default: break; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index c26871263f1f..2820c77e4846 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4983,6 +4983,16 @@ union bpf_attr { * Return * The number of loops performed, **-EINVAL** for invalid **flags**, * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. + * + * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2) + * Description + * Do strncmp() between **s1** and **s2**. **s1** doesn't need + * to be null-terminated and **s1_sz** is the maximum storage + * size of **s1**. **s2** must be a read-only string. + * Return + * An integer less than, equal to, or greater than zero + * if the first **s1_sz** bytes of **s1** is found to be + * less than, to match, or be greater than **s2**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5167,6 +5177,7 @@ union bpf_attr { FN(kallsyms_lookup_name), \ FN(find_vma), \ FN(loop), \ + FN(strncmp), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3-59-g8ed1b From 35d976802124303a5b3eb7ec3ed188d568204373 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:38 +0200 Subject: net: dsa: tag_ocelot: convert to tagger-owned data The felix driver makes very light use of dp->priv, and the tagger is effectively stateless. dp->priv is practically only needed to set up a callback to perform deferred xmit of PTP and STP packets using the ocelot-8021q tagging protocol (the main ocelot tagging protocol makes no use of dp->priv, although this driver sets up dp->priv irrespective of actual tagging protocol in use). struct felix_port (what used to be pointed to by dp->priv) is removed and replaced with a two-sided structure. The public side of this structure, visible to the switch driver, is ocelot_8021q_tagger_data. The private side is ocelot_8021q_tagger_private, and the latter structure physically encapsulates the former. The public half of the tagger data structure can be accessed through a helper of the same name (ocelot_8021q_tagger_data) which also sanity-checks the protocol currently in use by the switch. The public/private split was requested by Andrew Lunn. Suggested-by: Andrew Lunn Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/ocelot/felix.c | 64 ++++++++---------------------------- include/linux/dsa/ocelot.h | 12 +++++-- net/dsa/tag_ocelot_8021q.c | 73 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 94 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index a02fec33552d..f4fc403fbc1e 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -1155,38 +1155,22 @@ static void felix_port_deferred_xmit(struct kthread_work *work) kfree(xmit_work); } -static int felix_port_setup_tagger_data(struct dsa_switch *ds, int port) +static int felix_connect_tag_protocol(struct dsa_switch *ds, + enum dsa_tag_protocol proto) { - struct dsa_port *dp = dsa_to_port(ds, port); - struct ocelot *ocelot = ds->priv; - struct felix *felix = ocelot_to_felix(ocelot); - struct felix_port *felix_port; + struct ocelot_8021q_tagger_data *tagger_data; - if (!dsa_port_is_user(dp)) + switch (proto) { + case DSA_TAG_PROTO_OCELOT_8021Q: + tagger_data = ocelot_8021q_tagger_data(ds); + tagger_data->xmit_work_fn = felix_port_deferred_xmit; return 0; - - felix_port = kzalloc(sizeof(*felix_port), GFP_KERNEL); - if (!felix_port) - return -ENOMEM; - - felix_port->xmit_worker = felix->xmit_worker; - felix_port->xmit_work_fn = felix_port_deferred_xmit; - - dp->priv = felix_port; - - return 0; -} - -static void felix_port_teardown_tagger_data(struct dsa_switch *ds, int port) -{ - struct dsa_port *dp = dsa_to_port(ds, port); - struct felix_port *felix_port = dp->priv; - - if (!felix_port) - return; - - dp->priv = NULL; - kfree(felix_port); + case DSA_TAG_PROTO_OCELOT: + case DSA_TAG_PROTO_SEVILLE: + return 0; + default: + return -EPROTONOSUPPORT; + } } /* Hardware initialization done here so that we can allocate structures with @@ -1217,12 +1201,6 @@ static int felix_setup(struct dsa_switch *ds) } } - felix->xmit_worker = kthread_create_worker(0, "felix_xmit"); - if (IS_ERR(felix->xmit_worker)) { - err = PTR_ERR(felix->xmit_worker); - goto out_deinit_timestamp; - } - for (port = 0; port < ds->num_ports; port++) { if (dsa_is_unused_port(ds, port)) continue; @@ -1233,14 +1211,6 @@ static int felix_setup(struct dsa_switch *ds) * bits of vlan tag. */ felix_port_qos_map_init(ocelot, port); - - err = felix_port_setup_tagger_data(ds, port); - if (err) { - dev_err(ds->dev, - "port %d failed to set up tagger data: %pe\n", - port, ERR_PTR(err)); - goto out_deinit_ports; - } } err = ocelot_devlink_sb_register(ocelot); @@ -1268,13 +1238,9 @@ out_deinit_ports: if (dsa_is_unused_port(ds, port)) continue; - felix_port_teardown_tagger_data(ds, port); ocelot_deinit_port(ocelot, port); } - kthread_destroy_worker(felix->xmit_worker); - -out_deinit_timestamp: ocelot_deinit_timestamp(ocelot); ocelot_deinit(ocelot); @@ -1303,12 +1269,9 @@ static void felix_teardown(struct dsa_switch *ds) if (dsa_is_unused_port(ds, port)) continue; - felix_port_teardown_tagger_data(ds, port); ocelot_deinit_port(ocelot, port); } - kthread_destroy_worker(felix->xmit_worker); - ocelot_devlink_sb_unregister(ocelot); ocelot_deinit_timestamp(ocelot); ocelot_deinit(ocelot); @@ -1648,6 +1611,7 @@ felix_mrp_del_ring_role(struct dsa_switch *ds, int port, const struct dsa_switch_ops felix_switch_ops = { .get_tag_protocol = felix_get_tag_protocol, .change_tag_protocol = felix_change_tag_protocol, + .connect_tag_protocol = felix_connect_tag_protocol, .setup = felix_setup, .teardown = felix_teardown, .set_ageing_time = felix_set_ageing_time, diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h index 7ee708ad7df2..dca2969015d8 100644 --- a/include/linux/dsa/ocelot.h +++ b/include/linux/dsa/ocelot.h @@ -8,6 +8,7 @@ #include #include #include +#include struct ocelot_skb_cb { struct sk_buff *clone; @@ -168,11 +169,18 @@ struct felix_deferred_xmit_work { struct kthread_work work; }; -struct felix_port { +struct ocelot_8021q_tagger_data { void (*xmit_work_fn)(struct kthread_work *work); - struct kthread_worker *xmit_worker; }; +static inline struct ocelot_8021q_tagger_data * +ocelot_8021q_tagger_data(struct dsa_switch *ds) +{ + BUG_ON(ds->dst->tag_ops->proto != DSA_TAG_PROTO_OCELOT_8021Q); + + return ds->tagger_data; +} + static inline void ocelot_xfh_get_rew_val(void *extraction, u64 *rew_val) { packing(extraction, rew_val, 116, 85, OCELOT_TAG_LEN, UNPACK, 0); diff --git a/net/dsa/tag_ocelot_8021q.c b/net/dsa/tag_ocelot_8021q.c index a1919ea5e828..fe451f4de7ba 100644 --- a/net/dsa/tag_ocelot_8021q.c +++ b/net/dsa/tag_ocelot_8021q.c @@ -12,25 +12,39 @@ #include #include "dsa_priv.h" +struct ocelot_8021q_tagger_private { + struct ocelot_8021q_tagger_data data; /* Must be first */ + struct kthread_worker *xmit_worker; +}; + static struct sk_buff *ocelot_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) { + struct ocelot_8021q_tagger_private *priv = dp->ds->tagger_data; + struct ocelot_8021q_tagger_data *data = &priv->data; + void (*xmit_work_fn)(struct kthread_work *work); struct felix_deferred_xmit_work *xmit_work; - struct felix_port *felix_port = dp->priv; + struct kthread_worker *xmit_worker; + + xmit_work_fn = data->xmit_work_fn; + xmit_worker = priv->xmit_worker; + + if (!xmit_work_fn || !xmit_worker) + return NULL; xmit_work = kzalloc(sizeof(*xmit_work), GFP_ATOMIC); if (!xmit_work) return NULL; /* Calls felix_port_deferred_xmit in felix.c */ - kthread_init_work(&xmit_work->work, felix_port->xmit_work_fn); + kthread_init_work(&xmit_work->work, xmit_work_fn); /* Increase refcount so the kfree_skb in dsa_slave_xmit * won't really free the packet. */ xmit_work->dp = dp; xmit_work->skb = skb_get(skb); - kthread_queue_work(felix_port->xmit_worker, &xmit_work->work); + kthread_queue_work(xmit_worker, &xmit_work->work); return NULL; } @@ -67,11 +81,64 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb, return skb; } +static void ocelot_disconnect(struct dsa_switch_tree *dst) +{ + struct ocelot_8021q_tagger_private *priv; + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) { + priv = dp->ds->tagger_data; + + if (!priv) + continue; + + if (priv->xmit_worker) + kthread_destroy_worker(priv->xmit_worker); + + kfree(priv); + dp->ds->tagger_data = NULL; + } +} + +static int ocelot_connect(struct dsa_switch_tree *dst) +{ + struct ocelot_8021q_tagger_private *priv; + struct dsa_port *dp; + int err; + + list_for_each_entry(dp, &dst->ports, list) { + if (dp->ds->tagger_data) + continue; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + err = -ENOMEM; + goto out; + } + + priv->xmit_worker = kthread_create_worker(0, "felix_xmit"); + if (IS_ERR(priv->xmit_worker)) { + err = PTR_ERR(priv->xmit_worker); + goto out; + } + + dp->ds->tagger_data = priv; + } + + return 0; + +out: + ocelot_disconnect(dst); + return err; +} + static const struct dsa_device_ops ocelot_8021q_netdev_ops = { .name = "ocelot-8021q", .proto = DSA_TAG_PROTO_OCELOT_8021Q, .xmit = ocelot_xmit, .rcv = ocelot_rcv, + .connect = ocelot_connect, + .disconnect = ocelot_disconnect, .needed_headroom = VLAN_HLEN, .promisc_on_master = true, }; -- cgit v1.2.3-59-g8ed1b From d38049bbe7601f38d598f5da5ff09980483b290a Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:40 +0200 Subject: net: dsa: sja1105: bring deferred xmit implementation in line with ocelot-8021q When the ocelot-8021q driver was converted to deferred xmit as part of commit 8d5f7954b7c8 ("net: dsa: felix: break at first CPU port during init and teardown"), the deferred implementation was deliberately made subtly different from what sja1105 has. The implementation differences lied on the following observations: - There might be a race between these two lines in tag_sja1105.c: skb_queue_tail(&sp->xmit_queue, skb_get(skb)); kthread_queue_work(sp->xmit_worker, &sp->xmit_work); and the skb dequeue logic in sja1105_port_deferred_xmit(). For example, the xmit_work might be already queued, however the work item has just finished walking through the skb queue. Because we don't check the return code from kthread_queue_work, we don't do anything if the work item is already queued. However, nobody will take that skb and send it, at least until the next timestampable skb is sent. This creates additional (and avoidable) TX timestamping latency. To close that race, what the ocelot-8021q driver does is it doesn't keep a single work item per port, and a skb timestamping queue, but rather dynamically allocates a work item per packet. - It is also unnecessary to have more than one kthread that does the work. So delete the per-port kthread allocations and replace them with a single kthread which is global to the switch. This change brings the two implementations in line by applying those observations to the sja1105 driver as well. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105_main.c | 77 ++++++++++++++-------------------- include/linux/dsa/sja1105.h | 11 +++-- net/dsa/tag_sja1105.c | 21 ++++++++-- 3 files changed, 57 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index f7c88da377e4..c21822bb6834 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2675,10 +2675,8 @@ static int sja1105_mgmt_xmit(struct dsa_switch *ds, int port, int slot, return NETDEV_TX_OK; } -#define work_to_port(work) \ - container_of((work), struct sja1105_port, xmit_work) -#define tagger_to_sja1105(t) \ - container_of((t), struct sja1105_private, tagger_data) +#define work_to_xmit_work(w) \ + container_of((w), struct sja1105_deferred_xmit_work, work) /* Deferred work is unfortunately necessary because setting up the management * route cannot be done from atomit context (SPI transfer takes a sleepable @@ -2686,25 +2684,25 @@ static int sja1105_mgmt_xmit(struct dsa_switch *ds, int port, int slot, */ static void sja1105_port_deferred_xmit(struct kthread_work *work) { - struct sja1105_port *sp = work_to_port(work); - struct sja1105_tagger_data *tagger_data = sp->data; - struct sja1105_private *priv = tagger_to_sja1105(tagger_data); - int port = sp - priv->ports; - struct sk_buff *skb; + struct sja1105_deferred_xmit_work *xmit_work = work_to_xmit_work(work); + struct sk_buff *clone, *skb = xmit_work->skb; + struct dsa_switch *ds = xmit_work->dp->ds; + struct sja1105_private *priv = ds->priv; + int port = xmit_work->dp->index; - while ((skb = skb_dequeue(&sp->xmit_queue)) != NULL) { - struct sk_buff *clone = SJA1105_SKB_CB(skb)->clone; + clone = SJA1105_SKB_CB(skb)->clone; - mutex_lock(&priv->mgmt_lock); + mutex_lock(&priv->mgmt_lock); - sja1105_mgmt_xmit(priv->ds, port, 0, skb, !!clone); + sja1105_mgmt_xmit(ds, port, 0, skb, !!clone); - /* The clone, if there, was made by dsa_skb_tx_timestamp */ - if (clone) - sja1105_ptp_txtstamp_skb(priv->ds, port, clone); + /* The clone, if there, was made by dsa_skb_tx_timestamp */ + if (clone) + sja1105_ptp_txtstamp_skb(ds, port, clone); - mutex_unlock(&priv->mgmt_lock); - } + mutex_unlock(&priv->mgmt_lock); + + kfree(xmit_work); } /* The MAXAGE setting belongs to the L2 Forwarding Parameters table, @@ -3009,54 +3007,43 @@ static int sja1105_port_bridge_flags(struct dsa_switch *ds, int port, static void sja1105_teardown_ports(struct sja1105_private *priv) { - struct dsa_switch *ds = priv->ds; - int port; - - for (port = 0; port < ds->num_ports; port++) { - struct sja1105_port *sp = &priv->ports[port]; + struct sja1105_tagger_data *tagger_data = &priv->tagger_data; - if (sp->xmit_worker) - kthread_destroy_worker(sp->xmit_worker); - } + kthread_destroy_worker(tagger_data->xmit_worker); } static int sja1105_setup_ports(struct sja1105_private *priv) { struct sja1105_tagger_data *tagger_data = &priv->tagger_data; struct dsa_switch *ds = priv->ds; - int port, rc; + struct kthread_worker *worker; + int port; + + worker = kthread_create_worker(0, "dsa%d:%d_xmit", ds->dst->index, + ds->index); + if (IS_ERR(worker)) { + dev_err(ds->dev, + "failed to create deferred xmit thread: %pe\n", + worker); + return PTR_ERR(worker); + } + + tagger_data->xmit_worker = worker; + tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; /* Connections between dsa_port and sja1105_port */ for (port = 0; port < ds->num_ports; port++) { struct sja1105_port *sp = &priv->ports[port]; struct dsa_port *dp = dsa_to_port(ds, port); - struct kthread_worker *worker; - struct net_device *slave; if (!dsa_port_is_user(dp)) continue; dp->priv = sp; sp->data = tagger_data; - slave = dp->slave; - kthread_init_work(&sp->xmit_work, sja1105_port_deferred_xmit); - worker = kthread_create_worker(0, "%s_xmit", slave->name); - if (IS_ERR(worker)) { - rc = PTR_ERR(worker); - dev_err(ds->dev, - "failed to create deferred xmit thread: %d\n", - rc); - goto out_destroy_workers; - } - sp->xmit_worker = worker; - skb_queue_head_init(&sp->xmit_queue); } return 0; - -out_destroy_workers: - sja1105_teardown_ports(priv); - return rc; } /* The programming model for the SJA1105 switch is "all-at-once" via static diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index e6c78be40bde..acd9d2afccab 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -37,6 +37,12 @@ #define SJA1105_HWTS_RX_EN 0 +struct sja1105_deferred_xmit_work { + struct dsa_port *dp; + struct sk_buff *skb; + struct kthread_work work; +}; + /* Global tagger data: each struct sja1105_port has a reference to * the structure defined in struct sja1105_private. */ @@ -52,6 +58,8 @@ struct sja1105_tagger_data { * 2-step TX timestamps */ struct sk_buff_head skb_txtstamp_queue; + struct kthread_worker *xmit_worker; + void (*xmit_work_fn)(struct kthread_work *work); }; struct sja1105_skb_cb { @@ -65,9 +73,6 @@ struct sja1105_skb_cb { ((struct sja1105_skb_cb *)((skb)->cb)) struct sja1105_port { - struct kthread_worker *xmit_worker; - struct kthread_work xmit_work; - struct sk_buff_head xmit_queue; struct sja1105_tagger_data *data; bool hwts_tx_en; }; diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index 6c293c2a3008..7008952b6c1d 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -125,16 +125,29 @@ static inline bool sja1105_is_meta_frame(const struct sk_buff *skb) static struct sk_buff *sja1105_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) { + void (*xmit_work_fn)(struct kthread_work *work); + struct sja1105_deferred_xmit_work *xmit_work; struct sja1105_port *sp = dp->priv; + struct kthread_worker *xmit_worker; - if (!dsa_port_is_sja1105(dp)) - return skb; + xmit_work_fn = sp->data->xmit_work_fn; + xmit_worker = sp->data->xmit_worker; + + if (!xmit_work_fn || !xmit_worker) + return NULL; + xmit_work = kzalloc(sizeof(*xmit_work), GFP_ATOMIC); + if (!xmit_work) + return NULL; + + kthread_init_work(&xmit_work->work, xmit_work_fn); /* Increase refcount so the kfree_skb in dsa_slave_xmit * won't really free the packet. */ - skb_queue_tail(&sp->xmit_queue, skb_get(skb)); - kthread_queue_work(sp->xmit_worker, &sp->xmit_work); + xmit_work->dp = dp; + xmit_work->skb = skb_get(skb); + + kthread_queue_work(xmit_worker, &xmit_work->work); return NULL; } -- cgit v1.2.3-59-g8ed1b From 6f6770ab1ce2b56619264ec6be0b62f05564dcf6 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:41 +0200 Subject: net: dsa: sja1105: remove hwts_tx_en from tagger data This tagger property is in fact not used at all by the tagger, only by the switch driver. Therefore it makes sense to be moved to sja1105_private. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105.h | 1 + drivers/net/dsa/sja1105/sja1105_ptp.c | 9 ++++----- include/linux/dsa/sja1105.h | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105.h b/drivers/net/dsa/sja1105/sja1105.h index 21dba16af097..b0612c763ec0 100644 --- a/drivers/net/dsa/sja1105/sja1105.h +++ b/drivers/net/dsa/sja1105/sja1105.h @@ -249,6 +249,7 @@ struct sja1105_private { bool fixed_link[SJA1105_MAX_NUM_PORTS]; unsigned long ucast_egress_floods; unsigned long bcast_egress_floods; + unsigned long hwts_tx_en; const struct sja1105_info *info; size_t max_xfer_len; struct spi_device *spidev; diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index 54396992a919..b97bd4d948f5 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -98,10 +98,10 @@ int sja1105_hwtstamp_set(struct dsa_switch *ds, int port, struct ifreq *ifr) switch (config.tx_type) { case HWTSTAMP_TX_OFF: - priv->ports[port].hwts_tx_en = false; + priv->hwts_tx_en &= ~BIT(port); break; case HWTSTAMP_TX_ON: - priv->ports[port].hwts_tx_en = true; + priv->hwts_tx_en |= BIT(port); break; default: return -ERANGE; @@ -140,7 +140,7 @@ int sja1105_hwtstamp_get(struct dsa_switch *ds, int port, struct ifreq *ifr) struct hwtstamp_config config; config.flags = 0; - if (priv->ports[port].hwts_tx_en) + if (priv->hwts_tx_en & BIT(port)) config.tx_type = HWTSTAMP_TX_ON; else config.tx_type = HWTSTAMP_TX_OFF; @@ -486,10 +486,9 @@ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) void sja1105_port_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) { struct sja1105_private *priv = ds->priv; - struct sja1105_port *sp = &priv->ports[port]; struct sk_buff *clone; - if (!sp->hwts_tx_en) + if (!(priv->hwts_tx_en & BIT(port))) return; clone = skb_clone_sk(skb); diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index acd9d2afccab..32a8a1344cf6 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -74,7 +74,6 @@ struct sja1105_skb_cb { struct sja1105_port { struct sja1105_tagger_data *data; - bool hwts_tx_en; }; /* Timestamps are in units of 8 ns clock ticks (equivalent to -- cgit v1.2.3-59-g8ed1b From bfcf1425222008e7390c0784b0f3bb7b497fccaa Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:42 +0200 Subject: net: dsa: sja1105: make dp->priv point directly to sja1105_tagger_data The design of the sja1105 tagger dp->priv is that each port has a separate struct sja1105_port, and the sp->data pointer points to a common struct sja1105_tagger_data. We have removed all per-port members accessible by the tagger, and now only struct sja1105_tagger_data remains. Make dp->priv point directly to this. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105.h | 1 - drivers/net/dsa/sja1105/sja1105_main.c | 15 +++-------- drivers/net/dsa/sja1105/sja1105_ptp.c | 14 +++++----- include/linux/dsa/sja1105.h | 8 +----- net/dsa/tag_sja1105.c | 48 ++++++++++++++++++---------------- 5 files changed, 37 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105.h b/drivers/net/dsa/sja1105/sja1105.h index b0612c763ec0..6ef6fb4f30e6 100644 --- a/drivers/net/dsa/sja1105/sja1105.h +++ b/drivers/net/dsa/sja1105/sja1105.h @@ -257,7 +257,6 @@ struct sja1105_private { u16 bridge_pvid[SJA1105_MAX_NUM_PORTS]; u16 tag_8021q_pvid[SJA1105_MAX_NUM_PORTS]; struct sja1105_flow_block flow_block; - struct sja1105_port ports[SJA1105_MAX_NUM_PORTS]; /* Serializes transmission of management frames so that * the switch doesn't confuse them with one another. */ diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index c21822bb6834..e7a6cc7f681c 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -3017,7 +3017,7 @@ static int sja1105_setup_ports(struct sja1105_private *priv) struct sja1105_tagger_data *tagger_data = &priv->tagger_data; struct dsa_switch *ds = priv->ds; struct kthread_worker *worker; - int port; + struct dsa_port *dp; worker = kthread_create_worker(0, "dsa%d:%d_xmit", ds->dst->index, ds->index); @@ -3031,17 +3031,8 @@ static int sja1105_setup_ports(struct sja1105_private *priv) tagger_data->xmit_worker = worker; tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; - /* Connections between dsa_port and sja1105_port */ - for (port = 0; port < ds->num_ports; port++) { - struct sja1105_port *sp = &priv->ports[port]; - struct dsa_port *dp = dsa_to_port(ds, port); - - if (!dsa_port_is_user(dp)) - continue; - - dp->priv = sp; - sp->data = tagger_data; - } + dsa_switch_for_each_user_port(dp, ds) + dp->priv = tagger_data; return 0; } diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index b97bd4d948f5..9077067328c2 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -461,22 +461,24 @@ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) { struct sk_buff *clone = SJA1105_SKB_CB(skb)->clone; struct sja1105_private *priv = ds->priv; - struct sja1105_port *sp = &priv->ports[port]; + struct sja1105_tagger_data *tagger_data; u8 ts_id; + tagger_data = &priv->tagger_data; + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; - spin_lock(&sp->data->meta_lock); + spin_lock(&tagger_data->meta_lock); - ts_id = sp->data->ts_id; + ts_id = tagger_data->ts_id; /* Deal automatically with 8-bit wraparound */ - sp->data->ts_id++; + tagger_data->ts_id++; SJA1105_SKB_CB(clone)->ts_id = ts_id; - spin_unlock(&sp->data->meta_lock); + spin_unlock(&tagger_data->meta_lock); - skb_queue_tail(&sp->data->skb_txtstamp_queue, clone); + skb_queue_tail(&tagger_data->skb_txtstamp_queue, clone); } /* Called from dsa_skb_tx_timestamp. This callback is just to clone diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index 32a8a1344cf6..1dda9cce85d9 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -43,9 +43,7 @@ struct sja1105_deferred_xmit_work { struct kthread_work work; }; -/* Global tagger data: each struct sja1105_port has a reference to - * the structure defined in struct sja1105_private. - */ +/* Global tagger data */ struct sja1105_tagger_data { struct sk_buff *stampable_skb; /* Protects concurrent access to the meta state machine @@ -72,10 +70,6 @@ struct sja1105_skb_cb { #define SJA1105_SKB_CB(skb) \ ((struct sja1105_skb_cb *)((skb)->cb)) -struct sja1105_port { - struct sja1105_tagger_data *data; -}; - /* Timestamps are in units of 8 ns clock ticks (equivalent to * a fixed 125 MHz clock). */ diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index 7008952b6c1d..fc2af71b965c 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -125,13 +125,13 @@ static inline bool sja1105_is_meta_frame(const struct sk_buff *skb) static struct sk_buff *sja1105_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) { + struct sja1105_tagger_data *tagger_data = dp->priv; void (*xmit_work_fn)(struct kthread_work *work); struct sja1105_deferred_xmit_work *xmit_work; - struct sja1105_port *sp = dp->priv; struct kthread_worker *xmit_worker; - xmit_work_fn = sp->data->xmit_work_fn; - xmit_worker = sp->data->xmit_worker; + xmit_work_fn = tagger_data->xmit_work_fn; + xmit_worker = tagger_data->xmit_worker; if (!xmit_work_fn || !xmit_worker) return NULL; @@ -368,32 +368,32 @@ static struct sk_buff */ if (is_link_local) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_port *sp = dp->priv; + struct sja1105_tagger_data *tagger_data = dp->priv; if (unlikely(!dsa_port_is_sja1105(dp))) return skb; - if (!test_bit(SJA1105_HWTS_RX_EN, &sp->data->state)) + if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) /* Do normal processing. */ return skb; - spin_lock(&sp->data->meta_lock); + spin_lock(&tagger_data->meta_lock); /* Was this a link-local frame instead of the meta * that we were expecting? */ - if (sp->data->stampable_skb) { + if (tagger_data->stampable_skb) { dev_err_ratelimited(dp->ds->dev, "Expected meta frame, is %12llx " "in the DSA master multicast filter?\n", SJA1105_META_DMAC); - kfree_skb(sp->data->stampable_skb); + kfree_skb(tagger_data->stampable_skb); } /* Hold a reference to avoid dsa_switch_rcv * from freeing the skb. */ - sp->data->stampable_skb = skb_get(skb); - spin_unlock(&sp->data->meta_lock); + tagger_data->stampable_skb = skb_get(skb); + spin_unlock(&tagger_data->meta_lock); /* Tell DSA we got nothing */ return NULL; @@ -406,7 +406,7 @@ static struct sk_buff */ } else if (is_meta) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_port *sp = dp->priv; + struct sja1105_tagger_data *tagger_data = dp->priv; struct sk_buff *stampable_skb; if (unlikely(!dsa_port_is_sja1105(dp))) @@ -415,13 +415,13 @@ static struct sk_buff /* Drop the meta frame if we're not in the right state * to process it. */ - if (!test_bit(SJA1105_HWTS_RX_EN, &sp->data->state)) + if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) return NULL; - spin_lock(&sp->data->meta_lock); + spin_lock(&tagger_data->meta_lock); - stampable_skb = sp->data->stampable_skb; - sp->data->stampable_skb = NULL; + stampable_skb = tagger_data->stampable_skb; + tagger_data->stampable_skb = NULL; /* Was this a meta frame instead of the link-local * that we were expecting? @@ -429,14 +429,14 @@ static struct sk_buff if (!stampable_skb) { dev_err_ratelimited(dp->ds->dev, "Unexpected meta frame\n"); - spin_unlock(&sp->data->meta_lock); + spin_unlock(&tagger_data->meta_lock); return NULL; } if (stampable_skb->dev != skb->dev) { dev_err_ratelimited(dp->ds->dev, "Meta frame on wrong port\n"); - spin_unlock(&sp->data->meta_lock); + spin_unlock(&tagger_data->meta_lock); return NULL; } @@ -447,7 +447,7 @@ static struct sk_buff skb = stampable_skb; sja1105_transfer_meta(skb, meta); - spin_unlock(&sp->data->meta_lock); + spin_unlock(&tagger_data->meta_lock); } return skb; @@ -545,8 +545,8 @@ static void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, { struct sk_buff *skb, *skb_tmp, *skb_match = NULL; struct dsa_port *dp = dsa_to_port(ds, port); + struct sja1105_tagger_data *tagger_data; struct skb_shared_hwtstamps shwt = {0}; - struct sja1105_port *sp = dp->priv; if (!dsa_port_is_sja1105(dp)) return; @@ -555,19 +555,21 @@ static void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, if (dir == SJA1110_META_TSTAMP_RX) return; - spin_lock(&sp->data->skb_txtstamp_queue.lock); + tagger_data = dp->priv; - skb_queue_walk_safe(&sp->data->skb_txtstamp_queue, skb, skb_tmp) { + spin_lock(&tagger_data->skb_txtstamp_queue.lock); + + skb_queue_walk_safe(&tagger_data->skb_txtstamp_queue, skb, skb_tmp) { if (SJA1105_SKB_CB(skb)->ts_id != ts_id) continue; - __skb_unlink(skb, &sp->data->skb_txtstamp_queue); + __skb_unlink(skb, &tagger_data->skb_txtstamp_queue); skb_match = skb; break; } - spin_unlock(&sp->data->skb_txtstamp_queue.lock); + spin_unlock(&tagger_data->skb_txtstamp_queue.lock); if (WARN_ON(!skb_match)) return; -- cgit v1.2.3-59-g8ed1b From 22ee9f8e4011fb8a6d75dc2f9a43360d4f400235 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:43 +0200 Subject: net: dsa: sja1105: move ts_id from sja1105_tagger_data The TX timestamp ID is incremented by the SJA1110 PTP timestamping callback (->port_tx_timestamp) for every packet, when cloning it. It isn't used by the tagger at all, even though it sits inside the struct sja1105_tagger_data. Also, serialization to this structure is currently done through tagger_data->meta_lock, which is a cheap hack because the meta_lock isn't used for anything else on SJA1110 (sja1105_rcv_meta_state_machine isn't called). This change moves ts_id from sja1105_tagger_data to sja1105_private and introduces a dedicated spinlock for it, also in sja1105_private. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105.h | 3 +++ drivers/net/dsa/sja1105/sja1105_main.c | 1 + drivers/net/dsa/sja1105/sja1105_ptp.c | 8 ++++---- include/linux/dsa/sja1105.h | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105.h b/drivers/net/dsa/sja1105/sja1105.h index 6ef6fb4f30e6..850a7d3e69bb 100644 --- a/drivers/net/dsa/sja1105/sja1105.h +++ b/drivers/net/dsa/sja1105/sja1105.h @@ -261,6 +261,9 @@ struct sja1105_private { * the switch doesn't confuse them with one another. */ struct mutex mgmt_lock; + /* PTP two-step TX timestamp ID, and its serialization lock */ + spinlock_t ts_id_lock; + u8 ts_id; /* Serializes access to the dynamic config interface */ struct mutex dynamic_config_lock; struct devlink_region **regions; diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index e7a6cc7f681c..880f28ea184f 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -3348,6 +3348,7 @@ static int sja1105_probe(struct spi_device *spi) mutex_init(&priv->ptp_data.lock); mutex_init(&priv->dynamic_config_lock); mutex_init(&priv->mgmt_lock); + spin_lock_init(&priv->ts_id_lock); rc = sja1105_parse_dt(priv); if (rc < 0) { diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index 9077067328c2..0904ab10bd2f 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -468,15 +468,15 @@ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; - spin_lock(&tagger_data->meta_lock); + spin_lock(&priv->ts_id_lock); - ts_id = tagger_data->ts_id; + ts_id = priv->ts_id; /* Deal automatically with 8-bit wraparound */ - tagger_data->ts_id++; + priv->ts_id++; SJA1105_SKB_CB(clone)->ts_id = ts_id; - spin_unlock(&tagger_data->meta_lock); + spin_unlock(&priv->ts_id_lock); skb_queue_tail(&tagger_data->skb_txtstamp_queue, clone); } diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index 1dda9cce85d9..d8ee53085c09 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -51,7 +51,6 @@ struct sja1105_tagger_data { */ spinlock_t meta_lock; unsigned long state; - u8 ts_id; /* Used on SJA1110 where meta frames are generated only for * 2-step TX timestamps */ -- cgit v1.2.3-59-g8ed1b From c79e84866d2ac637fce921a28288f214e91d662b Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:44 +0200 Subject: net: dsa: tag_sja1105: convert to tagger-owned data Currently, struct sja1105_tagger_data is a part of struct sja1105_private, and is used by the sja1105 driver to populate dp->priv. With the movement towards tagger-owned storage, the sja1105 driver should not be the owner of this memory. This change implements the connection between the sja1105 switch driver and its tagging protocol, which means that sja1105_tagger_data no longer stays in dp->priv but in ds->tagger_data, and that the sja1105 driver now only populates the sja1105_port_deferred_xmit callback pointer. The kthread worker is now the responsibility of the tagger. The sja1105 driver also alters the tagger's state some more, especially with regard to the PTP RX timestamping state. This will be fixed up a bit in further changes. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105.h | 1 - drivers/net/dsa/sja1105/sja1105_main.c | 54 ++++++-------------- drivers/net/dsa/sja1105/sja1105_ptp.c | 35 ++++++------- include/linux/dsa/sja1105.h | 7 ++- net/dsa/tag_sja1105.c | 90 +++++++++++++++++++++++++++------- 5 files changed, 110 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105.h b/drivers/net/dsa/sja1105/sja1105.h index 850a7d3e69bb..9ba2ec2b966d 100644 --- a/drivers/net/dsa/sja1105/sja1105.h +++ b/drivers/net/dsa/sja1105/sja1105.h @@ -272,7 +272,6 @@ struct sja1105_private { struct mii_bus *mdio_base_tx; struct mii_bus *mdio_pcs; struct dw_xpcs *xpcs[SJA1105_MAX_NUM_PORTS]; - struct sja1105_tagger_data tagger_data; struct sja1105_ptp_data ptp_data; struct sja1105_tas_data tas_data; }; diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 880f28ea184f..4f5ea5d6a623 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2705,6 +2705,21 @@ static void sja1105_port_deferred_xmit(struct kthread_work *work) kfree(xmit_work); } +static int sja1105_connect_tag_protocol(struct dsa_switch *ds, + enum dsa_tag_protocol proto) +{ + struct sja1105_tagger_data *tagger_data; + + switch (proto) { + case DSA_TAG_PROTO_SJA1105: + tagger_data = sja1105_tagger_data(ds); + tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; + return 0; + default: + return -EPROTONOSUPPORT; + } +} + /* The MAXAGE setting belongs to the L2 Forwarding Parameters table, * which cannot be reconfigured at runtime. So a switch reset is required. */ @@ -3005,38 +3020,6 @@ static int sja1105_port_bridge_flags(struct dsa_switch *ds, int port, return 0; } -static void sja1105_teardown_ports(struct sja1105_private *priv) -{ - struct sja1105_tagger_data *tagger_data = &priv->tagger_data; - - kthread_destroy_worker(tagger_data->xmit_worker); -} - -static int sja1105_setup_ports(struct sja1105_private *priv) -{ - struct sja1105_tagger_data *tagger_data = &priv->tagger_data; - struct dsa_switch *ds = priv->ds; - struct kthread_worker *worker; - struct dsa_port *dp; - - worker = kthread_create_worker(0, "dsa%d:%d_xmit", ds->dst->index, - ds->index); - if (IS_ERR(worker)) { - dev_err(ds->dev, - "failed to create deferred xmit thread: %pe\n", - worker); - return PTR_ERR(worker); - } - - tagger_data->xmit_worker = worker; - tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; - - dsa_switch_for_each_user_port(dp, ds) - dp->priv = tagger_data; - - return 0; -} - /* The programming model for the SJA1105 switch is "all-at-once" via static * configuration tables. Some of these can be dynamically modified at runtime, * but not the xMII mode parameters table. @@ -3082,10 +3065,6 @@ static int sja1105_setup(struct dsa_switch *ds) } } - rc = sja1105_setup_ports(priv); - if (rc) - goto out_static_config_free; - sja1105_tas_setup(ds); sja1105_flower_setup(ds); @@ -3142,7 +3121,6 @@ out_ptp_clock_unregister: out_flower_teardown: sja1105_flower_teardown(ds); sja1105_tas_teardown(ds); - sja1105_teardown_ports(priv); out_static_config_free: sja1105_static_config_free(&priv->static_config); @@ -3162,12 +3140,12 @@ static void sja1105_teardown(struct dsa_switch *ds) sja1105_ptp_clock_unregister(ds); sja1105_flower_teardown(ds); sja1105_tas_teardown(ds); - sja1105_teardown_ports(priv); sja1105_static_config_free(&priv->static_config); } static const struct dsa_switch_ops sja1105_switch_ops = { .get_tag_protocol = sja1105_get_tag_protocol, + .connect_tag_protocol = sja1105_connect_tag_protocol, .setup = sja1105_setup, .teardown = sja1105_teardown, .set_ageing_time = sja1105_set_ageing_time, diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index 0904ab10bd2f..b34e4674e217 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -58,13 +58,13 @@ enum sja1105_ptp_clk_mode { #define ptp_data_to_sja1105(d) \ container_of((d), struct sja1105_private, ptp_data) -/* Must be called only with priv->tagger_data.state bit +/* Must be called only with the tagger_data->state bit * SJA1105_HWTS_RX_EN cleared */ static int sja1105_change_rxtstamping(struct sja1105_private *priv, + struct sja1105_tagger_data *tagger_data, bool on) { - struct sja1105_tagger_data *tagger_data = &priv->tagger_data; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; struct sja1105_general_params_entry *general_params; struct sja1105_table *table; @@ -75,9 +75,9 @@ static int sja1105_change_rxtstamping(struct sja1105_private *priv, general_params->send_meta0 = on; /* Initialize the meta state machine to a known state */ - if (priv->tagger_data.stampable_skb) { - kfree_skb(priv->tagger_data.stampable_skb); - priv->tagger_data.stampable_skb = NULL; + if (tagger_data->stampable_skb) { + kfree_skb(tagger_data->stampable_skb); + tagger_data->stampable_skb = NULL; } ptp_cancel_worker_sync(ptp_data->clock); skb_queue_purge(&tagger_data->skb_txtstamp_queue); @@ -88,6 +88,7 @@ static int sja1105_change_rxtstamping(struct sja1105_private *priv, int sja1105_hwtstamp_set(struct dsa_switch *ds, int port, struct ifreq *ifr) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sja1105_private *priv = ds->priv; struct hwtstamp_config config; bool rx_on; @@ -116,17 +117,17 @@ int sja1105_hwtstamp_set(struct dsa_switch *ds, int port, struct ifreq *ifr) break; } - if (rx_on != test_bit(SJA1105_HWTS_RX_EN, &priv->tagger_data.state)) { - clear_bit(SJA1105_HWTS_RX_EN, &priv->tagger_data.state); + if (rx_on != test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) { + clear_bit(SJA1105_HWTS_RX_EN, &tagger_data->state); - rc = sja1105_change_rxtstamping(priv, rx_on); + rc = sja1105_change_rxtstamping(priv, tagger_data, rx_on); if (rc < 0) { dev_err(ds->dev, "Failed to change RX timestamping: %d\n", rc); return rc; } if (rx_on) - set_bit(SJA1105_HWTS_RX_EN, &priv->tagger_data.state); + set_bit(SJA1105_HWTS_RX_EN, &tagger_data->state); } if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) @@ -136,6 +137,7 @@ int sja1105_hwtstamp_set(struct dsa_switch *ds, int port, struct ifreq *ifr) int sja1105_hwtstamp_get(struct dsa_switch *ds, int port, struct ifreq *ifr) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sja1105_private *priv = ds->priv; struct hwtstamp_config config; @@ -144,7 +146,7 @@ int sja1105_hwtstamp_get(struct dsa_switch *ds, int port, struct ifreq *ifr) config.tx_type = HWTSTAMP_TX_ON; else config.tx_type = HWTSTAMP_TX_OFF; - if (test_bit(SJA1105_HWTS_RX_EN, &priv->tagger_data.state)) + if (test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; else config.rx_filter = HWTSTAMP_FILTER_NONE; @@ -417,10 +419,11 @@ static long sja1105_rxtstamp_work(struct ptp_clock_info *ptp) bool sja1105_rxtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sja1105_private *priv = ds->priv; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; - if (!test_bit(SJA1105_HWTS_RX_EN, &priv->tagger_data.state)) + if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) return false; /* We need to read the full PTP clock to reconstruct the Rx @@ -459,13 +462,11 @@ bool sja1105_port_rxtstamp(struct dsa_switch *ds, int port, */ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sk_buff *clone = SJA1105_SKB_CB(skb)->clone; struct sja1105_private *priv = ds->priv; - struct sja1105_tagger_data *tagger_data; u8 ts_id; - tagger_data = &priv->tagger_data; - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; spin_lock(&priv->ts_id_lock); @@ -897,7 +898,6 @@ static struct ptp_pin_desc sja1105_ptp_pin = { int sja1105_ptp_clock_register(struct dsa_switch *ds) { struct sja1105_private *priv = ds->priv; - struct sja1105_tagger_data *tagger_data = &priv->tagger_data; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; ptp_data->caps = (struct ptp_clock_info) { @@ -919,9 +919,6 @@ int sja1105_ptp_clock_register(struct dsa_switch *ds) /* Only used on SJA1105 */ skb_queue_head_init(&ptp_data->skb_rxtstamp_queue); - /* Only used on SJA1110 */ - skb_queue_head_init(&tagger_data->skb_txtstamp_queue); - spin_lock_init(&tagger_data->meta_lock); ptp_data->clock = ptp_clock_register(&ptp_data->caps, ds->dev); if (IS_ERR_OR_NULL(ptp_data->clock)) @@ -937,8 +934,8 @@ int sja1105_ptp_clock_register(struct dsa_switch *ds) void sja1105_ptp_clock_unregister(struct dsa_switch *ds) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sja1105_private *priv = ds->priv; - struct sja1105_tagger_data *tagger_data = &priv->tagger_data; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; if (IS_ERR_OR_NULL(ptp_data->clock)) diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index d8ee53085c09..9f7d42cbbc08 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -84,9 +84,12 @@ static inline s64 sja1105_ticks_to_ns(s64 ticks) return ticks * SJA1105_TICK_NS; } -static inline bool dsa_port_is_sja1105(struct dsa_port *dp) +static inline struct sja1105_tagger_data * +sja1105_tagger_data(struct dsa_switch *ds) { - return true; + BUG_ON(ds->dst->tag_ops->proto != DSA_TAG_PROTO_SJA1105); + + return ds->tagger_data; } #endif /* _NET_DSA_SJA1105_H */ diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index fc2af71b965c..f3c1b31645f5 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -125,7 +125,7 @@ static inline bool sja1105_is_meta_frame(const struct sk_buff *skb) static struct sk_buff *sja1105_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) { - struct sja1105_tagger_data *tagger_data = dp->priv; + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(dp->ds); void (*xmit_work_fn)(struct kthread_work *work); struct sja1105_deferred_xmit_work *xmit_work; struct kthread_worker *xmit_worker; @@ -368,10 +368,10 @@ static struct sk_buff */ if (is_link_local) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_tagger_data *tagger_data = dp->priv; + struct sja1105_tagger_data *tagger_data; + struct dsa_switch *ds = dp->ds; - if (unlikely(!dsa_port_is_sja1105(dp))) - return skb; + tagger_data = sja1105_tagger_data(ds); if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) /* Do normal processing. */ @@ -382,7 +382,7 @@ static struct sk_buff * that we were expecting? */ if (tagger_data->stampable_skb) { - dev_err_ratelimited(dp->ds->dev, + dev_err_ratelimited(ds->dev, "Expected meta frame, is %12llx " "in the DSA master multicast filter?\n", SJA1105_META_DMAC); @@ -406,11 +406,11 @@ static struct sk_buff */ } else if (is_meta) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_tagger_data *tagger_data = dp->priv; + struct sja1105_tagger_data *tagger_data; + struct dsa_switch *ds = dp->ds; struct sk_buff *stampable_skb; - if (unlikely(!dsa_port_is_sja1105(dp))) - return skb; + tagger_data = sja1105_tagger_data(ds); /* Drop the meta frame if we're not in the right state * to process it. @@ -427,14 +427,14 @@ static struct sk_buff * that we were expecting? */ if (!stampable_skb) { - dev_err_ratelimited(dp->ds->dev, + dev_err_ratelimited(ds->dev, "Unexpected meta frame\n"); spin_unlock(&tagger_data->meta_lock); return NULL; } if (stampable_skb->dev != skb->dev) { - dev_err_ratelimited(dp->ds->dev, + dev_err_ratelimited(ds->dev, "Meta frame on wrong port\n"); spin_unlock(&tagger_data->meta_lock); return NULL; @@ -543,20 +543,14 @@ static void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, u8 ts_id, enum sja1110_meta_tstamp dir, u64 tstamp) { + struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sk_buff *skb, *skb_tmp, *skb_match = NULL; - struct dsa_port *dp = dsa_to_port(ds, port); - struct sja1105_tagger_data *tagger_data; struct skb_shared_hwtstamps shwt = {0}; - if (!dsa_port_is_sja1105(dp)) - return; - /* We don't care about RX timestamps on the CPU port */ if (dir == SJA1110_META_TSTAMP_RX) return; - tagger_data = dp->priv; - spin_lock(&tagger_data->skb_txtstamp_queue.lock); skb_queue_walk_safe(&tagger_data->skb_txtstamp_queue, skb, skb_tmp) { @@ -737,11 +731,71 @@ static void sja1110_flow_dissect(const struct sk_buff *skb, __be16 *proto, *proto = ((__be16 *)skb->data)[(VLAN_HLEN / 2) - 1]; } +static void sja1105_disconnect(struct dsa_switch_tree *dst) +{ + struct sja1105_tagger_data *tagger_data; + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) { + tagger_data = dp->ds->tagger_data; + + if (!tagger_data) + continue; + + if (tagger_data->xmit_worker) + kthread_destroy_worker(tagger_data->xmit_worker); + + kfree(tagger_data); + dp->ds->tagger_data = NULL; + } +} + +static int sja1105_connect(struct dsa_switch_tree *dst) +{ + struct sja1105_tagger_data *tagger_data; + struct kthread_worker *xmit_worker; + struct dsa_port *dp; + int err; + + list_for_each_entry(dp, &dst->ports, list) { + if (dp->ds->tagger_data) + continue; + + tagger_data = kzalloc(sizeof(*tagger_data), GFP_KERNEL); + if (!tagger_data) { + err = -ENOMEM; + goto out; + } + + /* Only used on SJA1110 */ + skb_queue_head_init(&tagger_data->skb_txtstamp_queue); + spin_lock_init(&tagger_data->meta_lock); + + xmit_worker = kthread_create_worker(0, "dsa%d:%d_xmit", + dst->index, dp->ds->index); + if (IS_ERR(xmit_worker)) { + err = PTR_ERR(xmit_worker); + goto out; + } + + tagger_data->xmit_worker = xmit_worker; + dp->ds->tagger_data = tagger_data; + } + + return 0; + +out: + sja1105_disconnect(dst); + return err; +} + static const struct dsa_device_ops sja1105_netdev_ops = { .name = "sja1105", .proto = DSA_TAG_PROTO_SJA1105, .xmit = sja1105_xmit, .rcv = sja1105_rcv, + .connect = sja1105_connect, + .disconnect = sja1105_disconnect, .needed_headroom = VLAN_HLEN, .flow_dissect = sja1105_flow_dissect, .promisc_on_master = true, @@ -755,6 +809,8 @@ static const struct dsa_device_ops sja1110_netdev_ops = { .proto = DSA_TAG_PROTO_SJA1110, .xmit = sja1110_xmit, .rcv = sja1110_rcv, + .connect = sja1105_connect, + .disconnect = sja1105_disconnect, .flow_dissect = sja1110_flow_dissect, .needed_headroom = SJA1110_HEADER_LEN + VLAN_HLEN, .needed_tailroom = SJA1110_RX_TRAILER_LEN + SJA1110_MAX_PADDING_LEN, -- cgit v1.2.3-59-g8ed1b From fcbf979a5b4b5784bfb5647ae6190cd5c2ae595d Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:45 +0200 Subject: Revert "net: dsa: move sja1110_process_meta_tstamp inside the tagging protocol driver" This reverts commit 6d709cadfde68dbd12bef12fcced6222226dcb06. The above change was done to avoid calling symbols exported by the switch driver from the tagging protocol driver. With the tagger-owned storage model, we have a new option on our hands, and that is for the switch driver to provide a data consumer handler in the form of a function pointer inside the ->connect_tag_protocol() method. Having a function pointer avoids the problems of the exported symbols approach. By creating a handler for metadata frames holding TX timestamps on SJA1110, we are able to eliminate an skb queue from the tagger data, and replace it with a simple, and stateless, function pointer. This skb queue is now handled exclusively by sja1105_ptp.c, which makes the code easier to follow, as it used to be before the reverted patch. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105_main.c | 1 + drivers/net/dsa/sja1105/sja1105_ptp.c | 44 ++++++++++++++++++++++++++---- drivers/net/dsa/sja1105/sja1105_ptp.h | 24 ++++++++++++++++ include/linux/dsa/sja1105.h | 26 +++++------------- net/dsa/tag_sja1105.c | 50 +++++----------------------------- 5 files changed, 78 insertions(+), 67 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 4f5ea5d6a623..9171fbea588c 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2714,6 +2714,7 @@ static int sja1105_connect_tag_protocol(struct dsa_switch *ds, case DSA_TAG_PROTO_SJA1105: tagger_data = sja1105_tagger_data(ds); tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; + tagger_data->meta_tstamp_handler = sja1110_process_meta_tstamp; return 0; default: return -EPROTONOSUPPORT; diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index b34e4674e217..a9f7e4ae0bb2 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -80,7 +80,7 @@ static int sja1105_change_rxtstamping(struct sja1105_private *priv, tagger_data->stampable_skb = NULL; } ptp_cancel_worker_sync(ptp_data->clock); - skb_queue_purge(&tagger_data->skb_txtstamp_queue); + skb_queue_purge(&ptp_data->skb_txtstamp_queue); skb_queue_purge(&ptp_data->skb_rxtstamp_queue); return sja1105_static_config_reload(priv, SJA1105_RX_HWTSTAMPING); @@ -456,15 +456,48 @@ bool sja1105_port_rxtstamp(struct dsa_switch *ds, int port, return priv->info->rxtstamp(ds, port, skb); } +void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, u8 ts_id, + enum sja1110_meta_tstamp dir, u64 tstamp) +{ + struct sja1105_private *priv = ds->priv; + struct sja1105_ptp_data *ptp_data = &priv->ptp_data; + struct sk_buff *skb, *skb_tmp, *skb_match = NULL; + struct skb_shared_hwtstamps shwt = {0}; + + /* We don't care about RX timestamps on the CPU port */ + if (dir == SJA1110_META_TSTAMP_RX) + return; + + spin_lock(&ptp_data->skb_txtstamp_queue.lock); + + skb_queue_walk_safe(&ptp_data->skb_txtstamp_queue, skb, skb_tmp) { + if (SJA1105_SKB_CB(skb)->ts_id != ts_id) + continue; + + __skb_unlink(skb, &ptp_data->skb_txtstamp_queue); + skb_match = skb; + + break; + } + + spin_unlock(&ptp_data->skb_txtstamp_queue.lock); + + if (WARN_ON(!skb_match)) + return; + + shwt.hwtstamp = ns_to_ktime(sja1105_ticks_to_ns(tstamp)); + skb_complete_tx_timestamp(skb_match, &shwt); +} + /* In addition to cloning the skb which is done by the common * sja1105_port_txtstamp, we need to generate a timestamp ID and save the * packet to the TX timestamping queue. */ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) { - struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sk_buff *clone = SJA1105_SKB_CB(skb)->clone; struct sja1105_private *priv = ds->priv; + struct sja1105_ptp_data *ptp_data = &priv->ptp_data; u8 ts_id; skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; @@ -479,7 +512,7 @@ void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) spin_unlock(&priv->ts_id_lock); - skb_queue_tail(&tagger_data->skb_txtstamp_queue, clone); + skb_queue_tail(&ptp_data->skb_txtstamp_queue, clone); } /* Called from dsa_skb_tx_timestamp. This callback is just to clone @@ -919,6 +952,8 @@ int sja1105_ptp_clock_register(struct dsa_switch *ds) /* Only used on SJA1105 */ skb_queue_head_init(&ptp_data->skb_rxtstamp_queue); + /* Only used on SJA1110 */ + skb_queue_head_init(&ptp_data->skb_txtstamp_queue); ptp_data->clock = ptp_clock_register(&ptp_data->caps, ds->dev); if (IS_ERR_OR_NULL(ptp_data->clock)) @@ -934,7 +969,6 @@ int sja1105_ptp_clock_register(struct dsa_switch *ds) void sja1105_ptp_clock_unregister(struct dsa_switch *ds) { - struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); struct sja1105_private *priv = ds->priv; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; @@ -943,7 +977,7 @@ void sja1105_ptp_clock_unregister(struct dsa_switch *ds) del_timer_sync(&ptp_data->extts_timer); ptp_cancel_worker_sync(ptp_data->clock); - skb_queue_purge(&tagger_data->skb_txtstamp_queue); + skb_queue_purge(&ptp_data->skb_txtstamp_queue); skb_queue_purge(&ptp_data->skb_rxtstamp_queue); ptp_clock_unregister(ptp_data->clock); ptp_data->clock = NULL; diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.h b/drivers/net/dsa/sja1105/sja1105_ptp.h index 3ae6b9fdd492..416461ee95d2 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.h +++ b/drivers/net/dsa/sja1105/sja1105_ptp.h @@ -8,6 +8,21 @@ #if IS_ENABLED(CONFIG_NET_DSA_SJA1105_PTP) +/* Timestamps are in units of 8 ns clock ticks (equivalent to + * a fixed 125 MHz clock). + */ +#define SJA1105_TICK_NS 8 + +static inline s64 ns_to_sja1105_ticks(s64 ns) +{ + return ns / SJA1105_TICK_NS; +} + +static inline s64 sja1105_ticks_to_ns(s64 ticks) +{ + return ticks * SJA1105_TICK_NS; +} + /* Calculate the first base_time in the future that satisfies this * relationship: * @@ -62,6 +77,10 @@ struct sja1105_ptp_data { struct timer_list extts_timer; /* Used only on SJA1105 to reconstruct partial timestamps */ struct sk_buff_head skb_rxtstamp_queue; + /* Used on SJA1110 where meta frames are generated only for + * 2-step TX timestamps + */ + struct sk_buff_head skb_txtstamp_queue; struct ptp_clock_info caps; struct ptp_clock *clock; struct sja1105_ptp_cmd cmd; @@ -112,6 +131,9 @@ bool sja1105_rxtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb); bool sja1110_rxtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb); void sja1110_txtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb); +void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, u8 ts_id, + enum sja1110_meta_tstamp dir, u64 tstamp); + #else struct sja1105_ptp_cmd; @@ -178,6 +200,8 @@ static inline int sja1105_ptp_commit(struct dsa_switch *ds, #define sja1110_rxtstamp NULL #define sja1110_txtstamp NULL +#define sja1110_process_meta_tstamp NULL + #endif /* IS_ENABLED(CONFIG_NET_DSA_SJA1105_PTP) */ #endif /* _SJA1105_PTP_H */ diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index 9f7d42cbbc08..d216211b64f8 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -37,6 +37,11 @@ #define SJA1105_HWTS_RX_EN 0 +enum sja1110_meta_tstamp { + SJA1110_META_TSTAMP_TX = 0, + SJA1110_META_TSTAMP_RX = 1, +}; + struct sja1105_deferred_xmit_work { struct dsa_port *dp; struct sk_buff *skb; @@ -51,12 +56,10 @@ struct sja1105_tagger_data { */ spinlock_t meta_lock; unsigned long state; - /* Used on SJA1110 where meta frames are generated only for - * 2-step TX timestamps - */ - struct sk_buff_head skb_txtstamp_queue; struct kthread_worker *xmit_worker; void (*xmit_work_fn)(struct kthread_work *work); + void (*meta_tstamp_handler)(struct dsa_switch *ds, int port, u8 ts_id, + enum sja1110_meta_tstamp dir, u64 tstamp); }; struct sja1105_skb_cb { @@ -69,21 +72,6 @@ struct sja1105_skb_cb { #define SJA1105_SKB_CB(skb) \ ((struct sja1105_skb_cb *)((skb)->cb)) -/* Timestamps are in units of 8 ns clock ticks (equivalent to - * a fixed 125 MHz clock). - */ -#define SJA1105_TICK_NS 8 - -static inline s64 ns_to_sja1105_ticks(s64 ns) -{ - return ns / SJA1105_TICK_NS; -} - -static inline s64 sja1105_ticks_to_ns(s64 ticks) -{ - return ticks * SJA1105_TICK_NS; -} - static inline struct sja1105_tagger_data * sja1105_tagger_data(struct dsa_switch *ds) { diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index f3c1b31645f5..fe6a6d95bb26 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include "dsa_priv.h" @@ -54,11 +53,6 @@ #define SJA1110_TX_TRAILER_LEN 4 #define SJA1110_MAX_PADDING_LEN 15 -enum sja1110_meta_tstamp { - SJA1110_META_TSTAMP_TX = 0, - SJA1110_META_TSTAMP_RX = 1, -}; - /* Similar to is_link_local_ether_addr(hdr->h_dest) but also covers PTP */ static inline bool sja1105_is_link_local(const struct sk_buff *skb) { @@ -539,44 +533,12 @@ static struct sk_buff *sja1105_rcv(struct sk_buff *skb, is_meta); } -static void sja1110_process_meta_tstamp(struct dsa_switch *ds, int port, - u8 ts_id, enum sja1110_meta_tstamp dir, - u64 tstamp) -{ - struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(ds); - struct sk_buff *skb, *skb_tmp, *skb_match = NULL; - struct skb_shared_hwtstamps shwt = {0}; - - /* We don't care about RX timestamps on the CPU port */ - if (dir == SJA1110_META_TSTAMP_RX) - return; - - spin_lock(&tagger_data->skb_txtstamp_queue.lock); - - skb_queue_walk_safe(&tagger_data->skb_txtstamp_queue, skb, skb_tmp) { - if (SJA1105_SKB_CB(skb)->ts_id != ts_id) - continue; - - __skb_unlink(skb, &tagger_data->skb_txtstamp_queue); - skb_match = skb; - - break; - } - - spin_unlock(&tagger_data->skb_txtstamp_queue.lock); - - if (WARN_ON(!skb_match)) - return; - - shwt.hwtstamp = ns_to_ktime(sja1105_ticks_to_ns(tstamp)); - skb_complete_tx_timestamp(skb_match, &shwt); -} - static struct sk_buff *sja1110_rcv_meta(struct sk_buff *skb, u16 rx_header) { u8 *buf = dsa_etype_header_pos_rx(skb) + SJA1110_HEADER_LEN; int switch_id = SJA1110_RX_HEADER_SWITCH_ID(rx_header); int n_ts = SJA1110_RX_HEADER_N_TS(rx_header); + struct sja1105_tagger_data *tagger_data; struct net_device *master = skb->dev; struct dsa_port *cpu_dp; struct dsa_switch *ds; @@ -590,6 +552,10 @@ static struct sk_buff *sja1110_rcv_meta(struct sk_buff *skb, u16 rx_header) return NULL; } + tagger_data = sja1105_tagger_data(ds); + if (!tagger_data->meta_tstamp_handler) + return NULL; + for (i = 0; i <= n_ts; i++) { u8 ts_id, source_port, dir; u64 tstamp; @@ -599,8 +565,8 @@ static struct sk_buff *sja1110_rcv_meta(struct sk_buff *skb, u16 rx_header) dir = (buf[1] & BIT(3)) >> 3; tstamp = be64_to_cpu(*(__be64 *)(buf + 2)); - sja1110_process_meta_tstamp(ds, source_port, ts_id, dir, - tstamp); + tagger_data->meta_tstamp_handler(ds, source_port, ts_id, dir, + tstamp); buf += SJA1110_META_TSTAMP_SIZE; } @@ -767,8 +733,6 @@ static int sja1105_connect(struct dsa_switch_tree *dst) goto out; } - /* Only used on SJA1110 */ - skb_queue_head_init(&tagger_data->skb_txtstamp_queue); spin_lock_init(&tagger_data->meta_lock); xmit_worker = kthread_create_worker(0, "dsa%d:%d_xmit", -- cgit v1.2.3-59-g8ed1b From 950a419d9de13668f86828394cb242a1f9dece74 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 10 Dec 2021 01:34:46 +0200 Subject: net: dsa: tag_sja1105: split sja1105_tagger_data into private and public sections The sja1105 driver messes with the tagging protocol's state when PTP RX timestamping is enabled/disabled. This is fundamentally necessary because the tagger needs to know what to do when it receives a PTP packet. If RX timestamping is enabled, then a metadata follow-up frame is expected, and this holds the (partial) timestamp. So the tagger plays hide-and-seek with the network stack until it also gets the metadata frame, and then presents a single packet, the timestamped PTP packet. But when RX timestamping isn't enabled, there is no metadata frame expected, so the hide-and-seek game must be turned off and the packet must be delivered right away to the network stack. Considering this, we create a pseudo isolation by devising two tagger methods callable by the switch: one to get the RX timestamping state, and one to set it. Since we can't export symbols between the tagger and the switch driver, these methods are exposed through function pointers. After this change, the public portion of the sja1105_tagger_data contains only function pointers. Signed-off-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105_ptp.c | 22 +++---- include/linux/dsa/sja1105.h | 13 ++-- net/dsa/tag_sja1105.c | 109 ++++++++++++++++++++++++---------- 3 files changed, 91 insertions(+), 53 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c index a9f7e4ae0bb2..be3068a935af 100644 --- a/drivers/net/dsa/sja1105/sja1105_ptp.c +++ b/drivers/net/dsa/sja1105/sja1105_ptp.c @@ -58,11 +58,10 @@ enum sja1105_ptp_clk_mode { #define ptp_data_to_sja1105(d) \ container_of((d), struct sja1105_private, ptp_data) -/* Must be called only with the tagger_data->state bit - * SJA1105_HWTS_RX_EN cleared +/* Must be called only while the RX timestamping state of the tagger + * is turned off */ static int sja1105_change_rxtstamping(struct sja1105_private *priv, - struct sja1105_tagger_data *tagger_data, bool on) { struct sja1105_ptp_data *ptp_data = &priv->ptp_data; @@ -74,11 +73,6 @@ static int sja1105_change_rxtstamping(struct sja1105_private *priv, general_params->send_meta1 = on; general_params->send_meta0 = on; - /* Initialize the meta state machine to a known state */ - if (tagger_data->stampable_skb) { - kfree_skb(tagger_data->stampable_skb); - tagger_data->stampable_skb = NULL; - } ptp_cancel_worker_sync(ptp_data->clock); skb_queue_purge(&ptp_data->skb_txtstamp_queue); skb_queue_purge(&ptp_data->skb_rxtstamp_queue); @@ -117,17 +111,17 @@ int sja1105_hwtstamp_set(struct dsa_switch *ds, int port, struct ifreq *ifr) break; } - if (rx_on != test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) { - clear_bit(SJA1105_HWTS_RX_EN, &tagger_data->state); + if (rx_on != tagger_data->rxtstamp_get_state(ds)) { + tagger_data->rxtstamp_set_state(ds, false); - rc = sja1105_change_rxtstamping(priv, tagger_data, rx_on); + rc = sja1105_change_rxtstamping(priv, rx_on); if (rc < 0) { dev_err(ds->dev, "Failed to change RX timestamping: %d\n", rc); return rc; } if (rx_on) - set_bit(SJA1105_HWTS_RX_EN, &tagger_data->state); + tagger_data->rxtstamp_set_state(ds, true); } if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) @@ -146,7 +140,7 @@ int sja1105_hwtstamp_get(struct dsa_switch *ds, int port, struct ifreq *ifr) config.tx_type = HWTSTAMP_TX_ON; else config.tx_type = HWTSTAMP_TX_OFF; - if (test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) + if (tagger_data->rxtstamp_get_state(ds)) config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; else config.rx_filter = HWTSTAMP_FILTER_NONE; @@ -423,7 +417,7 @@ bool sja1105_rxtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb) struct sja1105_private *priv = ds->priv; struct sja1105_ptp_data *ptp_data = &priv->ptp_data; - if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) + if (!tagger_data->rxtstamp_get_state(ds)) return false; /* We need to read the full PTP clock to reconstruct the Rx diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index d216211b64f8..e9cb1ae6d742 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -35,8 +35,6 @@ #define SJA1105_META_SMAC 0x222222222222ull #define SJA1105_META_DMAC 0x0180C200000Eull -#define SJA1105_HWTS_RX_EN 0 - enum sja1110_meta_tstamp { SJA1110_META_TSTAMP_TX = 0, SJA1110_META_TSTAMP_RX = 1, @@ -50,16 +48,13 @@ struct sja1105_deferred_xmit_work { /* Global tagger data */ struct sja1105_tagger_data { - struct sk_buff *stampable_skb; - /* Protects concurrent access to the meta state machine - * from taggers running on multiple ports on SMP systems - */ - spinlock_t meta_lock; - unsigned long state; - struct kthread_worker *xmit_worker; + /* Tagger to switch */ void (*xmit_work_fn)(struct kthread_work *work); void (*meta_tstamp_handler)(struct dsa_switch *ds, int port, u8 ts_id, enum sja1110_meta_tstamp dir, u64 tstamp); + /* Switch to tagger */ + bool (*rxtstamp_get_state)(struct dsa_switch *ds); + void (*rxtstamp_set_state)(struct dsa_switch *ds, bool on); }; struct sja1105_skb_cb { diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index fe6a6d95bb26..93d2484b2480 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -53,6 +53,25 @@ #define SJA1110_TX_TRAILER_LEN 4 #define SJA1110_MAX_PADDING_LEN 15 +#define SJA1105_HWTS_RX_EN 0 + +struct sja1105_tagger_private { + struct sja1105_tagger_data data; /* Must be first */ + unsigned long state; + /* Protects concurrent access to the meta state machine + * from taggers running on multiple ports on SMP systems + */ + spinlock_t meta_lock; + struct sk_buff *stampable_skb; + struct kthread_worker *xmit_worker; +}; + +static struct sja1105_tagger_private * +sja1105_tagger_private(struct dsa_switch *ds) +{ + return ds->tagger_data; +} + /* Similar to is_link_local_ether_addr(hdr->h_dest) but also covers PTP */ static inline bool sja1105_is_link_local(const struct sk_buff *skb) { @@ -120,12 +139,13 @@ static struct sk_buff *sja1105_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) { struct sja1105_tagger_data *tagger_data = sja1105_tagger_data(dp->ds); + struct sja1105_tagger_private *priv = sja1105_tagger_private(dp->ds); void (*xmit_work_fn)(struct kthread_work *work); struct sja1105_deferred_xmit_work *xmit_work; struct kthread_worker *xmit_worker; xmit_work_fn = tagger_data->xmit_work_fn; - xmit_worker = tagger_data->xmit_worker; + xmit_worker = priv->xmit_worker; if (!xmit_work_fn || !xmit_worker) return NULL; @@ -362,32 +382,32 @@ static struct sk_buff */ if (is_link_local) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_tagger_data *tagger_data; + struct sja1105_tagger_private *priv; struct dsa_switch *ds = dp->ds; - tagger_data = sja1105_tagger_data(ds); + priv = sja1105_tagger_private(ds); - if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) + if (!test_bit(SJA1105_HWTS_RX_EN, &priv->state)) /* Do normal processing. */ return skb; - spin_lock(&tagger_data->meta_lock); + spin_lock(&priv->meta_lock); /* Was this a link-local frame instead of the meta * that we were expecting? */ - if (tagger_data->stampable_skb) { + if (priv->stampable_skb) { dev_err_ratelimited(ds->dev, "Expected meta frame, is %12llx " "in the DSA master multicast filter?\n", SJA1105_META_DMAC); - kfree_skb(tagger_data->stampable_skb); + kfree_skb(priv->stampable_skb); } /* Hold a reference to avoid dsa_switch_rcv * from freeing the skb. */ - tagger_data->stampable_skb = skb_get(skb); - spin_unlock(&tagger_data->meta_lock); + priv->stampable_skb = skb_get(skb); + spin_unlock(&priv->meta_lock); /* Tell DSA we got nothing */ return NULL; @@ -400,22 +420,22 @@ static struct sk_buff */ } else if (is_meta) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct sja1105_tagger_data *tagger_data; + struct sja1105_tagger_private *priv; struct dsa_switch *ds = dp->ds; struct sk_buff *stampable_skb; - tagger_data = sja1105_tagger_data(ds); + priv = sja1105_tagger_private(ds); /* Drop the meta frame if we're not in the right state * to process it. */ - if (!test_bit(SJA1105_HWTS_RX_EN, &tagger_data->state)) + if (!test_bit(SJA1105_HWTS_RX_EN, &priv->state)) return NULL; - spin_lock(&tagger_data->meta_lock); + spin_lock(&priv->meta_lock); - stampable_skb = tagger_data->stampable_skb; - tagger_data->stampable_skb = NULL; + stampable_skb = priv->stampable_skb; + priv->stampable_skb = NULL; /* Was this a meta frame instead of the link-local * that we were expecting? @@ -423,14 +443,14 @@ static struct sk_buff if (!stampable_skb) { dev_err_ratelimited(ds->dev, "Unexpected meta frame\n"); - spin_unlock(&tagger_data->meta_lock); + spin_unlock(&priv->meta_lock); return NULL; } if (stampable_skb->dev != skb->dev) { dev_err_ratelimited(ds->dev, "Meta frame on wrong port\n"); - spin_unlock(&tagger_data->meta_lock); + spin_unlock(&priv->meta_lock); return NULL; } @@ -441,12 +461,36 @@ static struct sk_buff skb = stampable_skb; sja1105_transfer_meta(skb, meta); - spin_unlock(&tagger_data->meta_lock); + spin_unlock(&priv->meta_lock); } return skb; } +static bool sja1105_rxtstamp_get_state(struct dsa_switch *ds) +{ + struct sja1105_tagger_private *priv = sja1105_tagger_private(ds); + + return test_bit(SJA1105_HWTS_RX_EN, &priv->state); +} + +static void sja1105_rxtstamp_set_state(struct dsa_switch *ds, bool on) +{ + struct sja1105_tagger_private *priv = sja1105_tagger_private(ds); + + if (on) + set_bit(SJA1105_HWTS_RX_EN, &priv->state); + else + clear_bit(SJA1105_HWTS_RX_EN, &priv->state); + + /* Initialize the meta state machine to a known state */ + if (!priv->stampable_skb) + return; + + kfree_skb(priv->stampable_skb); + priv->stampable_skb = NULL; +} + static bool sja1105_skb_has_tag_8021q(const struct sk_buff *skb) { u16 tpid = ntohs(eth_hdr(skb)->h_proto); @@ -699,26 +743,27 @@ static void sja1110_flow_dissect(const struct sk_buff *skb, __be16 *proto, static void sja1105_disconnect(struct dsa_switch_tree *dst) { - struct sja1105_tagger_data *tagger_data; + struct sja1105_tagger_private *priv; struct dsa_port *dp; list_for_each_entry(dp, &dst->ports, list) { - tagger_data = dp->ds->tagger_data; + priv = dp->ds->tagger_data; - if (!tagger_data) + if (!priv) continue; - if (tagger_data->xmit_worker) - kthread_destroy_worker(tagger_data->xmit_worker); + if (priv->xmit_worker) + kthread_destroy_worker(priv->xmit_worker); - kfree(tagger_data); - dp->ds->tagger_data = NULL; + kfree(priv); + dp->ds->priv = NULL; } } static int sja1105_connect(struct dsa_switch_tree *dst) { struct sja1105_tagger_data *tagger_data; + struct sja1105_tagger_private *priv; struct kthread_worker *xmit_worker; struct dsa_port *dp; int err; @@ -727,13 +772,13 @@ static int sja1105_connect(struct dsa_switch_tree *dst) if (dp->ds->tagger_data) continue; - tagger_data = kzalloc(sizeof(*tagger_data), GFP_KERNEL); - if (!tagger_data) { + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { err = -ENOMEM; goto out; } - spin_lock_init(&tagger_data->meta_lock); + spin_lock_init(&priv->meta_lock); xmit_worker = kthread_create_worker(0, "dsa%d:%d_xmit", dst->index, dp->ds->index); @@ -742,8 +787,12 @@ static int sja1105_connect(struct dsa_switch_tree *dst) goto out; } - tagger_data->xmit_worker = xmit_worker; - dp->ds->tagger_data = tagger_data; + priv->xmit_worker = xmit_worker; + /* Export functions for switch driver use */ + tagger_data = &priv->data; + tagger_data->rxtstamp_get_state = sja1105_rxtstamp_get_state; + tagger_data->rxtstamp_set_state = sja1105_rxtstamp_set_state; + dp->ds->tagger_data = priv; } return 0; -- cgit v1.2.3-59-g8ed1b From 9020ef659885f2622cfb386cc229b6d618362895 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 17 Oct 2021 18:22:09 +0100 Subject: iio: trigger: Fix a scheduling whilst atomic issue seen on tsc2046 IIO triggers are software IRQ chips that split an incoming IRQ into separate IRQs routed to all devices using the trigger. When all consumers are done then a trigger callback reenable() is called. There are a few circumstances under which this can happen in atomic context. 1) A single user of the trigger that calls the iio_trigger_done() function from interrupt context. 2) A race between disconnecting the last device from a trigger and the trigger itself sucessfully being disabled. To avoid a resulting scheduling whilst atomic, close this second corner by using schedule_work() to ensure the reenable is not done in atomic context. Note that drivers must be careful to manage the interaction of set_state() and reenable() callbacks to ensure appropriate reference counting if they are relying on the same hardware controls. Deliberately taking this the slow path rather than via a fixes tree because the error has hard to hit and I would like it to soak for a while before hitting a release kernel. Signed-off-by: Jonathan Cameron Cc: Pengutronix Kernel Team Cc: Dmitry Torokhov Tested-by: Oleksij Rempel Cc: Link: https://lore.kernel.org/r/20211017172209.112387-1-jic23@kernel.org --- drivers/iio/industrialio-trigger.c | 36 +++++++++++++++++++++++++++++++++++- include/linux/iio/trigger.h | 2 ++ 2 files changed, 37 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index b23caa2f2aa1..d3bdc9800b4a 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -162,6 +162,39 @@ static struct iio_trigger *iio_trigger_acquire_by_name(const char *name) return trig; } +static void iio_reenable_work_fn(struct work_struct *work) +{ + struct iio_trigger *trig = container_of(work, struct iio_trigger, + reenable_work); + + /* + * This 'might' occur after the trigger state is set to disabled - + * in that case the driver should skip reenabling. + */ + trig->ops->reenable(trig); +} + +/* + * In general, reenable callbacks may need to sleep and this path is + * not performance sensitive, so just queue up a work item + * to reneable the trigger for us. + * + * Races that can cause this. + * 1) A handler occurs entirely in interrupt context so the counter + * the final decrement is still in this interrupt. + * 2) The trigger has been removed, but one last interrupt gets through. + * + * For (1) we must call reenable, but not in atomic context. + * For (2) it should be safe to call reenanble, if drivers never blindly + * reenable after state is off. + */ +static void iio_trigger_notify_done_atomic(struct iio_trigger *trig) +{ + if (atomic_dec_and_test(&trig->use_count) && trig->ops && + trig->ops->reenable) + schedule_work(&trig->reenable_work); +} + void iio_trigger_poll(struct iio_trigger *trig) { int i; @@ -173,7 +206,7 @@ void iio_trigger_poll(struct iio_trigger *trig) if (trig->subirqs[i].enabled) generic_handle_irq(trig->subirq_base + i); else - iio_trigger_notify_done(trig); + iio_trigger_notify_done_atomic(trig); } } } @@ -535,6 +568,7 @@ struct iio_trigger *viio_trigger_alloc(struct device *parent, trig->dev.type = &iio_trig_type; trig->dev.bus = &iio_bus_type; device_initialize(&trig->dev); + INIT_WORK(&trig->reenable_work, iio_reenable_work_fn); mutex_init(&trig->pool_lock); trig->subirq_base = irq_alloc_descs(-1, 0, diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index 096f68dd2e0c..4c69b144677b 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -55,6 +55,7 @@ struct iio_trigger_ops { * @attached_own_device:[INTERN] if we are using our own device as trigger, * i.e. if we registered a poll function to the same * device as the one providing the trigger. + * @reenable_work: [INTERN] work item used to ensure reenable can sleep. **/ struct iio_trigger { const struct iio_trigger_ops *ops; @@ -74,6 +75,7 @@ struct iio_trigger { unsigned long pool[BITS_TO_LONGS(CONFIG_IIO_CONSUMERS_PER_TRIGGER)]; struct mutex pool_lock; bool attached_own_device; + struct work_struct reenable_work; }; -- cgit v1.2.3-59-g8ed1b From 3ac27afefd5dd6a53e830542b899f092a58b6b51 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 5 Dec 2021 17:01:29 +0000 Subject: iio:dac:ad5755: Switch to generic firmware properties and drop pdata Lars pointed out that platform data can also be supported via the generic properties interface, so there is no point in continuing to support it separately. Hence squish the linux/platform_data/ad5755.h header into the c file and drop accessing the platform data directly. Done by inspection only. Mostly completely mechanical with the exception of a few places where default value handling is cleaner done by first setting the value, then calling the firmware reading function but and not checking the return value, as opposed to reading firmware then setting the default if an error occurs. Part of general attempt to move all of IIO over to generic device properties, both to enable other firmware types and to remove drivers that can be the source of of_ specific behaviour in new drivers. Suggested-by: Lars-Peter Clausen Signed-off-by: Jonathan Cameron Reviewed-by: Andy Shevchenko --- drivers/iio/dac/ad5755.c | 152 ++++++++++++++++++++++++++--------- include/linux/platform_data/ad5755.h | 102 ----------------------- 2 files changed, 116 insertions(+), 138 deletions(-) delete mode 100644 include/linux/platform_data/ad5755.h (limited to 'include/linux') diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c index cabc38d54085..7a62e6e1d5f1 100644 --- a/drivers/iio/dac/ad5755.c +++ b/drivers/iio/dac/ad5755.c @@ -13,10 +13,10 @@ #include #include #include -#include +#include + #include #include -#include #define AD5755_NUM_CHANNELS 4 @@ -63,6 +63,101 @@ #define AD5755_SLEW_RATE_SHIFT 3 #define AD5755_SLEW_ENABLE BIT(12) +enum ad5755_mode { + AD5755_MODE_VOLTAGE_0V_5V = 0, + AD5755_MODE_VOLTAGE_0V_10V = 1, + AD5755_MODE_VOLTAGE_PLUSMINUS_5V = 2, + AD5755_MODE_VOLTAGE_PLUSMINUS_10V = 3, + AD5755_MODE_CURRENT_4mA_20mA = 4, + AD5755_MODE_CURRENT_0mA_20mA = 5, + AD5755_MODE_CURRENT_0mA_24mA = 6, +}; + +enum ad5755_dc_dc_phase { + AD5755_DC_DC_PHASE_ALL_SAME_EDGE = 0, + AD5755_DC_DC_PHASE_A_B_SAME_EDGE_C_D_OPP_EDGE = 1, + AD5755_DC_DC_PHASE_A_C_SAME_EDGE_B_D_OPP_EDGE = 2, + AD5755_DC_DC_PHASE_90_DEGREE = 3, +}; + +enum ad5755_dc_dc_freq { + AD5755_DC_DC_FREQ_250kHZ = 0, + AD5755_DC_DC_FREQ_410kHZ = 1, + AD5755_DC_DC_FREQ_650kHZ = 2, +}; + +enum ad5755_dc_dc_maxv { + AD5755_DC_DC_MAXV_23V = 0, + AD5755_DC_DC_MAXV_24V5 = 1, + AD5755_DC_DC_MAXV_27V = 2, + AD5755_DC_DC_MAXV_29V5 = 3, +}; + +enum ad5755_slew_rate { + AD5755_SLEW_RATE_64k = 0, + AD5755_SLEW_RATE_32k = 1, + AD5755_SLEW_RATE_16k = 2, + AD5755_SLEW_RATE_8k = 3, + AD5755_SLEW_RATE_4k = 4, + AD5755_SLEW_RATE_2k = 5, + AD5755_SLEW_RATE_1k = 6, + AD5755_SLEW_RATE_500 = 7, + AD5755_SLEW_RATE_250 = 8, + AD5755_SLEW_RATE_125 = 9, + AD5755_SLEW_RATE_64 = 10, + AD5755_SLEW_RATE_32 = 11, + AD5755_SLEW_RATE_16 = 12, + AD5755_SLEW_RATE_8 = 13, + AD5755_SLEW_RATE_4 = 14, + AD5755_SLEW_RATE_0_5 = 15, +}; + +enum ad5755_slew_step_size { + AD5755_SLEW_STEP_SIZE_1 = 0, + AD5755_SLEW_STEP_SIZE_2 = 1, + AD5755_SLEW_STEP_SIZE_4 = 2, + AD5755_SLEW_STEP_SIZE_8 = 3, + AD5755_SLEW_STEP_SIZE_16 = 4, + AD5755_SLEW_STEP_SIZE_32 = 5, + AD5755_SLEW_STEP_SIZE_64 = 6, + AD5755_SLEW_STEP_SIZE_128 = 7, + AD5755_SLEW_STEP_SIZE_256 = 8, +}; + +/** + * struct ad5755_platform_data - AD5755 DAC driver platform data + * @ext_dc_dc_compenstation_resistor: Whether an external DC-DC converter + * compensation register is used. + * @dc_dc_phase: DC-DC converter phase. + * @dc_dc_freq: DC-DC converter frequency. + * @dc_dc_maxv: DC-DC maximum allowed boost voltage. + * @dac: Per DAC instance parameters. + * @dac.mode: The mode to be used for the DAC output. + * @dac.ext_current_sense_resistor: Whether an external current sense resistor + * is used. + * @dac.enable_voltage_overrange: Whether to enable 20% voltage output overrange. + * @dac.slew.enable: Whether to enable digital slew. + * @dac.slew.rate: Slew rate of the digital slew. + * @dac.slew.step_size: Slew step size of the digital slew. + **/ +struct ad5755_platform_data { + bool ext_dc_dc_compenstation_resistor; + enum ad5755_dc_dc_phase dc_dc_phase; + enum ad5755_dc_dc_freq dc_dc_freq; + enum ad5755_dc_dc_maxv dc_dc_maxv; + + struct { + enum ad5755_mode mode; + bool ext_current_sense_resistor; + bool enable_voltage_overrange; + struct { + bool enable; + enum ad5755_slew_rate rate; + enum ad5755_slew_step_size step_size; + } slew; + } dac[4]; +}; + /** * struct ad5755_chip_info - chip specific information * @channel_template: channel specification @@ -111,7 +206,6 @@ enum ad5755_type { ID_AD5737, }; -#ifdef CONFIG_OF static const int ad5755_dcdc_freq_table[][2] = { { 250000, AD5755_DC_DC_FREQ_250kHZ }, { 410000, AD5755_DC_DC_FREQ_410kHZ }, @@ -154,7 +248,6 @@ static const int ad5755_slew_step_table[][2] = { { 2, AD5755_SLEW_STEP_SIZE_2 }, { 1, AD5755_SLEW_STEP_SIZE_1 }, }; -#endif static int ad5755_write_unlocked(struct iio_dev *indio_dev, unsigned int reg, unsigned int val) @@ -604,30 +697,29 @@ static const struct ad5755_platform_data ad5755_default_pdata = { }, }; -#ifdef CONFIG_OF -static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) +static struct ad5755_platform_data *ad5755_parse_fw(struct device *dev) { - struct device_node *np = dev->of_node; - struct device_node *pp; + struct fwnode_handle *pp; struct ad5755_platform_data *pdata; unsigned int tmp; unsigned int tmparray[3]; int devnr, i; + if (!dev_fwnode(dev)) + return NULL; + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return NULL; pdata->ext_dc_dc_compenstation_resistor = - of_property_read_bool(np, "adi,ext-dc-dc-compenstation-resistor"); + device_property_read_bool(dev, "adi,ext-dc-dc-compenstation-resistor"); - if (!of_property_read_u32(np, "adi,dc-dc-phase", &tmp)) - pdata->dc_dc_phase = tmp; - else - pdata->dc_dc_phase = AD5755_DC_DC_PHASE_ALL_SAME_EDGE; + pdata->dc_dc_phase = AD5755_DC_DC_PHASE_ALL_SAME_EDGE; + device_property_read_u32(dev, "adi,dc-dc-phase", &pdata->dc_dc_phase); pdata->dc_dc_freq = AD5755_DC_DC_FREQ_410kHZ; - if (!of_property_read_u32(np, "adi,dc-dc-freq-hz", &tmp)) { + if (!device_property_read_u32(dev, "adi,dc-dc-freq-hz", &tmp)) { for (i = 0; i < ARRAY_SIZE(ad5755_dcdc_freq_table); i++) { if (tmp == ad5755_dcdc_freq_table[i][0]) { pdata->dc_dc_freq = ad5755_dcdc_freq_table[i][1]; @@ -641,7 +733,7 @@ static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) } pdata->dc_dc_maxv = AD5755_DC_DC_MAXV_23V; - if (!of_property_read_u32(np, "adi,dc-dc-max-microvolt", &tmp)) { + if (!device_property_read_u32(dev, "adi,dc-dc-max-microvolt", &tmp)) { for (i = 0; i < ARRAY_SIZE(ad5755_dcdc_maxv_table); i++) { if (tmp == ad5755_dcdc_maxv_table[i][0]) { pdata->dc_dc_maxv = ad5755_dcdc_maxv_table[i][1]; @@ -654,25 +746,23 @@ static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) } devnr = 0; - for_each_child_of_node(np, pp) { + device_for_each_child_node(dev, pp) { if (devnr >= AD5755_NUM_CHANNELS) { dev_err(dev, "There are too many channels defined in DT\n"); goto error_out; } - if (!of_property_read_u32(pp, "adi,mode", &tmp)) - pdata->dac[devnr].mode = tmp; - else - pdata->dac[devnr].mode = AD5755_MODE_CURRENT_4mA_20mA; + pdata->dac[devnr].mode = AD5755_MODE_CURRENT_4mA_20mA; + fwnode_property_read_u32(pp, "adi,mode", &pdata->dac[devnr].mode); pdata->dac[devnr].ext_current_sense_resistor = - of_property_read_bool(pp, "adi,ext-current-sense-resistor"); + fwnode_property_read_bool(pp, "adi,ext-current-sense-resistor"); pdata->dac[devnr].enable_voltage_overrange = - of_property_read_bool(pp, "adi,enable-voltage-overrange"); + fwnode_property_read_bool(pp, "adi,enable-voltage-overrange"); - if (!of_property_read_u32_array(pp, "adi,slew", tmparray, 3)) { + if (!fwnode_property_read_u32_array(pp, "adi,slew", tmparray, 3)) { pdata->dac[devnr].slew.enable = tmparray[0]; pdata->dac[devnr].slew.rate = AD5755_SLEW_RATE_64k; @@ -715,18 +805,11 @@ static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) devm_kfree(dev, pdata); return NULL; } -#else -static -struct ad5755_platform_data *ad5755_parse_dt(struct device *dev) -{ - return NULL; -} -#endif static int ad5755_probe(struct spi_device *spi) { enum ad5755_type type = spi_get_device_id(spi)->driver_data; - const struct ad5755_platform_data *pdata = dev_get_platdata(&spi->dev); + const struct ad5755_platform_data *pdata; struct iio_dev *indio_dev; struct ad5755_state *st; int ret; @@ -751,13 +834,10 @@ static int ad5755_probe(struct spi_device *spi) mutex_init(&st->lock); - if (spi->dev.of_node) - pdata = ad5755_parse_dt(&spi->dev); - else - pdata = spi->dev.platform_data; + pdata = ad5755_parse_fw(&spi->dev); if (!pdata) { - dev_warn(&spi->dev, "no platform data? using default\n"); + dev_warn(&spi->dev, "no firmware provided parameters? using default\n"); pdata = &ad5755_default_pdata; } diff --git a/include/linux/platform_data/ad5755.h b/include/linux/platform_data/ad5755.h deleted file mode 100644 index e371e08f04bc..000000000000 --- a/include/linux/platform_data/ad5755.h +++ /dev/null @@ -1,102 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2012 Analog Devices Inc. - */ -#ifndef __LINUX_PLATFORM_DATA_AD5755_H__ -#define __LINUX_PLATFORM_DATA_AD5755_H__ - -enum ad5755_mode { - AD5755_MODE_VOLTAGE_0V_5V = 0, - AD5755_MODE_VOLTAGE_0V_10V = 1, - AD5755_MODE_VOLTAGE_PLUSMINUS_5V = 2, - AD5755_MODE_VOLTAGE_PLUSMINUS_10V = 3, - AD5755_MODE_CURRENT_4mA_20mA = 4, - AD5755_MODE_CURRENT_0mA_20mA = 5, - AD5755_MODE_CURRENT_0mA_24mA = 6, -}; - -enum ad5755_dc_dc_phase { - AD5755_DC_DC_PHASE_ALL_SAME_EDGE = 0, - AD5755_DC_DC_PHASE_A_B_SAME_EDGE_C_D_OPP_EDGE = 1, - AD5755_DC_DC_PHASE_A_C_SAME_EDGE_B_D_OPP_EDGE = 2, - AD5755_DC_DC_PHASE_90_DEGREE = 3, -}; - -enum ad5755_dc_dc_freq { - AD5755_DC_DC_FREQ_250kHZ = 0, - AD5755_DC_DC_FREQ_410kHZ = 1, - AD5755_DC_DC_FREQ_650kHZ = 2, -}; - -enum ad5755_dc_dc_maxv { - AD5755_DC_DC_MAXV_23V = 0, - AD5755_DC_DC_MAXV_24V5 = 1, - AD5755_DC_DC_MAXV_27V = 2, - AD5755_DC_DC_MAXV_29V5 = 3, -}; - -enum ad5755_slew_rate { - AD5755_SLEW_RATE_64k = 0, - AD5755_SLEW_RATE_32k = 1, - AD5755_SLEW_RATE_16k = 2, - AD5755_SLEW_RATE_8k = 3, - AD5755_SLEW_RATE_4k = 4, - AD5755_SLEW_RATE_2k = 5, - AD5755_SLEW_RATE_1k = 6, - AD5755_SLEW_RATE_500 = 7, - AD5755_SLEW_RATE_250 = 8, - AD5755_SLEW_RATE_125 = 9, - AD5755_SLEW_RATE_64 = 10, - AD5755_SLEW_RATE_32 = 11, - AD5755_SLEW_RATE_16 = 12, - AD5755_SLEW_RATE_8 = 13, - AD5755_SLEW_RATE_4 = 14, - AD5755_SLEW_RATE_0_5 = 15, -}; - -enum ad5755_slew_step_size { - AD5755_SLEW_STEP_SIZE_1 = 0, - AD5755_SLEW_STEP_SIZE_2 = 1, - AD5755_SLEW_STEP_SIZE_4 = 2, - AD5755_SLEW_STEP_SIZE_8 = 3, - AD5755_SLEW_STEP_SIZE_16 = 4, - AD5755_SLEW_STEP_SIZE_32 = 5, - AD5755_SLEW_STEP_SIZE_64 = 6, - AD5755_SLEW_STEP_SIZE_128 = 7, - AD5755_SLEW_STEP_SIZE_256 = 8, -}; - -/** - * struct ad5755_platform_data - AD5755 DAC driver platform data - * @ext_dc_dc_compenstation_resistor: Whether an external DC-DC converter - * compensation register is used. - * @dc_dc_phase: DC-DC converter phase. - * @dc_dc_freq: DC-DC converter frequency. - * @dc_dc_maxv: DC-DC maximum allowed boost voltage. - * @dac.mode: The mode to be used for the DAC output. - * @dac.ext_current_sense_resistor: Whether an external current sense resistor - * is used. - * @dac.enable_voltage_overrange: Whether to enable 20% voltage output overrange. - * @dac.slew.enable: Whether to enable digital slew. - * @dac.slew.rate: Slew rate of the digital slew. - * @dac.slew.step_size: Slew step size of the digital slew. - **/ -struct ad5755_platform_data { - bool ext_dc_dc_compenstation_resistor; - enum ad5755_dc_dc_phase dc_dc_phase; - enum ad5755_dc_dc_freq dc_dc_freq; - enum ad5755_dc_dc_maxv dc_dc_maxv; - - struct { - enum ad5755_mode mode; - bool ext_current_sense_resistor; - bool enable_voltage_overrange; - struct { - bool enable; - enum ad5755_slew_rate rate; - enum ad5755_slew_step_size step_size; - } slew; - } dac[4]; -}; - -#endif -- cgit v1.2.3-59-g8ed1b From c537be0bfad6337f2afd618fe252c03217191405 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 3 Dec 2021 11:28:46 +0100 Subject: i2c: acpi: Add i2c_acpi_new_device_by_fwnode() function Change i2c_acpi_new_device() into i2c_acpi_new_device_by_fwnode() and add a static inline wrapper providing the old i2c_acpi_new_device() behavior. This is necessary because in some cases we may only have access to the fwnode / acpi_device and not to the matching physical-node struct device *. Suggested-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Acked-by: Mika Westerberg Acked-by: Wolfram Sang Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20211203102857.44539-4-hdegoede@redhat.com --- drivers/i2c/i2c-core-acpi.c | 17 +++++++++++------ include/linux/i2c.h | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index 04338cbd08a9..c87ce2276007 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -476,8 +476,8 @@ struct notifier_block i2c_acpi_notifier = { }; /** - * i2c_acpi_new_device - Create i2c-client for the Nth I2cSerialBus resource - * @dev: Device owning the ACPI resources to get the client from + * i2c_acpi_new_device_by_fwnode - Create i2c-client for the Nth I2cSerialBus resource + * @fwnode: fwnode with the ACPI resources to get the client from * @index: Index of ACPI resource to get * @info: describes the I2C device; note this is modified (addr gets set) * Context: can sleep @@ -493,15 +493,20 @@ struct notifier_block i2c_acpi_notifier = { * Returns a pointer to the new i2c-client, or error pointer in case of failure. * Specifically, -EPROBE_DEFER is returned if the adapter is not found. */ -struct i2c_client *i2c_acpi_new_device(struct device *dev, int index, - struct i2c_board_info *info) +struct i2c_client *i2c_acpi_new_device_by_fwnode(struct fwnode_handle *fwnode, + int index, + struct i2c_board_info *info) { - struct acpi_device *adev = ACPI_COMPANION(dev); struct i2c_acpi_lookup lookup; struct i2c_adapter *adapter; + struct acpi_device *adev; LIST_HEAD(resource_list); int ret; + adev = to_acpi_device_node(fwnode); + if (!adev) + return ERR_PTR(-ENODEV); + memset(&lookup, 0, sizeof(lookup)); lookup.info = info; lookup.device_handle = acpi_device_handle(adev); @@ -523,7 +528,7 @@ struct i2c_client *i2c_acpi_new_device(struct device *dev, int index, return i2c_new_client_device(adapter, info); } -EXPORT_SYMBOL_GPL(i2c_acpi_new_device); +EXPORT_SYMBOL_GPL(i2c_acpi_new_device_by_fwnode); bool i2c_acpi_waive_d0_probe(struct device *dev) { diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 16119ac1aa97..7d4f52ceb7b5 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -1025,8 +1025,9 @@ bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares, struct acpi_resource_i2c_serialbus **i2c); int i2c_acpi_client_count(struct acpi_device *adev); u32 i2c_acpi_find_bus_speed(struct device *dev); -struct i2c_client *i2c_acpi_new_device(struct device *dev, int index, - struct i2c_board_info *info); +struct i2c_client *i2c_acpi_new_device_by_fwnode(struct fwnode_handle *fwnode, + int index, + struct i2c_board_info *info); struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle); bool i2c_acpi_waive_d0_probe(struct device *dev); #else @@ -1043,8 +1044,9 @@ static inline u32 i2c_acpi_find_bus_speed(struct device *dev) { return 0; } -static inline struct i2c_client *i2c_acpi_new_device(struct device *dev, - int index, struct i2c_board_info *info) +static inline struct i2c_client *i2c_acpi_new_device_by_fwnode( + struct fwnode_handle *fwnode, int index, + struct i2c_board_info *info) { return ERR_PTR(-ENODEV); } @@ -1058,4 +1060,11 @@ static inline bool i2c_acpi_waive_d0_probe(struct device *dev) } #endif /* CONFIG_ACPI */ +static inline struct i2c_client *i2c_acpi_new_device(struct device *dev, + int index, + struct i2c_board_info *info) +{ + return i2c_acpi_new_device_by_fwnode(dev_fwnode(dev), index, info); +} + #endif /* _LINUX_I2C_H */ -- cgit v1.2.3-59-g8ed1b From 9dfa374cc6d04d2515adc21c39e356b64ee45a29 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 3 Dec 2021 11:28:47 +0100 Subject: platform_data: Add linux/platform_data/tps68470.h file The clk and regulator frameworks expect clk/regulator consumer-devices to have info about the consumed clks/regulators described in the device's fw_node. To work around cases where this info is not present in the firmware tables, which is often the case on x86/ACPI devices, both frameworks allow the provider-driver to attach info about consumers to the provider-device during probe/registration of the provider device. The TI TPS68470 PMIC is used x86/ACPI devices with the consumer-info missing from the ACPI tables. Thus the tps68470-clk and tps68470-regulator drivers must provide the consumer-info at probe time. Define tps68470_clk_platform_data and tps68470_regulator_platform_data structs to allow the x86 platform code to pass the necessary consumer info to these drivers. Reviewed-by: Andy Shevchenko Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20211203102857.44539-5-hdegoede@redhat.com --- include/linux/platform_data/tps68470.h | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 include/linux/platform_data/tps68470.h (limited to 'include/linux') diff --git a/include/linux/platform_data/tps68470.h b/include/linux/platform_data/tps68470.h new file mode 100644 index 000000000000..126d082c3f2e --- /dev/null +++ b/include/linux/platform_data/tps68470.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * TI TPS68470 PMIC platform data definition. + * + * Copyright (c) 2021 Red Hat Inc. + * + * Red Hat authors: + * Hans de Goede + */ +#ifndef __PDATA_TPS68470_H +#define __PDATA_TPS68470_H + +enum tps68470_regulators { + TPS68470_CORE, + TPS68470_ANA, + TPS68470_VCM, + TPS68470_VIO, + TPS68470_VSIO, + TPS68470_AUX1, + TPS68470_AUX2, + TPS68470_NUM_REGULATORS +}; + +struct regulator_init_data; + +struct tps68470_regulator_platform_data { + const struct regulator_init_data *reg_init_data[TPS68470_NUM_REGULATORS]; +}; + +struct tps68470_clk_platform_data { + const char *consumer_dev_name; + const char *consumer_con_id; +}; + +#endif -- cgit v1.2.3-59-g8ed1b From 3c118547f87e930d45a5787e386734015dd93b32 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 10 Dec 2021 21:29:59 +0100 Subject: u64_stats: Disable preemption on 32bit UP+SMP PREEMPT_RT during updates. On PREEMPT_RT the seqcount_t for synchronisation is required on 32bit architectures even on UP because the softirq (and the threaded IRQ handler) can be preempted. With the seqcount_t for synchronisation, a reader with higher priority can preempt the writer and then spin endlessly in read_seqcount_begin() while the writer can't make progress. To avoid such a lock up on PREEMPT_RT the writer must disable preemption during the update. There is no need to disable interrupts because no writer is using this API in hard-IRQ context on PREEMPT_RT. Disable preemption on 32bit-RT within the u64_stats write section. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: David S. Miller --- include/linux/u64_stats_sync.h | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index e8ec116c916b..6ad4e9032d53 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -66,7 +66,7 @@ #include struct u64_stats_sync { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) seqcount_t seq; #endif }; @@ -125,7 +125,7 @@ static inline void u64_stats_inc(u64_stats_t *p) } #endif -#if BITS_PER_LONG == 32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) #define u64_stats_init(syncp) seqcount_init(&(syncp)->seq) #else static inline void u64_stats_init(struct u64_stats_sync *syncp) @@ -135,15 +135,19 @@ static inline void u64_stats_init(struct u64_stats_sync *syncp) static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); write_seqcount_begin(&syncp->seq); #endif } static inline void u64_stats_update_end(struct u64_stats_sync *syncp) { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) write_seqcount_end(&syncp->seq); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); #endif } @@ -152,8 +156,11 @@ u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp) { unsigned long flags = 0; -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) - local_irq_save(flags); +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); + else + local_irq_save(flags); write_seqcount_begin(&syncp->seq); #endif return flags; @@ -163,15 +170,18 @@ static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp, unsigned long flags) { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) write_seqcount_end(&syncp->seq); - local_irq_restore(flags); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); + else + local_irq_restore(flags); #endif } static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) return read_seqcount_begin(&syncp->seq); #else return 0; @@ -180,7 +190,7 @@ static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync * static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG==32 && !defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT)) preempt_disable(); #endif return __u64_stats_fetch_begin(syncp); @@ -189,7 +199,7 @@ static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *sy static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, unsigned int start) { -#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) return read_seqcount_retry(&syncp->seq, start); #else return false; @@ -199,7 +209,7 @@ static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, unsigned int start) { -#if BITS_PER_LONG==32 && !defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT)) preempt_enable(); #endif return __u64_stats_fetch_retry(syncp, start); @@ -213,7 +223,9 @@ static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, */ static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG==32 && !defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) + preempt_disable(); +#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP) local_irq_disable(); #endif return __u64_stats_fetch_begin(syncp); @@ -222,7 +234,9 @@ static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp, unsigned int start) { -#if BITS_PER_LONG==32 && !defined(CONFIG_SMP) +#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) + preempt_enable(); +#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP) local_irq_enable(); #endif return __u64_stats_fetch_retry(syncp, start); -- cgit v1.2.3-59-g8ed1b From f92c1e183604c20ce00eb889315fdaa8f2d9e509 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Wed, 8 Dec 2021 20:32:44 +0100 Subject: bpf: Add get_func_[arg|ret|arg_cnt] helpers Adding following helpers for tracing programs: Get n-th argument of the traced function: long bpf_get_func_arg(void *ctx, u32 n, u64 *value) Get return value of the traced function: long bpf_get_func_ret(void *ctx, u64 *value) Get arguments count of the traced function: long bpf_get_func_arg_cnt(void *ctx) The trampoline now stores number of arguments on ctx-8 address, so it's easy to verify argument index and find return value argument's position. Moving function ip address on the trampoline stack behind the number of functions arguments, so it's now stored on ctx-16 address if it's needed. All helpers above are inlined by verifier. Also bit unrelated small change - using newly added function bpf_prog_has_trampoline in check_get_func_ip. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211208193245.172141-5-jolsa@kernel.org --- arch/x86/net/bpf_jit_comp.c | 15 +++++++- include/linux/bpf.h | 5 +++ include/uapi/linux/bpf.h | 28 +++++++++++++++ kernel/bpf/trampoline.c | 8 +++++ kernel/bpf/verifier.c | 77 +++++++++++++++++++++++++++++++++++++++--- kernel/trace/bpf_trace.c | 55 +++++++++++++++++++++++++++++- tools/include/uapi/linux/bpf.h | 28 +++++++++++++++ 7 files changed, 209 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 10fab8cb3fb5..4bbcded07415 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1941,7 +1941,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *orig_call) { int ret, i, nr_args = m->nr_args; - int regs_off, ip_off, stack_size = nr_args * 8; + int regs_off, ip_off, args_off, stack_size = nr_args * 8; struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN]; @@ -1968,6 +1968,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i * [ ... ] * RBP - regs_off [ reg_arg1 ] program's ctx pointer * + * RBP - args_off [ args count ] always + * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag */ @@ -1978,6 +1980,10 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i regs_off = stack_size; + /* args count */ + stack_size += 8; + args_off = stack_size; + if (flags & BPF_TRAMP_F_IP_ARG) stack_size += 8; /* room for IP address argument */ @@ -1996,6 +2002,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ EMIT1(0x53); /* push rbx */ + /* Store number of arguments of the traced function: + * mov rax, nr_args + * mov QWORD PTR [rbp - args_off], rax + */ + emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args); + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off); + if (flags & BPF_TRAMP_F_IP_ARG) { /* Store IP address of the traced function: * mov rax, QWORD PTR [rbp + 8] diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7a40022e3d00..965fffaf0308 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -777,6 +777,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); int bpf_jit_charge_modmem(u32 pages); void bpf_jit_uncharge_modmem(u32 pages); +bool bpf_prog_has_trampoline(const struct bpf_prog *prog); #else static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) @@ -805,6 +806,10 @@ static inline bool is_bpf_image_address(unsigned long address) { return false; } +static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) +{ + return false; +} #endif struct bpf_func_info_aux { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 2820c77e4846..b0383d371b9a 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4993,6 +4993,31 @@ union bpf_attr { * An integer less than, equal to, or greater than zero * if the first **s1_sz** bytes of **s1** is found to be * less than, to match, or be greater than **s2**. + * + * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) + * Description + * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * returned in **value**. + * + * Return + * 0 on success. + * **-EINVAL** if n >= arguments count of traced function. + * + * long bpf_get_func_ret(void *ctx, u64 *value) + * Description + * Get return value of the traced function (for tracing programs) + * in **value**. + * + * Return + * 0 on success. + * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. + * + * long bpf_get_func_arg_cnt(void *ctx) + * Description + * Get number of arguments of the traced function (for tracing programs). + * + * Return + * The number of arguments of the traced function. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5178,6 +5203,9 @@ union bpf_attr { FN(find_vma), \ FN(loop), \ FN(strncmp), \ + FN(get_func_arg), \ + FN(get_func_ret), \ + FN(get_func_arg_cnt), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e98de5e73ba5..4b6974a195c1 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -27,6 +27,14 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline_table */ static DEFINE_MUTEX(trampoline_mutex); +bool bpf_prog_has_trampoline(const struct bpf_prog *prog) +{ + enum bpf_attach_type eatype = prog->expected_attach_type; + + return eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || + eatype == BPF_MODIFY_RETURN; +} + void *bpf_jit_alloc_exec_page(void) { void *image; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b39de3ae50f5..d74e8a99412e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6395,13 +6395,11 @@ static int check_bpf_snprintf_call(struct bpf_verifier_env *env, static int check_get_func_ip(struct bpf_verifier_env *env) { - enum bpf_attach_type eatype = env->prog->expected_attach_type; enum bpf_prog_type type = resolve_prog_type(env->prog); int func_id = BPF_FUNC_get_func_ip; if (type == BPF_PROG_TYPE_TRACING) { - if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT && - eatype != BPF_MODIFY_RETURN) { + if (!bpf_prog_has_trampoline(env->prog)) { verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", func_id_name(func_id), func_id); return -ENOTSUPP; @@ -12997,6 +12995,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, static int do_misc_fixups(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog; + enum bpf_attach_type eatype = prog->expected_attach_type; bool expect_blinding = bpf_jit_blinding_enabled(prog); enum bpf_prog_type prog_type = resolve_prog_type(prog); struct bpf_insn *insn = prog->insnsi; @@ -13367,11 +13366,79 @@ patch_map_ops_generic: continue; } + /* Implement bpf_get_func_arg inline. */ + if (prog_type == BPF_PROG_TYPE_TRACING && + insn->imm == BPF_FUNC_get_func_arg) { + /* Load nr_args from ctx - 8 */ + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); + insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); + insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); + insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); + insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); + insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); + insn_buf[7] = BPF_JMP_A(1); + insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); + cnt = 9; + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; + } + + /* Implement bpf_get_func_ret inline. */ + if (prog_type == BPF_PROG_TYPE_TRACING && + insn->imm == BPF_FUNC_get_func_ret) { + if (eatype == BPF_TRACE_FEXIT || + eatype == BPF_MODIFY_RETURN) { + /* Load nr_args from ctx - 8 */ + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); + insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); + insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); + insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); + insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); + insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); + cnt = 6; + } else { + insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); + cnt = 1; + } + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; + } + + /* Implement get_func_arg_cnt inline. */ + if (prog_type == BPF_PROG_TYPE_TRACING && + insn->imm == BPF_FUNC_get_func_arg_cnt) { + /* Load nr_args from ctx - 8 */ + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); + if (!new_prog) + return -ENOMEM; + + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; + } + /* Implement bpf_get_func_ip inline. */ if (prog_type == BPF_PROG_TYPE_TRACING && insn->imm == BPF_FUNC_get_func_ip) { - /* Load IP address from ctx - 8 */ - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); + /* Load IP address from ctx - 16 */ + insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); if (!new_prog) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 623dd0684429..cea2ca6df949 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1012,7 +1012,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx) { /* This helper call is inlined by verifier. */ - return ((u64 *)ctx)[-1]; + return ((u64 *)ctx)[-2]; } static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = { @@ -1091,6 +1091,53 @@ static const struct bpf_func_proto bpf_get_branch_snapshot_proto = { .arg2_type = ARG_CONST_SIZE_OR_ZERO, }; +BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) +{ + /* This helper call is inlined by verifier. */ + u64 nr_args = ((u64 *)ctx)[-1]; + + if ((u64) n >= nr_args) + return -EINVAL; + *value = ((u64 *)ctx)[n]; + return 0; +} + +static const struct bpf_func_proto bpf_get_func_arg_proto = { + .func = get_func_arg, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_LONG, +}; + +BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value) +{ + /* This helper call is inlined by verifier. */ + u64 nr_args = ((u64 *)ctx)[-1]; + + *value = ((u64 *)ctx)[nr_args]; + return 0; +} + +static const struct bpf_func_proto bpf_get_func_ret_proto = { + .func = get_func_ret, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_LONG, +}; + +BPF_CALL_1(get_func_arg_cnt, void *, ctx) +{ + /* This helper call is inlined by verifier. */ + return ((u64 *)ctx)[-1]; +} + +static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = { + .func = get_func_arg_cnt, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, +}; + static const struct bpf_func_proto * bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -1629,6 +1676,12 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) NULL; case BPF_FUNC_d_path: return &bpf_d_path_proto; + case BPF_FUNC_get_func_arg: + return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL; + case BPF_FUNC_get_func_ret: + return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL; + case BPF_FUNC_get_func_arg_cnt: + return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL; default: fn = raw_tp_prog_func_proto(func_id, prog); if (!fn && prog->expected_attach_type == BPF_TRACE_ITER) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 2820c77e4846..b0383d371b9a 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4993,6 +4993,31 @@ union bpf_attr { * An integer less than, equal to, or greater than zero * if the first **s1_sz** bytes of **s1** is found to be * less than, to match, or be greater than **s2**. + * + * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) + * Description + * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * returned in **value**. + * + * Return + * 0 on success. + * **-EINVAL** if n >= arguments count of traced function. + * + * long bpf_get_func_ret(void *ctx, u64 *value) + * Description + * Get return value of the traced function (for tracing programs) + * in **value**. + * + * Return + * 0 on success. + * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. + * + * long bpf_get_func_arg_cnt(void *ctx) + * Description + * Get number of arguments of the traced function (for tracing programs). + * + * Return + * The number of arguments of the traced function. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5178,6 +5203,9 @@ union bpf_attr { FN(find_vma), \ FN(loop), \ FN(strncmp), \ + FN(get_func_arg), \ + FN(get_func_ret), \ + FN(get_func_arg_cnt), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- cgit v1.2.3-59-g8ed1b From 0e25498f8cd43c1b5aa327f373dd094e9a006da7 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 28 Jun 2021 14:52:01 -0500 Subject: exit: Add and use make_task_dead. There are two big uses of do_exit. The first is it's design use to be the guts of the exit(2) system call. The second use is to terminate a task after something catastrophic has happened like a NULL pointer in kernel code. Add a function make_task_dead that is initialy exactly the same as do_exit to cover the cases where do_exit is called to handle catastrophic failure. In time this can probably be reduced to just a light wrapper around do_task_dead. For now keep it exactly the same so that there will be no behavioral differences introducing this new concept. Replace all of the uses of do_exit that use it for catastraphic task cleanup with make_task_dead to make it clear what the code is doing. As part of this rename rewind_stack_do_exit rewind_stack_and_make_dead. Signed-off-by: "Eric W. Biederman" --- arch/alpha/kernel/traps.c | 6 +++--- arch/alpha/mm/fault.c | 2 +- arch/arm/kernel/traps.c | 2 +- arch/arm/mm/fault.c | 2 +- arch/arm64/kernel/traps.c | 2 +- arch/arm64/mm/fault.c | 2 +- arch/csky/abiv1/alignment.c | 2 +- arch/csky/kernel/traps.c | 2 +- arch/csky/mm/fault.c | 2 +- arch/h8300/kernel/traps.c | 2 +- arch/h8300/mm/fault.c | 2 +- arch/hexagon/kernel/traps.c | 2 +- arch/ia64/kernel/mca_drv.c | 2 +- arch/ia64/kernel/traps.c | 2 +- arch/ia64/mm/fault.c | 2 +- arch/m68k/kernel/traps.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/microblaze/kernel/exceptions.c | 4 ++-- arch/mips/kernel/traps.c | 2 +- arch/nds32/kernel/fpu.c | 2 +- arch/nds32/kernel/traps.c | 8 ++++---- arch/nios2/kernel/traps.c | 4 ++-- arch/openrisc/kernel/traps.c | 2 +- arch/parisc/kernel/traps.c | 2 +- arch/powerpc/kernel/traps.c | 8 ++++---- arch/riscv/kernel/traps.c | 2 +- arch/riscv/mm/fault.c | 2 +- arch/s390/kernel/dumpstack.c | 2 +- arch/s390/kernel/nmi.c | 2 +- arch/sh/kernel/traps.c | 2 +- arch/sparc/kernel/traps_32.c | 4 +--- arch/sparc/kernel/traps_64.c | 4 +--- arch/x86/entry/entry_32.S | 6 +++--- arch/x86/entry/entry_64.S | 6 +++--- arch/x86/kernel/dumpstack.c | 4 ++-- arch/xtensa/kernel/traps.c | 2 +- include/linux/sched/task.h | 1 + kernel/exit.c | 9 +++++++++ tools/objtool/check.c | 3 ++- 39 files changed, 63 insertions(+), 56 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index 2ae34702456c..8a66fe544c69 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -190,7 +190,7 @@ die_if_kernel(char * str, struct pt_regs *regs, long err, unsigned long *r9_15) local_irq_enable(); while (1); } - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } #ifndef CONFIG_MATHEMU @@ -575,7 +575,7 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, printk("Bad unaligned kernel access at %016lx: %p %lx %lu\n", pc, va, opcode, reg); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); got_exception: /* Ok, we caught the exception, but we don't want it. Is there @@ -630,7 +630,7 @@ got_exception: local_irq_enable(); while (1); } - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } /* diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index eee5102c3d88..e9193d52222e 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -204,7 +204,7 @@ retry: printk(KERN_ALERT "Unable to handle kernel paging request at " "virtual address %016lx\n", address); die_if_kernel("Oops", regs, cause, (unsigned long*)regs - 16); - do_exit(SIGKILL); + make_task_dead(SIGKILL); /* We ran out of memory, or some other thing happened to us that made us unable to handle the page fault gracefully. */ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 195dff58bafc..b4bd2e5f17c1 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -333,7 +333,7 @@ static void oops_end(unsigned long flags, struct pt_regs *regs, int signr) if (panic_on_oops) panic("Fatal exception"); if (signr) - do_exit(signr); + make_task_dead(signr); } /* diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index bc8779d54a64..bf1a0c618c49 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -111,7 +111,7 @@ static void die_kernel_fault(const char *msg, struct mm_struct *mm, show_pte(KERN_ALERT, mm, addr); die("Oops", regs, fsr); bust_spinlocks(0); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } /* diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 7b21213a570f..bdd456e4e7f4 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -235,7 +235,7 @@ void die(const char *str, struct pt_regs *regs, int err) raw_spin_unlock_irqrestore(&die_lock, flags); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } static void arm64_show_signal(int signo, const char *str) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 9ae24e3b72be..11a28cace2d2 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -302,7 +302,7 @@ static void die_kernel_fault(const char *msg, unsigned long addr, show_pte(addr); die("Oops", regs, esr); bust_spinlocks(0); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } #ifdef CONFIG_KASAN_HW_TAGS diff --git a/arch/csky/abiv1/alignment.c b/arch/csky/abiv1/alignment.c index cb2a0d94a144..5e2fb45d605c 100644 --- a/arch/csky/abiv1/alignment.c +++ b/arch/csky/abiv1/alignment.c @@ -294,7 +294,7 @@ bad_area: __func__, opcode, rz, rx, imm, addr); show_regs(regs); bust_spinlocks(0); - do_exit(SIGKILL); + make_dead_task(SIGKILL); } force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)addr); diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c index e5fbf8653a21..88a47035b925 100644 --- a/arch/csky/kernel/traps.c +++ b/arch/csky/kernel/traps.c @@ -109,7 +109,7 @@ void die(struct pt_regs *regs, const char *str) if (panic_on_oops) panic("Fatal exception"); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_dead_task(SIGSEGV); } void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index 466ad949818a..7215a46b6b8e 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -67,7 +67,7 @@ static inline void no_context(struct pt_regs *regs, unsigned long addr) pr_alert("Unable to handle kernel paging request at virtual " "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc); die(regs, "Oops"); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) diff --git a/arch/h8300/kernel/traps.c b/arch/h8300/kernel/traps.c index bdbe988d8dbc..3d4e0bde37ae 100644 --- a/arch/h8300/kernel/traps.c +++ b/arch/h8300/kernel/traps.c @@ -106,7 +106,7 @@ void die(const char *str, struct pt_regs *fp, unsigned long err) dump(fp); spin_unlock_irq(&die_lock); - do_exit(SIGSEGV); + make_dead_task(SIGSEGV); } static int kstack_depth_to_print = 24; diff --git a/arch/h8300/mm/fault.c b/arch/h8300/mm/fault.c index d4bc9c16f2df..0223528565dd 100644 --- a/arch/h8300/mm/fault.c +++ b/arch/h8300/mm/fault.c @@ -51,7 +51,7 @@ asmlinkage int do_page_fault(struct pt_regs *regs, unsigned long address, printk(" at virtual address %08lx\n", address); if (!user_mode(regs)) die("Oops", regs, error_code); - do_exit(SIGKILL); + make_dead_task(SIGKILL); return 1; } diff --git a/arch/hexagon/kernel/traps.c b/arch/hexagon/kernel/traps.c index edfc35dafeb1..6dd6cf0ab711 100644 --- a/arch/hexagon/kernel/traps.c +++ b/arch/hexagon/kernel/traps.c @@ -214,7 +214,7 @@ int die(const char *str, struct pt_regs *regs, long err) panic("Fatal exception"); oops_exit(); - do_exit(err); + make_dead_task(err); return 0; } diff --git a/arch/ia64/kernel/mca_drv.c b/arch/ia64/kernel/mca_drv.c index 5bfc79be4cef..23c203639a96 100644 --- a/arch/ia64/kernel/mca_drv.c +++ b/arch/ia64/kernel/mca_drv.c @@ -176,7 +176,7 @@ mca_handler_bh(unsigned long paddr, void *iip, unsigned long ipsr) spin_unlock(&mca_bh_lock); /* This process is about to be killed itself */ - do_exit(SIGKILL); + make_task_dead(SIGKILL); } /** diff --git a/arch/ia64/kernel/traps.c b/arch/ia64/kernel/traps.c index e13cb905930f..753642366e12 100644 --- a/arch/ia64/kernel/traps.c +++ b/arch/ia64/kernel/traps.c @@ -85,7 +85,7 @@ die (const char *str, struct pt_regs *regs, long err) if (panic_on_oops) panic("Fatal exception"); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); return 0; } diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 02de2e70c587..4796cccbf74f 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -259,7 +259,7 @@ retry: regs = NULL; bust_spinlocks(0); if (regs) - do_exit(SIGKILL); + make_task_dead(SIGKILL); return; out_of_memory: diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index 34d6458340b0..59fc63feb0dc 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -1131,7 +1131,7 @@ void die_if_kernel (char *str, struct pt_regs *fp, int nr) pr_crit("%s: %08x\n", str, nr); show_registers(fp); add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } asmlinkage void set_esp0(unsigned long ssp) diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index ef46e77e97a5..fcb3a0d8421c 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -48,7 +48,7 @@ int send_fault_sig(struct pt_regs *regs) pr_alert("Unable to handle kernel access"); pr_cont(" at virtual address %p\n", addr); die_if_kernel("Oops", regs, 0 /*error_code*/); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } return 1; diff --git a/arch/microblaze/kernel/exceptions.c b/arch/microblaze/kernel/exceptions.c index 908788497b28..fd153d5fab98 100644 --- a/arch/microblaze/kernel/exceptions.c +++ b/arch/microblaze/kernel/exceptions.c @@ -44,10 +44,10 @@ void die(const char *str, struct pt_regs *fp, long err) pr_warn("Oops: %s, sig: %ld\n", str, err); show_regs(fp); spin_unlock_irq(&die_lock); - /* do_exit() should take care of panic'ing from an interrupt + /* make_task_dead() should take care of panic'ing from an interrupt * context so we don't handle it here */ - do_exit(err); + make_task_dead(err); } /* for user application debugging */ diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index d26b0fb8ea06..a486486b2355 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -422,7 +422,7 @@ void __noreturn die(const char *str, struct pt_regs *regs) if (regs && kexec_should_crash(current)) crash_kexec(regs); - do_exit(sig); + make_task_dead(sig); } extern struct exception_table_entry __start___dbe_table[]; diff --git a/arch/nds32/kernel/fpu.c b/arch/nds32/kernel/fpu.c index 9edd7ed7d7bf..701c09a668de 100644 --- a/arch/nds32/kernel/fpu.c +++ b/arch/nds32/kernel/fpu.c @@ -223,7 +223,7 @@ inline void handle_fpu_exception(struct pt_regs *regs) } } else if (fpcsr & FPCSR_mskRIT) { if (!user_mode(regs)) - do_exit(SIGILL); + make_task_dead(SIGILL); si_signo = SIGILL; } diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c index ca75d475eda4..c0a8f3344fb9 100644 --- a/arch/nds32/kernel/traps.c +++ b/arch/nds32/kernel/traps.c @@ -141,7 +141,7 @@ void __noreturn die(const char *str, struct pt_regs *regs, int err) bust_spinlocks(0); spin_unlock_irq(&die_lock); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } EXPORT_SYMBOL(die); @@ -240,7 +240,7 @@ void unhandled_interruption(struct pt_regs *regs) pr_emerg("unhandled_interruption\n"); show_regs(regs); if (!user_mode(regs)) - do_exit(SIGKILL); + make_task_dead(SIGKILL); force_sig(SIGKILL); } @@ -251,7 +251,7 @@ void unhandled_exceptions(unsigned long entry, unsigned long addr, addr, type); show_regs(regs); if (!user_mode(regs)) - do_exit(SIGKILL); + make_task_dead(SIGKILL); force_sig(SIGKILL); } @@ -278,7 +278,7 @@ void do_revinsn(struct pt_regs *regs) pr_emerg("Reserved Instruction\n"); show_regs(regs); if (!user_mode(regs)) - do_exit(SIGILL); + make_task_dead(SIGILL); force_sig(SIGILL); } diff --git a/arch/nios2/kernel/traps.c b/arch/nios2/kernel/traps.c index 596986a74a26..85ac49d64cf7 100644 --- a/arch/nios2/kernel/traps.c +++ b/arch/nios2/kernel/traps.c @@ -37,10 +37,10 @@ void die(const char *str, struct pt_regs *regs, long err) show_regs(regs); spin_unlock_irq(&die_lock); /* - * do_exit() should take care of panic'ing from an interrupt + * make_task_dead() should take care of panic'ing from an interrupt * context so we don't handle it here */ - do_exit(err); + make_task_dead(err); } void _exception(int signo, struct pt_regs *regs, int code, unsigned long addr) diff --git a/arch/openrisc/kernel/traps.c b/arch/openrisc/kernel/traps.c index 0898cb159fac..0446a3c34372 100644 --- a/arch/openrisc/kernel/traps.c +++ b/arch/openrisc/kernel/traps.c @@ -212,7 +212,7 @@ void __noreturn die(const char *str, struct pt_regs *regs, long err) __asm__ __volatile__("l.nop 1"); do {} while (1); #endif - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } /* This is normally the 'Oops' routine */ diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index b11fb26ce299..df2122c50d78 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -269,7 +269,7 @@ void die_if_kernel(char *str, struct pt_regs *regs, long err) panic("Fatal exception"); oops_exit(); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } /* gdb uses break 4,8 */ diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 11741703d26e..a08bb7cefdc5 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -245,7 +245,7 @@ static void oops_end(unsigned long flags, struct pt_regs *regs, if (panic_on_oops) panic("Fatal exception"); - do_exit(signr); + make_task_dead(signr); } NOKPROBE_SYMBOL(oops_end); @@ -792,9 +792,9 @@ int machine_check_generic(struct pt_regs *regs) void die_mce(const char *str, struct pt_regs *regs, long err) { /* - * The machine check wants to kill the interrupted context, but - * do_exit() checks for in_interrupt() and panics in that case, so - * exit the irq/nmi before calling die. + * The machine check wants to kill the interrupted context, + * but make_task_dead() checks for in_interrupt() and panics + * in that case, so exit the irq/nmi before calling die. */ if (in_nmi()) nmi_exit(); diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 0daaa3e4630d..fe92e119e6a3 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -54,7 +54,7 @@ void die(struct pt_regs *regs, const char *str) if (panic_on_oops) panic("Fatal exception"); if (ret != NOTIFY_STOP) - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index aa08dd2f8fae..42118bc728f9 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -31,7 +31,7 @@ static void die_kernel_fault(const char *msg, unsigned long addr, bust_spinlocks(0); die(regs, "Oops"); - do_exit(SIGKILL); + make_task_dead(SIGKILL); } static inline void no_context(struct pt_regs *regs, unsigned long addr) diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index 0681c55e831d..1e3233eb510a 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -224,5 +224,5 @@ void __noreturn die(struct pt_regs *regs, const char *str) if (panic_on_oops) panic("Fatal exception: panic_on_oops"); oops_exit(); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 20f8e1868853..a4d8c058dd27 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -175,7 +175,7 @@ void __s390_handle_mcck(void) "malfunction (code 0x%016lx).\n", mcck.mcck_code); printk(KERN_EMERG "mcck: task: %s, pid: %d.\n", current->comm, current->pid); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } } diff --git a/arch/sh/kernel/traps.c b/arch/sh/kernel/traps.c index cbe3201d4f21..01884054aeb2 100644 --- a/arch/sh/kernel/traps.c +++ b/arch/sh/kernel/traps.c @@ -57,7 +57,7 @@ void __noreturn die(const char *str, struct pt_regs *regs, long err) if (panic_on_oops) panic("Fatal exception"); - do_exit(SIGSEGV); + make_task_dead(SIGSEGV); } void die_if_kernel(const char *str, struct pt_regs *regs, long err) diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index 5630e5a395e0..179aabfa712e 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -86,9 +86,7 @@ void __noreturn die_if_kernel(char *str, struct pt_regs *regs) } printk("Instruction DUMP:"); instruction_dump ((unsigned long *) regs->pc); - if(regs->psr & PSR_PS) - do_exit(SIGKILL); - do_exit(SIGSEGV); + make_task_dead((regs->psr & PSR_PS) ? SIGKILL : SIGSEGV); } void do_hw_interrupt(struct pt_regs *regs, unsigned long type) diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 6863025ed56d..21077821f427 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -2559,9 +2559,7 @@ void __noreturn die_if_kernel(char *str, struct pt_regs *regs) } if (panic_on_oops) panic("Fatal exception"); - if (regs->tstate & TSTATE_PRIV) - do_exit(SIGKILL); - do_exit(SIGSEGV); + make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV); } EXPORT_SYMBOL(die_if_kernel); diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index ccb9d32768f3..7738fad6a85e 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -1248,14 +1248,14 @@ SYM_CODE_START(asm_exc_nmi) SYM_CODE_END(asm_exc_nmi) .pushsection .text, "ax" -SYM_CODE_START(rewind_stack_do_exit) +SYM_CODE_START(rewind_stack_and_make_dead) /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp movl PER_CPU_VAR(cpu_current_top_of_stack), %esi leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp - call do_exit + call make_task_dead 1: jmp 1b -SYM_CODE_END(rewind_stack_do_exit) +SYM_CODE_END(rewind_stack_and_make_dead) .popsection diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index e38a4cf795d9..f09276457942 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1429,7 +1429,7 @@ SYM_CODE_END(ignore_sysret) #endif .pushsection .text, "ax" -SYM_CODE_START(rewind_stack_do_exit) +SYM_CODE_START(rewind_stack_and_make_dead) UNWIND_HINT_FUNC /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp @@ -1438,6 +1438,6 @@ SYM_CODE_START(rewind_stack_do_exit) leaq -PTREGS_SIZE(%rax), %rsp UNWIND_HINT_REGS - call do_exit -SYM_CODE_END(rewind_stack_do_exit) + call make_task_dead +SYM_CODE_END(rewind_stack_and_make_dead) .popsection diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index ea4fe192189d..53de044e5654 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -351,7 +351,7 @@ unsigned long oops_begin(void) } NOKPROBE_SYMBOL(oops_begin); -void __noreturn rewind_stack_do_exit(int signr); +void __noreturn rewind_stack_and_make_dead(int signr); void oops_end(unsigned long flags, struct pt_regs *regs, int signr) { @@ -386,7 +386,7 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr) * reuse the task stack and that existing poisons are invalid. */ kasan_unpoison_task_stack(current); - rewind_stack_do_exit(signr); + rewind_stack_and_make_dead(signr); } NOKPROBE_SYMBOL(oops_end); diff --git a/arch/xtensa/kernel/traps.c b/arch/xtensa/kernel/traps.c index 4b4dbeb2d612..9345007d474d 100644 --- a/arch/xtensa/kernel/traps.c +++ b/arch/xtensa/kernel/traps.c @@ -552,5 +552,5 @@ void __noreturn die(const char * str, struct pt_regs * regs, long err) if (panic_on_oops) panic("Fatal exception"); - do_exit(err); + make_task_dead(err); } diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index ba88a6987400..2d4bbd9c3278 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -59,6 +59,7 @@ extern void sched_post_fork(struct task_struct *p, extern void sched_dead(struct task_struct *p); void __noreturn do_task_dead(void); +void __noreturn make_task_dead(int signr); extern void proc_caches_init(void); diff --git a/kernel/exit.c b/kernel/exit.c index f702a6a63686..bfa513c5b227 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -884,6 +884,15 @@ void __noreturn do_exit(long code) } EXPORT_SYMBOL_GPL(do_exit); +void __noreturn make_task_dead(int signr) +{ + /* + * Take the task off the cpu after something catastrophic has + * happened. + */ + do_exit(signr); +} + void complete_and_exit(struct completion *comp, long code) { if (comp) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 21735829b860..e6ab5687770b 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -168,6 +168,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "panic", "do_exit", "do_task_dead", + "make_task_dead", "__module_put_and_exit", "complete_and_exit", "__reiserfs_panic", @@ -175,7 +176,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "fortify_panic", "usercopy_abort", "machine_real_restart", - "rewind_stack_do_exit", + "rewind_stack_and_make_dead" "kunit_try_catch_throw", "xen_start_kernel", "cpu_bringup_and_idle", -- cgit v1.2.3-59-g8ed1b From bbda86e988d4c124e4cfa816291cbd583ae8bfb1 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 22 Nov 2021 10:27:36 -0600 Subject: exit: Implement kthread_exit The way the per task_struct exit_code is used by kernel threads is not quite compatible how it is used by userspace applications. The low byte of the userspace exit_code value encodes the exit signal. While kthreads just use the value as an int holding ordinary kernel function exit status like -EPERM. Add kthread_exit to clearly separate the two kinds of uses. Signed-off-by: "Eric W. Biederman" --- include/linux/kthread.h | 1 + kernel/kthread.c | 23 +++++++++++++++++++---- tools/objtool/check.c | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 346b0f269161..22c43d419687 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -70,6 +70,7 @@ void *kthread_probe_data(struct task_struct *k); int kthread_park(struct task_struct *k); void kthread_unpark(struct task_struct *k); void kthread_parkme(void); +void kthread_exit(long result) __noreturn; int kthreadd(void *unused); extern struct task_struct *kthreadd_task; diff --git a/kernel/kthread.c b/kernel/kthread.c index 7113003fab63..77b7c3f23f18 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -268,6 +268,21 @@ void kthread_parkme(void) } EXPORT_SYMBOL_GPL(kthread_parkme); +/** + * kthread_exit - Cause the current kthread return @result to kthread_stop(). + * @result: The integer value to return to kthread_stop(). + * + * While kthread_exit can be called directly, it exists so that + * functions which do some additional work in non-modular code such as + * module_put_and_kthread_exit can be implemented. + * + * Does not return. + */ +void __noreturn kthread_exit(long result) +{ + do_exit(result); +} + static int kthread(void *_create) { static const struct sched_param param = { .sched_priority = 0 }; @@ -286,13 +301,13 @@ static int kthread(void *_create) done = xchg(&create->done, NULL); if (!done) { kfree(create); - do_exit(-EINTR); + kthread_exit(-EINTR); } if (!self) { create->result = ERR_PTR(-ENOMEM); complete(done); - do_exit(-ENOMEM); + kthread_exit(-ENOMEM); } self->threadfn = threadfn; @@ -326,7 +341,7 @@ static int kthread(void *_create) __kthread_parkme(self); ret = threadfn(data); } - do_exit(ret); + kthread_exit(ret); } /* called from kernel_clone() to get node information for about to be created task */ @@ -627,7 +642,7 @@ EXPORT_SYMBOL_GPL(kthread_park); * instead of calling wake_up_process(): the thread will exit without * calling threadfn(). * - * If threadfn() may call do_exit() itself, the caller must ensure + * If threadfn() may call kthread_exit() itself, the caller must ensure * task_struct can't go away. * * Returns the result of threadfn(), or %-EINTR if wake_up_process() diff --git a/tools/objtool/check.c b/tools/objtool/check.c index e6ab5687770b..90108fe5610d 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -168,6 +168,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "panic", "do_exit", "do_task_dead", + "kthread_exit", "make_task_dead", "__module_put_and_exit", "complete_and_exit", -- cgit v1.2.3-59-g8ed1b From ca3574bd653aba234a4b31955f2778947403be16 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Fri, 3 Dec 2021 11:00:19 -0600 Subject: exit: Rename module_put_and_exit to module_put_and_kthread_exit Update module_put_and_exit to call kthread_exit instead of do_exit. Change the name to reflect this change in functionality. All of the users of module_put_and_exit are causing the current kthread to exit so this change makes it clear what is happening. There is no functional change. Signed-off-by: "Eric W. Biederman" --- crypto/algboss.c | 4 ++-- fs/cifs/connect.c | 2 +- fs/nfs/callback.c | 4 ++-- fs/nfs/nfs4state.c | 2 +- fs/nfsd/nfssvc.c | 2 +- include/linux/module.h | 6 +++--- kernel/module.c | 6 +++--- net/bluetooth/bnep/core.c | 2 +- net/bluetooth/cmtp/core.c | 2 +- net/bluetooth/hidp/core.c | 2 +- tools/objtool/check.c | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/crypto/algboss.c b/crypto/algboss.c index 1814d2c5188a..eb5fe84efb83 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -67,7 +67,7 @@ out: complete_all(¶m->larval->completion); crypto_alg_put(¶m->larval->alg); kfree(param); - module_put_and_exit(0); + module_put_and_kthread_exit(0); } static int cryptomgr_schedule_probe(struct crypto_larval *larval) @@ -190,7 +190,7 @@ skiptest: crypto_alg_tested(param->driver, err); kfree(param); - module_put_and_exit(0); + module_put_and_kthread_exit(0); } static int cryptomgr_schedule_test(struct crypto_alg *alg) diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 82577a7a5bb1..39fbe9acbf51 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -1139,7 +1139,7 @@ next_pdu: } memalloc_noreclaim_restore(noreclaim_flag); - module_put_and_exit(0); + module_put_and_kthread_exit(0); } /* diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index 86d856de1389..3c86a559a321 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -93,7 +93,7 @@ nfs4_callback_svc(void *vrqstp) svc_process(rqstp); } svc_exit_thread(rqstp); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } @@ -137,7 +137,7 @@ nfs41_callback_svc(void *vrqstp) } } svc_exit_thread(rqstp); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index ecc4594299d6..ea41af731978 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2689,6 +2689,6 @@ static int nfs4_run_state_manager(void *ptr) allow_signal(SIGKILL); nfs4_state_manager(clp); nfs_put_client(clp); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 80431921e5d7..5ce9f14318c4 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -986,7 +986,7 @@ out: /* Release module */ mutex_unlock(&nfsd_mutex); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/include/linux/module.h b/include/linux/module.h index c9f1200b2312..f03be97e9ec1 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -595,9 +595,9 @@ int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type, /* Look for this name: can be of form module:name. */ unsigned long module_kallsyms_lookup_name(const char *name); -extern void __noreturn __module_put_and_exit(struct module *mod, +extern void __noreturn __module_put_and_kthread_exit(struct module *mod, long code); -#define module_put_and_exit(code) __module_put_and_exit(THIS_MODULE, code) +#define module_put_and_kthread_exit(code) __module_put_and_kthread_exit(THIS_MODULE, code) #ifdef CONFIG_MODULE_UNLOAD int module_refcount(struct module *mod); @@ -790,7 +790,7 @@ static inline int unregister_module_notifier(struct notifier_block *nb) return 0; } -#define module_put_and_exit(code) do_exit(code) +#define module_put_and_kthread_exit(code) kthread_exit(code) static inline void print_modules(void) { diff --git a/kernel/module.c b/kernel/module.c index 84a9141a5e15..a3aa00bf270d 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -337,12 +337,12 @@ static inline void add_taint_module(struct module *mod, unsigned flag, * A thread that wants to hold a reference to a module only while it * is running can call this to safely exit. nfsd and lockd use this. */ -void __noreturn __module_put_and_exit(struct module *mod, long code) +void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) { module_put(mod); - do_exit(code); + kthread_exit(code); } -EXPORT_SYMBOL(__module_put_and_exit); +EXPORT_SYMBOL(__module_put_and_kthread_exit); /* Find a module section: 0 means not found. */ static unsigned int find_sec(const struct load_info *info, const char *name) diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index c9add7753b9f..40baa6b7321a 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -535,7 +535,7 @@ static int bnep_session(void *arg) up_write(&bnep_session_sem); free_netdev(dev); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/net/bluetooth/cmtp/core.c b/net/bluetooth/cmtp/core.c index 0a2d78e811cf..9bfded6b74b3 100644 --- a/net/bluetooth/cmtp/core.c +++ b/net/bluetooth/cmtp/core.c @@ -323,7 +323,7 @@ static int cmtp_session(void *arg) up_write(&cmtp_session_sem); kfree(session); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 80848dfc01db..5940744a8cd8 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -1305,7 +1305,7 @@ static int hidp_session_thread(void *arg) l2cap_unregister_user(session->conn, &session->user); hidp_session_put(session); - module_put_and_exit(0); + module_put_and_kthread_exit(0); return 0; } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 90108fe5610d..120e9598c11a 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -170,7 +170,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "do_task_dead", "kthread_exit", "make_task_dead", - "__module_put_and_exit", + "__module_put_and_kthread_exit", "complete_and_exit", "__reiserfs_panic", "lbug_with_loc", -- cgit v1.2.3-59-g8ed1b From cead18552660702a4a46f58e65188fe5f36e9dfe Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 22 Nov 2021 11:15:19 -0600 Subject: exit: Rename complete_and_exit to kthread_complete_and_exit Update complete_and_exit to call kthread_exit instead of do_exit. Change the name to reflect this change in functionality. All of the users of complete_and_exit are causing the current kthread to exit so this change makes it clear what is happening. Move the implementation of kthread_complete_and_exit from kernel/exit.c to to kernel/kthread.c. As this function is kthread specific it makes most sense to live with the kthread functions. There are no functional change. Signed-off-by: "Eric W. Biederman" --- drivers/net/wireless/rsi/rsi_91x_coex.c | 2 +- drivers/net/wireless/rsi/rsi_91x_main.c | 2 +- drivers/net/wireless/rsi/rsi_91x_sdio_ops.c | 2 +- drivers/net/wireless/rsi/rsi_91x_usb_ops.c | 2 +- drivers/pnp/pnpbios/core.c | 6 +++--- drivers/staging/rts5208/rtsx.c | 16 ++++++++-------- drivers/usb/atm/usbatm.c | 2 +- drivers/usb/gadget/function/f_mass_storage.c | 2 +- fs/jffs2/background.c | 2 +- include/linux/kernel.h | 1 - include/linux/kthread.h | 1 + kernel/exit.c | 9 --------- kernel/kthread.c | 21 +++++++++++++++++++++ lib/kunit/try-catch.c | 4 ++-- tools/objtool/check.c | 2 +- 15 files changed, 43 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/rsi/rsi_91x_coex.c b/drivers/net/wireless/rsi/rsi_91x_coex.c index a0c5d02ae88c..8a3d86897ea8 100644 --- a/drivers/net/wireless/rsi/rsi_91x_coex.c +++ b/drivers/net/wireless/rsi/rsi_91x_coex.c @@ -63,7 +63,7 @@ static void rsi_coex_scheduler_thread(struct rsi_common *common) rsi_coex_sched_tx_pkts(coex_cb); } while (atomic_read(&coex_cb->coex_tx_thread.thread_done) == 0); - complete_and_exit(&coex_cb->coex_tx_thread.completion, 0); + kthread_complete_and_exit(&coex_cb->coex_tx_thread.completion, 0); } int rsi_coex_recv_pkt(struct rsi_common *common, u8 *msg) diff --git a/drivers/net/wireless/rsi/rsi_91x_main.c b/drivers/net/wireless/rsi/rsi_91x_main.c index f1bf71e6c608..c7f5cec5e446 100644 --- a/drivers/net/wireless/rsi/rsi_91x_main.c +++ b/drivers/net/wireless/rsi/rsi_91x_main.c @@ -260,7 +260,7 @@ static void rsi_tx_scheduler_thread(struct rsi_common *common) if (common->init_done) rsi_core_qos_processor(common); } while (atomic_read(&common->tx_thread.thread_done) == 0); - complete_and_exit(&common->tx_thread.completion, 0); + kthread_complete_and_exit(&common->tx_thread.completion, 0); } #ifdef CONFIG_RSI_COEX diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c b/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c index 8ace1874e5cb..b2b47a0abcbf 100644 --- a/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c +++ b/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c @@ -75,7 +75,7 @@ void rsi_sdio_rx_thread(struct rsi_common *common) rsi_dbg(INFO_ZONE, "%s: Terminated SDIO RX thread\n", __func__); atomic_inc(&sdev->rx_thread.thread_done); - complete_and_exit(&sdev->rx_thread.completion, 0); + kthread_complete_and_exit(&sdev->rx_thread.completion, 0); } /** diff --git a/drivers/net/wireless/rsi/rsi_91x_usb_ops.c b/drivers/net/wireless/rsi/rsi_91x_usb_ops.c index 4ffcdde1acb1..5130b0e72adc 100644 --- a/drivers/net/wireless/rsi/rsi_91x_usb_ops.c +++ b/drivers/net/wireless/rsi/rsi_91x_usb_ops.c @@ -56,6 +56,6 @@ void rsi_usb_rx_thread(struct rsi_common *common) out: rsi_dbg(INFO_ZONE, "%s: Terminated thread\n", __func__); skb_queue_purge(&dev->rx_q); - complete_and_exit(&dev->rx_thread.completion, 0); + kthread_complete_and_exit(&dev->rx_thread.completion, 0); } diff --git a/drivers/pnp/pnpbios/core.c b/drivers/pnp/pnpbios/core.c index 669ef4700c1a..f7e86ae9f72f 100644 --- a/drivers/pnp/pnpbios/core.c +++ b/drivers/pnp/pnpbios/core.c @@ -160,7 +160,7 @@ static int pnp_dock_thread(void *unused) * No dock to manage */ case PNP_FUNCTION_NOT_SUPPORTED: - complete_and_exit(&unload_sem, 0); + kthread_complete_and_exit(&unload_sem, 0); case PNP_SYSTEM_NOT_DOCKED: d = 0; break; @@ -170,7 +170,7 @@ static int pnp_dock_thread(void *unused) default: pnpbios_print_status("pnp_dock_thread", status); printk(KERN_WARNING "PnPBIOS: disabling dock monitoring.\n"); - complete_and_exit(&unload_sem, 0); + kthread_complete_and_exit(&unload_sem, 0); } if (d != docked) { if (pnp_dock_event(d, &now) == 0) { @@ -183,7 +183,7 @@ static int pnp_dock_thread(void *unused) } } } - complete_and_exit(&unload_sem, 0); + kthread_complete_and_exit(&unload_sem, 0); } static int pnpbios_get_resources(struct pnp_dev *dev) diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c index 91fcf85e150a..5a58dac76c88 100644 --- a/drivers/staging/rts5208/rtsx.c +++ b/drivers/staging/rts5208/rtsx.c @@ -450,13 +450,13 @@ skip_for_abort: * after the down() -- that's necessary for the thread-shutdown * case. * - * complete_and_exit() goes even further than this -- it is safe in - * the case that the thread of the caller is going away (not just - * the structure) -- this is necessary for the module-remove case. - * This is important in preemption kernels, which transfer the flow - * of execution immediately upon a complete(). + * kthread_complete_and_exit() goes even further than this -- + * it is safe in the case that the thread of the caller is going away + * (not just the structure) -- this is necessary for the module-remove + * case. This is important in preemption kernels, which transfer the + * flow of execution immediately upon a complete(). */ - complete_and_exit(&dev->control_exit, 0); + kthread_complete_and_exit(&dev->control_exit, 0); } static int rtsx_polling_thread(void *__dev) @@ -501,7 +501,7 @@ static int rtsx_polling_thread(void *__dev) mutex_unlock(&dev->dev_mutex); } - complete_and_exit(&dev->polling_exit, 0); + kthread_complete_and_exit(&dev->polling_exit, 0); } /* @@ -682,7 +682,7 @@ static int rtsx_scan_thread(void *__dev) /* Should we unbind if no devices were detected? */ } - complete_and_exit(&dev->scanning_done, 0); + kthread_complete_and_exit(&dev->scanning_done, 0); } static void rtsx_init_options(struct rtsx_chip *chip) diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c index da17be1ef64e..e3a49d837609 100644 --- a/drivers/usb/atm/usbatm.c +++ b/drivers/usb/atm/usbatm.c @@ -969,7 +969,7 @@ static int usbatm_do_heavy_init(void *arg) instance->thread = NULL; mutex_unlock(&instance->serialize); - complete_and_exit(&instance->thread_exited, ret); + kthread_complete_and_exit(&instance->thread_exited, ret); } static int usbatm_heavy_init(struct usbatm_data *instance) diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 752439690fda..46dd11dcb3a8 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -2547,7 +2547,7 @@ static int fsg_main_thread(void *common_) up_write(&common->filesem); /* Let fsg_unbind() know the thread has exited */ - complete_and_exit(&common->thread_notifier, 0); + kthread_complete_and_exit(&common->thread_notifier, 0); } diff --git a/fs/jffs2/background.c b/fs/jffs2/background.c index 2b4d5013dc5d..6da92ecaf66d 100644 --- a/fs/jffs2/background.c +++ b/fs/jffs2/background.c @@ -161,5 +161,5 @@ static int jffs2_garbage_collect_thread(void *_c) spin_lock(&c->erase_completion_lock); c->gc_task = NULL; spin_unlock(&c->erase_completion_lock); - complete_and_exit(&c->gc_thread_exit, 0); + kthread_complete_and_exit(&c->gc_thread_exit, 0); } diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 77755ac3e189..055eb203c00e 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -187,7 +187,6 @@ static inline void might_fault(void) { } #endif void do_exit(long error_code) __noreturn; -void complete_and_exit(struct completion *, long) __noreturn; extern int num_to_str(char *buf, int size, unsigned long long num, unsigned int width); diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 22c43d419687..d86a7e3b9a52 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -71,6 +71,7 @@ int kthread_park(struct task_struct *k); void kthread_unpark(struct task_struct *k); void kthread_parkme(void); void kthread_exit(long result) __noreturn; +void kthread_complete_and_exit(struct completion *, long) __noreturn; int kthreadd(void *unused); extern struct task_struct *kthreadd_task; diff --git a/kernel/exit.c b/kernel/exit.c index 57afac845a0a..6c4b04531f17 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -891,15 +891,6 @@ void __noreturn make_task_dead(int signr) do_exit(signr); } -void complete_and_exit(struct completion *comp, long code) -{ - if (comp) - complete(comp); - - do_exit(code); -} -EXPORT_SYMBOL(complete_and_exit); - SYSCALL_DEFINE1(exit, int, error_code) { do_exit((error_code&0xff)<<8); diff --git a/kernel/kthread.c b/kernel/kthread.c index 77b7c3f23f18..4388d6694a7f 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -283,6 +283,27 @@ void __noreturn kthread_exit(long result) do_exit(result); } +/** + * kthread_complete_and exit - Exit the current kthread. + * @comp: Completion to complete + * @code: The integer value to return to kthread_stop(). + * + * If present complete @comp and the reuturn code to kthread_stop(). + * + * A kernel thread whose module may be removed after the completion of + * @comp can use this function exit safely. + * + * Does not return. + */ +void __noreturn kthread_complete_and_exit(struct completion *comp, long code) +{ + if (comp) + complete(comp); + + kthread_exit(code); +} +EXPORT_SYMBOL(kthread_complete_and_exit); + static int kthread(void *_create) { static const struct sched_param param = { .sched_priority = 0 }; diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index 0dd434e40487..be38a2c5ecc2 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -17,7 +17,7 @@ void __noreturn kunit_try_catch_throw(struct kunit_try_catch *try_catch) { try_catch->try_result = -EFAULT; - complete_and_exit(try_catch->try_completion, -EFAULT); + kthread_complete_and_exit(try_catch->try_completion, -EFAULT); } EXPORT_SYMBOL_GPL(kunit_try_catch_throw); @@ -27,7 +27,7 @@ static int kunit_generic_run_threadfn_adapter(void *data) try_catch->try(try_catch->context); - complete_and_exit(try_catch->try_completion, 0); + kthread_complete_and_exit(try_catch->try_completion, 0); } static unsigned long kunit_test_timeout(void) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 120e9598c11a..282273a1ffa5 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -171,7 +171,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "kthread_exit", "make_task_dead", "__module_put_and_kthread_exit", - "complete_and_exit", + "kthread_complete_and_exit", "__reiserfs_panic", "lbug_with_loc", "fortify_panic", -- cgit v1.2.3-59-g8ed1b From 40966e316f86b8cfd83abd31ccb4df729309d3e7 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Thu, 2 Dec 2021 09:56:14 -0600 Subject: kthread: Ensure struct kthread is present for all kthreads Today the rules are a bit iffy and arbitrary about which kernel threads have struct kthread present. Both idle threads and thread started with create_kthread want struct kthread present so that is effectively all kernel threads. Make the rule that if PF_KTHREAD and the task is running then struct kthread is present. This will allow the kernel thread code to using tsk->exit_code with different semantics from ordinary processes. To make ensure that struct kthread is present for all kernel threads move it's allocation into copy_process. Add a deallocation of struct kthread in exec for processes that were kernel threads. Move the allocation of struct kthread for the initial thread earlier so that it is not repeated for each additional idle thread. Move the initialization of struct kthread into set_kthread_struct so that the structure is always and reliably initailized. Clear set_child_tid in free_kthread_struct to ensure the kthread struct is reliably freed during exec. The function free_kthread_struct does not need to clear vfork_done during exec as exec_mm_release called from exec_mmap has already cleared vfork_done. Signed-off-by: "Eric W. Biederman" --- fs/exec.c | 2 ++ include/linux/kthread.h | 2 +- kernel/fork.c | 4 ++++ kernel/kthread.c | 31 ++++++++++++++----------------- kernel/sched/core.c | 16 ++++++++-------- 5 files changed, 29 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/fs/exec.c b/fs/exec.c index 537d92c41105..59cac7c18178 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1307,6 +1307,8 @@ int begin_new_exec(struct linux_binprm * bprm) */ force_uaccess_begin(); + if (me->flags & PF_KTHREAD) + free_kthread_struct(me); me->flags &= ~(PF_RANDOMIZE | PF_FORKNOEXEC | PF_KTHREAD | PF_NOFREEZE | PF_NO_SETAFFINITY); flush_thread(); diff --git a/include/linux/kthread.h b/include/linux/kthread.h index d86a7e3b9a52..4f3433afb54b 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -33,7 +33,7 @@ struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), unsigned int cpu, const char *namefmt); -void set_kthread_struct(struct task_struct *p); +bool set_kthread_struct(struct task_struct *p); void kthread_set_per_cpu(struct task_struct *k, int cpu); bool kthread_is_per_cpu(struct task_struct *k); diff --git a/kernel/fork.c b/kernel/fork.c index 3244cc56b697..04fa3e5d97af 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2118,6 +2118,10 @@ static __latent_entropy struct task_struct *copy_process( p->io_context = NULL; audit_set_context(p, NULL); cgroup_fork(p); + if (p->flags & PF_KTHREAD) { + if (!set_kthread_struct(p)) + goto bad_fork_cleanup_threadgroup_lock; + } #ifdef CONFIG_NUMA p->mempolicy = mpol_dup(p->mempolicy); if (IS_ERR(p->mempolicy)) { diff --git a/kernel/kthread.c b/kernel/kthread.c index 4388d6694a7f..8e5f44bed027 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -93,20 +93,27 @@ static inline struct kthread *__to_kthread(struct task_struct *p) return kthread; } -void set_kthread_struct(struct task_struct *p) +bool set_kthread_struct(struct task_struct *p) { struct kthread *kthread; - if (__to_kthread(p)) - return; + if (WARN_ON_ONCE(to_kthread(p))) + return false; kthread = kzalloc(sizeof(*kthread), GFP_KERNEL); + if (!kthread) + return false; + + init_completion(&kthread->exited); + init_completion(&kthread->parked); + p->vfork_done = &kthread->exited; + /* * We abuse ->set_child_tid to avoid the new member and because it - * can't be wrongly copied by copy_process(). We also rely on fact - * that the caller can't exec, so PF_KTHREAD can't be cleared. + * can't be wrongly copied by copy_process(). */ p->set_child_tid = (__force void __user *)kthread; + return true; } void free_kthread_struct(struct task_struct *k) @@ -114,13 +121,13 @@ void free_kthread_struct(struct task_struct *k) struct kthread *kthread; /* - * Can be NULL if this kthread was created by kernel_thread() - * or if kmalloc() in kthread() failed. + * Can be NULL if kmalloc() in set_kthread_struct() failed. */ kthread = to_kthread(k); #ifdef CONFIG_BLK_CGROUP WARN_ON_ONCE(kthread && kthread->blkcg_css); #endif + k->set_child_tid = (__force void __user *)NULL; kfree(kthread); } @@ -315,7 +322,6 @@ static int kthread(void *_create) struct kthread *self; int ret; - set_kthread_struct(current); self = to_kthread(current); /* If user was SIGKILLed, I release the structure. */ @@ -325,17 +331,8 @@ static int kthread(void *_create) kthread_exit(-EINTR); } - if (!self) { - create->result = ERR_PTR(-ENOMEM); - complete(done); - kthread_exit(-ENOMEM); - } - self->threadfn = threadfn; self->data = data; - init_completion(&self->exited); - init_completion(&self->parked); - current->vfork_done = &self->exited; /* * The new thread inherited kthreadd's priority and CPU mask. Reset diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3c9b0fda64ac..0404a8c572a1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8599,14 +8599,6 @@ void __init init_idle(struct task_struct *idle, int cpu) __sched_fork(0, idle); - /* - * The idle task doesn't need the kthread struct to function, but it - * is dressed up as a per-CPU kthread and thus needs to play the part - * if we want to avoid special-casing it in code that deals with per-CPU - * kthreads. - */ - set_kthread_struct(idle); - raw_spin_lock_irqsave(&idle->pi_lock, flags); raw_spin_rq_lock(rq); @@ -9427,6 +9419,14 @@ void __init sched_init(void) mmgrab(&init_mm); enter_lazy_tlb(&init_mm, current); + /* + * The idle task doesn't need the kthread struct to function, but it + * is dressed up as a per-CPU kthread and thus needs to play the part + * if we want to avoid special-casing it in code that deals with per-CPU + * kthreads. + */ + WARN_ON(set_kthread_struct(current)); + /* * Make us the idle thread. Technically, schedule() should not be * called from this thread, however somewhere below it might be, -- cgit v1.2.3-59-g8ed1b From df5e49c880ea0776806b8a9f8ab95e035272cf6f Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: SUNRPC: change svc_get() to return the svc. It is common for 'get' functions to return the object that was 'got', and there are a couple of places where users of svc_get() would be a little simpler if svc_get() did that. Make it so. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/lockd/svc.c | 6 ++---- fs/nfs/callback.c | 6 ++---- include/linux/sunrpc/svc.h | 3 ++- 3 files changed, 6 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index b220e1b91726..2f50d5b2a8a4 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -430,14 +430,12 @@ static struct svc_serv *lockd_create_svc(void) /* * Check whether we're already up and running. */ - if (nlmsvc_rqst) { + if (nlmsvc_rqst) /* * Note: increase service usage, because later in case of error * svc_destroy() will be called. */ - svc_get(nlmsvc_rqst->rq_server); - return nlmsvc_rqst->rq_server; - } + return svc_get(nlmsvc_rqst->rq_server); /* * Sanity check: if there's no pid, diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index 86d856de1389..6e5e742a42b8 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -266,14 +266,12 @@ static struct svc_serv *nfs_callback_create_svc(int minorversion) /* * Check whether we're already up and running. */ - if (cb_info->serv) { + if (cb_info->serv) /* * Note: increase service usage, because later in case of error * svc_destroy() will be called. */ - svc_get(cb_info->serv); - return cb_info->serv; - } + return svc_get(cb_info->serv); switch (minorversion) { case 0: diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 0ae28ae6caf2..5d9568953fcd 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -120,9 +120,10 @@ struct svc_serv { * change the number of threads. Horrible, but there it is. * Should be called with the "service mutex" held. */ -static inline void svc_get(struct svc_serv *serv) +static inline struct svc_serv *svc_get(struct svc_serv *serv) { serv->sv_nrthreads++; + return serv; } /* -- cgit v1.2.3-59-g8ed1b From 8c62d12740a1450d2e8456d5747f440e10db281a Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: SUNRPC/NFSD: clean up get/put functions. svc_destroy() is poorly named - it doesn't necessarily destroy the svc, it might just reduce the ref count. nfsd_destroy() is poorly named for the same reason. This patch: - removes the refcount functionality from svc_destroy(), moving it to a new svc_put(). Almost all previous callers of svc_destroy() now call svc_put(). - renames nfsd_destroy() to nfsd_put() and improves the code, using the new svc_destroy() rather than svc_put() - removes a few comments that explain the important for balanced get/put calls. This should be obvious. The only non-trivial part of this is that svc_destroy() would call svc_sock_update() on a non-final decrement. It can no longer do that, and svc_put() isn't really a good place of it. This call is now made from svc_exit_thread() which seems like a good place. This makes the call *before* sv_nrthreads is decremented rather than after. This is not particularly important as the call just sets a flag which causes sv_nrthreads set be checked later. A subsequent patch will improve the ordering. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/lockd/svc.c | 6 +----- fs/nfs/callback.c | 14 ++------------ fs/nfsd/nfsctl.c | 4 ++-- fs/nfsd/nfsd.h | 2 +- fs/nfsd/nfssvc.c | 30 ++++++++++++++++-------------- include/linux/sunrpc/svc.h | 26 +++++++++++++++++++++++--- net/sunrpc/svc.c | 19 +++++-------------- 7 files changed, 50 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 2f50d5b2a8a4..135bd86ed3ad 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -431,10 +431,6 @@ static struct svc_serv *lockd_create_svc(void) * Check whether we're already up and running. */ if (nlmsvc_rqst) - /* - * Note: increase service usage, because later in case of error - * svc_destroy() will be called. - */ return svc_get(nlmsvc_rqst->rq_server); /* @@ -495,7 +491,7 @@ int lockd_up(struct net *net, const struct cred *cred) * so we exit through here on both success and failure. */ err_put: - svc_destroy(serv); + svc_put(serv); err_create: mutex_unlock(&nlmsvc_mutex); return error; diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index 6e5e742a42b8..edbc7579b4aa 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -267,10 +267,6 @@ static struct svc_serv *nfs_callback_create_svc(int minorversion) * Check whether we're already up and running. */ if (cb_info->serv) - /* - * Note: increase service usage, because later in case of error - * svc_destroy() will be called. - */ return svc_get(cb_info->serv); switch (minorversion) { @@ -333,16 +329,10 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt) goto err_start; cb_info->users++; - /* - * svc_create creates the svc_serv with sv_nrthreads == 1, and then - * svc_prepare_thread increments that. So we need to call svc_destroy - * on both success and failure so that the refcount is 1 when the - * thread exits. - */ err_net: if (!cb_info->users) cb_info->serv = NULL; - svc_destroy(serv); + svc_put(serv); err_create: mutex_unlock(&nfs_callback_mutex); return ret; @@ -368,7 +358,7 @@ void nfs_callback_down(int minorversion, struct net *net) if (cb_info->users == 0) { svc_get(serv); serv->sv_ops->svo_setup(serv, NULL, 0); - svc_destroy(serv); + svc_put(serv); dprintk("nfs_callback_down: service destroyed\n"); cb_info->serv = NULL; } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index bf4c9996ad92..17521fada83f 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -743,7 +743,7 @@ static ssize_t __write_ports_addfd(char *buf, struct net *net, const struct cred err = svc_addsock(nn->nfsd_serv, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred); if (err < 0 && list_empty(&nn->nfsd_serv->sv_permsocks)) { - nfsd_destroy(net); + nfsd_put(net); return err; } @@ -796,7 +796,7 @@ out_err: if (!list_empty(&nn->nfsd_serv->sv_permsocks)) nn->nfsd_serv->sv_nrthreads--; else - nfsd_destroy(net); + nfsd_put(net); return err; } diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 498e5a489826..3e5008b475ff 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -97,7 +97,7 @@ int nfsd_pool_stats_open(struct inode *, struct file *); int nfsd_pool_stats_release(struct inode *, struct file *); void nfsd_shutdown_threads(struct net *net); -void nfsd_destroy(struct net *net); +void nfsd_put(struct net *net); bool i_am_nfsd(void); diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 80431921e5d7..a0a7564e6c73 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -623,7 +623,7 @@ void nfsd_shutdown_threads(struct net *net) svc_get(serv); /* Kill outstanding nfsd threads */ serv->sv_ops->svo_setup(serv, NULL, 0); - nfsd_destroy(net); + nfsd_put(net); mutex_unlock(&nfsd_mutex); /* Wait for shutdown of nfsd_serv to complete */ wait_for_completion(&nn->nfsd_shutdown_complete); @@ -656,7 +656,10 @@ int nfsd_create_serv(struct net *net) nn->nfsd_serv->sv_maxconn = nn->max_connections; error = svc_bind(nn->nfsd_serv, net); if (error < 0) { - svc_destroy(nn->nfsd_serv); + /* NOT nfsd_put() as notifiers (see below) haven't + * been set up yet. + */ + svc_put(nn->nfsd_serv); nfsd_complete_shutdown(net); return error; } @@ -697,16 +700,16 @@ int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) return 0; } -void nfsd_destroy(struct net *net) +void nfsd_put(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - int destroy = (nn->nfsd_serv->sv_nrthreads == 1); - if (destroy) + nn->nfsd_serv->sv_nrthreads -= 1; + if (nn->nfsd_serv->sv_nrthreads == 0) { svc_shutdown_net(nn->nfsd_serv, net); - svc_destroy(nn->nfsd_serv); - if (destroy) + svc_destroy(nn->nfsd_serv); nfsd_complete_shutdown(net); + } } int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) @@ -758,7 +761,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) if (err) break; } - nfsd_destroy(net); + nfsd_put(net); return err; } @@ -795,7 +798,7 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) error = nfsd_startup_net(net, cred); if (error) - goto out_destroy; + goto out_put; error = nn->nfsd_serv->sv_ops->svo_setup(nn->nfsd_serv, NULL, nrservs); if (error) @@ -808,8 +811,8 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) out_shutdown: if (error < 0 && !nfsd_up_before) nfsd_shutdown_net(net); -out_destroy: - nfsd_destroy(net); /* Release server */ +out_put: + nfsd_put(net); out: mutex_unlock(&nfsd_mutex); return error; @@ -982,7 +985,7 @@ out: /* Release the thread */ svc_exit_thread(rqstp); - nfsd_destroy(net); + nfsd_put(net); /* Release module */ mutex_unlock(&nfsd_mutex); @@ -1109,8 +1112,7 @@ int nfsd_pool_stats_release(struct inode *inode, struct file *file) struct net *net = inode->i_sb->s_fs_info; mutex_lock(&nfsd_mutex); - /* this function really, really should have been called svc_put() */ - nfsd_destroy(net); + nfsd_put(net); mutex_unlock(&nfsd_mutex); return ret; } diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 5d9568953fcd..73d56d33a36d 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -114,8 +114,13 @@ struct svc_serv { #endif /* CONFIG_SUNRPC_BACKCHANNEL */ }; -/* - * We use sv_nrthreads as a reference count. svc_destroy() drops +/** + * svc_get() - increment reference count on a SUNRPC serv + * @serv: the svc_serv to have count incremented + * + * Returns: the svc_serv that was passed in. + * + * We use sv_nrthreads as a reference count. svc_put() drops * this refcount, so we need to bump it up around operations that * change the number of threads. Horrible, but there it is. * Should be called with the "service mutex" held. @@ -126,6 +131,22 @@ static inline struct svc_serv *svc_get(struct svc_serv *serv) return serv; } +void svc_destroy(struct svc_serv *serv); + +/** + * svc_put - decrement reference count on a SUNRPC serv + * @serv: the svc_serv to have count decremented + * + * When the reference count reaches zero, svc_destroy() + * is called to clean up and free the serv. + */ +static inline void svc_put(struct svc_serv *serv) +{ + serv->sv_nrthreads -= 1; + if (serv->sv_nrthreads == 0) + svc_destroy(serv); +} + /* * Maximum payload size supported by a kernel RPC server. * This is use to determine the max number of pages nfsd is @@ -515,7 +536,6 @@ struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); int svc_set_num_threads_sync(struct svc_serv *, struct svc_pool *, int); int svc_pool_stats_open(struct svc_serv *serv, struct file *file); -void svc_destroy(struct svc_serv *); void svc_shutdown_net(struct svc_serv *, struct net *); int svc_process(struct svc_rqst *); int bc_svc_process(struct svc_serv *, struct rpc_rqst *, diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 4292278a9552..55a1bf0d129f 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -528,17 +528,7 @@ EXPORT_SYMBOL_GPL(svc_shutdown_net); void svc_destroy(struct svc_serv *serv) { - dprintk("svc: svc_destroy(%s, %d)\n", - serv->sv_program->pg_name, - serv->sv_nrthreads); - - if (serv->sv_nrthreads) { - if (--(serv->sv_nrthreads) != 0) { - svc_sock_update_bufs(serv); - return; - } - } else - printk("svc_destroy: no threads for serv=%p!\n", serv); + dprintk("svc: svc_destroy(%s)\n", serv->sv_program->pg_name); del_timer_sync(&serv->sv_temptimer); @@ -892,9 +882,10 @@ svc_exit_thread(struct svc_rqst *rqstp) svc_rqst_free(rqstp); - /* Release the server */ - if (serv) - svc_destroy(serv); + if (!serv) + return; + svc_sock_update_bufs(serv); + svc_destroy(serv); } EXPORT_SYMBOL_GPL(svc_exit_thread); -- cgit v1.2.3-59-g8ed1b From ec52361df99b490f6af412b046df9799b92c1050 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: SUNRPC: stop using ->sv_nrthreads as a refcount The use of sv_nrthreads as a general refcount results in clumsy code, as is seen by various comments needed to explain the situation. This patch introduces a 'struct kref' and uses that for reference counting, leaving sv_nrthreads to be a pure count of threads. The kref is managed particularly in svc_get() and svc_put(), and also nfsd_put(); svc_destroy() now takes a pointer to the embedded kref, rather than to the serv. nfsd allows the svc_serv to exist with ->sv_nrhtreads being zero. This happens when a transport is created before the first thread is started. To support this, a 'keep_active' flag is introduced which holds a ref on the svc_serv. This is set when any listening socket is successfully added (unless there are running threads), and cleared when the number of threads is set. So when the last thread exits, the nfs_serv will be destroyed. The use of 'keep_active' replaces previous code which checked if there were any permanent sockets. We no longer clear ->rq_server when nfsd() exits. This was done to prevent svc_exit_thread() from calling svc_destroy(). Instead we take an extra reference to the svc_serv to prevent svc_destroy() from being called. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/lockd/svc.c | 4 ---- fs/nfs/callback.c | 2 +- fs/nfsd/netns.h | 7 +++++++ fs/nfsd/nfsctl.c | 22 ++++++++++------------ fs/nfsd/nfssvc.c | 42 ++++++++++++++++++++++++++---------------- include/linux/sunrpc/svc.h | 14 ++++---------- net/sunrpc/svc.c | 22 +++++++++++----------- 7 files changed, 59 insertions(+), 54 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 135bd86ed3ad..a9669b106dbd 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -486,10 +486,6 @@ int lockd_up(struct net *net, const struct cred *cred) goto err_put; } nlmsvc_users++; - /* - * Note: svc_serv structures have an initial use count of 1, - * so we exit through here on both success and failure. - */ err_put: svc_put(serv); err_create: diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index edbc7579b4aa..d9d78ffd1d65 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -169,7 +169,7 @@ static int nfs_callback_start_svc(int minorversion, struct rpc_xprt *xprt, if (nrservs < NFS4_MIN_NR_CALLBACK_THREADS) nrservs = NFS4_MIN_NR_CALLBACK_THREADS; - if (serv->sv_nrthreads-1 == nrservs) + if (serv->sv_nrthreads == nrservs) return 0; ret = serv->sv_ops->svo_setup(serv, NULL, nrservs); diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 935c1028c217..08bcd8f23b01 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -123,6 +123,13 @@ struct nfsd_net { u32 clverifier_counter; struct svc_serv *nfsd_serv; + /* When a listening socket is added to nfsd, keep_active is set + * and this justifies a reference on nfsd_serv. This stops + * nfsd_serv from being freed. When the number of threads is + * set, keep_active is cleared and the reference is dropped. So + * when the last thread exits, the service will be destroyed. + */ + int keep_active; wait_queue_head_t ntf_wq; atomic_t ntf_refcnt; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 17521fada83f..7b557eb8211a 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -742,13 +742,12 @@ static ssize_t __write_ports_addfd(char *buf, struct net *net, const struct cred return err; err = svc_addsock(nn->nfsd_serv, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred); - if (err < 0 && list_empty(&nn->nfsd_serv->sv_permsocks)) { - nfsd_put(net); - return err; - } - /* Decrease the count, but don't shut down the service */ - nn->nfsd_serv->sv_nrthreads--; + if (err >= 0 && + !nn->nfsd_serv->sv_nrthreads && !xchg(&nn->keep_active, 1)) + svc_get(nn->nfsd_serv); + + nfsd_put(net); return err; } @@ -783,8 +782,10 @@ static ssize_t __write_ports_addxprt(char *buf, struct net *net, const struct cr if (err < 0 && err != -EAFNOSUPPORT) goto out_close; - /* Decrease the count, but don't shut down the service */ - nn->nfsd_serv->sv_nrthreads--; + if (!nn->nfsd_serv->sv_nrthreads && !xchg(&nn->keep_active, 1)) + svc_get(nn->nfsd_serv); + + nfsd_put(net); return 0; out_close: xprt = svc_find_xprt(nn->nfsd_serv, transport, net, PF_INET, port); @@ -793,10 +794,7 @@ out_close: svc_xprt_put(xprt); } out_err: - if (!list_empty(&nn->nfsd_serv->sv_permsocks)) - nn->nfsd_serv->sv_nrthreads--; - else - nfsd_put(net); + nfsd_put(net); return err; } diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index a0a7564e6c73..5f605e7e8091 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -60,13 +60,13 @@ static __be32 nfsd_init_request(struct svc_rqst *, * extent ->sv_temp_socks and ->sv_permsocks. It also protects nfsdstats.th_cnt * * If (out side the lock) nn->nfsd_serv is non-NULL, then it must point to a - * properly initialised 'struct svc_serv' with ->sv_nrthreads > 0. That number - * of nfsd threads must exist and each must listed in ->sp_all_threads in each - * entry of ->sv_pools[]. + * properly initialised 'struct svc_serv' with ->sv_nrthreads > 0 (unless + * nn->keep_active is set). That number of nfsd threads must + * exist and each must be listed in ->sp_all_threads in some entry of + * ->sv_pools[]. * - * Transitions of the thread count between zero and non-zero are of particular - * interest since the svc_serv needs to be created and initialized at that - * point, or freed. + * Each active thread holds a counted reference on nn->nfsd_serv, as does + * the nn->keep_active flag and various transient calls to svc_get(). * * Finally, the nfsd_mutex also protects some of the global variables that are * accessed when nfsd starts and that are settable via the write_* routines in @@ -700,14 +700,22 @@ int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) return 0; } +/* This is the callback for kref_put() below. + * There is no code here as the first thing to be done is + * call svc_shutdown_net(), but we cannot get the 'net' from + * the kref. So do all the work when kref_put returns true. + */ +static void nfsd_noop(struct kref *ref) +{ +} + void nfsd_put(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->nfsd_serv->sv_nrthreads -= 1; - if (nn->nfsd_serv->sv_nrthreads == 0) { + if (kref_put(&nn->nfsd_serv->sv_refcnt, nfsd_noop)) { svc_shutdown_net(nn->nfsd_serv, net); - svc_destroy(nn->nfsd_serv); + svc_destroy(&nn->nfsd_serv->sv_refcnt); nfsd_complete_shutdown(net); } } @@ -803,15 +811,14 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) NULL, nrservs); if (error) goto out_shutdown; - /* We are holding a reference to nn->nfsd_serv which - * we don't want to count in the return value, - * so subtract 1 - */ - error = nn->nfsd_serv->sv_nrthreads - 1; + error = nn->nfsd_serv->sv_nrthreads; out_shutdown: if (error < 0 && !nfsd_up_before) nfsd_shutdown_net(net); out_put: + /* Threads now hold service active */ + if (xchg(&nn->keep_active, 0)) + nfsd_put(net); nfsd_put(net); out: mutex_unlock(&nfsd_mutex); @@ -980,11 +987,15 @@ nfsd(void *vrqstp) nfsdstats.th_cnt --; out: - rqstp->rq_server = NULL; + /* Take an extra ref so that the svc_put in svc_exit_thread() + * doesn't call svc_destroy() + */ + svc_get(nn->nfsd_serv); /* Release the thread */ svc_exit_thread(rqstp); + /* Now if needed we call svc_destroy in appropriate context */ nfsd_put(net); /* Release module */ @@ -1099,7 +1110,6 @@ int nfsd_pool_stats_open(struct inode *inode, struct file *file) mutex_unlock(&nfsd_mutex); return -ENODEV; } - /* bump up the psudo refcount while traversing */ svc_get(nn->nfsd_serv); ret = svc_pool_stats_open(nn->nfsd_serv, file); mutex_unlock(&nfsd_mutex); diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 73d56d33a36d..3903b4ae8ac5 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -85,6 +85,7 @@ struct svc_serv { struct svc_program * sv_program; /* RPC program */ struct svc_stat * sv_stats; /* RPC statistics */ spinlock_t sv_lock; + struct kref sv_refcnt; unsigned int sv_nrthreads; /* # of server threads */ unsigned int sv_maxconn; /* max connections allowed or * '0' causing max to be based @@ -119,19 +120,14 @@ struct svc_serv { * @serv: the svc_serv to have count incremented * * Returns: the svc_serv that was passed in. - * - * We use sv_nrthreads as a reference count. svc_put() drops - * this refcount, so we need to bump it up around operations that - * change the number of threads. Horrible, but there it is. - * Should be called with the "service mutex" held. */ static inline struct svc_serv *svc_get(struct svc_serv *serv) { - serv->sv_nrthreads++; + kref_get(&serv->sv_refcnt); return serv; } -void svc_destroy(struct svc_serv *serv); +void svc_destroy(struct kref *); /** * svc_put - decrement reference count on a SUNRPC serv @@ -142,9 +138,7 @@ void svc_destroy(struct svc_serv *serv); */ static inline void svc_put(struct svc_serv *serv) { - serv->sv_nrthreads -= 1; - if (serv->sv_nrthreads == 0) - svc_destroy(serv); + kref_put(&serv->sv_refcnt, svc_destroy); } /* diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 55a1bf0d129f..acddc6e12e9e 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -435,7 +435,7 @@ __svc_create(struct svc_program *prog, unsigned int bufsize, int npools, return NULL; serv->sv_name = prog->pg_name; serv->sv_program = prog; - serv->sv_nrthreads = 1; + kref_init(&serv->sv_refcnt); serv->sv_stats = prog->pg_stats; if (bufsize > RPCSVC_MAXPAYLOAD) bufsize = RPCSVC_MAXPAYLOAD; @@ -526,10 +526,11 @@ EXPORT_SYMBOL_GPL(svc_shutdown_net); * protect the sv_nrthreads, sv_permsocks and sv_tempsocks. */ void -svc_destroy(struct svc_serv *serv) +svc_destroy(struct kref *ref) { - dprintk("svc: svc_destroy(%s)\n", serv->sv_program->pg_name); + struct svc_serv *serv = container_of(ref, struct svc_serv, sv_refcnt); + dprintk("svc: svc_destroy(%s)\n", serv->sv_program->pg_name); del_timer_sync(&serv->sv_temptimer); /* @@ -637,6 +638,7 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) if (!rqstp) return ERR_PTR(-ENOMEM); + svc_get(serv); serv->sv_nrthreads++; spin_lock_bh(&pool->sp_lock); pool->sp_nrthreads++; @@ -776,8 +778,7 @@ int svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) { if (pool == NULL) { - /* The -1 assumes caller has done a svc_get() */ - nrservs -= (serv->sv_nrthreads-1); + nrservs -= serv->sv_nrthreads; } else { spin_lock_bh(&pool->sp_lock); nrservs -= pool->sp_nrthreads; @@ -814,8 +815,7 @@ int svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs) { if (pool == NULL) { - /* The -1 assumes caller has done a svc_get() */ - nrservs -= (serv->sv_nrthreads-1); + nrservs -= serv->sv_nrthreads; } else { spin_lock_bh(&pool->sp_lock); nrservs -= pool->sp_nrthreads; @@ -880,12 +880,12 @@ svc_exit_thread(struct svc_rqst *rqstp) list_del_rcu(&rqstp->rq_all); spin_unlock_bh(&pool->sp_lock); + serv->sv_nrthreads -= 1; + svc_sock_update_bufs(serv); + svc_rqst_free(rqstp); - if (!serv) - return; - svc_sock_update_bufs(serv); - svc_destroy(serv); + svc_put(serv); } EXPORT_SYMBOL_GPL(svc_exit_thread); -- cgit v1.2.3-59-g8ed1b From 3409e4f1e8f239f0ed81be0b068ecf4e73e2e826 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: NFSD: Make it possible to use svc_set_num_threads_sync nfsd cannot currently use svc_set_num_threads_sync. It instead uses svc_set_num_threads which does *not* wait for threads to all exit, and has a separate mechanism (nfsd_shutdown_complete) to wait for completion. The reason that nfsd is unlike other services is that nfsd threads can exit separately from svc_set_num_threads being called - they die on receipt of SIGKILL. Also, when the last thread exits, the service must be shut down (sockets closed). For this, the nfsd_mutex needs to be taken, and as that mutex needs to be held while svc_set_num_threads is called, the one cannot wait for the other. This patch changes the nfsd thread so that it can drop the ref on the service without blocking on nfsd_mutex, so that svc_set_num_threads_sync can be used: - if it can drop a non-last reference, it does that. This does not trigger shutdown and does not require a mutex. This will likely happen for all but the last thread signalled, and for all threads being shut down by nfsd_shutdown_threads() - if it can get the mutex without blocking (trylock), it does that and then drops the reference. This will likely happen for the last thread killed by SIGKILL - Otherwise there might be an unrelated task holding the mutex, possibly in another network namespace, or nfsd_shutdown_threads() might be just about to get a reference on the service, after which we can drop ours safely. We cannot conveniently get wakeup notifications on these events, and we are unlikely to need to, so we sleep briefly and check again. With this we can discard nfsd_shutdown_complete and nfsd_complete_shutdown(), and switch to svc_set_num_threads_sync. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfsd/netns.h | 3 --- fs/nfsd/nfssvc.c | 41 ++++++++++++++++++++--------------------- include/linux/sunrpc/svc.h | 13 +++++++++++++ 3 files changed, 33 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 08bcd8f23b01..1fd59eb0730b 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -134,9 +134,6 @@ struct nfsd_net { wait_queue_head_t ntf_wq; atomic_t ntf_refcnt; - /* Allow umount to wait for nfsd state cleanup */ - struct completion nfsd_shutdown_complete; - /* * clientid and stateid data for construction of net unique COPY * stateids. diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 097abd8b059c..d0d9107a1b93 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -593,20 +593,10 @@ static const struct svc_serv_ops nfsd_thread_sv_ops = { .svo_shutdown = nfsd_last_thread, .svo_function = nfsd, .svo_enqueue_xprt = svc_xprt_do_enqueue, - .svo_setup = svc_set_num_threads, + .svo_setup = svc_set_num_threads_sync, .svo_module = THIS_MODULE, }; -static void nfsd_complete_shutdown(struct net *net) -{ - struct nfsd_net *nn = net_generic(net, nfsd_net_id); - - WARN_ON(!mutex_is_locked(&nfsd_mutex)); - - nn->nfsd_serv = NULL; - complete(&nn->nfsd_shutdown_complete); -} - void nfsd_shutdown_threads(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); @@ -624,8 +614,6 @@ void nfsd_shutdown_threads(struct net *net) serv->sv_ops->svo_setup(serv, NULL, 0); nfsd_put(net); mutex_unlock(&nfsd_mutex); - /* Wait for shutdown of nfsd_serv to complete */ - wait_for_completion(&nn->nfsd_shutdown_complete); } bool i_am_nfsd(void) @@ -650,7 +638,6 @@ int nfsd_create_serv(struct net *net) &nfsd_thread_sv_ops); if (nn->nfsd_serv == NULL) return -ENOMEM; - init_completion(&nn->nfsd_shutdown_complete); nn->nfsd_serv->sv_maxconn = nn->max_connections; error = svc_bind(nn->nfsd_serv, net); @@ -659,7 +646,7 @@ int nfsd_create_serv(struct net *net) * been set up yet. */ svc_put(nn->nfsd_serv); - nfsd_complete_shutdown(net); + nn->nfsd_serv = NULL; return error; } @@ -715,7 +702,7 @@ void nfsd_put(struct net *net) if (kref_put(&nn->nfsd_serv->sv_refcnt, nfsd_noop)) { svc_shutdown_net(nn->nfsd_serv, net); svc_destroy(&nn->nfsd_serv->sv_refcnt); - nfsd_complete_shutdown(net); + nn->nfsd_serv = NULL; } } @@ -743,7 +730,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) if (tot > NFSD_MAXSERVS) { /* total too large: scale down requested numbers */ for (i = 0; i < n && tot > 0; i++) { - int new = nthreads[i] * NFSD_MAXSERVS / tot; + int new = nthreads[i] * NFSD_MAXSERVS / tot; tot -= (nthreads[i] - new); nthreads[i] = new; } @@ -989,10 +976,22 @@ out: /* Release the thread */ svc_exit_thread(rqstp); - /* Now if needed we call svc_destroy in appropriate context */ - mutex_lock(&nfsd_mutex); - nfsd_put(net); - mutex_unlock(&nfsd_mutex); + /* We need to drop a ref, but may not drop the last reference + * without holding nfsd_mutex, and we cannot wait for nfsd_mutex as that + * could deadlock with nfsd_shutdown_threads() waiting for us. + * So three options are: + * - drop a non-final reference, + * - get the mutex without waiting + * - sleep briefly andd try the above again + */ + while (!svc_put_not_last(nn->nfsd_serv)) { + if (mutex_trylock(&nfsd_mutex)) { + nfsd_put(net); + mutex_unlock(&nfsd_mutex); + break; + } + msleep(20); + } /* Release module */ module_put_and_exit(0); diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 3903b4ae8ac5..36bfc0281988 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -141,6 +141,19 @@ static inline void svc_put(struct svc_serv *serv) kref_put(&serv->sv_refcnt, svc_destroy); } +/** + * svc_put_not_last - decrement non-final reference count on SUNRPC serv + * @serv: the svc_serv to have count decremented + * + * Returns: %true is refcount was decremented. + * + * If the refcount is 1, it is not decremented and instead failure is reported. + */ +static inline bool svc_put_not_last(struct svc_serv *serv) +{ + return refcount_dec_not_one(&serv->sv_refcnt.refcount); +} + /* * Maximum payload size supported by a kernel RPC server. * This is use to determine the max number of pages nfsd is -- cgit v1.2.3-59-g8ed1b From 3ebdbe5203a874614819700d3f470724cb803709 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: SUNRPC: discard svo_setup and rename svc_set_num_threads_sync() The ->svo_setup callback serves no purpose. It is always called from within the same module that chooses which callback is needed. So discard it and call the relevant function directly. Now that svc_set_num_threads() is no longer used remove it and rename svc_set_num_threads_sync() to remove the "_sync" suffix. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/nfs/callback.c | 8 +++----- fs/nfsd/nfssvc.c | 11 +++++------ include/linux/sunrpc/svc.h | 4 ---- net/sunrpc/svc.c | 49 ++-------------------------------------------- 4 files changed, 10 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c index d9d78ffd1d65..6cdc9d18a7dd 100644 --- a/fs/nfs/callback.c +++ b/fs/nfs/callback.c @@ -172,9 +172,9 @@ static int nfs_callback_start_svc(int minorversion, struct rpc_xprt *xprt, if (serv->sv_nrthreads == nrservs) return 0; - ret = serv->sv_ops->svo_setup(serv, NULL, nrservs); + ret = svc_set_num_threads(serv, NULL, nrservs); if (ret) { - serv->sv_ops->svo_setup(serv, NULL, 0); + svc_set_num_threads(serv, NULL, 0); return ret; } dprintk("nfs_callback_up: service started\n"); @@ -235,14 +235,12 @@ err_bind: static const struct svc_serv_ops nfs40_cb_sv_ops = { .svo_function = nfs4_callback_svc, .svo_enqueue_xprt = svc_xprt_do_enqueue, - .svo_setup = svc_set_num_threads_sync, .svo_module = THIS_MODULE, }; #if defined(CONFIG_NFS_V4_1) static const struct svc_serv_ops nfs41_cb_sv_ops = { .svo_function = nfs41_callback_svc, .svo_enqueue_xprt = svc_xprt_do_enqueue, - .svo_setup = svc_set_num_threads_sync, .svo_module = THIS_MODULE, }; @@ -357,7 +355,7 @@ void nfs_callback_down(int minorversion, struct net *net) cb_info->users--; if (cb_info->users == 0) { svc_get(serv); - serv->sv_ops->svo_setup(serv, NULL, 0); + svc_set_num_threads(serv, NULL, 0); svc_put(serv); dprintk("nfs_callback_down: service destroyed\n"); cb_info->serv = NULL; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index d0d9107a1b93..020156e96bdb 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -593,7 +593,6 @@ static const struct svc_serv_ops nfsd_thread_sv_ops = { .svo_shutdown = nfsd_last_thread, .svo_function = nfsd, .svo_enqueue_xprt = svc_xprt_do_enqueue, - .svo_setup = svc_set_num_threads_sync, .svo_module = THIS_MODULE, }; @@ -611,7 +610,7 @@ void nfsd_shutdown_threads(struct net *net) svc_get(serv); /* Kill outstanding nfsd threads */ - serv->sv_ops->svo_setup(serv, NULL, 0); + svc_set_num_threads(serv, NULL, 0); nfsd_put(net); mutex_unlock(&nfsd_mutex); } @@ -750,8 +749,9 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) /* apply the new numbers */ svc_get(nn->nfsd_serv); for (i = 0; i < n; i++) { - err = nn->nfsd_serv->sv_ops->svo_setup(nn->nfsd_serv, - &nn->nfsd_serv->sv_pools[i], nthreads[i]); + err = svc_set_num_threads(nn->nfsd_serv, + &nn->nfsd_serv->sv_pools[i], + nthreads[i]); if (err) break; } @@ -793,8 +793,7 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) error = nfsd_startup_net(net, cred); if (error) goto out_put; - error = nn->nfsd_serv->sv_ops->svo_setup(nn->nfsd_serv, - NULL, nrservs); + error = svc_set_num_threads(nn->nfsd_serv, NULL, nrservs); if (error) goto out_shutdown; error = nn->nfsd_serv->sv_nrthreads; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 36bfc0281988..0b38c6eaf985 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -64,9 +64,6 @@ struct svc_serv_ops { /* queue up a transport for servicing */ void (*svo_enqueue_xprt)(struct svc_xprt *); - /* set up thread (or whatever) execution context */ - int (*svo_setup)(struct svc_serv *, struct svc_pool *, int); - /* optional module to count when adding threads (pooled svcs only) */ struct module *svo_module; }; @@ -541,7 +538,6 @@ void svc_pool_map_put(void); struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, const struct svc_serv_ops *); int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); -int svc_set_num_threads_sync(struct svc_serv *, struct svc_pool *, int); int svc_pool_stats_open(struct svc_serv *serv, struct file *file); void svc_shutdown_net(struct svc_serv *, struct net *); int svc_process(struct svc_rqst *); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 2b2042234e4b..5513f8c9a8d6 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -743,58 +743,13 @@ svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) return 0; } - -/* destroy old threads */ -static int -svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) -{ - struct task_struct *task; - unsigned int state = serv->sv_nrthreads-1; - - /* destroy old threads */ - do { - task = choose_victim(serv, pool, &state); - if (task == NULL) - break; - send_sig(SIGINT, task, 1); - nrservs++; - } while (nrservs < 0); - - return 0; -} - /* * Create or destroy enough new threads to make the number * of threads the given number. If `pool' is non-NULL, applies * only to threads in that pool, otherwise round-robins between * all pools. Caller must ensure that mutual exclusion between this and * server startup or shutdown. - * - * Destroying threads relies on the service threads filling in - * rqstp->rq_task, which only the nfs ones do. Assumes the serv - * has been created using svc_create_pooled(). - * - * Based on code that used to be in nfsd_svc() but tweaked - * to be pool-aware. */ -int -svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) -{ - if (pool == NULL) { - nrservs -= serv->sv_nrthreads; - } else { - spin_lock_bh(&pool->sp_lock); - nrservs -= pool->sp_nrthreads; - spin_unlock_bh(&pool->sp_lock); - } - - if (nrservs > 0) - return svc_start_kthreads(serv, pool, nrservs); - if (nrservs < 0) - return svc_signal_kthreads(serv, pool, nrservs); - return 0; -} -EXPORT_SYMBOL_GPL(svc_set_num_threads); /* destroy old threads */ static int @@ -815,7 +770,7 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) } int -svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs) +svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) { if (pool == NULL) { nrservs -= serv->sv_nrthreads; @@ -831,7 +786,7 @@ svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrser return svc_stop_kthreads(serv, pool, nrservs); return 0; } -EXPORT_SYMBOL_GPL(svc_set_num_threads_sync); +EXPORT_SYMBOL_GPL(svc_set_num_threads); /** * svc_rqst_replace_page - Replace one page in rq_pages[] -- cgit v1.2.3-59-g8ed1b From cf0e124e0a489944d08fcc3c694d2b234d2cc658 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: SUNRPC: move the pool_map definitions (back) into svc.c These definitions are not used outside of svc.c, and there is no evidence that they ever have been. So move them into svc.c and make the declarations 'static'. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 25 ------------------------- net/sunrpc/svc.c | 31 +++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 0b38c6eaf985..d69e6108cb83 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -494,29 +494,6 @@ struct svc_procedure { const char * pc_name; /* for display */ }; -/* - * Mode for mapping cpus to pools. - */ -enum { - SVC_POOL_AUTO = -1, /* choose one of the others */ - SVC_POOL_GLOBAL, /* no mapping, just a single global pool - * (legacy & UP mode) */ - SVC_POOL_PERCPU, /* one pool per cpu */ - SVC_POOL_PERNODE /* one pool per numa node */ -}; - -struct svc_pool_map { - int count; /* How many svc_servs use us */ - int mode; /* Note: int not enum to avoid - * warnings about "enumeration value - * not handled in switch" */ - unsigned int npools; - unsigned int *pool_to; /* maps pool id to cpu or node */ - unsigned int *to_pool; /* maps cpu or node to pool id */ -}; - -extern struct svc_pool_map svc_pool_map; - /* * Function prototypes. */ @@ -533,8 +510,6 @@ void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page); void svc_rqst_free(struct svc_rqst *); void svc_exit_thread(struct svc_rqst *); -unsigned int svc_pool_map_get(void); -void svc_pool_map_put(void); struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, const struct svc_serv_ops *); int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 5513f8c9a8d6..f0dd9ef7e0cd 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -41,14 +41,35 @@ static void svc_unregister(const struct svc_serv *serv, struct net *net); #define SVC_POOL_DEFAULT SVC_POOL_GLOBAL +/* + * Mode for mapping cpus to pools. + */ +enum { + SVC_POOL_AUTO = -1, /* choose one of the others */ + SVC_POOL_GLOBAL, /* no mapping, just a single global pool + * (legacy & UP mode) */ + SVC_POOL_PERCPU, /* one pool per cpu */ + SVC_POOL_PERNODE /* one pool per numa node */ +}; + /* * Structure for mapping cpus to pools and vice versa. * Setup once during sunrpc initialisation. */ -struct svc_pool_map svc_pool_map = { + +struct svc_pool_map { + int count; /* How many svc_servs use us */ + int mode; /* Note: int not enum to avoid + * warnings about "enumeration value + * not handled in switch" */ + unsigned int npools; + unsigned int *pool_to; /* maps pool id to cpu or node */ + unsigned int *to_pool; /* maps cpu or node to pool id */ +}; + +static struct svc_pool_map svc_pool_map = { .mode = SVC_POOL_DEFAULT }; -EXPORT_SYMBOL_GPL(svc_pool_map); static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */ @@ -222,7 +243,7 @@ svc_pool_map_init_pernode(struct svc_pool_map *m) * vice versa). Initialise the map if we're the first user. * Returns the number of pools. */ -unsigned int +static unsigned int svc_pool_map_get(void) { struct svc_pool_map *m = &svc_pool_map; @@ -257,7 +278,6 @@ svc_pool_map_get(void) mutex_unlock(&svc_pool_map_mutex); return m->npools; } -EXPORT_SYMBOL_GPL(svc_pool_map_get); /* * Drop a reference to the global map of cpus to pools. @@ -266,7 +286,7 @@ EXPORT_SYMBOL_GPL(svc_pool_map_get); * mode using the pool_mode module option without * rebooting or re-loading sunrpc.ko. */ -void +static void svc_pool_map_put(void) { struct svc_pool_map *m = &svc_pool_map; @@ -283,7 +303,6 @@ svc_pool_map_put(void) mutex_unlock(&svc_pool_map_mutex); } -EXPORT_SYMBOL_GPL(svc_pool_map_put); static int svc_pool_map_get_node(unsigned int pidx) { -- cgit v1.2.3-59-g8ed1b From 6b044fbaab02292fedb17565dbb3f2528083b169 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 29 Nov 2021 15:51:25 +1100 Subject: lockd: use svc_set_num_threads() for thread start and stop svc_set_num_threads() does everything that lockd_start_svc() does, except set sv_maxconn. It also (when passed 0) finds the threads and stops them with kthread_stop(). So move the setting for sv_maxconn, and use svc_set_num_thread() We now don't need nlmsvc_task. Now that we use svc_set_num_threads() it makes sense to set svo_module. This request that the thread exists with module_put_and_exit(). Also fix the documentation for svo_module to make this explicit. svc_prepare_thread is now only used where it is defined, so it can be made static. Signed-off-by: NeilBrown Signed-off-by: Chuck Lever --- fs/lockd/svc.c | 58 +++++++--------------------------------------- include/linux/sunrpc/svc.h | 6 ++--- net/sunrpc/svc.c | 3 +-- 3 files changed, 12 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 1a7c11118b32..4defefd89cbf 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -55,7 +55,6 @@ EXPORT_SYMBOL_GPL(nlmsvc_ops); static DEFINE_MUTEX(nlmsvc_mutex); static unsigned int nlmsvc_users; static struct svc_serv *nlmsvc_serv; -static struct task_struct *nlmsvc_task; unsigned long nlmsvc_timeout; unsigned int lockd_net_id; @@ -186,7 +185,7 @@ lockd(void *vrqstp) svc_exit_thread(rqstp); - return 0; + module_put_and_exit(0); } static int create_lockd_listener(struct svc_serv *serv, const char *name, @@ -292,8 +291,8 @@ static void lockd_down_net(struct svc_serv *serv, struct net *net) __func__, net->ns.inum); } } else { - pr_err("%s: no users! task=%p, net=%x\n", - __func__, nlmsvc_task, net->ns.inum); + pr_err("%s: no users! net=%x\n", + __func__, net->ns.inum); BUG(); } } @@ -351,49 +350,11 @@ static struct notifier_block lockd_inet6addr_notifier = { }; #endif -static int lockd_start_svc(struct svc_serv *serv) -{ - int error; - struct svc_rqst *rqst; - - /* - * Create the kernel thread and wait for it to start. - */ - rqst = svc_prepare_thread(serv, &serv->sv_pools[0], NUMA_NO_NODE); - if (IS_ERR(rqst)) { - error = PTR_ERR(rqst); - printk(KERN_WARNING - "lockd_up: svc_rqst allocation failed, error=%d\n", - error); - goto out_rqst; - } - - svc_sock_update_bufs(serv); - serv->sv_maxconn = nlm_max_connections; - - nlmsvc_task = kthread_create(lockd, rqst, "%s", serv->sv_name); - if (IS_ERR(nlmsvc_task)) { - error = PTR_ERR(nlmsvc_task); - printk(KERN_WARNING - "lockd_up: kthread_run failed, error=%d\n", error); - goto out_task; - } - rqst->rq_task = nlmsvc_task; - wake_up_process(nlmsvc_task); - - dprintk("lockd_up: service started\n"); - return 0; - -out_task: - svc_exit_thread(rqst); - nlmsvc_task = NULL; -out_rqst: - return error; -} - static const struct svc_serv_ops lockd_sv_ops = { .svo_shutdown = svc_rpcb_cleanup, + .svo_function = lockd, .svo_enqueue_xprt = svc_xprt_do_enqueue, + .svo_module = THIS_MODULE, }; static int lockd_get(void) @@ -425,7 +386,8 @@ static int lockd_get(void) return -ENOMEM; } - error = lockd_start_svc(serv); + serv->sv_maxconn = nlm_max_connections; + error = svc_set_num_threads(serv, NULL, 1); /* The thread now holds the only reference */ svc_put(serv); if (error < 0) @@ -453,11 +415,7 @@ static void lockd_put(void) unregister_inet6addr_notifier(&lockd_inet6addr_notifier); #endif - if (nlmsvc_task) { - kthread_stop(nlmsvc_task); - dprintk("lockd_down: service stopped\n"); - nlmsvc_task = NULL; - } + svc_set_num_threads(nlmsvc_serv, NULL, 0); nlmsvc_serv = NULL; dprintk("lockd_down: service destroyed\n"); } diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index d69e6108cb83..cf175d47c6b7 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -64,7 +64,9 @@ struct svc_serv_ops { /* queue up a transport for servicing */ void (*svo_enqueue_xprt)(struct svc_xprt *); - /* optional module to count when adding threads (pooled svcs only) */ + /* optional module to count when adding threads. + * Thread function must call module_put_and_exit() to exit. + */ struct module *svo_module; }; @@ -504,8 +506,6 @@ struct svc_serv *svc_create(struct svc_program *, unsigned int, const struct svc_serv_ops *); struct svc_rqst *svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node); -struct svc_rqst *svc_prepare_thread(struct svc_serv *serv, - struct svc_pool *pool, int node); void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page); void svc_rqst_free(struct svc_rqst *); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 5fbe7f55289e..2aabec2b4bec 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -652,7 +652,7 @@ out_enomem: } EXPORT_SYMBOL_GPL(svc_rqst_alloc); -struct svc_rqst * +static struct svc_rqst * svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) { struct svc_rqst *rqstp; @@ -672,7 +672,6 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) spin_unlock_bh(&pool->sp_lock); return rqstp; } -EXPORT_SYMBOL_GPL(svc_prepare_thread); /* * Choose a pool in which to create a new thread, for svc_set_num_threads -- cgit v1.2.3-59-g8ed1b From c8064e5b4adac5e1255cf4f3b374e75b5376e7ca Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Tue, 30 Nov 2021 11:08:07 +0100 Subject: bpf: Let bpf_warn_invalid_xdp_action() report more info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In non trivial scenarios, the action id alone is not sufficient to identify the program causing the warning. Before the previous patch, the generated stack-trace pointed out at least the involved device driver. Let's additionally include the program name and id, and the relevant device name. If the user needs additional infos, he can fetch them via a kernel probe, leveraging the arguments added here. Signed-off-by: Paolo Abeni Signed-off-by: Daniel Borkmann Acked-by: Toke Høiland-Jørgensen Link: https://lore.kernel.org/bpf/ddb96bb975cbfddb1546cf5da60e77d5100b533c.1638189075.git.pabeni@redhat.com --- drivers/net/ethernet/amazon/ena/ena_netdev.c | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 2 +- drivers/net/ethernet/cavium/thunder/nicvf_main.c | 2 +- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 2 +- drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c | 2 +- drivers/net/ethernet/freescale/enetc/enetc.c | 2 +- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 2 +- drivers/net/ethernet/intel/i40e/i40e_xsk.c | 2 +- drivers/net/ethernet/intel/ice/ice_txrx.c | 2 +- drivers/net/ethernet/intel/ice/ice_xsk.c | 2 +- drivers/net/ethernet/intel/igb/igb_main.c | 2 +- drivers/net/ethernet/intel/igc/igc_main.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- drivers/net/ethernet/marvell/mvneta.c | 2 +- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 2 +- drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c | 2 +- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c | 2 +- drivers/net/ethernet/microsoft/mana/mana_bpf.c | 2 +- drivers/net/ethernet/netronome/nfp/nfp_net_common.c | 2 +- drivers/net/ethernet/qlogic/qede/qede_fp.c | 2 +- drivers/net/ethernet/sfc/rx.c | 2 +- drivers/net/ethernet/socionext/netsec.c | 2 +- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 2 +- drivers/net/ethernet/ti/cpsw_priv.c | 2 +- drivers/net/hyperv/netvsc_bpf.c | 2 +- drivers/net/tun.c | 2 +- drivers/net/veth.c | 4 ++-- drivers/net/virtio_net.c | 4 ++-- drivers/net/xen-netfront.c | 2 +- include/linux/filter.h | 2 +- kernel/bpf/cpumap.c | 4 ++-- kernel/bpf/devmap.c | 4 ++-- net/core/dev.c | 2 +- net/core/filter.c | 6 +++--- 37 files changed, 43 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 7d5d885d85d5..3b46f1df5609 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -434,7 +434,7 @@ static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) xdp_stat = &rx_ring->rx_stats.xdp_pass; break; default: - bpf_warn_invalid_xdp_action(verdict); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict); xdp_stat = &rx_ring->rx_stats.xdp_invalid; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index c8083df5e0ab..52fad0fdeacf 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -195,7 +195,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, *event |= BNXT_REDIRECT_EVENT; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(bp->dev, xdp_prog, act); diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index bb45d5df2856..30450efccad7 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -590,7 +590,7 @@ static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog, nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len); return true; default: - bpf_warn_invalid_xdp_action(action); + bpf_warn_invalid_xdp_action(nic->netdev, prog, action); fallthrough; case XDP_ABORTED: trace_xdp_exception(nic->netdev, prog, action); diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index d6871437d951..c78883c3a2c8 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2623,7 +2623,7 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, } break; default: - bpf_warn_invalid_xdp_action(xdp_act); + bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); fallthrough; case XDP_ABORTED: trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 8e643567abce..d21ba70ef4a3 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -374,7 +374,7 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); break; default: - bpf_warn_invalid_xdp_action(xdp_act); + bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act); fallthrough; case XDP_ABORTED: trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 504e12554079..eacb41f86bdb 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -1547,7 +1547,7 @@ static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring, switch (xdp_act) { default: - bpf_warn_invalid_xdp_action(xdp_act); + bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act); fallthrough; case XDP_ABORTED: trace_xdp_exception(rx_ring->ndev, prog, xdp_act); diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index 10a83e5385c7..b399ca649f09 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -2322,7 +2322,7 @@ static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) result = I40E_XDP_REDIR; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index ea06e957393e..945b1bb9c6f4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -176,7 +176,7 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) goto out_failure; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index bc3ba19dc88f..56940bb908bc 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -561,7 +561,7 @@ ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, goto out_failure; return ICE_XDP_REDIR; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index bb9a80847298..ef4b213881f3 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -483,7 +483,7 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, goto out_failure; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index dd208930fbe4..337a7197e96f 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -8422,7 +8422,7 @@ static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, result = IGB_XDP_REDIR; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 142c57b7a451..7f2ce4b256d9 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2241,7 +2241,7 @@ static int __igc_xdp_run_prog(struct igc_adapter *adapter, return IGC_XDP_REDIRECT; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(adapter->netdev, prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 0f9f022260d7..265bc52aacf8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -2235,7 +2235,7 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, result = IXGBE_XDP_REDIR; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index db2bc58dfcfd..b3fd8e5cd85b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -131,7 +131,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, goto out_failure; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index b1dfbaff8b31..2459ecf65125 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1070,7 +1070,7 @@ static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, goto out_failure; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: out_failure: diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index ce810fc3c1a2..23cc4b874285 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2239,7 +2239,7 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, mvneta_xdp_put_buff(pp, rxq, xdp, sinfo, sync); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(pp->dev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(pp->dev, prog, act); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 8e5820d12362..7273a4c9dbb1 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -3823,7 +3823,7 @@ mvpp2_run_xdp(struct mvpp2_port *port, struct bpf_prog *prog, } break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(port->dev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(port->dev, prog, act); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 0cc6353254bf..7c4068c5d1ac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -1198,7 +1198,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, put_page(page); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act); break; case XDP_ABORTED: trace_xdp_exception(pfvf->netdev, prog, act); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 650e6a1844ae..8cfc649f226b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -812,7 +812,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud trace_xdp_exception(dev, xdp_prog, act); goto xdp_drop_no_cnt; /* Drop on xmit failure */ default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(dev, xdp_prog, act); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 2f0df5cc1a2d..338d65e2c9ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -151,7 +151,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, rq->stats->xdp_redirect++; return true; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rq->netdev, prog, act); fallthrough; case XDP_ABORTED: xdp_abort: diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c index 1bc8ff388341..1d2f948b5c00 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -60,7 +60,7 @@ u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(ndev, prog, act); } out: diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 6e38da4ad554..79257ec41987 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -1944,7 +1944,7 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) xdp_prog, act); continue; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(dp->netdev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(dp->netdev, xdp_prog, act); diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index 5ea9cb4311a1..b242000a77fd 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -1153,7 +1153,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, qede_rx_bd_ring_consume(rxq); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(edev->ndev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(edev->ndev, prog, act); diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index 606750938b89..2375cef577e4 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -338,7 +338,7 @@ static bool efx_do_xdp(struct efx_nic *efx, struct efx_channel *channel, break; default: - bpf_warn_invalid_xdp_action(xdp_act); + bpf_warn_invalid_xdp_action(efx->net_dev, xdp_prog, xdp_act); efx_free_rx_buffers(rx_queue, rx_buf, 1); channel->n_rx_xdp_bad_drops++; trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index de7d8bf2c226..25dcd8eda5fc 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -933,7 +933,7 @@ static u32 netsec_run_xdp(struct netsec_priv *priv, struct bpf_prog *prog, } break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(priv->ndev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(priv->ndev, prog, act); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 4e05c1d92935..7f62ddc90088 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -4714,7 +4714,7 @@ static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, res = STMMAC_XDP_REDIRECT; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(priv->dev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(priv->dev, prog, act); diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index c99dd9735087..67c4009fd16c 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -1366,7 +1366,7 @@ int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, xdp_do_flush_map(); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(ndev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(ndev, prog, act); diff --git a/drivers/net/hyperv/netvsc_bpf.c b/drivers/net/hyperv/netvsc_bpf.c index aa877da113f8..7856905414eb 100644 --- a/drivers/net/hyperv/netvsc_bpf.c +++ b/drivers/net/hyperv/netvsc_bpf.c @@ -68,7 +68,7 @@ u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan, break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(ndev, prog, act); } out: diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 1572878c3403..0e5d2272f63a 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1551,7 +1551,7 @@ static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, case XDP_PASS: break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(tun->dev, xdp_prog, act); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 38f6da24f460..700cc9374f9c 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -644,7 +644,7 @@ static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq, rcu_read_unlock(); goto xdp_xmit; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(rq->dev, xdp_prog, act); @@ -794,7 +794,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, rcu_read_unlock(); goto xdp_xmit; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(rq->dev, xdp_prog, act); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 088d9404c93d..d3e0b3a533cd 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -812,7 +812,7 @@ static struct sk_buff *receive_small(struct net_device *dev, rcu_read_unlock(); goto xdp_xmit; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(vi->dev, xdp_prog, act); @@ -1025,7 +1025,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, rcu_read_unlock(); goto xdp_xmit; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(vi->dev, xdp_prog, act); diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 911f43986a8c..7b7eb617051a 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -930,7 +930,7 @@ static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act); } return act; diff --git a/include/linux/filter.h b/include/linux/filter.h index 68c6b5c208e7..60eec80fa1d4 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1027,7 +1027,7 @@ void xdp_do_flush(void); */ #define xdp_do_flush_map xdp_do_flush -void bpf_warn_invalid_xdp_action(u32 act); +void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act); #ifdef CONFIG_INET struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 585b2b77ccc4..0421061d95f1 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -195,7 +195,7 @@ static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu, } return; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(skb->dev, rcpu->prog, act); @@ -254,7 +254,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, } break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act); fallthrough; case XDP_DROP: xdp_return_frame(xdpf); diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index f02d04540c0c..6feea293ff10 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -348,7 +348,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, frames[nframes++] = xdpf; break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(NULL, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(dev, xdp_prog, act); @@ -507,7 +507,7 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev __skb_push(skb, skb->mac_len); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(dst->dev, dst->xdp_prog, act); diff --git a/net/core/dev.c b/net/core/dev.c index 4420086f3aeb..c431c8925eed 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4708,7 +4708,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb, case XDP_PASS: break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(skb->dev, xdp_prog, act); diff --git a/net/core/filter.c b/net/core/filter.c index ad8619aa77b7..3f656391af7e 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8180,13 +8180,13 @@ static bool xdp_is_valid_access(int off, int size, return __is_valid_xdp_access(off, size); } -void bpf_warn_invalid_xdp_action(u32 act) +void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act) { const u32 act_max = XDP_REDIRECT; - pr_warn_once("%s XDP return value %u, expect packet loss!\n", + pr_warn_once("%s XDP return value %u on prog %s (id %d) dev %s, expect packet loss!\n", act > act_max ? "Illegal" : "Driver unsupported", - act); + act, prog->aux->name, prog->aux->id, dev ? dev->name : "N/A"); } EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action); -- cgit v1.2.3-59-g8ed1b From 22c3f2f56bd9c901e1da69040680c311f310635d Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Wed, 1 Dec 2021 11:36:18 -0800 Subject: net/mlx5: Separate FDB namespace This patch doesn't add an additional namespaces, but just separates the naming to be used by each FDB user, bypass and kernel. Downstream patches will actually split this up and allow to have more than single priority for the bypass users. Signed-off-by: Maor Gottlieb Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed Acked-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- drivers/infiniband/hw/mlx5/fs.c | 14 +++++++------- drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c | 4 +++- drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 1 + include/linux/mlx5/fs.h | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index b780185d9dc6..510ef85ef6e4 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -1508,7 +1508,7 @@ _get_flow_table(struct mlx5_ib_dev *dev, !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; break; - case MLX5_FLOW_NAMESPACE_FDB: + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: max_table_size = BIT( MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) @@ -1546,7 +1546,7 @@ _get_flow_table(struct mlx5_ib_dev *dev, case MLX5_FLOW_NAMESPACE_EGRESS: prio = &dev->flow_db->egress_prios[priority]; break; - case MLX5_FLOW_NAMESPACE_FDB: + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: prio = &dev->flow_db->fdb; break; case MLX5_FLOW_NAMESPACE_RDMA_RX: @@ -1937,7 +1937,7 @@ mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, *namespace = MLX5_FLOW_NAMESPACE_EGRESS; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: - *namespace = MLX5_FLOW_NAMESPACE_FDB; + *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS; break; case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; @@ -2029,8 +2029,8 @@ static int get_dests(struct uverbs_attr_bundle *attrs, } /* Allow only DEVX object, drop as dest for FDB */ - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || - (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && + !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) return -EINVAL; /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ @@ -2050,7 +2050,7 @@ static int get_dests(struct uverbs_attr_bundle *attrs, if (!is_flow_dest(devx_obj, dest_id, dest_type)) return -EINVAL; /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ - if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || + if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS || fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) return -EINVAL; @@ -2320,7 +2320,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( if (err) goto end; - if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB && + if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS && mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) { err = -EINVAL; goto end; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 750b21124a1a..762b9730a897 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -788,7 +788,8 @@ static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, int err; u32 *in; - if (namespace == MLX5_FLOW_NAMESPACE_FDB) + if (namespace == MLX5_FLOW_NAMESPACE_FDB || + namespace == MLX5_FLOW_NAMESPACE_FDB_BYPASS) max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size); else max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size); @@ -860,6 +861,7 @@ static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns, switch (namespace) { case MLX5_FLOW_NAMESPACE_FDB: + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions); table_type = FS_FT_FDB; break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 386ab9a2d490..2d26e16a67cd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -2220,6 +2220,7 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, switch (type) { case MLX5_FLOW_NAMESPACE_FDB: + case MLX5_FLOW_NAMESPACE_FDB_BYPASS: if (steering->fdb_root_ns) return &steering->fdb_root_ns->ns; return NULL; diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index cd2d4c572367..b1aad14689e3 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -73,6 +73,7 @@ enum mlx5_flow_namespace_type { MLX5_FLOW_NAMESPACE_KERNEL, MLX5_FLOW_NAMESPACE_LEFTOVERS, MLX5_FLOW_NAMESPACE_ANCHOR, + MLX5_FLOW_NAMESPACE_FDB_BYPASS, MLX5_FLOW_NAMESPACE_FDB, MLX5_FLOW_NAMESPACE_ESW_EGRESS, MLX5_FLOW_NAMESPACE_ESW_INGRESS, -- cgit v1.2.3-59-g8ed1b From 8aa45b544db9788b0fcc7a300eba8a3ade5d3d50 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Fri, 10 Dec 2021 13:11:34 +0200 Subject: HID: Add map_msc() to avoid boilerplate code Since we are going to have more MSC events too, add map_msc() that can be used to fill in necessary fields and avoid boilerplate code. Signed-off-by: Mika Westerberg Reviewed-by: Benjamin Tissoires Signed-off-by: Tero Kristo Signed-off-by: Benjamin Tissoires Link: https://lore.kernel.org/r/20211210111138.1248187-2-tero.kristo@linux.intel.com --- drivers/hid/hid-input.c | 6 ++---- include/linux/hid.h | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 217f2d1b91c5..fbbb82f65842 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -52,6 +52,7 @@ static const struct { #define map_rel(c) hid_map_usage(hidinput, usage, &bit, &max, EV_REL, (c)) #define map_key(c) hid_map_usage(hidinput, usage, &bit, &max, EV_KEY, (c)) #define map_led(c) hid_map_usage(hidinput, usage, &bit, &max, EV_LED, (c)) +#define map_msc(c) hid_map_usage(hidinput, usage, &bit, &max, EV_MSC, (c)) #define map_abs_clear(c) hid_map_usage_clear(hidinput, usage, &bit, \ &max, EV_ABS, (c)) @@ -874,10 +875,7 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel case 0x5b: /* TransducerSerialNumber */ case 0x6e: /* TransducerSerialNumber2 */ - usage->type = EV_MSC; - usage->code = MSC_SERIAL; - bit = input->mscbit; - max = MSC_MAX; + map_msc(MSC_SERIAL); break; default: goto unknown; diff --git a/include/linux/hid.h b/include/linux/hid.h index b2fea7fc54a1..f18e2cf5d74d 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -1010,6 +1010,10 @@ static inline void hid_map_usage(struct hid_input *hidinput, bmap = input->ledbit; limit = LED_MAX; break; + case EV_MSC: + bmap = input->mscbit; + limit = MSC_MAX; + break; } if (unlikely(c > limit || !bmap)) { -- cgit v1.2.3-59-g8ed1b From ae7fafa6896ae762ff489a5e71c8e5fabfeb61ee Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Fri, 10 Dec 2021 13:11:36 +0200 Subject: HID: Add hid usages for USI style pens Add usage codes for USI style pens, based on the USB-HID usage table: https://usb.org/document-library/hid-usage-tables-122 See chapter 16, Digitizers Page (0x0D) Signed-off-by: Tero Kristo Signed-off-by: Benjamin Tissoires Link: https://lore.kernel.org/r/20211210111138.1248187-4-tero.kristo@linux.intel.com --- include/linux/hid.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hid.h b/include/linux/hid.h index f18e2cf5d74d..fa2ddda84a53 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -241,6 +241,7 @@ struct hid_item { #define HID_DG_TOUCH 0x000d0033 #define HID_DG_UNTOUCH 0x000d0034 #define HID_DG_TAP 0x000d0035 +#define HID_DG_TRANSDUCER_INDEX 0x000d0038 #define HID_DG_TABLETFUNCTIONKEY 0x000d0039 #define HID_DG_PROGRAMCHANGEKEY 0x000d003a #define HID_DG_BATTERYSTRENGTH 0x000d003b @@ -253,6 +254,15 @@ struct hid_item { #define HID_DG_BARRELSWITCH 0x000d0044 #define HID_DG_ERASER 0x000d0045 #define HID_DG_TABLETPICK 0x000d0046 +#define HID_DG_PEN_COLOR 0x000d005c +#define HID_DG_PEN_LINE_WIDTH 0x000d005e +#define HID_DG_PEN_LINE_STYLE 0x000d0070 +#define HID_DG_PEN_LINE_STYLE_INK 0x000d0072 +#define HID_DG_PEN_LINE_STYLE_PENCIL 0x000d0073 +#define HID_DG_PEN_LINE_STYLE_HIGHLIGHTER 0x000d0074 +#define HID_DG_PEN_LINE_STYLE_CHISEL_MARKER 0x000d0075 +#define HID_DG_PEN_LINE_STYLE_BRUSH 0x000d0076 +#define HID_DG_PEN_LINE_STYLE_NO_PREFERENCE 0x000d0077 #define HID_CP_CONSUMERCONTROL 0x000c0001 #define HID_CP_NUMERICKEYPAD 0x000c0002 -- cgit v1.2.3-59-g8ed1b From 5904a3f9d756f108279f8c5b8f17ca6ddfb28d24 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Fri, 10 Dec 2021 13:11:37 +0200 Subject: HID: input: Make hidinput_find_field() static This function is not called outside of hid-input.c so we can make it static. Signed-off-by: Mika Westerberg Reviewed-by: Benjamin Tissoires Signed-off-by: Tero Kristo Signed-off-by: Benjamin Tissoires Link: https://lore.kernel.org/r/20211210111138.1248187-5-tero.kristo@linux.intel.com --- drivers/hid/hid-input.c | 4 ++-- include/linux/hid.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 93bbc33da3c3..4186143da7c6 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -1461,7 +1461,8 @@ void hidinput_report_event(struct hid_device *hid, struct hid_report *report) } EXPORT_SYMBOL_GPL(hidinput_report_event); -int hidinput_find_field(struct hid_device *hid, unsigned int type, unsigned int code, struct hid_field **field) +static int hidinput_find_field(struct hid_device *hid, unsigned int type, + unsigned int code, struct hid_field **field) { struct hid_report *report; int i, j; @@ -1476,7 +1477,6 @@ int hidinput_find_field(struct hid_device *hid, unsigned int type, unsigned int } return -1; } -EXPORT_SYMBOL_GPL(hidinput_find_field); struct hid_field *hidinput_get_led_field(struct hid_device *hid) { diff --git a/include/linux/hid.h b/include/linux/hid.h index fa2ddda84a53..ff8b6973022a 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -899,7 +899,6 @@ extern void hidinput_disconnect(struct hid_device *); int hid_set_field(struct hid_field *, unsigned, __s32); int hid_input_report(struct hid_device *, int type, u8 *, u32, int); -int hidinput_find_field(struct hid_device *hid, unsigned int type, unsigned int code, struct hid_field **field); struct hid_field *hidinput_get_led_field(struct hid_device *hid); unsigned int hidinput_count_leds(struct hid_device *hid); __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code); -- cgit v1.2.3-59-g8ed1b From fd8d135b2c5e88662f2729e034913f183455a667 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 8 Dec 2021 22:40:43 +1000 Subject: HID: quirks: Allow inverting the absolute X/Y values Add a HID_QUIRK_X_INVERT/HID_QUIRK_Y_INVERT quirk that can be used to invert the X/Y values. Signed-off-by: Alistair Francis [bentiss: silence checkpatch warning] Signed-off-by: Benjamin Tissoires Link: https://lore.kernel.org/r/20211208124045.61815-2-alistair@alistair23.me --- drivers/hid/hid-input.c | 6 ++++++ include/linux/hid.h | 2 ++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 4186143da7c6..5f357dddcc01 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -1329,6 +1329,12 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct input = field->hidinput->input; + if (usage->type == EV_ABS && + (((*quirks & HID_QUIRK_X_INVERT) && usage->code == ABS_X) || + ((*quirks & HID_QUIRK_Y_INVERT) && usage->code == ABS_Y))) { + value = field->logical_maximum - value; + } + if (usage->hat_min < usage->hat_max || usage->hat_dir) { int hat_dir = usage->hat_dir; if (!hat_dir) diff --git a/include/linux/hid.h b/include/linux/hid.h index ff8b6973022a..a14c089eb0c1 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -359,6 +359,8 @@ struct hid_item { /* BIT(9) reserved for backward compatibility, was NO_INIT_INPUT_REPORTS */ #define HID_QUIRK_ALWAYS_POLL BIT(10) #define HID_QUIRK_INPUT_PER_APP BIT(11) +#define HID_QUIRK_X_INVERT BIT(12) +#define HID_QUIRK_Y_INVERT BIT(13) #define HID_QUIRK_SKIP_OUTPUT_REPORTS BIT(16) #define HID_QUIRK_SKIP_OUTPUT_REPORT_ID BIT(17) #define HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP BIT(18) -- cgit v1.2.3-59-g8ed1b From 369461ce8fb6c8156206c7110d7da48e9fbc41bb Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Wed, 8 Dec 2021 14:11:20 -0600 Subject: x86: perf: Move RDPMC event flag to a common definition In preparation to enable user counter access on arm64 and to move some of the user access handling to perf core, create a common event flag for user counter access and convert x86 to use it. Since the architecture specific flags start at the LSB, starting at the MSB for common flags. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Arnaldo Carvalho de Melo Cc: Alexander Shishkin Cc: Jiri Olsa Cc: Namhyung Kim Cc: Kan Liang Cc: Thomas Gleixner Cc: Borislav Petkov Cc: x86@kernel.org Cc: "H. Peter Anvin" Cc: linux-perf-users@vger.kernel.org Reviewed-by: Mark Rutland Reviewed-by: Thomas Gleixner Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20211208201124.310740-2-robh@kernel.org Signed-off-by: Will Deacon --- arch/x86/events/core.c | 10 +++++----- arch/x86/events/perf_event.h | 2 +- include/linux/perf_event.h | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 38b2c779146f..68dea7ce6a22 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2476,7 +2476,7 @@ static int x86_pmu_event_init(struct perf_event *event) if (READ_ONCE(x86_pmu.attr_rdpmc) && !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) - event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; + event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; } @@ -2510,7 +2510,7 @@ void perf_clear_dirty_counters(void) static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) { - if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) return; /* @@ -2531,7 +2531,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) { - if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) return; if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) @@ -2542,7 +2542,7 @@ static int x86_pmu_event_idx(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; - if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED)) + if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) return 0; if (is_metric_idx(hwc->idx)) @@ -2725,7 +2725,7 @@ void arch_perf_update_userpage(struct perf_event *event, userpg->cap_user_time = 0; userpg->cap_user_time_zero = 0; userpg->cap_user_rdpmc = - !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); + !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT); userpg->pmc_width = x86_pmu.cntval_bits; if (!using_native_sched_clock() || !sched_clock_stable()) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 5480db242083..9d376e528dfc 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -74,7 +74,7 @@ static inline bool constraint_match(struct event_constraint *c, u64 ecode) #define PERF_X86_EVENT_PEBS_NA_HSW 0x0010 /* haswell style datala, unknown */ #define PERF_X86_EVENT_EXCL 0x0020 /* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0040 /* dynamic alloc'd constraint */ -#define PERF_X86_EVENT_RDPMC_ALLOWED 0x0080 /* grant rdpmc permission */ + #define PERF_X86_EVENT_EXCL_ACCT 0x0100 /* accounted EXCL event */ #define PERF_X86_EVENT_AUTO_RELOAD 0x0200 /* use PEBS auto-reload */ #define PERF_X86_EVENT_LARGE_PEBS 0x0400 /* use large PEBS */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 0dcfd265beed..ba9467972c09 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -129,6 +129,15 @@ struct hw_perf_event_extra { int idx; /* index in shared_regs->regs[] */ }; +/** + * hw_perf_event::flag values + * + * PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific + * usage. + */ +#define PERF_EVENT_FLAG_ARCH 0x0000ffff +#define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000 + /** * struct hw_perf_event - performance event hardware details: */ -- cgit v1.2.3-59-g8ed1b From 82ff0c022d19c2ad69a472692bb7ee01ac07a40b Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Wed, 8 Dec 2021 14:11:21 -0600 Subject: perf: Add a counter for number of user access events in context On arm64, user space counter access will be controlled differently compared to x86. On x86, access in the strictest mode is enabled for all tasks in an MM when any event is mmap'ed. For arm64, access is explicitly requested for an event and only enabled when the event's context is active. This avoids hooks into the arch context switch code and gives better control of when access is enabled. In order to configure user space access when the PMU is enabled, it is necessary to know if any event (currently active or not) in the current context has user space accessed enabled. Add a counter similar to other counters in the context to avoid walking the event list every time. Reviewed-by: Mark Rutland Reviewed-by: Thomas Gleixner Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20211208201124.310740-3-robh@kernel.org Signed-off-by: Will Deacon --- include/linux/perf_event.h | 1 + kernel/events/core.c | 4 ++++ 2 files changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ba9467972c09..411e34210fbf 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -831,6 +831,7 @@ struct perf_event_context { int nr_events; int nr_active; + int nr_user; int is_active; int nr_stat; int nr_freq; diff --git a/kernel/events/core.c b/kernel/events/core.c index 30d94f68c5bd..1362b9b770d8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1808,6 +1808,8 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) list_add_rcu(&event->event_entry, &ctx->event_list); ctx->nr_events++; + if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) + ctx->nr_user++; if (event->attr.inherit_stat) ctx->nr_stat++; @@ -1999,6 +2001,8 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) event->attach_state &= ~PERF_ATTACH_CONTEXT; ctx->nr_events--; + if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) + ctx->nr_user--; if (event->attr.inherit_stat) ctx->nr_stat--; -- cgit v1.2.3-59-g8ed1b From 8404b0fbc7fbd42e5c5d28cdedd450e70829c77a Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Thu, 2 Dec 2021 16:06:33 +0800 Subject: drivers/perf: hisi: Add driver for HiSilicon PCIe PMU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PCIe PMU Root Complex Integrated End Point(RCiEP) device is supported to sample bandwidth, latency, buffer occupation etc. Each PMU RCiEP device monitors multiple Root Ports, and each RCiEP is registered as a PMU in /sys/bus/event_source/devices, so users can select target PMU, and use filter to do further sets. Filtering options contains: event - select the event. port - select target Root Ports. Information of Root Ports are shown under sysfs. bdf - select requester_id of target EP device. trig_len - set trigger condition for starting event statistics. trig_mode - set trigger mode. 0 means starting to statistic when bigger than trigger condition, and 1 means smaller. thr_len - set threshold for statistics. thr_mode - set threshold mode. 0 means count when bigger than threshold, and 1 means smaller. Acked-by: Krzysztof Wilczyński Reviewed-by: John Garry Signed-off-by: Qi Liu Reviewed-by: Shaokun Zhang Link: https://lore.kernel.org/r/20211202080633.2919-3-liuqi115@huawei.com Signed-off-by: Will Deacon --- MAINTAINERS | 2 + drivers/perf/hisilicon/Kconfig | 9 + drivers/perf/hisilicon/Makefile | 2 + drivers/perf/hisilicon/hisi_pcie_pmu.c | 948 +++++++++++++++++++++++++++++++++ include/linux/cpuhotplug.h | 1 + 5 files changed, 962 insertions(+) create mode 100644 drivers/perf/hisilicon/hisi_pcie_pmu.c (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 360e9aa0205d..90a7d7b21982 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8607,8 +8607,10 @@ F: drivers/misc/hisi_hikey_usb.c HISILICON PMU DRIVER M: Shaokun Zhang +M: Qi Liu S: Supported W: http://www.hisilicon.com +F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst F: Documentation/admin-guide/perf/hisi-pmu.rst F: drivers/perf/hisilicon diff --git a/drivers/perf/hisilicon/Kconfig b/drivers/perf/hisilicon/Kconfig index c5d1b7019fff..5546218b5598 100644 --- a/drivers/perf/hisilicon/Kconfig +++ b/drivers/perf/hisilicon/Kconfig @@ -5,3 +5,12 @@ config HISI_PMU help Support for HiSilicon SoC L3 Cache performance monitor, Hydra Home Agent performance monitor and DDR Controller performance monitor. + +config HISI_PCIE_PMU + tristate "HiSilicon PCIE PERF PMU" + depends on PCI && ARM64 + help + Provide support for HiSilicon PCIe performance monitoring unit (PMU) + RCiEP devices. + Adds the PCIe PMU into perf events system for monitoring latency, + bandwidth etc. diff --git a/drivers/perf/hisilicon/Makefile b/drivers/perf/hisilicon/Makefile index 7643c9f93e36..506ed39e3266 100644 --- a/drivers/perf/hisilicon/Makefile +++ b/drivers/perf/hisilicon/Makefile @@ -2,3 +2,5 @@ obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o \ hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o hisi_uncore_sllc_pmu.o \ hisi_uncore_pa_pmu.o + +obj-$(CONFIG_HISI_PCIE_PMU) += hisi_pcie_pmu.o diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c new file mode 100644 index 000000000000..21771708597d --- /dev/null +++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c @@ -0,0 +1,948 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This driver adds support for PCIe PMU RCiEP device. Related + * perf events are bandwidth, latency etc. + * + * Copyright (C) 2021 HiSilicon Limited + * Author: Qi Liu + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "hisi_pcie_pmu" +/* Define registers */ +#define HISI_PCIE_GLOBAL_CTRL 0x00 +#define HISI_PCIE_EVENT_CTRL 0x010 +#define HISI_PCIE_CNT 0x090 +#define HISI_PCIE_EXT_CNT 0x110 +#define HISI_PCIE_INT_STAT 0x150 +#define HISI_PCIE_INT_MASK 0x154 +#define HISI_PCIE_REG_BDF 0xfe0 +#define HISI_PCIE_REG_VERSION 0xfe4 +#define HISI_PCIE_REG_INFO 0xfe8 + +/* Define command in HISI_PCIE_GLOBAL_CTRL */ +#define HISI_PCIE_GLOBAL_EN 0x01 +#define HISI_PCIE_GLOBAL_NONE 0 + +/* Define command in HISI_PCIE_EVENT_CTRL */ +#define HISI_PCIE_EVENT_EN BIT_ULL(20) +#define HISI_PCIE_RESET_CNT BIT_ULL(22) +#define HISI_PCIE_INIT_SET BIT_ULL(34) +#define HISI_PCIE_THR_EN BIT_ULL(26) +#define HISI_PCIE_TARGET_EN BIT_ULL(32) +#define HISI_PCIE_TRIG_EN BIT_ULL(52) + +/* Define offsets in HISI_PCIE_EVENT_CTRL */ +#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0) +#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27) +#define HISI_PCIE_THR_M GENMASK_ULL(31, 28) +#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36) +#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53) +#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56) + +#define HISI_PCIE_MAX_COUNTERS 8 +#define HISI_PCIE_REG_STEP 8 +#define HISI_PCIE_THR_MAX_VAL 10 +#define HISI_PCIE_TRIG_MAX_VAL 10 +#define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0)) +#define HISI_PCIE_INIT_VAL BIT_ULL(63) + +struct hisi_pcie_pmu { + struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS]; + struct hlist_node node; + struct pci_dev *pdev; + struct pmu pmu; + void __iomem *base; + int irq; + u32 identifier; + /* Minimum and maximum BDF of root ports monitored by PMU */ + u16 bdf_min; + u16 bdf_max; + int on_cpu; +}; + +struct hisi_pcie_reg_pair { + u16 lo; + u16 hi; +}; + +#define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu)) +#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) + +#define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \ + static u64 hisi_pcie_get_##_name(struct perf_event *event) \ + { \ + return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \ + } \ + +HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0); +HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0); +HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4); +HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5); +HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9); +HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0); +HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16); + +static ssize_t hisi_pcie_format_sysfs_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + + return sysfs_emit(buf, "%s\n", (char *)eattr->var); +} + +static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct perf_pmu_events_attr *pmu_attr = + container_of(attr, struct perf_pmu_events_attr, attr); + + return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); +} + +#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \ + (&((struct dev_ext_attribute[]){ \ + { .attr = __ATTR(_name, 0444, hisi_pcie_format_sysfs_show, \ + NULL), \ + .var = (void *)_format } \ + })[0].attr.attr) + +#define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id) + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); +} +static DEVICE_ATTR_RO(cpumask); + +static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return sysfs_emit(buf, "%#x\n", pcie_pmu->identifier); +} +static DEVICE_ATTR_RO(identifier); + +static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); + + return sysfs_emit(buf, "%#04x\n", PCI_BUS_NUM(pcie_pmu->bdf_min)); +} +static DEVICE_ATTR_RO(bus); + +static struct hisi_pcie_reg_pair +hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off) +{ + u32 val = readl_relaxed(pcie_pmu->base + reg_off); + struct hisi_pcie_reg_pair regs = { + .lo = val, + .hi = val >> 16, + }; + + return regs; +} + +/* + * Hardware counter and ext_counter work together for bandwidth, latency, bus + * utilization and buffer occupancy events. For example, RX memory write latency + * events(index = 0x0010), counter counts total delay cycles and ext_counter + * counts RX memory write PCIe packets number. + * + * As we don't want PMU driver to process these two data, "delay cycles" can + * be treated as an independent event(index = 0x0010), "RX memory write packets + * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15 + * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL. + */ +#define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16)) + +static u32 hisi_pcie_get_real_event(struct perf_event *event) +{ + return hisi_pcie_get_event(event) & GENMASK(15, 0); +} + +static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx) +{ + return offset + HISI_PCIE_REG_STEP * idx; +} + +static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, + u32 idx) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + return readl_relaxed(pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + writel_relaxed(val, pcie_pmu->base + offset); +} + +static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + return readq_relaxed(pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val) +{ + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); + + writeq_relaxed(val, pcie_pmu->base + offset); +} + +static void hisi_pcie_pmu_config_filter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + u64 reg = HISI_PCIE_INIT_SET; + u64 port, trig_len, thr_len; + + /* Config HISI_PCIE_EVENT_CTRL according to event. */ + reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event)); + + /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */ + port = hisi_pcie_get_port(event); + if (port) + reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port); + else + reg |= HISI_PCIE_TARGET_EN | + FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event)); + + /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */ + trig_len = hisi_pcie_get_trig_len(event); + if (trig_len) { + reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len); + reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event)); + reg |= HISI_PCIE_TRIG_EN; + } + + /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */ + thr_len = hisi_pcie_get_thr_len(event); + if (thr_len) { + reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len); + reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event)); + reg |= HISI_PCIE_THR_EN; + } + + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg); +} + +static void hisi_pcie_pmu_clear_filter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, HISI_PCIE_INIT_SET); +} + +static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf) +{ + struct pci_dev *root_port, *pdev; + u16 rp_bdf; + + pdev = pci_get_domain_bus_and_slot(pci_domain_nr(pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf), + GET_PCI_DEVFN(bdf)); + if (!pdev) + return false; + + root_port = pcie_find_root_port(pdev); + if (!root_port) { + pci_dev_put(pdev); + return false; + } + + pci_dev_put(pdev); + rp_bdf = pci_dev_id(root_port); + return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max; +} + +static bool hisi_pcie_pmu_valid_filter(struct perf_event *event, + struct hisi_pcie_pmu *pcie_pmu) +{ + u32 requester_id = hisi_pcie_get_bdf(event); + + if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL) + return false; + + if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL) + return false; + + if (requester_id) { + if (!hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id)) + return false; + } + + return true; +} + +static bool hisi_pcie_pmu_cmp_event(struct perf_event *target, + struct perf_event *event) +{ + return hisi_pcie_get_real_event(target) == hisi_pcie_get_real_event(event); +} + +static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) +{ + struct perf_event *sibling, *leader = event->group_leader; + struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS]; + int counters = 1; + int num; + + event_group[0] = leader; + if (!is_software_event(leader)) { + if (leader->pmu != event->pmu) + return false; + + if (leader != event && !hisi_pcie_pmu_cmp_event(leader, event)) + event_group[counters++] = event; + } + + for_each_sibling_event(sibling, event->group_leader) { + if (is_software_event(sibling)) + continue; + + if (sibling->pmu != event->pmu) + return false; + + for (num = 0; num < counters; num++) { + if (hisi_pcie_pmu_cmp_event(event_group[num], sibling)) + break; + } + + if (num == counters) + event_group[counters++] = sibling; + } + + return counters <= HISI_PCIE_MAX_COUNTERS; +} + +static int hisi_pcie_pmu_event_init(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + event->cpu = pcie_pmu->on_cpu; + + if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event))) + hwc->event_base = HISI_PCIE_EXT_CNT; + else + hwc->event_base = HISI_PCIE_CNT; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* Sampling is not supported. */ + if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) + return -EOPNOTSUPP; + + if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu)) + return -EINVAL; + + if (!hisi_pcie_pmu_validate_event_group(event)) + return -EINVAL; + + return 0; +} + +static u64 hisi_pcie_pmu_read_counter(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + u32 idx = event->hw.idx; + + return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx); +} + +static int hisi_pcie_pmu_find_related_event(struct hisi_pcie_pmu *pcie_pmu, + struct perf_event *event) +{ + struct perf_event *sibling; + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + sibling = pcie_pmu->hw_events[idx]; + if (!sibling) + continue; + + if (!hisi_pcie_pmu_cmp_event(sibling, event)) + continue; + + /* Related events must be used in group */ + if (sibling->group_leader == event->group_leader) + return idx; + else + return -EINVAL; + } + + return idx; +} + +static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu) +{ + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + if (!pcie_pmu->hw_events[idx]) + return idx; + } + + return -EINVAL; +} + +static void hisi_pcie_pmu_event_update(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 new_cnt, prev_cnt, delta; + + do { + prev_cnt = local64_read(&hwc->prev_count); + new_cnt = hisi_pcie_pmu_read_counter(event); + } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, + new_cnt) != prev_cnt); + + delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD; + local64_add(delta, &event->count); +} + +static void hisi_pcie_pmu_read(struct perf_event *event) +{ + hisi_pcie_pmu_event_update(event); +} + +static void hisi_pcie_pmu_set_period(struct perf_event *event) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL); +} + +static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + u64 val; + + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); + val |= HISI_PCIE_EVENT_EN; + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); +} + +static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + u64 val; + + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); + val &= ~HISI_PCIE_EVENT_EN; + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); +} + +static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 0); +} + +static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) +{ + u32 idx = hwc->idx; + + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 1); +} + +static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx) +{ + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT); + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET); +} + +static void hisi_pcie_pmu_start(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + u64 prev_cnt; + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + hwc->state = 0; + + hisi_pcie_pmu_config_filter(event); + hisi_pcie_pmu_enable_counter(pcie_pmu, hwc); + hisi_pcie_pmu_enable_int(pcie_pmu, hwc); + hisi_pcie_pmu_set_period(event); + + if (flags & PERF_EF_RELOAD) { + prev_cnt = local64_read(&hwc->prev_count); + hisi_pcie_pmu_writeq(pcie_pmu, hwc->event_base, idx, prev_cnt); + } + + perf_event_update_userpage(event); +} + +static void hisi_pcie_pmu_stop(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_event_update(event); + hisi_pcie_pmu_disable_int(pcie_pmu, hwc); + hisi_pcie_pmu_disable_counter(pcie_pmu, hwc); + hisi_pcie_pmu_clear_filter(event); + WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); + hwc->state |= PERF_HES_STOPPED; + + if (hwc->state & PERF_HES_UPTODATE) + return; + + hwc->state |= PERF_HES_UPTODATE; +} + +static int hisi_pcie_pmu_add(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + /* Check all working events to find a related event. */ + idx = hisi_pcie_pmu_find_related_event(pcie_pmu, event); + if (idx < 0) + return idx; + + /* Current event shares an enabled counter with the related event */ + if (idx < HISI_PCIE_MAX_COUNTERS) { + hwc->idx = idx; + goto start_count; + } + + idx = hisi_pcie_pmu_get_event_idx(pcie_pmu); + if (idx < 0) + return idx; + + hwc->idx = idx; + pcie_pmu->hw_events[idx] = event; + /* Reset Counter to avoid previous statistic interference. */ + hisi_pcie_pmu_reset_counter(pcie_pmu, idx); + +start_count: + if (flags & PERF_EF_START) + hisi_pcie_pmu_start(event, PERF_EF_RELOAD); + + return 0; +} + +static void hisi_pcie_pmu_del(struct perf_event *event, int flags) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + hisi_pcie_pmu_stop(event, PERF_EF_UPDATE); + pcie_pmu->hw_events[hwc->idx] = NULL; + perf_event_update_userpage(event); +} + +static void hisi_pcie_pmu_enable(struct pmu *pmu) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); + int num; + + for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) { + if (pcie_pmu->hw_events[num]) + break; + } + + if (num == HISI_PCIE_MAX_COUNTERS) + return; + + writel(HISI_PCIE_GLOBAL_EN, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); +} + +static void hisi_pcie_pmu_disable(struct pmu *pmu) +{ + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); + + writel(HISI_PCIE_GLOBAL_NONE, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); +} + +static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data) +{ + struct hisi_pcie_pmu *pcie_pmu = data; + irqreturn_t ret = IRQ_NONE; + struct perf_event *event; + u32 overflown; + int idx; + + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { + overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx); + if (!overflown) + continue; + + /* Clear status of interrupt. */ + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, 1); + event = pcie_pmu->hw_events[idx]; + if (!event) + continue; + + hisi_pcie_pmu_event_update(event); + hisi_pcie_pmu_set_period(event); + ret = IRQ_HANDLED; + } + + return ret; +} + +static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + int irq, ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + pci_err(pdev, "Failed to enable MSI vectors: %d\n", ret); + return ret; + } + + irq = pci_irq_vector(pdev, 0); + ret = request_irq(irq, hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME, + pcie_pmu); + if (ret) { + pci_err(pdev, "Failed to register IRQ: %d\n", ret); + pci_free_irq_vectors(pdev); + return ret; + } + + pcie_pmu->irq = irq; + + return 0; +} + +static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + free_irq(pcie_pmu->irq, pcie_pmu); + pci_free_irq_vectors(pdev); +} + +static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); + + if (pcie_pmu->on_cpu == -1) { + pcie_pmu->on_cpu = cpu; + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(cpu))); + } + + return 0; +} + +static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); + unsigned int target; + + /* Nothing to do if this CPU doesn't own the PMU */ + if (pcie_pmu->on_cpu != cpu) + return 0; + + pcie_pmu->on_cpu = -1; + /* Choose a new CPU from all online cpus. */ + target = cpumask_first(cpu_online_mask); + if (target >= nr_cpu_ids) { + pci_err(pcie_pmu->pdev, "There is no CPU to set\n"); + return 0; + } + + perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); + /* Use this CPU for event counting */ + pcie_pmu->on_cpu = target; + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target))); + + return 0; +} + +static struct attribute *hisi_pcie_pmu_events_attr[] = { + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010), + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x1005), + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x11005), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x2004), + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x12004), + NULL +}; + +static struct attribute_group hisi_pcie_pmu_events_group = { + .name = "events", + .attrs = hisi_pcie_pmu_events_attr, +}; + +static struct attribute *hisi_pcie_pmu_format_attr[] = { + HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16"), + HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3"), + HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"), + HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"), + HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"), + HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"), + HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"), + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_format_group = { + .name = "format", + .attrs = hisi_pcie_pmu_format_attr, +}; + +static struct attribute *hisi_pcie_pmu_bus_attrs[] = { + &dev_attr_bus.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_bus_attr_group = { + .attrs = hisi_pcie_pmu_bus_attrs, +}; + +static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = { + .attrs = hisi_pcie_pmu_cpumask_attrs, +}; + +static struct attribute *hisi_pcie_pmu_identifier_attrs[] = { + &dev_attr_identifier.attr, + NULL +}; + +static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = { + .attrs = hisi_pcie_pmu_identifier_attrs, +}; + +static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = { + &hisi_pcie_pmu_events_group, + &hisi_pcie_pmu_format_group, + &hisi_pcie_pmu_bus_attr_group, + &hisi_pcie_pmu_cpumask_attr_group, + &hisi_pcie_pmu_identifier_attr_group, + NULL +}; + +static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + struct hisi_pcie_reg_pair regs; + u16 sicl_id, core_id; + char *name; + + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF); + pcie_pmu->bdf_min = regs.lo; + pcie_pmu->bdf_max = regs.hi; + + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO); + sicl_id = regs.hi; + core_id = regs.lo; + + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_pcie%u_core%u", sicl_id, core_id); + if (!name) + return -ENOMEM; + + pcie_pmu->pdev = pdev; + pcie_pmu->on_cpu = -1; + pcie_pmu->identifier = readl(pcie_pmu->base + HISI_PCIE_REG_VERSION); + pcie_pmu->pmu = (struct pmu) { + .name = name, + .module = THIS_MODULE, + .event_init = hisi_pcie_pmu_event_init, + .pmu_enable = hisi_pcie_pmu_enable, + .pmu_disable = hisi_pcie_pmu_disable, + .add = hisi_pcie_pmu_add, + .del = hisi_pcie_pmu_del, + .start = hisi_pcie_pmu_start, + .stop = hisi_pcie_pmu_stop, + .read = hisi_pcie_pmu_read, + .task_ctx_nr = perf_invalid_context, + .attr_groups = hisi_pcie_pmu_attr_groups, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + }; + + return 0; +} + +static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) +{ + int ret; + + pcie_pmu->base = pci_ioremap_bar(pdev, 2); + if (!pcie_pmu->base) { + pci_err(pdev, "Ioremap failed for pcie_pmu resource\n"); + return -ENOMEM; + } + + ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu); + if (ret) + goto err_iounmap; + + ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu); + if (ret) + goto err_iounmap; + + ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + if (ret) { + pci_err(pdev, "Failed to register hotplug: %d\n", ret); + goto err_irq_unregister; + } + + ret = perf_pmu_register(&pcie_pmu->pmu, pcie_pmu->pmu.name, -1); + if (ret) { + pci_err(pdev, "Failed to register PCIe PMU: %d\n", ret); + goto err_hotplug_unregister; + } + + return ret; + +err_hotplug_unregister: + cpuhp_state_remove_instance_nocalls( + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + +err_irq_unregister: + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); + +err_iounmap: + iounmap(pcie_pmu->base); + + return ret; +} + +static void hisi_pcie_uninit_pmu(struct pci_dev *pdev) +{ + struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev); + + perf_pmu_unregister(&pcie_pmu->pmu); + cpuhp_state_remove_instance_nocalls( + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); + iounmap(pcie_pmu->base); +} + +static int hisi_pcie_init_dev(struct pci_dev *pdev) +{ + int ret; + + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "Failed to enable PCI device: %d\n", ret); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); + if (ret < 0) { + pci_err(pdev, "Failed to request PCI mem regions: %d\n", ret); + return ret; + } + + pci_set_master(pdev); + + return 0; +} + +static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_pcie_pmu *pcie_pmu; + int ret; + + pcie_pmu = devm_kzalloc(&pdev->dev, sizeof(*pcie_pmu), GFP_KERNEL); + if (!pcie_pmu) + return -ENOMEM; + + ret = hisi_pcie_init_dev(pdev); + if (ret) + return ret; + + ret = hisi_pcie_init_pmu(pdev, pcie_pmu); + if (ret) + return ret; + + pci_set_drvdata(pdev, pcie_pmu); + + return ret; +} + +static void hisi_pcie_pmu_remove(struct pci_dev *pdev) +{ + hisi_pcie_uninit_pmu(pdev); + pci_set_drvdata(pdev, NULL); +} + +static const struct pci_device_id hisi_pcie_pmu_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) }, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids); + +static struct pci_driver hisi_pcie_pmu_driver = { + .name = DRV_NAME, + .id_table = hisi_pcie_pmu_ids, + .probe = hisi_pcie_pmu_probe, + .remove = hisi_pcie_pmu_remove, +}; + +static int __init hisi_pcie_module_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, + "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE", + hisi_pcie_pmu_online_cpu, + hisi_pcie_pmu_offline_cpu); + if (ret) { + pr_err("Failed to setup PCIe PMU hotplug: %d\n", ret); + return ret; + } + + ret = pci_register_driver(&hisi_pcie_pmu_driver); + if (ret) + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); + + return ret; +} +module_init(hisi_pcie_module_init); + +static void __exit hisi_pcie_module_exit(void) +{ + pci_unregister_driver(&hisi_pcie_pmu_driver); + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); +} +module_exit(hisi_pcie_module_exit); + +MODULE_DESCRIPTION("HiSilicon PCIe PMU driver"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Qi Liu "); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 773c83730906..411a428ace4d 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -225,6 +225,7 @@ enum cpuhp_state { CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, CPUHP_AP_PERF_ARM_L2X0_ONLINE, CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, -- cgit v1.2.3-59-g8ed1b From c8a2a011cd04054a6577d8cf774adf9e09302876 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 14 Dec 2021 03:45:35 +0200 Subject: net: dsa: sja1105: fix broken connection with the sja1110 tagger The driver was incorrectly converted assuming that "sja1105" is the only tagger supported by this driver. This results in SJA1110 switches failing to probe: sja1105 spi1.0: Unable to connect to tag protocol "sja1110": -EPROTONOSUPPORT sja1105: probe of spi1.2 failed with error -93 Add DSA_TAG_PROTO_SJA1110 to the list of supported taggers by the sja1105 driver. The sja1105_tagger_data structure format is common for the two tagging protocols. Fixes: c79e84866d2a ("net: dsa: tag_sja1105: convert to tagger-owned data") Signed-off-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/dsa/sja1105/sja1105_main.c | 16 ++++++++-------- include/linux/dsa/sja1105.h | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index 9171fbea588c..b513713be610 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2708,17 +2708,17 @@ static void sja1105_port_deferred_xmit(struct kthread_work *work) static int sja1105_connect_tag_protocol(struct dsa_switch *ds, enum dsa_tag_protocol proto) { + struct sja1105_private *priv = ds->priv; struct sja1105_tagger_data *tagger_data; - switch (proto) { - case DSA_TAG_PROTO_SJA1105: - tagger_data = sja1105_tagger_data(ds); - tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; - tagger_data->meta_tstamp_handler = sja1110_process_meta_tstamp; - return 0; - default: + if (proto != priv->info->tag_proto) return -EPROTONOSUPPORT; - } + + tagger_data = sja1105_tagger_data(ds); + tagger_data->xmit_work_fn = sja1105_port_deferred_xmit; + tagger_data->meta_tstamp_handler = sja1110_process_meta_tstamp; + + return 0; } /* The MAXAGE setting belongs to the L2 Forwarding Parameters table, diff --git a/include/linux/dsa/sja1105.h b/include/linux/dsa/sja1105.h index e9cb1ae6d742..159e43171ccc 100644 --- a/include/linux/dsa/sja1105.h +++ b/include/linux/dsa/sja1105.h @@ -70,7 +70,8 @@ struct sja1105_skb_cb { static inline struct sja1105_tagger_data * sja1105_tagger_data(struct dsa_switch *ds) { - BUG_ON(ds->dst->tag_ops->proto != DSA_TAG_PROTO_SJA1105); + BUG_ON(ds->dst->tag_ops->proto != DSA_TAG_PROTO_SJA1105 && + ds->dst->tag_ops->proto != DSA_TAG_PROTO_SJA1110); return ds->tagger_data; } -- cgit v1.2.3-59-g8ed1b From 9280ac2e6f199cddcd746a9ba459136b8666287b Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 14 Dec 2021 07:15:15 -0800 Subject: net: dev_replace_track() cleanup Use existing helpers (netdev_tracker_free() and netdev_tracker_alloc()) to remove ifdefery. Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20211214151515.312535-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 235d5d082f1a..c06e9dc1a317 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3885,16 +3885,14 @@ static inline void dev_replace_track(struct net_device *odev, netdevice_tracker *tracker, gfp_t gfp) { -#ifdef CONFIG_NET_DEV_REFCNT_TRACKER if (odev) - ref_tracker_free(&odev->refcnt_tracker, tracker); -#endif + netdev_tracker_free(odev, tracker); + dev_hold(ndev); dev_put(odev); -#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + if (ndev) - ref_tracker_alloc(&ndev->refcnt_tracker, tracker, gfp); -#endif + netdev_tracker_alloc(ndev, tracker, gfp); } /* Carrier loss detection, dial on demand. The functions netif_carrier_on -- cgit v1.2.3-59-g8ed1b From ad69cd9972e79aba103ba5365de0acd35770c265 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 29 Nov 2021 22:15:27 +0200 Subject: fsnotify: clarify object type argument In preparation for separating object type from iterator type, rename some 'type' arguments in functions to 'obj_type' and remove the unused interface to clear marks by object type mask. Link: https://lore.kernel.org/r/20211129201537.1932819-2-amir73il@gmail.com Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/notify/fanotify/fanotify_user.c | 8 ++++---- fs/notify/group.c | 2 +- fs/notify/mark.c | 27 +++++++++++++++------------ include/linux/fsnotify_backend.h | 28 ++++++++++++---------------- 4 files changed, 32 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 559bc1e9926d..ab24c37f1fdf 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1057,7 +1057,7 @@ static __u32 fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, fsnotify_connp_t *connp, - unsigned int type, + unsigned int obj_type, __kernel_fsid_t *fsid) { struct ucounts *ucounts = group->fanotify_data.ucounts; @@ -1080,7 +1080,7 @@ static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, } fsnotify_init_mark(mark, group); - ret = fsnotify_add_mark_locked(mark, connp, type, 0, fsid); + ret = fsnotify_add_mark_locked(mark, connp, obj_type, 0, fsid); if (ret) { fsnotify_put_mark(mark); goto out_dec_ucounts; @@ -1105,7 +1105,7 @@ static int fanotify_group_init_error_pool(struct fsnotify_group *group) } static int fanotify_add_mark(struct fsnotify_group *group, - fsnotify_connp_t *connp, unsigned int type, + fsnotify_connp_t *connp, unsigned int obj_type, __u32 mask, unsigned int flags, __kernel_fsid_t *fsid) { @@ -1116,7 +1116,7 @@ static int fanotify_add_mark(struct fsnotify_group *group, mutex_lock(&group->mark_mutex); fsn_mark = fsnotify_find_mark(connp, group); if (!fsn_mark) { - fsn_mark = fanotify_add_new_mark(group, connp, type, fsid); + fsn_mark = fanotify_add_new_mark(group, connp, obj_type, fsid); if (IS_ERR(fsn_mark)) { mutex_unlock(&group->mark_mutex); return PTR_ERR(fsn_mark); diff --git a/fs/notify/group.c b/fs/notify/group.c index 6a297efc4788..b7d4d64f87c2 100644 --- a/fs/notify/group.c +++ b/fs/notify/group.c @@ -58,7 +58,7 @@ void fsnotify_destroy_group(struct fsnotify_group *group) fsnotify_group_stop_queueing(group); /* Clear all marks for this group and queue them for destruction */ - fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_ALL_TYPES_MASK); + fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_ANY); /* * Some marks can still be pinned when waiting for response from diff --git a/fs/notify/mark.c b/fs/notify/mark.c index fa1d99101f89..52af2e9dadc0 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -496,7 +496,7 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b) } static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, - unsigned int type, + unsigned int obj_type, __kernel_fsid_t *fsid) { struct inode *inode = NULL; @@ -507,7 +507,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, return -ENOMEM; spin_lock_init(&conn->lock); INIT_HLIST_HEAD(&conn->list); - conn->type = type; + conn->type = obj_type; conn->obj = connp; /* Cache fsid of filesystem containing the object */ if (fsid) { @@ -572,7 +572,8 @@ out: * priority, highest number first, and then by the group's location in memory. */ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, unsigned int type, + fsnotify_connp_t *connp, + unsigned int obj_type, int allow_dups, __kernel_fsid_t *fsid) { struct fsnotify_mark *lmark, *last = NULL; @@ -580,7 +581,7 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, int cmp; int err = 0; - if (WARN_ON(!fsnotify_valid_obj_type(type))) + if (WARN_ON(!fsnotify_valid_obj_type(obj_type))) return -EINVAL; /* Backend is expected to check for zero fsid (e.g. tmpfs) */ @@ -592,7 +593,8 @@ restart: conn = fsnotify_grab_connector(connp); if (!conn) { spin_unlock(&mark->lock); - err = fsnotify_attach_connector_to_object(connp, type, fsid); + err = fsnotify_attach_connector_to_object(connp, obj_type, + fsid); if (err) return err; goto restart; @@ -665,7 +667,7 @@ out_err: * event types should be delivered to which group. */ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, unsigned int type, + fsnotify_connp_t *connp, unsigned int obj_type, int allow_dups, __kernel_fsid_t *fsid) { struct fsnotify_group *group = mark->group; @@ -686,7 +688,7 @@ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, fsnotify_get_mark(mark); /* for g_list */ spin_unlock(&mark->lock); - ret = fsnotify_add_mark_list(mark, connp, type, allow_dups, fsid); + ret = fsnotify_add_mark_list(mark, connp, obj_type, allow_dups, fsid); if (ret) goto err; @@ -706,13 +708,14 @@ err: } int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp, - unsigned int type, int allow_dups, __kernel_fsid_t *fsid) + unsigned int obj_type, int allow_dups, + __kernel_fsid_t *fsid) { int ret; struct fsnotify_group *group = mark->group; mutex_lock(&group->mark_mutex); - ret = fsnotify_add_mark_locked(mark, connp, type, allow_dups, fsid); + ret = fsnotify_add_mark_locked(mark, connp, obj_type, allow_dups, fsid); mutex_unlock(&group->mark_mutex); return ret; } @@ -747,14 +750,14 @@ EXPORT_SYMBOL_GPL(fsnotify_find_mark); /* Clear any marks in a group with given type mask */ void fsnotify_clear_marks_by_group(struct fsnotify_group *group, - unsigned int type_mask) + unsigned int obj_type) { struct fsnotify_mark *lmark, *mark; LIST_HEAD(to_free); struct list_head *head = &to_free; /* Skip selection step if we want to clear all marks. */ - if (type_mask == FSNOTIFY_OBJ_ALL_TYPES_MASK) { + if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) { head = &group->marks_list; goto clear; } @@ -769,7 +772,7 @@ void fsnotify_clear_marks_by_group(struct fsnotify_group *group, */ mutex_lock_nested(&group->mark_mutex, SINGLE_DEPTH_NESTING); list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) { - if ((1U << mark->connector->type) & type_mask) + if (mark->connector->type == obj_type) list_move(&mark->g_list, &to_free); } mutex_unlock(&group->mark_mutex); diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 51ef2b079bfa..b9c84b1dbcc8 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -338,6 +338,7 @@ static inline struct fs_error_report *fsnotify_data_error_report( } enum fsnotify_obj_type { + FSNOTIFY_OBJ_TYPE_ANY = -1, FSNOTIFY_OBJ_TYPE_INODE, FSNOTIFY_OBJ_TYPE_PARENT, FSNOTIFY_OBJ_TYPE_VFSMOUNT, @@ -346,15 +347,9 @@ enum fsnotify_obj_type { FSNOTIFY_OBJ_TYPE_DETACHED = FSNOTIFY_OBJ_TYPE_COUNT }; -#define FSNOTIFY_OBJ_TYPE_INODE_FL (1U << FSNOTIFY_OBJ_TYPE_INODE) -#define FSNOTIFY_OBJ_TYPE_PARENT_FL (1U << FSNOTIFY_OBJ_TYPE_PARENT) -#define FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL (1U << FSNOTIFY_OBJ_TYPE_VFSMOUNT) -#define FSNOTIFY_OBJ_TYPE_SB_FL (1U << FSNOTIFY_OBJ_TYPE_SB) -#define FSNOTIFY_OBJ_ALL_TYPES_MASK ((1U << FSNOTIFY_OBJ_TYPE_COUNT) - 1) - -static inline bool fsnotify_valid_obj_type(unsigned int type) +static inline bool fsnotify_valid_obj_type(unsigned int obj_type) { - return (type < FSNOTIFY_OBJ_TYPE_COUNT); + return (obj_type < FSNOTIFY_OBJ_TYPE_COUNT); } struct fsnotify_iter_info { @@ -387,7 +382,7 @@ static inline void fsnotify_iter_set_report_type_mark( static inline struct fsnotify_mark *fsnotify_iter_##name##_mark( \ struct fsnotify_iter_info *iter_info) \ { \ - return (iter_info->report_mask & FSNOTIFY_OBJ_TYPE_##NAME##_FL) ? \ + return (iter_info->report_mask & (1U << FSNOTIFY_OBJ_TYPE_##NAME)) ? \ iter_info->marks[FSNOTIFY_OBJ_TYPE_##NAME] : NULL; \ } @@ -604,11 +599,11 @@ extern int fsnotify_get_conn_fsid(const struct fsnotify_mark_connector *conn, __kernel_fsid_t *fsid); /* attach the mark to the object */ extern int fsnotify_add_mark(struct fsnotify_mark *mark, - fsnotify_connp_t *connp, unsigned int type, + fsnotify_connp_t *connp, unsigned int obj_type, int allow_dups, __kernel_fsid_t *fsid); extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark, fsnotify_connp_t *connp, - unsigned int type, int allow_dups, + unsigned int obj_type, int allow_dups, __kernel_fsid_t *fsid); /* attach the mark to the inode */ @@ -637,22 +632,23 @@ extern void fsnotify_detach_mark(struct fsnotify_mark *mark); extern void fsnotify_free_mark(struct fsnotify_mark *mark); /* Wait until all marks queued for destruction are destroyed */ extern void fsnotify_wait_marks_destroyed(void); -/* run all the marks in a group, and clear all of the marks attached to given object type */ -extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group, unsigned int type); +/* Clear all of the marks of a group attached to a given object type */ +extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group, + unsigned int obj_type); /* run all the marks in a group, and clear all of the vfsmount marks */ static inline void fsnotify_clear_vfsmount_marks_by_group(struct fsnotify_group *group) { - fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL); + fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT); } /* run all the marks in a group, and clear all of the inode marks */ static inline void fsnotify_clear_inode_marks_by_group(struct fsnotify_group *group) { - fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE_FL); + fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE); } /* run all the marks in a group, and clear all of the sn marks */ static inline void fsnotify_clear_sb_marks_by_group(struct fsnotify_group *group) { - fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_SB_FL); + fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_SB); } extern void fsnotify_get_mark(struct fsnotify_mark *mark); extern void fsnotify_put_mark(struct fsnotify_mark *mark); -- cgit v1.2.3-59-g8ed1b From 1c9007d62bea6fd164285314f7553f73e5308863 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 29 Nov 2021 22:15:28 +0200 Subject: fsnotify: separate mark iterator type from object type enum They are two different types that use the same enum, so this confusing. Use the object type to indicate the type of object mark is attached to and the iter type to indicate the type of watch. A group can have two different watches of the same object type (parent and child watches) that match the same event. Link: https://lore.kernel.org/r/20211129201537.1932819-3-amir73il@gmail.com Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/notify/fanotify/fanotify.c | 6 +++--- fs/notify/fsnotify.c | 18 +++++++++--------- fs/notify/mark.c | 4 ++-- include/linux/fsnotify_backend.h | 41 +++++++++++++++++++++++++++------------- 4 files changed, 42 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index b6091775aa6e..652fe84cb8ac 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -299,7 +299,7 @@ static u32 fanotify_group_event_mask(struct fsnotify_group *group, return 0; } - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { if (!fsnotify_iter_should_report_type(iter_info, type)) continue; mark = iter_info->marks[type]; @@ -318,7 +318,7 @@ static u32 fanotify_group_event_mask(struct fsnotify_group *group, * If the event is on a child and this mark is on a parent not * watching children, don't send it! */ - if (type == FSNOTIFY_OBJ_TYPE_PARENT && + if (type == FSNOTIFY_ITER_TYPE_PARENT && !(mark->mask & FS_EVENT_ON_CHILD)) continue; @@ -746,7 +746,7 @@ static __kernel_fsid_t fanotify_get_fsid(struct fsnotify_iter_info *iter_info) int type; __kernel_fsid_t fsid = {}; - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { struct fsnotify_mark_connector *conn; if (!fsnotify_iter_should_report_type(iter_info, type)) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 4034ca566f95..0c94457c625e 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -330,7 +330,7 @@ static int send_to_group(__u32 mask, const void *data, int data_type, /* clear ignored on inode modification */ if (mask & FS_MODIFY) { - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { if (!fsnotify_iter_should_report_type(iter_info, type)) continue; mark = iter_info->marks[type]; @@ -340,7 +340,7 @@ static int send_to_group(__u32 mask, const void *data, int data_type, } } - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { if (!fsnotify_iter_should_report_type(iter_info, type)) continue; mark = iter_info->marks[type]; @@ -405,7 +405,7 @@ static unsigned int fsnotify_iter_select_report_types( int type; /* Choose max prio group among groups of all queue heads */ - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { mark = iter_info->marks[type]; if (mark && fsnotify_compare_groups(max_prio_group, mark->group) > 0) @@ -417,7 +417,7 @@ static unsigned int fsnotify_iter_select_report_types( /* Set the report mask for marks from same group as max prio group */ iter_info->report_mask = 0; - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { mark = iter_info->marks[type]; if (mark && fsnotify_compare_groups(max_prio_group, mark->group) == 0) @@ -435,7 +435,7 @@ static void fsnotify_iter_next(struct fsnotify_iter_info *iter_info) { int type; - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { if (fsnotify_iter_should_report_type(iter_info, type)) iter_info->marks[type] = fsnotify_next_mark(iter_info->marks[type]); @@ -519,18 +519,18 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); - iter_info.marks[FSNOTIFY_OBJ_TYPE_SB] = + iter_info.marks[FSNOTIFY_ITER_TYPE_SB] = fsnotify_first_mark(&sb->s_fsnotify_marks); if (mnt) { - iter_info.marks[FSNOTIFY_OBJ_TYPE_VFSMOUNT] = + iter_info.marks[FSNOTIFY_ITER_TYPE_VFSMOUNT] = fsnotify_first_mark(&mnt->mnt_fsnotify_marks); } if (inode) { - iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] = + iter_info.marks[FSNOTIFY_ITER_TYPE_INODE] = fsnotify_first_mark(&inode->i_fsnotify_marks); } if (parent) { - iter_info.marks[FSNOTIFY_OBJ_TYPE_PARENT] = + iter_info.marks[FSNOTIFY_ITER_TYPE_PARENT] = fsnotify_first_mark(&parent->i_fsnotify_marks); } diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 52af2e9dadc0..9007d6affff3 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -353,7 +353,7 @@ bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info) { int type; - fsnotify_foreach_obj_type(type) { + fsnotify_foreach_iter_type(type) { /* This can fail if mark is being removed */ if (!fsnotify_get_mark_safe(iter_info->marks[type])) { __release(&fsnotify_mark_srcu); @@ -382,7 +382,7 @@ void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info) int type; iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); - fsnotify_foreach_obj_type(type) + fsnotify_foreach_iter_type(type) fsnotify_put_mark_wake(iter_info->marks[type]); } diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index b9c84b1dbcc8..73739fee1710 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -337,10 +337,25 @@ static inline struct fs_error_report *fsnotify_data_error_report( } } +/* + * Index to merged marks iterator array that correlates to a type of watch. + * The type of watched object can be deduced from the iterator type, but not + * the other way around, because an event can match different watched objects + * of the same object type. + * For example, both parent and child are watching an object of type inode. + */ +enum fsnotify_iter_type { + FSNOTIFY_ITER_TYPE_INODE, + FSNOTIFY_ITER_TYPE_VFSMOUNT, + FSNOTIFY_ITER_TYPE_SB, + FSNOTIFY_ITER_TYPE_PARENT, + FSNOTIFY_ITER_TYPE_COUNT +}; + +/* The type of object that a mark is attached to */ enum fsnotify_obj_type { FSNOTIFY_OBJ_TYPE_ANY = -1, FSNOTIFY_OBJ_TYPE_INODE, - FSNOTIFY_OBJ_TYPE_PARENT, FSNOTIFY_OBJ_TYPE_VFSMOUNT, FSNOTIFY_OBJ_TYPE_SB, FSNOTIFY_OBJ_TYPE_COUNT, @@ -353,37 +368,37 @@ static inline bool fsnotify_valid_obj_type(unsigned int obj_type) } struct fsnotify_iter_info { - struct fsnotify_mark *marks[FSNOTIFY_OBJ_TYPE_COUNT]; + struct fsnotify_mark *marks[FSNOTIFY_ITER_TYPE_COUNT]; unsigned int report_mask; int srcu_idx; }; static inline bool fsnotify_iter_should_report_type( - struct fsnotify_iter_info *iter_info, int type) + struct fsnotify_iter_info *iter_info, int iter_type) { - return (iter_info->report_mask & (1U << type)); + return (iter_info->report_mask & (1U << iter_type)); } static inline void fsnotify_iter_set_report_type( - struct fsnotify_iter_info *iter_info, int type) + struct fsnotify_iter_info *iter_info, int iter_type) { - iter_info->report_mask |= (1U << type); + iter_info->report_mask |= (1U << iter_type); } static inline void fsnotify_iter_set_report_type_mark( - struct fsnotify_iter_info *iter_info, int type, + struct fsnotify_iter_info *iter_info, int iter_type, struct fsnotify_mark *mark) { - iter_info->marks[type] = mark; - iter_info->report_mask |= (1U << type); + iter_info->marks[iter_type] = mark; + iter_info->report_mask |= (1U << iter_type); } #define FSNOTIFY_ITER_FUNCS(name, NAME) \ static inline struct fsnotify_mark *fsnotify_iter_##name##_mark( \ struct fsnotify_iter_info *iter_info) \ { \ - return (iter_info->report_mask & (1U << FSNOTIFY_OBJ_TYPE_##NAME)) ? \ - iter_info->marks[FSNOTIFY_OBJ_TYPE_##NAME] : NULL; \ + return (iter_info->report_mask & (1U << FSNOTIFY_ITER_TYPE_##NAME)) ? \ + iter_info->marks[FSNOTIFY_ITER_TYPE_##NAME] : NULL; \ } FSNOTIFY_ITER_FUNCS(inode, INODE) @@ -391,8 +406,8 @@ FSNOTIFY_ITER_FUNCS(parent, PARENT) FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT) FSNOTIFY_ITER_FUNCS(sb, SB) -#define fsnotify_foreach_obj_type(type) \ - for (type = 0; type < FSNOTIFY_OBJ_TYPE_COUNT; type++) +#define fsnotify_foreach_iter_type(type) \ + for (type = 0; type < FSNOTIFY_ITER_TYPE_COUNT; type++) /* * fsnotify_connp_t is what we embed in objects which connector can be attached -- cgit v1.2.3-59-g8ed1b From d61fd650e9d206a71fda789f02a1ced4b19944c4 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 29 Nov 2021 22:15:29 +0200 Subject: fanotify: introduce group flag FAN_REPORT_TARGET_FID FAN_REPORT_FID is ambiguous in that it reports the fid of the child for some events and the fid of the parent for create/delete/move events. The new FAN_REPORT_TARGET_FID flag is an implicit request to report the fid of the target object of the operation (a.k.a the child inode) also in create/delete/move events in addition to the fid of the parent and the name of the child. To reduce the test matrix for uninteresting use cases, the new FAN_REPORT_TARGET_FID flag requires both FAN_REPORT_NAME and FAN_REPORT_FID. The convenience macro FAN_REPORT_DFID_NAME_TARGET combines FAN_REPORT_TARGET_FID with all the required flags. Link: https://lore.kernel.org/r/20211129201537.1932819-4-amir73il@gmail.com Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/notify/fanotify/fanotify.c | 48 ++++++++++++++++++++++++++++---------- fs/notify/fanotify/fanotify_user.c | 11 ++++++++- include/linux/fanotify.h | 2 +- include/uapi/linux/fanotify.h | 4 ++++ 4 files changed, 51 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 652fe84cb8ac..85e542b164c8 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -458,17 +458,41 @@ out_err: } /* - * The inode to use as identifier when reporting fid depends on the event. - * Report the modified directory inode on dirent modification events. - * Report the "victim" inode otherwise. + * FAN_REPORT_FID is ambiguous in that it reports the fid of the child for + * some events and the fid of the parent for create/delete/move events. + * + * With the FAN_REPORT_TARGET_FID flag, the fid of the child is reported + * also in create/delete/move events in addition to the fid of the parent + * and the name of the child. + */ +static inline bool fanotify_report_child_fid(unsigned int fid_mode, u32 mask) +{ + if (mask & ALL_FSNOTIFY_DIRENT_EVENTS) + return (fid_mode & FAN_REPORT_TARGET_FID); + + return (fid_mode & FAN_REPORT_FID) && !(mask & FAN_ONDIR); +} + +/* + * The inode to use as identifier when reporting fid depends on the event + * and the group flags. + * + * With the group flag FAN_REPORT_TARGET_FID, always report the child fid. + * + * Without the group flag FAN_REPORT_TARGET_FID, report the modified directory + * fid on dirent events and the child fid otherwise. + * * For example: - * FS_ATTRIB reports the child inode even if reported on a watched parent. - * FS_CREATE reports the modified dir inode and not the created inode. + * FS_ATTRIB reports the child fid even if reported on a watched parent. + * FS_CREATE reports the modified dir fid without FAN_REPORT_TARGET_FID. + * and reports the created child fid with FAN_REPORT_TARGET_FID. */ static struct inode *fanotify_fid_inode(u32 event_mask, const void *data, - int data_type, struct inode *dir) + int data_type, struct inode *dir, + unsigned int fid_mode) { - if (event_mask & ALL_FSNOTIFY_DIRENT_EVENTS) + if ((event_mask & ALL_FSNOTIFY_DIRENT_EVENTS) && + !(fid_mode & FAN_REPORT_TARGET_FID)) return dir; return fsnotify_data_inode(data, data_type); @@ -647,10 +671,11 @@ static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group, { struct fanotify_event *event = NULL; gfp_t gfp = GFP_KERNEL_ACCOUNT; - struct inode *id = fanotify_fid_inode(mask, data, data_type, dir); + unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); + struct inode *id = fanotify_fid_inode(mask, data, data_type, dir, + fid_mode); struct inode *dirid = fanotify_dfid_inode(mask, data, data_type, dir); const struct path *path = fsnotify_data_path(data, data_type); - unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); struct mem_cgroup *old_memcg; struct inode *child = NULL; bool name_event = false; @@ -660,11 +685,10 @@ static struct fanotify_event *fanotify_alloc_event(struct fsnotify_group *group, if ((fid_mode & FAN_REPORT_DIR_FID) && dirid) { /* - * With both flags FAN_REPORT_DIR_FID and FAN_REPORT_FID, we - * report the child fid for events reported on a non-dir child + * For certain events and group flags, report the child fid * in addition to reporting the parent fid and maybe child name. */ - if ((fid_mode & FAN_REPORT_FID) && id != dirid && !ondir) + if (fanotify_report_child_fid(fid_mode, mask) && id != dirid) child = id; id = dirid; diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index ab24c37f1fdf..00bbc29712bb 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1275,6 +1275,15 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) if ((fid_mode & FAN_REPORT_NAME) && !(fid_mode & FAN_REPORT_DIR_FID)) return -EINVAL; + /* + * FAN_REPORT_TARGET_FID requires FAN_REPORT_NAME and FAN_REPORT_FID + * and is used as an indication to report both dir and child fid on all + * dirent events. + */ + if ((fid_mode & FAN_REPORT_TARGET_FID) && + (!(fid_mode & FAN_REPORT_NAME) || !(fid_mode & FAN_REPORT_FID))) + return -EINVAL; + f_flags = O_RDWR | FMODE_NONOTIFY; if (flags & FAN_CLOEXEC) f_flags |= O_CLOEXEC; @@ -1667,7 +1676,7 @@ static int __init fanotify_user_setup(void) FANOTIFY_DEFAULT_MAX_USER_MARKS); BUILD_BUG_ON(FANOTIFY_INIT_FLAGS & FANOTIFY_INTERNAL_GROUP_FLAGS); - BUILD_BUG_ON(HWEIGHT32(FANOTIFY_INIT_FLAGS) != 11); + BUILD_BUG_ON(HWEIGHT32(FANOTIFY_INIT_FLAGS) != 12); BUILD_BUG_ON(HWEIGHT32(FANOTIFY_MARK_FLAGS) != 9); fanotify_mark_cache = KMEM_CACHE(fsnotify_mark, diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h index 616af2ea20f3..376e050e6f38 100644 --- a/include/linux/fanotify.h +++ b/include/linux/fanotify.h @@ -25,7 +25,7 @@ extern struct ctl_table fanotify_table[]; /* for sysctl */ #define FANOTIFY_CLASS_BITS (FAN_CLASS_NOTIF | FANOTIFY_PERM_CLASSES) -#define FANOTIFY_FID_BITS (FAN_REPORT_FID | FAN_REPORT_DFID_NAME) +#define FANOTIFY_FID_BITS (FAN_REPORT_DFID_NAME_TARGET) #define FANOTIFY_INFO_MODES (FANOTIFY_FID_BITS | FAN_REPORT_PIDFD) diff --git a/include/uapi/linux/fanotify.h b/include/uapi/linux/fanotify.h index bd1932c2074d..60f73639a896 100644 --- a/include/uapi/linux/fanotify.h +++ b/include/uapi/linux/fanotify.h @@ -57,9 +57,13 @@ #define FAN_REPORT_FID 0x00000200 /* Report unique file id */ #define FAN_REPORT_DIR_FID 0x00000400 /* Report unique directory id */ #define FAN_REPORT_NAME 0x00000800 /* Report events with name */ +#define FAN_REPORT_TARGET_FID 0x00001000 /* Report dirent target id */ /* Convenience macro - FAN_REPORT_NAME requires FAN_REPORT_DIR_FID */ #define FAN_REPORT_DFID_NAME (FAN_REPORT_DIR_FID | FAN_REPORT_NAME) +/* Convenience macro - FAN_REPORT_TARGET_FID requires all other FID flags */ +#define FAN_REPORT_DFID_NAME_TARGET (FAN_REPORT_DFID_NAME | \ + FAN_REPORT_FID | FAN_REPORT_TARGET_FID) /* Deprecated - do not use this in programs and do not add new flags here! */ #define FAN_ALL_INIT_FLAGS (FAN_CLOEXEC | FAN_NONBLOCK | \ -- cgit v1.2.3-59-g8ed1b From e54183fa7047c15819bc155f4c58501d9a9a3489 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 29 Nov 2021 22:15:30 +0200 Subject: fsnotify: generate FS_RENAME event with rich information The dnotify FS_DN_RENAME event is used to request notification about a move within the same parent directory and was always coupled with the FS_MOVED_FROM event. Rename the FS_DN_RENAME event flag to FS_RENAME, decouple it from FS_MOVED_FROM and report it with the moved dentry instead of the moved inode, so it has the information about both old and new parent and name. Generate the FS_RENAME event regardless of same parent dir and apply the "same parent" rule in the generic fsnotify_handle_event() helper that is used to call backends with ->handle_inode_event() method (i.e. dnotify). The ->handle_inode_event() method is not rich enough to report both old and new parent and name anyway. The enriched event is reported to fanotify over the ->handle_event() method with the old and new dir inode marks in marks array slots for ITER_TYPE_INODE and a new iter type slot ITER_TYPE_INODE2. The enriched event will be used for reporting old and new parent+name to fanotify groups with FAN_RENAME events. Link: https://lore.kernel.org/r/20211129201537.1932819-5-amir73il@gmail.com Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/notify/dnotify/dnotify.c | 2 +- fs/notify/fsnotify.c | 37 +++++++++++++++++++++++++++++-------- include/linux/dnotify.h | 2 +- include/linux/fsnotify.h | 9 ++++++--- include/linux/fsnotify_backend.h | 7 ++++--- 5 files changed, 41 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c index e85e13c50d6d..d5ebebb034ff 100644 --- a/fs/notify/dnotify/dnotify.c +++ b/fs/notify/dnotify/dnotify.c @@ -196,7 +196,7 @@ static __u32 convert_arg(unsigned long arg) if (arg & DN_ATTRIB) new_mask |= FS_ATTRIB; if (arg & DN_RENAME) - new_mask |= FS_DN_RENAME; + new_mask |= FS_RENAME; if (arg & DN_CREATE) new_mask |= (FS_CREATE | FS_MOVED_TO); diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 0c94457c625e..ab81a0776ece 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -279,6 +279,18 @@ static int fsnotify_handle_event(struct fsnotify_group *group, __u32 mask, WARN_ON_ONCE(fsnotify_iter_vfsmount_mark(iter_info))) return 0; + /* + * For FS_RENAME, 'dir' is old dir and 'data' is new dentry. + * The only ->handle_inode_event() backend that supports FS_RENAME is + * dnotify, where it means file was renamed within same parent. + */ + if (mask & FS_RENAME) { + struct dentry *moved = fsnotify_data_dentry(data, data_type); + + if (dir != moved->d_parent->d_inode) + return 0; + } + if (parent_mark) { /* * parent_mark indicates that the parent inode is watching @@ -469,7 +481,9 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, struct super_block *sb = fsnotify_data_sb(data, data_type); struct fsnotify_iter_info iter_info = {}; struct mount *mnt = NULL; - struct inode *parent = NULL; + struct inode *inode2 = NULL; + struct dentry *moved; + int inode2_type; int ret = 0; __u32 test_mask, marks_mask; @@ -479,12 +493,19 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, if (!inode) { /* Dirent event - report on TYPE_INODE to dir */ inode = dir; + /* For FS_RENAME, inode is old_dir and inode2 is new_dir */ + if (mask & FS_RENAME) { + moved = fsnotify_data_dentry(data, data_type); + inode2 = moved->d_parent->d_inode; + inode2_type = FSNOTIFY_ITER_TYPE_INODE2; + } } else if (mask & FS_EVENT_ON_CHILD) { /* * Event on child - report on TYPE_PARENT to dir if it is * watching children and on TYPE_INODE to child. */ - parent = dir; + inode2 = dir; + inode2_type = FSNOTIFY_ITER_TYPE_PARENT; } /* @@ -497,7 +518,7 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, if (!sb->s_fsnotify_marks && (!mnt || !mnt->mnt_fsnotify_marks) && (!inode || !inode->i_fsnotify_marks) && - (!parent || !parent->i_fsnotify_marks)) + (!inode2 || !inode2->i_fsnotify_marks)) return 0; marks_mask = sb->s_fsnotify_mask; @@ -505,8 +526,8 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, marks_mask |= mnt->mnt_fsnotify_mask; if (inode) marks_mask |= inode->i_fsnotify_mask; - if (parent) - marks_mask |= parent->i_fsnotify_mask; + if (inode2) + marks_mask |= inode2->i_fsnotify_mask; /* @@ -529,9 +550,9 @@ int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, iter_info.marks[FSNOTIFY_ITER_TYPE_INODE] = fsnotify_first_mark(&inode->i_fsnotify_marks); } - if (parent) { - iter_info.marks[FSNOTIFY_ITER_TYPE_PARENT] = - fsnotify_first_mark(&parent->i_fsnotify_marks); + if (inode2) { + iter_info.marks[inode2_type] = + fsnotify_first_mark(&inode2->i_fsnotify_marks); } /* diff --git a/include/linux/dnotify.h b/include/linux/dnotify.h index 0aad774beaec..b87c3b85a166 100644 --- a/include/linux/dnotify.h +++ b/include/linux/dnotify.h @@ -26,7 +26,7 @@ struct dnotify_struct { FS_MODIFY | FS_MODIFY_CHILD |\ FS_ACCESS | FS_ACCESS_CHILD |\ FS_ATTRIB | FS_ATTRIB_CHILD |\ - FS_CREATE | FS_DN_RENAME |\ + FS_CREATE | FS_RENAME |\ FS_MOVED_FROM | FS_MOVED_TO) extern int dir_notify_enable; diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 787545e87eeb..3a2d7dc3c607 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -144,16 +144,19 @@ static inline void fsnotify_move(struct inode *old_dir, struct inode *new_dir, u32 fs_cookie = fsnotify_get_cookie(); __u32 old_dir_mask = FS_MOVED_FROM; __u32 new_dir_mask = FS_MOVED_TO; + __u32 rename_mask = FS_RENAME; const struct qstr *new_name = &moved->d_name; - if (old_dir == new_dir) - old_dir_mask |= FS_DN_RENAME; - if (isdir) { old_dir_mask |= FS_ISDIR; new_dir_mask |= FS_ISDIR; + rename_mask |= FS_ISDIR; } + /* Event with information about both old and new parent+name */ + fsnotify_name(rename_mask, moved, FSNOTIFY_EVENT_DENTRY, + old_dir, old_name, 0); + fsnotify_name(old_dir_mask, source, FSNOTIFY_EVENT_INODE, old_dir, old_name, fs_cookie); fsnotify_name(new_dir_mask, source, FSNOTIFY_EVENT_INODE, diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 73739fee1710..790c31844db5 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -63,7 +63,7 @@ */ #define FS_EVENT_ON_CHILD 0x08000000 -#define FS_DN_RENAME 0x10000000 /* file renamed */ +#define FS_RENAME 0x10000000 /* File was renamed */ #define FS_DN_MULTISHOT 0x20000000 /* dnotify multishot */ #define FS_ISDIR 0x40000000 /* event occurred against dir */ #define FS_IN_ONESHOT 0x80000000 /* only send event once */ @@ -76,7 +76,7 @@ * The watching parent may get an FS_ATTRIB|FS_EVENT_ON_CHILD event * when a directory entry inside a child subdir changes. */ -#define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE) +#define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE | FS_RENAME) #define ALL_FSNOTIFY_PERM_EVENTS (FS_OPEN_PERM | FS_ACCESS_PERM | \ FS_OPEN_EXEC_PERM) @@ -101,7 +101,7 @@ /* Events that can be reported to backends */ #define ALL_FSNOTIFY_EVENTS (ALL_FSNOTIFY_DIRENT_EVENTS | \ FS_EVENTS_POSS_ON_CHILD | \ - FS_DELETE_SELF | FS_MOVE_SELF | FS_DN_RENAME | \ + FS_DELETE_SELF | FS_MOVE_SELF | \ FS_UNMOUNT | FS_Q_OVERFLOW | FS_IN_IGNORED | \ FS_ERROR) @@ -349,6 +349,7 @@ enum fsnotify_iter_type { FSNOTIFY_ITER_TYPE_VFSMOUNT, FSNOTIFY_ITER_TYPE_SB, FSNOTIFY_ITER_TYPE_PARENT, + FSNOTIFY_ITER_TYPE_INODE2, FSNOTIFY_ITER_TYPE_COUNT }; -- cgit v1.2.3-59-g8ed1b From 8cc3b1ccd930fe6971e1527f0c4f1bdc8cb56026 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Mon, 29 Nov 2021 22:15:37 +0200 Subject: fanotify: wire up FAN_RENAME event FAN_RENAME is the successor of FAN_MOVED_FROM and FAN_MOVED_TO and can be used to get the old and new parent+name information in a single event. FAN_MOVED_FROM and FAN_MOVED_TO are still supported for backward compatibility, but it makes little sense to use them together with FAN_RENAME in the same group. FAN_RENAME uses special info type records to report the old and new parent+name, so reporting only old and new parent id is less useful and was not implemented. Therefore, FAN_REANAME requires a group with flag FAN_REPORT_NAME. Link: https://lore.kernel.org/r/20211129201537.1932819-12-amir73il@gmail.com Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/notify/fanotify/fanotify.c | 2 +- fs/notify/fanotify/fanotify_user.c | 8 ++++++++ include/linux/fanotify.h | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 0da305b6f3e2..985e995d2a39 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -930,7 +930,7 @@ static int fanotify_handle_event(struct fsnotify_group *group, u32 mask, BUILD_BUG_ON(FAN_FS_ERROR != FS_ERROR); BUILD_BUG_ON(FAN_RENAME != FS_RENAME); - BUILD_BUG_ON(HWEIGHT32(ALL_FANOTIFY_EVENT_BITS) != 20); + BUILD_BUG_ON(HWEIGHT32(ALL_FANOTIFY_EVENT_BITS) != 21); mask = fanotify_group_event_mask(group, iter_info, &match_mask, mask, data, data_type, dir); diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index eb2a0251b318..73a3e939c921 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1586,6 +1586,14 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, (!fid_mode || mark_type == FAN_MARK_MOUNT)) goto fput_and_out; + /* + * FAN_RENAME uses special info type records to report the old and + * new parent+name. Reporting only old and new parent id is less + * useful and was not implemented. + */ + if (mask & FAN_RENAME && !(fid_mode & FAN_REPORT_NAME)) + goto fput_and_out; + if (flags & FAN_MARK_FLUSH) { ret = 0; if (mark_type == FAN_MARK_MOUNT) diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h index 376e050e6f38..3afdf339d53c 100644 --- a/include/linux/fanotify.h +++ b/include/linux/fanotify.h @@ -82,7 +82,8 @@ extern struct ctl_table fanotify_table[]; /* for sysctl */ * Directory entry modification events - reported only to directory * where entry is modified and not to a watching parent. */ -#define FANOTIFY_DIRENT_EVENTS (FAN_MOVE | FAN_CREATE | FAN_DELETE) +#define FANOTIFY_DIRENT_EVENTS (FAN_MOVE | FAN_CREATE | FAN_DELETE | \ + FAN_RENAME) /* Events that can be reported with event->fd */ #define FANOTIFY_FD_EVENTS (FANOTIFY_PATH_EVENTS | FANOTIFY_PERM_EVENTS) -- cgit v1.2.3-59-g8ed1b From f1d9268e061863ead77b07f5a6807d063e28a1c2 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 14 Dec 2021 07:09:33 -0800 Subject: net: add net device refcount tracker to struct packet_type Most notable changes are in af_packet, tipc ones are trivial. Signed-off-by: Eric Dumazet Cc: Jon Maloy Cc: Ying Xue Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + net/packet/af_packet.c | 14 +++++++++++--- net/tipc/bearer.c | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c06e9dc1a317..a419718612c6 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2533,6 +2533,7 @@ struct packet_type { __be16 type; /* This is really htons(ether_type). */ bool ignore_outgoing; struct net_device *dev; /* NULL is wildcarded here */ + netdevice_tracker dev_tracker; int (*func) (struct sk_buff *, struct net_device *, struct packet_type *, diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index a1ffdb48cc47..71854a16afbb 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3109,7 +3109,7 @@ static int packet_release(struct socket *sock) packet_cached_dev_reset(po); if (po->prot_hook.dev) { - dev_put(po->prot_hook.dev); + dev_put_track(po->prot_hook.dev, &po->prot_hook.dev_tracker); po->prot_hook.dev = NULL; } spin_unlock(&po->bind_lock); @@ -3217,18 +3217,25 @@ static int packet_do_bind(struct sock *sk, const char *name, int ifindex, WRITE_ONCE(po->num, proto); po->prot_hook.type = proto; + dev_put_track(dev_curr, &po->prot_hook.dev_tracker); + dev_curr = NULL; + if (unlikely(unlisted)) { dev_put(dev); po->prot_hook.dev = NULL; WRITE_ONCE(po->ifindex, -1); packet_cached_dev_reset(po); } else { + if (dev) + netdev_tracker_alloc(dev, + &po->prot_hook.dev_tracker, + GFP_ATOMIC); po->prot_hook.dev = dev; WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0); packet_cached_dev_assign(po, dev); } } - dev_put(dev_curr); + dev_put_track(dev_curr, &po->prot_hook.dev_tracker); if (proto == 0 || !need_rehook) goto out_unlock; @@ -4138,7 +4145,8 @@ static int packet_notifier(struct notifier_block *this, if (msg == NETDEV_UNREGISTER) { packet_cached_dev_reset(po); WRITE_ONCE(po->ifindex, -1); - dev_put(po->prot_hook.dev); + dev_put_track(po->prot_hook.dev, + &po->prot_hook.dev_tracker); po->prot_hook.dev = NULL; } spin_unlock(&po->bind_lock); diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c index 60bc74b76adc..473a790f5894 100644 --- a/net/tipc/bearer.c +++ b/net/tipc/bearer.c @@ -787,7 +787,7 @@ int tipc_attach_loopback(struct net *net) if (!dev) return -ENODEV; - dev_hold(dev); + dev_hold_track(dev, &tn->loopback_pt.dev_tracker, GFP_KERNEL); tn->loopback_pt.dev = dev; tn->loopback_pt.type = htons(ETH_P_TIPC); tn->loopback_pt.func = tipc_loopback_rcv_pkt; @@ -800,7 +800,7 @@ void tipc_detach_loopback(struct net *net) struct tipc_net *tn = tipc_net(net); dev_remove_pack(&tn->loopback_pt); - dev_put(net->loopback_dev); + dev_put_track(net->loopback_dev, &tn->loopback_pt.dev_tracker); } /* Caller should hold rtnl_lock to protect the bearer */ -- cgit v1.2.3-59-g8ed1b From 685b1afd7911676691c4167f420e16a957f5a38e Mon Sep 17 00:00:00 2001 From: Shay Drory Date: Thu, 9 Dec 2021 12:09:23 +0200 Subject: net/mlx5: Introduce log_max_current_uc_list_wr_supported bit Downstream patch will use this bit in order to know whether the device supports changing of max_uc_list. Signed-off-by: Shay Drory Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 3636df90899a..d3899fc33fd7 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1621,7 +1621,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 ext_stride_num_range[0x1]; u8 roce_rw_supported[0x1]; - u8 reserved_at_3a2[0x1]; + u8 log_max_current_uc_list_wr_supported[0x1]; u8 log_max_stride_sz_rq[0x5]; u8 reserved_at_3a8[0x3]; u8 log_min_stride_sz_rq[0x5]; -- cgit v1.2.3-59-g8ed1b From ff5f87cb6a75dbf6d30668d2464e46249dd5c47f Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 3 Dec 2021 11:28:49 +0100 Subject: clk: Introduce clk-tps68470 driver The TPS68470 PMIC provides Clocks, GPIOs and Regulators. At present in the kernel the Regulators and Clocks are controlled by an OpRegion driver designed to work with power control methods defined in ACPI, but some platforms lack those methods, meaning drivers need to be able to consume the resources of these chips through the usual frameworks. This commit adds a driver for the clocks provided by the tps68470, and is designed to bind to the platform_device registered by the intel_skl_int3472 module. This is based on this out of tree driver written by Intel: https://github.com/intel/linux-intel-lts/blob/4.14/base/drivers/clk/clk-tps68470.c with various cleanups added. Reviewed-by: Andy Shevchenko Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20211203102857.44539-7-hdegoede@redhat.com Signed-off-by: Stephen Boyd --- drivers/clk/Kconfig | 8 ++ drivers/clk/Makefile | 1 + drivers/clk/clk-tps68470.c | 261 +++++++++++++++++++++++++++++++++++++++++++ include/linux/mfd/tps68470.h | 11 ++ 4 files changed, 281 insertions(+) create mode 100644 drivers/clk/clk-tps68470.c (limited to 'include/linux') diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index c5b3dc97396a..4e9098d79249 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -169,6 +169,14 @@ config COMMON_CLK_CDCE706 help This driver supports TI CDCE706 programmable 3-PLL clock synthesizer. +config COMMON_CLK_TPS68470 + tristate "Clock Driver for TI TPS68470 PMIC" + depends on I2C + depends on INTEL_SKL_INT3472 || COMPILE_TEST + select REGMAP_I2C + help + This driver supports the clocks provided by the TPS68470 PMIC. + config COMMON_CLK_CDCE925 tristate "Clock driver for TI CDCE913/925/937/949 devices" depends on I2C diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index e42312121e51..6b6a88ae1425 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -63,6 +63,7 @@ obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o obj-$(CONFIG_COMMON_CLK_STM32F) += clk-stm32f4.o obj-$(CONFIG_COMMON_CLK_STM32H7) += clk-stm32h7.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o +obj-$(CONFIG_COMMON_CLK_TPS68470) += clk-tps68470.o obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_VC5) += clk-versaclock5.o diff --git a/drivers/clk/clk-tps68470.c b/drivers/clk/clk-tps68470.c new file mode 100644 index 000000000000..e5fbefd6ac2d --- /dev/null +++ b/drivers/clk/clk-tps68470.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Clock driver for TPS68470 PMIC + * + * Copyright (c) 2021 Red Hat Inc. + * Copyright (C) 2018 Intel Corporation + * + * Authors: + * Hans de Goede + * Zaikuo Wang + * Tianshu Qiu + * Jian Xu Zheng + * Yuning Pu + * Antti Laakso + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define TPS68470_CLK_NAME "tps68470-clk" + +#define to_tps68470_clkdata(clkd) \ + container_of(clkd, struct tps68470_clkdata, clkout_hw) + +static struct tps68470_clkout_freqs { + unsigned long freq; + unsigned int xtaldiv; + unsigned int plldiv; + unsigned int postdiv; + unsigned int buckdiv; + unsigned int boostdiv; +} clk_freqs[] = { +/* + * The PLL is used to multiply the crystal oscillator + * frequency range of 3 MHz to 27 MHz by a programmable + * factor of F = (M/N)*(1/P) such that the output + * available at the HCLK_A or HCLK_B pins are in the range + * of 4 MHz to 64 MHz in increments of 0.1 MHz. + * + * hclk_# = osc_in * (((plldiv*2)+320) / (xtaldiv+30)) * (1 / 2^postdiv) + * + * PLL_REF_CLK should be as close as possible to 100kHz + * PLL_REF_CLK = input clk / XTALDIV[7:0] + 30) + * + * PLL_VCO_CLK = (PLL_REF_CLK * (plldiv*2 + 320)) + * + * BOOST should be as close as possible to 2Mhz + * BOOST = PLL_VCO_CLK / (BOOSTDIV[4:0] + 16) * + * + * BUCK should be as close as possible to 5.2Mhz + * BUCK = PLL_VCO_CLK / (BUCKDIV[3:0] + 5) + * + * osc_in xtaldiv plldiv postdiv hclk_# + * 20Mhz 170 32 1 19.2Mhz + * 20Mhz 170 40 1 20Mhz + * 20Mhz 170 80 1 24Mhz + */ + { 19200000, 170, 32, 1, 2, 3 }, + { 20000000, 170, 40, 1, 3, 4 }, + { 24000000, 170, 80, 1, 4, 8 }, +}; + +struct tps68470_clkdata { + struct clk_hw clkout_hw; + struct regmap *regmap; + unsigned long rate; +}; + +static int tps68470_clk_is_prepared(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + int val; + + if (regmap_read(clkdata->regmap, TPS68470_REG_PLLCTL, &val)) + return 0; + + return val & TPS68470_PLL_EN_MASK; +} + +static int tps68470_clk_prepare(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG1, + (TPS68470_PLL_OUTPUT_ENABLE << TPS68470_OUTPUT_A_SHIFT) | + (TPS68470_PLL_OUTPUT_ENABLE << TPS68470_OUTPUT_B_SHIFT)); + + regmap_update_bits(clkdata->regmap, TPS68470_REG_PLLCTL, + TPS68470_PLL_EN_MASK, TPS68470_PLL_EN_MASK); + + /* + * The PLLCTL reg lock bit is set by the PMIC after approx. 4ms and + * does not indicate a true lock, so just wait 4 ms. + */ + usleep_range(4000, 5000); + + return 0; +} + +static void tps68470_clk_unprepare(struct clk_hw *hw) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + /* Disable clock first ... */ + regmap_update_bits(clkdata->regmap, TPS68470_REG_PLLCTL, TPS68470_PLL_EN_MASK, 0); + + /* ... and then tri-state the clock outputs. */ + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG1, 0); +} + +static unsigned long tps68470_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + + return clkdata->rate; +} + +/* + * This returns the index of the clk_freqs[] cfg with the closest rate for + * use in tps68470_clk_round_rate(). tps68470_clk_set_rate() checks that + * the rate of the returned cfg is an exact match. + */ +static unsigned int tps68470_clk_cfg_lookup(unsigned long rate) +{ + long diff, best_diff = LONG_MAX; + unsigned int i, best_idx = 0; + + for (i = 0; i < ARRAY_SIZE(clk_freqs); i++) { + diff = clk_freqs[i].freq - rate; + if (diff == 0) + return i; + + diff = abs(diff); + if (diff < best_diff) { + best_diff = diff; + best_idx = i; + } + } + + return best_idx; +} + +static long tps68470_clk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + unsigned int idx = tps68470_clk_cfg_lookup(rate); + + return clk_freqs[idx].freq; +} + +static int tps68470_clk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct tps68470_clkdata *clkdata = to_tps68470_clkdata(hw); + unsigned int idx = tps68470_clk_cfg_lookup(rate); + + if (rate != clk_freqs[idx].freq) + return -EINVAL; + + regmap_write(clkdata->regmap, TPS68470_REG_BOOSTDIV, clk_freqs[idx].boostdiv); + regmap_write(clkdata->regmap, TPS68470_REG_BUCKDIV, clk_freqs[idx].buckdiv); + regmap_write(clkdata->regmap, TPS68470_REG_PLLSWR, TPS68470_PLLSWR_DEFAULT); + regmap_write(clkdata->regmap, TPS68470_REG_XTALDIV, clk_freqs[idx].xtaldiv); + regmap_write(clkdata->regmap, TPS68470_REG_PLLDIV, clk_freqs[idx].plldiv); + regmap_write(clkdata->regmap, TPS68470_REG_POSTDIV, clk_freqs[idx].postdiv); + regmap_write(clkdata->regmap, TPS68470_REG_POSTDIV2, clk_freqs[idx].postdiv); + regmap_write(clkdata->regmap, TPS68470_REG_CLKCFG2, TPS68470_CLKCFG2_DRV_STR_2MA); + + regmap_write(clkdata->regmap, TPS68470_REG_PLLCTL, + TPS68470_OSC_EXT_CAP_DEFAULT << TPS68470_OSC_EXT_CAP_SHIFT | + TPS68470_CLK_SRC_XTAL << TPS68470_CLK_SRC_SHIFT); + + clkdata->rate = rate; + + return 0; +} + +static const struct clk_ops tps68470_clk_ops = { + .is_prepared = tps68470_clk_is_prepared, + .prepare = tps68470_clk_prepare, + .unprepare = tps68470_clk_unprepare, + .recalc_rate = tps68470_clk_recalc_rate, + .round_rate = tps68470_clk_round_rate, + .set_rate = tps68470_clk_set_rate, +}; + +static int tps68470_clk_probe(struct platform_device *pdev) +{ + struct tps68470_clk_platform_data *pdata = pdev->dev.platform_data; + struct clk_init_data tps68470_clk_initdata = { + .name = TPS68470_CLK_NAME, + .ops = &tps68470_clk_ops, + /* Changing the dividers when the PLL is on is not allowed */ + .flags = CLK_SET_RATE_GATE, + }; + struct tps68470_clkdata *tps68470_clkdata; + int ret; + + tps68470_clkdata = devm_kzalloc(&pdev->dev, sizeof(*tps68470_clkdata), + GFP_KERNEL); + if (!tps68470_clkdata) + return -ENOMEM; + + tps68470_clkdata->regmap = dev_get_drvdata(pdev->dev.parent); + tps68470_clkdata->clkout_hw.init = &tps68470_clk_initdata; + + /* Set initial rate */ + tps68470_clk_set_rate(&tps68470_clkdata->clkout_hw, clk_freqs[0].freq, 0); + + ret = devm_clk_hw_register(&pdev->dev, &tps68470_clkdata->clkout_hw); + if (ret) + return ret; + + ret = devm_clk_hw_register_clkdev(&pdev->dev, &tps68470_clkdata->clkout_hw, + TPS68470_CLK_NAME, NULL); + if (ret) + return ret; + + if (pdata) { + ret = devm_clk_hw_register_clkdev(&pdev->dev, + &tps68470_clkdata->clkout_hw, + pdata->consumer_con_id, + pdata->consumer_dev_name); + } + + return ret; +} + +static struct platform_driver tps68470_clk_driver = { + .driver = { + .name = TPS68470_CLK_NAME, + }, + .probe = tps68470_clk_probe, +}; + +/* + * The ACPI tps68470 probe-ordering depends on the clk/gpio/regulator drivers + * registering before the drivers for the camera-sensors which use them bind. + * subsys_initcall() ensures this when the drivers are builtin. + */ +static int __init tps68470_clk_init(void) +{ + return platform_driver_register(&tps68470_clk_driver); +} +subsys_initcall(tps68470_clk_init); + +static void __exit tps68470_clk_exit(void) +{ + platform_driver_unregister(&tps68470_clk_driver); +} +module_exit(tps68470_clk_exit); + +MODULE_ALIAS("platform:tps68470-clk"); +MODULE_DESCRIPTION("clock driver for TPS68470 pmic"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/mfd/tps68470.h b/include/linux/mfd/tps68470.h index ffe81127d91c..7807fa329db0 100644 --- a/include/linux/mfd/tps68470.h +++ b/include/linux/mfd/tps68470.h @@ -75,6 +75,17 @@ #define TPS68470_CLKCFG1_MODE_A_MASK GENMASK(1, 0) #define TPS68470_CLKCFG1_MODE_B_MASK GENMASK(3, 2) +#define TPS68470_CLKCFG2_DRV_STR_2MA 0x05 +#define TPS68470_PLL_OUTPUT_ENABLE 0x02 +#define TPS68470_CLK_SRC_XTAL BIT(0) +#define TPS68470_PLLSWR_DEFAULT GENMASK(1, 0) +#define TPS68470_OSC_EXT_CAP_DEFAULT 0x05 + +#define TPS68470_OUTPUT_A_SHIFT 0x00 +#define TPS68470_OUTPUT_B_SHIFT 0x02 +#define TPS68470_CLK_SRC_SHIFT GENMASK(2, 0) +#define TPS68470_OSC_EXT_CAP_SHIFT BIT(2) + #define TPS68470_GPIO_CTL_REG_A(x) (TPS68470_REG_GPCTL0A + (x) * 2) #define TPS68470_GPIO_CTL_REG_B(x) (TPS68470_REG_GPCTL0B + (x) * 2) #define TPS68470_GPIO_MODE_MASK GENMASK(1, 0) -- cgit v1.2.3-59-g8ed1b From d1e86325af377129adb7fc6f34eb044ca6068b47 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 15 Dec 2021 15:34:15 +0000 Subject: net: phylink: add mac_select_pcs() method to phylink_mac_ops mac_select_pcs() allows us to have an explicit point to query which PCS the MAC wishes to use for a particular PHY interface mode, thereby allowing us to add support to validate the link settings with the PCS. Phylink will also use this to select the PCS to be used during a major configuration event without the MAC driver needing to call phylink_set_pcs(). Note that if mac_select_pcs() is present, the supported_interfaces bitmap must be filled in; this avoids mac_select_pcs() being called with PHY_INTERFACE_MODE_NA when we want to get support for all interface types. Phylink will return an error in phylink_create() unless this condition is satisfied. Signed-off-by: Russell King (Oracle) Signed-off-by: David S. Miller --- drivers/net/phy/phylink.c | 68 ++++++++++++++++++++++++++++++++++++++++------- include/linux/phylink.h | 18 +++++++++++++ 2 files changed, 77 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 20df8af3e201..c7035d65e159 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -419,6 +419,23 @@ void phylink_generic_validate(struct phylink_config *config, } EXPORT_SYMBOL_GPL(phylink_generic_validate); +static int phylink_validate_mac_and_pcs(struct phylink *pl, + unsigned long *supported, + struct phylink_link_state *state) +{ + struct phylink_pcs *pcs; + + if (pl->mac_ops->mac_select_pcs) { + pcs = pl->mac_ops->mac_select_pcs(pl->config, state->interface); + if (IS_ERR(pcs)) + return PTR_ERR(pcs); + } + + pl->mac_ops->validate(pl->config, supported, state); + + return phylink_is_empty_linkmode(supported) ? -EINVAL : 0; +} + static int phylink_validate_any(struct phylink *pl, unsigned long *supported, struct phylink_link_state *state) { @@ -434,9 +451,10 @@ static int phylink_validate_any(struct phylink *pl, unsigned long *supported, t = *state; t.interface = intf; - pl->mac_ops->validate(pl->config, s, &t); - linkmode_or(all_s, all_s, s); - linkmode_or(all_adv, all_adv, t.advertising); + if (!phylink_validate_mac_and_pcs(pl, s, &t)) { + linkmode_or(all_s, all_s, s); + linkmode_or(all_adv, all_adv, t.advertising); + } } } @@ -458,9 +476,7 @@ static int phylink_validate(struct phylink *pl, unsigned long *supported, return -EINVAL; } - pl->mac_ops->validate(pl->config, supported, state); - - return phylink_is_empty_linkmode(supported) ? -EINVAL : 0; + return phylink_validate_mac_and_pcs(pl, supported, state); } static int phylink_parse_fixedlink(struct phylink *pl, @@ -750,10 +766,21 @@ static void phylink_mac_pcs_an_restart(struct phylink *pl) static void phylink_major_config(struct phylink *pl, bool restart, const struct phylink_link_state *state) { + struct phylink_pcs *pcs = NULL; int err; phylink_dbg(pl, "major config %s\n", phy_modes(state->interface)); + if (pl->mac_ops->mac_select_pcs) { + pcs = pl->mac_ops->mac_select_pcs(pl->config, state->interface); + if (IS_ERR(pcs)) { + phylink_err(pl, + "mac_select_pcs unexpectedly failed: %pe\n", + pcs); + return; + } + } + if (pl->mac_ops->mac_prepare) { err = pl->mac_ops->mac_prepare(pl->config, pl->cur_link_an_mode, state->interface); @@ -764,6 +791,12 @@ static void phylink_major_config(struct phylink *pl, bool restart, } } + /* If we have a new PCS, switch to the new PCS after preparing the MAC + * for the change. + */ + if (pcs) + phylink_set_pcs(pl, pcs); + phylink_mac_config(pl, state); if (pl->pcs_ops) { @@ -1155,6 +1188,14 @@ struct phylink *phylink_create(struct phylink_config *config, struct phylink *pl; int ret; + /* Validate the supplied configuration */ + if (mac_ops->mac_select_pcs && + phy_interface_empty(config->supported_interfaces)) { + dev_err(config->dev, + "phylink: error: empty supported_interfaces but mac_select_pcs() method present\n"); + return ERR_PTR(-EINVAL); + } + pl = kzalloc(sizeof(*pl), GFP_KERNEL); if (!pl) return ERR_PTR(-ENOMEM); @@ -1222,9 +1263,10 @@ EXPORT_SYMBOL_GPL(phylink_create); * @pl: a pointer to a &struct phylink returned from phylink_create() * @pcs: a pointer to the &struct phylink_pcs * - * Bind the MAC PCS to phylink. This may be called after phylink_create(), - * in mac_prepare() or mac_config() methods if it is desired to dynamically - * change the PCS. + * Bind the MAC PCS to phylink. This may be called after phylink_create(). + * If it is desired to dynamically change the PCS, then the preferred method + * is to use mac_select_pcs(), but it may also be called in mac_prepare() + * or mac_config(). * * Please note that there are behavioural changes with the mac_config() * callback if a PCS is present (denoting a newer setup) so removing a PCS @@ -1235,6 +1277,14 @@ void phylink_set_pcs(struct phylink *pl, struct phylink_pcs *pcs) { pl->pcs = pcs; pl->pcs_ops = pcs->ops; + + if (!pl->phylink_disable_state && + pl->cfg_link_an_mode == MLO_AN_INBAND) { + if (pl->config->pcs_poll || pcs->poll) + mod_timer(&pl->link_poll, jiffies + HZ); + else + del_timer(&pl->link_poll); + } } EXPORT_SYMBOL_GPL(phylink_set_pcs); diff --git a/include/linux/phylink.h b/include/linux/phylink.h index a2f266cc3442..b3086dcafeaf 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -112,6 +112,7 @@ struct phylink_config { /** * struct phylink_mac_ops - MAC operations structure. * @validate: Validate and update the link configuration. + * @mac_select_pcs: Select a PCS for the interface mode. * @mac_pcs_get_state: Read the current link state from the hardware. * @mac_prepare: prepare for a major reconfiguration of the interface. * @mac_config: configure the MAC for the selected mode and state. @@ -126,6 +127,8 @@ struct phylink_mac_ops { void (*validate)(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state); + struct phylink_pcs *(*mac_select_pcs)(struct phylink_config *config, + phy_interface_t interface); void (*mac_pcs_get_state)(struct phylink_config *config, struct phylink_link_state *state); int (*mac_prepare)(struct phylink_config *config, unsigned int mode, @@ -178,6 +181,21 @@ struct phylink_mac_ops { */ void validate(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state); +/** + * mac_select_pcs: Select a PCS for the interface mode. + * @config: a pointer to a &struct phylink_config. + * @interface: PHY interface mode for PCS + * + * Return the &struct phylink_pcs for the specified interface mode, or + * NULL if none is required, or an error pointer on error. + * + * This must not modify any state. It is used to query which PCS should + * be used. Phylink will use this during validation to ensure that the + * configuration is valid, and when setting a configuration to internally + * set the PCS that will be used. + */ +struct phylink_pcs *mac_select_pcs(struct phylink_config *config, + phy_interface_t interface); /** * mac_pcs_get_state() - Read the current inband link state from the hardware -- cgit v1.2.3-59-g8ed1b From 0d22d4b626a4eaa3196019092eb6c1919e9f8caa Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 15 Dec 2021 15:34:20 +0000 Subject: net: phylink: add pcs_validate() method Add a hook for PCS to validate the link parameters. This avoids MAC drivers having to have knowledge of their PCS in their validate() method, thereby allowing several MAC drivers to be simplfied. Signed-off-by: Russell King (Oracle) Signed-off-by: David S. Miller --- drivers/net/phy/phylink.c | 31 +++++++++++++++++++++++++++++++ include/linux/phylink.h | 20 ++++++++++++++++++++ 2 files changed, 51 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index c7035d65e159..420201858564 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -424,13 +424,44 @@ static int phylink_validate_mac_and_pcs(struct phylink *pl, struct phylink_link_state *state) { struct phylink_pcs *pcs; + int ret; + /* Get the PCS for this interface mode */ if (pl->mac_ops->mac_select_pcs) { pcs = pl->mac_ops->mac_select_pcs(pl->config, state->interface); if (IS_ERR(pcs)) return PTR_ERR(pcs); + } else { + pcs = pl->pcs; + } + + if (pcs) { + /* The PCS, if present, must be setup before phylink_create() + * has been called. If the ops is not initialised, print an + * error and backtrace rather than oopsing the kernel. + */ + if (!pcs->ops) { + phylink_err(pl, "interface %s: uninitialised PCS\n", + phy_modes(state->interface)); + dump_stack(); + return -EINVAL; + } + + /* Validate the link parameters with the PCS */ + if (pcs->ops->pcs_validate) { + ret = pcs->ops->pcs_validate(pcs, supported, state); + if (ret < 0 || phylink_is_empty_linkmode(supported)) + return -EINVAL; + + /* Ensure the advertising mask is a subset of the + * supported mask. + */ + linkmode_and(state->advertising, state->advertising, + supported); + } } + /* Then validate the link parameters with the MAC */ pl->mac_ops->validate(pl->config, supported, state); return phylink_is_empty_linkmode(supported) ? -EINVAL : 0; diff --git a/include/linux/phylink.h b/include/linux/phylink.h index b3086dcafeaf..713a0c928b7c 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -416,6 +416,7 @@ struct phylink_pcs { /** * struct phylink_pcs_ops - MAC PCS operations structure. + * @pcs_validate: validate the link configuration. * @pcs_get_state: read the current MAC PCS link state from the hardware. * @pcs_config: configure the MAC PCS for the selected mode and state. * @pcs_an_restart: restart 802.3z BaseX autonegotiation. @@ -423,6 +424,8 @@ struct phylink_pcs { * (where necessary). */ struct phylink_pcs_ops { + int (*pcs_validate)(struct phylink_pcs *pcs, unsigned long *supported, + const struct phylink_link_state *state); void (*pcs_get_state)(struct phylink_pcs *pcs, struct phylink_link_state *state); int (*pcs_config)(struct phylink_pcs *pcs, unsigned int mode, @@ -435,6 +438,23 @@ struct phylink_pcs_ops { }; #if 0 /* For kernel-doc purposes only. */ +/** + * pcs_validate() - validate the link configuration. + * @pcs: a pointer to a &struct phylink_pcs. + * @supported: ethtool bitmask for supported link modes. + * @state: a const pointer to a &struct phylink_link_state. + * + * Validate the interface mode, and advertising's autoneg bit, removing any + * media ethtool link modes that would not be supportable from the supported + * mask. Phylink will propagate the changes to the advertising mask. See the + * &struct phylink_mac_ops validate() method. + * + * Returns -EINVAL if the interface mode/autoneg mode is not supported. + * Returns non-zero positive if the link state can be supported. + */ +int pcs_validate(struct phylink_pcs *pcs, unsigned long *supported, + const struct phylink_link_state *state); + /** * pcs_get_state() - Read the current inband link state from the hardware * @pcs: a pointer to a &struct phylink_pcs. -- cgit v1.2.3-59-g8ed1b From c6aeaf56f468a565f6d2f27325fc07d35cdcd3cb Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Thu, 9 Sep 2021 15:51:24 +0200 Subject: drm/tegra: Implement correct DMA-BUF semantics DMA-BUF requires that each device that accesses a DMA-BUF attaches to it separately. To do so the host1x_bo_pin() and host1x_bo_unpin() functions need to be reimplemented so that they can return a mapping, which either represents an attachment or a map of the driver's own GEM object. Signed-off-by: Thierry Reding --- drivers/gpu/drm/tegra/gem.c | 163 ++++++++++++++++++++++++++--------------- drivers/gpu/drm/tegra/plane.c | 60 ++++----------- drivers/gpu/drm/tegra/plane.h | 2 +- drivers/gpu/drm/tegra/submit.c | 62 +++++++++++----- drivers/gpu/drm/tegra/uapi.c | 68 +++++++---------- drivers/gpu/drm/tegra/uapi.h | 5 +- drivers/gpu/host1x/job.c | 160 ++++++++++++++-------------------------- drivers/gpu/host1x/job.h | 6 +- include/linux/host1x.h | 30 +++++--- 9 files changed, 268 insertions(+), 288 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index d38fd7e12b57..f00938d8bc88 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -23,86 +23,96 @@ MODULE_IMPORT_NS(DMA_BUF); -static void tegra_bo_put(struct host1x_bo *bo) +static unsigned int sg_dma_count_chunks(struct scatterlist *sgl, unsigned int nents) { - struct tegra_bo *obj = host1x_to_tegra_bo(bo); + dma_addr_t next = ~(dma_addr_t)0; + unsigned int count = 0, i; + struct scatterlist *s; - drm_gem_object_put(&obj->gem); -} + for_each_sg(sgl, s, nents, i) { + /* sg_dma_address(s) is only valid for entries that have sg_dma_len(s) != 0. */ + if (!sg_dma_len(s)) + continue; -/* XXX move this into lib/scatterlist.c? */ -static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg, - unsigned int nents, gfp_t gfp_mask) -{ - struct scatterlist *dst; - unsigned int i; - int err; + if (sg_dma_address(s) != next) { + next = sg_dma_address(s) + sg_dma_len(s); + count++; + } + } - err = sg_alloc_table(sgt, nents, gfp_mask); - if (err < 0) - return err; + return count; +} - dst = sgt->sgl; +static inline unsigned int sgt_dma_count_chunks(struct sg_table *sgt) +{ + return sg_dma_count_chunks(sgt->sgl, sgt->nents); +} - for (i = 0; i < nents; i++) { - sg_set_page(dst, sg_page(sg), sg->length, 0); - dst = sg_next(dst); - sg = sg_next(sg); - } +static void tegra_bo_put(struct host1x_bo *bo) +{ + struct tegra_bo *obj = host1x_to_tegra_bo(bo); - return 0; + drm_gem_object_put(&obj->gem); } -static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, - dma_addr_t *phys) +static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, + enum dma_data_direction direction) { struct tegra_bo *obj = host1x_to_tegra_bo(bo); - struct sg_table *sgt; + struct drm_gem_object *gem = &obj->gem; + struct host1x_bo_mapping *map; int err; + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) + return ERR_PTR(-ENOMEM); + + map->bo = host1x_bo_get(bo); + map->direction = direction; + map->dev = dev; + /* - * If we've manually mapped the buffer object through the IOMMU, make - * sure to return the IOVA address of our mapping. - * - * Similarly, for buffers that have been allocated by the DMA API the - * physical address can be used for devices that are not attached to - * an IOMMU. For these devices, callers must pass a valid pointer via - * the @phys argument. - * - * Imported buffers were also already mapped at import time, so the - * existing mapping can be reused. + * Imported buffers need special treatment to satisfy the semantics of DMA-BUF. */ - if (phys) { - *phys = obj->iova; - return NULL; + if (gem->import_attach) { + struct dma_buf *buf = gem->import_attach->dmabuf; + + map->attach = dma_buf_attach(buf, dev); + if (IS_ERR(map->attach)) { + err = PTR_ERR(map->attach); + goto free; + } + + map->sgt = dma_buf_map_attachment(map->attach, direction); + if (IS_ERR(map->sgt)) { + dma_buf_detach(buf, map->attach); + err = PTR_ERR(map->sgt); + goto free; + } + + err = sgt_dma_count_chunks(map->sgt); + map->size = gem->size; + + goto out; } /* * If we don't have a mapping for this buffer yet, return an SG table * so that host1x can do the mapping for us via the DMA API. */ - sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); - if (!sgt) - return ERR_PTR(-ENOMEM); + map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); + if (!map->sgt) { + err = -ENOMEM; + goto free; + } if (obj->pages) { /* * If the buffer object was allocated from the explicit IOMMU * API code paths, construct an SG table from the pages. */ - err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages, - 0, obj->gem.size, GFP_KERNEL); - if (err < 0) - goto free; - } else if (obj->sgt) { - /* - * If the buffer object already has an SG table but no pages - * were allocated for it, it means the buffer was imported and - * the SG table needs to be copied to avoid overwriting any - * other potential users of the original SG table. - */ - err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl, - obj->sgt->orig_nents, GFP_KERNEL); + err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size, + GFP_KERNEL); if (err < 0) goto free; } else { @@ -111,25 +121,56 @@ static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, * not imported, it had to be allocated with the DMA API, so * the DMA API helper can be used. */ - err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova, - obj->gem.size); + err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size); if (err < 0) goto free; } - return sgt; + err = dma_map_sgtable(dev, map->sgt, direction, 0); + if (err) + goto free_sgt; + +out: + /* + * If we've manually mapped the buffer object through the IOMMU, make sure to return the + * existing IOVA address of our mapping. + */ + if (!obj->mm) { + map->phys = sg_dma_address(map->sgt->sgl); + map->chunks = err; + } else { + map->phys = obj->iova; + map->chunks = 1; + } + + map->size = gem->size; + return map; + +free_sgt: + sg_free_table(map->sgt); free: - kfree(sgt); + kfree(map->sgt); + kfree(map); return ERR_PTR(err); } -static void tegra_bo_unpin(struct device *dev, struct sg_table *sgt) +static void tegra_bo_unpin(struct host1x_bo_mapping *map) { - if (sgt) { - sg_free_table(sgt); - kfree(sgt); + if (!map) + return; + + if (map->attach) { + dma_buf_unmap_attachment(map->attach, map->sgt, map->direction); + dma_buf_detach(map->attach->dmabuf, map->attach); + } else { + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); + sg_free_table(map->sgt); + kfree(map->sgt); } + + host1x_bo_put(map->bo); + kfree(map); } static void *tegra_bo_mmap(struct host1x_bo *bo) diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c index 16a1cdc28657..e579d966e8dc 100644 --- a/drivers/gpu/drm/tegra/plane.c +++ b/drivers/gpu/drm/tegra/plane.c @@ -74,7 +74,7 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane) for (i = 0; i < 3; i++) { copy->iova[i] = DMA_MAPPING_ERROR; - copy->sgt[i] = NULL; + copy->map[i] = NULL; } return ©->base; @@ -138,55 +138,37 @@ const struct drm_plane_funcs tegra_plane_funcs = { static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) { - struct iommu_domain *domain = iommu_get_domain_for_dev(dc->dev); unsigned int i; int err; for (i = 0; i < state->base.fb->format->num_planes; i++) { struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); - dma_addr_t phys_addr, *phys; - struct sg_table *sgt; + struct host1x_bo_mapping *map; - /* - * If we're not attached to a domain, we already stored the - * physical address when the buffer was allocated. If we're - * part of a group that's shared between all display - * controllers, we've also already mapped the framebuffer - * through the SMMU. In both cases we can short-circuit the - * code below and retrieve the stored IOV address. - */ - if (!domain || dc->client.group) - phys = &phys_addr; - else - phys = NULL; - - sgt = host1x_bo_pin(dc->dev, &bo->base, phys); - if (IS_ERR(sgt)) { - err = PTR_ERR(sgt); + map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE); + if (IS_ERR(map)) { + err = PTR_ERR(map); goto unpin; } - if (sgt) { - err = dma_map_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); - if (err) - goto unpin; - + if (!dc->client.group) { /* * The display controller needs contiguous memory, so * fail if the buffer is discontiguous and we fail to * map its SG table to a single contiguous chunk of * I/O virtual memory. */ - if (sgt->nents > 1) { + if (map->chunks > 1) { err = -EINVAL; goto unpin; } - state->iova[i] = sg_dma_address(sgt->sgl); - state->sgt[i] = sgt; + state->iova[i] = map->phys; } else { - state->iova[i] = phys_addr; + state->iova[i] = bo->iova; } + + state->map[i] = map; } return 0; @@ -195,15 +177,9 @@ unpin: dev_err(dc->dev, "failed to map plane %u: %d\n", i, err); while (i--) { - struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); - struct sg_table *sgt = state->sgt[i]; - - if (sgt) - dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); - - host1x_bo_unpin(dc->dev, &bo->base, sgt); + host1x_bo_unpin(state->map[i]); state->iova[i] = DMA_MAPPING_ERROR; - state->sgt[i] = NULL; + state->map[i] = NULL; } return err; @@ -214,15 +190,9 @@ static void tegra_dc_unpin(struct tegra_dc *dc, struct tegra_plane_state *state) unsigned int i; for (i = 0; i < state->base.fb->format->num_planes; i++) { - struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); - struct sg_table *sgt = state->sgt[i]; - - if (sgt) - dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0); - - host1x_bo_unpin(dc->dev, &bo->base, sgt); + host1x_bo_unpin(state->map[i]); state->iova[i] = DMA_MAPPING_ERROR; - state->sgt[i] = NULL; + state->map[i] = NULL; } } diff --git a/drivers/gpu/drm/tegra/plane.h b/drivers/gpu/drm/tegra/plane.h index d9470780c803..dfb20712fbd7 100644 --- a/drivers/gpu/drm/tegra/plane.h +++ b/drivers/gpu/drm/tegra/plane.h @@ -43,7 +43,7 @@ struct tegra_plane_legacy_blending_state { struct tegra_plane_state { struct drm_plane_state base; - struct sg_table *sgt[3]; + struct host1x_bo_mapping *map[3]; dma_addr_t iova[3]; struct tegra_bo_tiling tiling; diff --git a/drivers/gpu/drm/tegra/submit.c b/drivers/gpu/drm/tegra/submit.c index 776f825df52f..6b950388527b 100644 --- a/drivers/gpu/drm/tegra/submit.c +++ b/drivers/gpu/drm/tegra/submit.c @@ -64,33 +64,61 @@ static void gather_bo_put(struct host1x_bo *host_bo) kref_put(&bo->ref, gather_bo_release); } -static struct sg_table * -gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys) +static struct host1x_bo_mapping * +gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) { - struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); - struct sg_table *sgt; + struct gather_bo *gather = container_of(bo, struct gather_bo, base); + struct host1x_bo_mapping *map; int err; - sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); - if (!sgt) + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) return ERR_PTR(-ENOMEM); - err = dma_get_sgtable(bo->dev, sgt, bo->gather_data, bo->gather_data_dma, - bo->gather_data_words * 4); - if (err) { - kfree(sgt); - return ERR_PTR(err); + map->bo = host1x_bo_get(bo); + map->direction = direction; + map->dev = dev; + + map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); + if (!map->sgt) { + err = -ENOMEM; + goto free; } - return sgt; + err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, + gather->gather_data_words * 4); + if (err) + goto free_sgt; + + err = dma_map_sgtable(dev, map->sgt, direction, 0); + if (err) + goto free_sgt; + + map->phys = sg_dma_address(map->sgt->sgl); + map->size = gather->gather_data_words * 4; + map->chunks = err; + + return map; + +free_sgt: + sg_free_table(map->sgt); + kfree(map->sgt); +free: + kfree(map); + return ERR_PTR(err); } -static void gather_bo_unpin(struct device *dev, struct sg_table *sgt) +static void gather_bo_unpin(struct host1x_bo_mapping *map) { - if (sgt) { - sg_free_table(sgt); - kfree(sgt); - } + if (!map) + return; + + dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); + sg_free_table(map->sgt); + kfree(map->sgt); + host1x_bo_put(map->bo); + + kfree(map); } static void *gather_bo_mmap(struct host1x_bo *host_bo) diff --git a/drivers/gpu/drm/tegra/uapi.c b/drivers/gpu/drm/tegra/uapi.c index 690a339c52ec..d31df8e129c5 100644 --- a/drivers/gpu/drm/tegra/uapi.c +++ b/drivers/gpu/drm/tegra/uapi.c @@ -17,11 +17,7 @@ static void tegra_drm_mapping_release(struct kref *ref) struct tegra_drm_mapping *mapping = container_of(ref, struct tegra_drm_mapping, ref); - if (mapping->sgt) - dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, - DMA_ATTR_SKIP_CPU_SYNC); - - host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); + host1x_bo_unpin(mapping->map); host1x_bo_put(mapping->bo); kfree(mapping); @@ -159,6 +155,7 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f struct drm_tegra_channel_map *args = data; struct tegra_drm_mapping *mapping; struct tegra_drm_context *context; + enum dma_data_direction direction; int err = 0; if (args->flags & ~DRM_TEGRA_CHANNEL_MAP_READ_WRITE) @@ -180,68 +177,53 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f kref_init(&mapping->ref); - mapping->dev = context->client->base.dev; mapping->bo = tegra_gem_lookup(file, args->handle); if (!mapping->bo) { err = -EINVAL; - goto unlock; + goto free; } - if (context->client->base.group) { - /* IOMMU domain managed directly using IOMMU API */ - host1x_bo_pin(mapping->dev, mapping->bo, &mapping->iova); - } else { - switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { - case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: - mapping->direction = DMA_BIDIRECTIONAL; - break; - - case DRM_TEGRA_CHANNEL_MAP_WRITE: - mapping->direction = DMA_FROM_DEVICE; - break; - - case DRM_TEGRA_CHANNEL_MAP_READ: - mapping->direction = DMA_TO_DEVICE; - break; + switch (args->flags & DRM_TEGRA_CHANNEL_MAP_READ_WRITE) { + case DRM_TEGRA_CHANNEL_MAP_READ_WRITE: + direction = DMA_BIDIRECTIONAL; + break; - default: - return -EINVAL; - } + case DRM_TEGRA_CHANNEL_MAP_WRITE: + direction = DMA_FROM_DEVICE; + break; - mapping->sgt = host1x_bo_pin(mapping->dev, mapping->bo, NULL); - if (IS_ERR(mapping->sgt)) { - err = PTR_ERR(mapping->sgt); - goto put_gem; - } + case DRM_TEGRA_CHANNEL_MAP_READ: + direction = DMA_TO_DEVICE; + break; - err = dma_map_sgtable(mapping->dev, mapping->sgt, mapping->direction, - DMA_ATTR_SKIP_CPU_SYNC); - if (err) - goto unpin; + default: + err = -EINVAL; + goto put_gem; + } - mapping->iova = sg_dma_address(mapping->sgt->sgl); + mapping->map = host1x_bo_pin(context->client->base.dev, mapping->bo, direction); + if (IS_ERR(mapping->map)) { + err = PTR_ERR(mapping->map); + goto put_gem; } + mapping->iova = mapping->map->phys; mapping->iova_end = mapping->iova + host1x_to_tegra_bo(mapping->bo)->gem.size; err = xa_alloc(&context->mappings, &args->mapping, mapping, XA_LIMIT(1, U32_MAX), GFP_KERNEL); if (err < 0) - goto unmap; + goto unpin; mutex_unlock(&fpriv->lock); return 0; -unmap: - if (mapping->sgt) { - dma_unmap_sgtable(mapping->dev, mapping->sgt, mapping->direction, - DMA_ATTR_SKIP_CPU_SYNC); - } unpin: - host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt); + host1x_bo_unpin(mapping->map); put_gem: host1x_bo_put(mapping->bo); +free: kfree(mapping); unlock: mutex_unlock(&fpriv->lock); diff --git a/drivers/gpu/drm/tegra/uapi.h b/drivers/gpu/drm/tegra/uapi.h index 12adad770ad3..92ff1e44ff15 100644 --- a/drivers/gpu/drm/tegra/uapi.h +++ b/drivers/gpu/drm/tegra/uapi.h @@ -27,10 +27,9 @@ struct tegra_drm_file { struct tegra_drm_mapping { struct kref ref; - struct device *dev; + struct host1x_bo_mapping *map; struct host1x_bo *bo; - struct sg_table *sgt; - enum dma_data_direction direction; + dma_addr_t iova; dma_addr_t iova_end; }; diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index 0eef6df7c89e..64bcc3d83efc 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -134,20 +134,20 @@ EXPORT_SYMBOL(host1x_job_add_wait); static unsigned int pin_job(struct host1x *host, struct host1x_job *job) { + unsigned long mask = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE; struct host1x_client *client = job->client; struct device *dev = client->dev; struct host1x_job_gather *g; - struct iommu_domain *domain; - struct sg_table *sgt; unsigned int i; int err; - domain = iommu_get_domain_for_dev(dev); job->num_unpins = 0; for (i = 0; i < job->num_relocs; i++) { struct host1x_reloc *reloc = &job->relocs[i]; - dma_addr_t phys_addr, *phys; + enum dma_data_direction direction; + struct host1x_bo_mapping *map; + struct host1x_bo *bo; reloc->target.bo = host1x_bo_get(reloc->target.bo); if (!reloc->target.bo) { @@ -155,64 +155,44 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - /* - * If the client device is not attached to an IOMMU, the - * physical address of the buffer object can be used. - * - * Similarly, when an IOMMU domain is shared between all - * host1x clients, the IOVA is already available, so no - * need to map the buffer object again. - * - * XXX Note that this isn't always safe to do because it - * relies on an assumption that no cache maintenance is - * needed on the buffer objects. - */ - if (!domain || client->group) - phys = &phys_addr; - else - phys = NULL; - - sgt = host1x_bo_pin(dev, reloc->target.bo, phys); - if (IS_ERR(sgt)) { - err = PTR_ERR(sgt); - goto unpin; - } + bo = reloc->target.bo; - if (sgt) { - unsigned long mask = HOST1X_RELOC_READ | - HOST1X_RELOC_WRITE; - enum dma_data_direction dir; - - switch (reloc->flags & mask) { - case HOST1X_RELOC_READ: - dir = DMA_TO_DEVICE; - break; + switch (reloc->flags & mask) { + case HOST1X_RELOC_READ: + direction = DMA_TO_DEVICE; + break; - case HOST1X_RELOC_WRITE: - dir = DMA_FROM_DEVICE; - break; + case HOST1X_RELOC_WRITE: + direction = DMA_FROM_DEVICE; + break; - case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: - dir = DMA_BIDIRECTIONAL; - break; + case HOST1X_RELOC_READ | HOST1X_RELOC_WRITE: + direction = DMA_BIDIRECTIONAL; + break; - default: - err = -EINVAL; - goto unpin; - } + default: + err = -EINVAL; + goto unpin; + } - err = dma_map_sgtable(dev, sgt, dir, 0); - if (err) - goto unpin; + map = host1x_bo_pin(dev, bo, direction); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto unpin; + } - job->unpins[job->num_unpins].dev = dev; - job->unpins[job->num_unpins].dir = dir; - phys_addr = sg_dma_address(sgt->sgl); + /* + * host1x clients are generally not able to do scatter-gather themselves, so fail + * if the buffer is discontiguous and we fail to map its SG table to a single + * contiguous chunk of I/O virtual memory. + */ + if (map->chunks > 1) { + err = -EINVAL; + goto unpin; } - job->addr_phys[job->num_unpins] = phys_addr; - job->unpins[job->num_unpins].bo = reloc->target.bo; - job->unpins[job->num_unpins].sgt = sgt; + job->addr_phys[job->num_unpins] = map->phys; + job->unpins[job->num_unpins].map = map; job->num_unpins++; } @@ -224,12 +204,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) return 0; for (i = 0; i < job->num_cmds; i++) { + struct host1x_bo_mapping *map; size_t gather_size = 0; struct scatterlist *sg; - dma_addr_t phys_addr; unsigned long shift; struct iova *alloc; - dma_addr_t *phys; unsigned int j; if (job->cmds[i].is_wait) @@ -243,25 +222,16 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - /** - * If the host1x is not attached to an IOMMU, there is no need - * to map the buffer object for the host1x, since the physical - * address can simply be used. - */ - if (!iommu_get_domain_for_dev(host->dev)) - phys = &phys_addr; - else - phys = NULL; - - sgt = host1x_bo_pin(host->dev, g->bo, phys); - if (IS_ERR(sgt)) { - err = PTR_ERR(sgt); - goto put; + map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto unpin; } if (host->domain) { - for_each_sgtable_sg(sgt, sg, j) + for_each_sgtable_sg(map->sgt, sg, j) gather_size += sg->length; + gather_size = iova_align(&host->iova, gather_size); shift = iova_shift(&host->iova); @@ -272,33 +242,23 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto put; } - err = iommu_map_sgtable(host->domain, - iova_dma_addr(&host->iova, alloc), - sgt, IOMMU_READ); + err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), + map->sgt, IOMMU_READ); if (err == 0) { __free_iova(&host->iova, alloc); err = -EINVAL; goto put; } - job->unpins[job->num_unpins].size = gather_size; - phys_addr = iova_dma_addr(&host->iova, alloc); - } else if (sgt) { - err = dma_map_sgtable(host->dev, sgt, DMA_TO_DEVICE, 0); - if (err) - goto put; - - job->unpins[job->num_unpins].dir = DMA_TO_DEVICE; - job->unpins[job->num_unpins].dev = host->dev; - phys_addr = sg_dma_address(sgt->sgl); + map->phys = iova_dma_addr(&host->iova, alloc); + map->size = gather_size; } - job->addr_phys[job->num_unpins] = phys_addr; - job->gather_addr_phys[i] = phys_addr; - - job->unpins[job->num_unpins].bo = g->bo; - job->unpins[job->num_unpins].sgt = sgt; + job->addr_phys[job->num_unpins] = map->phys; + job->unpins[job->num_unpins].map = map; job->num_unpins++; + + job->gather_addr_phys[i] = map->phys; } return 0; @@ -690,22 +650,16 @@ void host1x_job_unpin(struct host1x_job *job) unsigned int i; for (i = 0; i < job->num_unpins; i++) { - struct host1x_job_unpin_data *unpin = &job->unpins[i]; - struct device *dev = unpin->dev ?: host->dev; - struct sg_table *sgt = unpin->sgt; - - if (!job->enable_firewall && unpin->size && host->domain) { - iommu_unmap(host->domain, job->addr_phys[i], - unpin->size); - free_iova(&host->iova, - iova_pfn(&host->iova, job->addr_phys[i])); - } + struct host1x_bo_mapping *map = job->unpins[i].map; + struct host1x_bo *bo = map->bo; - if (unpin->dev && sgt) - dma_unmap_sgtable(unpin->dev, sgt, unpin->dir, 0); + if (!job->enable_firewall && map->size && host->domain) { + iommu_unmap(host->domain, job->addr_phys[i], map->size); + free_iova(&host->iova, iova_pfn(&host->iova, job->addr_phys[i])); + } - host1x_bo_unpin(dev, unpin->bo, sgt); - host1x_bo_put(unpin->bo); + host1x_bo_unpin(map); + host1x_bo_put(bo); } job->num_unpins = 0; diff --git a/drivers/gpu/host1x/job.h b/drivers/gpu/host1x/job.h index b4428c5495c9..dad5a1946693 100644 --- a/drivers/gpu/host1x/job.h +++ b/drivers/gpu/host1x/job.h @@ -35,11 +35,7 @@ struct host1x_job_cmd { }; struct host1x_job_unpin_data { - struct host1x_bo *bo; - struct sg_table *sgt; - struct device *dev; - size_t size; - enum dma_data_direction dir; + struct host1x_bo_mapping *map; }; /* diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 7bccf589aba7..157326074df8 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -7,6 +7,7 @@ #define __LINUX_HOST1X_H #include +#include #include enum host1x_class { @@ -82,12 +83,23 @@ struct host1x_client { struct host1x_bo; struct sg_table; +struct host1x_bo_mapping { + struct dma_buf_attachment *attach; + enum dma_data_direction direction; + struct host1x_bo *bo; + struct sg_table *sgt; + unsigned int chunks; + struct device *dev; + dma_addr_t phys; + size_t size; +}; + struct host1x_bo_ops { struct host1x_bo *(*get)(struct host1x_bo *bo); void (*put)(struct host1x_bo *bo); - struct sg_table *(*pin)(struct device *dev, struct host1x_bo *bo, - dma_addr_t *phys); - void (*unpin)(struct device *dev, struct sg_table *sgt); + struct host1x_bo_mapping *(*pin)(struct device *dev, struct host1x_bo *bo, + enum dma_data_direction dir); + void (*unpin)(struct host1x_bo_mapping *map); void *(*mmap)(struct host1x_bo *bo); void (*munmap)(struct host1x_bo *bo, void *addr); }; @@ -112,17 +124,15 @@ static inline void host1x_bo_put(struct host1x_bo *bo) bo->ops->put(bo); } -static inline struct sg_table *host1x_bo_pin(struct device *dev, - struct host1x_bo *bo, - dma_addr_t *phys) +static inline struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, + enum dma_data_direction dir) { - return bo->ops->pin(dev, bo, phys); + return bo->ops->pin(dev, bo, dir); } -static inline void host1x_bo_unpin(struct device *dev, struct host1x_bo *bo, - struct sg_table *sgt) +static inline void host1x_bo_unpin(struct host1x_bo_mapping *map) { - bo->ops->unpin(dev, sgt); + map->bo->ops->unpin(map); } static inline void *host1x_bo_mmap(struct host1x_bo *bo) -- cgit v1.2.3-59-g8ed1b From 1f39b1dfa53c84b56d7ad37fed44afda7004959d Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Fri, 7 Feb 2020 16:50:52 +0100 Subject: drm/tegra: Implement buffer object cache This cache is used to avoid mapping and unmapping buffer objects unnecessarily. Mappings are cached per client and stay hot until the buffer object is destroyed. Signed-off-by: Thierry Reding --- drivers/gpu/drm/tegra/gem.c | 14 ++++++-- drivers/gpu/drm/tegra/plane.c | 2 +- drivers/gpu/drm/tegra/submit.c | 1 + drivers/gpu/drm/tegra/uapi.c | 2 +- drivers/gpu/host1x/bus.c | 78 ++++++++++++++++++++++++++++++++++++++++++ drivers/gpu/host1x/dev.c | 2 ++ drivers/gpu/host1x/dev.h | 2 ++ drivers/gpu/host1x/job.c | 4 +-- include/linux/host1x.h | 53 ++++++++++++++++++++++------ 9 files changed, 141 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index f00938d8bc88..fce0e52973c2 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -67,6 +67,7 @@ static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_ if (!map) return ERR_PTR(-ENOMEM); + kref_init(&map->ref); map->bo = host1x_bo_get(bo); map->direction = direction; map->dev = dev; @@ -157,9 +158,6 @@ free: static void tegra_bo_unpin(struct host1x_bo_mapping *map) { - if (!map) - return; - if (map->attach) { dma_buf_unmap_attachment(map->attach, map->sgt, map->direction); dma_buf_detach(map->attach->dmabuf, map->attach); @@ -493,8 +491,18 @@ free: void tegra_bo_free_object(struct drm_gem_object *gem) { struct tegra_drm *tegra = gem->dev->dev_private; + struct host1x_bo_mapping *mapping, *tmp; struct tegra_bo *bo = to_tegra_bo(gem); + /* remove all mappings of this buffer object from any caches */ + list_for_each_entry_safe(mapping, tmp, &bo->base.mappings, list) { + if (mapping->cache) + host1x_bo_unpin(mapping); + else + dev_err(gem->dev->dev, "mapping %p stale for device %s\n", mapping, + dev_name(mapping->dev)); + } + if (tegra->domain) tegra_bo_iommu_unmap(tegra, bo); diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c index e579d966e8dc..f8e8afcbcbf2 100644 --- a/drivers/gpu/drm/tegra/plane.c +++ b/drivers/gpu/drm/tegra/plane.c @@ -145,7 +145,7 @@ static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state) struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i); struct host1x_bo_mapping *map; - map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE); + map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE, &dc->client.cache); if (IS_ERR(map)) { err = PTR_ERR(map); goto unpin; diff --git a/drivers/gpu/drm/tegra/submit.c b/drivers/gpu/drm/tegra/submit.c index 6b950388527b..c32698404e36 100644 --- a/drivers/gpu/drm/tegra/submit.c +++ b/drivers/gpu/drm/tegra/submit.c @@ -75,6 +75,7 @@ gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction if (!map) return ERR_PTR(-ENOMEM); + kref_init(&map->ref); map->bo = host1x_bo_get(bo); map->direction = direction; map->dev = dev; diff --git a/drivers/gpu/drm/tegra/uapi.c b/drivers/gpu/drm/tegra/uapi.c index d31df8e129c5..9ab9179d2026 100644 --- a/drivers/gpu/drm/tegra/uapi.c +++ b/drivers/gpu/drm/tegra/uapi.c @@ -201,7 +201,7 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data, struct drm_f goto put_gem; } - mapping->map = host1x_bo_pin(context->client->base.dev, mapping->bo, direction); + mapping->map = host1x_bo_pin(context->client->base.dev, mapping->bo, direction, NULL); if (IS_ERR(mapping->map)) { err = PTR_ERR(mapping->map); goto put_gem; diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c index 218e3718fd68..0b67d894470d 100644 --- a/drivers/gpu/host1x/bus.c +++ b/drivers/gpu/host1x/bus.c @@ -742,6 +742,7 @@ EXPORT_SYMBOL(host1x_driver_unregister); */ void __host1x_client_init(struct host1x_client *client, struct lock_class_key *key) { + host1x_bo_cache_init(&client->cache); INIT_LIST_HEAD(&client->list); __mutex_init(&client->lock, "host1x client lock", key); client->usecount = 0; @@ -830,6 +831,8 @@ int host1x_client_unregister(struct host1x_client *client) mutex_unlock(&clients_lock); + host1x_bo_cache_destroy(&client->cache); + return 0; } EXPORT_SYMBOL(host1x_client_unregister); @@ -904,3 +907,78 @@ unlock: return err; } EXPORT_SYMBOL(host1x_client_resume); + +struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, + enum dma_data_direction dir, + struct host1x_bo_cache *cache) +{ + struct host1x_bo_mapping *mapping; + + if (cache) { + mutex_lock(&cache->lock); + + list_for_each_entry(mapping, &cache->mappings, entry) { + if (mapping->bo == bo && mapping->direction == dir) { + kref_get(&mapping->ref); + goto unlock; + } + } + } + + mapping = bo->ops->pin(dev, bo, dir); + if (IS_ERR(mapping)) + goto unlock; + + spin_lock(&mapping->bo->lock); + list_add_tail(&mapping->list, &bo->mappings); + spin_unlock(&mapping->bo->lock); + + if (cache) { + INIT_LIST_HEAD(&mapping->entry); + mapping->cache = cache; + + list_add_tail(&mapping->entry, &cache->mappings); + + /* bump reference count to track the copy in the cache */ + kref_get(&mapping->ref); + } + +unlock: + if (cache) + mutex_unlock(&cache->lock); + + return mapping; +} +EXPORT_SYMBOL(host1x_bo_pin); + +static void __host1x_bo_unpin(struct kref *ref) +{ + struct host1x_bo_mapping *mapping = to_host1x_bo_mapping(ref); + + /* + * When the last reference of the mapping goes away, make sure to remove the mapping from + * the cache. + */ + if (mapping->cache) + list_del(&mapping->entry); + + spin_lock(&mapping->bo->lock); + list_del(&mapping->list); + spin_unlock(&mapping->bo->lock); + + mapping->bo->ops->unpin(mapping); +} + +void host1x_bo_unpin(struct host1x_bo_mapping *mapping) +{ + struct host1x_bo_cache *cache = mapping->cache; + + if (cache) + mutex_lock(&cache->lock); + + kref_put(&mapping->ref, __host1x_bo_unpin); + + if (cache) + mutex_unlock(&cache->lock); +} +EXPORT_SYMBOL(host1x_bo_unpin); diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c index fbb6447b8659..85eb3d19bbdb 100644 --- a/drivers/gpu/host1x/dev.c +++ b/drivers/gpu/host1x/dev.c @@ -386,6 +386,7 @@ static int host1x_probe(struct platform_device *pdev) if (syncpt_irq < 0) return syncpt_irq; + host1x_bo_cache_init(&host->cache); mutex_init(&host->devices_lock); INIT_LIST_HEAD(&host->devices); INIT_LIST_HEAD(&host->list); @@ -512,6 +513,7 @@ static int host1x_remove(struct platform_device *pdev) reset_control_assert(host->rst); clk_disable_unprepare(host->clk); host1x_iommu_exit(host); + host1x_bo_cache_destroy(&host->cache); return 0; } diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h index fa6d4bc46e98..5b7fdea5d169 100644 --- a/drivers/gpu/host1x/dev.h +++ b/drivers/gpu/host1x/dev.h @@ -149,6 +149,8 @@ struct host1x { struct list_head list; struct device_dma_parameters dma_parms; + + struct host1x_bo_cache cache; }; void host1x_hypervisor_writel(struct host1x *host1x, u32 r, u32 v); diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index 64bcc3d83efc..5e8c183167b7 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -175,7 +175,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - map = host1x_bo_pin(dev, bo, direction); + map = host1x_bo_pin(dev, bo, direction, &client->cache); if (IS_ERR(map)) { err = PTR_ERR(map); goto unpin; @@ -222,7 +222,7 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto unpin; } - map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE); + map = host1x_bo_pin(host->dev, g->bo, DMA_TO_DEVICE, &host->cache); if (IS_ERR(map)) { err = PTR_ERR(map); goto unpin; diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 157326074df8..d0bb16cdd005 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -8,6 +8,7 @@ #include #include +#include #include enum host1x_class { @@ -24,6 +25,28 @@ struct iommu_group; u64 host1x_get_dma_mask(struct host1x *host1x); +/** + * struct host1x_bo_cache - host1x buffer object cache + * @mappings: list of mappings + * @lock: synchronizes accesses to the list of mappings + */ +struct host1x_bo_cache { + struct list_head mappings; + struct mutex lock; +}; + +static inline void host1x_bo_cache_init(struct host1x_bo_cache *cache) +{ + INIT_LIST_HEAD(&cache->mappings); + mutex_init(&cache->lock); +} + +static inline void host1x_bo_cache_destroy(struct host1x_bo_cache *cache) +{ + /* XXX warn if not empty? */ + mutex_destroy(&cache->lock); +} + /** * struct host1x_client_ops - host1x client operations * @early_init: host1x client early initialization code @@ -74,6 +97,8 @@ struct host1x_client { struct host1x_client *parent; unsigned int usecount; struct mutex lock; + + struct host1x_bo_cache cache; }; /* @@ -84,16 +109,26 @@ struct host1x_bo; struct sg_table; struct host1x_bo_mapping { + struct kref ref; struct dma_buf_attachment *attach; enum dma_data_direction direction; + struct list_head list; struct host1x_bo *bo; struct sg_table *sgt; unsigned int chunks; struct device *dev; dma_addr_t phys; size_t size; + + struct host1x_bo_cache *cache; + struct list_head entry; }; +static inline struct host1x_bo_mapping *to_host1x_bo_mapping(struct kref *ref) +{ + return container_of(ref, struct host1x_bo_mapping, ref); +} + struct host1x_bo_ops { struct host1x_bo *(*get)(struct host1x_bo *bo); void (*put)(struct host1x_bo *bo); @@ -106,11 +141,15 @@ struct host1x_bo_ops { struct host1x_bo { const struct host1x_bo_ops *ops; + struct list_head mappings; + spinlock_t lock; }; static inline void host1x_bo_init(struct host1x_bo *bo, const struct host1x_bo_ops *ops) { + INIT_LIST_HEAD(&bo->mappings); + spin_lock_init(&bo->lock); bo->ops = ops; } @@ -124,16 +163,10 @@ static inline void host1x_bo_put(struct host1x_bo *bo) bo->ops->put(bo); } -static inline struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, - enum dma_data_direction dir) -{ - return bo->ops->pin(dev, bo, dir); -} - -static inline void host1x_bo_unpin(struct host1x_bo_mapping *map) -{ - map->bo->ops->unpin(map); -} +struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo, + enum dma_data_direction dir, + struct host1x_bo_cache *cache); +void host1x_bo_unpin(struct host1x_bo_mapping *map); static inline void *host1x_bo_mmap(struct host1x_bo *bo) { -- cgit v1.2.3-59-g8ed1b From 46f226c93d35b936aeec6eb31da932dc2e86f413 Mon Sep 17 00:00:00 2001 From: Mikko Perttunen Date: Thu, 16 Sep 2021 17:55:17 +0300 Subject: drm/tegra: Add NVDEC driver Add support for booting and using NVDEC on Tegra210, Tegra186 and Tegra194 to the Host1x and TegraDRM drivers. Booting in secure mode is not currently supported. Signed-off-by: Mikko Perttunen Signed-off-by: Thierry Reding --- drivers/gpu/drm/tegra/Makefile | 3 +- drivers/gpu/drm/tegra/drm.c | 4 + drivers/gpu/drm/tegra/drm.h | 1 + drivers/gpu/drm/tegra/nvdec.c | 464 +++++++++++++++++++++++++++++++++++++++++ drivers/gpu/host1x/dev.c | 18 ++ include/linux/host1x.h | 2 + 6 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/tegra/nvdec.c (limited to 'include/linux') diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile index d801909182cf..df6cc986aeba 100644 --- a/drivers/gpu/drm/tegra/Makefile +++ b/drivers/gpu/drm/tegra/Makefile @@ -23,7 +23,8 @@ tegra-drm-y := \ gr2d.o \ gr3d.o \ falcon.o \ - vic.o + vic.o \ + nvdec.o tegra-drm-y += trace.o diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c index 8d37d6b00562..2e7da2a7505d 100644 --- a/drivers/gpu/drm/tegra/drm.c +++ b/drivers/gpu/drm/tegra/drm.c @@ -1344,15 +1344,18 @@ static const struct of_device_id host1x_drm_subdevs[] = { { .compatible = "nvidia,tegra210-sor", }, { .compatible = "nvidia,tegra210-sor1", }, { .compatible = "nvidia,tegra210-vic", }, + { .compatible = "nvidia,tegra210-nvdec", }, { .compatible = "nvidia,tegra186-display", }, { .compatible = "nvidia,tegra186-dc", }, { .compatible = "nvidia,tegra186-sor", }, { .compatible = "nvidia,tegra186-sor1", }, { .compatible = "nvidia,tegra186-vic", }, + { .compatible = "nvidia,tegra186-nvdec", }, { .compatible = "nvidia,tegra194-display", }, { .compatible = "nvidia,tegra194-dc", }, { .compatible = "nvidia,tegra194-sor", }, { .compatible = "nvidia,tegra194-vic", }, + { .compatible = "nvidia,tegra194-nvdec", }, { /* sentinel */ } }; @@ -1376,6 +1379,7 @@ static struct platform_driver * const drivers[] = { &tegra_gr2d_driver, &tegra_gr3d_driver, &tegra_vic_driver, + &tegra_nvdec_driver, }; static int __init host1x_drm_init(void) diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h index 8b28327c931c..fc0a19554eac 100644 --- a/drivers/gpu/drm/tegra/drm.h +++ b/drivers/gpu/drm/tegra/drm.h @@ -202,5 +202,6 @@ extern struct platform_driver tegra_sor_driver; extern struct platform_driver tegra_gr2d_driver; extern struct platform_driver tegra_gr3d_driver; extern struct platform_driver tegra_vic_driver; +extern struct platform_driver tegra_nvdec_driver; #endif /* HOST1X_DRM_H */ diff --git a/drivers/gpu/drm/tegra/nvdec.c b/drivers/gpu/drm/tegra/nvdec.c new file mode 100644 index 000000000000..c3b6fe7fb454 --- /dev/null +++ b/drivers/gpu/drm/tegra/nvdec.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, NVIDIA Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "drm.h" +#include "falcon.h" +#include "vic.h" + +struct nvdec_config { + const char *firmware; + unsigned int version; + bool supports_sid; +}; + +struct nvdec { + struct falcon falcon; + + void __iomem *regs; + struct tegra_drm_client client; + struct host1x_channel *channel; + struct device *dev; + struct clk *clk; + + /* Platform configuration */ + const struct nvdec_config *config; +}; + +static inline struct nvdec *to_nvdec(struct tegra_drm_client *client) +{ + return container_of(client, struct nvdec, client); +} + +static void nvdec_writel(struct nvdec *nvdec, u32 value, unsigned int offset) +{ + writel(value, nvdec->regs + offset); +} + +static int nvdec_boot(struct nvdec *nvdec) +{ +#ifdef CONFIG_IOMMU_API + struct iommu_fwspec *spec = dev_iommu_fwspec_get(nvdec->dev); +#endif + int err; + +#ifdef CONFIG_IOMMU_API + if (nvdec->config->supports_sid && spec) { + u32 value; + + value = TRANSCFG_ATT(1, TRANSCFG_SID_FALCON) | TRANSCFG_ATT(0, TRANSCFG_SID_HW); + nvdec_writel(nvdec, value, VIC_TFBIF_TRANSCFG); + + if (spec->num_ids > 0) { + value = spec->ids[0] & 0xffff; + + nvdec_writel(nvdec, value, VIC_THI_STREAMID0); + nvdec_writel(nvdec, value, VIC_THI_STREAMID1); + } + } +#endif + + err = falcon_boot(&nvdec->falcon); + if (err < 0) + return err; + + err = falcon_wait_idle(&nvdec->falcon); + if (err < 0) { + dev_err(nvdec->dev, "falcon boot timed out\n"); + return err; + } + + return 0; +} + +static int nvdec_init(struct host1x_client *client) +{ + struct tegra_drm_client *drm = host1x_to_drm_client(client); + struct drm_device *dev = dev_get_drvdata(client->host); + struct tegra_drm *tegra = dev->dev_private; + struct nvdec *nvdec = to_nvdec(drm); + int err; + + err = host1x_client_iommu_attach(client); + if (err < 0 && err != -ENODEV) { + dev_err(nvdec->dev, "failed to attach to domain: %d\n", err); + return err; + } + + nvdec->channel = host1x_channel_request(client); + if (!nvdec->channel) { + err = -ENOMEM; + goto detach; + } + + client->syncpts[0] = host1x_syncpt_request(client, 0); + if (!client->syncpts[0]) { + err = -ENOMEM; + goto free_channel; + } + + err = tegra_drm_register_client(tegra, drm); + if (err < 0) + goto free_syncpt; + + /* + * Inherit the DMA parameters (such as maximum segment size) from the + * parent host1x device. + */ + client->dev->dma_parms = client->host->dma_parms; + + return 0; + +free_syncpt: + host1x_syncpt_put(client->syncpts[0]); +free_channel: + host1x_channel_put(nvdec->channel); +detach: + host1x_client_iommu_detach(client); + + return err; +} + +static int nvdec_exit(struct host1x_client *client) +{ + struct tegra_drm_client *drm = host1x_to_drm_client(client); + struct drm_device *dev = dev_get_drvdata(client->host); + struct tegra_drm *tegra = dev->dev_private; + struct nvdec *nvdec = to_nvdec(drm); + int err; + + /* avoid a dangling pointer just in case this disappears */ + client->dev->dma_parms = NULL; + + err = tegra_drm_unregister_client(tegra, drm); + if (err < 0) + return err; + + host1x_syncpt_put(client->syncpts[0]); + host1x_channel_put(nvdec->channel); + host1x_client_iommu_detach(client); + + if (client->group) { + dma_unmap_single(nvdec->dev, nvdec->falcon.firmware.phys, + nvdec->falcon.firmware.size, DMA_TO_DEVICE); + tegra_drm_free(tegra, nvdec->falcon.firmware.size, + nvdec->falcon.firmware.virt, + nvdec->falcon.firmware.iova); + } else { + dma_free_coherent(nvdec->dev, nvdec->falcon.firmware.size, + nvdec->falcon.firmware.virt, + nvdec->falcon.firmware.iova); + } + + return 0; +} + +static const struct host1x_client_ops nvdec_client_ops = { + .init = nvdec_init, + .exit = nvdec_exit, +}; + +static int nvdec_load_firmware(struct nvdec *nvdec) +{ + struct host1x_client *client = &nvdec->client.base; + struct tegra_drm *tegra = nvdec->client.drm; + dma_addr_t iova; + size_t size; + void *virt; + int err; + + if (nvdec->falcon.firmware.virt) + return 0; + + err = falcon_read_firmware(&nvdec->falcon, nvdec->config->firmware); + if (err < 0) + return err; + + size = nvdec->falcon.firmware.size; + + if (!client->group) { + virt = dma_alloc_coherent(nvdec->dev, size, &iova, GFP_KERNEL); + + err = dma_mapping_error(nvdec->dev, iova); + if (err < 0) + return err; + } else { + virt = tegra_drm_alloc(tegra, size, &iova); + } + + nvdec->falcon.firmware.virt = virt; + nvdec->falcon.firmware.iova = iova; + + err = falcon_load_firmware(&nvdec->falcon); + if (err < 0) + goto cleanup; + + /* + * In this case we have received an IOVA from the shared domain, so we + * need to make sure to get the physical address so that the DMA API + * knows what memory pages to flush the cache for. + */ + if (client->group) { + dma_addr_t phys; + + phys = dma_map_single(nvdec->dev, virt, size, DMA_TO_DEVICE); + + err = dma_mapping_error(nvdec->dev, phys); + if (err < 0) + goto cleanup; + + nvdec->falcon.firmware.phys = phys; + } + + return 0; + +cleanup: + if (!client->group) + dma_free_coherent(nvdec->dev, size, virt, iova); + else + tegra_drm_free(tegra, size, virt, iova); + + return err; +} + + +static int nvdec_runtime_resume(struct device *dev) +{ + struct nvdec *nvdec = dev_get_drvdata(dev); + int err; + + err = clk_prepare_enable(nvdec->clk); + if (err < 0) + return err; + + usleep_range(10, 20); + + err = nvdec_load_firmware(nvdec); + if (err < 0) + goto disable; + + err = nvdec_boot(nvdec); + if (err < 0) + goto disable; + + return 0; + +disable: + clk_disable_unprepare(nvdec->clk); + return err; +} + +static int nvdec_runtime_suspend(struct device *dev) +{ + struct nvdec *nvdec = dev_get_drvdata(dev); + + clk_disable_unprepare(nvdec->clk); + + return 0; +} + +static int nvdec_open_channel(struct tegra_drm_client *client, + struct tegra_drm_context *context) +{ + struct nvdec *nvdec = to_nvdec(client); + int err; + + err = pm_runtime_get_sync(nvdec->dev); + if (err < 0) { + pm_runtime_put(nvdec->dev); + return err; + } + + context->channel = host1x_channel_get(nvdec->channel); + if (!context->channel) { + pm_runtime_put(nvdec->dev); + return -ENOMEM; + } + + return 0; +} + +static void nvdec_close_channel(struct tegra_drm_context *context) +{ + struct nvdec *nvdec = to_nvdec(context->client); + + host1x_channel_put(context->channel); + pm_runtime_put(nvdec->dev); +} + +static const struct tegra_drm_client_ops nvdec_ops = { + .open_channel = nvdec_open_channel, + .close_channel = nvdec_close_channel, + .submit = tegra_drm_submit, +}; + +#define NVIDIA_TEGRA_210_NVDEC_FIRMWARE "nvidia/tegra210/nvdec.bin" + +static const struct nvdec_config nvdec_t210_config = { + .firmware = NVIDIA_TEGRA_210_NVDEC_FIRMWARE, + .version = 0x21, + .supports_sid = false, +}; + +#define NVIDIA_TEGRA_186_NVDEC_FIRMWARE "nvidia/tegra186/nvdec.bin" + +static const struct nvdec_config nvdec_t186_config = { + .firmware = NVIDIA_TEGRA_186_NVDEC_FIRMWARE, + .version = 0x18, + .supports_sid = true, +}; + +#define NVIDIA_TEGRA_194_NVDEC_FIRMWARE "nvidia/tegra194/nvdec.bin" + +static const struct nvdec_config nvdec_t194_config = { + .firmware = NVIDIA_TEGRA_194_NVDEC_FIRMWARE, + .version = 0x19, + .supports_sid = true, +}; + +static const struct of_device_id tegra_nvdec_of_match[] = { + { .compatible = "nvidia,tegra210-nvdec", .data = &nvdec_t210_config }, + { .compatible = "nvidia,tegra186-nvdec", .data = &nvdec_t186_config }, + { .compatible = "nvidia,tegra194-nvdec", .data = &nvdec_t194_config }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra_nvdec_of_match); + +static int nvdec_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct host1x_syncpt **syncpts; + struct nvdec *nvdec; + u32 host_class; + int err; + + /* inherit DMA mask from host1x parent */ + err = dma_coerce_mask_and_coherent(dev, *dev->parent->dma_mask); + if (err < 0) { + dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err); + return err; + } + + nvdec = devm_kzalloc(dev, sizeof(*nvdec), GFP_KERNEL); + if (!nvdec) + return -ENOMEM; + + nvdec->config = of_device_get_match_data(dev); + + syncpts = devm_kzalloc(dev, sizeof(*syncpts), GFP_KERNEL); + if (!syncpts) + return -ENOMEM; + + nvdec->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(nvdec->regs)) + return PTR_ERR(nvdec->regs); + + nvdec->clk = devm_clk_get(dev, NULL); + if (IS_ERR(nvdec->clk)) { + dev_err(&pdev->dev, "failed to get clock\n"); + return PTR_ERR(nvdec->clk); + } + + err = of_property_read_u32(dev->of_node, "nvidia,host1x-class", &host_class); + if (err < 0) + host_class = HOST1X_CLASS_NVDEC; + + nvdec->falcon.dev = dev; + nvdec->falcon.regs = nvdec->regs; + + err = falcon_init(&nvdec->falcon); + if (err < 0) + return err; + + platform_set_drvdata(pdev, nvdec); + + INIT_LIST_HEAD(&nvdec->client.base.list); + nvdec->client.base.ops = &nvdec_client_ops; + nvdec->client.base.dev = dev; + nvdec->client.base.class = host_class; + nvdec->client.base.syncpts = syncpts; + nvdec->client.base.num_syncpts = 1; + nvdec->dev = dev; + + INIT_LIST_HEAD(&nvdec->client.list); + nvdec->client.version = nvdec->config->version; + nvdec->client.ops = &nvdec_ops; + + err = host1x_client_register(&nvdec->client.base); + if (err < 0) { + dev_err(dev, "failed to register host1x client: %d\n", err); + goto exit_falcon; + } + + pm_runtime_enable(&pdev->dev); + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); + pm_runtime_use_autosuspend(&pdev->dev); + + return 0; + +exit_falcon: + falcon_exit(&nvdec->falcon); + + return err; +} + +static int nvdec_remove(struct platform_device *pdev) +{ + struct nvdec *nvdec = platform_get_drvdata(pdev); + int err; + + err = host1x_client_unregister(&nvdec->client.base); + if (err < 0) { + dev_err(&pdev->dev, "failed to unregister host1x client: %d\n", + err); + return err; + } + + if (pm_runtime_enabled(&pdev->dev)) + pm_runtime_disable(&pdev->dev); + else + nvdec_runtime_suspend(&pdev->dev); + + falcon_exit(&nvdec->falcon); + + return 0; +} + +static const struct dev_pm_ops nvdec_pm_ops = { + SET_RUNTIME_PM_OPS(nvdec_runtime_suspend, nvdec_runtime_resume, NULL) +}; + +struct platform_driver tegra_nvdec_driver = { + .driver = { + .name = "tegra-nvdec", + .of_match_table = tegra_nvdec_of_match, + .pm = &nvdec_pm_ops + }, + .probe = nvdec_probe, + .remove = nvdec_remove, +}; + +#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC) +MODULE_FIRMWARE(NVIDIA_TEGRA_210_NVDEC_FIRMWARE); +#endif +#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) +MODULE_FIRMWARE(NVIDIA_TEGRA_186_NVDEC_FIRMWARE); +#endif +#if IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) +MODULE_FIRMWARE(NVIDIA_TEGRA_194_NVDEC_FIRMWARE); +#endif diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c index 85eb3d19bbdb..3d4cabdbc78d 100644 --- a/drivers/gpu/host1x/dev.c +++ b/drivers/gpu/host1x/dev.c @@ -132,6 +132,12 @@ static const struct host1x_sid_entry tegra186_sid_table[] = { .offset = 0x30, .limit = 0x34 }, + { + /* NVDEC */ + .base = 0x1b00, + .offset = 0x30, + .limit = 0x34 + }, }; static const struct host1x_info host1x06_info = { @@ -156,6 +162,18 @@ static const struct host1x_sid_entry tegra194_sid_table[] = { .offset = 0x30, .limit = 0x34 }, + { + /* NVDEC */ + .base = 0x1b00, + .offset = 0x30, + .limit = 0x34 + }, + { + /* NVDEC1 */ + .base = 0x1bc0, + .offset = 0x30, + .limit = 0x34 + }, }; static const struct host1x_info host1x07_info = { diff --git a/include/linux/host1x.h b/include/linux/host1x.h index d0bb16cdd005..2ca53d7ed7ca 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -17,6 +17,8 @@ enum host1x_class { HOST1X_CLASS_GR2D_SB = 0x52, HOST1X_CLASS_VIC = 0x5D, HOST1X_CLASS_GR3D = 0x60, + HOST1X_CLASS_NVDEC = 0xF0, + HOST1X_CLASS_NVDEC1 = 0xF5, }; struct host1x; -- cgit v1.2.3-59-g8ed1b From 9ca790f44606109071ab1a3a37ed99e91794c37c Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Wed, 1 Dec 2021 02:23:16 +0300 Subject: gpu: host1x: Add host1x_channel_stop() Add host1x_channel_stop() which waits till channel becomes idle and then stops the channel hardware. This is needed for supporting suspend/resume by host1x drivers since the hardware state is lost after power-gating, thus the channel needs to be stopped before client enters into suspend. Tested-by: Peter Geis # Ouya T30 Tested-by: Paul Fertser # PAZ00 T20 Tested-by: Nicolas Chauvet # PAZ00 T20 and TK1 T124 Tested-by: Matt Merhar # Ouya T30 Signed-off-by: Dmitry Osipenko Signed-off-by: Thierry Reding --- drivers/gpu/host1x/channel.c | 8 ++++++++ include/linux/host1x.h | 1 + 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/drivers/gpu/host1x/channel.c b/drivers/gpu/host1x/channel.c index 4cd212bb570d..2a9a3a8d5931 100644 --- a/drivers/gpu/host1x/channel.c +++ b/drivers/gpu/host1x/channel.c @@ -75,6 +75,14 @@ struct host1x_channel *host1x_channel_get_index(struct host1x *host, return ch; } +void host1x_channel_stop(struct host1x_channel *channel) +{ + struct host1x *host = dev_get_drvdata(channel->dev->parent); + + host1x_hw_cdma_stop(host, &channel->cdma); +} +EXPORT_SYMBOL(host1x_channel_stop); + static void release_channel(struct kref *kref) { struct host1x_channel *channel = diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 2ca53d7ed7ca..e8dc5bc41f79 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -226,6 +226,7 @@ struct host1x_job; struct host1x_channel *host1x_channel_request(struct host1x_client *client); struct host1x_channel *host1x_channel_get(struct host1x_channel *channel); +void host1x_channel_stop(struct host1x_channel *channel); void host1x_channel_put(struct host1x_channel *channel); int host1x_job_submit(struct host1x_job *job); -- cgit v1.2.3-59-g8ed1b From c0cdc89072a3e1ae3981437f385de14b7bba8fd8 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 27 Oct 2021 16:15:04 +0100 Subject: irqchip/gic-v3-its: Give the percpu rdist struct its own flags field Later patches will require tracking some per-rdist status. Reuse the bytes "lost" to padding within the __percpu rdist struct as a flags field, and re-encode ->lpi_enabled within said flags. No change in functionality intended. Signed-off-by: Valentin Schneider Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20211027151506.2085066-2-valentin.schneider@arm.com --- drivers/irqchip/irq-gic-v3-its.c | 8 +++++--- include/linux/irqchip/arm-gic-v3.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index eb0882d15366..74c22741f3ce 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -46,6 +46,8 @@ #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) +#define RD_LOCAL_LPI_ENABLED BIT(0) + static u32 lpi_id_bits; /* @@ -3044,7 +3046,7 @@ static void its_cpu_init_lpis(void) phys_addr_t paddr; u64 val, tmp; - if (gic_data_rdist()->lpi_enabled) + if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) return; val = readl_relaxed(rbase + GICR_CTLR); @@ -3158,7 +3160,7 @@ static void its_cpu_init_lpis(void) /* Make sure the GIC has seen the above */ dsb(sy); out: - gic_data_rdist()->lpi_enabled = true; + gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", smp_processor_id(), gic_data_rdist()->pend_page ? "allocated" : "reserved", @@ -5138,7 +5140,7 @@ static int redist_disable_lpis(void) * * If running with preallocated tables, there is nothing to do. */ - if (gic_data_rdist()->lpi_enabled || + if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) || (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) return 0; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 81cbf85f73de..0dc34d7d735a 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -615,7 +615,7 @@ struct rdists { void __iomem *rd_base; struct page *pend_page; phys_addr_t phys_base; - bool lpi_enabled; + u64 flags; cpumask_t *vpe_table_mask; void *vpe_l1_base; } __percpu *rdist; -- cgit v1.2.3-59-g8ed1b From d23bc2bc1d634658d7fa96395419c1c553a784f0 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 27 Oct 2021 16:15:05 +0100 Subject: irqchip/gic-v3-its: Postpone LPI pending table freeing and memreserve Memory used by the LPI tables have to be made persistent for kexec to have a chance to work, as explained in [1]. If they have been made persistent and we are booting into a kexec'd kernel, we also need to free the pages that were preemptively allocated by the new kernel for those tables. Both of those operations currently happen during its_cpu_init(), which happens in a _STARTING (IOW atomic) cpuhp callback for secondary CPUs. efi_mem_reserve_iomem() issues a GFP_ATOMIC allocation, which unfortunately doesn't work under PREEMPT_RT (this ends up grabbing a non-raw spinlock, which can sleep under PREEMPT_RT). Similarly, freeing the pages ends up grabbing a sleepable spinlock. Since the memreserve is only required by kexec, it doesn't have to be done so early in the secondary boot process. Issue the reservation in a new CPUHP_AP_ONLINE_DYN cpuhp callback, and piggy-back the page freeing on top of it. A CPU gets to run the body of this new callback exactly once. As kexec issues a machine_shutdown() prior to machine_kexec(), it will be serialized vs a CPU being plugged to life by the hotplug machinery - either the CPU will have been brought up and have had its redistributor's pending table memreserved, or it never went online and will have its table allocated by the new kernel. [1]: https://lore.kernel.org/lkml/20180921195954.21574-1-marc.zyngier@arm.com/ Signed-off-by: Valentin Schneider Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20211027151506.2085066-3-valentin.schneider@arm.com --- drivers/irqchip/irq-gic-v3-its.c | 58 +++++++++++++++++++++++++++++++++++--- drivers/irqchip/irq-gic-v3.c | 1 + include/linux/irqchip/arm-gic-v3.h | 1 + 3 files changed, 56 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 74c22741f3ce..f860733d3e4e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -47,6 +47,8 @@ #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) #define RD_LOCAL_LPI_ENABLED BIT(0) +#define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1) +#define RD_LOCAL_MEMRESERVE_DONE BIT(2) static u32 lpi_id_bits; @@ -3065,15 +3067,13 @@ static void its_cpu_init_lpis(void) paddr &= GENMASK_ULL(51, 16); WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ)); - its_free_pending_table(gic_data_rdist()->pend_page); - gic_data_rdist()->pend_page = NULL; + gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED; goto out; } pend_page = gic_data_rdist()->pend_page; paddr = page_to_phys(pend_page); - WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); /* set PROPBASE */ val = (gic_rdists->prop_table_pa | @@ -3163,7 +3163,8 @@ out: gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", smp_processor_id(), - gic_data_rdist()->pend_page ? "allocated" : "reserved", + gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ? + "reserved" : "allocated", &paddr); } @@ -5202,6 +5203,38 @@ int its_cpu_init(void) return 0; } +static int its_cpu_memreserve_lpi(unsigned int cpu) +{ + struct page *pend_page; + int ret = 0; + + /* This gets to run exactly once per CPU */ + if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE) + return 0; + + pend_page = gic_data_rdist()->pend_page; + if (WARN_ON(!pend_page)) { + ret = -ENOMEM; + goto out; + } + /* + * If the pending table was pre-programmed, free the memory we + * preemptively allocated. Otherwise, reserve that memory for + * later kexecs. + */ + if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) { + its_free_pending_table(pend_page); + gic_data_rdist()->pend_page = NULL; + } else { + phys_addr_t paddr = page_to_phys(pend_page); + WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); + } + +out: + gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; + return ret; +} + static const struct of_device_id its_device_id[] = { { .compatible = "arm,gic-v3-its", }, {}, @@ -5385,6 +5418,23 @@ static void __init its_acpi_probe(void) static void __init its_acpi_probe(void) { } #endif +int __init its_lpi_memreserve_init(void) +{ + int state; + + if (!efi_enabled(EFI_CONFIG_TABLES)) + return 0; + + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "irqchip/arm/gicv3/memreserve:online", + its_cpu_memreserve_lpi, + NULL); + if (state < 0) + return state; + + return 0; +} + int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *parent_domain) { diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index daec3309b014..9fa3e1d16721 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1802,6 +1802,7 @@ static int __init gic_init_bases(void __iomem *dist_base, if (gic_dist_supports_lpis()) { its_init(handle, &gic_data.rdists, gic_data.domain); its_cpu_init(); + its_lpi_memreserve_init(); } else { if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) gicv2m_init(handle, gic_data.domain); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 0dc34d7d735a..51b85506ae90 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -632,6 +632,7 @@ struct rdists { struct irq_domain; struct fwnode_handle; +int __init its_lpi_memreserve_init(void); int its_cpu_init(void); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain); -- cgit v1.2.3-59-g8ed1b From 835f442fdbce33a47a6bde356643fd7e3ef7ec1b Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 27 Oct 2021 16:15:06 +0100 Subject: irqchip/gic-v3-its: Limit memreserve cpuhp state lifetime The new memreserve cpuhp callback only needs to survive up until a point where every CPU in the system has booted once. Beyond that, it becomes a no-op and can be put in the bin. Signed-off-by: Valentin Schneider Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20211027151506.2085066-4-valentin.schneider@arm.com --- drivers/irqchip/irq-gic-v3-its.c | 16 ++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 1 + 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index f860733d3e4e..ee83eb377d7e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -5203,6 +5203,15 @@ int its_cpu_init(void) return 0; } +static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work) +{ + cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); + gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; +} + +static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work, + rdist_memreserve_cpuhp_cleanup_workfn); + static int its_cpu_memreserve_lpi(unsigned int cpu) { struct page *pend_page; @@ -5231,6 +5240,10 @@ static int its_cpu_memreserve_lpi(unsigned int cpu) } out: + /* Last CPU being brought up gets to issue the cleanup */ + if (cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask)) + schedule_work(&rdist_memreserve_cpuhp_cleanup_work); + gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; return ret; } @@ -5425,6 +5438,7 @@ int __init its_lpi_memreserve_init(void) if (!efi_enabled(EFI_CONFIG_TABLES)) return 0; + gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "irqchip/arm/gicv3/memreserve:online", its_cpu_memreserve_lpi, @@ -5432,6 +5446,8 @@ int __init its_lpi_memreserve_init(void) if (state < 0) return state; + gic_rdists->cpuhp_memreserve_state = state; + return 0; } diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 51b85506ae90..12d91f0dedf9 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -624,6 +624,7 @@ struct rdists { u64 flags; u32 gicd_typer; u32 gicd_typer2; + int cpuhp_memreserve_state; bool has_vlpis; bool has_rvpeid; bool has_direct_lpi; -- cgit v1.2.3-59-g8ed1b From 3c67d44de787dff288d7f2a51c372b22f7356db6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 3 Dec 2021 06:48:53 -0700 Subject: block: add mq_ops->queue_rqs hook If we have a list of requests in our plug list, send it to the driver in one go, if possible. The driver must set mq_ops->queue_rqs() to support this, if not the usual one-by-one path is used. Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 26 +++++++++++++++++++++++--- include/linux/blk-mq.h | 8 ++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index 75154cc788db..51991232824a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2553,6 +2553,7 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) { struct blk_mq_hw_ctx *this_hctx; struct blk_mq_ctx *this_ctx; + struct request *rq; unsigned int depth; LIST_HEAD(list); @@ -2561,7 +2562,28 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) plug->rq_count = 0; if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) { - struct request_queue *q = rq_list_peek(&plug->mq_list)->q; + struct request_queue *q; + + rq = rq_list_peek(&plug->mq_list); + q = rq->q; + + /* + * Peek first request and see if we have a ->queue_rqs() hook. + * If we do, we can dispatch the whole plug list in one go. We + * already know at this point that all requests belong to the + * same queue, caller must ensure that's the case. + * + * Since we pass off the full list to the driver at this point, + * we do not increment the active request count for the queue. + * Bypass shared tags for now because of that. + */ + if (q->mq_ops->queue_rqs && + !(rq->mq_hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) { + blk_mq_run_dispatch_ops(q, + q->mq_ops->queue_rqs(&plug->mq_list)); + if (rq_list_empty(plug->mq_list)) + return; + } blk_mq_run_dispatch_ops(q, blk_mq_plug_issue_direct(plug, false)); @@ -2573,8 +2595,6 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) this_ctx = NULL; depth = 0; do { - struct request *rq; - rq = rq_list_pop(&plug->mq_list); if (!this_hctx) { diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 772f8f921526..550996cf419c 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -492,6 +492,14 @@ struct blk_mq_ops { */ void (*commit_rqs)(struct blk_mq_hw_ctx *); + /** + * @queue_rqs: Queue a list of new requests. Driver is guaranteed + * that each request belongs to the same queue. If the driver doesn't + * empty the @rqlist completely, then the rest will be queued + * individually by the block layer upon return. + */ + void (*queue_rqs)(struct request **rqlist); + /** * @get_budget: Reserve budget before queue request, once .queue_rq is * run, it is driver's responsibility to release the -- cgit v1.2.3-59-g8ed1b From 8a2ba1785c5803d59a63b6320ff54fd4a37a41ce Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 9 Dec 2021 07:31:21 +0100 Subject: block: remove the nr_task field from struct io_context Nothing ever looks at ->nr_tasks, so remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211209063131.18537-2-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ioc.c | 3 --- include/linux/iocontext.h | 1 - 2 files changed, 4 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 536fb496ad76..96336c2134ef 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -207,7 +207,6 @@ void exit_io_context(struct task_struct *task) task->io_context = NULL; task_unlock(task); - atomic_dec(&ioc->nr_tasks); put_io_context_active(ioc); } @@ -259,7 +258,6 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) return NULL; atomic_long_set(&ioc->refcount, 1); - atomic_set(&ioc->nr_tasks, 1); atomic_set(&ioc->active_ref, 1); spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); @@ -339,7 +337,6 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) if (clone_flags & CLONE_IO) { atomic_long_inc(&ioc->refcount); atomic_inc(&ioc->active_ref); - atomic_inc(&ioc->nr_tasks); tsk->io_context = ioc; } else if (ioprio_valid(ioc->ioprio)) { tsk->io_context = alloc_io_context(GFP_KERNEL, NUMA_NO_NODE); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index c1229fbd6691..82c7f4f5f4f5 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -99,7 +99,6 @@ struct io_cq { struct io_context { atomic_long_t refcount; atomic_t active_ref; - atomic_t nr_tasks; /* all the fields below are protected by this lock */ spinlock_t lock; -- cgit v1.2.3-59-g8ed1b From a411cd3cfdc5bbd1329d5b33dbf39e2b5213969d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 9 Dec 2021 07:31:27 +0100 Subject: block: move set_task_ioprio to blk-ioc.c Keep set_task_ioprio with the other low-level code that accesses the io_context structure. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211209063131.18537-8-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-ioc.c | 34 ++++++++++++++++++++++++++++++++-- block/ioprio.c | 32 -------------------------------- include/linux/iocontext.h | 2 -- 3 files changed, 32 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/block/blk-ioc.c b/block/blk-ioc.c index f98a29ee8f36..c25ce2f3eb19 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "blk.h" @@ -280,8 +281,8 @@ static struct io_context *create_task_io_context(struct task_struct *task, * This function always goes through task_lock() and it's better to use * %current->io_context + get_io_context() for %current. */ -struct io_context *get_task_io_context(struct task_struct *task, - gfp_t gfp_flags, int node) +static struct io_context *get_task_io_context(struct task_struct *task, + gfp_t gfp_flags, int node) { struct io_context *ioc; @@ -298,6 +299,35 @@ struct io_context *get_task_io_context(struct task_struct *task, return ioc; } +int set_task_ioprio(struct task_struct *task, int ioprio) +{ + int err; + struct io_context *ioc; + const struct cred *cred = current_cred(), *tcred; + + rcu_read_lock(); + tcred = __task_cred(task); + if (!uid_eq(tcred->uid, cred->euid) && + !uid_eq(tcred->uid, cred->uid) && !capable(CAP_SYS_NICE)) { + rcu_read_unlock(); + return -EPERM; + } + rcu_read_unlock(); + + err = security_task_setioprio(task, ioprio); + if (err) + return err; + + ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE); + if (ioc) { + ioc->ioprio = ioprio; + put_io_context(ioc); + } + + return err; +} +EXPORT_SYMBOL_GPL(set_task_ioprio); + int __copy_io(unsigned long clone_flags, struct task_struct *tsk) { struct io_context *ioc = current->io_context; diff --git a/block/ioprio.c b/block/ioprio.c index 313c14a70bbd..e118f4bf2dc6 100644 --- a/block/ioprio.c +++ b/block/ioprio.c @@ -22,46 +22,14 @@ */ #include #include -#include #include #include #include #include -#include -#include #include #include #include -int set_task_ioprio(struct task_struct *task, int ioprio) -{ - int err; - struct io_context *ioc; - const struct cred *cred = current_cred(), *tcred; - - rcu_read_lock(); - tcred = __task_cred(task); - if (!uid_eq(tcred->uid, cred->euid) && - !uid_eq(tcred->uid, cred->uid) && !capable(CAP_SYS_NICE)) { - rcu_read_unlock(); - return -EPERM; - } - rcu_read_unlock(); - - err = security_task_setioprio(task, ioprio); - if (err) - return err; - - ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE); - if (ioc) { - ioc->ioprio = ioprio; - put_io_context(ioc); - } - - return err; -} -EXPORT_SYMBOL_GPL(set_task_ioprio); - int ioprio_check_cap(int ioprio) { int class = IOPRIO_PRIO_CLASS(ioprio); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 82c7f4f5f4f5..648331f35fc6 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -116,8 +116,6 @@ struct task_struct; #ifdef CONFIG_BLOCK void put_io_context(struct io_context *ioc); void exit_io_context(struct task_struct *task); -struct io_context *get_task_io_context(struct task_struct *task, - gfp_t gfp_flags, int node); int __copy_io(unsigned long clone_flags, struct task_struct *tsk); static inline int copy_io(unsigned long clone_flags, struct task_struct *tsk) { -- cgit v1.2.3-59-g8ed1b From 5ef1630586317e92c9ebd7b4ce48f393b7ff790f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 9 Dec 2021 07:31:31 +0100 Subject: block: only build the icq tracking code when needed Only bfq needs to code to track icq, so make it conditional. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211209063131.18537-12-hch@lst.de Signed-off-by: Jens Axboe --- block/Kconfig | 3 +++ block/Kconfig.iosched | 1 + block/blk-ioc.c | 68 ++++++++++++++++++++++++++++------------------- block/blk.h | 6 +++++ include/linux/iocontext.h | 6 +++-- 5 files changed, 55 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/block/Kconfig b/block/Kconfig index c6ce41a5e5b2..d5d4197b7ed2 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -35,6 +35,9 @@ config BLK_CGROUP_RWSTAT config BLK_DEV_BSG_COMMON tristate +config BLK_ICQ + bool + config BLK_DEV_BSGLIB bool "Block layer SG support v4 helper lib" select BLK_DEV_BSG_COMMON diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched index 885fee86dfca..615516146086 100644 --- a/block/Kconfig.iosched +++ b/block/Kconfig.iosched @@ -18,6 +18,7 @@ config MQ_IOSCHED_KYBER config IOSCHED_BFQ tristate "BFQ I/O scheduler" + select BLK_ICQ help BFQ I/O scheduler for BLK-MQ. BFQ distributes the bandwidth of of the device among all processes according to their weights, diff --git a/block/blk-ioc.c b/block/blk-ioc.c index dc7fb064fd5f..87bdc9ca8295 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -19,6 +19,7 @@ */ static struct kmem_cache *iocontext_cachep; +#ifdef CONFIG_BLK_ICQ /** * get_io_context - increment reference count to io_context * @ioc: io_context to get @@ -162,6 +163,42 @@ static bool ioc_delay_free(struct io_context *ioc) return false; } +/** + * ioc_clear_queue - break any ioc association with the specified queue + * @q: request_queue being cleared + * + * Walk @q->icq_list and exit all io_cq's. + */ +void ioc_clear_queue(struct request_queue *q) +{ + LIST_HEAD(icq_list); + + spin_lock_irq(&q->queue_lock); + list_splice_init(&q->icq_list, &icq_list); + spin_unlock_irq(&q->queue_lock); + + rcu_read_lock(); + while (!list_empty(&icq_list)) { + struct io_cq *icq = + list_entry(icq_list.next, struct io_cq, q_node); + + spin_lock_irq(&icq->ioc->lock); + if (!(icq->flags & ICQ_DESTROYED)) + ioc_destroy_icq(icq); + spin_unlock_irq(&icq->ioc->lock); + } + rcu_read_unlock(); +} +#else /* CONFIG_BLK_ICQ */ +static inline void ioc_exit_icqs(struct io_context *ioc) +{ +} +static inline bool ioc_delay_free(struct io_context *ioc) +{ + return false; +} +#endif /* CONFIG_BLK_ICQ */ + /** * put_io_context - put a reference of io_context * @ioc: io_context to put @@ -193,33 +230,6 @@ void exit_io_context(struct task_struct *task) } } -/** - * ioc_clear_queue - break any ioc association with the specified queue - * @q: request_queue being cleared - * - * Walk @q->icq_list and exit all io_cq's. - */ -void ioc_clear_queue(struct request_queue *q) -{ - LIST_HEAD(icq_list); - - spin_lock_irq(&q->queue_lock); - list_splice_init(&q->icq_list, &icq_list); - spin_unlock_irq(&q->queue_lock); - - rcu_read_lock(); - while (!list_empty(&icq_list)) { - struct io_cq *icq = - list_entry(icq_list.next, struct io_cq, q_node); - - spin_lock_irq(&icq->ioc->lock); - if (!(icq->flags & ICQ_DESTROYED)) - ioc_destroy_icq(icq); - spin_unlock_irq(&icq->ioc->lock); - } - rcu_read_unlock(); -} - static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) { struct io_context *ioc; @@ -231,10 +241,12 @@ static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) atomic_long_set(&ioc->refcount, 1); atomic_set(&ioc->active_ref, 1); +#ifdef CONFIG_BLK_ICQ spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); +#endif return ioc; } @@ -300,6 +312,7 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) return 0; } +#ifdef CONFIG_BLK_ICQ /** * ioc_lookup_icq - lookup io_cq from ioc * @q: the associated request_queue @@ -428,6 +441,7 @@ struct io_cq *ioc_find_get_icq(struct request_queue *q) return icq; } EXPORT_SYMBOL_GPL(ioc_find_get_icq); +#endif /* CONFIG_BLK_ICQ */ static int __init blk_ioc_init(void) { diff --git a/block/blk.h b/block/blk.h index 7ccb7c7d86b3..8bd43b3ad33d 100644 --- a/block/blk.h +++ b/block/blk.h @@ -366,7 +366,13 @@ static inline unsigned int bio_aligned_discard_max_sectors( */ struct io_cq *ioc_find_get_icq(struct request_queue *q); struct io_cq *ioc_lookup_icq(struct request_queue *q); +#ifdef CONFIG_BLK_ICQ void ioc_clear_queue(struct request_queue *q); +#else +static inline void ioc_clear_queue(struct request_queue *q) +{ +} +#endif /* CONFIG_BLK_ICQ */ #ifdef CONFIG_BLK_DEV_THROTTLING_LOW extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 648331f35fc6..14f7eaf1b443 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -100,16 +100,18 @@ struct io_context { atomic_long_t refcount; atomic_t active_ref; + unsigned short ioprio; + +#ifdef CONFIG_BLK_ICQ /* all the fields below are protected by this lock */ spinlock_t lock; - unsigned short ioprio; - struct radix_tree_root icq_tree; struct io_cq __rcu *icq_hint; struct hlist_head icq_list; struct work_struct release_work; +#endif /* CONFIG_BLK_ICQ */ }; struct task_struct; -- cgit v1.2.3-59-g8ed1b From 85f5a74c2b9ba213d4102dc12ccbfdbe26472abb Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 8 Apr 2021 01:33:45 -0400 Subject: block: Add bio_add_folio() This is a thin wrapper around bio_add_page(). The main advantage here is the documentation that folios larger than 2GiB are not supported. It's not currently possible to allocate folios that large, but if it ever becomes possible, this function will fail gracefully instead of doing I/O to the wrong bytes. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jens Axboe Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- block/bio.c | 22 ++++++++++++++++++++++ include/linux/bio.h | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index 15ab0d6d1c06..4b3087e20d51 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1033,6 +1033,28 @@ int bio_add_page(struct bio *bio, struct page *page, } EXPORT_SYMBOL(bio_add_page); +/** + * bio_add_folio - Attempt to add part of a folio to a bio. + * @bio: BIO to add to. + * @folio: Folio to add. + * @len: How many bytes from the folio to add. + * @off: First byte in this folio to add. + * + * Filesystems that use folios can call this function instead of calling + * bio_add_page() for each page in the folio. If @off is bigger than + * PAGE_SIZE, this function can create a bio_vec that starts in a page + * after the bv_page. BIOs do not support folios that are 4GiB or larger. + * + * Return: Whether the addition was successful. + */ +bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, + size_t off) +{ + if (len > UINT_MAX || off > UINT_MAX) + return 0; + return bio_add_page(bio, &folio->page, len, off) > 0; +} + void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct bvec_iter_all iter_all; diff --git a/include/linux/bio.h b/include/linux/bio.h index fe6bdfbbef66..a783cac49978 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -409,7 +409,8 @@ extern void bio_uninit(struct bio *); extern void bio_reset(struct bio *); void bio_chain(struct bio *, struct bio *); -extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); +int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off); +bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); int bio_add_zone_append_page(struct bio *bio, struct page *page, -- cgit v1.2.3-59-g8ed1b From 640d1930bef4f87ec8d8d2b05f0f6edc1dfcf662 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Jan 2021 10:58:17 -0500 Subject: block: Add bio_for_each_folio_all() Allow callers to iterate over each folio instead of each page. The bio need not have been constructed using folios originally. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jens Axboe Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- Documentation/core-api/kernel-api.rst | 1 + include/linux/bio.h | 53 ++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst index 2e7186805148..7f0cb604b6ab 100644 --- a/Documentation/core-api/kernel-api.rst +++ b/Documentation/core-api/kernel-api.rst @@ -279,6 +279,7 @@ Accounting Framework Block Devices ============= +.. kernel-doc:: include/linux/bio.h .. kernel-doc:: block/blk-core.c :export: diff --git a/include/linux/bio.h b/include/linux/bio.h index a783cac49978..e3c9e8207f12 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -166,7 +166,7 @@ static inline void bio_advance(struct bio *bio, unsigned int nbytes) */ #define bio_for_each_bvec_all(bvl, bio, i) \ for (i = 0, bvl = bio_first_bvec_all(bio); \ - i < (bio)->bi_vcnt; i++, bvl++) \ + i < (bio)->bi_vcnt; i++, bvl++) #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) @@ -260,6 +260,57 @@ static inline struct bio_vec *bio_last_bvec_all(struct bio *bio) return &bio->bi_io_vec[bio->bi_vcnt - 1]; } +/** + * struct folio_iter - State for iterating all folios in a bio. + * @folio: The current folio we're iterating. NULL after the last folio. + * @offset: The byte offset within the current folio. + * @length: The number of bytes in this iteration (will not cross folio + * boundary). + */ +struct folio_iter { + struct folio *folio; + size_t offset; + size_t length; + /* private: for use by the iterator */ + size_t _seg_count; + int _i; +}; + +static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, + int i) +{ + struct bio_vec *bvec = bio_first_bvec_all(bio) + i; + + fi->folio = page_folio(bvec->bv_page); + fi->offset = bvec->bv_offset + + PAGE_SIZE * (bvec->bv_page - &fi->folio->page); + fi->_seg_count = bvec->bv_len; + fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count); + fi->_i = i; +} + +static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio) +{ + fi->_seg_count -= fi->length; + if (fi->_seg_count) { + fi->folio = folio_next(fi->folio); + fi->offset = 0; + fi->length = min(folio_size(fi->folio), fi->_seg_count); + } else if (fi->_i + 1 < bio->bi_vcnt) { + bio_first_folio(fi, bio, fi->_i + 1); + } else { + fi->folio = NULL; + } +} + +/** + * bio_for_each_folio_all - Iterate over each folio in a bio. + * @fi: struct folio_iter which is updated for each folio. + * @bio: struct bio to iterate over. + */ +#define bio_for_each_folio_all(fi, bio) \ + for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio)) + enum bip_flags { BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ -- cgit v1.2.3-59-g8ed1b From 8306a5f56305521d8b307b4ee1f69949fbb49279 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 28 Apr 2021 07:51:36 -0400 Subject: iomap: Add iomap_invalidate_folio Keep iomap_invalidatepage around as a wrapper for use in address_space operations. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- fs/iomap/buffered-io.c | 20 ++++++++++++-------- include/linux/iomap.h | 1 + 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index b0192b148c9f..de7ce1909527 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -479,23 +479,27 @@ iomap_releasepage(struct page *page, gfp_t gfp_mask) } EXPORT_SYMBOL_GPL(iomap_releasepage); -void -iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len) +void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len) { - struct folio *folio = page_folio(page); - - trace_iomap_invalidatepage(page->mapping->host, offset, len); + trace_iomap_invalidatepage(folio->mapping->host, offset, len); /* * If we're invalidating the entire page, clear the dirty state from it * and release it to avoid unnecessary buildup of the LRU. */ - if (offset == 0 && len == PAGE_SIZE) { - WARN_ON_ONCE(PageWriteback(page)); - cancel_dirty_page(page); + if (offset == 0 && len == folio_size(folio)) { + WARN_ON_ONCE(folio_test_writeback(folio)); + folio_cancel_dirty(folio); iomap_page_release(folio); } } +EXPORT_SYMBOL_GPL(iomap_invalidate_folio); + +void iomap_invalidatepage(struct page *page, unsigned int offset, + unsigned int len) +{ + iomap_invalidate_folio(page_folio(page), offset, len); +} EXPORT_SYMBOL_GPL(iomap_invalidatepage); #ifdef CONFIG_MIGRATION diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 6d1b08d0ae93..29491fb9c5ba 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -225,6 +225,7 @@ void iomap_readahead(struct readahead_control *, const struct iomap_ops *ops); int iomap_is_partially_uptodate(struct page *page, unsigned long from, unsigned long count); int iomap_releasepage(struct page *page, gfp_t gfp_mask); +void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); void iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len); #ifdef CONFIG_MIGRATION -- cgit v1.2.3-59-g8ed1b From ea6fa4961aab8f90a8aa03575a98b4bda368d4b6 Mon Sep 17 00:00:00 2001 From: Mateusz Jończyk Date: Fri, 10 Dec 2021 21:01:26 +0100 Subject: rtc: mc146818-lib: fix RTC presence check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To prevent an infinite loop in mc146818_get_time(), commit 211e5db19d15 ("rtc: mc146818: Detect and handle broken RTCs") added a check for RTC availability. Together with a later fix, it checked if bit 6 in register 0x0d is cleared. This, however, caused a false negative on a motherboard with an AMD SB710 southbridge; according to the specification [1], bit 6 of register 0x0d of this chipset is a scratchbit. This caused a regression in Linux 5.11 - the RTC was determined broken by the kernel and not used by rtc-cmos.c [3]. This problem was also reported in Fedora [4]. As a better alternative, check whether the UIP ("Update-in-progress") bit is set for longer then 10ms. If that is the case, then apparently the RTC is either absent (and all register reads return 0xff) or broken. Also limit the number of loop iterations in mc146818_get_time() to 10 to prevent an infinite loop there. The functions mc146818_get_time() and mc146818_does_rtc_work() will be refactored later in this patch series, in order to fix a separate problem with reading / setting the RTC alarm time. This is done so to avoid a confusion about what is being fixed when. In a previous approach to this problem, I implemented a check whether the RTC_HOURS register contains a value <= 24. This, however, sometimes did not work correctly on my Intel Kaby Lake laptop. According to Intel's documentation [2], "the time and date RAM locations (0-9) are disconnected from the external bus" during the update cycle so reading this register without checking the UIP bit is incorrect. [1] AMD SB700/710/750 Register Reference Guide, page 308, https://developer.amd.com/wordpress/media/2012/10/43009_sb7xx_rrg_pub_1.00.pdf [2] 7th Generation Intel ® Processor Family I/O for U/Y Platforms [...] Datasheet Volume 1 of 2, page 209 Intel's Document Number: 334658-006, https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/7th-and-8th-gen-core-family-mobile-u-y-processor-lines-i-o-datasheet-vol-1.pdf [3] Functions in arch/x86/kernel/rtc.c apparently were using it. [4] https://bugzilla.redhat.com/show_bug.cgi?id=1936688 Fixes: 211e5db19d15 ("rtc: mc146818: Detect and handle broken RTCs") Fixes: ebb22a059436 ("rtc: mc146818: Dont test for bit 0-5 in Register D") Signed-off-by: Mateusz Jończyk Cc: Thomas Gleixner Cc: Alessandro Zummo Cc: Alexandre Belloni Signed-off-by: Alexandre Belloni Link: https://lore.kernel.org/r/20211210200131.153887-5-mat.jonczyk@o2.pl --- drivers/rtc/rtc-cmos.c | 10 ++++------ drivers/rtc/rtc-mc146818-lib.c | 34 ++++++++++++++++++++++++++++++---- include/linux/mc146818rtc.h | 1 + 3 files changed, 35 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index d0f58cca5c20..b90a603d6b12 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -800,16 +800,14 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) rename_region(ports, dev_name(&cmos_rtc.rtc->dev)); - spin_lock_irq(&rtc_lock); - - /* Ensure that the RTC is accessible. Bit 6 must be 0! */ - if ((CMOS_READ(RTC_VALID) & 0x40) != 0) { - spin_unlock_irq(&rtc_lock); - dev_warn(dev, "not accessible\n"); + if (!mc146818_does_rtc_work()) { + dev_warn(dev, "broken or not accessible\n"); retval = -ENXIO; goto cleanup1; } + spin_lock_irq(&rtc_lock); + if (!(flags & CMOS_RTC_FLAGS_NOFREQ)) { /* force periodic irq to CMOS reset default of 1024Hz; * diff --git a/drivers/rtc/rtc-mc146818-lib.c b/drivers/rtc/rtc-mc146818-lib.c index ccd974b8a75a..d8e67a01220e 100644 --- a/drivers/rtc/rtc-mc146818-lib.c +++ b/drivers/rtc/rtc-mc146818-lib.c @@ -8,10 +8,36 @@ #include #endif +/* + * If the UIP (Update-in-progress) bit of the RTC is set for more then + * 10ms, the RTC is apparently broken or not present. + */ +bool mc146818_does_rtc_work(void) +{ + int i; + unsigned char val; + unsigned long flags; + + for (i = 0; i < 10; i++) { + spin_lock_irqsave(&rtc_lock, flags); + val = CMOS_READ(RTC_FREQ_SELECT); + spin_unlock_irqrestore(&rtc_lock, flags); + + if ((val & RTC_UIP) == 0) + return true; + + mdelay(1); + } + + return false; +} +EXPORT_SYMBOL_GPL(mc146818_does_rtc_work); + unsigned int mc146818_get_time(struct rtc_time *time) { unsigned char ctrl; unsigned long flags; + unsigned int iter_count = 0; unsigned char century = 0; bool retry; @@ -20,13 +46,13 @@ unsigned int mc146818_get_time(struct rtc_time *time) #endif again: - spin_lock_irqsave(&rtc_lock, flags); - /* Ensure that the RTC is accessible. Bit 6 must be 0! */ - if (WARN_ON_ONCE((CMOS_READ(RTC_VALID) & 0x40) != 0)) { - spin_unlock_irqrestore(&rtc_lock, flags); + if (iter_count > 10) { memset(time, 0, sizeof(*time)); return -EIO; } + iter_count++; + + spin_lock_irqsave(&rtc_lock, flags); /* * Check whether there is an update in progress during which the diff --git a/include/linux/mc146818rtc.h b/include/linux/mc146818rtc.h index 0661af17a758..69c80c4325bf 100644 --- a/include/linux/mc146818rtc.h +++ b/include/linux/mc146818rtc.h @@ -123,6 +123,7 @@ struct cmos_rtc_board_info { #define RTC_IO_EXTENT_USED RTC_IO_EXTENT #endif /* ARCH_RTC_LOCATION */ +bool mc146818_does_rtc_work(void); unsigned int mc146818_get_time(struct rtc_time *time); int mc146818_set_time(struct rtc_time *time); -- cgit v1.2.3-59-g8ed1b From ec5895c0f2d87b9bf4185db1915e40fa6fcfc0ac Mon Sep 17 00:00:00 2001 From: Mateusz Jończyk Date: Fri, 10 Dec 2021 21:01:27 +0100 Subject: rtc: mc146818-lib: extract mc146818_avoid_UIP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Function mc146818_get_time() contains an elaborate mechanism of reading the RTC time while no RTC update is in progress. It turns out that reading the RTC alarm clock also requires avoiding the RTC update. Therefore, the mechanism in mc146818_get_time() should be reused - so extract it into a separate function. The logic in mc146818_avoid_UIP() is same as in mc146818_get_time() except that after every if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { there is now "mdelay(1)". To avoid producing a very unreadable patch, mc146818_get_time() will be refactored to use mc146818_avoid_UIP() in the next patch. Signed-off-by: Mateusz Jończyk Cc: Alessandro Zummo Cc: Alexandre Belloni Signed-off-by: Alexandre Belloni Link: https://lore.kernel.org/r/20211210200131.153887-6-mat.jonczyk@o2.pl --- drivers/rtc/rtc-mc146818-lib.c | 70 ++++++++++++++++++++++++++++++++++++++++++ include/linux/mc146818rtc.h | 3 ++ 2 files changed, 73 insertions(+) (limited to 'include/linux') diff --git a/drivers/rtc/rtc-mc146818-lib.c b/drivers/rtc/rtc-mc146818-lib.c index d8e67a01220e..b20f4ebb2f3a 100644 --- a/drivers/rtc/rtc-mc146818-lib.c +++ b/drivers/rtc/rtc-mc146818-lib.c @@ -8,6 +8,76 @@ #include #endif +/* + * Execute a function while the UIP (Update-in-progress) bit of the RTC is + * unset. + * + * Warning: callback may be executed more then once. + */ +bool mc146818_avoid_UIP(void (*callback)(unsigned char seconds, void *param), + void *param) +{ + int i; + unsigned long flags; + unsigned char seconds; + + for (i = 0; i < 10; i++) { + spin_lock_irqsave(&rtc_lock, flags); + + /* + * Check whether there is an update in progress during which the + * readout is unspecified. The maximum update time is ~2ms. Poll + * every msec for completion. + * + * Store the second value before checking UIP so a long lasting + * NMI which happens to hit after the UIP check cannot make + * an update cycle invisible. + */ + seconds = CMOS_READ(RTC_SECONDS); + + if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { + spin_unlock_irqrestore(&rtc_lock, flags); + mdelay(1); + continue; + } + + /* Revalidate the above readout */ + if (seconds != CMOS_READ(RTC_SECONDS)) { + spin_unlock_irqrestore(&rtc_lock, flags); + continue; + } + + if (callback) + callback(seconds, param); + + /* + * Check for the UIP bit again. If it is set now then + * the above values may contain garbage. + */ + if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { + spin_unlock_irqrestore(&rtc_lock, flags); + mdelay(1); + continue; + } + + /* + * A NMI might have interrupted the above sequence so check + * whether the seconds value has changed which indicates that + * the NMI took longer than the UIP bit was set. Unlikely, but + * possible and there is also virt... + */ + if (seconds != CMOS_READ(RTC_SECONDS)) { + spin_unlock_irqrestore(&rtc_lock, flags); + continue; + } + spin_unlock_irqrestore(&rtc_lock, flags); + + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(mc146818_avoid_UIP); + /* * If the UIP (Update-in-progress) bit of the RTC is set for more then * 10ms, the RTC is apparently broken or not present. diff --git a/include/linux/mc146818rtc.h b/include/linux/mc146818rtc.h index 69c80c4325bf..67fb0a12becc 100644 --- a/include/linux/mc146818rtc.h +++ b/include/linux/mc146818rtc.h @@ -127,4 +127,7 @@ bool mc146818_does_rtc_work(void); unsigned int mc146818_get_time(struct rtc_time *time); int mc146818_set_time(struct rtc_time *time); +bool mc146818_avoid_UIP(void (*callback)(unsigned char seconds, void *param), + void *param); + #endif /* _MC146818RTC_H */ -- cgit v1.2.3-59-g8ed1b From 34fff62827b254f8a43633cc878deb04bf11297c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:18:54 +0100 Subject: device: Move MSI related data into a struct The only unconditional part of MSI data in struct device is the irqdomain pointer. Everything else can be allocated on demand. Create a data structure and move the irqdomain pointer into it. The other MSI specific parts are going to be removed from struct device in later steps. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Arnd Bergmann Link: https://lore.kernel.org/r/20211210221813.617178827@linutronix.de --- drivers/base/platform-msi.c | 12 ++++++------ drivers/dma/ti/k3-udma.c | 4 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +- drivers/irqchip/irq-mvebu-icu.c | 6 +++--- drivers/soc/ti/k3-ringacc.c | 4 ++-- drivers/soc/ti/ti_sci_inta_msi.c | 2 +- include/linux/device.h | 20 ++++++++++++++------ 7 files changed, 29 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 3d6c8f9caf43..f1f0876510a4 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -210,10 +210,10 @@ platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, * accordingly (which would impact the max number of MSI * capable devices). */ - if (!dev->msi_domain || !write_msi_msg || !nvec || nvec > MAX_DEV_MSIS) + if (!dev->msi.domain || !write_msi_msg || !nvec || nvec > MAX_DEV_MSIS) return ERR_PTR(-EINVAL); - if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { + if (dev->msi.domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { dev_err(dev, "Incompatible msi_domain, giving up\n"); return ERR_PTR(-EINVAL); } @@ -269,7 +269,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, if (err) goto out_free_priv_data; - err = msi_domain_alloc_irqs(dev->msi_domain, dev, nvec); + err = msi_domain_alloc_irqs(dev->msi.domain, dev, nvec); if (err) goto out_free_desc; @@ -282,7 +282,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, return 0; out_free_irqs: - msi_domain_free_irqs(dev->msi_domain, dev); + msi_domain_free_irqs(dev->msi.domain, dev); out_free_desc: platform_msi_free_descs(dev, 0, nvec); out_free_priv_data: @@ -306,7 +306,7 @@ void platform_msi_domain_free_irqs(struct device *dev) platform_msi_free_priv_data(desc->platform.msi_priv_data); } - msi_domain_free_irqs(dev->msi_domain, dev); + msi_domain_free_irqs(dev->msi.domain, dev); platform_msi_free_descs(dev, 0, MAX_DEV_MSIS); } EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); @@ -354,7 +354,7 @@ __platform_msi_create_device_domain(struct device *dev, return NULL; data->host_data = host_data; - domain = irq_domain_create_hierarchy(dev->msi_domain, 0, + domain = irq_domain_create_hierarchy(dev->msi.domain, 0, is_tree ? 0 : nvec, dev->fwnode, ops, data); if (!domain) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 041d8e32d630..20edd0bf3e8b 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -5279,9 +5279,9 @@ static int udma_probe(struct platform_device *pdev) if (IS_ERR(ud->ringacc)) return PTR_ERR(ud->ringacc); - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_TI_SCI_INTA_MSI); - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_err(dev, "Failed to get MSI domain\n"); return -EPROBE_DEFER; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f5848b351b19..5ba9006ec550 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3170,7 +3170,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) if (!(smmu->features & ARM_SMMU_FEAT_MSI)) return; - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); return; } diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c index 3e7297fc5948..4a4a65a6f632 100644 --- a/drivers/irqchip/irq-mvebu-icu.c +++ b/drivers/irqchip/irq-mvebu-icu.c @@ -314,12 +314,12 @@ static int mvebu_icu_subset_probe(struct platform_device *pdev) msi_data->subset_data = of_device_get_match_data(dev); } - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_PLATFORM_MSI); - if (!dev->msi_domain) + if (!dev->msi.domain) return -EPROBE_DEFER; - msi_parent_dn = irq_domain_get_of_node(dev->msi_domain); + msi_parent_dn = irq_domain_get_of_node(dev->msi.domain); if (!msi_parent_dn) return -ENODEV; diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 312ba0f98ad7..26159a5eef88 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -1356,9 +1356,9 @@ static int k3_ringacc_init(struct platform_device *pdev, struct resource *res; int ret, i; - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_TI_SCI_INTA_MSI); - if (!dev->msi_domain) { + if (!dev->msi.domain) { dev_err(dev, "Failed to get MSI domain\n"); return -EPROBE_DEFER; } diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c index a1d9c027022a..428a482df0f2 100644 --- a/drivers/soc/ti/ti_sci_inta_msi.c +++ b/drivers/soc/ti/ti_sci_inta_msi.c @@ -140,7 +140,7 @@ EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_alloc_irqs); void ti_sci_inta_msi_domain_free_irqs(struct device *dev) { - msi_domain_free_irqs(dev->msi_domain, dev); + msi_domain_free_irqs(dev->msi.domain, dev); ti_sci_inta_msi_free_descs(dev); } EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_free_irqs); diff --git a/include/linux/device.h b/include/linux/device.h index 2a22875238a6..f212b7a7b156 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -371,6 +371,16 @@ struct dev_links_info { enum dl_dev_state status; }; +/** + * struct dev_msi_info - Device data related to MSI + * @domain: The MSI interrupt domain associated to the device + */ +struct dev_msi_info { +#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN + struct irq_domain *domain; +#endif +}; + /** * struct device - The basic device structure * @parent: The device's "parent" device, the device to which it is attached. @@ -407,8 +417,8 @@ struct dev_links_info { * @em_pd: device's energy model performance domain * @pins: For device pin management. * See Documentation/driver-api/pin-control.rst for details. + * @msi: MSI related data * @msi_list: Hosts MSI descriptors - * @msi_domain: The generic MSI domain this device is using. * @numa_node: NUMA node this device is close to. * @dma_ops: DMA mapping operations for this device. * @dma_mask: Dma mask (if dma'ble device). @@ -500,12 +510,10 @@ struct device { struct em_perf_domain *em_pd; #endif -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - struct irq_domain *msi_domain; -#endif #ifdef CONFIG_PINCTRL struct dev_pin_info *pins; #endif + struct dev_msi_info msi; #ifdef CONFIG_GENERIC_MSI_IRQ struct list_head msi_list; #endif @@ -666,7 +674,7 @@ static inline void set_dev_node(struct device *dev, int node) static inline struct irq_domain *dev_get_msi_domain(const struct device *dev) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - return dev->msi_domain; + return dev->msi.domain; #else return NULL; #endif @@ -675,7 +683,7 @@ static inline struct irq_domain *dev_get_msi_domain(const struct device *dev) static inline void dev_set_msi_domain(struct device *dev, struct irq_domain *d) { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - dev->msi_domain = d; + dev->msi.domain = d; #endif } -- cgit v1.2.3-59-g8ed1b From 013bd8e543c2c777b586cf033c588ea82bd502db Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:18:55 +0100 Subject: device: Add device:: Msi_data pointer and struct msi_device_data Create struct msi_device_data and add a pointer of that type to struct dev_msi_info, which is part of struct device. Provide an allocator function which can be invoked from the MSI interrupt allocation code pathes. Add a properties field to the data structure as a first member so the allocation size is not zero bytes. The field will be uses later on. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221813.676660809@linutronix.de --- include/linux/device.h | 5 +++++ include/linux/msi.h | 18 ++++++++++++++++++ kernel/irq/msi.c | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) (limited to 'include/linux') diff --git a/include/linux/device.h b/include/linux/device.h index f212b7a7b156..f0033cd93631 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -45,6 +45,7 @@ struct iommu_ops; struct iommu_group; struct dev_pin_info; struct dev_iommu; +struct msi_device_data; /** * struct subsys_interface - interfaces to device functions @@ -374,11 +375,15 @@ struct dev_links_info { /** * struct dev_msi_info - Device data related to MSI * @domain: The MSI interrupt domain associated to the device + * @data: Pointer to MSI device data */ struct dev_msi_info { #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN struct irq_domain *domain; #endif +#ifdef CONFIG_GENERIC_MSI_IRQ + struct msi_device_data *data; +#endif }; /** diff --git a/include/linux/msi.h b/include/linux/msi.h index ba4a39c430b5..7e4c8fd7c65d 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -171,6 +171,16 @@ struct msi_desc { }; }; +/** + * msi_device_data - MSI per device data + * @properties: MSI properties which are interesting to drivers + */ +struct msi_device_data { + unsigned long properties; +}; + +int msi_setup_device_data(struct device *dev); + /* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) #define dev_to_msi_list(dev) (&(dev)->msi_list) @@ -233,10 +243,16 @@ void pci_msi_mask_irq(struct irq_data *data); void pci_msi_unmask_irq(struct irq_data *data); #ifdef CONFIG_SYSFS +int msi_device_populate_sysfs(struct device *dev); +void msi_device_destroy_sysfs(struct device *dev); + const struct attribute_group **msi_populate_sysfs(struct device *dev); void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups); #else +static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } +static inline void msi_device_destroy_sysfs(struct device *dev) { } + static inline const struct attribute_group **msi_populate_sysfs(struct device *dev) { return NULL; @@ -384,6 +400,8 @@ enum { MSI_FLAG_MUST_REACTIVATE = (1 << 5), /* Is level-triggered capable, using two messages */ MSI_FLAG_LEVEL_CAPABLE = (1 << 6), + /* Populate sysfs on alloc() and destroy it on free() */ + MSI_FLAG_DEV_SYSFS = (1 << 7), }; int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask, diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index b3f73ef0376c..6bca6ad9bf69 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -73,6 +73,38 @@ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) } EXPORT_SYMBOL_GPL(get_cached_msi_msg); +static void msi_device_data_release(struct device *dev, void *res) +{ + WARN_ON_ONCE(!list_empty(&dev->msi_list)); + dev->msi.data = NULL; +} + +/** + * msi_setup_device_data - Setup MSI device data + * @dev: Device for which MSI device data should be set up + * + * Return: 0 on success, appropriate error code otherwise + * + * This can be called more than once for @dev. If the MSI device data is + * already allocated the call succeeds. The allocated memory is + * automatically released when the device is destroyed. + */ +int msi_setup_device_data(struct device *dev) +{ + struct msi_device_data *md; + + if (dev->msi.data) + return 0; + + md = devres_alloc(msi_device_data_release, sizeof(*md), GFP_KERNEL); + if (!md) + return -ENOMEM; + + dev->msi.data = md; + devres_add(dev, md); + return 0; +} + #ifdef CONFIG_SYSFS static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) -- cgit v1.2.3-59-g8ed1b From 3f35d2cf9fbc656db82579d849cc69c373b1ad0d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 15 Dec 2021 18:16:44 +0100 Subject: PCI/MSI: Decouple MSI[-X] disable from pcim_release() The MSI core will introduce runtime allocation of MSI related data. This data will be devres managed and has to be set up before enabling PCI/MSI[-X]. This would introduce an ordering issue vs. pcim_release(). The setup order is: pcim_enable_device() devres_alloc(pcim_release...); ... pci_irq_alloc() msi_setup_device_data() devres_alloc(msi_device_data_release, ...) and once the device is released these release functions are invoked in the opposite order: msi_device_data_release() ... pcim_release() pci_disable_msi[x]() which is obviously wrong, because pci_disable_msi[x]() requires the MSI data to be available to tear down the MSI[-X] interrupts. Remove the MSI[-X] teardown from pcim_release() and add an explicit action to be installed on the attempt of enabling PCI/MSI[-X]. This allows the MSI core data allocation to be ordered correctly in a subsequent step. Reported-by: Nishanth Menon Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/87tuf9rdoj.ffs@tglx --- drivers/pci/msi/msi.c | 33 +++++++++++++++++++++++++++++++++ drivers/pci/pci.c | 5 ----- include/linux/pci.h | 3 ++- 3 files changed, 35 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 5af8d9b6d424..358b63e507e9 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -341,6 +341,31 @@ void pci_restore_msi_state(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_restore_msi_state); +static void pcim_msi_release(void *pcidev) +{ + struct pci_dev *dev = pcidev; + + dev->is_msi_managed = false; + pci_free_irq_vectors(dev); +} + +/* + * Needs to be separate from pcim_release to prevent an ordering problem + * vs. msi_device_data_release() in the MSI core code. + */ +static int pcim_setup_msi_release(struct pci_dev *dev) +{ + int ret; + + if (!pci_is_managed(dev) || dev->is_msi_managed) + return 0; + + ret = devm_add_action(&dev->dev, pcim_msi_release, dev); + if (!ret) + dev->is_msi_managed = true; + return ret; +} + static struct msi_desc * msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) { @@ -884,6 +909,10 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, if (nvec > maxvec) nvec = maxvec; + rc = pcim_setup_msi_release(dev); + if (rc) + return rc; + for (;;) { if (affd) { nvec = irq_calc_affinity_vectors(minvec, nvec, affd); @@ -927,6 +956,10 @@ static int __pci_enable_msix_range(struct pci_dev *dev, if (WARN_ON_ONCE(dev->msix_enabled)) return -EINVAL; + rc = pcim_setup_msi_release(dev); + if (rc) + return rc; + for (;;) { if (affd) { nvec = irq_calc_affinity_vectors(minvec, nvec, affd); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 3d2fb394986a..f3f606c232a8 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2024,11 +2024,6 @@ static void pcim_release(struct device *gendev, void *res) struct pci_devres *this = res; int i; - if (dev->msi_enabled) - pci_disable_msi(dev); - if (dev->msix_enabled) - pci_disable_msix(dev); - for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) if (this->region_mask & (1 << i)) pci_release_region(dev, i); diff --git a/include/linux/pci.h b/include/linux/pci.h index 5cc46baef519..a09736d3e05e 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -425,7 +425,8 @@ struct pci_dev { unsigned int ats_enabled:1; /* Address Translation Svc */ unsigned int pasid_enabled:1; /* Process Address Space ID */ unsigned int pri_enabled:1; /* Page Request Interface */ - unsigned int is_managed:1; + unsigned int is_managed:1; /* Managed via devres */ + unsigned int is_msi_managed:1; /* MSI release via devres installed */ unsigned int needs_freset:1; /* Requires fundamental reset */ unsigned int state_saved:1; unsigned int is_physfn:1; -- cgit v1.2.3-59-g8ed1b From bf6e054e0e3fbc9614355b760e18c8a14f952a4e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:03 +0100 Subject: genirq/msi: Provide msi_device_populate/destroy_sysfs() Add new allocation functions which can be activated by domain info flags. They store the groups pointer in struct msi_device_data. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221813.988659194@linutronix.de --- include/linux/msi.h | 4 ++++ kernel/irq/msi.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 7e4c8fd7c65d..1b96dc483b88 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -56,6 +56,8 @@ struct irq_data; struct msi_desc; struct pci_dev; struct platform_msi_priv_data; +struct attribute_group; + void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg); #ifdef CONFIG_GENERIC_MSI_IRQ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg); @@ -174,9 +176,11 @@ struct msi_desc { /** * msi_device_data - MSI per device data * @properties: MSI properties which are interesting to drivers + * @attrs: Pointer to the sysfs attribute group */ struct msi_device_data { unsigned long properties; + const struct attribute_group **attrs; }; int msi_setup_device_data(struct device *dev); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 6bca6ad9bf69..dd65e678a46c 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -199,6 +199,20 @@ error_attrs: return ERR_PTR(ret); } +/** + * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device + * @dev: The device (PCI, platform etc) which will get sysfs entries + */ +int msi_device_populate_sysfs(struct device *dev) +{ + const struct attribute_group **group = msi_populate_sysfs(dev); + + if (IS_ERR(group)) + return PTR_ERR(group); + dev->msi.data->attrs = group; + return 0; +} + /** * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices * @dev: The device(PCI, platform etc) who will remove sysfs entries @@ -225,6 +239,17 @@ void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_ir kfree(msi_irq_groups); } } + +/** + * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device + * @dev: The device (PCI, platform etc) for which to remove + * sysfs entries + */ +void msi_device_destroy_sysfs(struct device *dev) +{ + msi_destroy_sysfs(dev, dev->msi.data->attrs); + dev->msi.data->attrs = NULL; +} #endif #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN @@ -672,8 +697,19 @@ int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; + int ret; + + ret = ops->domain_alloc_irqs(domain, dev, nvec); + if (ret) + return ret; + + if (!(info->flags & MSI_FLAG_DEV_SYSFS)) + return 0; - return ops->domain_alloc_irqs(domain, dev, nvec); + ret = msi_device_populate_sysfs(dev); + if (ret) + msi_domain_free_irqs(domain, dev); + return ret; } void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) @@ -712,7 +748,9 @@ void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; - return ops->domain_free_irqs(domain, dev); + if (info->flags & MSI_FLAG_DEV_SYSFS) + msi_device_destroy_sysfs(dev); + ops->domain_free_irqs(domain, dev); } /** -- cgit v1.2.3-59-g8ed1b From ffd84485e6beb9cad3e5a133d88201b995298c33 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:05 +0100 Subject: PCI/MSI: Let the irq code handle sysfs groups Set the domain info flag which makes the core code handle sysfs groups and put an explicit invocation into the legacy code. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211210221814.048612053@linutronix.de --- drivers/pci/msi/irqdomain.c | 2 +- drivers/pci/msi/legacy.c | 6 +++++- drivers/pci/msi/msi.c | 23 ----------------------- include/linux/pci.h | 1 - 4 files changed, 6 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index a5546900244d..26b23a1c4f60 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -159,7 +159,7 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) pci_msi_domain_update_chip_ops(info); - info->flags |= MSI_FLAG_ACTIVATE_EARLY; + info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) info->flags |= MSI_FLAG_MUST_REACTIVATE; diff --git a/drivers/pci/msi/legacy.c b/drivers/pci/msi/legacy.c index d52cff17b694..773f35122bbb 100644 --- a/drivers/pci/msi/legacy.c +++ b/drivers/pci/msi/legacy.c @@ -70,10 +70,14 @@ int pci_msi_legacy_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) { int ret = arch_setup_msi_irqs(dev, nvec, type); - return pci_msi_setup_check_result(dev, type, ret); + ret = pci_msi_setup_check_result(dev, type, ret); + if (!ret) + ret = msi_device_populate_sysfs(&dev->dev); + return ret; } void pci_msi_legacy_teardown_msi_irqs(struct pci_dev *dev) { + msi_device_destroy_sysfs(&dev->dev); arch_teardown_msi_irqs(dev); } diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 369e3c5827b2..76c15be8e1cc 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -233,11 +233,6 @@ static void free_msi_irqs(struct pci_dev *dev) for (i = 0; i < entry->nvec_used; i++) BUG_ON(irq_has_action(entry->irq + i)); - if (dev->msi_irq_groups) { - msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups); - dev->msi_irq_groups = NULL; - } - pci_msi_teardown_msi_irqs(dev); list_for_each_entry_safe(entry, tmp, msi_list, list) { @@ -455,7 +450,6 @@ static int msi_verify_entries(struct pci_dev *dev) static int msi_capability_init(struct pci_dev *dev, int nvec, struct irq_affinity *affd) { - const struct attribute_group **groups; struct msi_desc *entry; int ret; @@ -486,14 +480,6 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, if (ret) goto err; - groups = msi_populate_sysfs(&dev->dev); - if (IS_ERR(groups)) { - ret = PTR_ERR(groups); - goto err; - } - - dev->msi_irq_groups = groups; - /* Set MSI enabled bits */ pci_intx_for_msi(dev, 0); pci_msi_set_enable(dev, 1); @@ -622,7 +608,6 @@ static void msix_mask_all(void __iomem *base, int tsize) static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { - const struct attribute_group **groups; void __iomem *base; int ret, tsize; u16 control; @@ -664,14 +649,6 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, msix_update_entries(dev, entries); - groups = msi_populate_sysfs(&dev->dev); - if (IS_ERR(groups)) { - ret = PTR_ERR(groups); - goto out_free; - } - - dev->msi_irq_groups = groups; - /* Disable INTX */ pci_intx_for_msi(dev, 0); diff --git a/include/linux/pci.h b/include/linux/pci.h index a09736d3e05e..0a7b6b2f163b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -476,7 +476,6 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; - const struct attribute_group **msi_irq_groups; #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC -- cgit v1.2.3-59-g8ed1b From 24cff375fdb663c2238f06693a067b9219596fdc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:08 +0100 Subject: genirq/msi: Remove the original sysfs interfaces No more users. Refactor the core code accordingly and move the global interface under CONFIG_PCI_MSI_ARCH_FALLBACKS. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.168362229@linutronix.de --- include/linux/msi.h | 18 +++--------------- kernel/irq/msi.c | 53 ++++++++++++++++++++--------------------------------- 2 files changed, 23 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 1b96dc483b88..634a12962e72 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -249,22 +249,10 @@ void pci_msi_unmask_irq(struct irq_data *data); #ifdef CONFIG_SYSFS int msi_device_populate_sysfs(struct device *dev); void msi_device_destroy_sysfs(struct device *dev); - -const struct attribute_group **msi_populate_sysfs(struct device *dev); -void msi_destroy_sysfs(struct device *dev, - const struct attribute_group **msi_irq_groups); -#else +#else /* CONFIG_SYSFS */ static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } static inline void msi_device_destroy_sysfs(struct device *dev) { } - -static inline const struct attribute_group **msi_populate_sysfs(struct device *dev) -{ - return NULL; -} -static inline void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups) -{ -} -#endif +#endif /* !CONFIG_SYSFS */ /* * The arch hooks to setup up msi irqs. Default functions are implemented @@ -279,7 +267,7 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc); void arch_teardown_msi_irq(unsigned int irq); int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); void arch_teardown_msi_irqs(struct pci_dev *dev); -#endif +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ /* * The restore hook is still available even for fully irq domain based diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index dd65e678a46c..8e433f1a24fb 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -118,12 +118,8 @@ static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, /** * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices * @dev: The device(PCI, platform etc) who will get sysfs entries - * - * Return attribute_group ** so that specific bus MSI can save it to - * somewhere during initilizing msi irqs. If devices has no MSI irq, - * return NULL; if it fails to populate sysfs, return ERR_PTR */ -const struct attribute_group **msi_populate_sysfs(struct device *dev) +static const struct attribute_group **msi_populate_sysfs(struct device *dev) { const struct attribute_group **msi_irq_groups; struct attribute **msi_attrs, *msi_attr; @@ -213,33 +209,6 @@ int msi_device_populate_sysfs(struct device *dev) return 0; } -/** - * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices - * @dev: The device(PCI, platform etc) who will remove sysfs entries - * @msi_irq_groups: attribute_group for device msi_irqs entries - */ -void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups) -{ - struct device_attribute *dev_attr; - struct attribute **msi_attrs; - int count = 0; - - if (msi_irq_groups) { - sysfs_remove_groups(&dev->kobj, msi_irq_groups); - msi_attrs = msi_irq_groups[0]->attrs; - while (msi_attrs[count]) { - dev_attr = container_of(msi_attrs[count], - struct device_attribute, attr); - kfree(dev_attr->attr.name); - kfree(dev_attr); - ++count; - } - kfree(msi_attrs); - kfree(msi_irq_groups[0]); - kfree(msi_irq_groups); - } -} - /** * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device * @dev: The device (PCI, platform etc) for which to remove @@ -247,8 +216,26 @@ void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_ir */ void msi_device_destroy_sysfs(struct device *dev) { - msi_destroy_sysfs(dev, dev->msi.data->attrs); + const struct attribute_group **msi_irq_groups = dev->msi.data->attrs; + struct device_attribute *dev_attr; + struct attribute **msi_attrs; + int count = 0; + dev->msi.data->attrs = NULL; + if (!msi_irq_groups) + return; + + sysfs_remove_groups(&dev->kobj, msi_irq_groups); + msi_attrs = msi_irq_groups[0]->attrs; + while (msi_attrs[count]) { + dev_attr = container_of(msi_attrs[count], struct device_attribute, attr); + kfree(dev_attr->attr.name); + kfree(dev_attr); + ++count; + } + kfree(msi_attrs); + kfree(msi_irq_groups[0]); + kfree(msi_irq_groups); } #endif -- cgit v1.2.3-59-g8ed1b From 9835cec6d557b0bff3d48bd91cd0484aba59386c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:09 +0100 Subject: platform-msi: Rename functions and clarify comments It's hard to distinguish what platform_msi_domain_alloc() and platform_msi_domain_alloc_irqs() are about. Make the distinction more explicit and add comments which explain the use cases properly. Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.228706214@linutronix.de --- drivers/base/platform-msi.c | 36 +++++++++++++++++++++--------------- drivers/irqchip/irq-mbigen.c | 4 ++-- drivers/irqchip/irq-mvebu-icu.c | 6 +++--- include/linux/msi.h | 8 ++++---- 4 files changed, 30 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 5d50793dded6..46ee9554717b 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -313,17 +313,18 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); * a platform-msi domain * @domain: The platform-msi domain * - * Returns the private data provided when calling - * platform_msi_create_device_domain. + * Return: The private data provided when calling + * platform_msi_create_device_domain(). */ void *platform_msi_get_host_data(struct irq_domain *domain) { struct platform_msi_priv_data *data = domain->host_data; + return data->host_data; } /** - * __platform_msi_create_device_domain - Create a platform-msi domain + * __platform_msi_create_device_domain - Create a platform-msi device domain * * @dev: The device generating the MSIs * @nvec: The number of MSIs that need to be allocated @@ -332,7 +333,11 @@ void *platform_msi_get_host_data(struct irq_domain *domain) * @ops: The hierarchy domain operations to use * @host_data: Private data associated to this domain * - * Returns an irqdomain for @nvec interrupts + * Return: An irqdomain for @nvec interrupts on success, NULL in case of error. + * + * This is for interrupt domains which stack on a platform-msi domain + * created by platform_msi_create_irq_domain(). @dev->msi.domain points to + * that platform-msi domain which is the parent for the new domain. */ struct irq_domain * __platform_msi_create_device_domain(struct device *dev, @@ -372,18 +377,19 @@ free_priv: } /** - * platform_msi_domain_free - Free interrupts associated with a platform-msi - * domain + * platform_msi_device_domain_free - Free interrupts associated with a platform-msi + * device domain * - * @domain: The platform-msi domain + * @domain: The platform-msi device domain * @virq: The base irq from which to perform the free operation * @nvec: How many interrupts to free from @virq */ -void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nvec) +void platform_msi_device_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nvec) { struct platform_msi_priv_data *data = domain->host_data; struct msi_desc *desc, *tmp; + for_each_msi_entry_safe(desc, tmp, data->dev) { if (WARN_ON(!desc->irq || desc->nvec_used != 1)) return; @@ -397,10 +403,10 @@ void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, } /** - * platform_msi_domain_alloc - Allocate interrupts associated with - * a platform-msi domain + * platform_msi_device_domain_alloc - Allocate interrupts associated with + * a platform-msi device domain * - * @domain: The platform-msi domain + * @domain: The platform-msi device domain * @virq: The base irq from which to perform the allocate operation * @nr_irqs: How many interrupts to free from @virq * @@ -408,8 +414,8 @@ void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, * with irq_domain_mutex held (which can only be done as part of a * top-level interrupt allocation). */ -int platform_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) +int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) { struct platform_msi_priv_data *data = domain->host_data; int err; @@ -421,7 +427,7 @@ int platform_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, err = msi_domain_populate_irqs(domain->parent, data->dev, virq, nr_irqs, &data->arg); if (err) - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); return err; } diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 12df2162108e..f3faf5c99770 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -207,7 +207,7 @@ static int mbigen_irq_domain_alloc(struct irq_domain *domain, if (err) return err; - err = platform_msi_domain_alloc(domain, virq, nr_irqs); + err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); if (err) return err; @@ -223,7 +223,7 @@ static int mbigen_irq_domain_alloc(struct irq_domain *domain, static void mbigen_irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); } static const struct irq_domain_ops mbigen_domain_ops = { diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c index 4a4a65a6f632..497da344717c 100644 --- a/drivers/irqchip/irq-mvebu-icu.c +++ b/drivers/irqchip/irq-mvebu-icu.c @@ -221,7 +221,7 @@ mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, icu_irqd->icu_group = msi_data->subset_data->icu_group; icu_irqd->icu = icu; - err = platform_msi_domain_alloc(domain, virq, nr_irqs); + err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); if (err) { dev_err(icu->dev, "failed to allocate ICU interrupt in parent domain\n"); goto free_irqd; @@ -245,7 +245,7 @@ mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; free_msi: - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); free_irqd: kfree(icu_irqd); return err; @@ -260,7 +260,7 @@ mvebu_icu_irq_domain_free(struct irq_domain *domain, unsigned int virq, kfree(icu_irqd); - platform_msi_domain_free(domain, virq, nr_irqs); + platform_msi_device_domain_free(domain, virq, nr_irqs); } static const struct irq_domain_ops mvebu_icu_domain_ops = { diff --git a/include/linux/msi.h b/include/linux/msi.h index 634a12962e72..12dd28629c43 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -435,10 +435,10 @@ __platform_msi_create_device_domain(struct device *dev, #define platform_msi_create_device_tree_domain(dev, nvec, write, ops, data) \ __platform_msi_create_device_domain(dev, nvec, true, write, ops, data) -int platform_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs); -void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nvec); +int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs); +void platform_msi_device_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nvec); void *platform_msi_get_host_data(struct irq_domain *domain); #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */ -- cgit v1.2.3-59-g8ed1b From fc22e7dbcdb3e06a3d3ce05fc91c6a2345411f9b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:11 +0100 Subject: platform-msi: Store platform private data pointer in msi_device_data Storing the platform private data in a MSI descriptor is sloppy at best. The data belongs to the device and not to the descriptor. Add a pointer to struct msi_device_data and store the pointer there. Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.287680528@linutronix.de --- drivers/base/platform-msi.c | 79 ++++++++++++++++++--------------------------- include/linux/msi.h | 4 +-- 2 files changed, 34 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 46ee9554717b..1487906ad5e0 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -38,9 +38,7 @@ static DEFINE_IDA(platform_msi_devid_ida); */ static irq_hw_number_t platform_msi_calc_hwirq(struct msi_desc *desc) { - u32 devid; - - devid = desc->platform.msi_priv_data->devid; + u32 devid = desc->dev->msi.data->platform_data->devid; return (devid << (32 - DEV_ID_SHIFT)) | desc->platform.msi_index; } @@ -85,11 +83,8 @@ static void platform_msi_update_dom_ops(struct msi_domain_info *info) static void platform_msi_write_msg(struct irq_data *data, struct msi_msg *msg) { struct msi_desc *desc = irq_data_get_msi_desc(data); - struct platform_msi_priv_data *priv_data; - - priv_data = desc->platform.msi_priv_data; - priv_data->write_msg(desc, msg); + desc->dev->msi.data->platform_data->write_msg(desc, msg); } static void platform_msi_update_chip_ops(struct msi_domain_info *info) @@ -126,9 +121,7 @@ static void platform_msi_free_descs(struct device *dev, int base, int nvec) } static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, - int nvec, - struct platform_msi_priv_data *data) - + int nvec) { struct msi_desc *desc; int i, base = 0; @@ -144,7 +137,6 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, if (!desc) break; - desc->platform.msi_priv_data = data; desc->platform.msi_index = base + i; desc->irq = virq ? virq + i : 0; @@ -161,11 +153,9 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, return 0; } -static int platform_msi_alloc_descs(struct device *dev, int nvec, - struct platform_msi_priv_data *data) - +static int platform_msi_alloc_descs(struct device *dev, int nvec) { - return platform_msi_alloc_descs_with_irq(dev, 0, nvec, data); + return platform_msi_alloc_descs_with_irq(dev, 0, nvec); } /** @@ -199,9 +189,8 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, return domain; } -static struct platform_msi_priv_data * -platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, - irq_write_msi_msg_t write_msi_msg) +static int platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, + irq_write_msi_msg_t write_msi_msg) { struct platform_msi_priv_data *datap; int err; @@ -213,41 +202,44 @@ platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, * capable devices). */ if (!dev->msi.domain || !write_msi_msg || !nvec || nvec > MAX_DEV_MSIS) - return ERR_PTR(-EINVAL); + return -EINVAL; if (dev->msi.domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { dev_err(dev, "Incompatible msi_domain, giving up\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } err = msi_setup_device_data(dev); if (err) - return ERR_PTR(err); + return err; - /* Already had a helping of MSI? Greed... */ - if (!list_empty(dev_to_msi_list(dev))) - return ERR_PTR(-EBUSY); + /* Already initialized? */ + if (dev->msi.data->platform_data) + return -EBUSY; datap = kzalloc(sizeof(*datap), GFP_KERNEL); if (!datap) - return ERR_PTR(-ENOMEM); + return -ENOMEM; datap->devid = ida_simple_get(&platform_msi_devid_ida, 0, 1 << DEV_ID_SHIFT, GFP_KERNEL); if (datap->devid < 0) { err = datap->devid; kfree(datap); - return ERR_PTR(err); + return err; } datap->write_msg = write_msi_msg; datap->dev = dev; - - return datap; + dev->msi.data->platform_data = datap; + return 0; } -static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) +static void platform_msi_free_priv_data(struct device *dev) { + struct platform_msi_priv_data *data = dev->msi.data->platform_data; + + dev->msi.data->platform_data = NULL; ida_simple_remove(&platform_msi_devid_ida, data->devid); kfree(data); } @@ -264,14 +256,13 @@ static void platform_msi_free_priv_data(struct platform_msi_priv_data *data) int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, irq_write_msi_msg_t write_msi_msg) { - struct platform_msi_priv_data *priv_data; int err; - priv_data = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); - if (IS_ERR(priv_data)) - return PTR_ERR(priv_data); + err = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); + if (err) + return err; - err = platform_msi_alloc_descs(dev, nvec, priv_data); + err = platform_msi_alloc_descs(dev, nvec); if (err) goto out_free_priv_data; @@ -284,8 +275,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, out_free_desc: platform_msi_free_descs(dev, 0, nvec); out_free_priv_data: - platform_msi_free_priv_data(priv_data); - + platform_msi_free_priv_data(dev); return err; } EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); @@ -296,15 +286,9 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); */ void platform_msi_domain_free_irqs(struct device *dev) { - if (!list_empty(dev_to_msi_list(dev))) { - struct msi_desc *desc; - - desc = first_msi_entry(dev); - platform_msi_free_priv_data(desc->platform.msi_priv_data); - } - msi_domain_free_irqs(dev->msi.domain, dev); platform_msi_free_descs(dev, 0, MAX_DEV_MSIS); + platform_msi_free_priv_data(dev); } EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs); @@ -351,10 +335,11 @@ __platform_msi_create_device_domain(struct device *dev, struct irq_domain *domain; int err; - data = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); - if (IS_ERR(data)) + err = platform_msi_alloc_priv_data(dev, nvec, write_msi_msg); + if (err) return NULL; + data = dev->msi.data->platform_data; data->host_data = host_data; domain = irq_domain_create_hierarchy(dev->msi.domain, 0, is_tree ? 0 : nvec, @@ -372,7 +357,7 @@ __platform_msi_create_device_domain(struct device *dev, free_domain: irq_domain_remove(domain); free_priv: - platform_msi_free_priv_data(data); + platform_msi_free_priv_data(dev); return NULL; } @@ -420,7 +405,7 @@ int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int vir struct platform_msi_priv_data *data = domain->host_data; int err; - err = platform_msi_alloc_descs_with_irq(data->dev, virq, nr_irqs, data); + err = platform_msi_alloc_descs_with_irq(data->dev, virq, nr_irqs); if (err) return err; diff --git a/include/linux/msi.h b/include/linux/msi.h index 12dd28629c43..cdf0d09c3ad4 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -108,11 +108,9 @@ struct pci_msi_desc { /** * platform_msi_desc - Platform device specific msi descriptor data - * @msi_priv_data: Pointer to platform private data * @msi_index: The index of the MSI descriptor for multi MSI */ struct platform_msi_desc { - struct platform_msi_priv_data *msi_priv_data; u16 msi_index; }; @@ -177,10 +175,12 @@ struct msi_desc { * msi_device_data - MSI per device data * @properties: MSI properties which are interesting to drivers * @attrs: Pointer to the sysfs attribute group + * @platform_data: Platform-MSI specific data */ struct msi_device_data { unsigned long properties; const struct attribute_group **attrs; + struct platform_msi_priv_data *platform_data; }; int msi_setup_device_data(struct device *dev); -- cgit v1.2.3-59-g8ed1b From 20c6d424cfe641659c9a025db8a8608701b27246 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:12 +0100 Subject: genirq/msi: Consolidate MSI descriptor data All non PCI/MSI usage variants have data structures in struct msi_desc with only one member: xxx_index. PCI/MSI has a entry_nr member. Add a common msi_index member to struct msi_desc so all implementations can share it which allows further consolidation. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.350967317@linutronix.de --- include/linux/msi.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index cdf0d09c3ad4..ee8fe49dedd1 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -143,6 +143,7 @@ struct ti_sci_inta_msi_desc { * address or data changes * @write_msi_msg_data: Data parameter for the callback. * + * @msi_index: Index of the msi descriptor * @pci: [PCI] PCI speficic msi descriptor data * @platform: [platform] Platform device specific msi descriptor data * @fsl_mc: [fsl-mc] FSL MC device specific msi descriptor data @@ -163,6 +164,7 @@ struct msi_desc { void (*write_msi_msg)(struct msi_desc *entry, void *data); void *write_msi_msg_data; + u16 msi_index; union { struct pci_msi_desc pci; struct platform_msi_desc platform; -- cgit v1.2.3-59-g8ed1b From dba27c7fa36f468e7eb29b216879f8c33bf0955d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:14 +0100 Subject: platform-msi: Use msi_desc::msi_index Use the common msi_index member and get rid of the pointless wrapper struct. Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.413638645@linutronix.de --- drivers/base/platform-msi.c | 10 +++++----- drivers/dma/qcom/hidma.c | 4 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- drivers/mailbox/bcm-flexrm-mailbox.c | 4 ++-- include/linux/msi.h | 10 ---------- 5 files changed, 11 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 1487906ad5e0..88bdc4b19916 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -40,7 +40,7 @@ static irq_hw_number_t platform_msi_calc_hwirq(struct msi_desc *desc) { u32 devid = desc->dev->msi.data->platform_data->devid; - return (devid << (32 - DEV_ID_SHIFT)) | desc->platform.msi_index; + return (devid << (32 - DEV_ID_SHIFT)) | desc->msi_index; } static void platform_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) @@ -112,8 +112,8 @@ static void platform_msi_free_descs(struct device *dev, int base, int nvec) struct msi_desc *desc, *tmp; list_for_each_entry_safe(desc, tmp, dev_to_msi_list(dev), list) { - if (desc->platform.msi_index >= base && - desc->platform.msi_index < (base + nvec)) { + if (desc->msi_index >= base && + desc->msi_index < (base + nvec)) { list_del(&desc->list); free_msi_entry(desc); } @@ -129,7 +129,7 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, if (!list_empty(dev_to_msi_list(dev))) { desc = list_last_entry(dev_to_msi_list(dev), struct msi_desc, list); - base = desc->platform.msi_index + 1; + base = desc->msi_index + 1; } for (i = 0; i < nvec; i++) { @@ -137,7 +137,7 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq, if (!desc) break; - desc->platform.msi_index = base + i; + desc->msi_index = base + i; desc->irq = virq ? virq + i : 0; list_add_tail(&desc->list, dev_to_msi_list(dev)); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 23d64489d25f..db8241bfe199 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -666,7 +666,7 @@ static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) struct device *dev = msi_desc_to_dev(desc); struct hidma_dev *dmadev = dev_get_drvdata(dev); - if (!desc->platform.msi_index) { + if (!desc->msi_index) { writel(msg->address_lo, dmadev->dev_evca + 0x118); writel(msg->address_hi, dmadev->dev_evca + 0x11C); writel(msg->data, dmadev->dev_evca + 0x120); @@ -702,7 +702,7 @@ static int hidma_request_msi(struct hidma_dev *dmadev, return rc; for_each_msi_entry(desc, &pdev->dev) { - if (!desc->platform.msi_index) + if (!desc->msi_index) dmadev->msi_virqbase = desc->irq; rc = devm_request_irq(&pdev->dev, desc->irq, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 5ba9006ec550..beaf91bca30a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3142,7 +3142,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) phys_addr_t doorbell; struct device *dev = msi_desc_to_dev(desc); struct arm_smmu_device *smmu = dev_get_drvdata(dev); - phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index]; + phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index]; doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; doorbell &= MSI_CFG0_ADDR_MASK; @@ -3183,7 +3183,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) } for_each_msi_entry(desc, dev) { - switch (desc->platform.msi_index) { + switch (desc->msi_index) { case EVTQ_MSI_INDEX: smmu->evtq.q.irq = desc->irq; break; diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 78073ad1f2f1..53a84733d8cc 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1484,7 +1484,7 @@ static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg) { struct device *dev = msi_desc_to_dev(desc); struct flexrm_mbox *mbox = dev_get_drvdata(dev); - struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index]; + struct flexrm_ring *ring = &mbox->rings[desc->msi_index]; /* Configure per-Ring MSI registers */ writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS); @@ -1609,7 +1609,7 @@ static int flexrm_mbox_probe(struct platform_device *pdev) /* Save alloced IRQ numbers for each ring */ for_each_msi_entry(desc, dev) { - ring = &mbox->rings[desc->platform.msi_index]; + ring = &mbox->rings[desc->msi_index]; ring->irq = desc->irq; } diff --git a/include/linux/msi.h b/include/linux/msi.h index ee8fe49dedd1..1d85e954e130 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -106,14 +106,6 @@ struct pci_msi_desc { }; }; -/** - * platform_msi_desc - Platform device specific msi descriptor data - * @msi_index: The index of the MSI descriptor for multi MSI - */ -struct platform_msi_desc { - u16 msi_index; -}; - /** * fsl_mc_msi_desc - FSL-MC device specific msi descriptor data * @msi_index: The index of the MSI descriptor @@ -145,7 +137,6 @@ struct ti_sci_inta_msi_desc { * * @msi_index: Index of the msi descriptor * @pci: [PCI] PCI speficic msi descriptor data - * @platform: [platform] Platform device specific msi descriptor data * @fsl_mc: [fsl-mc] FSL MC device specific msi descriptor data * @inta: [INTA] TISCI based INTA specific msi descriptor data */ @@ -167,7 +158,6 @@ struct msi_desc { u16 msi_index; union { struct pci_msi_desc pci; - struct platform_msi_desc platform; struct fsl_mc_msi_desc fsl_mc; struct ti_sci_inta_msi_desc inta; }; -- cgit v1.2.3-59-g8ed1b From 78ee9fb4b8b126ed84a819a6e1732fd3039b525a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:15 +0100 Subject: bus: fsl-mc-msi: Use msi_desc::msi_index Use the common msi_index member and get rid of the pointless wrapper struct. Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.477386185@linutronix.de --- drivers/bus/fsl-mc/fsl-mc-allocator.c | 2 +- drivers/bus/fsl-mc/fsl-mc-msi.c | 6 +++--- include/linux/msi.h | 10 ---------- 3 files changed, 4 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/fsl-mc/fsl-mc-allocator.c b/drivers/bus/fsl-mc/fsl-mc-allocator.c index 6c513556911e..3370b63bad5f 100644 --- a/drivers/bus/fsl-mc/fsl-mc-allocator.c +++ b/drivers/bus/fsl-mc/fsl-mc-allocator.c @@ -393,7 +393,7 @@ int fsl_mc_populate_irq_pool(struct fsl_mc_device *mc_bus_dev, } for_each_msi_entry(msi_desc, &mc_bus_dev->dev) { - mc_dev_irq = &irq_resources[msi_desc->fsl_mc.msi_index]; + mc_dev_irq = &irq_resources[msi_desc->msi_index]; mc_dev_irq->msi_desc = msi_desc; mc_dev_irq->resource.id = msi_desc->irq; } diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c index c6215b7fe27c..2ffb70550a39 100644 --- a/drivers/bus/fsl-mc/fsl-mc-msi.c +++ b/drivers/bus/fsl-mc/fsl-mc-msi.c @@ -29,7 +29,7 @@ static irq_hw_number_t fsl_mc_domain_calc_hwirq(struct fsl_mc_device *dev, * Make the base hwirq value for ICID*10000 so it is readable * as a decimal value in /proc/interrupts. */ - return (irq_hw_number_t)(desc->fsl_mc.msi_index + (dev->icid * 10000)); + return (irq_hw_number_t)(desc->msi_index + (dev->icid * 10000)); } static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg, @@ -122,7 +122,7 @@ static void fsl_mc_msi_write_msg(struct irq_data *irq_data, struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev); struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev); struct fsl_mc_device_irq *mc_dev_irq = - &mc_bus->irq_resources[msi_desc->fsl_mc.msi_index]; + &mc_bus->irq_resources[msi_desc->msi_index]; msi_desc->msg = *msg; @@ -235,7 +235,7 @@ static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count) goto cleanup_msi_descs; } - msi_desc->fsl_mc.msi_index = i; + msi_desc->msi_index = i; INIT_LIST_HEAD(&msi_desc->list); list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); } diff --git a/include/linux/msi.h b/include/linux/msi.h index 1d85e954e130..25edf83ede41 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -106,14 +106,6 @@ struct pci_msi_desc { }; }; -/** - * fsl_mc_msi_desc - FSL-MC device specific msi descriptor data - * @msi_index: The index of the MSI descriptor - */ -struct fsl_mc_msi_desc { - u16 msi_index; -}; - /** * ti_sci_inta_msi_desc - TISCI based INTA specific msi descriptor data * @dev_index: TISCI device index @@ -137,7 +129,6 @@ struct ti_sci_inta_msi_desc { * * @msi_index: Index of the msi descriptor * @pci: [PCI] PCI speficic msi descriptor data - * @fsl_mc: [fsl-mc] FSL MC device specific msi descriptor data * @inta: [INTA] TISCI based INTA specific msi descriptor data */ struct msi_desc { @@ -158,7 +149,6 @@ struct msi_desc { u16 msi_index; union { struct pci_msi_desc pci; - struct fsl_mc_msi_desc fsl_mc; struct ti_sci_inta_msi_desc inta; }; }; -- cgit v1.2.3-59-g8ed1b From 0f18095871fc59c89a281caf6f18538cf9e50fbf Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:17 +0100 Subject: soc: ti: ti_sci_inta_msi: Use msi_desc::msi_index Use the common msi_index member and get rid of the pointless wrapper struct. Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Nishanth Menon Link: https://lore.kernel.org/r/20211210221814.540704224@linutronix.de --- drivers/irqchip/irq-ti-sci-inta.c | 2 +- drivers/soc/ti/ti_sci_inta_msi.c | 6 +++--- include/linux/msi.h | 16 ++-------------- 3 files changed, 6 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c index 8eba08db33b2..5fdbb4358dd0 100644 --- a/drivers/irqchip/irq-ti-sci-inta.c +++ b/drivers/irqchip/irq-ti-sci-inta.c @@ -595,7 +595,7 @@ static void ti_sci_inta_msi_set_desc(msi_alloc_info_t *arg, struct platform_device *pdev = to_platform_device(desc->dev); arg->desc = desc; - arg->hwirq = TO_HWIRQ(pdev->id, desc->inta.dev_index); + arg->hwirq = TO_HWIRQ(pdev->id, desc->msi_index); } static struct msi_domain_ops ti_sci_inta_msi_ops = { diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c index a89bcbcd4694..9d23daac74c3 100644 --- a/drivers/soc/ti/ti_sci_inta_msi.c +++ b/drivers/soc/ti/ti_sci_inta_msi.c @@ -84,7 +84,7 @@ static int ti_sci_inta_msi_alloc_descs(struct device *dev, return -ENOMEM; } - msi_desc->inta.dev_index = res->desc[set].start + i; + msi_desc->msi_index = res->desc[set].start + i; INIT_LIST_HEAD(&msi_desc->list); list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); count++; @@ -96,7 +96,7 @@ static int ti_sci_inta_msi_alloc_descs(struct device *dev, return -ENOMEM; } - msi_desc->inta.dev_index = res->desc[set].start_sec + i; + msi_desc->msi_index = res->desc[set].start_sec + i; INIT_LIST_HEAD(&msi_desc->list); list_add_tail(&msi_desc->list, dev_to_msi_list(dev)); count++; @@ -154,7 +154,7 @@ unsigned int ti_sci_inta_msi_get_virq(struct device *dev, u32 dev_index) struct msi_desc *desc; for_each_msi_entry(desc, dev) - if (desc->inta.dev_index == dev_index) + if (desc->msi_index == dev_index) return desc->irq; return -ENODEV; diff --git a/include/linux/msi.h b/include/linux/msi.h index 25edf83ede41..45ec5d07a5f3 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -106,14 +106,6 @@ struct pci_msi_desc { }; }; -/** - * ti_sci_inta_msi_desc - TISCI based INTA specific msi descriptor data - * @dev_index: TISCI device index - */ -struct ti_sci_inta_msi_desc { - u16 dev_index; -}; - /** * struct msi_desc - Descriptor structure for MSI based interrupts * @list: List head for management @@ -128,8 +120,7 @@ struct ti_sci_inta_msi_desc { * @write_msi_msg_data: Data parameter for the callback. * * @msi_index: Index of the msi descriptor - * @pci: [PCI] PCI speficic msi descriptor data - * @inta: [INTA] TISCI based INTA specific msi descriptor data + * @pci: PCI specific msi descriptor data */ struct msi_desc { /* Shared device/bus type independent data */ @@ -147,10 +138,7 @@ struct msi_desc { void *write_msi_msg_data; u16 msi_index; - union { - struct pci_msi_desc pci; - struct ti_sci_inta_msi_desc inta; - }; + struct pci_msi_desc pci; }; /** -- cgit v1.2.3-59-g8ed1b From 173ffad79d177d9a91fbf3be6bf67ca81e0f765a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:18 +0100 Subject: PCI/MSI: Use msi_desc::msi_index The usage of msi_desc::pci::entry_nr is confusing at best. It's the index into the MSI[X] descriptor table. Use msi_desc::msi_index which is shared between all MSI incarnations instead of having a PCI specific storage for no value. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211210221814.602911509@linutronix.de --- arch/powerpc/platforms/pseries/msi.c | 4 ++-- arch/x86/pci/xen.c | 2 +- drivers/pci/msi/irqdomain.c | 2 +- drivers/pci/msi/msi.c | 20 ++++++++------------ drivers/pci/xen-pcifront.c | 2 +- include/linux/msi.h | 2 -- 6 files changed, 13 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index dc8cf3603f9a..27cd1fb7e02b 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -332,7 +332,7 @@ static int check_msix_entries(struct pci_dev *pdev) expected = 0; for_each_pci_msi_entry(entry, pdev) { - if (entry->pci.msi_attrib.entry_nr != expected) { + if (entry->msi_index != expected) { pr_debug("rtas_msi: bad MSI-X entries.\n"); return -EINVAL; } @@ -579,7 +579,7 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq int hwirq; int i, ret; - hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->pci.msi_attrib.entry_nr); + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); return hwirq; diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index bfd87b46bc51..ded032053479 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -306,7 +306,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; map_irq.table_base = pci_resource_start(dev, bir); - map_irq.entry_nr = msidesc->pci.msi_attrib.entry_nr; + map_irq.entry_nr = msidesc->msi_index; } ret = -EINVAL; diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 26b23a1c4f60..422420276d6f 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -57,7 +57,7 @@ static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) { struct pci_dev *dev = msi_desc_to_pci_dev(desc); - return (irq_hw_number_t)desc->pci.msi_attrib.entry_nr | + return (irq_hw_number_t)desc->msi_index | pci_dev_id(dev) << 11 | (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; } diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 76c15be8e1cc..8fb4dd2a18b9 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -44,7 +44,7 @@ static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask) static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) { - return desc->pci.mask_base + desc->pci.msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; + return desc->pci.mask_base + desc->msi_index * PCI_MSIX_ENTRY_SIZE; } /* @@ -394,13 +394,10 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING) control |= PCI_MSI_FLAGS_MASKBIT; - entry->pci.msi_attrib.is_msix = 0; - entry->pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); - entry->pci.msi_attrib.is_virtual = 0; - entry->pci.msi_attrib.entry_nr = 0; + entry->pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); entry->pci.msi_attrib.can_mask = !pci_msi_ignore_mask && !!(control & PCI_MSI_FLAGS_MASKBIT); - entry->pci.msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ + entry->pci.msi_attrib.default_irq = dev->irq; entry->pci.msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; entry->pci.msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); @@ -542,12 +539,11 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, entry->pci.msi_attrib.is_64 = 1; if (entries) - entry->pci.msi_attrib.entry_nr = entries[i].entry; + entry->msi_index = entries[i].entry; else - entry->pci.msi_attrib.entry_nr = i; + entry->msi_index = i; - entry->pci.msi_attrib.is_virtual = - entry->pci.msi_attrib.entry_nr >= vec_count; + entry->pci.msi_attrib.is_virtual = entry->msi_index >= vec_count; entry->pci.msi_attrib.can_mask = !pci_msi_ignore_mask && !entry->pci.msi_attrib.is_virtual; @@ -1088,7 +1084,7 @@ int pci_irq_vector(struct pci_dev *dev, unsigned int nr) struct msi_desc *entry; for_each_pci_msi_entry(entry, dev) { - if (entry->pci.msi_attrib.entry_nr == nr) + if (entry->msi_index == nr) return entry->irq; } WARN_ON_ONCE(1); @@ -1127,7 +1123,7 @@ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) struct msi_desc *entry; for_each_pci_msi_entry(entry, dev) { - if (entry->pci.msi_attrib.entry_nr == nr) + if (entry->msi_index == nr) return &entry->affinity->mask; } WARN_ON_ONCE(1); diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index 699cc9544424..bacf51152168 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -263,7 +263,7 @@ static int pci_frontend_enable_msix(struct pci_dev *dev, i = 0; for_each_pci_msi_entry(entry, dev) { - op.msix_entries[i].entry = entry->pci.msi_attrib.entry_nr; + op.msix_entries[i].entry = entry->msi_index; /* Vector is useless at this point. */ op.msix_entries[i].vector = -1; i++; diff --git a/include/linux/msi.h b/include/linux/msi.h index 45ec5d07a5f3..b3d3b0bf59fe 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -80,7 +80,6 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc, * @multi_cap: [PCI MSI/X] log2 num of messages supported * @can_mask: [PCI MSI/X] Masking supported? * @is_64: [PCI MSI/X] Address size: 0=32bit 1=64bit - * @entry_nr: [PCI MSI/X] Entry which is described by this descriptor * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq * @mask_pos: [PCI MSI] Mask register position * @mask_base: [PCI MSI-X] Mask register base address @@ -97,7 +96,6 @@ struct pci_msi_desc { u8 can_mask : 1; u8 is_64 : 1; u8 is_virtual : 1; - u16 entry_nr; unsigned default_irq; } msi_attrib; union { -- cgit v1.2.3-59-g8ed1b From 7a823443e9b4ed1ff4a3026d184f09d23fd6d9c9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:20 +0100 Subject: PCI/MSI: Provide MSI_FLAG_MSIX_CONTIGUOUS Provide a domain info flag which makes the core code check for a contiguous MSI-X index on allocation. That's simpler than checking it at some other domain callback in architecture code. Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20211210221814.662401116@linutronix.de --- drivers/pci/msi/irqdomain.c | 16 ++++++++++++++-- include/linux/msi.h | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 422420276d6f..180b2a18dc19 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -89,9 +89,21 @@ static int pci_msi_domain_check_cap(struct irq_domain *domain, if (pci_msi_desc_is_multi_msi(desc) && !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) return 1; - else if (desc->pci.msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) - return -ENOTSUPP; + if (desc->pci.msi_attrib.is_msix) { + if (!(info->flags & MSI_FLAG_PCI_MSIX)) + return -ENOTSUPP; + + if (info->flags & MSI_FLAG_MSIX_CONTIGUOUS) { + unsigned int idx = 0; + + /* Check for gaps in the entry indices */ + for_each_msi_entry(desc, dev) { + if (desc->msi_index != idx++) + return -ENOTSUPP; + } + } + } return 0; } diff --git a/include/linux/msi.h b/include/linux/msi.h index b3d3b0bf59fe..d206239e6fa8 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -362,6 +362,8 @@ enum { MSI_FLAG_LEVEL_CAPABLE = (1 << 6), /* Populate sysfs on alloc() and destroy it on free() */ MSI_FLAG_DEV_SYSFS = (1 << 7), + /* MSI-X entries must be contiguous */ + MSI_FLAG_MSIX_CONTIGUOUS = (1 << 8), }; int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask, -- cgit v1.2.3-59-g8ed1b From cf15f43acaad31dabb2646cef170a506a1d663eb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:23 +0100 Subject: genirq/msi: Provide interface to retrieve Linux interrupt number This allows drivers to retrieve the Linux interrupt number instead of fiddling with MSI descriptors. msi_get_virq() returns the Linux interrupt number or 0 in case that there is no entry for the given MSI index. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221814.780824745@linutronix.de --- include/linux/msi.h | 2 ++ kernel/irq/msi.c | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index d206239e6fa8..7593fc383dba 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -153,6 +153,8 @@ struct msi_device_data { int msi_setup_device_data(struct device *dev); +unsigned int msi_get_virq(struct device *dev, unsigned int index); + /* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) #define dev_to_msi_list(dev) (&(dev)->msi_list) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 8e433f1a24fb..ab5e83f41188 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -105,6 +105,42 @@ int msi_setup_device_data(struct device *dev) return 0; } +/** + * msi_get_virq - Return Linux interrupt number of a MSI interrupt + * @dev: Device to operate on + * @index: MSI interrupt index to look for (0-based) + * + * Return: The Linux interrupt number on success (> 0), 0 if not found + */ +unsigned int msi_get_virq(struct device *dev, unsigned int index) +{ + struct msi_desc *desc; + bool pcimsi; + + if (!dev->msi.data) + return 0; + + pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false; + + for_each_msi_entry(desc, dev) { + /* PCI-MSI has only one descriptor for multiple interrupts. */ + if (pcimsi) { + if (desc->irq && index < desc->nvec_used) + return desc->irq + index; + break; + } + + /* + * PCI-MSIX and platform MSI use a descriptor per + * interrupt. + */ + if (desc->msi_index == index) + return desc->irq; + } + return 0; +} +EXPORT_SYMBOL_GPL(msi_get_virq); + #ifdef CONFIG_SYSFS static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) -- cgit v1.2.3-59-g8ed1b From d86a6d47bcc6b41fe2a4e13313d66a772d00382f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:34 +0100 Subject: bus: fsl-mc: fsl-mc-allocator: Rework MSI handling Storing a pointer to the MSI descriptor just to track the Linux interrupt number is daft. Just store the interrupt number and be done with it. Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211210221815.207838579@linutronix.de --- drivers/bus/fsl-mc/dprc-driver.c | 8 ++++---- drivers/bus/fsl-mc/fsl-mc-allocator.c | 9 ++------- drivers/bus/fsl-mc/fsl-mc-msi.c | 6 +++--- drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c | 4 ++-- drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c | 4 +--- drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c | 5 ++--- drivers/soc/fsl/dpio/dpio-driver.c | 8 ++++---- drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c | 4 ++-- include/linux/fsl/mc.h | 4 ++-- 9 files changed, 22 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 315e830b6ecd..5e70f9775a0e 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -400,7 +400,7 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); struct fsl_mc_io *mc_io = mc_dev->mc_io; - struct msi_desc *msi_desc = mc_dev->irqs[0]->msi_desc; + int irq = mc_dev->irqs[0]->virq; dev_dbg(dev, "DPRC IRQ %d triggered on CPU %u\n", irq_num, smp_processor_id()); @@ -409,7 +409,7 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) return IRQ_HANDLED; mutex_lock(&mc_bus->scan_mutex); - if (!msi_desc || msi_desc->irq != (u32)irq_num) + if (irq != (u32)irq_num) goto out; status = 0; @@ -521,7 +521,7 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) * function that programs the MSI physically in the device */ error = devm_request_threaded_irq(&mc_dev->dev, - irq->msi_desc->irq, + irq->virq, dprc_irq0_handler, dprc_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, @@ -771,7 +771,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) (void)disable_dprc_irq(mc_dev); - devm_free_irq(&mc_dev->dev, irq->msi_desc->irq, &mc_dev->dev); + devm_free_irq(&mc_dev->dev, irq->virq, &mc_dev->dev); fsl_mc_free_irqs(mc_dev); } diff --git a/drivers/bus/fsl-mc/fsl-mc-allocator.c b/drivers/bus/fsl-mc/fsl-mc-allocator.c index 3370b63bad5f..dced427ca8ba 100644 --- a/drivers/bus/fsl-mc/fsl-mc-allocator.c +++ b/drivers/bus/fsl-mc/fsl-mc-allocator.c @@ -350,7 +350,6 @@ int fsl_mc_populate_irq_pool(struct fsl_mc_device *mc_bus_dev, unsigned int irq_count) { unsigned int i; - struct msi_desc *msi_desc; struct fsl_mc_device_irq *irq_resources; struct fsl_mc_device_irq *mc_dev_irq; int error; @@ -388,16 +387,12 @@ int fsl_mc_populate_irq_pool(struct fsl_mc_device *mc_bus_dev, mc_dev_irq->resource.type = res_pool->type; mc_dev_irq->resource.data = mc_dev_irq; mc_dev_irq->resource.parent_pool = res_pool; + mc_dev_irq->virq = msi_get_virq(&mc_bus_dev->dev, i); + mc_dev_irq->resource.id = mc_dev_irq->virq; INIT_LIST_HEAD(&mc_dev_irq->resource.node); list_add_tail(&mc_dev_irq->resource.node, &res_pool->free_list); } - for_each_msi_entry(msi_desc, &mc_bus_dev->dev) { - mc_dev_irq = &irq_resources[msi_desc->msi_index]; - mc_dev_irq->msi_desc = msi_desc; - mc_dev_irq->resource.id = msi_desc->irq; - } - res_pool->max_count = irq_count; res_pool->free_count = irq_count; mc_bus->irq_resources = irq_resources; diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c index 2ffb70550a39..4823947d7d6f 100644 --- a/drivers/bus/fsl-mc/fsl-mc-msi.c +++ b/drivers/bus/fsl-mc/fsl-mc-msi.c @@ -58,11 +58,11 @@ static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info) } static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev, - struct fsl_mc_device_irq *mc_dev_irq) + struct fsl_mc_device_irq *mc_dev_irq, + struct msi_desc *msi_desc) { int error; struct fsl_mc_device *owner_mc_dev = mc_dev_irq->mc_dev; - struct msi_desc *msi_desc = mc_dev_irq->msi_desc; struct dprc_irq_cfg irq_cfg; /* @@ -129,7 +129,7 @@ static void fsl_mc_msi_write_msg(struct irq_data *irq_data, /* * Program the MSI (paddr, value) pair in the device: */ - __fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq); + __fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq, msi_desc); } static void fsl_mc_msi_update_chip_ops(struct msi_domain_info *info) diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 8e643567abce..2c53f957e373 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -4246,7 +4246,7 @@ static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) } irq = ls_dev->irqs[0]; - err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, + err = devm_request_threaded_irq(&ls_dev->dev, irq->virq, NULL, dpni_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, dev_name(&ls_dev->dev), &ls_dev->dev); @@ -4273,7 +4273,7 @@ static int dpaa2_eth_setup_irqs(struct fsl_mc_device *ls_dev) return 0; free_irq: - devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); + devm_free_irq(&ls_dev->dev, irq->virq, &ls_dev->dev); free_mc_irq: fsl_mc_free_irqs(ls_dev); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c index 32b5faa87bb8..5f5f8c53c4a0 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c @@ -129,7 +129,6 @@ static irqreturn_t dpaa2_ptp_irq_handler_thread(int irq, void *priv) static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev) { struct device *dev = &mc_dev->dev; - struct fsl_mc_device_irq *irq; struct ptp_qoriq *ptp_qoriq; struct device_node *node; void __iomem *base; @@ -177,8 +176,7 @@ static int dpaa2_ptp_probe(struct fsl_mc_device *mc_dev) goto err_unmap; } - irq = mc_dev->irqs[0]; - ptp_qoriq->irq = irq->msi_desc->irq; + ptp_qoriq->irq = mc_dev->irqs[0]->virq; err = request_threaded_irq(ptp_qoriq->irq, NULL, dpaa2_ptp_irq_handler_thread, diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index d039457928b0..084cc4d2d876 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1553,8 +1553,7 @@ static int dpaa2_switch_setup_irqs(struct fsl_mc_device *sw_dev) irq = sw_dev->irqs[DPSW_IRQ_INDEX_IF]; - err = devm_request_threaded_irq(dev, irq->msi_desc->irq, - NULL, + err = devm_request_threaded_irq(dev, irq->virq, NULL, dpaa2_switch_irq0_handler_thread, IRQF_NO_SUSPEND | IRQF_ONESHOT, dev_name(dev), dev); @@ -1580,7 +1579,7 @@ static int dpaa2_switch_setup_irqs(struct fsl_mc_device *sw_dev) return 0; free_devm_irq: - devm_free_irq(dev, irq->msi_desc->irq, dev); + devm_free_irq(dev, irq->virq, dev); free_irq: fsl_mc_free_irqs(sw_dev); return err; diff --git a/drivers/soc/fsl/dpio/dpio-driver.c b/drivers/soc/fsl/dpio/dpio-driver.c index dd948889eeab..5a2edc48dd79 100644 --- a/drivers/soc/fsl/dpio/dpio-driver.c +++ b/drivers/soc/fsl/dpio/dpio-driver.c @@ -88,7 +88,7 @@ static void unregister_dpio_irq_handlers(struct fsl_mc_device *dpio_dev) irq = dpio_dev->irqs[0]; /* clear the affinity hint */ - irq_set_affinity_hint(irq->msi_desc->irq, NULL); + irq_set_affinity_hint(irq->virq, NULL); } static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) @@ -98,7 +98,7 @@ static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) irq = dpio_dev->irqs[0]; error = devm_request_irq(&dpio_dev->dev, - irq->msi_desc->irq, + irq->virq, dpio_irq_handler, 0, dev_name(&dpio_dev->dev), @@ -111,10 +111,10 @@ static int register_dpio_irq_handlers(struct fsl_mc_device *dpio_dev, int cpu) } /* set the affinity hint */ - if (irq_set_affinity_hint(irq->msi_desc->irq, cpumask_of(cpu))) + if (irq_set_affinity_hint(irq->virq, cpumask_of(cpu))) dev_err(&dpio_dev->dev, "irq_set_affinity failed irq %d cpu %d\n", - irq->msi_desc->irq, cpu); + irq->virq, cpu); return 0; } diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c index 77e584093a23..7b428eac3d3e 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc_intr.c @@ -67,7 +67,7 @@ static int vfio_set_trigger(struct vfio_fsl_mc_device *vdev, int hwirq; int ret; - hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + hwirq = vdev->mc_dev->irqs[index]->virq; if (irq->trigger) { free_irq(hwirq, irq); kfree(irq->name); @@ -137,7 +137,7 @@ static int vfio_fsl_mc_set_irq_trigger(struct vfio_fsl_mc_device *vdev, return vfio_set_trigger(vdev, index, fd); } - hwirq = vdev->mc_dev->irqs[index]->msi_desc->irq; + hwirq = vdev->mc_dev->irqs[index]->virq; irq = &vdev->mc_irqs[index]; diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index e026f6c48b49..7b6c42bfb660 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -91,13 +91,13 @@ struct fsl_mc_resource { /** * struct fsl_mc_device_irq - MC object device message-based interrupt - * @msi_desc: pointer to MSI descriptor allocated by fsl_mc_msi_alloc_descs() + * @virq: Linux virtual interrupt number * @mc_dev: MC object device that owns this interrupt * @dev_irq_index: device-relative IRQ index * @resource: MC generic resource associated with the interrupt */ struct fsl_mc_device_irq { - struct msi_desc *msi_desc; + unsigned int virq; struct fsl_mc_device *mc_dev; u8 dev_irq_index; struct fsl_mc_resource resource; -- cgit v1.2.3-59-g8ed1b From 89e0032ec201f76c86d6e3e6f94574dfb8e39b71 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 10 Dec 2021 23:19:35 +0100 Subject: soc: ti: ti_sci_inta_msi: Get rid of ti_sci_inta_msi_get_virq() Just use the core function msi_get_virq(). Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Acked-by: Arnd Bergmann Acked-by: Vinod Koul Acked-by: Nishanth Menon Link: https://lore.kernel.org/r/20211210221815.269468319@linutronix.de --- drivers/dma/ti/k3-udma-private.c | 6 ++---- drivers/dma/ti/k3-udma.c | 10 ++++------ drivers/soc/ti/k3-ringacc.c | 2 +- drivers/soc/ti/ti_sci_inta_msi.c | 12 ------------ include/linux/soc/ti/ti_sci_inta_msi.h | 1 - 5 files changed, 7 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index aada84f40723..d4f1e4e9603a 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -168,8 +168,7 @@ int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id + - oes->pktdma_tchan_flow); + return msi_get_virq(ud->dev, udma_tflow_id + oes->pktdma_tchan_flow); } EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq); @@ -177,7 +176,6 @@ int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id + - oes->pktdma_rchan_flow); + return msi_get_virq(ud->dev, udma_rflow_id + oes->pktdma_rchan_flow); } EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 20edd0bf3e8b..86f7a0ae2648 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -2313,8 +2313,7 @@ static int udma_alloc_chan_resources(struct dma_chan *chan) /* Event from UDMA (TR events) only needed for slave TR mode channels */ if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", irq_udma_idx); @@ -2486,7 +2485,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; } - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); @@ -2503,8 +2502,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) /* Event from BCDMA (TR events) only needed for slave channels */ if (is_slave_direction(uc->config.dir)) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", irq_udma_idx); @@ -2672,7 +2670,7 @@ static int pktdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c index 26159a5eef88..56be39161489 100644 --- a/drivers/soc/ti/k3-ringacc.c +++ b/drivers/soc/ti/k3-ringacc.c @@ -647,7 +647,7 @@ int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) if (!ring) return -EINVAL; - irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); + irq_num = msi_get_virq(ring->parent->dev, ring->ring_id); if (irq_num <= 0) irq_num = -EINVAL; return irq_num; diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c index 9d23daac74c3..b68cc6d042b2 100644 --- a/drivers/soc/ti/ti_sci_inta_msi.c +++ b/drivers/soc/ti/ti_sci_inta_msi.c @@ -148,15 +148,3 @@ void ti_sci_inta_msi_domain_free_irqs(struct device *dev) ti_sci_inta_msi_free_descs(dev); } EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_free_irqs); - -unsigned int ti_sci_inta_msi_get_virq(struct device *dev, u32 dev_index) -{ - struct msi_desc *desc; - - for_each_msi_entry(desc, dev) - if (desc->msi_index == dev_index) - return desc->irq; - - return -ENODEV; -} -EXPORT_SYMBOL_GPL(ti_sci_inta_msi_get_virq); diff --git a/include/linux/soc/ti/ti_sci_inta_msi.h b/include/linux/soc/ti/ti_sci_inta_msi.h index e3aa8b14612e..25ea78a8ea5c 100644 --- a/include/linux/soc/ti/ti_sci_inta_msi.h +++ b/include/linux/soc/ti/ti_sci_inta_msi.h @@ -18,6 +18,5 @@ struct irq_domain struct irq_domain *parent); int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev, struct ti_sci_resource *res); -unsigned int ti_sci_inta_msi_get_virq(struct device *dev, u32 index); void ti_sci_inta_msi_domain_free_irqs(struct device *dev); #endif /* __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H */ -- cgit v1.2.3-59-g8ed1b From 125282cd4f33ecd53a24ae4807409da0e5e90fd4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:04 +0100 Subject: genirq/msi: Move descriptor list to struct msi_device_data It's only required when MSI is in use. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.650487479@linutronix.de --- drivers/base/core.c | 3 --- include/linux/device.h | 4 ---- include/linux/msi.h | 4 +++- kernel/irq/msi.c | 5 ++++- 4 files changed, 7 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index f26c668092d6..f8987867789f 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2874,9 +2874,6 @@ void device_initialize(struct device *dev) INIT_LIST_HEAD(&dev->devres_head); device_pm_init(dev); set_dev_node(dev, NUMA_NO_NODE); -#ifdef CONFIG_GENERIC_MSI_IRQ - INIT_LIST_HEAD(&dev->msi_list); -#endif INIT_LIST_HEAD(&dev->links.consumers); INIT_LIST_HEAD(&dev->links.suppliers); INIT_LIST_HEAD(&dev->links.defer_sync); diff --git a/include/linux/device.h b/include/linux/device.h index f0033cd93631..93459724dcde 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -423,7 +423,6 @@ struct dev_msi_info { * @pins: For device pin management. * See Documentation/driver-api/pin-control.rst for details. * @msi: MSI related data - * @msi_list: Hosts MSI descriptors * @numa_node: NUMA node this device is close to. * @dma_ops: DMA mapping operations for this device. * @dma_mask: Dma mask (if dma'ble device). @@ -519,9 +518,6 @@ struct device { struct dev_pin_info *pins; #endif struct dev_msi_info msi; -#ifdef CONFIG_GENERIC_MSI_IRQ - struct list_head msi_list; -#endif #ifdef CONFIG_DMA_OPS const struct dma_map_ops *dma_ops; #endif diff --git a/include/linux/msi.h b/include/linux/msi.h index 7593fc383dba..4223e47103ed 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -144,11 +144,13 @@ struct msi_desc { * @properties: MSI properties which are interesting to drivers * @attrs: Pointer to the sysfs attribute group * @platform_data: Platform-MSI specific data + * @list: List of MSI descriptors associated to the device */ struct msi_device_data { unsigned long properties; const struct attribute_group **attrs; struct platform_msi_priv_data *platform_data; + struct list_head list; }; int msi_setup_device_data(struct device *dev); @@ -157,7 +159,7 @@ unsigned int msi_get_virq(struct device *dev, unsigned int index); /* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) -#define dev_to_msi_list(dev) (&(dev)->msi_list) +#define dev_to_msi_list(dev) (&(dev)->msi.data->list) #define first_msi_entry(dev) \ list_first_entry(dev_to_msi_list((dev)), struct msi_desc, list) #define for_each_msi_entry(desc, dev) \ diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index ab5e83f41188..c66787daee57 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -75,7 +75,9 @@ EXPORT_SYMBOL_GPL(get_cached_msi_msg); static void msi_device_data_release(struct device *dev, void *res) { - WARN_ON_ONCE(!list_empty(&dev->msi_list)); + struct msi_device_data *md = res; + + WARN_ON_ONCE(!list_empty(&md->list)); dev->msi.data = NULL; } @@ -100,6 +102,7 @@ int msi_setup_device_data(struct device *dev) if (!md) return -ENOMEM; + INIT_LIST_HEAD(&md->list); dev->msi.data = md; devres_add(dev, md); return 0; -- cgit v1.2.3-59-g8ed1b From b5f687f97d1e112493fe0447a1fb09fbd93c334b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:05 +0100 Subject: genirq/msi: Add mutex for MSI list protection For upcoming runtime extensions of MSI-X interrupts it's required to protect the MSI descriptor list. Add a mutex to struct msi_device_data and provide lock/unlock functions. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.708877269@linutronix.de --- include/linux/msi.h | 5 +++++ kernel/irq/msi.c | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 4223e47103ed..2cf6c530588d 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -3,6 +3,7 @@ #define LINUX_MSI_H #include +#include #include #include @@ -145,17 +146,21 @@ struct msi_desc { * @attrs: Pointer to the sysfs attribute group * @platform_data: Platform-MSI specific data * @list: List of MSI descriptors associated to the device + * @mutex: Mutex protecting the MSI list */ struct msi_device_data { unsigned long properties; const struct attribute_group **attrs; struct platform_msi_priv_data *platform_data; struct list_head list; + struct mutex mutex; }; int msi_setup_device_data(struct device *dev); unsigned int msi_get_virq(struct device *dev, unsigned int index); +void msi_lock_descs(struct device *dev); +void msi_unlock_descs(struct device *dev); /* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index c66787daee57..97ec245803f0 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -103,11 +103,32 @@ int msi_setup_device_data(struct device *dev) return -ENOMEM; INIT_LIST_HEAD(&md->list); + mutex_init(&md->mutex); dev->msi.data = md; devres_add(dev, md); return 0; } +/** + * msi_lock_descs - Lock the MSI descriptor storage of a device + * @dev: Device to operate on + */ +void msi_lock_descs(struct device *dev) +{ + mutex_lock(&dev->msi.data->mutex); +} +EXPORT_SYMBOL_GPL(msi_lock_descs); + +/** + * msi_unlock_descs - Unlock the MSI descriptor storage of a device + * @dev: Device to operate on + */ +void msi_unlock_descs(struct device *dev) +{ + mutex_unlock(&dev->msi.data->mutex); +} +EXPORT_SYMBOL_GPL(msi_unlock_descs); + /** * msi_get_virq - Return Linux interrupt number of a MSI interrupt * @dev: Device to operate on -- cgit v1.2.3-59-g8ed1b From 0f62d941acf9ac3b6025692ce649b1f282b89e7f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:07 +0100 Subject: genirq/msi: Provide msi_domain_alloc/free_irqs_descs_locked() Usage sites which do allocations of the MSI descriptors before invoking msi_domain_alloc_irqs() require to lock the MSI decriptors accross the operation. Provide entry points which can be called with the MSI mutex held and lock the mutex in the existing entry points. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.765371053@linutronix.de --- include/linux/msi.h | 3 +++ kernel/irq/msi.c | 74 +++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 2cf6c530588d..69c588efe85b 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -383,9 +383,12 @@ struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode, struct irq_domain *parent); int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec); +int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev, + int nvec); int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec); void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); +void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev); void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev); struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 97ec245803f0..3b21e99bb793 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -672,10 +672,8 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, dev_to_node(dev), &arg, false, desc->affinity); - if (virq < 0) { - ret = msi_handle_pci_fail(domain, desc, allocated); - goto cleanup; - } + if (virq < 0) + return msi_handle_pci_fail(domain, desc, allocated); for (i = 0; i < desc->nvec_used; i++) { irq_set_msi_desc_off(virq, i, desc); @@ -709,7 +707,7 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, } ret = irq_domain_activate_irq(irq_data, can_reserve); if (ret) - goto cleanup; + return ret; } skip_activate: @@ -724,38 +722,63 @@ skip_activate: } } return 0; - -cleanup: - msi_domain_free_irqs(domain, dev); - return ret; } /** - * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain + * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain * @domain: The domain to allocate from * @dev: Pointer to device struct of the device for which the interrupts * are allocated * @nvec: The number of interrupts to allocate * + * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() + * pair. Use this for MSI irqdomains which implement their own vector + * allocation/free. + * * Return: %0 on success or an error code. */ -int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, - int nvec) +int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev, + int nvec) { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; int ret; + lockdep_assert_held(&dev->msi.data->mutex); + ret = ops->domain_alloc_irqs(domain, dev, nvec); if (ret) - return ret; + goto cleanup; if (!(info->flags & MSI_FLAG_DEV_SYSFS)) return 0; ret = msi_device_populate_sysfs(dev); if (ret) - msi_domain_free_irqs(domain, dev); + goto cleanup; + return 0; + +cleanup: + msi_domain_free_irqs_descs_locked(domain, dev); + return ret; +} + +/** + * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain + * @domain: The domain to allocate from + * @dev: Pointer to device struct of the device for which the interrupts + * are allocated + * @nvec: The number of interrupts to allocate + * + * Return: %0 on success or an error code. + */ +int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec) +{ + int ret; + + msi_lock_descs(dev); + ret = msi_domain_alloc_irqs_descs_locked(domain, dev, nvec); + msi_unlock_descs(dev); return ret; } @@ -785,21 +808,40 @@ void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) } /** - * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev + * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev * @domain: The domain to managing the interrupts * @dev: Pointer to device struct of the device for which the interrupts * are free + * + * Must be invoked from within a msi_lock_descs() / msi_unlock_descs() + * pair. Use this for MSI irqdomains which implement their own vector + * allocation. */ -void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) +void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev) { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; + lockdep_assert_held(&dev->msi.data->mutex); + if (info->flags & MSI_FLAG_DEV_SYSFS) msi_device_destroy_sysfs(dev); ops->domain_free_irqs(domain, dev); } +/** + * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev + * @domain: The domain to managing the interrupts + * @dev: Pointer to device struct of the device for which the interrupts + * are free + */ +void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) +{ + msi_lock_descs(dev); + msi_domain_free_irqs_descs_locked(domain, dev); + msi_unlock_descs(dev); +} + /** * msi_get_domain_info - Get the MSI interrupt domain info for @domain * @domain: The interrupt domain to retrieve data from -- cgit v1.2.3-59-g8ed1b From 1046f71d7268b1680d7b044dea83c664403f6302 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:08 +0100 Subject: genirq/msi: Provide a set of advanced MSI accessors and iterators In preparation for dynamic handling of MSI-X interrupts provide a new set of MSI descriptor accessor functions and iterators. They are benefitial per se as they allow to cleanup quite some code in various MSI domain implementations. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.818635078@linutronix.de --- include/linux/msi.h | 33 ++++++++++++++++++ kernel/irq/msi.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 69c588efe85b..703221f7e9ea 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -140,6 +140,18 @@ struct msi_desc { struct pci_msi_desc pci; }; +/* + * Filter values for the MSI descriptor iterators and accessor functions. + */ +enum msi_desc_filter { + /* All descriptors */ + MSI_DESC_ALL, + /* Descriptors which have no interrupt associated */ + MSI_DESC_NOTASSOCIATED, + /* Descriptors which have an interrupt associated */ + MSI_DESC_ASSOCIATED, +}; + /** * msi_device_data - MSI per device data * @properties: MSI properties which are interesting to drivers @@ -147,6 +159,7 @@ struct msi_desc { * @platform_data: Platform-MSI specific data * @list: List of MSI descriptors associated to the device * @mutex: Mutex protecting the MSI list + * @__next: Cached pointer to the next entry for iterators */ struct msi_device_data { unsigned long properties; @@ -154,6 +167,7 @@ struct msi_device_data { struct platform_msi_priv_data *platform_data; struct list_head list; struct mutex mutex; + struct msi_desc *__next; }; int msi_setup_device_data(struct device *dev); @@ -162,6 +176,25 @@ unsigned int msi_get_virq(struct device *dev, unsigned int index); void msi_lock_descs(struct device *dev); void msi_unlock_descs(struct device *dev); +struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter); +struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter); + +/** + * msi_for_each_desc - Iterate the MSI descriptors + * + * @desc: struct msi_desc pointer used as iterator + * @dev: struct device pointer - device to iterate + * @filter: Filter for descriptor selection + * + * Notes: + * - The loop must be protected with a msi_lock_descs()/msi_unlock_descs() + * pair. + * - It is safe to remove a retrieved MSI descriptor in the loop. + */ +#define msi_for_each_desc(desc, dev, filter) \ + for ((desc) = msi_first_desc((dev), (filter)); (desc); \ + (desc) = msi_next_desc((dev), (filter))) + /* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) #define dev_to_msi_list(dev) (&(dev)->msi.data->list) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 3b21e99bb793..bc67b2cafc9d 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -125,10 +125,106 @@ EXPORT_SYMBOL_GPL(msi_lock_descs); */ void msi_unlock_descs(struct device *dev) { + /* Clear the next pointer which was cached by the iterator */ + dev->msi.data->__next = NULL; mutex_unlock(&dev->msi.data->mutex); } EXPORT_SYMBOL_GPL(msi_unlock_descs); +static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) +{ + switch (filter) { + case MSI_DESC_ALL: + return true; + case MSI_DESC_NOTASSOCIATED: + return !desc->irq; + case MSI_DESC_ASSOCIATED: + return !!desc->irq; + } + WARN_ON_ONCE(1); + return false; +} + +static struct msi_desc *msi_find_first_desc(struct device *dev, enum msi_desc_filter filter) +{ + struct msi_desc *desc; + + list_for_each_entry(desc, dev_to_msi_list(dev), list) { + if (msi_desc_match(desc, filter)) + return desc; + } + return NULL; +} + +/** + * msi_first_desc - Get the first MSI descriptor of a device + * @dev: Device to operate on + * @filter: Descriptor state filter + * + * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs() + * must be invoked before the call. + * + * Return: Pointer to the first MSI descriptor matching the search + * criteria, NULL if none found. + */ +struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter) +{ + struct msi_desc *desc; + + if (WARN_ON_ONCE(!dev->msi.data)) + return NULL; + + lockdep_assert_held(&dev->msi.data->mutex); + + desc = msi_find_first_desc(dev, filter); + dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; + return desc; +} +EXPORT_SYMBOL_GPL(msi_first_desc); + +static struct msi_desc *__msi_next_desc(struct device *dev, enum msi_desc_filter filter, + struct msi_desc *from) +{ + struct msi_desc *desc = from; + + list_for_each_entry_from(desc, dev_to_msi_list(dev), list) { + if (msi_desc_match(desc, filter)) + return desc; + } + return NULL; +} + +/** + * msi_next_desc - Get the next MSI descriptor of a device + * @dev: Device to operate on + * + * The first invocation of msi_next_desc() has to be preceeded by a + * successful incovation of __msi_first_desc(). Consecutive invocations are + * only valid if the previous one was successful. All these operations have + * to be done within the same MSI mutex held region. + * + * Return: Pointer to the next MSI descriptor matching the search + * criteria, NULL if none found. + */ +struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter) +{ + struct msi_device_data *data = dev->msi.data; + struct msi_desc *desc; + + if (WARN_ON_ONCE(!data)) + return NULL; + + lockdep_assert_held(&data->mutex); + + if (!data->__next) + return NULL; + + desc = __msi_next_desc(dev, filter, data->__next); + dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; + return desc; +} +EXPORT_SYMBOL_GPL(msi_next_desc); + /** * msi_get_virq - Return Linux interrupt number of a MSI interrupt * @dev: Device to operate on -- cgit v1.2.3-59-g8ed1b From 602905253607ba892336f7bba8bb45b5be819d87 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:10 +0100 Subject: genirq/msi: Provide msi_alloc_msi_desc() and a simple allocator Provide msi_alloc_msi_desc() which takes a template MSI descriptor for initializing a newly allocated descriptor. This allows to simplify various usage sites of alloc_msi_entry() and moves the storage handling into the core code. For simple cases where only a linear vector space is required provide msi_add_simple_msi_descs() which just allocates a linear range of MSI descriptors and fills msi_desc::msi_index accordingly. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.873833567@linutronix.de --- include/linux/msi.h | 2 ++ kernel/irq/msi.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 703221f7e9ea..bbb8c1e2c18b 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -247,6 +247,8 @@ static inline void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) } #endif /* CONFIG_PCI_MSI */ +int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc); + struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, const struct irq_affinity_desc *affinity); void free_msi_entry(struct msi_desc *entry); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index bc67b2cafc9d..6ffe75eeba59 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -60,6 +60,65 @@ void free_msi_entry(struct msi_desc *entry) kfree(entry); } +/** + * msi_add_msi_desc - Allocate and initialize a MSI descriptor + * @dev: Pointer to the device for which the descriptor is allocated + * @init_desc: Pointer to an MSI descriptor to initialize the new descriptor + * + * Return: 0 on success or an appropriate failure code. + */ +int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) +{ + struct msi_desc *desc; + + lockdep_assert_held(&dev->msi.data->mutex); + + desc = alloc_msi_entry(dev, init_desc->nvec_used, init_desc->affinity); + if (!desc) + return -ENOMEM; + + /* Copy the MSI index and type specific data to the new descriptor. */ + desc->msi_index = init_desc->msi_index; + desc->pci = init_desc->pci; + + list_add_tail(&desc->list, &dev->msi.data->list); + return 0; +} + +/** + * msi_add_simple_msi_descs - Allocate and initialize MSI descriptors + * @dev: Pointer to the device for which the descriptors are allocated + * @index: Index for the first MSI descriptor + * @ndesc: Number of descriptors to allocate + * + * Return: 0 on success or an appropriate failure code. + */ +static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc) +{ + struct msi_desc *desc, *tmp; + LIST_HEAD(list); + unsigned int i; + + lockdep_assert_held(&dev->msi.data->mutex); + + for (i = 0; i < ndesc; i++) { + desc = alloc_msi_entry(dev, 1, NULL); + if (!desc) + goto fail; + desc->msi_index = index + i; + list_add_tail(&desc->list, &list); + } + list_splice_tail(&list, &dev->msi.data->list); + return 0; + +fail: + list_for_each_entry_safe(desc, tmp, &list, list) { + list_del(&desc->list); + free_msi_entry(desc); + } + return -ENOMEM; +} + void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { *msg = entry->msg; -- cgit v1.2.3-59-g8ed1b From 645474e2cee450131e8b8d8a69a5d9bbabd43f3f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:12 +0100 Subject: genirq/msi: Provide domain flags to allocate/free MSI descriptors automatically Provide domain info flags which tell the core to allocate simple descriptors or to free descriptors when the interrupts are freed and implement the required functionality. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210747.928198636@linutronix.de --- include/linux/msi.h | 17 +++++++++++++++++ kernel/irq/msi.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index bbb8c1e2c18b..17e47ab8d57a 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -105,6 +105,8 @@ struct pci_msi_desc { }; }; +#define MSI_MAX_INDEX ((unsigned int)USHRT_MAX) + /** * struct msi_desc - Descriptor structure for MSI based interrupts * @list: List head for management @@ -248,6 +250,17 @@ static inline void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) #endif /* CONFIG_PCI_MSI */ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc); +void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, + unsigned int first_index, unsigned int last_index); + +/** + * msi_free_msi_descs - Free MSI descriptors of a device + * @dev: Device to free the descriptors + */ +static inline void msi_free_msi_descs(struct device *dev) +{ + msi_free_msi_descs_range(dev, MSI_DESC_ALL, 0, MSI_MAX_INDEX); +} struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, const struct irq_affinity_desc *affinity); @@ -408,6 +421,10 @@ enum { MSI_FLAG_DEV_SYSFS = (1 << 7), /* MSI-X entries must be contiguous */ MSI_FLAG_MSIX_CONTIGUOUS = (1 << 8), + /* Allocate simple MSI descriptors */ + MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS = (1 << 9), + /* Free MSI descriptors */ + MSI_FLAG_FREE_MSI_DESCS = (1 << 10), }; int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask, diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 6ffe75eeba59..b511dc1a0219 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -119,6 +119,32 @@ fail: return -ENOMEM; } +/** + * msi_free_msi_descs_range - Free MSI descriptors of a device + * @dev: Device to free the descriptors + * @filter: Descriptor state filter + * @first_index: Index to start freeing from + * @last_index: Last index to be freed + */ +void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, + unsigned int first_index, unsigned int last_index) +{ + struct msi_desc *desc; + + lockdep_assert_held(&dev->msi.data->mutex); + + msi_for_each_desc(desc, dev, filter) { + /* + * Stupid for now to handle MSI device domain until the + * storage is switched over to an xarray. + */ + if (desc->msi_index < first_index || desc->msi_index > last_index) + continue; + list_del(&desc->list); + free_msi_entry(desc); + } +} + void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { *msg = entry->msg; @@ -879,6 +905,16 @@ skip_activate: return 0; } +static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info, + struct device *dev, + unsigned int num_descs) +{ + if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS)) + return 0; + + return msi_add_simple_msi_descs(dev, 0, num_descs); +} + /** * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain * @domain: The domain to allocate from @@ -901,6 +937,10 @@ int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device lockdep_assert_held(&dev->msi.data->mutex); + ret = msi_domain_add_simple_msi_descs(info, dev, nvec); + if (ret) + return ret; + ret = ops->domain_alloc_irqs(domain, dev, nvec); if (ret) goto cleanup; @@ -962,6 +1002,13 @@ void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) } } +static void msi_domain_free_msi_descs(struct msi_domain_info *info, + struct device *dev) +{ + if (info->flags & MSI_FLAG_FREE_MSI_DESCS) + msi_free_msi_descs(dev); +} + /** * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev * @domain: The domain to managing the interrupts @@ -982,6 +1029,7 @@ void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device if (info->flags & MSI_FLAG_DEV_SYSFS) msi_device_destroy_sysfs(dev); ops->domain_free_irqs(domain, dev); + msi_domain_free_msi_descs(info, dev); } /** -- cgit v1.2.3-59-g8ed1b From 7ad321a5eadb52b4af1c577dda51783e08235ea7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:37 +0100 Subject: soc: ti: ti_sci_inta_msi: Remove ti_sci_inta_msi_domain_free_irqs() The function has no users and is pointless now that the core frees the MSI descriptors, which means potential users can just use msi_domain_free_irqs(). Signed-off-by: Thomas Gleixner Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210748.793119155@linutronix.de --- drivers/soc/ti/ti_sci_inta_msi.c | 6 ------ include/linux/soc/ti/ti_sci_inta_msi.h | 1 - 2 files changed, 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c index 31399118fcc6..991c78b34745 100644 --- a/drivers/soc/ti/ti_sci_inta_msi.c +++ b/drivers/soc/ti/ti_sci_inta_msi.c @@ -122,9 +122,3 @@ unlock: return ret; } EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_alloc_irqs); - -void ti_sci_inta_msi_domain_free_irqs(struct device *dev) -{ - msi_domain_free_irqs(dev->msi.domain, dev); -} -EXPORT_SYMBOL_GPL(ti_sci_inta_msi_domain_free_irqs); diff --git a/include/linux/soc/ti/ti_sci_inta_msi.h b/include/linux/soc/ti/ti_sci_inta_msi.h index 25ea78a8ea5c..4dba2f2aff6f 100644 --- a/include/linux/soc/ti/ti_sci_inta_msi.h +++ b/include/linux/soc/ti/ti_sci_inta_msi.h @@ -18,5 +18,4 @@ struct irq_domain struct irq_domain *parent); int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev, struct ti_sci_resource *res); -void ti_sci_inta_msi_domain_free_irqs(struct device *dev); #endif /* __INCLUDE_LINUX_IRQCHIP_TI_SCI_INTA_H */ -- cgit v1.2.3-59-g8ed1b From ef8dd01538ea2553ab101ddce6a85a321406d9c0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:44 +0100 Subject: genirq/msi: Make interrupt allocation less convoluted There is no real reason to do several loops over the MSI descriptors instead of just doing one loop. In case of an error everything is undone anyway so it does not matter whether it's a partial or a full rollback. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210749.010234767@linutronix.de --- .clang-format | 1 - include/linux/msi.h | 6 --- kernel/irq/msi.c | 129 ++++++++++++++++++++++++++++------------------------ 3 files changed, 69 insertions(+), 67 deletions(-) (limited to 'include/linux') diff --git a/.clang-format b/.clang-format index 15d4eaabc6b5..fa959436bcfd 100644 --- a/.clang-format +++ b/.clang-format @@ -216,7 +216,6 @@ ForEachMacros: - 'for_each_migratetype_order' - 'for_each_msi_entry' - 'for_each_msi_entry_safe' - - 'for_each_msi_vector' - 'for_each_net' - 'for_each_net_continue_reverse' - 'for_each_netdev' diff --git a/include/linux/msi.h b/include/linux/msi.h index 17e47ab8d57a..e8dd0be17e89 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -206,12 +206,6 @@ struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter); list_for_each_entry((desc), dev_to_msi_list((dev)), list) #define for_each_msi_entry_safe(desc, tmp, dev) \ list_for_each_entry_safe((desc), (tmp), dev_to_msi_list((dev)), list) -#define for_each_msi_vector(desc, __irq, dev) \ - for_each_msi_entry((desc), (dev)) \ - if ((desc)->irq) \ - for (__irq = (desc)->irq; \ - __irq < ((desc)->irq + (desc)->nvec_used); \ - __irq++) #ifdef CONFIG_IRQ_MSI_IOMMU static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 09f34e17e891..bbe36e20a986 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -828,23 +828,74 @@ static int msi_handle_pci_fail(struct irq_domain *domain, struct msi_desc *desc, return allocated ? allocated : -ENOSPC; } +#define VIRQ_CAN_RESERVE 0x01 +#define VIRQ_ACTIVATE 0x02 +#define VIRQ_NOMASK_QUIRK 0x04 + +static int msi_init_virq(struct irq_domain *domain, int virq, unsigned int vflags) +{ + struct irq_data *irqd = irq_domain_get_irq_data(domain, virq); + int ret; + + if (!(vflags & VIRQ_CAN_RESERVE)) { + irqd_clr_can_reserve(irqd); + if (vflags & VIRQ_NOMASK_QUIRK) + irqd_set_msi_nomask_quirk(irqd); + } + + if (!(vflags & VIRQ_ACTIVATE)) + return 0; + + ret = irq_domain_activate_irq(irqd, vflags & VIRQ_CAN_RESERVE); + if (ret) + return ret; + /* + * If the interrupt uses reservation mode, clear the activated bit + * so request_irq() will assign the final vector. + */ + if (vflags & VIRQ_CAN_RESERVE) + irqd_clr_activated(irqd); + return 0; +} + int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec) { struct msi_domain_info *info = domain->host_data; struct msi_domain_ops *ops = info->ops; - struct irq_data *irq_data; - struct msi_desc *desc; msi_alloc_info_t arg = { }; + unsigned int vflags = 0; + struct msi_desc *desc; int allocated = 0; int i, ret, virq; - bool can_reserve; ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg); if (ret) return ret; - for_each_msi_entry(desc, dev) { + /* + * This flag is set by the PCI layer as we need to activate + * the MSI entries before the PCI layer enables MSI in the + * card. Otherwise the card latches a random msi message. + */ + if (info->flags & MSI_FLAG_ACTIVATE_EARLY) + vflags |= VIRQ_ACTIVATE; + + /* + * Interrupt can use a reserved vector and will not occupy + * a real device vector until the interrupt is requested. + */ + if (msi_check_reservation_mode(domain, info, dev)) { + vflags |= VIRQ_CAN_RESERVE; + /* + * MSI affinity setting requires a special quirk (X86) when + * reservation mode is active. + */ + if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) + vflags |= VIRQ_NOMASK_QUIRK; + } + + msi_for_each_desc(desc, dev, MSI_DESC_NOTASSOCIATED) { ops->set_desc(&arg, desc); virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used, @@ -856,49 +907,12 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, for (i = 0; i < desc->nvec_used; i++) { irq_set_msi_desc_off(virq, i, desc); irq_debugfs_copy_devname(virq + i, dev); + ret = msi_init_virq(domain, virq + i, vflags); + if (ret) + return ret; } allocated++; } - - can_reserve = msi_check_reservation_mode(domain, info, dev); - - /* - * This flag is set by the PCI layer as we need to activate - * the MSI entries before the PCI layer enables MSI in the - * card. Otherwise the card latches a random msi message. - */ - if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY)) - goto skip_activate; - - for_each_msi_vector(desc, i, dev) { - if (desc->irq == i) { - virq = desc->irq; - dev_dbg(dev, "irq [%d-%d] for MSI\n", - virq, virq + desc->nvec_used - 1); - } - - irq_data = irq_domain_get_irq_data(domain, i); - if (!can_reserve) { - irqd_clr_can_reserve(irq_data); - if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) - irqd_set_msi_nomask_quirk(irq_data); - } - ret = irq_domain_activate_irq(irq_data, can_reserve); - if (ret) - return ret; - } - -skip_activate: - /* - * If these interrupts use reservation mode, clear the activated bit - * so request_irq() will assign the final vector. - */ - if (can_reserve) { - for_each_msi_vector(desc, i, dev) { - irq_data = irq_domain_get_irq_data(domain, i); - irqd_clr_activated(irq_data); - } - } return 0; } @@ -976,26 +990,21 @@ int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nve void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) { - struct irq_data *irq_data; + struct irq_data *irqd; struct msi_desc *desc; int i; - for_each_msi_vector(desc, i, dev) { - irq_data = irq_domain_get_irq_data(domain, i); - if (irqd_is_activated(irq_data)) - irq_domain_deactivate_irq(irq_data); - } - - for_each_msi_entry(desc, dev) { - /* - * We might have failed to allocate an MSI early - * enough that there is no IRQ associated to this - * entry. If that's the case, don't do anything. - */ - if (desc->irq) { - irq_domain_free_irqs(desc->irq, desc->nvec_used); - desc->irq = 0; + /* Only handle MSI entries which have an interrupt associated */ + msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { + /* Make sure all interrupts are deactivated */ + for (i = 0; i < desc->nvec_used; i++) { + irqd = irq_domain_get_irq_data(domain, desc->irq + i); + if (irqd && irqd_is_activated(irqd)) + irq_domain_deactivate_irq(irqd); } + + irq_domain_free_irqs(desc->irq, desc->nvec_used); + desc->irq = 0; } } -- cgit v1.2.3-59-g8ed1b From cc9a246dbf6bdef56d9eee296a1db52dd0607976 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:47 +0100 Subject: genirq/msi: Mop up old interfaces Get rid of the old iterators, alloc/free functions and adjust the core code accordingly. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210749.117395027@linutronix.de --- include/linux/msi.h | 15 --------------- kernel/irq/msi.c | 31 +++++++++++++++---------------- 2 files changed, 15 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index e8dd0be17e89..b54010ba7b0d 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -197,15 +197,7 @@ struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter); for ((desc) = msi_first_desc((dev), (filter)); (desc); \ (desc) = msi_next_desc((dev), (filter))) -/* Helpers to hide struct msi_desc implementation details */ #define msi_desc_to_dev(desc) ((desc)->dev) -#define dev_to_msi_list(dev) (&(dev)->msi.data->list) -#define first_msi_entry(dev) \ - list_first_entry(dev_to_msi_list((dev)), struct msi_desc, list) -#define for_each_msi_entry(desc, dev) \ - list_for_each_entry((desc), dev_to_msi_list((dev)), list) -#define for_each_msi_entry_safe(desc, tmp, dev) \ - list_for_each_entry_safe((desc), (tmp), dev_to_msi_list((dev)), list) #ifdef CONFIG_IRQ_MSI_IOMMU static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc) @@ -231,10 +223,6 @@ static inline void msi_desc_set_iommu_cookie(struct msi_desc *desc, #endif #ifdef CONFIG_PCI_MSI -#define first_pci_msi_entry(pdev) first_msi_entry(&(pdev)->dev) -#define for_each_pci_msi_entry(desc, pdev) \ - for_each_msi_entry((desc), &(pdev)->dev) - struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc); void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg); #else /* CONFIG_PCI_MSI */ @@ -256,9 +244,6 @@ static inline void msi_free_msi_descs(struct device *dev) msi_free_msi_descs_range(dev, MSI_DESC_ALL, 0, MSI_MAX_INDEX); } -struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, - const struct irq_affinity_desc *affinity); -void free_msi_entry(struct msi_desc *entry); void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 745434efb557..e8c19740ca0c 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -19,8 +19,10 @@ #include "internals.h" +#define dev_to_msi_list(dev) (&(dev)->msi.data->list) + /** - * alloc_msi_entry - Allocate an initialized msi_desc + * msi_alloc_desc - Allocate an initialized msi_desc * @dev: Pointer to the device for which this is allocated * @nvec: The number of vectors used in this entry * @affinity: Optional pointer to an affinity mask array size of @nvec @@ -30,12 +32,11 @@ * * Return: pointer to allocated &msi_desc on success or %NULL on failure */ -struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, - const struct irq_affinity_desc *affinity) +static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, + const struct irq_affinity_desc *affinity) { - struct msi_desc *desc; + struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); - desc = kzalloc(sizeof(*desc), GFP_KERNEL); if (!desc) return NULL; @@ -43,21 +44,19 @@ struct msi_desc *alloc_msi_entry(struct device *dev, int nvec, desc->dev = dev; desc->nvec_used = nvec; if (affinity) { - desc->affinity = kmemdup(affinity, - nvec * sizeof(*desc->affinity), GFP_KERNEL); + desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL); if (!desc->affinity) { kfree(desc); return NULL; } } - return desc; } -void free_msi_entry(struct msi_desc *entry) +static void msi_free_desc(struct msi_desc *desc) { - kfree(entry->affinity); - kfree(entry); + kfree(desc->affinity); + kfree(desc); } /** @@ -73,7 +72,7 @@ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) lockdep_assert_held(&dev->msi.data->mutex); - desc = alloc_msi_entry(dev, init_desc->nvec_used, init_desc->affinity); + desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity); if (!desc) return -ENOMEM; @@ -102,7 +101,7 @@ static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsi lockdep_assert_held(&dev->msi.data->mutex); for (i = 0; i < ndesc; i++) { - desc = alloc_msi_entry(dev, 1, NULL); + desc = msi_alloc_desc(dev, 1, NULL); if (!desc) goto fail; desc->msi_index = index + i; @@ -114,7 +113,7 @@ static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsi fail: list_for_each_entry_safe(desc, tmp, &list, list) { list_del(&desc->list); - free_msi_entry(desc); + msi_free_desc(desc); } return -ENOMEM; } @@ -141,7 +140,7 @@ void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, if (desc->msi_index < first_index || desc->msi_index > last_index) continue; list_del(&desc->list); - free_msi_entry(desc); + msi_free_desc(desc); } } @@ -745,7 +744,7 @@ int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, msi_lock_descs(dev); for (virq = virq_base; virq < virq_base + nvec; virq++) { - desc = alloc_msi_entry(dev, 1, NULL); + desc = msi_alloc_desc(dev, 1, NULL); if (!desc) { ret = -ENOMEM; goto fail; -- cgit v1.2.3-59-g8ed1b From ef3350c53d2aac65cf1c4ecc968bbb1de5f421ea Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:49 +0100 Subject: genirq/msi: Add abuse prevention comment to msi header Hope dies last. Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210749.170847844@linutronix.de --- include/linux/msi.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index b54010ba7b0d..70cc6a555a8e 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -2,6 +2,20 @@ #ifndef LINUX_MSI_H #define LINUX_MSI_H +/* + * This header file contains MSI data structures and functions which are + * only relevant for: + * - Interrupt core code + * - PCI/MSI core code + * - MSI interrupt domain implementations + * - IOMMU, low level VFIO, NTB and other justified exceptions + * dealing with low level MSI details. + * + * Regular device drivers have no business with any of these functions and + * especially storing MSI descriptor pointers in random code is considered + * abuse. The only function which is relevant for drivers is msi_get_virq(). + */ + #include #include #include -- cgit v1.2.3-59-g8ed1b From bf5e758f02fc739589dcc6a3395c3a3eb77b5c90 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:50 +0100 Subject: genirq/msi: Simplify sysfs handling The sysfs handling for MSI is a convoluted maze and it is in the way of supporting dynamic expansion of the MSI-X vectors because it only supports a one off bulk population/free of the sysfs entries. Change it to do: 1) Creating an empty sysfs attribute group when msi_device_data is allocated 2) Populate the entries when the MSI descriptor is initialized 3) Free the entries when a MSI descriptor is detached from a Linux interrupt. 4) Provide functions for the legacy non-irqdomain fallback code to do a bulk population/free. This code won't support dynamic expansion. This makes the code simpler and reduces the number of allocations as the empty attribute group can be shared. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210749.224917330@linutronix.de --- include/linux/msi.h | 23 +++--- kernel/irq/msi.c | 198 ++++++++++++++++++++++++---------------------------- 2 files changed, 103 insertions(+), 118 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 70cc6a555a8e..1a00367d2cfa 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -71,7 +71,7 @@ struct irq_data; struct msi_desc; struct pci_dev; struct platform_msi_priv_data; -struct attribute_group; +struct device_attribute; void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg); #ifdef CONFIG_GENERIC_MSI_IRQ @@ -129,6 +129,7 @@ struct pci_msi_desc { * @dev: Pointer to the device which uses this descriptor * @msg: The last set MSI message cached for reuse * @affinity: Optional pointer to a cpu affinity mask for this descriptor + * @sysfs_attr: Pointer to sysfs device attribute * * @write_msi_msg: Callback that may be called when the MSI message * address or data changes @@ -148,6 +149,9 @@ struct msi_desc { #ifdef CONFIG_IRQ_MSI_IOMMU const void *iommu_cookie; #endif +#ifdef CONFIG_SYSFS + struct device_attribute *sysfs_attrs; +#endif void (*write_msi_msg)(struct msi_desc *entry, void *data); void *write_msi_msg_data; @@ -171,7 +175,6 @@ enum msi_desc_filter { /** * msi_device_data - MSI per device data * @properties: MSI properties which are interesting to drivers - * @attrs: Pointer to the sysfs attribute group * @platform_data: Platform-MSI specific data * @list: List of MSI descriptors associated to the device * @mutex: Mutex protecting the MSI list @@ -179,7 +182,6 @@ enum msi_desc_filter { */ struct msi_device_data { unsigned long properties; - const struct attribute_group **attrs; struct platform_msi_priv_data *platform_data; struct list_head list; struct mutex mutex; @@ -264,14 +266,6 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg); void pci_msi_mask_irq(struct irq_data *data); void pci_msi_unmask_irq(struct irq_data *data); -#ifdef CONFIG_SYSFS -int msi_device_populate_sysfs(struct device *dev); -void msi_device_destroy_sysfs(struct device *dev); -#else /* CONFIG_SYSFS */ -static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } -static inline void msi_device_destroy_sysfs(struct device *dev) { } -#endif /* !CONFIG_SYSFS */ - /* * The arch hooks to setup up msi irqs. Default functions are implemented * as weak symbols so that they /can/ be overriden by architecture specific @@ -285,6 +279,13 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc); void arch_teardown_msi_irq(unsigned int irq); int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); void arch_teardown_msi_irqs(struct pci_dev *dev); +#ifdef CONFIG_SYSFS +int msi_device_populate_sysfs(struct device *dev); +void msi_device_destroy_sysfs(struct device *dev); +#else /* CONFIG_SYSFS */ +static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } +static inline void msi_device_destroy_sysfs(struct device *dev) { } +#endif /* !CONFIG_SYSFS */ #endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ /* diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index e8c19740ca0c..d290e09258bc 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -19,6 +19,7 @@ #include "internals.h" +static inline int msi_sysfs_create_group(struct device *dev); #define dev_to_msi_list(dev) (&(dev)->msi.data->list) /** @@ -178,6 +179,7 @@ static void msi_device_data_release(struct device *dev, void *res) int msi_setup_device_data(struct device *dev) { struct msi_device_data *md; + int ret; if (dev->msi.data) return 0; @@ -186,6 +188,12 @@ int msi_setup_device_data(struct device *dev) if (!md) return -ENOMEM; + ret = msi_sysfs_create_group(dev); + if (ret) { + devres_free(md); + return ret; + } + INIT_LIST_HEAD(&md->list); mutex_init(&md->mutex); dev->msi.data = md; @@ -351,6 +359,20 @@ unsigned int msi_get_virq(struct device *dev, unsigned int index) EXPORT_SYMBOL_GPL(msi_get_virq); #ifdef CONFIG_SYSFS +static struct attribute *msi_dev_attrs[] = { + NULL +}; + +static const struct attribute_group msi_irqs_group = { + .name = "msi_irqs", + .attrs = msi_dev_attrs, +}; + +static inline int msi_sysfs_create_group(struct device *dev) +{ + return devm_device_add_group(dev, &msi_irqs_group); +} + static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -360,97 +382,74 @@ static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi"); } -/** - * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices - * @dev: The device(PCI, platform etc) who will get sysfs entries - */ -static const struct attribute_group **msi_populate_sysfs(struct device *dev) +static void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { - const struct attribute_group **msi_irq_groups; - struct attribute **msi_attrs, *msi_attr; - struct device_attribute *msi_dev_attr; - struct attribute_group *msi_irq_group; - struct msi_desc *entry; - int ret = -ENOMEM; - int num_msi = 0; - int count = 0; + struct device_attribute *attrs = desc->sysfs_attrs; int i; - /* Determine how many msi entries we have */ - msi_for_each_desc(entry, dev, MSI_DESC_ALL) - num_msi += entry->nvec_used; - if (!num_msi) - return NULL; + if (!attrs) + return; - /* Dynamically create the MSI attributes for the device */ - msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL); - if (!msi_attrs) - return ERR_PTR(-ENOMEM); - - msi_for_each_desc(entry, dev, MSI_DESC_ALL) { - for (i = 0; i < entry->nvec_used; i++) { - msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); - if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - - sysfs_attr_init(&msi_dev_attr->attr); - msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", - entry->irq + i); - if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode = 0444; - msi_dev_attr->show = msi_mode_show; - ++count; - } + desc->sysfs_attrs = NULL; + for (i = 0; i < desc->nvec_used; i++) { + if (attrs[i].show) + sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); + kfree(attrs[i].attr.name); } + kfree(attrs); +} - msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); - if (!msi_irq_group) - goto error_attrs; - msi_irq_group->name = "msi_irqs"; - msi_irq_group->attrs = msi_attrs; +static int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) +{ + struct device_attribute *attrs; + int ret, i; - msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL); - if (!msi_irq_groups) - goto error_irq_group; - msi_irq_groups[0] = msi_irq_group; + attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL); + if (!attrs) + return -ENOMEM; - ret = sysfs_create_groups(&dev->kobj, msi_irq_groups); - if (ret) - goto error_irq_groups; - - return msi_irq_groups; - -error_irq_groups: - kfree(msi_irq_groups); -error_irq_group: - kfree(msi_irq_group); -error_attrs: - count = 0; - msi_attr = msi_attrs[count]; - while (msi_attr) { - msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); - kfree(msi_attr->name); - kfree(msi_dev_attr); - ++count; - msi_attr = msi_attrs[count]; + desc->sysfs_attrs = attrs; + for (i = 0; i < desc->nvec_used; i++) { + sysfs_attr_init(&attrs[i].attr); + attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i); + if (!attrs[i].attr.name) { + ret = -ENOMEM; + goto fail; + } + + attrs[i].attr.mode = 0444; + attrs[i].show = msi_mode_show; + + ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name); + if (ret) { + attrs[i].show = NULL; + goto fail; + } } - kfree(msi_attrs); - return ERR_PTR(ret); + return 0; + +fail: + msi_sysfs_remove_desc(dev, desc); + return ret; } +#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS /** * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device * @dev: The device (PCI, platform etc) which will get sysfs entries */ int msi_device_populate_sysfs(struct device *dev) { - const struct attribute_group **group = msi_populate_sysfs(dev); + struct msi_desc *desc; + int ret; - if (IS_ERR(group)) - return PTR_ERR(group); - dev->msi.data->attrs = group; + msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { + if (desc->sysfs_attrs) + continue; + ret = msi_sysfs_populate_desc(dev, desc); + if (ret) + return ret; + } return 0; } @@ -461,28 +460,17 @@ int msi_device_populate_sysfs(struct device *dev) */ void msi_device_destroy_sysfs(struct device *dev) { - const struct attribute_group **msi_irq_groups = dev->msi.data->attrs; - struct device_attribute *dev_attr; - struct attribute **msi_attrs; - int count = 0; - - dev->msi.data->attrs = NULL; - if (!msi_irq_groups) - return; + struct msi_desc *desc; - sysfs_remove_groups(&dev->kobj, msi_irq_groups); - msi_attrs = msi_irq_groups[0]->attrs; - while (msi_attrs[count]) { - dev_attr = container_of(msi_attrs[count], struct device_attribute, attr); - kfree(dev_attr->attr.name); - kfree(dev_attr); - ++count; - } - kfree(msi_attrs); - kfree(msi_irq_groups[0]); - kfree(msi_irq_groups); + msi_for_each_desc(desc, dev, MSI_DESC_ALL) + msi_sysfs_remove_desc(dev, desc); } -#endif +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */ +#else /* CONFIG_SYSFS */ +static inline int msi_sysfs_create_group(struct device *dev) { return 0; } +static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } +static inline void msi_sysfs_remove_desc(struct device *dev, struct msi_desc *desc) { } +#endif /* !CONFIG_SYSFS */ #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN static inline void irq_chip_write_msi_msg(struct irq_data *data, @@ -914,6 +902,12 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, ret = msi_init_virq(domain, virq + i, vflags); if (ret) return ret; + + if (info->flags & MSI_FLAG_DEV_SYSFS) { + ret = msi_sysfs_populate_desc(dev, desc); + if (ret) + return ret; + } } allocated++; } @@ -958,18 +952,7 @@ int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device ret = ops->domain_alloc_irqs(domain, dev, nvec); if (ret) - goto cleanup; - - if (!(info->flags & MSI_FLAG_DEV_SYSFS)) - return 0; - - ret = msi_device_populate_sysfs(dev); - if (ret) - goto cleanup; - return 0; - -cleanup: - msi_domain_free_irqs_descs_locked(domain, dev); + msi_domain_free_irqs_descs_locked(domain, dev); return ret; } @@ -994,6 +977,7 @@ int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nve void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) { + struct msi_domain_info *info = domain->host_data; struct irq_data *irqd; struct msi_desc *desc; int i; @@ -1008,6 +992,8 @@ void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev) } irq_domain_free_irqs(desc->irq, desc->nvec_used); + if (info->flags & MSI_FLAG_DEV_SYSFS) + msi_sysfs_remove_desc(dev, desc); desc->irq = 0; } } @@ -1036,8 +1022,6 @@ void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device lockdep_assert_held(&dev->msi.data->mutex); - if (info->flags & MSI_FLAG_DEV_SYSFS) - msi_device_destroy_sysfs(dev); ops->domain_free_irqs(domain, dev); msi_domain_free_msi_descs(info, dev); } -- cgit v1.2.3-59-g8ed1b From cd6cf06590b9792340dceaa285138777f3cc4d90 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 6 Dec 2021 23:51:52 +0100 Subject: genirq/msi: Convert storage to xarray The current linked list storage for MSI descriptors is suboptimal in several ways: 1) Looking up a MSI desciptor requires a O(n) list walk in the worst case 2) The upcoming support of runtime expansion of MSI-X vectors would need to do a full list walk to figure out whether a particular index is already associated. 3) Runtime expansion of sparse allocations is even more complex as the current implementation assumes an ordered list (increasing MSI index). Use an xarray which solves all of the above problems nicely. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Tested-by: Nishanth Menon Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20211206210749.280627070@linutronix.de --- include/linux/msi.h | 13 ++-- kernel/irq/msi.c | 169 ++++++++++++++++++++++++---------------------------- 2 files changed, 83 insertions(+), 99 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 1a00367d2cfa..fc918a658d48 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -123,7 +124,6 @@ struct pci_msi_desc { /** * struct msi_desc - Descriptor structure for MSI based interrupts - * @list: List head for management * @irq: The base interrupt number * @nvec_used: The number of vectors used * @dev: Pointer to the device which uses this descriptor @@ -140,7 +140,6 @@ struct pci_msi_desc { */ struct msi_desc { /* Shared device/bus type independent data */ - struct list_head list; unsigned int irq; unsigned int nvec_used; struct device *dev; @@ -176,16 +175,16 @@ enum msi_desc_filter { * msi_device_data - MSI per device data * @properties: MSI properties which are interesting to drivers * @platform_data: Platform-MSI specific data - * @list: List of MSI descriptors associated to the device - * @mutex: Mutex protecting the MSI list - * @__next: Cached pointer to the next entry for iterators + * @mutex: Mutex protecting the MSI descriptor store + * @__store: Xarray for storing MSI descriptor pointers + * @__iter_idx: Index to search the next entry for iterators */ struct msi_device_data { unsigned long properties; struct platform_msi_priv_data *platform_data; - struct list_head list; struct mutex mutex; - struct msi_desc *__next; + struct xarray __store; + unsigned long __iter_idx; }; int msi_setup_device_data(struct device *dev); diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index d290e09258bc..173bc04f9fe5 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -20,7 +20,6 @@ #include "internals.h" static inline int msi_sysfs_create_group(struct device *dev); -#define dev_to_msi_list(dev) (&(dev)->msi.data->list) /** * msi_alloc_desc - Allocate an initialized msi_desc @@ -41,7 +40,6 @@ static struct msi_desc *msi_alloc_desc(struct device *dev, int nvec, if (!desc) return NULL; - INIT_LIST_HEAD(&desc->list); desc->dev = dev; desc->nvec_used = nvec; if (affinity) { @@ -60,6 +58,17 @@ static void msi_free_desc(struct msi_desc *desc) kfree(desc); } +static int msi_insert_desc(struct msi_device_data *md, struct msi_desc *desc, unsigned int index) +{ + int ret; + + desc->msi_index = index; + ret = xa_insert(&md->__store, index, desc, GFP_KERNEL); + if (ret) + msi_free_desc(desc); + return ret; +} + /** * msi_add_msi_desc - Allocate and initialize a MSI descriptor * @dev: Pointer to the device for which the descriptor is allocated @@ -77,12 +86,9 @@ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) if (!desc) return -ENOMEM; - /* Copy the MSI index and type specific data to the new descriptor. */ - desc->msi_index = init_desc->msi_index; + /* Copy type specific data to the new descriptor. */ desc->pci = init_desc->pci; - - list_add_tail(&desc->list, &dev->msi.data->list); - return 0; + return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index); } /** @@ -95,28 +101,41 @@ int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc) */ static int msi_add_simple_msi_descs(struct device *dev, unsigned int index, unsigned int ndesc) { - struct msi_desc *desc, *tmp; - LIST_HEAD(list); - unsigned int i; + unsigned int idx, last = index + ndesc - 1; + struct msi_desc *desc; + int ret; lockdep_assert_held(&dev->msi.data->mutex); - for (i = 0; i < ndesc; i++) { + for (idx = index; idx <= last; idx++) { desc = msi_alloc_desc(dev, 1, NULL); if (!desc) + goto fail_mem; + ret = msi_insert_desc(dev->msi.data, desc, idx); + if (ret) goto fail; - desc->msi_index = index + i; - list_add_tail(&desc->list, &list); } - list_splice_tail(&list, &dev->msi.data->list); return 0; +fail_mem: + ret = -ENOMEM; fail: - list_for_each_entry_safe(desc, tmp, &list, list) { - list_del(&desc->list); - msi_free_desc(desc); + msi_free_msi_descs_range(dev, MSI_DESC_NOTASSOCIATED, index, last); + return ret; +} + +static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) +{ + switch (filter) { + case MSI_DESC_ALL: + return true; + case MSI_DESC_NOTASSOCIATED: + return !desc->irq; + case MSI_DESC_ASSOCIATED: + return !!desc->irq; } - return -ENOMEM; + WARN_ON_ONCE(1); + return false; } /** @@ -129,19 +148,17 @@ fail: void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter, unsigned int first_index, unsigned int last_index) { + struct xarray *xa = &dev->msi.data->__store; struct msi_desc *desc; + unsigned long idx; lockdep_assert_held(&dev->msi.data->mutex); - msi_for_each_desc(desc, dev, filter) { - /* - * Stupid for now to handle MSI device domain until the - * storage is switched over to an xarray. - */ - if (desc->msi_index < first_index || desc->msi_index > last_index) - continue; - list_del(&desc->list); - msi_free_desc(desc); + xa_for_each_range(xa, idx, desc, first_index, last_index) { + if (msi_desc_match(desc, filter)) { + xa_erase(xa, idx); + msi_free_desc(desc); + } } } @@ -162,7 +179,8 @@ static void msi_device_data_release(struct device *dev, void *res) { struct msi_device_data *md = res; - WARN_ON_ONCE(!list_empty(&md->list)); + WARN_ON_ONCE(!xa_empty(&md->__store)); + xa_destroy(&md->__store); dev->msi.data = NULL; } @@ -194,7 +212,7 @@ int msi_setup_device_data(struct device *dev) return ret; } - INIT_LIST_HEAD(&md->list); + xa_init(&md->__store); mutex_init(&md->mutex); dev->msi.data = md; devres_add(dev, md); @@ -217,34 +235,21 @@ EXPORT_SYMBOL_GPL(msi_lock_descs); */ void msi_unlock_descs(struct device *dev) { - /* Clear the next pointer which was cached by the iterator */ - dev->msi.data->__next = NULL; + /* Invalidate the index wich was cached by the iterator */ + dev->msi.data->__iter_idx = MSI_MAX_INDEX; mutex_unlock(&dev->msi.data->mutex); } EXPORT_SYMBOL_GPL(msi_unlock_descs); -static bool msi_desc_match(struct msi_desc *desc, enum msi_desc_filter filter) -{ - switch (filter) { - case MSI_DESC_ALL: - return true; - case MSI_DESC_NOTASSOCIATED: - return !desc->irq; - case MSI_DESC_ASSOCIATED: - return !!desc->irq; - } - WARN_ON_ONCE(1); - return false; -} - -static struct msi_desc *msi_find_first_desc(struct device *dev, enum msi_desc_filter filter) +static struct msi_desc *msi_find_desc(struct msi_device_data *md, enum msi_desc_filter filter) { struct msi_desc *desc; - list_for_each_entry(desc, dev_to_msi_list(dev), list) { + xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) { if (msi_desc_match(desc, filter)) return desc; } + md->__iter_idx = MSI_MAX_INDEX; return NULL; } @@ -261,37 +266,24 @@ static struct msi_desc *msi_find_first_desc(struct device *dev, enum msi_desc_fi */ struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter) { - struct msi_desc *desc; + struct msi_device_data *md = dev->msi.data; - if (WARN_ON_ONCE(!dev->msi.data)) + if (WARN_ON_ONCE(!md)) return NULL; - lockdep_assert_held(&dev->msi.data->mutex); + lockdep_assert_held(&md->mutex); - desc = msi_find_first_desc(dev, filter); - dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; - return desc; + md->__iter_idx = 0; + return msi_find_desc(md, filter); } EXPORT_SYMBOL_GPL(msi_first_desc); -static struct msi_desc *__msi_next_desc(struct device *dev, enum msi_desc_filter filter, - struct msi_desc *from) -{ - struct msi_desc *desc = from; - - list_for_each_entry_from(desc, dev_to_msi_list(dev), list) { - if (msi_desc_match(desc, filter)) - return desc; - } - return NULL; -} - /** * msi_next_desc - Get the next MSI descriptor of a device * @dev: Device to operate on * * The first invocation of msi_next_desc() has to be preceeded by a - * successful incovation of __msi_first_desc(). Consecutive invocations are + * successful invocation of __msi_first_desc(). Consecutive invocations are * only valid if the previous one was successful. All these operations have * to be done within the same MSI mutex held region. * @@ -300,20 +292,18 @@ static struct msi_desc *__msi_next_desc(struct device *dev, enum msi_desc_filter */ struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter) { - struct msi_device_data *data = dev->msi.data; - struct msi_desc *desc; + struct msi_device_data *md = dev->msi.data; - if (WARN_ON_ONCE(!data)) + if (WARN_ON_ONCE(!md)) return NULL; - lockdep_assert_held(&data->mutex); + lockdep_assert_held(&md->mutex); - if (!data->__next) + if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX) return NULL; - desc = __msi_next_desc(dev, filter, data->__next); - dev->msi.data->__next = desc ? list_next_entry(desc, list) : NULL; - return desc; + md->__iter_idx++; + return msi_find_desc(md, filter); } EXPORT_SYMBOL_GPL(msi_next_desc); @@ -336,21 +326,18 @@ unsigned int msi_get_virq(struct device *dev, unsigned int index) pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false; msi_lock_descs(dev); - msi_for_each_desc(desc, dev, MSI_DESC_ASSOCIATED) { - /* PCI-MSI has only one descriptor for multiple interrupts. */ - if (pcimsi) { - if (index < desc->nvec_used) - ret = desc->irq + index; - break; - } - + desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index); + if (desc && desc->irq) { /* + * PCI-MSI has only one descriptor for multiple interrupts. * PCI-MSIX and platform MSI use a descriptor per * interrupt. */ - if (desc->msi_index == index) { + if (pcimsi) { + if (index < desc->nvec_used) + ret = desc->irq + index; + } else { ret = desc->irq; - break; } } msi_unlock_descs(dev); @@ -731,16 +718,13 @@ int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev, int ret, virq; msi_lock_descs(dev); - for (virq = virq_base; virq < virq_base + nvec; virq++) { - desc = msi_alloc_desc(dev, 1, NULL); - if (!desc) { - ret = -ENOMEM; - goto fail; - } + ret = msi_add_simple_msi_descs(dev, virq_base, nvec); + if (ret) + goto unlock; - desc->msi_index = virq; + for (virq = virq_base; virq < virq_base + nvec; virq++) { + desc = xa_load(&dev->msi.data->__store, virq); desc->irq = virq; - list_add_tail(&desc->list, &dev->msi.data->list); ops->set_desc(arg, desc); ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg); @@ -756,6 +740,7 @@ fail: for (--virq; virq >= virq_base; virq--) irq_domain_free_irqs_common(domain, virq, 1); msi_free_msi_descs_range(dev, MSI_DESC_ALL, virq_base, virq_base + nvec - 1); +unlock: msi_unlock_descs(dev); return ret; } -- cgit v1.2.3-59-g8ed1b From 60f20d84dc813f1342771a3e4f06d89da26dc412 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 18 Nov 2021 12:12:10 -0600 Subject: of/fdt: Rework early_init_dt_scan_chosen() to call directly Use of the of_scan_flat_dt() function predates libfdt and is discouraged as libfdt provides a nicer set of APIs. Rework early_init_dt_scan_chosen() to be called directly and use libfdt. Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Frank Rowand Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Rob Herring Reviewed-by: Frank Rowand Link: https://lore.kernel.org/r/20211118181213.1433346-2-robh@kernel.org --- arch/powerpc/kernel/prom.c | 2 +- arch/powerpc/mm/nohash/kaslr_booke.c | 4 +--- drivers/of/fdt.c | 39 ++++++++++++++++++------------------ include/linux/of_fdt.h | 3 +-- 4 files changed, 22 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 2e67588f6f6e..c6c398ccd98a 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -402,7 +402,7 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node, const unsigned long *lprop; /* All these set by kernel, so no need to convert endian */ /* Use common scan routine to determine if this is the chosen node */ - if (early_init_dt_scan_chosen(node, uname, depth, data) == 0) + if (early_init_dt_scan_chosen(data) < 0) return 0; #ifdef CONFIG_PPC64 diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c index 8fc49b1b4a91..90debe19ab4c 100644 --- a/arch/powerpc/mm/nohash/kaslr_booke.c +++ b/arch/powerpc/mm/nohash/kaslr_booke.c @@ -44,9 +44,7 @@ struct regions __initdata regions; static __init void kaslr_get_cmdline(void *fdt) { - int node = fdt_path_offset(fdt, "/chosen"); - - early_init_dt_scan_chosen(node, "chosen", 1, boot_command_line); + early_init_dt_scan_chosen(boot_command_line); } static unsigned long __init rotate_xor(unsigned long hash, const void *area, diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index bdca35284ceb..1f1705f76263 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -1124,18 +1124,18 @@ int __init early_init_dt_scan_memory(unsigned long node, const char *uname, return 0; } -int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, - int depth, void *data) +int __init early_init_dt_scan_chosen(char *cmdline) { - int l; + int l, node; const char *p; const void *rng_seed; + const void *fdt = initial_boot_params; - pr_debug("search \"chosen\", depth: %d, uname: %s\n", depth, uname); - - if (depth != 1 || !data || - (strcmp(uname, "chosen") != 0 && strcmp(uname, "chosen@0") != 0)) - return 0; + node = fdt_path_offset(fdt, "/chosen"); + if (node < 0) + node = fdt_path_offset(fdt, "/chosen@0"); + if (node < 0) + return -ENOENT; early_init_dt_check_for_initrd(node); early_init_dt_check_for_elfcorehdr(node); @@ -1144,7 +1144,7 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, /* Retrieve command line */ p = of_get_flat_dt_prop(node, "bootargs", &l); if (p != NULL && l > 0) - strlcpy(data, p, min(l, COMMAND_LINE_SIZE)); + strlcpy(cmdline, p, min(l, COMMAND_LINE_SIZE)); /* * CONFIG_CMDLINE is meant to be a default in case nothing else @@ -1153,18 +1153,18 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, */ #ifdef CONFIG_CMDLINE #if defined(CONFIG_CMDLINE_EXTEND) - strlcat(data, " ", COMMAND_LINE_SIZE); - strlcat(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + strlcat(cmdline, " ", COMMAND_LINE_SIZE); + strlcat(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #elif defined(CONFIG_CMDLINE_FORCE) - strlcpy(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + strlcpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #else /* No arguments from boot loader, use kernel's cmdl*/ - if (!((char *)data)[0]) - strlcpy(data, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + if (!((char *)cmdline)[0]) + strlcpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #endif #endif /* CONFIG_CMDLINE */ - pr_debug("Command line is: %s\n", (char *)data); + pr_debug("Command line is: %s\n", (char *)cmdline); rng_seed = of_get_flat_dt_prop(node, "rng-seed", &l); if (rng_seed && l > 0) { @@ -1178,8 +1178,7 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, fdt_totalsize(initial_boot_params)); } - /* break now */ - return 1; + return 0; } #ifndef MIN_MEMBLOCK_ADDR @@ -1261,14 +1260,14 @@ bool __init early_init_dt_verify(void *params) void __init early_init_dt_scan_nodes(void) { - int rc = 0; + int rc; /* Initialize {size,address}-cells info */ of_scan_flat_dt(early_init_dt_scan_root, NULL); /* Retrieve various information from the /chosen node */ - rc = of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line); - if (!rc) + rc = early_init_dt_scan_chosen(boot_command_line); + if (rc) pr_warn("No chosen node found, continuing without\n"); /* Setup memory, calling early_init_dt_add_memory_arch */ diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index cf48983d3c86..654722235df6 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -58,8 +58,7 @@ extern int of_flat_dt_is_compatible(unsigned long node, const char *name); extern unsigned long of_get_flat_dt_root(void); extern uint32_t of_get_flat_dt_phandle(unsigned long node); -extern int early_init_dt_scan_chosen(unsigned long node, const char *uname, - int depth, void *data); +extern int early_init_dt_scan_chosen(char *cmdline); extern int early_init_dt_scan_memory(unsigned long node, const char *uname, int depth, void *data); extern int early_init_dt_scan_chosen_stdout(void); -- cgit v1.2.3-59-g8ed1b From d665881d2171b62ca1ea23be89be6f2a8a330bb2 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Thu, 18 Nov 2021 12:12:11 -0600 Subject: of/fdt: Rework early_init_dt_scan_root() to call directly Use of the of_scan_flat_dt() function predates libfdt and is discouraged as libfdt provides a nicer set of APIs. Rework early_init_dt_scan_root() to be called directly and use libfdt. Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Frank Rowand Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Rob Herring Reviewed-by: Frank Rowand Link: https://lore.kernel.org/r/20211118181213.1433346-3-robh@kernel.org --- arch/powerpc/kernel/prom.c | 4 ++-- drivers/of/fdt.c | 14 +++++++------- include/linux/of_fdt.h | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index c6c398ccd98a..6e1a106f02eb 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -748,7 +748,7 @@ void __init early_init_devtree(void *params) of_scan_flat_dt(early_init_dt_scan_chosen_ppc, boot_command_line); /* Scan memory nodes and rebuild MEMBLOCKs */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); + early_init_dt_scan_root(); of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); parse_early_param(); @@ -857,7 +857,7 @@ void __init early_get_first_memblock_info(void *params, phys_addr_t *size) * mess the memblock. */ add_mem_to_memblock = 0; - of_scan_flat_dt(early_init_dt_scan_root, NULL); + early_init_dt_scan_root(); of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); add_mem_to_memblock = 1; diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 1f1705f76263..5e216555fe4f 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -1042,13 +1042,14 @@ int __init early_init_dt_scan_chosen_stdout(void) /* * early_init_dt_scan_root - fetch the top level address and size cells */ -int __init early_init_dt_scan_root(unsigned long node, const char *uname, - int depth, void *data) +int __init early_init_dt_scan_root(void) { const __be32 *prop; + const void *fdt = initial_boot_params; + int node = fdt_path_offset(fdt, "/"); - if (depth != 0) - return 0; + if (node < 0) + return -ENODEV; dt_root_size_cells = OF_ROOT_NODE_SIZE_CELLS_DEFAULT; dt_root_addr_cells = OF_ROOT_NODE_ADDR_CELLS_DEFAULT; @@ -1063,8 +1064,7 @@ int __init early_init_dt_scan_root(unsigned long node, const char *uname, dt_root_addr_cells = be32_to_cpup(prop); pr_debug("dt_root_addr_cells = %x\n", dt_root_addr_cells); - /* break now */ - return 1; + return 0; } u64 __init dt_mem_next_cell(int s, const __be32 **cellp) @@ -1263,7 +1263,7 @@ void __init early_init_dt_scan_nodes(void) int rc; /* Initialize {size,address}-cells info */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); + early_init_dt_scan_root(); /* Retrieve various information from the /chosen node */ rc = early_init_dt_scan_chosen(boot_command_line); diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index 654722235df6..df3d31926c3c 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -68,8 +68,7 @@ extern void early_init_dt_add_memory_arch(u64 base, u64 size); extern u64 dt_mem_next_cell(int s, const __be32 **cellp); /* Early flat tree scan hooks */ -extern int early_init_dt_scan_root(unsigned long node, const char *uname, - int depth, void *data); +extern int early_init_dt_scan_root(void); extern bool early_init_dt_scan(void *params); extern bool early_init_dt_verify(void *params); -- cgit v1.2.3-59-g8ed1b From 1f012283e9360fb4007308f04cfaeb205e34b684 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Wed, 15 Dec 2021 09:01:02 -0600 Subject: of/fdt: Rework early_init_dt_scan_memory() to call directly Use of the of_scan_flat_dt() function predates libfdt and is discouraged as libfdt provides a nicer set of APIs. Rework early_init_dt_scan_memory() to be called directly and use libfdt. Cc: John Crispin Cc: Thomas Bogendoerfer Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Frank Rowand Cc: linux-mips@vger.kernel.org Cc: linuxppc-dev@lists.ozlabs.org Reviewed-by: Frank Rowand Signed-off-by: Rob Herring Tested-by: Michael Ellerman Link: https://lore.kernel.org/r/20211215150102.1303588-1-robh@kernel.org --- arch/mips/ralink/of.c | 19 +++---------- arch/powerpc/kernel/prom.c | 21 +++++++------- drivers/of/fdt.c | 68 ++++++++++++++++++++++++---------------------- include/linux/of_fdt.h | 3 +- 4 files changed, 51 insertions(+), 60 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/ralink/of.c b/arch/mips/ralink/of.c index 0135376c5de5..35a87a2da10b 100644 --- a/arch/mips/ralink/of.c +++ b/arch/mips/ralink/of.c @@ -53,17 +53,6 @@ void __init device_tree_init(void) unflatten_and_copy_device_tree(); } -static int memory_dtb; - -static int __init early_init_dt_find_memory(unsigned long node, - const char *uname, int depth, void *data) -{ - if (depth == 1 && !strcmp(uname, "memory@0")) - memory_dtb = 1; - - return 0; -} - void __init plat_mem_setup(void) { void *dtb; @@ -77,10 +66,10 @@ void __init plat_mem_setup(void) dtb = get_fdt(); __dt_setup_arch(dtb); - of_scan_flat_dt(early_init_dt_find_memory, NULL); - if (memory_dtb) - of_scan_flat_dt(early_init_dt_scan_memory, NULL); - else if (soc_info.mem_detect) + if (!early_init_dt_scan_memory()) + return; + + if (soc_info.mem_detect) soc_info.mem_detect(); else if (soc_info.mem_size) memblock_add(soc_info.mem_base, soc_info.mem_size * SZ_1M); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 6e1a106f02eb..ad1230c4f3fe 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -532,19 +532,18 @@ static int __init early_init_drmem_lmb(struct drmem_lmb *lmb, } #endif /* CONFIG_PPC_PSERIES */ -static int __init early_init_dt_scan_memory_ppc(unsigned long node, - const char *uname, - int depth, void *data) +static int __init early_init_dt_scan_memory_ppc(void) { #ifdef CONFIG_PPC_PSERIES - if (depth == 1 && - strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) { + const void *fdt = initial_boot_params; + int node = fdt_path_offset(fdt, "/ibm,dynamic-reconfiguration-memory"); + + if (node > 0) walk_drmem_lmbs_early(node, NULL, early_init_drmem_lmb); - return 0; - } + #endif - - return early_init_dt_scan_memory(node, uname, depth, data); + + return early_init_dt_scan_memory(); } /* @@ -749,7 +748,7 @@ void __init early_init_devtree(void *params) /* Scan memory nodes and rebuild MEMBLOCKs */ early_init_dt_scan_root(); - of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); + early_init_dt_scan_memory_ppc(); parse_early_param(); @@ -858,7 +857,7 @@ void __init early_get_first_memblock_info(void *params, phys_addr_t *size) */ add_mem_to_memblock = 0; early_init_dt_scan_root(); - of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); + early_init_dt_scan_memory_ppc(); add_mem_to_memblock = 1; if (size) diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 5e216555fe4f..a835c458f50a 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -1078,49 +1078,53 @@ u64 __init dt_mem_next_cell(int s, const __be32 **cellp) /* * early_init_dt_scan_memory - Look for and parse memory nodes */ -int __init early_init_dt_scan_memory(unsigned long node, const char *uname, - int depth, void *data) +int __init early_init_dt_scan_memory(void) { - const char *type = of_get_flat_dt_prop(node, "device_type", NULL); - const __be32 *reg, *endp; - int l; - bool hotpluggable; + int node; + const void *fdt = initial_boot_params; - /* We are scanning "memory" nodes only */ - if (type == NULL || strcmp(type, "memory") != 0) - return 0; + fdt_for_each_subnode(node, fdt, 0) { + const char *type = of_get_flat_dt_prop(node, "device_type", NULL); + const __be32 *reg, *endp; + int l; + bool hotpluggable; - reg = of_get_flat_dt_prop(node, "linux,usable-memory", &l); - if (reg == NULL) - reg = of_get_flat_dt_prop(node, "reg", &l); - if (reg == NULL) - return 0; + /* We are scanning "memory" nodes only */ + if (type == NULL || strcmp(type, "memory") != 0) + continue; - endp = reg + (l / sizeof(__be32)); - hotpluggable = of_get_flat_dt_prop(node, "hotpluggable", NULL); + reg = of_get_flat_dt_prop(node, "linux,usable-memory", &l); + if (reg == NULL) + reg = of_get_flat_dt_prop(node, "reg", &l); + if (reg == NULL) + continue; - pr_debug("memory scan node %s, reg size %d,\n", uname, l); + endp = reg + (l / sizeof(__be32)); + hotpluggable = of_get_flat_dt_prop(node, "hotpluggable", NULL); - while ((endp - reg) >= (dt_root_addr_cells + dt_root_size_cells)) { - u64 base, size; + pr_debug("memory scan node %s, reg size %d,\n", + fdt_get_name(fdt, node, NULL), l); - base = dt_mem_next_cell(dt_root_addr_cells, ®); - size = dt_mem_next_cell(dt_root_size_cells, ®); + while ((endp - reg) >= (dt_root_addr_cells + dt_root_size_cells)) { + u64 base, size; - if (size == 0) - continue; - pr_debug(" - %llx, %llx\n", base, size); + base = dt_mem_next_cell(dt_root_addr_cells, ®); + size = dt_mem_next_cell(dt_root_size_cells, ®); - early_init_dt_add_memory_arch(base, size); + if (size == 0) + continue; + pr_debug(" - %llx, %llx\n", base, size); - if (!hotpluggable) - continue; + early_init_dt_add_memory_arch(base, size); - if (memblock_mark_hotplug(base, size)) - pr_warn("failed to mark hotplug range 0x%llx - 0x%llx\n", - base, base + size); - } + if (!hotpluggable) + continue; + if (memblock_mark_hotplug(base, size)) + pr_warn("failed to mark hotplug range 0x%llx - 0x%llx\n", + base, base + size); + } + } return 0; } @@ -1271,7 +1275,7 @@ void __init early_init_dt_scan_nodes(void) pr_warn("No chosen node found, continuing without\n"); /* Setup memory, calling early_init_dt_add_memory_arch */ - of_scan_flat_dt(early_init_dt_scan_memory, NULL); + early_init_dt_scan_memory(); /* Handle linux,usable-memory-range property */ memblock_cap_memory_range(cap_mem_addr, cap_mem_size); diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index df3d31926c3c..914739f3c192 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -59,8 +59,7 @@ extern unsigned long of_get_flat_dt_root(void); extern uint32_t of_get_flat_dt_phandle(unsigned long node); extern int early_init_dt_scan_chosen(char *cmdline); -extern int early_init_dt_scan_memory(unsigned long node, const char *uname, - int depth, void *data); +extern int early_init_dt_scan_memory(void); extern int early_init_dt_scan_chosen_stdout(void); extern void early_init_fdt_scan_reserved_mem(void); extern void early_init_fdt_reserve_self(void); -- cgit v1.2.3-59-g8ed1b From f7ea534a0920dbaf71a8003936e178e14ec9271d Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 15 Dec 2021 18:55:36 -0800 Subject: add includes masked by cgroup -> bpf dependency cgroup pulls in BPF which pulls in a lot of includes. We're about to break that chain so fix those who were depending on it. Signed-off-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211216025538.1649516-2-kuba@kernel.org --- arch/s390/mm/hugetlbpage.c | 1 + include/linux/perf_event.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include/linux') diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index da36d13ffc16..082793d497ec 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -9,6 +9,7 @@ #define KMSG_COMPONENT "hugetlb" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include #include #include #include diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 0dcfd265beed..4a021149eaf0 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -611,6 +611,7 @@ struct swevent_hlist { #define PERF_ATTACH_SCHED_CB 0x20 #define PERF_ATTACH_CHILD 0x40 +struct bpf_prog; struct perf_cgroup; struct perf_buffer; -- cgit v1.2.3-59-g8ed1b From fd1740b6abac39f68ce12e201697f106e0f1d519 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 15 Dec 2021 18:55:38 -0800 Subject: bpf: Remove the cgroup -> bpf header dependecy Remove the dependency from cgroup-defs.h to bpf-cgroup.h and bpf.h. This reduces the incremental build size of x86 allmodconfig after bpf.h was touched from ~17k objects rebuilt to ~5k objects. bpf.h is 2.2kLoC and is modified relatively often. We need a new header with just the definition of struct cgroup_bpf and enum cgroup_bpf_attach_type, this is akin to cgroup-defs.h. Signed-off-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov Acked-by: Tejun Heo Link: https://lore.kernel.org/bpf/20211216025538.1649516-4-kuba@kernel.org --- include/linux/bpf-cgroup-defs.h | 70 +++++++++++++++++++++++++++++++++++++++++ include/linux/bpf-cgroup.h | 57 +-------------------------------- include/linux/cgroup-defs.h | 2 +- 3 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 include/linux/bpf-cgroup-defs.h (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h new file mode 100644 index 000000000000..695d1224a71b --- /dev/null +++ b/include/linux/bpf-cgroup-defs.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BPF_CGROUP_DEFS_H +#define _BPF_CGROUP_DEFS_H + +#ifdef CONFIG_CGROUP_BPF + +#include +#include +#include + +struct bpf_prog_array; + +enum cgroup_bpf_attach_type { + CGROUP_BPF_ATTACH_TYPE_INVALID = -1, + CGROUP_INET_INGRESS = 0, + CGROUP_INET_EGRESS, + CGROUP_INET_SOCK_CREATE, + CGROUP_SOCK_OPS, + CGROUP_DEVICE, + CGROUP_INET4_BIND, + CGROUP_INET6_BIND, + CGROUP_INET4_CONNECT, + CGROUP_INET6_CONNECT, + CGROUP_INET4_POST_BIND, + CGROUP_INET6_POST_BIND, + CGROUP_UDP4_SENDMSG, + CGROUP_UDP6_SENDMSG, + CGROUP_SYSCTL, + CGROUP_UDP4_RECVMSG, + CGROUP_UDP6_RECVMSG, + CGROUP_GETSOCKOPT, + CGROUP_SETSOCKOPT, + CGROUP_INET4_GETPEERNAME, + CGROUP_INET6_GETPEERNAME, + CGROUP_INET4_GETSOCKNAME, + CGROUP_INET6_GETSOCKNAME, + CGROUP_INET_SOCK_RELEASE, + MAX_CGROUP_BPF_ATTACH_TYPE +}; + +struct cgroup_bpf { + /* array of effective progs in this cgroup */ + struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE]; + + /* attached progs to this cgroup and attach flags + * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will + * have either zero or one element + * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS + */ + struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE]; + u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE]; + + /* list of cgroup shared storages */ + struct list_head storages; + + /* temp storage for effective prog array used by prog_attach/detach */ + struct bpf_prog_array *inactive; + + /* reference counter used to detach bpf programs after cgroup removal */ + struct percpu_ref refcnt; + + /* cgroup_bpf is released using a work queue */ + struct work_struct release_work; +}; + +#else /* CONFIG_CGROUP_BPF */ +struct cgroup_bpf {}; +#endif /* CONFIG_CGROUP_BPF */ + +#endif diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 11820a430d6c..b525d8cdc25b 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -3,10 +3,10 @@ #define _BPF_CGROUP_H #include +#include #include #include #include -#include #include #include @@ -23,33 +23,6 @@ struct ctl_table_header; struct task_struct; #ifdef CONFIG_CGROUP_BPF -enum cgroup_bpf_attach_type { - CGROUP_BPF_ATTACH_TYPE_INVALID = -1, - CGROUP_INET_INGRESS = 0, - CGROUP_INET_EGRESS, - CGROUP_INET_SOCK_CREATE, - CGROUP_SOCK_OPS, - CGROUP_DEVICE, - CGROUP_INET4_BIND, - CGROUP_INET6_BIND, - CGROUP_INET4_CONNECT, - CGROUP_INET6_CONNECT, - CGROUP_INET4_POST_BIND, - CGROUP_INET6_POST_BIND, - CGROUP_UDP4_SENDMSG, - CGROUP_UDP6_SENDMSG, - CGROUP_SYSCTL, - CGROUP_UDP4_RECVMSG, - CGROUP_UDP6_RECVMSG, - CGROUP_GETSOCKOPT, - CGROUP_SETSOCKOPT, - CGROUP_INET4_GETPEERNAME, - CGROUP_INET6_GETPEERNAME, - CGROUP_INET4_GETSOCKNAME, - CGROUP_INET6_GETSOCKNAME, - CGROUP_INET_SOCK_RELEASE, - MAX_CGROUP_BPF_ATTACH_TYPE -}; #define CGROUP_ATYPE(type) \ case BPF_##type: return type @@ -127,33 +100,6 @@ struct bpf_prog_list { struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; }; -struct bpf_prog_array; - -struct cgroup_bpf { - /* array of effective progs in this cgroup */ - struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE]; - - /* attached progs to this cgroup and attach flags - * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will - * have either zero or one element - * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS - */ - struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE]; - u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE]; - - /* list of cgroup shared storages */ - struct list_head storages; - - /* temp storage for effective prog array used by prog_attach/detach */ - struct bpf_prog_array *inactive; - - /* reference counter used to detach bpf programs after cgroup removal */ - struct percpu_ref refcnt; - - /* cgroup_bpf is released using a work queue */ - struct work_struct release_work; -}; - int cgroup_bpf_inherit(struct cgroup *cgrp); void cgroup_bpf_offline(struct cgroup *cgrp); @@ -451,7 +397,6 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr); #else -struct cgroup_bpf {}; static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; } static inline void cgroup_bpf_offline(struct cgroup *cgrp) {} diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index db2e147e069f..411684c80cf3 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #ifdef CONFIG_CGROUPS -- cgit v1.2.3-59-g8ed1b From 0f55f9ed21f96630c6ec96805d42f92c0b458b37 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Thu, 16 Dec 2021 13:33:56 -0800 Subject: bpf: Only print scratched registers and stack slots to verifier logs. When printing verifier state for any log level, print full verifier state only on function calls or on errors. Otherwise, only print the registers and stack slots that were accessed. Log size differences: verif_scale_loop6 before: 234566564 verif_scale_loop6 after: 72143943 69% size reduction kfree_skb before: 166406 kfree_skb after: 55386 69% size reduction Before: 156: (61) r0 = *(u32 *)(r1 +0) 157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=invP0 R10=fp0 fp-8_w=00000000 fp-16_w=00\ 000000 fp-24_w=00000000 fp-32_w=00000000 fp-40_w=00000000 fp-48_w=00000000 fp-56_w=00000000 fp-64_w=00000000 fp-72_w=00000000 fp-80_w=00000\ 000 fp-88_w=00000000 fp-96_w=00000000 fp-104_w=00000000 fp-112_w=00000000 fp-120_w=00000000 fp-128_w=00000000 fp-136_w=00000000 fp-144_w=00\ 000000 fp-152_w=00000000 fp-160_w=00000000 fp-168_w=00000000 fp-176_w=00000000 fp-184_w=00000000 fp-192_w=00000000 fp-200_w=00000000 fp-208\ _w=00000000 fp-216_w=00000000 fp-224_w=00000000 fp-232_w=00000000 fp-240_w=00000000 fp-248_w=00000000 fp-256_w=00000000 fp-264_w=00000000 f\ p-272_w=00000000 fp-280_w=00000000 fp-288_w=00000000 fp-296_w=00000000 fp-304_w=00000000 fp-312_w=00000000 fp-320_w=00000000 fp-328_w=00000\ 000 fp-336_w=00000000 fp-344_w=00000000 fp-352_w=00000000 fp-360_w=00000000 fp-368_w=00000000 fp-376_w=00000000 fp-384_w=00000000 fp-392_w=\ 00000000 fp-400_w=00000000 fp-408_w=00000000 fp-416_w=00000000 fp-424_w=00000000 fp-432_w=00000000 fp-440_w=00000000 fp-448_w=00000000 ; return skb->len; 157: (95) exit Func#4 is safe for any args that match its prototype Validating get_constant() func#5... 158: R1=invP(id=0) R10=fp0 ; int get_constant(long val) 158: (bf) r0 = r1 159: R0_w=invP(id=1) R1=invP(id=1) R10=fp0 ; return val - 122; 159: (04) w0 += -122 160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=1) R10=fp0 ; return val - 122; 160: (95) exit Func#5 is safe for any args that match its prototype Validating get_skb_ifindex() func#6... 161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0 ; int get_skb_ifindex(int val, struct __sk_buff *skb, int var) 161: (bc) w0 = w3 162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0 After: 156: (61) r0 = *(u32 *)(r1 +0) 157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) ; return skb->len; 157: (95) exit Func#4 is safe for any args that match its prototype Validating get_constant() func#5... 158: R1=invP(id=0) R10=fp0 ; int get_constant(long val) 158: (bf) r0 = r1 159: R0_w=invP(id=1) R1=invP(id=1) ; return val - 122; 159: (04) w0 += -122 160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return val - 122; 160: (95) exit Func#5 is safe for any args that match its prototype Validating get_skb_ifindex() func#6... 161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0 ; int get_skb_ifindex(int val, struct __sk_buff *skb, int var) 161: (bc) w0 = w3 162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R3=invP(id=0) Signed-off-by: Christy Lee Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20211216213358.3374427-2-christylee@fb.com --- include/linux/bpf_verifier.h | 7 +++ kernel/bpf/verifier.c | 83 +++++++++++++++++++++----- tools/testing/selftests/bpf/prog_tests/align.c | 30 +++++----- 3 files changed, 91 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 182b16a91084..c66f238c538d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -474,6 +474,13 @@ struct bpf_verifier_env { /* longest register parentage chain walked for liveness marking */ u32 longest_mark_read_walk; bpfptr_t fd_array; + + /* bit mask to keep track of whether a register has been accessed + * since the last time the function state was printed + */ + u32 scratched_regs; + /* Same as scratched_regs but for stack slots */ + u64 scratched_stack_slots; }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f0604796132f..ded6e816dcd9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -609,6 +609,44 @@ static const char *kernel_type_name(const struct btf* btf, u32 id) return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); } +static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) +{ + env->scratched_regs |= 1U << regno; +} + +static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) +{ + env->scratched_stack_slots |= 1UL << spi; +} + +static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) +{ + return (env->scratched_regs >> regno) & 1; +} + +static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +{ + return (env->scratched_stack_slots >> regno) & 1; +} + +static bool verifier_state_scratched(const struct bpf_verifier_env *env) +{ + return env->scratched_regs || env->scratched_stack_slots; +} + +static void mark_verifier_state_clean(struct bpf_verifier_env *env) +{ + env->scratched_regs = 0U; + env->scratched_stack_slots = 0UL; +} + +/* Used for printing the entire verifier state. */ +static void mark_verifier_state_scratched(struct bpf_verifier_env *env) +{ + env->scratched_regs = ~0U; + env->scratched_stack_slots = ~0UL; +} + /* The reg state of a pointer or a bounded scalar was saved when * it was spilled to the stack. */ @@ -624,7 +662,8 @@ static void scrub_spilled_slot(u8 *stype) } static void print_verifier_state(struct bpf_verifier_env *env, - const struct bpf_func_state *state) + const struct bpf_func_state *state, + bool print_all) { const struct bpf_reg_state *reg; enum bpf_reg_type t; @@ -637,6 +676,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, t = reg->type; if (t == NOT_INIT) continue; + if (!print_all && !reg_scratched(env, i)) + continue; verbose(env, " R%d", i); print_liveness(env, reg->live); verbose(env, "=%s", reg_type_str[t]); @@ -726,6 +767,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, types_buf[BPF_REG_SIZE] = 0; if (!valid) continue; + if (!print_all && !stack_slot_scratched(env, i)) + continue; verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); print_liveness(env, state->stack[i].spilled_ptr.live); if (is_spilled_reg(&state->stack[i])) { @@ -751,6 +794,7 @@ static void print_verifier_state(struct bpf_verifier_env *env, if (state->in_async_callback_fn) verbose(env, " async_cb"); verbose(env, "\n"); + mark_verifier_state_clean(env); } /* copy array src of length n * size bytes to dst. dst is reallocated if it's too @@ -1541,6 +1585,7 @@ static void init_func_state(struct bpf_verifier_env *env, state->frameno = frameno; state->subprogno = subprogno; init_reg_state(env, state); + mark_verifier_state_scratched(env); } /* Similar to push_stack(), but for async callbacks */ @@ -2228,6 +2273,8 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, return -EINVAL; } + mark_reg_scratched(env, regno); + reg = ®s[regno]; rw64 = is_reg64(env, insn, regno, reg, t); if (t == SRC_OP) { @@ -2678,7 +2725,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, reg->precise = true; } if (env->log.level & BPF_LOG_LEVEL) { - print_verifier_state(env, func); + print_verifier_state(env, func, false); verbose(env, "parent %s regs=%x stack=%llx marks\n", new_marks ? "didn't have" : "already had", reg_mask, stack_mask); @@ -2837,6 +2884,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, env->insn_aux_data[insn_idx].sanitize_stack_spill = true; } + mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && !register_is_null(reg) && env->bpf_capable) { if (dst_reg != BPF_REG_FP) { @@ -2958,6 +3006,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, slot = -i - 1; spi = slot / BPF_REG_SIZE; stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + mark_stack_slot_scratched(env, spi); if (!env->allow_ptr_leaks && *stype != NOT_INIT @@ -3376,7 +3425,7 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, * to make sure our theoretical access will be safe. */ if (env->log.level & BPF_LOG_LEVEL) - print_verifier_state(env, state); + print_verifier_state(env, state, false); /* The minimum value is only important with signed * comparisons where we can't assume the floor of a @@ -6011,9 +6060,9 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "caller:\n"); - print_verifier_state(env, caller); + print_verifier_state(env, caller, true); verbose(env, "callee:\n"); - print_verifier_state(env, callee); + print_verifier_state(env, callee, true); } return 0; } @@ -6228,9 +6277,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) *insn_idx = callee->callsite + 1; if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "returning from callee:\n"); - print_verifier_state(env, callee); + print_verifier_state(env, callee, true); verbose(env, "to caller at %d:\n", *insn_idx); - print_verifier_state(env, caller); + print_verifier_state(env, caller, true); } /* clear everything in the callee */ free_func_state(callee); @@ -8249,12 +8298,12 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, /* Got here implies adding two SCALAR_VALUEs */ if (WARN_ON_ONCE(ptr_reg)) { - print_verifier_state(env, state); + print_verifier_state(env, state, true); verbose(env, "verifier internal error: unexpected ptr_reg\n"); return -EINVAL; } if (WARN_ON(!src_reg)) { - print_verifier_state(env, state); + print_verifier_state(env, state, true); verbose(env, "verifier internal error: no src_reg\n"); return -EINVAL; } @@ -9389,7 +9438,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EACCES; } if (env->log.level & BPF_LOG_LEVEL) - print_verifier_state(env, this_branch->frame[this_branch->curframe]); + print_verifier_state(env, this_branch->frame[this_branch->curframe], false); return 0; } @@ -11259,14 +11308,17 @@ static int do_check(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2 || (env->log.level & BPF_LOG_LEVEL && do_print_state)) { - if (env->log.level & BPF_LOG_LEVEL2) - verbose(env, "%d:", env->insn_idx); - else + if (env->log.level & BPF_LOG_LEVEL2) { + if (verifier_state_scratched(env)) + verbose(env, "%d:", env->insn_idx); + } else { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, env->cur_state->speculative ? " (speculative execution)" : ""); - print_verifier_state(env, state->frame[state->curframe]); + } + print_verifier_state(env, state->frame[state->curframe], + false); do_print_state = false; } @@ -11488,6 +11540,7 @@ static int do_check(struct bpf_verifier_env *env) if (err) return err; process_bpf_exit: + mark_verifier_state_scratched(env); update_branch_counts(env, env->cur_state); err = pop_stack(env, &prev_insn_idx, &env->insn_idx, pop_log); @@ -14148,6 +14201,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) } } + mark_verifier_state_clean(env); + if (IS_ERR(btf_vmlinux)) { /* Either gcc or pahole or kernel are broken. */ verbose(env, "in-kernel BTF is malformed\n"); diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c index 837f67c6bfda..aeb2080a67f7 100644 --- a/tools/testing/selftests/bpf/prog_tests/align.c +++ b/tools/testing/selftests/bpf/prog_tests/align.c @@ -39,8 +39,8 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {1, "R1=ctx(id=0,off=0,imm=0)"}, - {1, "R10=fp0"}, + {0, "R1=ctx(id=0,off=0,imm=0)"}, + {0, "R10=fp0"}, {1, "R3_w=inv2"}, {2, "R3_w=inv4"}, {3, "R3_w=inv8"}, @@ -67,8 +67,8 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {1, "R1=ctx(id=0,off=0,imm=0)"}, - {1, "R10=fp0"}, + {0, "R1=ctx(id=0,off=0,imm=0)"}, + {0, "R10=fp0"}, {1, "R3_w=inv1"}, {2, "R3_w=inv2"}, {3, "R3_w=inv4"}, @@ -96,8 +96,8 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {1, "R1=ctx(id=0,off=0,imm=0)"}, - {1, "R10=fp0"}, + {0, "R1=ctx(id=0,off=0,imm=0)"}, + {0, "R10=fp0"}, {1, "R3_w=inv4"}, {2, "R3_w=inv8"}, {3, "R3_w=inv10"}, @@ -118,8 +118,8 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {1, "R1=ctx(id=0,off=0,imm=0)"}, - {1, "R10=fp0"}, + {0, "R1=ctx(id=0,off=0,imm=0)"}, + {0, "R10=fp0"}, {1, "R3_w=inv7"}, {2, "R3_w=inv7"}, {3, "R3_w=inv14"}, @@ -161,13 +161,13 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"}, + {6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"}, {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, - {18, "R3=pkt_end(id=0,off=0,imm=0)"}, + {13, "R3_w=pkt_end(id=0,off=0,imm=0)"}, {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"}, {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, @@ -234,10 +234,10 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"}, + {3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"}, {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"}, {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"}, - {10, "R2=pkt(id=0,off=0,r=18,imm=0)"}, + {9, "R2=pkt(id=0,off=0,r=18,imm=0)"}, {10, "R5=pkt(id=0,off=14,r=18,imm=0)"}, {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, {14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, @@ -296,7 +296,7 @@ static struct bpf_align_test tests[] = { /* Calculated offset in R6 has unknown value, but known * alignment of 4. */ - {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, + {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Offset is added to packet pointer R5, resulting in * known fixed offset, and variable offset from R6. @@ -386,7 +386,7 @@ static struct bpf_align_test tests[] = { /* Calculated offset in R6 has unknown value, but known * alignment of 4. */ - {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, + {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Adding 14 makes R6 be (4n+2) */ {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, @@ -458,7 +458,7 @@ static struct bpf_align_test tests[] = { /* Checked s>=0 */ {9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, /* packet pointer + nonnegative (4n+2) */ - {11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, + {12, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, {13, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, /* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine. * We checked the bounds, but it might have been able -- cgit v1.2.3-59-g8ed1b From 2e5766483c8c5cf886b4dc647a1741738dde7d79 Mon Sep 17 00:00:00 2001 From: Christy Lee Date: Thu, 16 Dec 2021 19:42:45 -0800 Subject: bpf: Right align verifier states in verifier logs. Make the verifier logs more readable, print the verifier states on the corresponding instruction line. If the previous line was not a bpf instruction, then print the verifier states on its own line. Before: Validating test_pkt_access_subprog3() func#3... 86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0 ; int test_pkt_access_subprog3(int val, struct __sk_buff *skb) 86: (bf) r6 = r2 87: R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0) 87: (bc) w7 = w1 88: R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6 89: R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0) 89: (85) call pc+9 Func#4 is global and valid. Skipping. 90: R0_w=invP(id=0) 90: (bc) w8 = w0 91: R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123 92: R1_w=invP123 92: (85) call pc+65 Func#5 is global and valid. Skipping. 93: R0=invP(id=0) After: 86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0 ; int test_pkt_access_subprog3(int val, struct __sk_buff *skb) 86: (bf) r6 = r2 ; R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0) 87: (bc) w7 = w1 ; R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6 ; R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0) 89: (85) call pc+9 Func#4 is global and valid. Skipping. 90: R0_w=invP(id=0) 90: (bc) w8 = w0 ; R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123 ; R1_w=invP123 92: (85) call pc+65 Func#5 is global and valid. Skipping. 93: R0=invP(id=0) Signed-off-by: Christy Lee Acked-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 3 + kernel/bpf/verifier.c | 57 ++++++--- tools/testing/selftests/bpf/prog_tests/align.c | 169 ++++++++++++++----------- 3 files changed, 131 insertions(+), 98 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c66f238c538d..ee931398f311 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -388,6 +388,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) #define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2) #define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS) #define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */ +#define BPF_LOG_MIN_ALIGNMENT 8U +#define BPF_LOG_ALIGNMENT 40U static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) { @@ -481,6 +483,7 @@ struct bpf_verifier_env { u32 scratched_regs; /* Same as scratched_regs but for stack slots */ u64 scratched_stack_slots; + u32 prev_log_len, prev_insn_print_len; }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ded6e816dcd9..72c4a6bbb5bc 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -797,6 +797,25 @@ static void print_verifier_state(struct bpf_verifier_env *env, mark_verifier_state_clean(env); } +static inline u32 vlog_alignment(u32 pos) +{ + return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), + BPF_LOG_MIN_ALIGNMENT) - pos - 1; +} + +static void print_insn_state(struct bpf_verifier_env *env, + const struct bpf_func_state *state) +{ + if (env->prev_log_len && env->prev_log_len == env->log.len_used) { + /* remove new line character */ + bpf_vlog_reset(&env->log, env->prev_log_len - 1); + verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); + } else { + verbose(env, "%d:", env->insn_idx); + } + print_verifier_state(env, state, false); +} + /* copy array src of length n * size bytes to dst. dst is reallocated if it's too * small to hold src. This is different from krealloc since we don't want to preserve * the contents of dst. @@ -2725,10 +2744,10 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, reg->precise = true; } if (env->log.level & BPF_LOG_LEVEL) { - print_verifier_state(env, func, false); - verbose(env, "parent %s regs=%x stack=%llx marks\n", + verbose(env, "parent %s regs=%x stack=%llx marks:", new_marks ? "didn't have" : "already had", reg_mask, stack_mask); + print_verifier_state(env, func, true); } if (!reg_mask && !stack_mask) @@ -3423,11 +3442,8 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, /* We may have adjusted the register pointing to memory region, so we * need to try adding each of min_value and max_value to off * to make sure our theoretical access will be safe. - */ - if (env->log.level & BPF_LOG_LEVEL) - print_verifier_state(env, state, false); - - /* The minimum value is only important with signed + * + * The minimum value is only important with signed * comparisons where we can't assume the floor of a * value is 0. If we are using signed variables for our * index'es we need to make sure that whatever we use @@ -9438,7 +9454,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EACCES; } if (env->log.level & BPF_LOG_LEVEL) - print_verifier_state(env, this_branch->frame[this_branch->curframe], false); + print_insn_state(env, this_branch->frame[this_branch->curframe]); return 0; } @@ -11306,19 +11322,12 @@ static int do_check(struct bpf_verifier_env *env) if (need_resched()) cond_resched(); - if (env->log.level & BPF_LOG_LEVEL2 || - (env->log.level & BPF_LOG_LEVEL && do_print_state)) { - if (env->log.level & BPF_LOG_LEVEL2) { - if (verifier_state_scratched(env)) - verbose(env, "%d:", env->insn_idx); - } else { - verbose(env, "\nfrom %d to %d%s:", - env->prev_insn_idx, env->insn_idx, - env->cur_state->speculative ? - " (speculative execution)" : ""); - } - print_verifier_state(env, state->frame[state->curframe], - false); + if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { + verbose(env, "\nfrom %d to %d%s:", + env->prev_insn_idx, env->insn_idx, + env->cur_state->speculative ? + " (speculative execution)" : ""); + print_verifier_state(env, state->frame[state->curframe], true); do_print_state = false; } @@ -11329,9 +11338,15 @@ static int do_check(struct bpf_verifier_env *env) .private_data = env, }; + if (verifier_state_scratched(env)) + print_insn_state(env, state->frame[state->curframe]); + verbose_linfo(env, env->insn_idx, "; "); + env->prev_log_len = env->log.len_used; verbose(env, "%d: ", env->insn_idx); print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + env->prev_insn_print_len = env->log.len_used - env->prev_log_len; + env->prev_log_len = env->log.len_used; } if (bpf_prog_is_dev_bound(env->prog->aux)) { diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c index aeb2080a67f7..0ee29e11eaee 100644 --- a/tools/testing/selftests/bpf/prog_tests/align.c +++ b/tools/testing/selftests/bpf/prog_tests/align.c @@ -41,11 +41,11 @@ static struct bpf_align_test tests[] = { .matches = { {0, "R1=ctx(id=0,off=0,imm=0)"}, {0, "R10=fp0"}, - {1, "R3_w=inv2"}, - {2, "R3_w=inv4"}, - {3, "R3_w=inv8"}, - {4, "R3_w=inv16"}, - {5, "R3_w=inv32"}, + {0, "R3_w=inv2"}, + {1, "R3_w=inv4"}, + {2, "R3_w=inv8"}, + {3, "R3_w=inv16"}, + {4, "R3_w=inv32"}, }, }, { @@ -69,17 +69,17 @@ static struct bpf_align_test tests[] = { .matches = { {0, "R1=ctx(id=0,off=0,imm=0)"}, {0, "R10=fp0"}, - {1, "R3_w=inv1"}, - {2, "R3_w=inv2"}, - {3, "R3_w=inv4"}, - {4, "R3_w=inv8"}, - {5, "R3_w=inv16"}, - {6, "R3_w=inv1"}, - {7, "R4_w=inv32"}, - {8, "R4_w=inv16"}, - {9, "R4_w=inv8"}, - {10, "R4_w=inv4"}, - {11, "R4_w=inv2"}, + {0, "R3_w=inv1"}, + {1, "R3_w=inv2"}, + {2, "R3_w=inv4"}, + {3, "R3_w=inv8"}, + {4, "R3_w=inv16"}, + {5, "R3_w=inv1"}, + {6, "R4_w=inv32"}, + {7, "R4_w=inv16"}, + {8, "R4_w=inv8"}, + {9, "R4_w=inv4"}, + {10, "R4_w=inv2"}, }, }, { @@ -98,12 +98,12 @@ static struct bpf_align_test tests[] = { .matches = { {0, "R1=ctx(id=0,off=0,imm=0)"}, {0, "R10=fp0"}, - {1, "R3_w=inv4"}, - {2, "R3_w=inv8"}, - {3, "R3_w=inv10"}, - {4, "R4_w=inv8"}, - {5, "R4_w=inv12"}, - {6, "R4_w=inv14"}, + {0, "R3_w=inv4"}, + {1, "R3_w=inv8"}, + {2, "R3_w=inv10"}, + {3, "R4_w=inv8"}, + {4, "R4_w=inv12"}, + {5, "R4_w=inv14"}, }, }, { @@ -120,10 +120,10 @@ static struct bpf_align_test tests[] = { .matches = { {0, "R1=ctx(id=0,off=0,imm=0)"}, {0, "R10=fp0"}, + {0, "R3_w=inv7"}, {1, "R3_w=inv7"}, - {2, "R3_w=inv7"}, - {3, "R3_w=inv14"}, - {4, "R3_w=inv56"}, + {2, "R3_w=inv14"}, + {3, "R3_w=inv56"}, }, }, @@ -162,18 +162,18 @@ static struct bpf_align_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { {6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"}, - {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, - {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, - {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, - {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, - {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, - {13, "R3_w=pkt_end(id=0,off=0,imm=0)"}, - {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, - {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"}, - {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, - {21, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, - {22, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, - {23, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, + {6, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, + {7, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, + {8, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {9, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, + {10, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, + {12, "R3_w=pkt_end(id=0,off=0,imm=0)"}, + {17, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, + {18, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"}, + {19, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, + {20, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, + {21, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {22, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, }, }, { @@ -194,16 +194,16 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, - {8, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, - {9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, - {10, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, - {11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, - {12, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, - {13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, - {14, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, - {15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, - {16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, + {6, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, + {7, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, + {8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, + {9, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, + {10, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, + {11, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, + {12, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {13, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"}, + {14, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, + {15, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, }, }, { @@ -234,14 +234,14 @@ static struct bpf_align_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .matches = { - {3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"}, - {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"}, - {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"}, + {2, "R5_w=pkt(id=0,off=0,r=0,imm=0)"}, + {4, "R5_w=pkt(id=0,off=14,r=0,imm=0)"}, + {5, "R4_w=pkt(id=0,off=14,r=0,imm=0)"}, {9, "R2=pkt(id=0,off=0,r=18,imm=0)"}, {10, "R5=pkt(id=0,off=14,r=18,imm=0)"}, {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, + {13, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, {14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, - {15, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, }, }, { @@ -297,7 +297,7 @@ static struct bpf_align_test tests[] = { * alignment of 4. */ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, - {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {7, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Offset is added to packet pointer R5, resulting in * known fixed offset, and variable offset from R6. */ @@ -313,11 +313,11 @@ static struct bpf_align_test tests[] = { /* Variable offset is added to R5 packet pointer, * resulting in auxiliary alignment of 4. */ - {18, "R5_w=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {17, "R5_w=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Constant offset is added to R5, resulting in * reg->off of 14. */ - {19, "R5_w=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {18, "R5_w=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off * (14) which is 16. Then the variable offset is 4-byte @@ -329,18 +329,18 @@ static struct bpf_align_test tests[] = { /* Constant offset is added to R5 packet pointer, * resulting in reg->off value of 14. */ - {26, "R5_w=pkt(id=0,off=14,r=8"}, + {25, "R5_w=pkt(id=0,off=14,r=8"}, /* Variable offset is added to R5, resulting in a * variable offset of (4n). */ - {27, "R5_w=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {26, "R5_w=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Constant is added to R5 again, setting reg->off to 18. */ - {28, "R5_w=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {27, "R5_w=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* And once more we add a variable; resulting var_off * is still (4n), fixed offset is not changed. * Also, we create a new reg->id. */ - {29, "R5_w=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc)"}, + {28, "R5_w=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc)"}, /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (18) * which is 20. Then the variable offset is (4n), so @@ -387,12 +387,12 @@ static struct bpf_align_test tests[] = { * alignment of 4. */ {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, - {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {7, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Adding 14 makes R6 be (4n+2) */ - {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, + {8, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, /* Packet pointer has (4n+2) offset */ {11, "R5_w=pkt(id=1,off=0,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"}, - {13, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"}, + {12, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc)"}, /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so @@ -403,12 +403,12 @@ static struct bpf_align_test tests[] = { /* Newly read value in R6 was shifted left by 2, so has * known alignment of 4. */ - {18, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {17, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Added (4n) to packet pointer's (4n+2) var_off, giving * another (4n+2). */ {19, "R5_w=pkt(id=2,off=0,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"}, - {21, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"}, + {20, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc)"}, /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so @@ -448,18 +448,18 @@ static struct bpf_align_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, .matches = { - {4, "R5_w=pkt_end(id=0,off=0,imm=0)"}, + {3, "R5_w=pkt_end(id=0,off=0,imm=0)"}, /* (ptr - ptr) << 2 == unknown, (4n) */ - {6, "R5_w=inv(id=0,smax_value=9223372036854775804,umax_value=18446744073709551612,var_off=(0x0; 0xfffffffffffffffc)"}, + {5, "R5_w=inv(id=0,smax_value=9223372036854775804,umax_value=18446744073709551612,var_off=(0x0; 0xfffffffffffffffc)"}, /* (4n) + 14 == (4n+2). We blow our bounds, because * the add could overflow. */ - {7, "R5_w=inv(id=0,smin_value=-9223372036854775806,smax_value=9223372036854775806,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"}, + {6, "R5_w=inv(id=0,smin_value=-9223372036854775806,smax_value=9223372036854775806,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"}, /* Checked s>=0 */ {9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, /* packet pointer + nonnegative (4n+2) */ - {12, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, - {13, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, + {11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, + {12, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"}, /* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine. * We checked the bounds, but it might have been able * to overflow if the packet pointer started in the @@ -502,14 +502,14 @@ static struct bpf_align_test tests[] = { /* Calculated offset in R6 has unknown value, but known * alignment of 4. */ - {7, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, - {9, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, + {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Adding 14 makes R6 be (4n+2) */ - {10, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, + {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, /* New unknown value in R7 is (4n) */ - {11, "R7_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, + {10, "R7_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, /* Subtracting it from R6 blows our unsigned bounds */ - {12, "R6=inv(id=0,smin_value=-1006,smax_value=1034,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"}, + {11, "R6=inv(id=0,smin_value=-1006,smax_value=1034,umin_value=2,umax_value=18446744073709551614,var_off=(0x2; 0xfffffffffffffffc)"}, /* Checked s>= 0 */ {14, "R6=inv(id=0,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc))"}, /* At the time the word size load is performed from R5, @@ -556,14 +556,14 @@ static struct bpf_align_test tests[] = { /* Calculated offset in R6 has unknown value, but known * alignment of 4. */ - {7, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, - {10, "R6_w=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"}, + {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"}, + {9, "R6_w=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"}, /* Adding 14 makes R6 be (4n+2) */ - {11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"}, + {10, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"}, /* Subtracting from packet pointer overflows ubounds */ {13, "R5_w=pkt(id=2,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"}, /* New unknown value in R7 is (4n), >= 76 */ - {15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"}, + {14, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"}, /* Adding it to packet pointer gives nice bounds again */ {16, "R5_w=pkt(id=3,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"}, /* At the time the word size load is performed from R5, @@ -625,12 +625,15 @@ static int do_test_single(struct bpf_align_test *test) line_ptr = strtok(bpf_vlog_copy, "\n"); for (i = 0; i < MAX_MATCHES; i++) { struct bpf_reg_match m = test->matches[i]; + int tmp; if (!m.match) break; while (line_ptr) { cur_line = -1; sscanf(line_ptr, "%u: ", &cur_line); + if (cur_line == -1) + sscanf(line_ptr, "from %u to %u: ", &tmp, &cur_line); if (cur_line == m.line) break; line_ptr = strtok(NULL, "\n"); @@ -642,7 +645,19 @@ static int do_test_single(struct bpf_align_test *test) printf("%s", bpf_vlog); break; } + /* Check the next line as well in case the previous line + * did not have a corresponding bpf insn. Example: + * func#0 @0 + * 0: R1=ctx(id=0,off=0,imm=0) R10=fp0 + * 0: (b7) r3 = 2 ; R3_w=inv2 + */ if (!strstr(line_ptr, m.match)) { + cur_line = -1; + line_ptr = strtok(NULL, "\n"); + sscanf(line_ptr, "%u: ", &cur_line); + } + if (cur_line != m.line || !line_ptr || + !strstr(line_ptr, m.match)) { printf("Failed to find match %u: %s\n", m.line, m.match); ret = 1; -- cgit v1.2.3-59-g8ed1b From 03de6b273805b3c552ff158f8688555937375926 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 22 Nov 2021 23:22:00 +0100 Subject: dmaengine: qcom-adm: stop abusing slave_id config The slave_id was previously used to pick one DMA slave instead of another, but this is now done through the DMA descriptors in device tree. For the qcom_adm driver, the configuration is documented in the DT binding to contain a tuple of device identifier and a "crci" field, but the implementation ends up using only a single cell for identifying the slave, with the crci getting passed in nonstandard properties of the device, and passed through the dma driver using the old slave_id field. Part of the problem apparently is that the nand driver ends up using only a single DMA request ID, but requires distinct values for "crci" depending on the type of transfer. Change both the dmaengine driver and the two slave drivers to allow the documented binding to work in addition to the ad-hoc passing of crci values. In order to no longer abuse the slave_id field, pass the data using the "peripheral_config" mechanism instead. Signed-off-by: Arnd Bergmann Acked-by: Mark Brown Link: https://lore.kernel.org/r/20211122222203.4103644-9-arnd@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/qcom/qcom_adm.c | 56 ++++++++++++++++++++++++++++++++++----- drivers/mtd/nand/raw/qcom_nandc.c | 14 ++++++++-- drivers/tty/serial/msm_serial.c | 15 +++++++++-- include/linux/dma/qcom_adm.h | 12 +++++++++ 4 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 include/linux/dma/qcom_adm.h (limited to 'include/linux') diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c index ee78bed8d60d..facdacf8aede 100644 --- a/drivers/dma/qcom/qcom_adm.c +++ b/drivers/dma/qcom/qcom_adm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -140,6 +141,8 @@ struct adm_chan { struct adm_async_desc *curr_txd; struct dma_slave_config slave; + u32 crci; + u32 mux; struct list_head node; int error; @@ -379,8 +382,8 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, return ERR_PTR(-EINVAL); } - crci = achan->slave.slave_id & 0xf; - if (!crci || achan->slave.slave_id > 0x1f) { + crci = achan->crci & 0xf; + if (!crci || achan->crci > 0x1f) { dev_err(adev->dev, "invalid crci value\n"); return ERR_PTR(-EINVAL); } @@ -403,9 +406,7 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, if (!async_desc) return ERR_PTR(-ENOMEM); - if (crci) - async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ? - ADM_CRCI_CTL_MUX_SEL : 0; + async_desc->mux = achan->mux ? ADM_CRCI_CTL_MUX_SEL : 0; async_desc->crci = crci; async_desc->blk_size = blk_size; async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) + @@ -488,10 +489,13 @@ static int adm_terminate_all(struct dma_chan *chan) static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { struct adm_chan *achan = to_adm_chan(chan); + struct qcom_adm_peripheral_config *config = cfg->peripheral_config; unsigned long flag; spin_lock_irqsave(&achan->vc.lock, flag); memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); + if (cfg->peripheral_size == sizeof(config)) + achan->crci = config->crci; spin_unlock_irqrestore(&achan->vc.lock, flag); return 0; @@ -694,6 +698,45 @@ static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan, achan->vc.desc_free = adm_dma_free_desc; } +/** + * adm_dma_xlate + * @dma_spec: pointer to DMA specifier as found in the device tree + * @ofdma: pointer to DMA controller data + * + * This can use either 1-cell or 2-cell formats, the first cell + * identifies the slave device, while the optional second cell + * contains the crci value. + * + * Returns pointer to appropriate dma channel on success or NULL on error. + */ +static struct dma_chan *adm_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dma_device *dev = ofdma->of_dma_data; + struct dma_chan *chan, *candidate = NULL; + struct adm_chan *achan; + + if (!dev || dma_spec->args_count > 2) + return NULL; + + list_for_each_entry(chan, &dev->channels, device_node) + if (chan->chan_id == dma_spec->args[0]) { + candidate = chan; + break; + } + + if (!candidate) + return NULL; + + achan = to_adm_chan(candidate); + if (dma_spec->args_count == 2) + achan->crci = dma_spec->args[1]; + else + achan->crci = 0; + + return dma_get_slave_channel(candidate); +} + static int adm_dma_probe(struct platform_device *pdev) { struct adm_device *adev; @@ -838,8 +881,7 @@ static int adm_dma_probe(struct platform_device *pdev) goto err_disable_clks; } - ret = of_dma_controller_register(pdev->dev.of_node, - of_dma_xlate_by_chan_id, + ret = of_dma_controller_register(pdev->dev.of_node, adm_dma_xlate, &adev->common); if (ret) goto err_unregister_dma; diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index 04e6f7b26706..7c6efa3b6255 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -952,6 +953,7 @@ static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, struct dma_async_tx_descriptor *dma_desc; struct scatterlist *sgl; struct dma_slave_config slave_conf; + struct qcom_adm_peripheral_config periph_conf = {}; enum dma_transfer_direction dir_eng; int ret; @@ -983,11 +985,19 @@ static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read, if (read) { slave_conf.src_maxburst = 16; slave_conf.src_addr = nandc->base_dma + reg_off; - slave_conf.slave_id = nandc->data_crci; + if (nandc->data_crci) { + periph_conf.crci = nandc->data_crci; + slave_conf.peripheral_config = &periph_conf; + slave_conf.peripheral_size = sizeof(periph_conf); + } } else { slave_conf.dst_maxburst = 16; slave_conf.dst_addr = nandc->base_dma + reg_off; - slave_conf.slave_id = nandc->cmd_crci; + if (nandc->cmd_crci) { + periph_conf.crci = nandc->cmd_crci; + slave_conf.peripheral_config = &periph_conf; + slave_conf.peripheral_size = sizeof(periph_conf); + } } ret = dmaengine_slave_config(nandc->chan, &slave_conf); diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c index fcef7a961430..c6be09f44dc1 100644 --- a/drivers/tty/serial/msm_serial.c +++ b/drivers/tty/serial/msm_serial.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -290,6 +291,7 @@ static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base) { struct device *dev = msm_port->uart.dev; struct dma_slave_config conf; + struct qcom_adm_peripheral_config periph_conf = {}; struct msm_dma *dma; u32 crci = 0; int ret; @@ -308,7 +310,11 @@ static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base) conf.device_fc = true; conf.dst_addr = base + UARTDM_TF; conf.dst_maxburst = UARTDM_BURST_SIZE; - conf.slave_id = crci; + if (crci) { + conf.peripheral_config = &periph_conf; + conf.peripheral_size = sizeof(periph_conf); + periph_conf.crci = crci; + } ret = dmaengine_slave_config(dma->chan, &conf); if (ret) @@ -333,6 +339,7 @@ static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base) { struct device *dev = msm_port->uart.dev; struct dma_slave_config conf; + struct qcom_adm_peripheral_config periph_conf = {}; struct msm_dma *dma; u32 crci = 0; int ret; @@ -355,7 +362,11 @@ static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base) conf.device_fc = true; conf.src_addr = base + UARTDM_RF; conf.src_maxburst = UARTDM_BURST_SIZE; - conf.slave_id = crci; + if (crci) { + conf.peripheral_config = &periph_conf; + conf.peripheral_size = sizeof(periph_conf); + periph_conf.crci = crci; + } ret = dmaengine_slave_config(dma->chan, &conf); if (ret) diff --git a/include/linux/dma/qcom_adm.h b/include/linux/dma/qcom_adm.h new file mode 100644 index 000000000000..af20df674f0c --- /dev/null +++ b/include/linux/dma/qcom_adm.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only +#ifndef __LINUX_DMA_QCOM_ADM_H +#define __LINUX_DMA_QCOM_ADM_H + +#include + +struct qcom_adm_peripheral_config { + u32 crci; + u32 mux; +}; + +#endif /* __LINUX_DMA_QCOM_ADM_H */ -- cgit v1.2.3-59-g8ed1b From 93cdb5b0dc56cc7a8b87a61146495f3bdc93d7ba Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 22 Nov 2021 23:22:01 +0100 Subject: dmaengine: xilinx_dpdma: stop using slave_id field The display driver wants to pass a custom flag to the DMA engine driver, which it started doing by using the slave_id field that was traditionally used for a different purpose. As there is no longer a correct use for the slave_id field, it should really be removed, and the remaining users changed over to something different. The new mechanism for passing nonstandard settings is using the .peripheral_config field, so use that to pass a newly defined structure here, making it clear that this will not work in portable drivers. Reviewed-by: Laurent Pinchart Signed-off-by: Arnd Bergmann Acked-by: Mark Brown Link: https://lore.kernel.org/r/20211122222203.4103644-10-arnd@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dpdma.c | 17 +++++++++++------ drivers/gpu/drm/xlnx/zynqmp_disp.c | 9 +++++++-- include/linux/dma/xilinx_dpdma.h | 11 +++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 include/linux/dma/xilinx_dpdma.h (limited to 'include/linux') diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c index ce5c66e6897d..b0f4948b00a5 100644 --- a/drivers/dma/xilinx/xilinx_dpdma.c +++ b/drivers/dma/xilinx/xilinx_dpdma.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1273,6 +1274,7 @@ static int xilinx_dpdma_config(struct dma_chan *dchan, struct dma_slave_config *config) { struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dpdma_peripheral_config *pconfig; unsigned long flags; /* @@ -1282,15 +1284,18 @@ static int xilinx_dpdma_config(struct dma_chan *dchan, * fixed both on the DPDMA side and on the DP controller side. */ - spin_lock_irqsave(&chan->lock, flags); - /* - * Abuse the slave_id to indicate that the channel is part of a video - * group. + * Use the peripheral_config to indicate that the channel is part + * of a video group. This requires matching use of the custom + * structure in each driver. */ - if (chan->id <= ZYNQMP_DPDMA_VIDEO2) - chan->video_group = config->slave_id != 0; + pconfig = config->peripheral_config; + if (WARN_ON(pconfig && config->peripheral_size != sizeof(*pconfig))) + return -EINVAL; + spin_lock_irqsave(&chan->lock, flags); + if (chan->id <= ZYNQMP_DPDMA_VIDEO2 && pconfig) + chan->video_group = pconfig->video_group; spin_unlock_irqrestore(&chan->lock, flags); return 0; diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.c b/drivers/gpu/drm/xlnx/zynqmp_disp.c index ff2b308d8651..11c409cbc88e 100644 --- a/drivers/gpu/drm/xlnx/zynqmp_disp.c +++ b/drivers/gpu/drm/xlnx/zynqmp_disp.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -1058,14 +1059,18 @@ static void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer, zynqmp_disp_avbuf_set_format(layer->disp, layer, layer->disp_fmt); /* - * Set slave_id for each DMA channel to indicate they're part of a + * Set pconfig for each DMA channel to indicate they're part of a * video group. */ for (i = 0; i < info->num_planes; i++) { struct zynqmp_disp_layer_dma *dma = &layer->dmas[i]; + struct xilinx_dpdma_peripheral_config pconfig = { + .video_group = true, + }; struct dma_slave_config config = { .direction = DMA_MEM_TO_DEV, - .slave_id = 1, + .peripheral_config = &pconfig, + .peripheral_size = sizeof(pconfig), }; dmaengine_slave_config(dma->chan, &config); diff --git a/include/linux/dma/xilinx_dpdma.h b/include/linux/dma/xilinx_dpdma.h new file mode 100644 index 000000000000..83a1377f03f8 --- /dev/null +++ b/include/linux/dma/xilinx_dpdma.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __LINUX_DMA_XILINX_DPDMA_H +#define __LINUX_DMA_XILINX_DPDMA_H + +#include + +struct xilinx_dpdma_peripheral_config { + bool video_group; +}; + +#endif /* __LINUX_DMA_XILINX_DPDMA_H */ -- cgit v1.2.3-59-g8ed1b From 3c219644075795a99271d345efdfa8b256e55161 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 22 Nov 2021 23:22:03 +0100 Subject: dmaengine: remove slave_id config field All references to the slave_id field have been removed, so remove the field as well to prevent new references from creeping in again. Originally this allowed slave DMA drivers to configure which device is accessed with the dmaengine_slave_config() call, but this was inconsistent, as the same information is also passed while requesting a channel, and never changes in practice. In modern kernels, the device is always selected when requesting the channel, so the .slave_id field is no longer useful. Reviewed-by: Laurent Pinchart Signed-off-by: Arnd Bergmann Acked-by: Mark Brown Link: https://lore.kernel.org/r/20211122222203.4103644-12-arnd@kernel.org Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 9000f3ffce8b..0349b35235e6 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -418,9 +418,6 @@ enum dma_slave_buswidth { * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill * with 'true' if peripheral should be flow controller. Direction will be * selected at Runtime. - * @slave_id: Slave requester id. Only valid for slave channels. The dma - * slave peripheral will have unique id as dma requester which need to be - * pass as slave config. * @peripheral_config: peripheral configuration for programming peripheral * for dmaengine transfer * @peripheral_size: peripheral configuration buffer size @@ -448,7 +445,6 @@ struct dma_slave_config { u32 src_port_window_size; u32 dst_port_window_size; bool device_fc; - unsigned int slave_id; void *peripheral_config; size_t peripheral_size; }; -- cgit v1.2.3-59-g8ed1b From 3d725965f836a7acbd1674e33644bec18373de53 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Tue, 7 Dec 2021 15:33:06 -0800 Subject: crypto: ccp - Add SEV_INIT_EX support Add new module parameter to allow users to use SEV_INIT_EX instead of SEV_INIT. This helps users who lock their SPI bus to use the PSP for SEV functionality. The 'init_ex_path' parameter defaults to NULL which means the kernel will use SEV_INIT, if a path is specified SEV_INIT_EX will be used with the data found at the path. On certain PSP commands this file is written to as the PSP updates the NV memory region. Depending on file system initialization this file open may fail during module init but the CCP driver for SEV already has sufficient retries for platform initialization. During normal operation of PSP system and SEV commands if the PSP has not been initialized it is at run time. If the file at 'init_ex_path' does not exist the PSP will not be initialized. The user must create the file prior to use with 32Kb of 0xFFs per spec. Signed-off-by: David Rientjes Co-developed-by: Peter Gonda Signed-off-by: Peter Gonda Reviewed-by: Marc Orr Reported-by: kernel test robot Acked-by: Brijesh Singh Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: Paolo Bonzini Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- Documentation/virt/kvm/amd-memory-encryption.rst | 6 + drivers/crypto/ccp/sev-dev.c | 193 +++++++++++++++++++++-- include/linux/psp-sev.h | 21 +++ 3 files changed, 205 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/Documentation/virt/kvm/amd-memory-encryption.rst b/Documentation/virt/kvm/amd-memory-encryption.rst index 5c081c8c7164..1c6847fff304 100644 --- a/Documentation/virt/kvm/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/amd-memory-encryption.rst @@ -85,6 +85,12 @@ guests, such as launching, running, snapshotting, migrating and decommissioning. The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform context. In a typical workflow, this command should be the first command issued. +The firmware can be initialized either by using its own non-volatile storage or +the OS can manage the NV storage for the firmware using the module parameter +``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create +a new NV storage file allocate the file with 32KB bytes of 0xFF as required by +the SEV spec. + Returns: 0 on success, -negative on error 2. KVM_SEV_LAUNCH_START diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 686b16e69de7..991cc5aaaa0f 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -43,6 +44,10 @@ static int psp_probe_timeout = 5; module_param(psp_probe_timeout, int, 0644); MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); +static char *init_ex_path; +module_param(init_ex_path, charp, 0444); +MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX"); + static bool psp_init_on_probe = true; module_param(psp_init_on_probe, bool, 0444); MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); @@ -62,6 +67,14 @@ static int psp_timeout; #define SEV_ES_TMR_SIZE (1024 * 1024) static void *sev_es_tmr; +/* INIT_EX NV Storage: + * The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page + * allocator to allocate the memory, which will return aligned memory for the + * specified allocation order. + */ +#define NV_LENGTH (32 * 1024) +static void *sev_init_ex_buffer; + static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { struct sev_device *sev = psp_master->sev_data; @@ -111,6 +124,7 @@ static int sev_cmd_buffer_len(int cmd) { switch (cmd) { case SEV_CMD_INIT: return sizeof(struct sev_data_init); + case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex); case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status); case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr); case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import); @@ -156,6 +170,101 @@ static void *sev_fw_alloc(unsigned long len) return page_address(page); } +static int sev_read_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + ssize_t nread; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return -EOPNOTSUPP; + + fp = filp_open(init_ex_path, O_RDONLY, 0); + if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + + dev_err(sev->dev, + "SEV: could not open %s for read, error %d\n", + init_ex_path, ret); + return ret; + } + + nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); + if (nread != NV_LENGTH) { + dev_err(sev->dev, + "SEV: failed to read %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nread); + return -EIO; + } + + dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); + filp_close(fp, NULL); + + return 0; +} + +static void sev_write_init_ex_file(void) +{ + struct sev_device *sev = psp_master->sev_data; + struct file *fp; + loff_t offset = 0; + ssize_t nwrite; + + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return; + + fp = filp_open(init_ex_path, O_CREAT | O_WRONLY, 0600); + if (IS_ERR(fp)) { + dev_err(sev->dev, + "SEV: could not open file for write, error %ld\n", + PTR_ERR(fp)); + return; + } + + nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); + vfs_fsync(fp, 0); + filp_close(fp, NULL); + + if (nwrite != NV_LENGTH) { + dev_err(sev->dev, + "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", + NV_LENGTH, nwrite); + return; + } + + dev_dbg(sev->dev, "SEV: write successful to NV file\n"); +} + +static void sev_write_init_ex_file_if_required(int cmd_id) +{ + lockdep_assert_held(&sev_cmd_mutex); + + if (!sev_init_ex_buffer) + return; + + /* + * Only a few platform commands modify the SPI/NV area, but none of the + * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN, + * PEK_CERT_IMPORT, and PDH_GEN do. + */ + switch (cmd_id) { + case SEV_CMD_FACTORY_RESET: + case SEV_CMD_INIT_EX: + case SEV_CMD_PDH_GEN: + case SEV_CMD_PEK_CERT_IMPORT: + case SEV_CMD_PEK_GEN: + break; + default: + return; + }; + + sev_write_init_ex_file(); +} + static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) { struct psp_device *psp = psp_master; @@ -225,6 +334,8 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n", cmd, reg & PSP_CMDRESP_ERR_MASK); ret = -EIO; + } else { + sev_write_init_ex_file_if_required(cmd); } print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, @@ -251,37 +362,71 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } -static int __sev_platform_init_locked(int *error) +static int __sev_init_locked(int *error) { - struct psp_device *psp = psp_master; struct sev_data_init data; - struct sev_device *sev; - int psp_ret, rc = 0; - if (!psp || !psp->sev_data) - return -ENODEV; + memset(&data, 0, sizeof(data)); + if (sev_es_tmr) { + /* + * Do not include the encryption mask on the physical + * address of the TMR (firmware should clear it anyway). + */ + data.tmr_address = __pa(sev_es_tmr); - sev = psp->sev_data; + data.flags |= SEV_INIT_FLAGS_SEV_ES; + data.tmr_len = SEV_ES_TMR_SIZE; + } - if (sev->state == SEV_STATE_INIT) - return 0; + return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error); +} + +static int __sev_init_ex_locked(int *error) +{ + struct sev_data_init_ex data; + int ret; memset(&data, 0, sizeof(data)); - if (sev_es_tmr) { - u64 tmr_pa; + data.length = sizeof(data); + data.nv_address = __psp_pa(sev_init_ex_buffer); + data.nv_len = NV_LENGTH; + + ret = sev_read_init_ex_file(); + if (ret) + return ret; + if (sev_es_tmr) { /* * Do not include the encryption mask on the physical * address of the TMR (firmware should clear it anyway). */ - tmr_pa = __pa(sev_es_tmr); + data.tmr_address = __pa(sev_es_tmr); data.flags |= SEV_INIT_FLAGS_SEV_ES; - data.tmr_address = tmr_pa; data.tmr_len = SEV_ES_TMR_SIZE; } - rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error); +} + +static int __sev_platform_init_locked(int *error) +{ + struct psp_device *psp = psp_master; + struct sev_device *sev; + int rc, psp_ret; + int (*init_function)(int *error); + + if (!psp || !psp->sev_data) + return -ENODEV; + + sev = psp->sev_data; + + if (sev->state == SEV_STATE_INIT) + return 0; + + init_function = sev_init_ex_buffer ? __sev_init_ex_locked : + __sev_init_locked; + rc = init_function(&psp_ret); if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { /* * Initialization command returned an integrity check failure @@ -291,7 +436,7 @@ static int __sev_platform_init_locked(int *error) * with a reset state. */ dev_dbg(sev->dev, "SEV: retrying INIT command"); - rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, &psp_ret); + rc = init_function(&psp_ret); } if (error) *error = psp_ret; @@ -1066,6 +1211,12 @@ static void sev_firmware_shutdown(struct sev_device *sev) get_order(SEV_ES_TMR_SIZE)); sev_es_tmr = NULL; } + + if (sev_init_ex_buffer) { + free_pages((unsigned long)sev_init_ex_buffer, + get_order(NV_LENGTH)); + sev_init_ex_buffer = NULL; + } } void sev_dev_destroy(struct psp_device *psp) @@ -1110,6 +1261,18 @@ void sev_pci_init(void) sev_update_firmware(sev->dev) == 0) sev_get_api_version(); + /* If an init_ex_path is provided rely on INIT_EX for PSP initialization + * instead of INIT. + */ + if (init_ex_path) { + sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH); + if (!sev_init_ex_buffer) { + dev_err(sev->dev, + "SEV: INIT_EX NV memory allocation failed\n"); + goto err; + } + } + /* Obtain the TMR memory area for SEV-ES use */ sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE); if (!sev_es_tmr) diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index d48a7192e881..1595088c428b 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -52,6 +52,7 @@ enum sev_cmd { SEV_CMD_DF_FLUSH = 0x00A, SEV_CMD_DOWNLOAD_FIRMWARE = 0x00B, SEV_CMD_GET_ID = 0x00C, + SEV_CMD_INIT_EX = 0x00D, /* Guest commands */ SEV_CMD_DECOMMISSION = 0x020, @@ -102,6 +103,26 @@ struct sev_data_init { u32 tmr_len; /* In */ } __packed; +/** + * struct sev_data_init_ex - INIT_EX command parameters + * + * @length: len of the command buffer read by the PSP + * @flags: processing flags + * @tmr_address: system physical address used for SEV-ES + * @tmr_len: len of tmr_address + * @nv_address: system physical address used for PSP NV storage + * @nv_len: len of nv_address + */ +struct sev_data_init_ex { + u32 length; /* In */ + u32 flags; /* In */ + u64 tmr_address; /* In */ + u32 tmr_len; /* In */ + u32 reserved; /* In */ + u64 nv_address; /* In/Out */ + u32 nv_len; /* In */ +} __packed; + #define SEV_INIT_FLAGS_SEV_ES 0x01 /** -- cgit v1.2.3-59-g8ed1b From 9dfa5b6f5efb85efe69fd3b7b0b912004d9547f1 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 16 Dec 2021 09:17:03 +0800 Subject: iommu/vt-d: Remove unused macros These macros has no reference in the tree anymore. Cleanup them. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211216011703.763331-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/intel-svm.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel-svm.h b/include/linux/intel-svm.h index 57cceecbe37f..1b73bab7eeff 100644 --- a/include/linux/intel-svm.h +++ b/include/linux/intel-svm.h @@ -8,12 +8,6 @@ #ifndef __INTEL_SVM_H__ #define __INTEL_SVM_H__ -/* Values for rxwp in fault_cb callback */ -#define SVM_REQ_READ (1<<3) -#define SVM_REQ_WRITE (1<<2) -#define SVM_REQ_EXEC (1<<1) -#define SVM_REQ_PRIV (1<<0) - /* Page Request Queue depth */ #define PRQ_ORDER 2 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) -- cgit v1.2.3-59-g8ed1b From dd61b29207ca4f346fbd9c06bc49f093e3369185 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 7 Dec 2021 10:34:06 +0100 Subject: gpiolib: provide gpiod_remove_hogs() Currently all users of gpiod_add_hogs() call it only once at system init so there never was any need for a mechanism allowing to remove them. Now the upcoming gpio-sim will need to tear down chips with hogged lines so provide a function that allows to remove hogs. Signed-off-by: Bartosz Golaszewski Reviewed-by: Andy Shevchenko Acked-by: Linus Walleij --- drivers/gpio/gpiolib.c | 11 +++++++++++ include/linux/gpio/machine.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 535eb1b509c1..467f4a8586a0 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -3557,6 +3557,17 @@ void gpiod_add_hogs(struct gpiod_hog *hogs) } EXPORT_SYMBOL_GPL(gpiod_add_hogs); +void gpiod_remove_hogs(struct gpiod_hog *hogs) +{ + struct gpiod_hog *hog; + + mutex_lock(&gpio_machine_hogs_mutex); + for (hog = &hogs[0]; hog->chip_label; hog++) + list_del(&hog->list); + mutex_unlock(&gpio_machine_hogs_mutex); +} +EXPORT_SYMBOL_GPL(gpiod_remove_hogs); + static struct gpiod_lookup_table *gpiod_find_lookup_table(struct device *dev) { const char *dev_id = dev ? dev_name(dev) : NULL; diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h index d755e529c1e3..2647dd10b541 100644 --- a/include/linux/gpio/machine.h +++ b/include/linux/gpio/machine.h @@ -100,6 +100,7 @@ void gpiod_add_lookup_table(struct gpiod_lookup_table *table); void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n); void gpiod_remove_lookup_table(struct gpiod_lookup_table *table); void gpiod_add_hogs(struct gpiod_hog *hogs); +void gpiod_remove_hogs(struct gpiod_hog *hogs); #else /* ! CONFIG_GPIOLIB */ static inline void gpiod_add_lookup_table(struct gpiod_lookup_table *table) {} @@ -108,6 +109,7 @@ void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n) {} static inline void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) {} static inline void gpiod_add_hogs(struct gpiod_hog *hogs) {} +static inline void gpiod_remove_hogs(struct gpiod_hog *hogs) {} #endif /* CONFIG_GPIOLIB */ #endif /* __LINUX_GPIO_MACHINE_H */ -- cgit v1.2.3-59-g8ed1b From 990f6756bb64756d2d1033118cded6333b43397d Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 13 Dec 2021 11:16:41 +0100 Subject: gpiolib: allow to specify the firmware node in struct gpio_chip Software nodes allow us to represent hierarchies for device components that don't have their struct device representation yet - for instance: banks of GPIOs under a common GPIO expander. The core gpiolib core however doesn't offer any way of passing this information from the drivers. This extends struct gpio_chip with a pointer to fwnode that can be set by the driver and used to pass device properties for child nodes. This is similar to how we handle device-tree sub-nodes with CONFIG_OF_GPIO enabled. Signed-off-by: Bartosz Golaszewski Reviewed-by: Andy Shevchenko Acked-by: Linus Walleij --- drivers/gpio/gpiolib.c | 7 ++++++- include/linux/gpio/driver.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 467f4a8586a0..4d666767c9ab 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -593,7 +593,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct lock_class_key *lock_key, struct lock_class_key *request_key) { - struct fwnode_handle *fwnode = gc->parent ? dev_fwnode(gc->parent) : NULL; + struct fwnode_handle *fwnode = NULL; struct gpio_device *gdev; unsigned long flags; int base = gc->base; @@ -601,6 +601,11 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, int ret = 0; u32 ngpios; + if (gc->fwnode) + fwnode = gc->fwnode; + else if (gc->parent) + fwnode = dev_fwnode(gc->parent); + /* * First: allocate and populate the internal stat container, and * set up the struct device. diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index a673a359e20b..b0728c8ad90c 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -289,6 +289,7 @@ struct gpio_irq_chip { * number or the name of the SoC IP-block implementing it. * @gpiodev: the internal state holder, opaque struct * @parent: optional parent device providing the GPIOs + * @fwnode: optional fwnode providing this controller's properties * @owner: helps prevent removal of modules exporting active GPIOs * @request: optional hook for chip-specific activation, such as * enabling module power and clock; may sleep @@ -377,6 +378,7 @@ struct gpio_chip { const char *label; struct gpio_device *gpiodev; struct device *parent; + struct fwnode_handle *fwnode; struct module *owner; int (*request)(struct gpio_chip *gc, -- cgit v1.2.3-59-g8ed1b From c06ef740d401d0f4ab188882bf6f8d9cf0f75eaf Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 7 Dec 2021 00:20:59 +0000 Subject: PM: core: Redefine pm_ptr() macro The pm_ptr() macro was previously conditionally defined, according to the value of the CONFIG_PM option. This meant that the pointed structure was either referenced (if CONFIG_PM was set), or never referenced (if CONFIG_PM was not set), causing it to be detected as unused by the compiler. This worked fine, but required the __maybe_unused compiler attribute to be used to every symbol pointed to by a pointer wrapped with pm_ptr(). We can do better. With this change, the pm_ptr() is now defined the same, independently of the value of CONFIG_PM. It now uses the (?:) ternary operator to conditionally resolve to its argument. Since the condition is known at compile time, the compiler will then choose to discard the unused symbols, which won't need to be tagged with __maybe_unused anymore. This pm_ptr() macro is usually used with pointers to dev_pm_ops structures created with SIMPLE_DEV_PM_OPS() or similar macros. These do use a __maybe_unused flag, which is now useless with this change, so it later can be removed. However in the meantime it causes no harm, and all the drivers still compile fine with the new pm_ptr() macro. Signed-off-by: Paul Cercueil Reviewed-by: Jonathan Cameron Reviewed-by: Arnd Bergmann Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index 1d8209c09686..b88ac7dcf2a2 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -373,11 +373,7 @@ const struct dev_pm_ops __maybe_unused name = { \ SET_RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ } -#ifdef CONFIG_PM -#define pm_ptr(_ptr) (_ptr) -#else -#define pm_ptr(_ptr) NULL -#endif +#define pm_ptr(_ptr) PTR_IF(IS_ENABLED(CONFIG_PM), (_ptr)) /* * PM_EVENT_ messages -- cgit v1.2.3-59-g8ed1b From 1a3c7bb088266fa2db017be299f91f1c1894c857 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 7 Dec 2021 00:21:00 +0000 Subject: PM: core: Add new *_PM_OPS macros, deprecate old ones This commit introduces the following macros: SYSTEM_SLEEP_PM_OPS() LATE_SYSTEM_SLEEP_PM_OPS() NOIRQ_SYSTEM_SLEEP_PM_OPS() RUNTIME_PM_OPS() These new macros are very similar to their SET_*_PM_OPS() equivalent. They however differ in the fact that the callbacks they set will always be seen as referenced by the compiler. This means that the callback functions don't need to be wrapped with a #ifdef CONFIG_PM guard, or tagged with __maybe_unused, to prevent the compiler from complaining about unused static symbols. The compiler will then simply evaluate at compile time whether or not these symbols are dead code. The callbacks that are only useful with CONFIG_PM_SLEEP is enabled, are now also wrapped with a new pm_sleep_ptr() macro, which is inspired from pm_ptr(). This is needed for drivers that use different callbacks for sleep and runtime PM, to handle the case where CONFIG_PM is set and CONFIG_PM_SLEEP is not. This commit also deprecates the following macros: SIMPLE_DEV_PM_OPS() UNIVERSAL_DEV_PM_OPS() And introduces the following macros: DEFINE_SIMPLE_DEV_PM_OPS() DEFINE_UNIVERSAL_DEV_PM_OPS() These macros are similar to the functions they were created to replace, with the following differences: - They use the new macros introduced above, and as such always reference the provided callback functions. - They are not tagged with __maybe_unused. They are meant to be used with pm_ptr() or pm_sleep_ptr() for DEFINE_UNIVERSAL_DEV_PM_OPS() and DEFINE_SIMPLE_DEV_PM_OPS() respectively. - They declare the symbol static, since every driver seems to do that anyway; and if a non-static use-case is needed an indirection pointer could be used. The point of this change, is to progressively switch from a code model where PM callbacks are all protected behind CONFIG_PM guards, to a code model where the PM callbacks are always seen by the compiler, but discarded if not used. Signed-off-by: Paul Cercueil Reviewed-by: Jonathan Cameron Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 74 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index b88ac7dcf2a2..fc9691cb01b4 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -300,47 +300,59 @@ struct dev_pm_ops { int (*runtime_idle)(struct device *dev); }; +#define SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + .suspend = pm_sleep_ptr(suspend_fn), \ + .resume = pm_sleep_ptr(resume_fn), \ + .freeze = pm_sleep_ptr(suspend_fn), \ + .thaw = pm_sleep_ptr(resume_fn), \ + .poweroff = pm_sleep_ptr(suspend_fn), \ + .restore = pm_sleep_ptr(resume_fn), + +#define LATE_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + .suspend_late = pm_sleep_ptr(suspend_fn), \ + .resume_early = pm_sleep_ptr(resume_fn), \ + .freeze_late = pm_sleep_ptr(suspend_fn), \ + .thaw_early = pm_sleep_ptr(resume_fn), \ + .poweroff_late = pm_sleep_ptr(suspend_fn), \ + .restore_early = pm_sleep_ptr(resume_fn), + +#define NOIRQ_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + .suspend_noirq = pm_sleep_ptr(suspend_fn), \ + .resume_noirq = pm_sleep_ptr(resume_fn), \ + .freeze_noirq = pm_sleep_ptr(suspend_fn), \ + .thaw_noirq = pm_sleep_ptr(resume_fn), \ + .poweroff_noirq = pm_sleep_ptr(suspend_fn), \ + .restore_noirq = pm_sleep_ptr(resume_fn), + +#define RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ + .runtime_suspend = suspend_fn, \ + .runtime_resume = resume_fn, \ + .runtime_idle = idle_fn, + #ifdef CONFIG_PM_SLEEP #define SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ - .suspend = suspend_fn, \ - .resume = resume_fn, \ - .freeze = suspend_fn, \ - .thaw = resume_fn, \ - .poweroff = suspend_fn, \ - .restore = resume_fn, + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #else #define SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #endif #ifdef CONFIG_PM_SLEEP #define SET_LATE_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ - .suspend_late = suspend_fn, \ - .resume_early = resume_fn, \ - .freeze_late = suspend_fn, \ - .thaw_early = resume_fn, \ - .poweroff_late = suspend_fn, \ - .restore_early = resume_fn, + LATE_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #else #define SET_LATE_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #endif #ifdef CONFIG_PM_SLEEP #define SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ - .suspend_noirq = suspend_fn, \ - .resume_noirq = resume_fn, \ - .freeze_noirq = suspend_fn, \ - .thaw_noirq = resume_fn, \ - .poweroff_noirq = suspend_fn, \ - .restore_noirq = resume_fn, + NOIRQ_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #else #define SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) #endif #ifdef CONFIG_PM #define SET_RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ - .runtime_suspend = suspend_fn, \ - .runtime_resume = resume_fn, \ - .runtime_idle = idle_fn, + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) #else #define SET_RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) #endif @@ -349,9 +361,9 @@ struct dev_pm_ops { * Use this if you want to use the same suspend and resume callbacks for suspend * to RAM and hibernation. */ -#define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ -const struct dev_pm_ops __maybe_unused name = { \ - SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ +#define DEFINE_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ +static const struct dev_pm_ops name = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ } /* @@ -367,6 +379,19 @@ const struct dev_pm_ops __maybe_unused name = { \ * .resume_early(), to the same routines as .runtime_suspend() and * .runtime_resume(), respectively (and analogously for hibernation). */ +#define DEFINE_UNIVERSAL_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ +static const struct dev_pm_ops name = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ +} + +/* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */ +#define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ +const struct dev_pm_ops __maybe_unused name = { \ + SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ +} + +/* Deprecated. Use DEFINE_UNIVERSAL_DEV_PM_OPS() instead. */ #define UNIVERSAL_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ const struct dev_pm_ops __maybe_unused name = { \ SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ @@ -374,6 +399,7 @@ const struct dev_pm_ops __maybe_unused name = { \ } #define pm_ptr(_ptr) PTR_IF(IS_ENABLED(CONFIG_PM), (_ptr)) +#define pm_sleep_ptr(_ptr) PTR_IF(IS_ENABLED(CONFIG_PM_SLEEP), (_ptr)) /* * PM_EVENT_ messages -- cgit v1.2.3-59-g8ed1b From 931da6a0de5d620425af4425344259e6ff46b654 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Tue, 7 Dec 2021 21:17:34 +0800 Subject: powercap: intel_rapl: support new layout of Psys PowerLimit Register on SPR On Sapphire Rapids, the layout of the Psys domain Power Limit Register is different from from what it was before. Enhance the code to support the new Psys PL register layout. Signed-off-by: Zhang Rui Reported-and-tested-by: Alkattan Dana [ rjw: Subject and changelog edits ] Signed-off-by: Rafael J. Wysocki --- drivers/powercap/intel_rapl_common.c | 61 ++++++++++++++++++++++++++++++++++-- include/linux/intel_rapl.h | 6 ++++ 2 files changed, 65 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 7c0099e7a6d7..07611a00b78f 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -61,6 +61,20 @@ #define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff #define PP_POLICY_MASK 0x1F +/* + * SPR has different layout for Psys Domain PowerLimit registers. + * There are 17 bits of PL1 and PL2 instead of 15 bits. + * The Enable bits and TimeWindow bits are also shifted as a result. + */ +#define PSYS_POWER_LIMIT1_MASK 0x1FFFF +#define PSYS_POWER_LIMIT1_ENABLE BIT(17) + +#define PSYS_POWER_LIMIT2_MASK (0x1FFFFULL<<32) +#define PSYS_POWER_LIMIT2_ENABLE BIT_ULL(49) + +#define PSYS_TIME_WINDOW1_MASK (0x7FULL<<19) +#define PSYS_TIME_WINDOW2_MASK (0x7FULL<<51) + /* Non HW constants */ #define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */ #define RAPL_PRIMITIVE_DUMMY BIT(2) @@ -97,6 +111,7 @@ struct rapl_defaults { bool to_raw); unsigned int dram_domain_energy_unit; unsigned int psys_domain_energy_unit; + bool spr_psys_bits; }; static struct rapl_defaults *rapl_defaults; @@ -669,12 +684,51 @@ static struct rapl_primitive_info rpi[] = { RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0), PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0, RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0, + RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32, + RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19, + RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), + PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51, + RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), /* non-hardware */ PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT, RAPL_PRIMITIVE_DERIVED), {NULL, 0, 0, 0}, }; +static enum rapl_primitives +prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) +{ + if (!rapl_defaults->spr_psys_bits) + return prim; + + if (rd->id != RAPL_DOMAIN_PLATFORM) + return prim; + + switch (prim) { + case POWER_LIMIT1: + return PSYS_POWER_LIMIT1; + case POWER_LIMIT2: + return PSYS_POWER_LIMIT2; + case PL1_ENABLE: + return PSYS_PL1_ENABLE; + case PL2_ENABLE: + return PSYS_PL2_ENABLE; + case TIME_WINDOW1: + return PSYS_TIME_WINDOW1; + case TIME_WINDOW2: + return PSYS_TIME_WINDOW2; + default: + return prim; + } +} + /* Read primitive data based on its related struct rapl_primitive_info. * if xlate flag is set, return translated data based on data units, i.e. * time, energy, and power. @@ -692,7 +746,8 @@ static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, bool xlate, u64 *data) { u64 value; - struct rapl_primitive_info *rp = &rpi[prim]; + enum rapl_primitives prim_fixed = prim_fixups(rd, prim); + struct rapl_primitive_info *rp = &rpi[prim_fixed]; struct reg_action ra; int cpu; @@ -738,7 +793,8 @@ static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value) { - struct rapl_primitive_info *rp = &rpi[prim]; + enum rapl_primitives prim_fixed = prim_fixups(rd, prim); + struct rapl_primitive_info *rp = &rpi[prim_fixed]; int cpu; u64 bits; struct reg_action ra; @@ -981,6 +1037,7 @@ static const struct rapl_defaults rapl_defaults_spr_server = { .compute_time_window = rapl_compute_time_window_core, .dram_domain_energy_unit = 15300, .psys_domain_energy_unit = 1000000000, + .spr_psys_bits = true, }; static const struct rapl_defaults rapl_defaults_byt = { diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 93780834fc8f..9f4b6f5b822f 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -58,6 +58,12 @@ enum rapl_primitives { THROTTLED_TIME, PRIORITY_LEVEL, + PSYS_POWER_LIMIT1, + PSYS_POWER_LIMIT2, + PSYS_PL1_ENABLE, + PSYS_PL2_ENABLE, + PSYS_TIME_WINDOW1, + PSYS_TIME_WINDOW2, /* below are not raw primitive data */ AVERAGE_POWER, NR_RAPL_PRIMITIVES, -- cgit v1.2.3-59-g8ed1b From c24efa6732788f0be22cdf5d2aedd5e3117e983f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 7 Dec 2021 19:54:32 +0100 Subject: PM: runtime: Capture device status before disabling runtime PM In some cases (for example, during system-wide suspend and resume of devices) it is useful to know whether or not runtime PM has ever been enabled for a given device and, if so, what the runtime PM status of it had been right before runtime PM was disabled for it last time. For this reason, introduce a new struct dev_pm_info field called last_status that will be used for capturing the runtime PM status of the device when its power.disable_depth counter changes from 0 to 1. The new field will be set to RPM_INVALID to start with and whenever power.disable_depth changes from 1 to 0, so it will be valid only when runtime PM of the device is currently disabled, but it has been enabled at least once. Immediately use power.last_status in rpm_resume() to make it handle the case when PM runtime is disabled for the device, but its runtime PM status is RPM_ACTIVE more consistently. Namely, make it return 1 if power.last_status is also equal to RPM_ACTIVE in that case (the idea being that if the status was RPM_ACTIVE last time when power.disable_depth was changing from 0 to 1 and it is still RPM_ACTIVE, it can be assumed to reflect what happened to the device last time when it was using runtime PM) and -EACCES otherwise. Update the documentation to provide a description of last_status and change the description of pm_runtime_resume() in it to reflect the new behavior of rpm_active(). While at it, rearrange the code in pm_runtime_enable() to be more straightforward and replace the WARN() macro in it with a pr_warn() invocation which is less disruptive. Link: https://lore.kernel.org/linux-pm/20211026222626.39222-1-ulf.hansson@linaro.org/t/#u Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- Documentation/power/runtime_pm.rst | 14 ++++++++---- drivers/base/power/runtime.c | 45 +++++++++++++++++++++----------------- include/linux/pm.h | 2 ++ 3 files changed, 37 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index d6bf84f061f4..65b86e487afe 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -265,6 +265,10 @@ defined in include/linux/pm.h: RPM_SUSPENDED, which means that each device is initially regarded by the PM core as 'suspended', regardless of its real hardware status + `enum rpm_status last_status;` + - the last runtime PM status of the device captured before disabling runtime + PM for it (invalid initially and when disable_depth is 0) + `unsigned int runtime_auto;` - if set, indicates that the user space has allowed the device driver to power manage the device at run time via the /sys/devices/.../power/control @@ -333,10 +337,12 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: `int pm_runtime_resume(struct device *dev);` - execute the subsystem-level resume callback for the device; returns 0 on - success, 1 if the device's runtime PM status was already 'active' or - error code on failure, where -EAGAIN means it may be safe to attempt to - resume the device again in future, but 'power.runtime_error' should be - checked additionally, and -EACCES means that 'power.disable_depth' is + success, 1 if the device's runtime PM status is already 'active' (also if + 'power.disable_depth' is nonzero, but the status was 'active' when it was + changing from 0 to 1) or error code on failure, where -EAGAIN means it may + be safe to attempt to resume the device again in future, but + 'power.runtime_error' should be checked additionally, and -EACCES means + that the callback could not be run, because 'power.disable_depth' was different from 0 `int pm_runtime_resume_and_get(struct device *dev);` diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index d504cd4ab3cb..844b6b811610 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -742,13 +742,15 @@ static int rpm_resume(struct device *dev, int rpmflags) trace_rpm_resume_rcuidle(dev, rpmflags); repeat: - if (dev->power.runtime_error) + if (dev->power.runtime_error) { retval = -EINVAL; - else if (dev->power.disable_depth == 1 && dev->power.is_suspended - && dev->power.runtime_status == RPM_ACTIVE) - retval = 1; - else if (dev->power.disable_depth > 0) - retval = -EACCES; + } else if (dev->power.disable_depth > 0) { + if (dev->power.runtime_status == RPM_ACTIVE && + dev->power.last_status == RPM_ACTIVE) + retval = 1; + else + retval = -EACCES; + } if (retval) goto out; @@ -1410,8 +1412,10 @@ void __pm_runtime_disable(struct device *dev, bool check_resume) /* Update time accounting before disabling PM-runtime. */ update_pm_runtime_accounting(dev); - if (!dev->power.disable_depth++) + if (!dev->power.disable_depth++) { __pm_runtime_barrier(dev); + dev->power.last_status = dev->power.runtime_status; + } out: spin_unlock_irq(&dev->power.lock); @@ -1428,23 +1432,23 @@ void pm_runtime_enable(struct device *dev) spin_lock_irqsave(&dev->power.lock, flags); - if (dev->power.disable_depth > 0) { - dev->power.disable_depth--; - - /* About to enable runtime pm, set accounting_timestamp to now */ - if (!dev->power.disable_depth) - dev->power.accounting_timestamp = ktime_get_mono_fast_ns(); - } else { + if (!dev->power.disable_depth) { dev_warn(dev, "Unbalanced %s!\n", __func__); + goto out; } - WARN(!dev->power.disable_depth && - dev->power.runtime_status == RPM_SUSPENDED && - !dev->power.ignore_children && - atomic_read(&dev->power.child_count) > 0, - "Enabling runtime PM for inactive device (%s) with active children\n", - dev_name(dev)); + if (--dev->power.disable_depth > 0) + goto out; + dev->power.last_status = RPM_INVALID; + dev->power.accounting_timestamp = ktime_get_mono_fast_ns(); + + if (dev->power.runtime_status == RPM_SUSPENDED && + !dev->power.ignore_children && + atomic_read(&dev->power.child_count) > 0) + dev_warn(dev, "Enabling runtime PM for inactive device with active children\n"); + +out: spin_unlock_irqrestore(&dev->power.lock, flags); } EXPORT_SYMBOL_GPL(pm_runtime_enable); @@ -1640,6 +1644,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend); void pm_runtime_init(struct device *dev) { dev->power.runtime_status = RPM_SUSPENDED; + dev->power.last_status = RPM_INVALID; dev->power.idle_notification = false; dev->power.disable_depth = 1; diff --git a/include/linux/pm.h b/include/linux/pm.h index fc9691cb01b4..e1e9402180b9 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -521,6 +521,7 @@ const struct dev_pm_ops __maybe_unused name = { \ */ enum rpm_status { + RPM_INVALID = -1, RPM_ACTIVE = 0, RPM_RESUMING, RPM_SUSPENDED, @@ -634,6 +635,7 @@ struct dev_pm_info { unsigned int links_count; enum rpm_request request; enum rpm_status runtime_status; + enum rpm_status last_status; int runtime_error; int autosuspend_delay; u64 last_busy; -- cgit v1.2.3-59-g8ed1b From d1579e61192e0e686faa4208500ef4c3b529b16c Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 10 Dec 2021 17:10:13 +0100 Subject: PM: runtime: Add safety net to supplier device release Because refcount_dec_not_one() returns true if the target refcount becomes saturated, it is generally unsafe to use its return value as a loop termination condition, but that is what happens when a device link's supplier device is released during runtime PM suspend operations and on device link removal. To address this, introduce pm_runtime_release_supplier() to be used in the above cases which will check the supplier device's runtime PM usage counter in addition to the refcount_dec_not_one() return value, so the loop can be terminated in case the rpm_active refcount value becomes invalid, and update the code in question to use it as appropriate. This change is not expected to have any visible functional impact. Reported-by: Peter Zijlstra Signed-off-by: Rafael J. Wysocki Acked-by: Greg Kroah-Hartman Acked-by: Peter Zijlstra (Intel) --- drivers/base/core.c | 3 +-- drivers/base/power/runtime.c | 41 ++++++++++++++++++++++++++++++----------- include/linux/pm_runtime.h | 3 +++ 3 files changed, 34 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index fd034d742447..b191bd17de89 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -485,8 +485,7 @@ static void device_link_release_fn(struct work_struct *work) /* Ensure that all references to the link object have been dropped. */ device_link_synchronize_removal(); - while (refcount_dec_not_one(&link->rpm_active)) - pm_runtime_put(link->supplier); + pm_runtime_release_supplier(link, true); put_device(link->consumer); put_device(link->supplier); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 844b6b811610..f1b856ea8e10 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -305,19 +305,40 @@ static int rpm_get_suppliers(struct device *dev) return 0; } +/** + * pm_runtime_release_supplier - Drop references to device link's supplier. + * @link: Target device link. + * @check_idle: Whether or not to check if the supplier device is idle. + * + * Drop all runtime PM references associated with @link to its supplier device + * and if @check_idle is set, check if that device is idle (and so it can be + * suspended). + */ +void pm_runtime_release_supplier(struct device_link *link, bool check_idle) +{ + struct device *supplier = link->supplier; + + /* + * The additional power.usage_count check is a safety net in case + * the rpm_active refcount becomes saturated, in which case + * refcount_dec_not_one() would return true forever, but it is not + * strictly necessary. + */ + while (refcount_dec_not_one(&link->rpm_active) && + atomic_read(&supplier->power.usage_count) > 0) + pm_runtime_put_noidle(supplier); + + if (check_idle) + pm_request_idle(supplier); +} + static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend) { struct device_link *link; list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, - device_links_read_lock_held()) { - - while (refcount_dec_not_one(&link->rpm_active)) - pm_runtime_put_noidle(link->supplier); - - if (try_to_suspend) - pm_request_idle(link->supplier); - } + device_links_read_lock_held()) + pm_runtime_release_supplier(link, try_to_suspend); } static void rpm_put_suppliers(struct device *dev) @@ -1777,9 +1798,7 @@ void pm_runtime_drop_link(struct device_link *link) return; pm_runtime_drop_link_count(link->consumer); - - while (refcount_dec_not_one(&link->rpm_active)) - pm_runtime_put(link->supplier); + pm_runtime_release_supplier(link, true); } static bool pm_runtime_need_not_resume(struct device *dev) diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index eddd66d426ca..016de5776b6d 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -58,6 +58,7 @@ extern void pm_runtime_get_suppliers(struct device *dev); extern void pm_runtime_put_suppliers(struct device *dev); extern void pm_runtime_new_link(struct device *dev); extern void pm_runtime_drop_link(struct device_link *link); +extern void pm_runtime_release_supplier(struct device_link *link, bool check_idle); extern int devm_pm_runtime_enable(struct device *dev); @@ -283,6 +284,8 @@ static inline void pm_runtime_get_suppliers(struct device *dev) {} static inline void pm_runtime_put_suppliers(struct device *dev) {} static inline void pm_runtime_new_link(struct device *dev) {} static inline void pm_runtime_drop_link(struct device_link *link) {} +static inline void pm_runtime_release_supplier(struct device_link *link, + bool check_idle) {} #endif /* !CONFIG_PM */ -- cgit v1.2.3-59-g8ed1b From 227fee5fc99eeb74d43bf68832f6d59d30ac07d8 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Thu, 16 Dec 2021 13:42:25 +0530 Subject: bus: mhi: core: Add an API for auto queueing buffers for DL channel Add a new API "mhi_prepare_for_transfer_autoqueue" for using with client drivers like QRTR to request MHI core to autoqueue buffers for the DL channel along with starting both UL and DL channels. So far, the "auto_queue" flag specified by the controller drivers in channel definition served this purpose but this will be removed at some point in future. Cc: netdev@vger.kernel.org Cc: Jakub Kicinski Cc: David S. Miller Cc: Greg Kroah-Hartman Co-developed-by: Loic Poulain Acked-by: Jakub Kicinski Signed-off-by: Loic Poulain Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20211216081227.237749-9-manivannan.sadhasivam@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/bus/mhi/core/internal.h | 6 +++++- drivers/bus/mhi/core/main.c | 21 +++++++++++++++++---- include/linux/mhi.h | 21 ++++++++++++++++----- net/qrtr/mhi.c | 2 +- 4 files changed, 39 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h index 9d72b1d1e986..e2e10474a9d9 100644 --- a/drivers/bus/mhi/core/internal.h +++ b/drivers/bus/mhi/core/internal.h @@ -682,8 +682,12 @@ void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl); void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, struct image_info *img_info); void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl); + +/* Automatically allocate and queue inbound buffers */ +#define MHI_CH_INBOUND_ALLOC_BUFS BIT(0) int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, - struct mhi_chan *mhi_chan); + struct mhi_chan *mhi_chan, unsigned int flags); + int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan); void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 930aba666b67..ffde617f93a3 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -1430,7 +1430,7 @@ exit_unprepare_channel: } int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, - struct mhi_chan *mhi_chan) + struct mhi_chan *mhi_chan, unsigned int flags) { int ret = 0; struct device *dev = &mhi_chan->mhi_dev->dev; @@ -1455,6 +1455,9 @@ int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, if (ret) goto error_pm_state; + if (mhi_chan->dir == DMA_FROM_DEVICE) + mhi_chan->pre_alloc = !!(flags & MHI_CH_INBOUND_ALLOC_BUFS); + /* Pre-allocate buffer for xfer ring */ if (mhi_chan->pre_alloc) { int nr_el = get_nr_avail_ring_elements(mhi_cntrl, @@ -1610,8 +1613,7 @@ void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan) read_unlock_bh(&mhi_cntrl->pm_lock); } -/* Move channel to start state */ -int mhi_prepare_for_transfer(struct mhi_device *mhi_dev) +static int __mhi_prepare_for_transfer(struct mhi_device *mhi_dev, unsigned int flags) { int ret, dir; struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; @@ -1622,7 +1624,7 @@ int mhi_prepare_for_transfer(struct mhi_device *mhi_dev) if (!mhi_chan) continue; - ret = mhi_prepare_channel(mhi_cntrl, mhi_chan); + ret = mhi_prepare_channel(mhi_cntrl, mhi_chan, flags); if (ret) goto error_open_chan; } @@ -1640,8 +1642,19 @@ error_open_chan: return ret; } + +int mhi_prepare_for_transfer(struct mhi_device *mhi_dev) +{ + return __mhi_prepare_for_transfer(mhi_dev, 0); +} EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer); +int mhi_prepare_for_transfer_autoqueue(struct mhi_device *mhi_dev) +{ + return __mhi_prepare_for_transfer(mhi_dev, MHI_CH_INBOUND_ALLOC_BUFS); +} +EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer_autoqueue); + void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev) { struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; diff --git a/include/linux/mhi.h b/include/linux/mhi.h index a5cc4cdf9cc8..a5441ad33c74 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -730,15 +730,26 @@ void mhi_device_put(struct mhi_device *mhi_dev); /** * mhi_prepare_for_transfer - Setup UL and DL channels for data transfer. - * Allocate and initialize the channel context and - * also issue the START channel command to both - * channels. Channels can be started only if both - * host and device execution environments match and - * channels are in a DISABLED state. * @mhi_dev: Device associated with the channels + * + * Allocate and initialize the channel context and also issue the START channel + * command to both channels. Channels can be started only if both host and + * device execution environments match and channels are in a DISABLED state. */ int mhi_prepare_for_transfer(struct mhi_device *mhi_dev); +/** + * mhi_prepare_for_transfer_autoqueue - Setup UL and DL channels with auto queue + * buffers for DL traffic + * @mhi_dev: Device associated with the channels + * + * Allocate and initialize the channel context and also issue the START channel + * command to both channels. Channels can be started only if both host and + * device execution environments match and channels are in a DISABLED state. + * The MHI core will automatically allocate and queue buffers for the DL traffic. + */ +int mhi_prepare_for_transfer_autoqueue(struct mhi_device *mhi_dev); + /** * mhi_unprepare_from_transfer - Reset UL and DL channels for data transfer. * Issue the RESET channel command and let the diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index fa611678af05..18196e1c8c2f 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -79,7 +79,7 @@ static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, int rc; /* start channels */ - rc = mhi_prepare_for_transfer(mhi_dev); + rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); if (rc) return rc; -- cgit v1.2.3-59-g8ed1b From 49f39cb0ef198ae3c73765c9b9ee3034e4c9f076 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Wed, 1 Dec 2021 14:59:30 +0200 Subject: device property: Fix documentation for FWNODE_GRAPH_DEVICE_DISABLED FWNODE_GRAPH_DEVICE_DISABLED flag was meant for also returning endpoints connected to disabled devices, but it also may return endpoints that are not connected. Fix this in documentation. Also fwnode_graph_get_endpoint_by_id() was affeced by this. Also improve the language a little bit. Fixes: 0fcc2bdc8aff ("device property: Add fwnode_graph_get_endpoint_by_id()") Signed-off-by: Sakari Ailus Reviewed-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/base/property.c | 4 ++-- include/linux/property.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index b7b3a7b86006..e2860abf9889 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -1063,8 +1063,8 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_node); * has not been found, look for the closest endpoint ID greater than the * specified one and return the endpoint that corresponds to it, if present. * - * Do not return endpoints that belong to disabled devices, unless - * FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags. + * Does not return endpoints that belong to disabled devices or endpoints that + * are unconnected, unless FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags. * * The returned endpoint needs to be released by calling fwnode_handle_put() on * it when it is not needed any more. diff --git a/include/linux/property.h b/include/linux/property.h index 16f736c698a2..7a2df45ec3ae 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -414,7 +414,8 @@ static inline bool fwnode_graph_is_endpoint(struct fwnode_handle *fwnode) * one. * @FWNODE_GRAPH_DEVICE_DISABLED: That the device to which the remote * endpoint of the given endpoint belongs to, - * may be disabled. + * may be disabled, or that the endpoint is not + * connected. */ #define FWNODE_GRAPH_ENDPOINT_NEXT BIT(0) #define FWNODE_GRAPH_DEVICE_DISABLED BIT(1) -- cgit v1.2.3-59-g8ed1b From c87b8fc569667610b4891cad1e4a663e5a94d8f8 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Wed, 1 Dec 2021 14:59:33 +0200 Subject: device property: Implement fwnode_graph_get_endpoint_count() Add fwnode_graph_get_endpoint_count() function to provide generic implementation of of_graph_get_endpoint_count(). The former by default only counts endpoints to available devices which is consistent with the rest of the fwnode graph API. By providing FWNODE_GRAPH_DEVICE_DISABLED flag, also unconnected endpoints and endpoints to disabled devices are counted. Signed-off-by: Sakari Ailus Reviewed-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/base/property.c | 49 ++++++++++++++++++++++++++++++++++++++---------- include/linux/property.h | 2 ++ 2 files changed, 41 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index e2860abf9889..39435ba64dac 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -1049,6 +1049,18 @@ fwnode_graph_get_remote_node(const struct fwnode_handle *fwnode, u32 port_id, } EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_node); +static bool fwnode_graph_remote_available(struct fwnode_handle *ep) +{ + struct fwnode_handle *dev_node; + bool available; + + dev_node = fwnode_graph_get_remote_port_parent(ep); + available = fwnode_device_is_available(dev_node); + fwnode_handle_put(dev_node); + + return available; +} + /** * fwnode_graph_get_endpoint_by_id - get endpoint by port and endpoint numbers * @fwnode: parent fwnode_handle containing the graph @@ -1082,16 +1094,8 @@ fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, struct fwnode_endpoint fwnode_ep = { 0 }; int ret; - if (enabled_only) { - struct fwnode_handle *dev_node; - bool available; - - dev_node = fwnode_graph_get_remote_port_parent(ep); - available = fwnode_device_is_available(dev_node); - fwnode_handle_put(dev_node); - if (!available) - continue; - } + if (enabled_only && !fwnode_graph_remote_available(ep)) + continue; ret = fwnode_graph_parse_endpoint(ep, &fwnode_ep); if (ret < 0) @@ -1124,6 +1128,31 @@ fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, } EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id); +/** + * fwnode_graph_get_endpoint_count - Count endpoints on a device node + * @fwnode: The node related to a device + * @flags: fwnode lookup flags + * Count endpoints in a device node. + * + * If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints + * and endpoints connected to disabled devices are counted. + */ +unsigned int fwnode_graph_get_endpoint_count(struct fwnode_handle *fwnode, + unsigned long flags) +{ + struct fwnode_handle *ep; + unsigned int count = 0; + + fwnode_graph_for_each_endpoint(fwnode, ep) { + if (flags & FWNODE_GRAPH_DEVICE_DISABLED || + fwnode_graph_remote_available(ep)) + count++; + } + + return count; +} +EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_count); + /** * fwnode_graph_parse_endpoint - parse common endpoint node properties * @fwnode: pointer to endpoint fwnode_handle diff --git a/include/linux/property.h b/include/linux/property.h index 7a2df45ec3ae..8c0104871252 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -423,6 +423,8 @@ static inline bool fwnode_graph_is_endpoint(struct fwnode_handle *fwnode) struct fwnode_handle * fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, u32 port, u32 endpoint, unsigned long flags); +unsigned int fwnode_graph_get_endpoint_count(struct fwnode_handle *fwnode, + unsigned long flags); #define fwnode_graph_for_each_endpoint(fwnode, child) \ for (child = NULL; \ -- cgit v1.2.3-59-g8ed1b From c49eea6ffec626c059ace085fce1bf501b05dbc7 Mon Sep 17 00:00:00 2001 From: Sakari Ailus Date: Wed, 1 Dec 2021 15:01:15 +0200 Subject: device property: Drop fwnode_graph_get_remote_node() fwnode_graph_get_remote_node() is only used by the tegra-video driver. Convert it to use newer fwnode_graph_get_endpoint_by_id() and drop now-unused fwnode_graph_get_remote_node(). Signed-off-by: Sakari Ailus Reviewed-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/base/property.c | 38 ---------------------------------- drivers/staging/media/tegra-video/vi.c | 12 +++++++---- include/linux/property.h | 3 --- 3 files changed, 8 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index 3ff44dd92e52..5379eae478b1 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -1011,44 +1011,6 @@ fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_endpoint); -/** - * fwnode_graph_get_remote_node - get remote parent node for given port/endpoint - * @fwnode: pointer to parent fwnode_handle containing graph port/endpoint - * @port_id: identifier of the parent port node - * @endpoint_id: identifier of the endpoint node - * - * Return: Remote fwnode handle associated with remote endpoint node linked - * to @node. Use fwnode_node_put() on it when done. - */ -struct fwnode_handle * -fwnode_graph_get_remote_node(const struct fwnode_handle *fwnode, u32 port_id, - u32 endpoint_id) -{ - struct fwnode_handle *endpoint; - - fwnode_graph_for_each_endpoint(fwnode, endpoint) { - struct fwnode_endpoint fwnode_ep; - struct fwnode_handle *remote; - int ret; - - ret = fwnode_graph_parse_endpoint(endpoint, &fwnode_ep); - if (ret < 0) - continue; - - if (fwnode_ep.port != port_id || fwnode_ep.id != endpoint_id) - continue; - - remote = fwnode_graph_get_remote_port_parent(endpoint); - if (!remote) - return NULL; - - return fwnode_device_is_available(remote) ? remote : NULL; - } - - return NULL; -} -EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_node); - static bool fwnode_graph_remote_available(struct fwnode_handle *ep) { struct fwnode_handle *dev_node; diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index 69d9787d5338..d1f43f465c22 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -1845,7 +1845,6 @@ static int tegra_vi_graph_init(struct tegra_vi *vi) struct tegra_vi_channel *chan; struct fwnode_handle *fwnode = dev_fwnode(vi->dev); int ret; - struct fwnode_handle *remote = NULL; /* * Walk the links to parse the full graph. Each channel will have @@ -1857,11 +1856,16 @@ static int tegra_vi_graph_init(struct tegra_vi *vi) * next channels. */ list_for_each_entry(chan, &vi->vi_chans, list) { - remote = fwnode_graph_get_remote_node(fwnode, chan->portnos[0], - 0); - if (!remote) + struct fwnode_handle *ep, *remote; + + ep = fwnode_graph_get_endpoint_by_id(fwnode, + chan->portnos[0], 0, 0); + if (!ep) continue; + remote = fwnode_graph_get_remote_port_parent(ep); + fwnode_handle_put(ep); + ret = tegra_vi_graph_parse_one(chan, remote); fwnode_handle_put(remote); if (ret < 0 || list_empty(&chan->notifier.asd_list)) diff --git a/include/linux/property.h b/include/linux/property.h index 8c0104871252..8355f99ebd47 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -397,9 +397,6 @@ struct fwnode_handle *fwnode_graph_get_remote_port( const struct fwnode_handle *fwnode); struct fwnode_handle *fwnode_graph_get_remote_endpoint( const struct fwnode_handle *fwnode); -struct fwnode_handle * -fwnode_graph_get_remote_node(const struct fwnode_handle *fwnode, u32 port, - u32 endpoint); static inline bool fwnode_graph_is_endpoint(struct fwnode_handle *fwnode) { -- cgit v1.2.3-59-g8ed1b From bd0b536dc2e1e9828a85b1e3470ee7bafc3b36f6 Mon Sep 17 00:00:00 2001 From: Brett Creeley Date: Mon, 29 Nov 2021 16:15:59 -0800 Subject: virtchnl: Add support for new VLAN capabilities Currently VIRTCHNL only allows for VLAN filtering and offloads to happen on a single 802.1Q VLAN. Add support to filter and offload on inner, outer, and/or inner + outer VLANs. This is done by introducing the new capability VIRTCHNL_VF_OFFLOAD_VLAN_V2. The flow to negotiate this new capability is shown below. 1. VF - sets the VIRTCHNL_VF_OFFLOAD_VLAN_V2 bit in the virtchnl_vf_resource.vf_caps_flags during the VIRTCHNL_OP_GET_VF_RESOURCES request message. The VF should also set the VIRTCHNL_VF_OFFLOAD_VLAN bit in case the PF driver doesn't support the new capability. 2. PF - sets the VLAN capability bit it supports in the VIRTCHNL_OP_GET_VF_RESOURCES response message. This will either be VIRTCHNL_VF_OFFLOAD_VLAN_V2, VIRTCHNL_VF_OFFLOAD_VLAN, or none. 3. VF - If the VIRTCHNL_VF_OFFLOAD_VLAN_V2 capability was ACK'd by the PF, then the VF needs to request the VLAN capabilities of the PF/Device by issuing a VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS request. If the VIRTCHNL_VF_OFFLOAD_VLAN capability was ACK'd then the VF knows only single 802.1Q VLAN filtering/offloads are supported. If no VLAN capability is ACK'd then the PF/Device doesn't support hardware VLAN filtering/offloads for this VF. 4. PF - Populates the virtchnl_vlan_caps structure based on what it allows/supports for that VF and sends that response via VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS. After VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS is successfully negotiated the VF driver needs to interpret the capabilities supported by the underlying PF/Device. The VF will be allowed to filter/offload the inner 802.1Q, outer (various ethertype), inner 802.1Q + outer (various ethertypes), or none based on which fields are set. The VF will also need to interpret where the VLAN tag should be inserted and/or stripped based on the negotiated capabilities. Signed-off-by: Brett Creeley Tested-by: Konrad Jankowski Signed-off-by: Tony Nguyen --- include/linux/avf/virtchnl.h | 377 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 377 insertions(+) (limited to 'include/linux') diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h index b30a1bc74fc7..2ce27e8e4f19 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/avf/virtchnl.h @@ -141,6 +141,13 @@ enum virtchnl_ops { VIRTCHNL_OP_DEL_RSS_CFG = 46, VIRTCHNL_OP_ADD_FDIR_FILTER = 47, VIRTCHNL_OP_DEL_FDIR_FILTER = 48, + VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS = 51, + VIRTCHNL_OP_ADD_VLAN_V2 = 52, + VIRTCHNL_OP_DEL_VLAN_V2 = 53, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 = 54, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 = 55, + VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 = 56, + VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2 = 57, VIRTCHNL_OP_MAX, }; @@ -246,6 +253,7 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); #define VIRTCHNL_VF_OFFLOAD_REQ_QUEUES BIT(6) /* used to negotiate communicating link speeds in Mbps */ #define VIRTCHNL_VF_CAP_ADV_LINK_SPEED BIT(7) +#define VIRTCHNL_VF_OFFLOAD_VLAN_V2 BIT(15) #define VIRTCHNL_VF_OFFLOAD_VLAN BIT(16) #define VIRTCHNL_VF_OFFLOAD_RX_POLLING BIT(17) #define VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 BIT(18) @@ -475,6 +483,351 @@ struct virtchnl_vlan_filter_list { VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_vlan_filter_list); +/* This enum is used for all of the VIRTCHNL_VF_OFFLOAD_VLAN_V2_CAPS related + * structures and opcodes. + * + * VIRTCHNL_VLAN_UNSUPPORTED - This field is not supported and if a VF driver + * populates it the PF should return VIRTCHNL_STATUS_ERR_NOT_SUPPORTED. + * + * VIRTCHNL_VLAN_ETHERTYPE_8100 - This field supports 0x8100 ethertype. + * VIRTCHNL_VLAN_ETHERTYPE_88A8 - This field supports 0x88A8 ethertype. + * VIRTCHNL_VLAN_ETHERTYPE_9100 - This field supports 0x9100 ethertype. + * + * VIRTCHNL_VLAN_ETHERTYPE_AND - Used when multiple ethertypes can be supported + * by the PF concurrently. For example, if the PF can support + * VIRTCHNL_VLAN_ETHERTYPE_8100 AND VIRTCHNL_VLAN_ETHERTYPE_88A8 filters it + * would OR the following bits: + * + * VIRTHCNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * The VF would interpret this as VLAN filtering can be supported on both 0x8100 + * and 0x88A8 VLAN ethertypes. + * + * VIRTCHNL_ETHERTYPE_XOR - Used when only a single ethertype can be supported + * by the PF concurrently. For example if the PF can support + * VIRTCHNL_VLAN_ETHERTYPE_8100 XOR VIRTCHNL_VLAN_ETHERTYPE_88A8 stripping + * offload it would OR the following bits: + * + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * The VF would interpret this as VLAN stripping can be supported on either + * 0x8100 or 0x88a8 VLAN ethertypes. So when requesting VLAN stripping via + * VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 the specified ethertype will override + * the previously set value. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1 - Used to tell the VF to insert and/or + * strip the VLAN tag using the L2TAG1 field of the Tx/Rx descriptors. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 - Used to tell the VF to insert hardware + * offloaded VLAN tags using the L2TAG2 field of the Tx descriptor. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 - Used to tell the VF to strip hardware + * offloaded VLAN tags using the L2TAG2_2 field of the Rx descriptor. + * + * VIRTCHNL_VLAN_PRIO - This field supports VLAN priority bits. This is used for + * VLAN filtering if the underlying PF supports it. + * + * VIRTCHNL_VLAN_TOGGLE_ALLOWED - This field is used to say whether a + * certain VLAN capability can be toggled. For example if the underlying PF/CP + * allows the VF to toggle VLAN filtering, stripping, and/or insertion it should + * set this bit along with the supported ethertypes. + */ +enum virtchnl_vlan_support { + VIRTCHNL_VLAN_UNSUPPORTED = 0, + VIRTCHNL_VLAN_ETHERTYPE_8100 = BIT(0), + VIRTCHNL_VLAN_ETHERTYPE_88A8 = BIT(1), + VIRTCHNL_VLAN_ETHERTYPE_9100 = BIT(2), + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1 = BIT(8), + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 = BIT(9), + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2 = BIT(10), + VIRTCHNL_VLAN_PRIO = BIT(24), + VIRTCHNL_VLAN_FILTER_MASK = BIT(28), + VIRTCHNL_VLAN_ETHERTYPE_AND = BIT(29), + VIRTCHNL_VLAN_ETHERTYPE_XOR = BIT(30), + VIRTCHNL_VLAN_TOGGLE = BIT(31), +}; + +/* This structure is used as part of the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS + * for filtering, insertion, and stripping capabilities. + * + * If only outer capabilities are supported (for filtering, insertion, and/or + * stripping) then this refers to the outer most or single VLAN from the VF's + * perspective. + * + * If only inner capabilities are supported (for filtering, insertion, and/or + * stripping) then this refers to the outer most or single VLAN from the VF's + * perspective. Functionally this is the same as if only outer capabilities are + * supported. The VF driver is just forced to use the inner fields when + * adding/deleting filters and enabling/disabling offloads (if supported). + * + * If both outer and inner capabilities are supported (for filtering, insertion, + * and/or stripping) then outer refers to the outer most or single VLAN and + * inner refers to the second VLAN, if it exists, in the packet. + * + * There is no support for tunneled VLAN offloads, so outer or inner are never + * referring to a tunneled packet from the VF's perspective. + */ +struct virtchnl_vlan_supported_caps { + u32 outer; + u32 inner; +}; + +/* The PF populates these fields based on the supported VLAN filtering. If a + * field is VIRTCHNL_VLAN_UNSUPPORTED then it's not supported and the PF will + * reject any VIRTCHNL_OP_ADD_VLAN_V2 or VIRTCHNL_OP_DEL_VLAN_V2 messages using + * the unsupported fields. + * + * Also, a VF is only allowed to toggle its VLAN filtering setting if the + * VIRTCHNL_VLAN_TOGGLE bit is set. + * + * The ethertype(s) specified in the ethertype_init field are the ethertypes + * enabled for VLAN filtering. VLAN filtering in this case refers to the outer + * most VLAN from the VF's perspective. If both inner and outer filtering are + * allowed then ethertype_init only refers to the outer most VLAN as only + * VLAN ethertype supported for inner VLAN filtering is + * VIRTCHNL_VLAN_ETHERTYPE_8100. By default, inner VLAN filtering is disabled + * when both inner and outer filtering are allowed. + * + * The max_filters field tells the VF how many VLAN filters it's allowed to have + * at any one time. If it exceeds this amount and tries to add another filter, + * then the request will be rejected by the PF. To prevent failures, the VF + * should keep track of how many VLAN filters it has added and not attempt to + * add more than max_filters. + */ +struct virtchnl_vlan_filtering_caps { + struct virtchnl_vlan_supported_caps filtering_support; + u32 ethertype_init; + u16 max_filters; + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vlan_filtering_caps); + +/* This enum is used for the virtchnl_vlan_offload_caps structure to specify + * if the PF supports a different ethertype for stripping and insertion. + * + * VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION - The ethertype(s) specified + * for stripping affect the ethertype(s) specified for insertion and visa versa + * as well. If the VF tries to configure VLAN stripping via + * VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 with VIRTCHNL_VLAN_ETHERTYPE_8100 then + * that will be the ethertype for both stripping and insertion. + * + * VIRTCHNL_ETHERTYPE_MATCH_NOT_REQUIRED - The ethertype(s) specified for + * stripping do not affect the ethertype(s) specified for insertion and visa + * versa. + */ +enum virtchnl_vlan_ethertype_match { + VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION = 0, + VIRTCHNL_ETHERTYPE_MATCH_NOT_REQUIRED = 1, +}; + +/* The PF populates these fields based on the supported VLAN offloads. If a + * field is VIRTCHNL_VLAN_UNSUPPORTED then it's not supported and the PF will + * reject any VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 or + * VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 messages using the unsupported fields. + * + * Also, a VF is only allowed to toggle its VLAN offload setting if the + * VIRTCHNL_VLAN_TOGGLE_ALLOWED bit is set. + * + * The VF driver needs to be aware of how the tags are stripped by hardware and + * inserted by the VF driver based on the level of offload support. The PF will + * populate these fields based on where the VLAN tags are expected to be + * offloaded via the VIRTHCNL_VLAN_TAG_LOCATION_* bits. The VF will need to + * interpret these fields. See the definition of the + * VIRTCHNL_VLAN_TAG_LOCATION_* bits above the virtchnl_vlan_support + * enumeration. + */ +struct virtchnl_vlan_offload_caps { + struct virtchnl_vlan_supported_caps stripping_support; + struct virtchnl_vlan_supported_caps insertion_support; + u32 ethertype_init; + u8 ethertype_match; + u8 pad[3]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_vlan_offload_caps); + +/* VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS + * VF sends this message to determine its VLAN capabilities. + * + * PF will mark which capabilities it supports based on hardware support and + * current configuration. For example, if a port VLAN is configured the PF will + * not allow outer VLAN filtering, stripping, or insertion to be configured so + * it will block these features from the VF. + * + * The VF will need to cross reference its capabilities with the PFs + * capabilities in the response message from the PF to determine the VLAN + * support. + */ +struct virtchnl_vlan_caps { + struct virtchnl_vlan_filtering_caps filtering; + struct virtchnl_vlan_offload_caps offloads; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_vlan_caps); + +struct virtchnl_vlan { + u16 tci; /* tci[15:13] = PCP and tci[11:0] = VID */ + u16 tci_mask; /* only valid if VIRTCHNL_VLAN_FILTER_MASK set in + * filtering caps + */ + u16 tpid; /* 0x8100, 0x88a8, etc. and only type(s) set in + * filtering caps. Note that tpid here does not refer to + * VIRTCHNL_VLAN_ETHERTYPE_*, but it refers to the + * actual 2-byte VLAN TPID + */ + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_vlan); + +struct virtchnl_vlan_filter { + struct virtchnl_vlan inner; + struct virtchnl_vlan outer; + u8 pad[16]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(32, virtchnl_vlan_filter); + +/* VIRTCHNL_OP_ADD_VLAN_V2 + * VIRTCHNL_OP_DEL_VLAN_V2 + * + * VF sends these messages to add/del one or more VLAN tag filters for Rx + * traffic. + * + * The PF attempts to add the filters and returns status. + * + * The VF should only ever attempt to add/del virtchnl_vlan_filter(s) using the + * supported fields negotiated via VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS. + */ +struct virtchnl_vlan_filter_list_v2 { + u16 vport_id; + u16 num_elements; + u8 pad[4]; + struct virtchnl_vlan_filter filters[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_vlan_filter_list_v2); + +/* VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 + * VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 + * VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 + * VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2 + * + * VF sends this message to enable or disable VLAN stripping or insertion. It + * also needs to specify an ethertype. The VF knows which VLAN ethertypes are + * allowed and whether or not it's allowed to enable/disable the specific + * offload via the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS message. The VF needs to + * parse the virtchnl_vlan_caps.offloads fields to determine which offload + * messages are allowed. + * + * For example, if the PF populates the virtchnl_vlan_caps.offloads in the + * following manner the VF will be allowed to enable and/or disable 0x8100 inner + * VLAN insertion and/or stripping via the opcodes listed above. Inner in this + * case means the outer most or single VLAN from the VF's perspective. This is + * because no outer offloads are supported. See the comments above the + * virtchnl_vlan_supported_caps structure for more details. + * + * virtchnl_vlan_caps.offloads.stripping_support.inner = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * virtchnl_vlan_caps.offloads.insertion_support.inner = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * In order to enable inner (again note that in this case inner is the outer + * most or single VLAN from the VF's perspective) VLAN stripping for 0x8100 + * VLANs, the VF would populate the virtchnl_vlan_setting structure in the + * following manner and send the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 message. + * + * virtchnl_vlan_setting.inner_ethertype_setting = + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + * + * The reason that VLAN TPID(s) are not being used for the + * outer_ethertype_setting and inner_ethertype_setting fields is because it's + * possible a device could support VLAN insertion and/or stripping offload on + * multiple ethertypes concurrently, so this method allows a VF to request + * multiple ethertypes in one message using the virtchnl_vlan_support + * enumeration. + * + * For example, if the PF populates the virtchnl_vlan_caps.offloads in the + * following manner the VF will be allowed to enable 0x8100 and 0x88a8 outer + * VLAN insertion and stripping simultaneously. The + * virtchnl_vlan_caps.offloads.ethertype_match field will also have to be + * populated based on what the PF can support. + * + * virtchnl_vlan_caps.offloads.stripping_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * virtchnl_vlan_caps.offloads.insertion_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * In order to enable outer VLAN stripping for 0x8100 and 0x88a8 VLANs, the VF + * would populate the virthcnl_vlan_offload_structure in the following manner + * and send the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 message. + * + * virtchnl_vlan_setting.outer_ethertype_setting = + * VIRTHCNL_VLAN_ETHERTYPE_8100 | + * VIRTHCNL_VLAN_ETHERTYPE_88A8; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + * + * There is also the case where a PF and the underlying hardware can support + * VLAN offloads on multiple ethertypes, but not concurrently. For example, if + * the PF populates the virtchnl_vlan_caps.offloads in the following manner the + * VF will be allowed to enable and/or disable 0x8100 XOR 0x88a8 outer VLAN + * offloads. The ethertypes must match for stripping and insertion. + * + * virtchnl_vlan_caps.offloads.stripping_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * virtchnl_vlan_caps.offloads.insertion_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * virtchnl_vlan_caps.offloads.ethertype_match = + * VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION; + * + * In order to enable outer VLAN stripping for 0x88a8 VLANs, the VF would + * populate the virtchnl_vlan_setting structure in the following manner and send + * the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2. Also, this will change the + * ethertype for VLAN insertion if it's enabled. So, for completeness, a + * VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 with the same ethertype should be sent. + * + * virtchnl_vlan_setting.outer_ethertype_setting = VIRTHCNL_VLAN_ETHERTYPE_88A8; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + */ +struct virtchnl_vlan_setting { + u32 outer_ethertype_setting; + u32 inner_ethertype_setting; + u16 vport_id; + u8 pad[6]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vlan_setting); + /* VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE * VF sends VSI id and flags. * PF returns status code in retval. @@ -1156,6 +1509,30 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, case VIRTCHNL_OP_DEL_FDIR_FILTER: valid_len = sizeof(struct virtchnl_fdir_del); break; + case VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS: + break; + case VIRTCHNL_OP_ADD_VLAN_V2: + case VIRTCHNL_OP_DEL_VLAN_V2: + valid_len = sizeof(struct virtchnl_vlan_filter_list_v2); + if (msglen >= valid_len) { + struct virtchnl_vlan_filter_list_v2 *vfl = + (struct virtchnl_vlan_filter_list_v2 *)msg; + + valid_len += (vfl->num_elements - 1) * + sizeof(struct virtchnl_vlan_filter); + + if (vfl->num_elements == 0) { + err_msg_format = true; + break; + } + } + break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2: + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2: + case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2: + case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2: + valid_len = sizeof(struct virtchnl_vlan_setting); + break; /* These are always errors coming from the VF. */ case VIRTCHNL_OP_EVENT: case VIRTCHNL_OP_UNKNOWN: -- cgit v1.2.3-59-g8ed1b From 877fee2a0c65a3b0b6ac0e90d7d7718b5a0341d3 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 17 Dec 2021 15:15:15 +0100 Subject: PCI: Convert pci_dev_present() stub to static inline Change the pci_dev_present() stub which is used when CONFIG_PCI is not set from a #define to a static inline stub. Thix should fix clang -Werror builds failing due to errors like this: drivers/platform/x86/thinkpad_acpi.c:4475:35: error: unused variable 'fwbug_cards_ids' [-Werror,-Wunused-const-variable] Where fwbug_cards_ids is an array of pci_device_id passed to pci_dev_present() during a quirk check. Link: https://lore.kernel.org/r/20211217141515.379586-1-hdegoede@redhat.com Reported-by: kernel test robot Signed-off-by: Hans de Goede Signed-off-by: Bjorn Helgaas Reviewed-by: Andy Shevchenko Cc: platform-driver-x86@vger.kernel.org --- include/linux/pci.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pci.h b/include/linux/pci.h index 18a75c8e615c..7d825637d7ca 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1775,7 +1775,10 @@ static inline struct pci_dev *pci_get_class(unsigned int class, struct pci_dev *from) { return NULL; } -#define pci_dev_present(ids) (0) + +static inline int pci_dev_present(const struct pci_device_id *ids) +{ return 0; } + #define no_pci_devices() (1) #define pci_dev_put(dev) do { } while (0) -- cgit v1.2.3-59-g8ed1b From 6e478521df535b9d5ef5eb84d4352f235bbbef99 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 30 Jul 2021 09:56:05 -0400 Subject: iomap,xfs: Convert ->discard_page to ->discard_folio XFS has the only implementation of ->discard_page today, so convert it to use folios in the same patch as converting the API. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong --- fs/iomap/buffered-io.c | 4 ++-- fs/xfs/xfs_aops.c | 24 ++++++++++++------------ include/linux/iomap.h | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index b78a456e696f..b403b83eedaf 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -1360,8 +1360,8 @@ iomap_writepage_map(struct iomap_writepage_ctx *wpc, * won't be affected by I/O completion and we must unlock it * now. */ - if (wpc->ops->discard_page) - wpc->ops->discard_page(page, file_offset); + if (wpc->ops->discard_folio) + wpc->ops->discard_folio(folio, file_offset); if (!count) { ClearPageUptodate(page); unlock_page(page); diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index c8c15c3c3147..4098a9875c5b 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -437,37 +437,37 @@ xfs_prepare_ioend( * see a ENOSPC in writeback). */ static void -xfs_discard_page( - struct page *page, - loff_t fileoff) +xfs_discard_folio( + struct folio *folio, + loff_t pos) { - struct inode *inode = page->mapping->host; + struct inode *inode = folio->mapping->host; struct xfs_inode *ip = XFS_I(inode); struct xfs_mount *mp = ip->i_mount; - unsigned int pageoff = offset_in_page(fileoff); - xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, fileoff); - xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, pageoff); + size_t offset = offset_in_folio(folio, pos); + xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, pos); + xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, offset); int error; if (xfs_is_shutdown(mp)) goto out_invalidate; xfs_alert_ratelimited(mp, - "page discard on page "PTR_FMT", inode 0x%llx, offset %llu.", - page, ip->i_ino, fileoff); + "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.", + folio, ip->i_ino, pos); error = xfs_bmap_punch_delalloc_range(ip, start_fsb, - i_blocks_per_page(inode, page) - pageoff_fsb); + i_blocks_per_folio(inode, folio) - pageoff_fsb); if (error && !xfs_is_shutdown(mp)) xfs_alert(mp, "page discard unable to remove delalloc mapping."); out_invalidate: - iomap_invalidatepage(page, pageoff, PAGE_SIZE - pageoff); + iomap_invalidate_folio(folio, offset, folio_size(folio) - offset); } static const struct iomap_writeback_ops xfs_writeback_ops = { .map_blocks = xfs_map_blocks, .prepare_ioend = xfs_prepare_ioend, - .discard_page = xfs_discard_page, + .discard_folio = xfs_discard_folio, }; STATIC int diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 29491fb9c5ba..5ef5088dbbd8 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -285,7 +285,7 @@ struct iomap_writeback_ops { * Optional, allows the file system to discard state on a page where * we failed to submit any I/O. */ - void (*discard_page)(struct page *page, loff_t fileoff); + void (*discard_folio)(struct folio *folio, loff_t pos); }; struct iomap_writepage_ctx { -- cgit v1.2.3-59-g8ed1b From e17f7a0bc4daa44a4809f5f2f947aa2aa74d1369 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Dec 2021 09:45:05 +0100 Subject: uio: remove copy_from_iter_flushcache() and copy_mc_to_iter() These two wrappers are never used. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211215084508.435401-2-hch@lst.de Signed-off-by: Dan Williams --- drivers/nvdimm/pmem.c | 4 +--- include/linux/uio.h | 20 +------------------- 2 files changed, 2 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 4190c8c46ca8..d225bcfa67cf 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -302,9 +302,7 @@ static long pmem_dax_direct_access(struct dax_device *dax_dev, } /* - * Use the 'no check' versions of copy_from_iter_flushcache() and - * copy_mc_to_iter() to bypass HARDENED_USERCOPY overhead. Bounds - * checking, both file offset and device offset, is handled by + * Bounds checking, both file offset and device offset, is handled by * dax_iomap_actor() */ static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, diff --git a/include/linux/uio.h b/include/linux/uio.h index 6350354f97e9..494d552c1d66 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -196,7 +196,7 @@ bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE /* * Note, users like pmem that depend on the stricter semantics of - * copy_from_iter_flushcache() than copy_from_iter_nocache() must check for + * _copy_from_iter_flushcache() than _copy_from_iter_nocache() must check for * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the * destination is flushed from the cache on return. */ @@ -211,24 +211,6 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i); #define _copy_mc_to_iter _copy_to_iter #endif -static __always_inline __must_check -size_t copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) -{ - if (unlikely(!check_copy_size(addr, bytes, false))) - return 0; - else - return _copy_from_iter_flushcache(addr, bytes, i); -} - -static __always_inline __must_check -size_t copy_mc_to_iter(void *addr, size_t bytes, struct iov_iter *i) -{ - if (unlikely(!check_copy_size(addr, bytes, true))) - return 0; - else - return _copy_mc_to_iter(addr, bytes, i); -} - size_t iov_iter_zero(size_t bytes, struct iov_iter *); unsigned long iov_iter_alignment(const struct iov_iter *i); unsigned long iov_iter_gap_alignment(const struct iov_iter *i); -- cgit v1.2.3-59-g8ed1b From fd1d00ec92002d8fe28ca981a72395eaa7ae3d11 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Dec 2021 09:45:06 +0100 Subject: dax: simplify dax_synchronous and set_dax_synchronous Remove the pointless wrappers. Signed-off-by: Christoph Hellwig Reviewed-by: Pankaj Gupta Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20211215084508.435401-3-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/super.c | 8 ++++---- include/linux/dax.h | 12 ++---------- 2 files changed, 6 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e7152a6c4cc4..e18155f43a63 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -208,17 +208,17 @@ bool dax_write_cache_enabled(struct dax_device *dax_dev) } EXPORT_SYMBOL_GPL(dax_write_cache_enabled); -bool __dax_synchronous(struct dax_device *dax_dev) +bool dax_synchronous(struct dax_device *dax_dev) { return test_bit(DAXDEV_SYNC, &dax_dev->flags); } -EXPORT_SYMBOL_GPL(__dax_synchronous); +EXPORT_SYMBOL_GPL(dax_synchronous); -void __set_dax_synchronous(struct dax_device *dax_dev) +void set_dax_synchronous(struct dax_device *dax_dev) { set_bit(DAXDEV_SYNC, &dax_dev->flags); } -EXPORT_SYMBOL_GPL(__set_dax_synchronous); +EXPORT_SYMBOL_GPL(set_dax_synchronous); bool dax_alive(struct dax_device *dax_dev) { diff --git a/include/linux/dax.h b/include/linux/dax.h index 87ae4c9b1d65..3bd1fdb5d5f4 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -48,16 +48,8 @@ void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); bool dax_write_cache_enabled(struct dax_device *dax_dev); -bool __dax_synchronous(struct dax_device *dax_dev); -static inline bool dax_synchronous(struct dax_device *dax_dev) -{ - return __dax_synchronous(dax_dev); -} -void __set_dax_synchronous(struct dax_device *dax_dev); -static inline void set_dax_synchronous(struct dax_device *dax_dev) -{ - __set_dax_synchronous(dax_dev); -} +bool dax_synchronous(struct dax_device *dax_dev); +void set_dax_synchronous(struct dax_device *dax_dev); /* * Check if given mapping is supported by the file / underlying device. */ -- cgit v1.2.3-59-g8ed1b From 30c6828a17a572aeb9e3a3bacce05fdcf1106541 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Dec 2021 09:45:07 +0100 Subject: dax: remove the DAXDEV_F_SYNC flag Remove the DAXDEV_F_SYNC flag and thus the flags argument to alloc_dax and just let the drivers call set_dax_synchronous directly. Signed-off-by: Christoph Hellwig Reviewed-by: Pankaj Gupta Reviewed-by: Dan Williams Link: https://lore.kernel.org/r/20211215084508.435401-4-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/bus.c | 3 ++- drivers/dax/super.c | 6 +----- drivers/md/dm.c | 2 +- drivers/nvdimm/pmem.c | 7 +++---- drivers/s390/block/dcssblk.c | 4 ++-- fs/fuse/virtio_fs.c | 2 +- include/linux/dax.h | 8 ++------ 7 files changed, 12 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 6683d42c32c5..da2a14d096d2 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -1324,11 +1324,12 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) * No dax_operations since there is no access to this device outside of * mmap of the resulting character device. */ - dax_dev = alloc_dax(dev_dax, NULL, DAXDEV_F_SYNC); + dax_dev = alloc_dax(dev_dax, NULL); if (IS_ERR(dax_dev)) { rc = PTR_ERR(dax_dev); goto err_alloc_dax; } + set_dax_synchronous(dax_dev); /* a device_dax instance is dead while the driver is not attached */ kill_dax(dax_dev); diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e18155f43a63..e81d5ee57390 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -345,8 +345,7 @@ static struct dax_device *dax_dev_get(dev_t devt) return dax_dev; } -struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, - unsigned long flags) +struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) { struct dax_device *dax_dev; dev_t devt; @@ -366,9 +365,6 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, dax_dev->ops = ops; dax_dev->private = private; - if (flags & DAXDEV_F_SYNC) - set_dax_synchronous(dax_dev); - return dax_dev; err_dev: diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 4e997c02bb0a..f4b972af1092 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1765,7 +1765,7 @@ static struct mapped_device *alloc_dev(int minor) sprintf(md->disk->disk_name, "dm-%d", minor); if (IS_ENABLED(CONFIG_FS_DAX)) { - md->dax_dev = alloc_dax(md, &dm_dax_ops, 0); + md->dax_dev = alloc_dax(md, &dm_dax_ops); if (IS_ERR(md->dax_dev)) { md->dax_dev = NULL; goto bad; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index d225bcfa67cf..18b1d9c55831 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -400,7 +400,6 @@ static int pmem_attach_disk(struct device *dev, struct gendisk *disk; void *addr; int rc; - unsigned long flags = 0UL; pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL); if (!pmem) @@ -493,13 +492,13 @@ static int pmem_attach_disk(struct device *dev, nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_range); disk->bb = &pmem->bb; - if (is_nvdimm_sync(nd_region)) - flags = DAXDEV_F_SYNC; - dax_dev = alloc_dax(pmem, &pmem_dax_ops, flags); + dax_dev = alloc_dax(pmem, &pmem_dax_ops); if (IS_ERR(dax_dev)) { rc = PTR_ERR(dax_dev); goto out; } + if (is_nvdimm_sync(nd_region)) + set_dax_synchronous(dax_dev); rc = dax_add_host(dax_dev, disk); if (rc) goto out_cleanup_dax; diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index e65e83764d1c..10823debc09b 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -686,13 +686,13 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char if (rc) goto put_dev; - dev_info->dax_dev = alloc_dax(dev_info, &dcssblk_dax_ops, - DAXDEV_F_SYNC); + dev_info->dax_dev = alloc_dax(dev_info, &dcssblk_dax_ops); if (IS_ERR(dev_info->dax_dev)) { rc = PTR_ERR(dev_info->dax_dev); dev_info->dax_dev = NULL; goto put_dev; } + set_dax_synchronous(dev_info->dax_dev); rc = dax_add_host(dev_info->dax_dev, dev_info->gd); if (rc) goto out_dax; diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 242cc1c0d7ed..5c03a0364a9b 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -850,7 +850,7 @@ static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); - fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops, 0); + fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops); if (IS_ERR(fs->dax_dev)) return PTR_ERR(fs->dax_dev); diff --git a/include/linux/dax.h b/include/linux/dax.h index 3bd1fdb5d5f4..c04f46478e3b 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -6,9 +6,6 @@ #include #include -/* Flag for synchronous flush */ -#define DAXDEV_F_SYNC (1UL << 0) - typedef unsigned long dax_entry_t; struct dax_device; @@ -42,8 +39,7 @@ struct dax_operations { }; #if IS_ENABLED(CONFIG_DAX) -struct dax_device *alloc_dax(void *private, const struct dax_operations *ops, - unsigned long flags); +struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); @@ -64,7 +60,7 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, } #else static inline struct dax_device *alloc_dax(void *private, - const struct dax_operations *ops, unsigned long flags) + const struct dax_operations *ops) { /* * Callers should check IS_ENABLED(CONFIG_DAX) to know if this -- cgit v1.2.3-59-g8ed1b From 7ac5360cd4d02cc7e0eaf10867f599e041822f12 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 15 Dec 2021 09:45:08 +0100 Subject: dax: remove the copy_from_iter and copy_to_iter methods These methods indirect the actual DAX read/write path. In the end pmem uses magic flush and mc safe variants and fuse and dcssblk use plain ones while device mapper picks redirects to the underlying device. Add set_dax_nocache() and set_dax_nomc() APIs to control which copy routines are used to remove indirect call from the read/write fast path as well as a lot of boilerplate code. Signed-off-by: Christoph Hellwig Reviewed-by: Vivek Goyal [virtiofs] Link: https://lore.kernel.org/r/20211215084508.435401-5-hch@lst.de Signed-off-by: Dan Williams --- drivers/dax/bus.c | 2 ++ drivers/dax/super.c | 36 ++++++++++++++++--- drivers/md/dm-linear.c | 20 ----------- drivers/md/dm-log-writes.c | 80 ------------------------------------------- drivers/md/dm-stripe.c | 20 ----------- drivers/md/dm.c | 52 ++-------------------------- drivers/nvdimm/pmem.c | 20 ++--------- drivers/s390/block/dcssblk.c | 14 -------- fs/dax.c | 5 --- fs/fuse/virtio_fs.c | 16 --------- include/linux/dax.h | 9 ++--- include/linux/device-mapper.h | 4 --- 12 files changed, 41 insertions(+), 237 deletions(-) (limited to 'include/linux') diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index da2a14d096d2..ee4568ef757c 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -1330,6 +1330,8 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data) goto err_alloc_dax; } set_dax_synchronous(dax_dev); + set_dax_nocache(dax_dev); + set_dax_nomc(dax_dev); /* a device_dax instance is dead while the driver is not attached */ kill_dax(dax_dev); diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e81d5ee57390..e3029389d809 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -105,6 +105,10 @@ enum dax_device_flags { DAXDEV_WRITE_CACHE, /* flag to check if device supports synchronous flush */ DAXDEV_SYNC, + /* do not leave the caches dirty after writes */ + DAXDEV_NOCACHE, + /* handle CPU fetch exceptions during reads */ + DAXDEV_NOMC, }; /** @@ -146,9 +150,15 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, if (!dax_alive(dax_dev)) return 0; - return dax_dev->ops->copy_from_iter(dax_dev, pgoff, addr, bytes, i); + /* + * The userspace address for the memory copy has already been validated + * via access_ok() in vfs_write, so use the 'no check' version to bypass + * the HARDENED_USERCOPY overhead. + */ + if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags)) + return _copy_from_iter_flushcache(addr, bytes, i); + return _copy_from_iter(addr, bytes, i); } -EXPORT_SYMBOL_GPL(dax_copy_from_iter); size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) @@ -156,9 +166,15 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, if (!dax_alive(dax_dev)) return 0; - return dax_dev->ops->copy_to_iter(dax_dev, pgoff, addr, bytes, i); + /* + * The userspace address for the memory copy has already been validated + * via access_ok() in vfs_red, so use the 'no check' version to bypass + * the HARDENED_USERCOPY overhead. + */ + if (test_bit(DAXDEV_NOMC, &dax_dev->flags)) + return _copy_mc_to_iter(addr, bytes, i); + return _copy_to_iter(addr, bytes, i); } -EXPORT_SYMBOL_GPL(dax_copy_to_iter); int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) @@ -220,6 +236,18 @@ void set_dax_synchronous(struct dax_device *dax_dev) } EXPORT_SYMBOL_GPL(set_dax_synchronous); +void set_dax_nocache(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_NOCACHE, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(set_dax_nocache); + +void set_dax_nomc(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_NOMC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(set_dax_nomc); + bool dax_alive(struct dax_device *dax_dev) { lockdep_assert_held(&dax_srcu); diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 90de42f6743a..1b97a11d7151 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -180,22 +180,6 @@ static long linear_dax_direct_access(struct dm_target *ti, pgoff_t pgoff, return dax_direct_access(dax_dev, pgoff, nr_pages, kaddr, pfn); } -static size_t linear_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct dax_device *dax_dev = linear_dax_pgoff(ti, &pgoff); - - return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i); -} - -static size_t linear_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct dax_device *dax_dev = linear_dax_pgoff(ti, &pgoff); - - return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i); -} - static int linear_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, size_t nr_pages) { @@ -206,8 +190,6 @@ static int linear_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, #else #define linear_dax_direct_access NULL -#define linear_dax_copy_from_iter NULL -#define linear_dax_copy_to_iter NULL #define linear_dax_zero_page_range NULL #endif @@ -225,8 +207,6 @@ static struct target_type linear_target = { .prepare_ioctl = linear_prepare_ioctl, .iterate_devices = linear_iterate_devices, .direct_access = linear_dax_direct_access, - .dax_copy_from_iter = linear_dax_copy_from_iter, - .dax_copy_to_iter = linear_dax_copy_to_iter, .dax_zero_page_range = linear_dax_zero_page_range, }; diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index cdb22e7a1d0d..139b09b06eda 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -902,51 +902,6 @@ static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limit } #if IS_ENABLED(CONFIG_FS_DAX) -static int log_dax(struct log_writes_c *lc, sector_t sector, size_t bytes, - struct iov_iter *i) -{ - struct pending_block *block; - - if (!bytes) - return 0; - - block = kzalloc(sizeof(struct pending_block), GFP_KERNEL); - if (!block) { - DMERR("Error allocating dax pending block"); - return -ENOMEM; - } - - block->data = kzalloc(bytes, GFP_KERNEL); - if (!block->data) { - DMERR("Error allocating dax data space"); - kfree(block); - return -ENOMEM; - } - - /* write data provided via the iterator */ - if (!copy_from_iter(block->data, bytes, i)) { - DMERR("Error copying dax data"); - kfree(block->data); - kfree(block); - return -EIO; - } - - /* rewind the iterator so that the block driver can use it */ - iov_iter_revert(i, bytes); - - block->datalen = bytes; - block->sector = bio_to_dev_sectors(lc, sector); - block->nr_sectors = ALIGN(bytes, lc->sectorsize) >> lc->sectorshift; - - atomic_inc(&lc->pending_blocks); - spin_lock_irq(&lc->blocks_lock); - list_add_tail(&block->list, &lc->unflushed_blocks); - spin_unlock_irq(&lc->blocks_lock); - wake_up_process(lc->log_kthread); - - return 0; -} - static struct dax_device *log_writes_dax_pgoff(struct dm_target *ti, pgoff_t *pgoff) { @@ -964,37 +919,6 @@ static long log_writes_dax_direct_access(struct dm_target *ti, pgoff_t pgoff, return dax_direct_access(dax_dev, pgoff, nr_pages, kaddr, pfn); } -static size_t log_writes_dax_copy_from_iter(struct dm_target *ti, - pgoff_t pgoff, void *addr, size_t bytes, - struct iov_iter *i) -{ - struct log_writes_c *lc = ti->private; - sector_t sector = pgoff * PAGE_SECTORS; - struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff); - int err; - - /* Don't bother doing anything if logging has been disabled */ - if (!lc->logging_enabled) - goto dax_copy; - - err = log_dax(lc, sector, bytes, i); - if (err) { - DMWARN("Error %d logging DAX write", err); - return 0; - } -dax_copy: - return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i); -} - -static size_t log_writes_dax_copy_to_iter(struct dm_target *ti, - pgoff_t pgoff, void *addr, size_t bytes, - struct iov_iter *i) -{ - struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff); - - return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i); -} - static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, size_t nr_pages) { @@ -1005,8 +929,6 @@ static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, #else #define log_writes_dax_direct_access NULL -#define log_writes_dax_copy_from_iter NULL -#define log_writes_dax_copy_to_iter NULL #define log_writes_dax_zero_page_range NULL #endif @@ -1024,8 +946,6 @@ static struct target_type log_writes_target = { .iterate_devices = log_writes_iterate_devices, .io_hints = log_writes_io_hints, .direct_access = log_writes_dax_direct_access, - .dax_copy_from_iter = log_writes_dax_copy_from_iter, - .dax_copy_to_iter = log_writes_dax_copy_to_iter, .dax_zero_page_range = log_writes_dax_zero_page_range, }; diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c index 50dba3f39274..e566115ec0bb 100644 --- a/drivers/md/dm-stripe.c +++ b/drivers/md/dm-stripe.c @@ -324,22 +324,6 @@ static long stripe_dax_direct_access(struct dm_target *ti, pgoff_t pgoff, return dax_direct_access(dax_dev, pgoff, nr_pages, kaddr, pfn); } -static size_t stripe_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct dax_device *dax_dev = stripe_dax_pgoff(ti, &pgoff); - - return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i); -} - -static size_t stripe_dax_copy_to_iter(struct dm_target *ti, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct dax_device *dax_dev = stripe_dax_pgoff(ti, &pgoff); - - return dax_copy_to_iter(dax_dev, pgoff, addr, bytes, i); -} - static int stripe_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, size_t nr_pages) { @@ -350,8 +334,6 @@ static int stripe_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff, #else #define stripe_dax_direct_access NULL -#define stripe_dax_copy_from_iter NULL -#define stripe_dax_copy_to_iter NULL #define stripe_dax_zero_page_range NULL #endif @@ -488,8 +470,6 @@ static struct target_type stripe_target = { .iterate_devices = stripe_iterate_devices, .io_hints = stripe_io_hints, .direct_access = stripe_dax_direct_access, - .dax_copy_from_iter = stripe_dax_copy_from_iter, - .dax_copy_to_iter = stripe_dax_copy_to_iter, .dax_zero_page_range = stripe_dax_zero_page_range, }; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index f4b972af1092..ce250bd274f3 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1027,54 +1027,6 @@ static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, return ret; } -static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct mapped_device *md = dax_get_private(dax_dev); - sector_t sector = pgoff * PAGE_SECTORS; - struct dm_target *ti; - long ret = 0; - int srcu_idx; - - ti = dm_dax_get_live_target(md, sector, &srcu_idx); - - if (!ti) - goto out; - if (!ti->type->dax_copy_from_iter) { - ret = copy_from_iter(addr, bytes, i); - goto out; - } - ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i); - out: - dm_put_live_table(md, srcu_idx); - - return ret; -} - -static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - struct mapped_device *md = dax_get_private(dax_dev); - sector_t sector = pgoff * PAGE_SECTORS; - struct dm_target *ti; - long ret = 0; - int srcu_idx; - - ti = dm_dax_get_live_target(md, sector, &srcu_idx); - - if (!ti) - goto out; - if (!ti->type->dax_copy_to_iter) { - ret = copy_to_iter(addr, bytes, i); - goto out; - } - ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i); - out: - dm_put_live_table(md, srcu_idx); - - return ret; -} - static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) { @@ -1770,6 +1722,8 @@ static struct mapped_device *alloc_dev(int minor) md->dax_dev = NULL; goto bad; } + set_dax_nocache(md->dax_dev); + set_dax_nomc(md->dax_dev); if (dax_add_host(md->dax_dev, md->disk)) goto bad; } @@ -3024,8 +2978,6 @@ static const struct block_device_operations dm_rq_blk_dops = { static const struct dax_operations dm_dax_ops = { .direct_access = dm_dax_direct_access, - .copy_from_iter = dm_dax_copy_from_iter, - .copy_to_iter = dm_dax_copy_to_iter, .zero_page_range = dm_dax_zero_page_range, }; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 18b1d9c55831..58d95242a836 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -301,26 +301,8 @@ static long pmem_dax_direct_access(struct dax_device *dax_dev, return __pmem_direct_access(pmem, pgoff, nr_pages, kaddr, pfn); } -/* - * Bounds checking, both file offset and device offset, is handled by - * dax_iomap_actor() - */ -static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - return _copy_from_iter_flushcache(addr, bytes, i); -} - -static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i) -{ - return _copy_mc_to_iter(addr, bytes, i); -} - static const struct dax_operations pmem_dax_ops = { .direct_access = pmem_dax_direct_access, - .copy_from_iter = pmem_copy_from_iter, - .copy_to_iter = pmem_copy_to_iter, .zero_page_range = pmem_dax_zero_page_range, }; @@ -497,6 +479,8 @@ static int pmem_attach_disk(struct device *dev, rc = PTR_ERR(dax_dev); goto out; } + set_dax_nocache(dax_dev); + set_dax_nomc(dax_dev); if (is_nvdimm_sync(nd_region)) set_dax_synchronous(dax_dev); rc = dax_add_host(dax_dev, disk); diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 10823debc09b..d614843caf6c 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -44,18 +44,6 @@ static const struct block_device_operations dcssblk_devops = { .release = dcssblk_release, }; -static size_t dcssblk_dax_copy_from_iter(struct dax_device *dax_dev, - pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) -{ - return copy_from_iter(addr, bytes, i); -} - -static size_t dcssblk_dax_copy_to_iter(struct dax_device *dax_dev, - pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) -{ - return copy_to_iter(addr, bytes, i); -} - static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) { @@ -72,8 +60,6 @@ static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev, static const struct dax_operations dcssblk_dax_ops = { .direct_access = dcssblk_dax_direct_access, - .copy_from_iter = dcssblk_dax_copy_from_iter, - .copy_to_iter = dcssblk_dax_copy_to_iter, .zero_page_range = dcssblk_dax_zero_page_range, }; diff --git a/fs/dax.c b/fs/dax.c index e0eecd8e3a8f..cd03485867a7 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1260,11 +1260,6 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi, if (map_len > end - pos) map_len = end - pos; - /* - * The userspace address for the memory copy has already been - * validated via access_ok() in either vfs_read() or - * vfs_write(), depending on which operation we are doing. - */ if (iov_iter_rw(iter) == WRITE) xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr, map_len, iter); diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 5c03a0364a9b..3928cd8ceba6 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -753,20 +753,6 @@ static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; } -static size_t virtio_fs_copy_from_iter(struct dax_device *dax_dev, - pgoff_t pgoff, void *addr, - size_t bytes, struct iov_iter *i) -{ - return copy_from_iter(addr, bytes, i); -} - -static size_t virtio_fs_copy_to_iter(struct dax_device *dax_dev, - pgoff_t pgoff, void *addr, - size_t bytes, struct iov_iter *i) -{ - return copy_to_iter(addr, bytes, i); -} - static int virtio_fs_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) { @@ -783,8 +769,6 @@ static int virtio_fs_zero_page_range(struct dax_device *dax_dev, static const struct dax_operations virtio_fs_dax_ops = { .direct_access = virtio_fs_direct_access, - .copy_from_iter = virtio_fs_copy_from_iter, - .copy_to_iter = virtio_fs_copy_to_iter, .zero_page_range = virtio_fs_zero_page_range, }; diff --git a/include/linux/dax.h b/include/linux/dax.h index c04f46478e3b..9fc5f99a0ae2 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -28,12 +28,6 @@ struct dax_operations { */ bool (*dax_supported)(struct dax_device *, struct block_device *, int, sector_t, sector_t); - /* copy_from_iter: required operation for fs-dax direct-i/o */ - size_t (*copy_from_iter)(struct dax_device *, pgoff_t, void *, size_t, - struct iov_iter *); - /* copy_to_iter: required operation for fs-dax direct-i/o */ - size_t (*copy_to_iter)(struct dax_device *, pgoff_t, void *, size_t, - struct iov_iter *); /* zero_page_range: required operation. Zero page range */ int (*zero_page_range)(struct dax_device *, pgoff_t, size_t); }; @@ -95,6 +89,9 @@ static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, } #endif +void set_dax_nocache(struct dax_device *dax_dev); +void set_dax_nomc(struct dax_device *dax_dev); + struct writeback_control; #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index a7df155ea49b..b26fecf6c8e8 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -147,8 +147,6 @@ typedef int (*dm_busy_fn) (struct dm_target *ti); */ typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn); -typedef size_t (*dm_dax_copy_iter_fn)(struct dm_target *ti, pgoff_t pgoff, - void *addr, size_t bytes, struct iov_iter *i); typedef int (*dm_dax_zero_page_range_fn)(struct dm_target *ti, pgoff_t pgoff, size_t nr_pages); @@ -200,8 +198,6 @@ struct target_type { dm_iterate_devices_fn iterate_devices; dm_io_hints_fn io_hints; dm_dax_direct_access_fn direct_access; - dm_dax_copy_iter_fn dax_copy_from_iter; - dm_dax_copy_iter_fn dax_copy_to_iter; dm_dax_zero_page_range_fn dax_zero_page_range; /* For internal device-mapper use. */ -- cgit v1.2.3-59-g8ed1b From d639b9d13a39cf15639cbe6e8b2c43eb60148a73 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:44 -0800 Subject: bpf: Introduce composable reg, ret and arg types. There are some common properties shared between bpf reg, ret and arg values. For instance, a value may be a NULL pointer, or a pointer to a read-only memory. Previously, to express these properties, enumeration was used. For example, in order to test whether a reg value can be NULL, reg_type_may_be_null() simply enumerates all types that are possibly NULL. The problem of this approach is that it's not scalable and causes a lot of duplication. These properties can be combined, for example, a type could be either MAYBE_NULL or RDONLY, or both. This patch series rewrites the layout of reg_type, arg_type and ret_type, so that common properties can be extracted and represented as composable flag. For example, one can write ARG_PTR_TO_MEM | PTR_MAYBE_NULL which is equivalent to the previous ARG_PTR_TO_MEM_OR_NULL The type ARG_PTR_TO_MEM are called "base type" in this patch. Base types can be extended with flags. A flag occupies the higher bits while base types sits in the lower bits. This patch in particular sets up a set of macro for this purpose. The following patches will rewrite arg_types, ret_types and reg_types respectively. Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-2-haoluo@google.com --- include/linux/bpf.h | 42 ++++++++++++++++++++++++++++++++++++++++++ include/linux/bpf_verifier.h | 13 +++++++++++++ 2 files changed, 55 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 965fffaf0308..41bb3687cc85 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -297,6 +297,29 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0, extern const struct bpf_map_ops bpf_map_offload_ops; +/* bpf_type_flag contains a set of flags that are applicable to the values of + * arg_type, ret_type and reg_type. For example, a pointer value may be null, + * or a memory is read-only. We classify types into two categories: base types + * and extended types. Extended types are base types combined with a type flag. + * + * Currently there are no more than 32 base types in arg_type, ret_type and + * reg_types. + */ +#define BPF_BASE_TYPE_BITS 8 + +enum bpf_type_flag { + /* PTR may be NULL. */ + PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), + + __BPF_TYPE_LAST_FLAG = PTR_MAYBE_NULL, +}; + +/* Max number of base types. */ +#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS) + +/* Max number of all types. */ +#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1)) + /* function argument constraints */ enum bpf_arg_type { ARG_DONTCARE = 0, /* unused argument in helper function */ @@ -343,7 +366,13 @@ enum bpf_arg_type { ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ __BPF_ARG_TYPE_MAX, + + /* This must be the last entry. Its purpose is to ensure the enum is + * wide enough to hold the higher bits reserved for bpf_type_flag. + */ + __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT, }; +static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* type of values returned from helper functions */ enum bpf_return_type { @@ -359,7 +388,14 @@ enum bpf_return_type { RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */ RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ + __BPF_RET_TYPE_MAX, + + /* This must be the last entry. Its purpose is to ensure the enum is + * wide enough to hold the higher bits reserved for bpf_type_flag. + */ + __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT, }; +static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL @@ -461,7 +497,13 @@ enum bpf_reg_type { PTR_TO_FUNC, /* reg points to a bpf program function */ PTR_TO_MAP_KEY, /* reg points to a map element key */ __BPF_REG_TYPE_MAX, + + /* This must be the last entry. Its purpose is to ensure the enum is + * wide enough to hold the higher bits reserved for bpf_type_flag. + */ + __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT, }; +static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* The information passed from prog-specific *_is_valid_access * back to the verifier. diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ee931398f311..34e4ceaca3c7 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -546,5 +546,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_attach_target_info *tgt_info); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); +#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0) + +/* extract base type from bpf_{arg, return, reg}_type. */ +static inline u32 base_type(u32 type) +{ + return type & BPF_BASE_TYPE_MASK; +} + +/* extract flags from an extended type. See bpf_type_flag in bpf.h. */ +static inline u32 type_flag(u32 type) +{ + return type & ~BPF_BASE_TYPE_MASK; +} #endif /* _LINUX_BPF_VERIFIER_H */ -- cgit v1.2.3-59-g8ed1b From 48946bd6a5d695c50b34546864b79c1f910a33c1 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:45 -0800 Subject: bpf: Replace ARG_XXX_OR_NULL with ARG_XXX | PTR_MAYBE_NULL We have introduced a new type to make bpf_arg composable, by reserving high bits of bpf_arg to represent flags of a type. One of the flags is PTR_MAYBE_NULL which indicates a pointer may be NULL. When applying this flag to an arg_type, it means the arg can take NULL pointer. This patch switches the qualified arg_types to use this flag. The arg_types changed in this patch include: 1. ARG_PTR_TO_MAP_VALUE_OR_NULL 2. ARG_PTR_TO_MEM_OR_NULL 3. ARG_PTR_TO_CTX_OR_NULL 4. ARG_PTR_TO_SOCKET_OR_NULL 5. ARG_PTR_TO_ALLOC_MEM_OR_NULL 6. ARG_PTR_TO_STACK_OR_NULL This patch does not eliminate the use of these arg_types, instead it makes them an alias to the 'ARG_XXX | PTR_MAYBE_NULL'. Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-3-haoluo@google.com --- include/linux/bpf.h | 15 +++++++++------ kernel/bpf/verifier.c | 39 ++++++++++++++------------------------- 2 files changed, 23 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 41bb3687cc85..765bd7cc4272 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -331,13 +331,11 @@ enum bpf_arg_type { ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ - ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ /* the following constraints used to prototype bpf_memcmp() and other * functions that access data on eBPF program stack */ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ - ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, * helper function must fill all bytes or clear * them in error case. @@ -347,26 +345,31 @@ enum bpf_arg_type { ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ - ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */ ARG_ANYTHING, /* any (initialized) argument is ok */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_INT, /* pointer to int */ ARG_PTR_TO_LONG, /* pointer to long */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ - ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */ ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ - ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ - ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */ + ARG_PTR_TO_STACK, /* pointer to stack */ ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ __BPF_ARG_TYPE_MAX, + /* Extended arg_types. */ + ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE, + ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM, + ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX, + ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET, + ARG_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_ALLOC_MEM, + ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK, + /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 07d7d6124a18..6095a75c1118 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -475,14 +475,9 @@ static bool arg_type_may_be_refcounted(enum bpf_arg_type type) return type == ARG_PTR_TO_SOCK_COMMON; } -static bool arg_type_may_be_null(enum bpf_arg_type type) +static bool type_may_be_null(u32 type) { - return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || - type == ARG_PTR_TO_MEM_OR_NULL || - type == ARG_PTR_TO_CTX_OR_NULL || - type == ARG_PTR_TO_SOCKET_OR_NULL || - type == ARG_PTR_TO_ALLOC_MEM_OR_NULL || - type == ARG_PTR_TO_STACK_OR_NULL; + return type & PTR_MAYBE_NULL; } /* Determine whether the function releases some resources allocated by another @@ -5016,9 +5011,8 @@ static int process_timer_func(struct bpf_verifier_env *env, int regno, static bool arg_type_is_mem_ptr(enum bpf_arg_type type) { - return type == ARG_PTR_TO_MEM || - type == ARG_PTR_TO_MEM_OR_NULL || - type == ARG_PTR_TO_UNINIT_MEM; + return base_type(type) == ARG_PTR_TO_MEM || + base_type(type) == ARG_PTR_TO_UNINIT_MEM; } static bool arg_type_is_mem_size(enum bpf_arg_type type) @@ -5155,31 +5149,26 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, - [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, [ARG_CONST_SIZE] = &scalar_types, [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, - [ARG_PTR_TO_CTX_OR_NULL] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, #ifdef CONFIG_NET [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, #endif [ARG_PTR_TO_SOCKET] = &fullsock_types, - [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, [ARG_PTR_TO_MEM] = &mem_types, - [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, [ARG_PTR_TO_UNINIT_MEM] = &mem_types, [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, - [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, [ARG_PTR_TO_INT] = &int_ptr_types, [ARG_PTR_TO_LONG] = &int_ptr_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, - [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types, + [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, [ARG_PTR_TO_TIMER] = &timer_types, }; @@ -5193,7 +5182,7 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, const struct bpf_reg_types *compatible; int i, j; - compatible = compatible_reg_types[arg_type]; + compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); return -EFAULT; @@ -5274,15 +5263,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EACCES; } - if (arg_type == ARG_PTR_TO_MAP_VALUE || - arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || - arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { + if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE || + base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) { err = resolve_map_arg_type(env, meta, &arg_type); if (err) return err; } - if (register_is_null(reg) && arg_type_may_be_null(arg_type)) + if (register_is_null(reg) && type_may_be_null(arg_type)) /* A NULL register has a SCALAR_VALUE type, so skip * type checking. */ @@ -5351,10 +5339,11 @@ skip_type_check: err = check_helper_mem_access(env, regno, meta->map_ptr->key_size, false, NULL); - } else if (arg_type == ARG_PTR_TO_MAP_VALUE || - (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && - !register_is_null(reg)) || - arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { + } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE || + base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) { + if (type_may_be_null(arg_type) && register_is_null(reg)) + return 0; + /* bpf_map_xxx(..., map_ptr, ..., value) call: * check [value, value + map->value_size) validity */ -- cgit v1.2.3-59-g8ed1b From 3c4807322660d4290ac9062c034aed6b87243861 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:46 -0800 Subject: bpf: Replace RET_XXX_OR_NULL with RET_XXX | PTR_MAYBE_NULL We have introduced a new type to make bpf_ret composable, by reserving high bits to represent flags. One of the flag is PTR_MAYBE_NULL, which indicates a pointer may be NULL. When applying this flag to ret_types, it means the returned value could be a NULL pointer. This patch switches the qualified arg_types to use this flag. The ret_types changed in this patch include: 1. RET_PTR_TO_MAP_VALUE_OR_NULL 2. RET_PTR_TO_SOCKET_OR_NULL 3. RET_PTR_TO_TCP_SOCK_OR_NULL 4. RET_PTR_TO_SOCK_COMMON_OR_NULL 5. RET_PTR_TO_ALLOC_MEM_OR_NULL 6. RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL 7. RET_PTR_TO_BTF_ID_OR_NULL This patch doesn't eliminate the use of these names, instead it makes them aliases to 'RET_PTR_TO_XXX | PTR_MAYBE_NULL'. Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-4-haoluo@google.com --- include/linux/bpf.h | 19 ++++++++++++------- kernel/bpf/helpers.c | 2 +- kernel/bpf/verifier.c | 52 +++++++++++++++++++++++++-------------------------- 3 files changed, 39 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 765bd7cc4272..975a1d5951bd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -382,17 +382,22 @@ enum bpf_return_type { RET_INTEGER, /* function returns integer */ RET_VOID, /* function doesn't return anything */ RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ - RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ - RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ - RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ - RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ - RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */ - RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */ - RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */ + RET_PTR_TO_SOCKET, /* returns a pointer to a socket */ + RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */ + RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */ + RET_PTR_TO_ALLOC_MEM, /* returns a pointer to dynamically allocated memory */ RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ __BPF_RET_TYPE_MAX, + /* Extended ret_types. */ + RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE, + RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, + RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, + RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, + RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM, + RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, + /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 34d6f91dec1c..c49dc5cbe0a7 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -682,7 +682,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) const struct bpf_func_proto bpf_per_cpu_ptr_proto = { .func = bpf_per_cpu_ptr, .gpl_only = false, - .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, + .ret_type = RET_PTR_TO_MEM_OR_BTF_ID | PTR_MAYBE_NULL, .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID, .arg2_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6095a75c1118..ccc068c5c5f2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6473,6 +6473,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn int *insn_idx_p) { const struct bpf_func_proto *fn = NULL; + enum bpf_return_type ret_type; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; int insn_idx = *insn_idx_p; @@ -6612,13 +6613,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; /* update return register (already marked as written above) */ - if (fn->ret_type == RET_INTEGER) { + ret_type = fn->ret_type; + if (ret_type == RET_INTEGER) { /* sets type to SCALAR_VALUE */ mark_reg_unknown(env, regs, BPF_REG_0); - } else if (fn->ret_type == RET_VOID) { + } else if (ret_type == RET_VOID) { regs[BPF_REG_0].type = NOT_INIT; - } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || - fn->ret_type == RET_PTR_TO_MAP_VALUE) { + } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) { /* There is no offset yet applied, variable or fixed */ mark_reg_known_zero(env, regs, BPF_REG_0); /* remember map_ptr, so that check_map_access() @@ -6632,28 +6633,27 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } regs[BPF_REG_0].map_ptr = meta.map_ptr; regs[BPF_REG_0].map_uid = meta.map_uid; - if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { + if (type_may_be_null(ret_type)) { + regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; + } else { regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; if (map_value_has_spin_lock(meta.map_ptr)) regs[BPF_REG_0].id = ++env->id_gen; - } else { - regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; } - } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { + } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; - } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { + } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; - } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { + } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; - } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { + } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; regs[BPF_REG_0].mem_size = meta.mem_size; - } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || - fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { + } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) { const struct btf_type *t; mark_reg_known_zero(env, regs, BPF_REG_0); @@ -6672,28 +6672,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } regs[BPF_REG_0].type = - fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? - PTR_TO_MEM : PTR_TO_MEM_OR_NULL; + (ret_type & PTR_MAYBE_NULL) ? + PTR_TO_MEM_OR_NULL : PTR_TO_MEM; regs[BPF_REG_0].mem_size = tsize; } else { regs[BPF_REG_0].type = - fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? - PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; + (ret_type & PTR_MAYBE_NULL) ? + PTR_TO_BTF_ID_OR_NULL : PTR_TO_BTF_ID; regs[BPF_REG_0].btf = meta.ret_btf; regs[BPF_REG_0].btf_id = meta.ret_btf_id; } - } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || - fn->ret_type == RET_PTR_TO_BTF_ID) { + } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) { int ret_btf_id; mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? - PTR_TO_BTF_ID : - PTR_TO_BTF_ID_OR_NULL; + regs[BPF_REG_0].type = (ret_type & PTR_MAYBE_NULL) ? + PTR_TO_BTF_ID_OR_NULL : + PTR_TO_BTF_ID; ret_btf_id = *fn->ret_btf_id; if (ret_btf_id == 0) { - verbose(env, "invalid return type %d of func %s#%d\n", - fn->ret_type, func_id_name(func_id), func_id); + verbose(env, "invalid return type %u of func %s#%d\n", + base_type(ret_type), func_id_name(func_id), + func_id); return -EINVAL; } /* current BPF helper definitions are only coming from @@ -6702,8 +6702,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].btf = btf_vmlinux; regs[BPF_REG_0].btf_id = ret_btf_id; } else { - verbose(env, "unknown return type %d of func %s#%d\n", - fn->ret_type, func_id_name(func_id), func_id); + verbose(env, "unknown return type %u of func %s#%d\n", + base_type(ret_type), func_id_name(func_id), func_id); return -EINVAL; } -- cgit v1.2.3-59-g8ed1b From c25b2ae136039ffa820c26138ed4a5e5f3ab3841 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:47 -0800 Subject: bpf: Replace PTR_TO_XXX_OR_NULL with PTR_TO_XXX | PTR_MAYBE_NULL We have introduced a new type to make bpf_reg composable, by allocating bits in the type to represent flags. One of the flags is PTR_MAYBE_NULL which indicates a pointer may be NULL. This patch switches the qualified reg_types to use this flag. The reg_types changed in this patch include: 1. PTR_TO_MAP_VALUE_OR_NULL 2. PTR_TO_SOCKET_OR_NULL 3. PTR_TO_SOCK_COMMON_OR_NULL 4. PTR_TO_TCP_SOCK_OR_NULL 5. PTR_TO_BTF_ID_OR_NULL 6. PTR_TO_MEM_OR_NULL 7. PTR_TO_RDONLY_BUF_OR_NULL 8. PTR_TO_RDWR_BUF_OR_NULL Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/r/20211217003152.48334-5-haoluo@google.com --- include/linux/bpf.h | 18 +-- include/linux/bpf_verifier.h | 4 + kernel/bpf/btf.c | 7 +- kernel/bpf/map_iter.c | 4 +- kernel/bpf/verifier.c | 298 ++++++++++++++++++------------------------- net/core/bpf_sk_storage.c | 2 +- net/core/sock_map.c | 2 +- 7 files changed, 147 insertions(+), 188 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 975a1d5951bd..c3de62267b84 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -465,18 +465,15 @@ enum bpf_reg_type { PTR_TO_CTX, /* reg points to bpf_context */ CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ PTR_TO_MAP_VALUE, /* reg points to map element value */ - PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ + PTR_TO_MAP_KEY, /* reg points to a map element key */ PTR_TO_STACK, /* reg == frame_pointer + offset */ PTR_TO_PACKET_META, /* skb->data - meta_len */ PTR_TO_PACKET, /* reg points to skb->data */ PTR_TO_PACKET_END, /* skb->data + headlen */ PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ PTR_TO_SOCKET, /* reg points to struct bpf_sock */ - PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ PTR_TO_SOCK_COMMON, /* reg points to sock_common */ - PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ - PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ /* PTR_TO_BTF_ID points to a kernel struct that does not need @@ -494,18 +491,21 @@ enum bpf_reg_type { * been checked for null. Used primarily to inform the verifier * an explicit null check is required for this struct. */ - PTR_TO_BTF_ID_OR_NULL, PTR_TO_MEM, /* reg points to valid memory region */ - PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ - PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */ PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ - PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */ PTR_TO_FUNC, /* reg points to a bpf program function */ - PTR_TO_MAP_KEY, /* reg points to a map element key */ __BPF_REG_TYPE_MAX, + /* Extended reg_types. */ + PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE, + PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET, + PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, + PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, + PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, + PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MEM, + /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 34e4ceaca3c7..143401d4c9d9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -18,6 +18,8 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) +/* size of type_str_buf in bpf_verifier. */ +#define TYPE_STR_BUF_LEN 64 /* Liveness marks, used for registers and spilled-regs (in stack slots). * Read marks propagate upwards until they find a write mark; they record that @@ -484,6 +486,8 @@ struct bpf_verifier_env { /* Same as scratched_regs but for stack slots */ u64 scratched_stack_slots; u32 prev_log_len, prev_insn_print_len; + /* buffer used in reg_type_str() to generate reg_type string */ + char type_str_buf[TYPE_STR_BUF_LEN]; }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a17de71abd2e..4e2ca7bea6c4 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4940,10 +4940,13 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; + u32 type, flag; + type = base_type(ctx_arg_info->reg_type); + flag = type_flag(ctx_arg_info->reg_type); if (ctx_arg_info->offset == off && - (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL || - ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) { + (type == PTR_TO_RDWR_BUF || type == PTR_TO_RDONLY_BUF) && + (flag & PTR_MAYBE_NULL)) { info->reg_type = ctx_arg_info->reg_type; return true; } diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 6a9542af4212..631f0e44b7a9 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__bpf_map_elem, key), - PTR_TO_RDONLY_BUF_OR_NULL }, + PTR_TO_RDONLY_BUF | PTR_MAYBE_NULL }, { offsetof(struct bpf_iter__bpf_map_elem, value), - PTR_TO_RDWR_BUF_OR_NULL }, + PTR_TO_RDWR_BUF | PTR_MAYBE_NULL }, }, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ccc068c5c5f2..97e9d3f31443 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -442,18 +442,6 @@ static bool reg_type_not_null(enum bpf_reg_type type) type == PTR_TO_SOCK_COMMON; } -static bool reg_type_may_be_null(enum bpf_reg_type type) -{ - return type == PTR_TO_MAP_VALUE_OR_NULL || - type == PTR_TO_SOCKET_OR_NULL || - type == PTR_TO_SOCK_COMMON_OR_NULL || - type == PTR_TO_TCP_SOCK_OR_NULL || - type == PTR_TO_BTF_ID_OR_NULL || - type == PTR_TO_MEM_OR_NULL || - type == PTR_TO_RDONLY_BUF_OR_NULL || - type == PTR_TO_RDWR_BUF_OR_NULL; -} - static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) { return reg->type == PTR_TO_MAP_VALUE && @@ -462,12 +450,9 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) { - return type == PTR_TO_SOCKET || - type == PTR_TO_SOCKET_OR_NULL || - type == PTR_TO_TCP_SOCK || - type == PTR_TO_TCP_SOCK_OR_NULL || - type == PTR_TO_MEM || - type == PTR_TO_MEM_OR_NULL; + return base_type(type) == PTR_TO_SOCKET || + base_type(type) == PTR_TO_TCP_SOCK || + base_type(type) == PTR_TO_MEM; } static bool arg_type_may_be_refcounted(enum bpf_arg_type type) @@ -537,39 +522,52 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) insn->imm == BPF_CMPXCHG; } -/* string representation of 'enum bpf_reg_type' */ -static const char * const reg_type_str[] = { - [NOT_INIT] = "?", - [SCALAR_VALUE] = "inv", - [PTR_TO_CTX] = "ctx", - [CONST_PTR_TO_MAP] = "map_ptr", - [PTR_TO_MAP_VALUE] = "map_value", - [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", - [PTR_TO_STACK] = "fp", - [PTR_TO_PACKET] = "pkt", - [PTR_TO_PACKET_META] = "pkt_meta", - [PTR_TO_PACKET_END] = "pkt_end", - [PTR_TO_FLOW_KEYS] = "flow_keys", - [PTR_TO_SOCKET] = "sock", - [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", - [PTR_TO_SOCK_COMMON] = "sock_common", - [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", - [PTR_TO_TCP_SOCK] = "tcp_sock", - [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", - [PTR_TO_TP_BUFFER] = "tp_buffer", - [PTR_TO_XDP_SOCK] = "xdp_sock", - [PTR_TO_BTF_ID] = "ptr_", - [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", - [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", - [PTR_TO_MEM] = "mem", - [PTR_TO_MEM_OR_NULL] = "mem_or_null", - [PTR_TO_RDONLY_BUF] = "rdonly_buf", - [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", - [PTR_TO_RDWR_BUF] = "rdwr_buf", - [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", - [PTR_TO_FUNC] = "func", - [PTR_TO_MAP_KEY] = "map_key", -}; +/* string representation of 'enum bpf_reg_type' + * + * Note that reg_type_str() can not appear more than once in a single verbose() + * statement. + */ +static const char *reg_type_str(struct bpf_verifier_env *env, + enum bpf_reg_type type) +{ + char postfix[16] = {0}; + static const char * const str[] = { + [NOT_INIT] = "?", + [SCALAR_VALUE] = "inv", + [PTR_TO_CTX] = "ctx", + [CONST_PTR_TO_MAP] = "map_ptr", + [PTR_TO_MAP_VALUE] = "map_value", + [PTR_TO_STACK] = "fp", + [PTR_TO_PACKET] = "pkt", + [PTR_TO_PACKET_META] = "pkt_meta", + [PTR_TO_PACKET_END] = "pkt_end", + [PTR_TO_FLOW_KEYS] = "flow_keys", + [PTR_TO_SOCKET] = "sock", + [PTR_TO_SOCK_COMMON] = "sock_common", + [PTR_TO_TCP_SOCK] = "tcp_sock", + [PTR_TO_TP_BUFFER] = "tp_buffer", + [PTR_TO_XDP_SOCK] = "xdp_sock", + [PTR_TO_BTF_ID] = "ptr_", + [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", + [PTR_TO_MEM] = "mem", + [PTR_TO_RDONLY_BUF] = "rdonly_buf", + [PTR_TO_RDWR_BUF] = "rdwr_buf", + [PTR_TO_FUNC] = "func", + [PTR_TO_MAP_KEY] = "map_key", + }; + + if (type & PTR_MAYBE_NULL) { + if (base_type(type) == PTR_TO_BTF_ID || + base_type(type) == PTR_TO_PERCPU_BTF_ID) + strncpy(postfix, "or_null_", 16); + else + strncpy(postfix, "_or_null", 16); + } + + snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s", + str[base_type(type)], postfix); + return env->type_str_buf; +} static char slot_type_char[] = { [STACK_INVALID] = '?', @@ -675,7 +673,7 @@ static void print_verifier_state(struct bpf_verifier_env *env, continue; verbose(env, " R%d", i); print_liveness(env, reg->live); - verbose(env, "=%s", reg_type_str[t]); + verbose(env, "=%s", reg_type_str(env, t)); if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && @@ -683,9 +681,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, /* reg->off should be 0 for SCALAR_VALUE */ verbose(env, "%lld", reg->var_off.value + reg->off); } else { - if (t == PTR_TO_BTF_ID || - t == PTR_TO_BTF_ID_OR_NULL || - t == PTR_TO_PERCPU_BTF_ID) + if (base_type(t) == PTR_TO_BTF_ID || + base_type(t) == PTR_TO_PERCPU_BTF_ID) verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); verbose(env, "(id=%d", reg->id); if (reg_type_may_be_refcounted_or_null(t)) @@ -694,10 +691,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, verbose(env, ",off=%d", reg->off); if (type_is_pkt_pointer(t)) verbose(env, ",r=%d", reg->range); - else if (t == CONST_PTR_TO_MAP || - t == PTR_TO_MAP_KEY || - t == PTR_TO_MAP_VALUE || - t == PTR_TO_MAP_VALUE_OR_NULL) + else if (base_type(t) == CONST_PTR_TO_MAP || + base_type(t) == PTR_TO_MAP_KEY || + base_type(t) == PTR_TO_MAP_VALUE) verbose(env, ",ks=%d,vs=%d", reg->map_ptr->key_size, reg->map_ptr->value_size); @@ -769,7 +765,7 @@ static void print_verifier_state(struct bpf_verifier_env *env, if (is_spilled_reg(&state->stack[i])) { reg = &state->stack[i].spilled_ptr; t = reg->type; - verbose(env, "=%s", reg_type_str[t]); + verbose(env, "=%s", reg_type_str(env, t)); if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) @@ -1202,8 +1198,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) { - switch (reg->type) { - case PTR_TO_MAP_VALUE_OR_NULL: { + if (base_type(reg->type) == PTR_TO_MAP_VALUE) { const struct bpf_map *map = reg->map_ptr; if (map->inner_map_meta) { @@ -1222,32 +1217,10 @@ static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) } else { reg->type = PTR_TO_MAP_VALUE; } - break; - } - case PTR_TO_SOCKET_OR_NULL: - reg->type = PTR_TO_SOCKET; - break; - case PTR_TO_SOCK_COMMON_OR_NULL: - reg->type = PTR_TO_SOCK_COMMON; - break; - case PTR_TO_TCP_SOCK_OR_NULL: - reg->type = PTR_TO_TCP_SOCK; - break; - case PTR_TO_BTF_ID_OR_NULL: - reg->type = PTR_TO_BTF_ID; - break; - case PTR_TO_MEM_OR_NULL: - reg->type = PTR_TO_MEM; - break; - case PTR_TO_RDONLY_BUF_OR_NULL: - reg->type = PTR_TO_RDONLY_BUF; - break; - case PTR_TO_RDWR_BUF_OR_NULL: - reg->type = PTR_TO_RDWR_BUF; - break; - default: - WARN_ONCE(1, "unknown nullable register type"); + return; } + + reg->type &= ~PTR_MAYBE_NULL; } static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) @@ -2103,7 +2076,7 @@ static int mark_reg_read(struct bpf_verifier_env *env, break; if (parent->live & REG_LIVE_DONE) { verbose(env, "verifier BUG type %s var_off %lld off %d\n", - reg_type_str[parent->type], + reg_type_str(env, parent->type), parent->var_off.value, parent->off); return -EFAULT; } @@ -2768,9 +2741,8 @@ static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) static bool is_spillable_regtype(enum bpf_reg_type type) { - switch (type) { + switch (base_type(type)) { case PTR_TO_MAP_VALUE: - case PTR_TO_MAP_VALUE_OR_NULL: case PTR_TO_STACK: case PTR_TO_CTX: case PTR_TO_PACKET: @@ -2779,21 +2751,14 @@ static bool is_spillable_regtype(enum bpf_reg_type type) case PTR_TO_FLOW_KEYS: case CONST_PTR_TO_MAP: case PTR_TO_SOCKET: - case PTR_TO_SOCKET_OR_NULL: case PTR_TO_SOCK_COMMON: - case PTR_TO_SOCK_COMMON_OR_NULL: case PTR_TO_TCP_SOCK: - case PTR_TO_TCP_SOCK_OR_NULL: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID_OR_NULL: case PTR_TO_RDONLY_BUF: - case PTR_TO_RDONLY_BUF_OR_NULL: case PTR_TO_RDWR_BUF: - case PTR_TO_RDWR_BUF_OR_NULL: case PTR_TO_PERCPU_BTF_ID: case PTR_TO_MEM: - case PTR_TO_MEM_OR_NULL: case PTR_TO_FUNC: case PTR_TO_MAP_KEY: return true; @@ -3633,7 +3598,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, */ *reg_type = info.reg_type; - if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { + if (base_type(*reg_type) == PTR_TO_BTF_ID) { *btf = info.btf; *btf_id = info.btf_id; } else { @@ -3701,7 +3666,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, } verbose(env, "R%d invalid %s access off=%d size=%d\n", - regno, reg_type_str[reg->type], off, size); + regno, reg_type_str(env, reg->type), off, size); return -EACCES; } @@ -4466,7 +4431,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else { mark_reg_known_zero(env, regs, value_regno); - if (reg_type_may_be_null(reg_type)) + if (type_may_be_null(reg_type)) regs[value_regno].id = ++env->id_gen; /* A load of ctx field could have different * actual load size with the one encoded in the @@ -4474,8 +4439,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn * a sub-register. */ regs[value_regno].subreg_def = DEF_NOT_SUBREG; - if (reg_type == PTR_TO_BTF_ID || - reg_type == PTR_TO_BTF_ID_OR_NULL) { + if (base_type(reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = btf; regs[value_regno].btf_id = btf_id; } @@ -4528,7 +4492,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (type_is_sk_pointer(reg->type)) { if (t == BPF_WRITE) { verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str[reg->type]); + regno, reg_type_str(env, reg->type)); return -EACCES; } err = check_sock_access(env, insn_idx, regno, off, size, t); @@ -4547,7 +4511,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (reg->type == PTR_TO_RDONLY_BUF) { if (t == BPF_WRITE) { verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str[reg->type]); + regno, reg_type_str(env, reg->type)); return -EACCES; } err = check_buffer_access(env, reg, regno, off, size, false, @@ -4563,7 +4527,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn mark_reg_unknown(env, regs, value_regno); } else { verbose(env, "R%d invalid mem access '%s'\n", regno, - reg_type_str[reg->type]); + reg_type_str(env, reg->type)); return -EACCES; } @@ -4630,7 +4594,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i is_sk_reg(env, insn->dst_reg)) { verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", insn->dst_reg, - reg_type_str[reg_state(env, insn->dst_reg)->type]); + reg_type_str(env, reg_state(env, insn->dst_reg)->type)); return -EACCES; } @@ -4850,9 +4814,9 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, register_is_null(reg)) return 0; - verbose(env, "R%d type=%s expected=%s\n", regno, - reg_type_str[reg->type], - reg_type_str[PTR_TO_STACK]); + verbose(env, "R%d type=%s ", regno, + reg_type_str(env, reg->type)); + verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); return -EACCES; } } @@ -4863,7 +4827,7 @@ int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, if (register_is_null(reg)) return 0; - if (reg_type_may_be_null(reg->type)) { + if (type_may_be_null(reg->type)) { /* Assuming that the register contains a value check if the memory * access is safe. Temporarily save and restore the register's state as * the conversion shouldn't be visible to a caller. @@ -5197,10 +5161,10 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, goto found; } - verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); + verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, type)); for (j = 0; j + 1 < i; j++) - verbose(env, "%s, ", reg_type_str[compatible->types[j]]); - verbose(env, "%s\n", reg_type_str[compatible->types[j]]); + verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); + verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); return -EACCES; found: @@ -6474,6 +6438,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn { const struct bpf_func_proto *fn = NULL; enum bpf_return_type ret_type; + enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; int insn_idx = *insn_idx_p; @@ -6614,6 +6579,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* update return register (already marked as written above) */ ret_type = fn->ret_type; + ret_flag = type_flag(fn->ret_type); if (ret_type == RET_INTEGER) { /* sets type to SCALAR_VALUE */ mark_reg_unknown(env, regs, BPF_REG_0); @@ -6633,25 +6599,23 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } regs[BPF_REG_0].map_ptr = meta.map_ptr; regs[BPF_REG_0].map_uid = meta.map_uid; - if (type_may_be_null(ret_type)) { - regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; - } else { - regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; - if (map_value_has_spin_lock(meta.map_ptr)) - regs[BPF_REG_0].id = ++env->id_gen; + regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; + if (!type_may_be_null(ret_type) && + map_value_has_spin_lock(meta.map_ptr)) { + regs[BPF_REG_0].id = ++env->id_gen; } } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) { mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; + regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) { mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; + regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) { mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; + regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) { mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; + regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; regs[BPF_REG_0].mem_size = meta.mem_size; } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) { const struct btf_type *t; @@ -6671,14 +6635,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn tname, PTR_ERR(ret)); return -EINVAL; } - regs[BPF_REG_0].type = - (ret_type & PTR_MAYBE_NULL) ? - PTR_TO_MEM_OR_NULL : PTR_TO_MEM; + regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; regs[BPF_REG_0].mem_size = tsize; } else { - regs[BPF_REG_0].type = - (ret_type & PTR_MAYBE_NULL) ? - PTR_TO_BTF_ID_OR_NULL : PTR_TO_BTF_ID; + regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; regs[BPF_REG_0].btf = meta.ret_btf; regs[BPF_REG_0].btf_id = meta.ret_btf_id; } @@ -6686,9 +6646,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn int ret_btf_id; mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].type = (ret_type & PTR_MAYBE_NULL) ? - PTR_TO_BTF_ID_OR_NULL : - PTR_TO_BTF_ID; + regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; ret_btf_id = *fn->ret_btf_id; if (ret_btf_id == 0) { verbose(env, "invalid return type %u of func %s#%d\n", @@ -6707,7 +6665,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (reg_type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type)) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id)) { @@ -6916,25 +6874,25 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { verbose(env, "math between %s pointer and %lld is not allowed\n", - reg_type_str[type], val); + reg_type_str(env, type), val); return false; } if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { verbose(env, "%s pointer offset %d is not allowed\n", - reg_type_str[type], reg->off); + reg_type_str(env, type), reg->off); return false; } if (smin == S64_MIN) { verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", - reg_type_str[type]); + reg_type_str(env, type)); return false; } if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { verbose(env, "value %lld makes %s pointer be out of bounds\n", - smin, reg_type_str[type]); + smin, reg_type_str(env, type)); return false; } @@ -7311,11 +7269,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - switch (ptr_reg->type) { - case PTR_TO_MAP_VALUE_OR_NULL: + if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", - dst, reg_type_str[ptr_reg->type]); + dst, reg_type_str(env, ptr_reg->type)); return -EACCES; + } + + switch (base_type(ptr_reg->type)) { case CONST_PTR_TO_MAP: /* smin_val represents the known value */ if (known && smin_val == 0 && opcode == BPF_ADD) @@ -7323,14 +7283,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, fallthrough; case PTR_TO_PACKET_END: case PTR_TO_SOCKET: - case PTR_TO_SOCKET_OR_NULL: case PTR_TO_SOCK_COMMON: - case PTR_TO_SOCK_COMMON_OR_NULL: case PTR_TO_TCP_SOCK: - case PTR_TO_TCP_SOCK_OR_NULL: case PTR_TO_XDP_SOCK: verbose(env, "R%d pointer arithmetic on %s prohibited\n", - dst, reg_type_str[ptr_reg->type]); + dst, reg_type_str(env, ptr_reg->type)); return -EACCES; default: break; @@ -9049,7 +9006,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, struct bpf_reg_state *reg, u32 id, bool is_null) { - if (reg_type_may_be_null(reg->type) && reg->id == id && + if (type_may_be_null(reg->type) && reg->id == id && !WARN_ON_ONCE(!reg->id)) { /* Old offset (both fixed and variable parts) should * have been known-zero, because we don't allow pointer @@ -9427,7 +9384,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, */ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && - reg_type_may_be_null(dst_reg->type)) { + type_may_be_null(dst_reg->type)) { /* Mark all identical registers in each branch as either * safe or unknown depending R == 0 or R != 0 conditional. */ @@ -9681,7 +9638,7 @@ static int check_return_code(struct bpf_verifier_env *env) /* enforce return zero from async callbacks like timer */ if (reg->type != SCALAR_VALUE) { verbose(env, "In async callback the register R0 is not a known value (%s)\n", - reg_type_str[reg->type]); + reg_type_str(env, reg->type)); return -EINVAL; } @@ -9695,7 +9652,7 @@ static int check_return_code(struct bpf_verifier_env *env) if (is_subprog) { if (reg->type != SCALAR_VALUE) { verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str[reg->type]); + reg_type_str(env, reg->type)); return -EINVAL; } return 0; @@ -9759,7 +9716,7 @@ static int check_return_code(struct bpf_verifier_env *env) if (reg->type != SCALAR_VALUE) { verbose(env, "At program exit the register R0 is not a known value (%s)\n", - reg_type_str[reg->type]); + reg_type_str(env, reg->type)); return -EINVAL; } @@ -10616,7 +10573,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, return true; if (rcur->type == NOT_INIT) return false; - switch (rold->type) { + switch (base_type(rold->type)) { case SCALAR_VALUE: if (env->explore_alu_limits) return false; @@ -10638,6 +10595,22 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, } case PTR_TO_MAP_KEY: case PTR_TO_MAP_VALUE: + /* a PTR_TO_MAP_VALUE could be safe to use as a + * PTR_TO_MAP_VALUE_OR_NULL into the same map. + * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- + * checked, doing so could have affected others with the same + * id, and we can't check for that because we lost the id when + * we converted to a PTR_TO_MAP_VALUE. + */ + if (type_may_be_null(rold->type)) { + if (!type_may_be_null(rcur->type)) + return false; + if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) + return false; + /* Check our ids match any regs they're supposed to */ + return check_ids(rold->id, rcur->id, idmap); + } + /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. * 'id' is not compared, since it's only used for maps with @@ -10649,20 +10622,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); - case PTR_TO_MAP_VALUE_OR_NULL: - /* a PTR_TO_MAP_VALUE could be safe to use as a - * PTR_TO_MAP_VALUE_OR_NULL into the same map. - * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- - * checked, doing so could have affected others with the same - * id, and we can't check for that because we lost the id when - * we converted to a PTR_TO_MAP_VALUE. - */ - if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) - return false; - if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) - return false; - /* Check our ids match any regs they're supposed to */ - return check_ids(rold->id, rcur->id, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: if (rcur->type != rold->type) @@ -10691,11 +10650,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_PACKET_END: case PTR_TO_FLOW_KEYS: case PTR_TO_SOCKET: - case PTR_TO_SOCKET_OR_NULL: case PTR_TO_SOCK_COMMON: - case PTR_TO_SOCK_COMMON_OR_NULL: case PTR_TO_TCP_SOCK: - case PTR_TO_TCP_SOCK_OR_NULL: case PTR_TO_XDP_SOCK: /* Only valid matches are exact, which memcmp() above * would have accepted @@ -11221,17 +11177,13 @@ next: /* Return true if it's OK to have the same insn return a different type. */ static bool reg_type_mismatch_ok(enum bpf_reg_type type) { - switch (type) { + switch (base_type(type)) { case PTR_TO_CTX: case PTR_TO_SOCKET: - case PTR_TO_SOCKET_OR_NULL: case PTR_TO_SOCK_COMMON: - case PTR_TO_SOCK_COMMON_OR_NULL: case PTR_TO_TCP_SOCK: - case PTR_TO_TCP_SOCK_OR_NULL: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID_OR_NULL: return false; default: return true; @@ -11457,7 +11409,7 @@ static int do_check(struct bpf_verifier_env *env) if (is_ctx_reg(env, insn->dst_reg)) { verbose(env, "BPF_ST stores into R%d %s is not allowed\n", insn->dst_reg, - reg_type_str[reg_state(env, insn->dst_reg)->type]); + reg_type_str(env, reg_state(env, insn->dst_reg)->type)); return -EACCES; } diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 68d2cbf8331a..4cb5ef8eddbc 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -929,7 +929,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), PTR_TO_BTF_ID_OR_NULL }, { offsetof(struct bpf_iter__bpf_sk_storage_map, value), - PTR_TO_RDWR_BUF_OR_NULL }, + PTR_TO_RDWR_BUF | PTR_MAYBE_NULL }, }, .seq_info = &iter_seq_info, }; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 4ca4b11f4e5f..96d4ea7e6918 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1564,7 +1564,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__sockmap, key), - PTR_TO_RDONLY_BUF_OR_NULL }, + PTR_TO_RDONLY_BUF | PTR_MAYBE_NULL }, { offsetof(struct bpf_iter__sockmap, sk), PTR_TO_BTF_ID_OR_NULL }, }, -- cgit v1.2.3-59-g8ed1b From 20b2aff4bc15bda809f994761d5719827d66c0b4 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:48 -0800 Subject: bpf: Introduce MEM_RDONLY flag This patch introduce a flag MEM_RDONLY to tag a reg value pointing to read-only memory. It makes the following changes: 1. PTR_TO_RDWR_BUF -> PTR_TO_BUF 2. PTR_TO_RDONLY_BUF -> PTR_TO_BUF | MEM_RDONLY Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-6-haoluo@google.com --- include/linux/bpf.h | 8 +++-- kernel/bpf/btf.c | 3 +- kernel/bpf/map_iter.c | 4 +-- kernel/bpf/verifier.c | 84 ++++++++++++++++++++++++++++------------------- net/core/bpf_sk_storage.c | 2 +- net/core/sock_map.c | 2 +- 6 files changed, 60 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c3de62267b84..126048110bdb 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -311,7 +311,10 @@ enum bpf_type_flag { /* PTR may be NULL. */ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), - __BPF_TYPE_LAST_FLAG = PTR_MAYBE_NULL, + /* MEM is read-only. */ + MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), + + __BPF_TYPE_LAST_FLAG = MEM_RDONLY, }; /* Max number of base types. */ @@ -492,8 +495,7 @@ enum bpf_reg_type { * an explicit null check is required for this struct. */ PTR_TO_MEM, /* reg points to valid memory region */ - PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ - PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ + PTR_TO_BUF, /* reg points to a read/write buffer */ PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */ PTR_TO_FUNC, /* reg points to a bpf program function */ __BPF_REG_TYPE_MAX, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4e2ca7bea6c4..d1447b075c73 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4944,8 +4944,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, type = base_type(ctx_arg_info->reg_type); flag = type_flag(ctx_arg_info->reg_type); - if (ctx_arg_info->offset == off && - (type == PTR_TO_RDWR_BUF || type == PTR_TO_RDONLY_BUF) && + if (ctx_arg_info->offset == off && type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)) { info->reg_type = ctx_arg_info->reg_type; return true; diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 631f0e44b7a9..b0fa190b0979 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -174,9 +174,9 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__bpf_map_elem, key), - PTR_TO_RDONLY_BUF | PTR_MAYBE_NULL }, + PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY }, { offsetof(struct bpf_iter__bpf_map_elem, value), - PTR_TO_RDWR_BUF | PTR_MAYBE_NULL }, + PTR_TO_BUF | PTR_MAYBE_NULL }, }, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 97e9d3f31443..e0a8a55ea3df 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -455,6 +455,11 @@ static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) base_type(type) == PTR_TO_MEM; } +static bool type_is_rdonly_mem(u32 type) +{ + return type & MEM_RDONLY; +} + static bool arg_type_may_be_refcounted(enum bpf_arg_type type) { return type == ARG_PTR_TO_SOCK_COMMON; @@ -530,7 +535,7 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) static const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type) { - char postfix[16] = {0}; + char postfix[16] = {0}, prefix[16] = {0}; static const char * const str[] = { [NOT_INIT] = "?", [SCALAR_VALUE] = "inv", @@ -550,8 +555,7 @@ static const char *reg_type_str(struct bpf_verifier_env *env, [PTR_TO_BTF_ID] = "ptr_", [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", [PTR_TO_MEM] = "mem", - [PTR_TO_RDONLY_BUF] = "rdonly_buf", - [PTR_TO_RDWR_BUF] = "rdwr_buf", + [PTR_TO_BUF] = "buf", [PTR_TO_FUNC] = "func", [PTR_TO_MAP_KEY] = "map_key", }; @@ -564,8 +568,11 @@ static const char *reg_type_str(struct bpf_verifier_env *env, strncpy(postfix, "_or_null", 16); } - snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s", - str[base_type(type)], postfix); + if (type & MEM_RDONLY) + strncpy(prefix, "rdonly_", 16); + + snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", + prefix, str[base_type(type)], postfix); return env->type_str_buf; } @@ -2755,8 +2762,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) case PTR_TO_TCP_SOCK: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: - case PTR_TO_RDONLY_BUF: - case PTR_TO_RDWR_BUF: + case PTR_TO_BUF: case PTR_TO_PERCPU_BTF_ID: case PTR_TO_MEM: case PTR_TO_FUNC: @@ -4508,22 +4514,28 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (reg->type == CONST_PTR_TO_MAP) { err = check_ptr_to_map_access(env, regs, regno, off, size, t, value_regno); - } else if (reg->type == PTR_TO_RDONLY_BUF) { - if (t == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str(env, reg->type)); - return -EACCES; + } else if (base_type(reg->type) == PTR_TO_BUF) { + bool rdonly_mem = type_is_rdonly_mem(reg->type); + const char *buf_info; + u32 *max_access; + + if (rdonly_mem) { + if (t == BPF_WRITE) { + verbose(env, "R%d cannot write into %s\n", + regno, reg_type_str(env, reg->type)); + return -EACCES; + } + buf_info = "rdonly"; + max_access = &env->prog->aux->max_rdonly_access; + } else { + buf_info = "rdwr"; + max_access = &env->prog->aux->max_rdwr_access; } + err = check_buffer_access(env, reg, regno, off, size, false, - "rdonly", - &env->prog->aux->max_rdonly_access); - if (!err && value_regno >= 0) - mark_reg_unknown(env, regs, value_regno); - } else if (reg->type == PTR_TO_RDWR_BUF) { - err = check_buffer_access(env, reg, regno, off, size, false, - "rdwr", - &env->prog->aux->max_rdwr_access); - if (!err && t == BPF_READ && value_regno >= 0) + buf_info, max_access); + + if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); } else { verbose(env, "R%d invalid mem access '%s'\n", regno, @@ -4771,8 +4783,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; + const char *buf_info; + u32 *max_access; - switch (reg->type) { + switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: return check_packet_access(env, regno, reg->off, access_size, @@ -4791,18 +4805,20 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, return check_mem_region_access(env, regno, reg->off, access_size, reg->mem_size, zero_size_allowed); - case PTR_TO_RDONLY_BUF: - if (meta && meta->raw_mode) - return -EACCES; - return check_buffer_access(env, reg, regno, reg->off, - access_size, zero_size_allowed, - "rdonly", - &env->prog->aux->max_rdonly_access); - case PTR_TO_RDWR_BUF: + case PTR_TO_BUF: + if (type_is_rdonly_mem(reg->type)) { + if (meta && meta->raw_mode) + return -EACCES; + + buf_info = "rdonly"; + max_access = &env->prog->aux->max_rdonly_access; + } else { + buf_info = "rdwr"; + max_access = &env->prog->aux->max_rdwr_access; + } return check_buffer_access(env, reg, regno, reg->off, access_size, zero_size_allowed, - "rdwr", - &env->prog->aux->max_rdwr_access); + buf_info, max_access); case PTR_TO_STACK: return check_stack_range_initialized( env, @@ -5081,8 +5097,8 @@ static const struct bpf_reg_types mem_types = { PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE, PTR_TO_MEM, - PTR_TO_RDONLY_BUF, - PTR_TO_RDWR_BUF, + PTR_TO_BUF, + PTR_TO_BUF | MEM_RDONLY, }, }; diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 4cb5ef8eddbc..ea61dfe19c86 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -929,7 +929,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), PTR_TO_BTF_ID_OR_NULL }, { offsetof(struct bpf_iter__bpf_sk_storage_map, value), - PTR_TO_RDWR_BUF | PTR_MAYBE_NULL }, + PTR_TO_BUF | PTR_MAYBE_NULL }, }, .seq_info = &iter_seq_info, }; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 96d4ea7e6918..9618ab6d7cc9 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1564,7 +1564,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__sockmap, key), - PTR_TO_RDONLY_BUF | PTR_MAYBE_NULL }, + PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY }, { offsetof(struct bpf_iter__sockmap, sk), PTR_TO_BTF_ID_OR_NULL }, }, -- cgit v1.2.3-59-g8ed1b From cf9f2f8d62eca810afbd1ee6cc0800202b000e57 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:49 -0800 Subject: bpf: Convert PTR_TO_MEM_OR_NULL to composable types. Remove PTR_TO_MEM_OR_NULL and replace it with PTR_TO_MEM combined with flag PTR_MAYBE_NULL. Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-7-haoluo@google.com --- include/linux/bpf.h | 1 - kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 126048110bdb..567d83bf28f9 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -506,7 +506,6 @@ enum bpf_reg_type { PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, - PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MEM, /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d1447b075c73..d948b5be3bb8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5859,7 +5859,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, return -EINVAL; } - reg->type = PTR_TO_MEM_OR_NULL; + reg->type = PTR_TO_MEM | PTR_MAYBE_NULL; reg->id = ++env->id_gen; continue; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e0a8a55ea3df..9073337ac66f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13578,7 +13578,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) mark_reg_known_zero(env, regs, i); else if (regs[i].type == SCALAR_VALUE) mark_reg_unknown(env, regs, i); - else if (regs[i].type == PTR_TO_MEM_OR_NULL) { + else if (base_type(regs[i].type) == PTR_TO_MEM) { const u32 mem_size = regs[i].mem_size; mark_reg_known_zero(env, regs, i); -- cgit v1.2.3-59-g8ed1b From 216e3cd2f28dbbf1fe86848e0e29e6693b9f0a20 Mon Sep 17 00:00:00 2001 From: Hao Luo Date: Thu, 16 Dec 2021 16:31:51 -0800 Subject: bpf: Add MEM_RDONLY for helper args that are pointers to rdonly mem. Some helper functions may modify its arguments, for example, bpf_d_path, bpf_get_stack etc. Previously, their argument types were marked as ARG_PTR_TO_MEM, which is compatible with read-only mem types, such as PTR_TO_RDONLY_BUF. Therefore it's legitimate, but technically incorrect, to modify a read-only memory by passing it into one of such helper functions. This patch tags the bpf_args compatible with immutable memory with MEM_RDONLY flag. The arguments that don't have this flag will be only compatible with mutable memory types, preventing the helper from modifying a read-only memory. The bpf_args that have MEM_RDONLY are compatible with both mutable memory and immutable memory. Signed-off-by: Hao Luo Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211217003152.48334-9-haoluo@google.com --- include/linux/bpf.h | 4 ++- kernel/bpf/btf.c | 2 +- kernel/bpf/cgroup.c | 2 +- kernel/bpf/helpers.c | 8 +++--- kernel/bpf/ringbuf.c | 2 +- kernel/bpf/syscall.c | 2 +- kernel/bpf/verifier.c | 20 ++++++++++++--- kernel/trace/bpf_trace.c | 26 ++++++++++---------- net/core/filter.c | 64 ++++++++++++++++++++++++------------------------ 9 files changed, 73 insertions(+), 57 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 567d83bf28f9..26753139d5b4 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -311,7 +311,9 @@ enum bpf_type_flag { /* PTR may be NULL. */ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), - /* MEM is read-only. */ + /* MEM is read-only. When applied on bpf_arg, it indicates the arg is + * compatible with both mutable and immutable memory. + */ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), __BPF_TYPE_LAST_FLAG = MEM_RDONLY, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d948b5be3bb8..b3fddfb5bc84 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6353,7 +6353,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .func = bpf_btf_find_by_name_kind, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 43eb3501721b..514b4681a90a 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1789,7 +1789,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, }; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6a65e2a62b01..01cfdf40c838 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -531,7 +531,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .func = bpf_strtol, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_LONG, @@ -559,7 +559,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .func = bpf_strtoul, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_LONG, @@ -645,7 +645,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -1026,7 +1026,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .arg1_type = ARG_PTR_TO_MEM_OR_NULL, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, - .arg4_type = ARG_PTR_TO_MEM_OR_NULL, + .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 9e0c10c6892a..638d7fd7b375 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -444,7 +444,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .func = bpf_ringbuf_output, .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index da07bdf71697..fa4505f9b611 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4773,7 +4773,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f49b3d334f4e..ca5cd0de804c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5113,7 +5113,6 @@ static const struct bpf_reg_types mem_types = { PTR_TO_MAP_VALUE, PTR_TO_MEM, PTR_TO_BUF, - PTR_TO_BUF | MEM_RDONLY, }, }; @@ -5183,6 +5182,21 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, return -EFAULT; } + /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, + * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY + * + * Same for MAYBE_NULL: + * + * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, + * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL + * + * Therefore we fold these flags depending on the arg_type before comparison. + */ + if (arg_type & MEM_RDONLY) + type &= ~MEM_RDONLY; + if (arg_type & PTR_MAYBE_NULL) + type &= ~PTR_MAYBE_NULL; + for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { expected = compatible->types[i]; if (expected == NOT_INIT) @@ -5192,14 +5206,14 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, goto found; } - verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, type)); + verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); return -EACCES; found: - if (type == PTR_TO_BTF_ID) { + if (reg->type == PTR_TO_BTF_ID) { if (!arg_btf_id) { if (!compatible->btf_id) { verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index cea2ca6df949..21aa30644219 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -345,7 +345,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, }; @@ -394,7 +394,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .func = bpf_trace_printk, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, }; @@ -450,9 +450,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { .func = bpf_trace_vprintk, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, - .arg3_type = ARG_PTR_TO_MEM_OR_NULL, + .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -492,9 +492,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, - .arg4_type = ARG_PTR_TO_MEM_OR_NULL, + .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -509,7 +509,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -533,7 +533,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -694,7 +694,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -1004,7 +1004,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM, .arg2_type = ARG_CONST_SIZE, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1334,7 +1334,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -1556,7 +1556,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -1610,7 +1610,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/net/core/filter.c b/net/core/filter.c index 3f656391af7e..606ab5a98a1a 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1712,7 +1712,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -2017,9 +2017,9 @@ static const struct bpf_func_proto bpf_csum_diff_proto = { .gpl_only = false, .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL, + .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE_OR_ZERO, - .arg3_type = ARG_PTR_TO_MEM_OR_NULL, + .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -2540,7 +2540,7 @@ static const struct bpf_func_proto bpf_redirect_neigh_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL, + .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -4173,7 +4173,7 @@ static const struct bpf_func_proto bpf_skb_event_output_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -4187,7 +4187,7 @@ const struct bpf_func_proto bpf_skb_output_proto = { .arg1_btf_id = &bpf_skb_output_btf_ids[0], .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -4370,7 +4370,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -4396,7 +4396,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, }; @@ -4566,7 +4566,7 @@ static const struct bpf_func_proto bpf_xdp_event_output_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -4580,7 +4580,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = { .arg1_btf_id = &bpf_xdp_output_btf_ids[0], .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -5066,7 +5066,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_proto = { .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE, }; @@ -5100,7 +5100,7 @@ static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE, }; @@ -5134,7 +5134,7 @@ static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE, }; @@ -5309,7 +5309,7 @@ static const struct bpf_func_proto bpf_bind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, }; @@ -5897,7 +5897,7 @@ static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE }; @@ -5907,7 +5907,7 @@ static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE }; @@ -5950,7 +5950,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE }; @@ -6038,7 +6038,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_action_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE }; @@ -6263,7 +6263,7 @@ static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6282,7 +6282,7 @@ static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6301,7 +6301,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6338,7 +6338,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6361,7 +6361,7 @@ static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6384,7 +6384,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = { .pkt_access = true, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6403,7 +6403,7 @@ static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = { .gpl_only = false, .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6422,7 +6422,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = { .gpl_only = false, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6441,7 +6441,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = { .gpl_only = false, .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -6754,9 +6754,9 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = { .pkt_access = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE, }; @@ -6823,9 +6823,9 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { .pkt_access = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, - .arg4_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg5_type = ARG_CONST_SIZE, }; @@ -7054,7 +7054,7 @@ static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; -- cgit v1.2.3-59-g8ed1b From 8cbfe939abe905280279e84a297b1cb34e0d0ec9 Mon Sep 17 00:00:00 2001 From: Baowen Zheng Date: Fri, 17 Dec 2021 19:16:22 +0100 Subject: flow_offload: allow user to offload tc action to net device Use flow_indr_dev_register/flow_indr_dev_setup_offload to offload tc action. We need to call tc_cleanup_flow_action to clean up tc action entry since in tc_setup_action, some actions may hold dev refcnt, especially the mirror action. Signed-off-by: Baowen Zheng Signed-off-by: Louis Peens Signed-off-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + include/net/flow_offload.h | 17 +++++++++ include/net/pkt_cls.h | 5 +++ net/core/flow_offload.c | 42 +++++++++++++++++---- net/sched/act_api.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++ net/sched/act_csum.c | 4 +- net/sched/act_ct.c | 4 +- net/sched/act_gact.c | 13 ++++++- net/sched/act_gate.c | 4 +- net/sched/act_mirred.c | 13 ++++++- net/sched/act_mpls.c | 16 +++++++- net/sched/act_police.c | 4 +- net/sched/act_sample.c | 4 +- net/sched/act_skbedit.c | 11 +++++- net/sched/act_tunnel_key.c | 9 ++++- net/sched/act_vlan.c | 16 +++++++- net/sched/cls_api.c | 21 +++++++++-- 17 files changed, 254 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a419718612c6..8b0bdeb4734e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -920,6 +920,7 @@ enum tc_setup_type { TC_SETUP_QDISC_TBF, TC_SETUP_QDISC_FIFO, TC_SETUP_QDISC_HTB, + TC_SETUP_ACT, }; /* These structures hold the attributes of bpf state that are being passed diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h index 2271da5aa8ee..5b8c54eb7a6b 100644 --- a/include/net/flow_offload.h +++ b/include/net/flow_offload.h @@ -551,6 +551,23 @@ struct flow_cls_offload { u32 classid; }; +enum offload_act_command { + FLOW_ACT_REPLACE, + FLOW_ACT_DESTROY, + FLOW_ACT_STATS, +}; + +struct flow_offload_action { + struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ + enum offload_act_command command; + enum flow_action_id id; + u32 index; + struct flow_stats stats; + struct flow_action action; +}; + +struct flow_offload_action *offload_action_alloc(unsigned int num_actions); + static inline struct flow_rule * flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) { diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index 5d4ff76d37e2..1bfb616ea759 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -262,6 +262,9 @@ static inline void tcf_exts_put_net(struct tcf_exts *exts) for (; 0; (void)(i), (void)(a), (void)(exts)) #endif +#define tcf_act_for_each_action(i, a, actions) \ + for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++) + static inline void tcf_exts_stats_update(const struct tcf_exts *exts, u64 bytes, u64 packets, u64 drops, u64 lastuse, @@ -539,6 +542,8 @@ tcf_match_indev(struct sk_buff *skb, int ifindex) int tc_setup_offload_action(struct flow_action *flow_action, const struct tcf_exts *exts); void tc_cleanup_offload_action(struct flow_action *flow_action); +int tc_setup_action(struct flow_action *flow_action, + struct tc_action *actions[]); int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, void *type_data, bool err_stop, bool rtnl_held); diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c index 6beaea13564a..022c945817fa 100644 --- a/net/core/flow_offload.c +++ b/net/core/flow_offload.c @@ -27,6 +27,26 @@ struct flow_rule *flow_rule_alloc(unsigned int num_actions) } EXPORT_SYMBOL(flow_rule_alloc); +struct flow_offload_action *offload_action_alloc(unsigned int num_actions) +{ + struct flow_offload_action *fl_action; + int i; + + fl_action = kzalloc(struct_size(fl_action, action.entries, num_actions), + GFP_KERNEL); + if (!fl_action) + return NULL; + + fl_action->action.num_entries = num_actions; + /* Pre-fill each action hw_stats with DONT_CARE. + * Caller can override this if it wants stats for a given action. + */ + for (i = 0; i < num_actions; i++) + fl_action->action.entries[i].hw_stats = FLOW_ACTION_HW_STATS_DONT_CARE; + + return fl_action; +} + #define FLOW_DISSECTOR_MATCH(__rule, __type, __out) \ const struct flow_match *__m = &(__rule)->match; \ struct flow_dissector *__d = (__m)->dissector; \ @@ -549,19 +569,25 @@ int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, void (*cleanup)(struct flow_block_cb *block_cb)) { struct flow_indr_dev *this; + u32 count = 0; + int err; mutex_lock(&flow_indr_block_lock); + if (bo) { + if (bo->command == FLOW_BLOCK_BIND) + indir_dev_add(data, dev, sch, type, cleanup, bo); + else if (bo->command == FLOW_BLOCK_UNBIND) + indir_dev_remove(data); + } - if (bo->command == FLOW_BLOCK_BIND) - indir_dev_add(data, dev, sch, type, cleanup, bo); - else if (bo->command == FLOW_BLOCK_UNBIND) - indir_dev_remove(data); - - list_for_each_entry(this, &flow_block_indr_dev_list, list) - this->cb(dev, sch, this->cb_priv, type, bo, data, cleanup); + list_for_each_entry(this, &flow_block_indr_dev_list, list) { + err = this->cb(dev, sch, this->cb_priv, type, bo, data, cleanup); + if (!err) + count++; + } mutex_unlock(&flow_indr_block_lock); - return list_empty(&bo->cb_list) ? -EOPNOTSUPP : 0; + return (bo && list_empty(&bo->cb_list)) ? -EOPNOTSUPP : count; } EXPORT_SYMBOL(flow_indr_dev_setup_offload); diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 3258da3d5bed..5c21401b0555 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -19,8 +19,10 @@ #include #include #include +#include #include #include +#include #ifdef CONFIG_INET DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count); @@ -129,8 +131,92 @@ static void free_tcf(struct tc_action *p) kfree(p); } +static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) +{ + if (is_tcf_pedit(act)) + return tcf_pedit_nkeys(act); + else + return 1; +} + +static int offload_action_init(struct flow_offload_action *fl_action, + struct tc_action *act, + enum offload_act_command cmd, + struct netlink_ext_ack *extack) +{ + fl_action->extack = extack; + fl_action->command = cmd; + fl_action->index = act->tcfa_index; + + if (act->ops->offload_act_setup) + return act->ops->offload_act_setup(act, fl_action, NULL, false); + + return -EOPNOTSUPP; +} + +static int tcf_action_offload_cmd(struct flow_offload_action *fl_act, + struct netlink_ext_ack *extack) +{ + int err; + + err = flow_indr_dev_setup_offload(NULL, NULL, TC_SETUP_ACT, + fl_act, NULL, NULL); + if (err < 0) + return err; + + return 0; +} + +/* offload the tc action after it is inserted */ +static int tcf_action_offload_add(struct tc_action *action, + struct netlink_ext_ack *extack) +{ + struct tc_action *actions[TCA_ACT_MAX_PRIO] = { + [0] = action, + }; + struct flow_offload_action *fl_action; + int num, err = 0; + + num = tcf_offload_act_num_actions_single(action); + fl_action = offload_action_alloc(num); + if (!fl_action) + return -ENOMEM; + + err = offload_action_init(fl_action, action, FLOW_ACT_REPLACE, extack); + if (err) + goto fl_err; + + err = tc_setup_action(&fl_action->action, actions); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to setup tc actions for offload\n"); + goto fl_err; + } + + err = tcf_action_offload_cmd(fl_action, extack); + tc_cleanup_offload_action(&fl_action->action); + +fl_err: + kfree(fl_action); + + return err; +} + +static int tcf_action_offload_del(struct tc_action *action) +{ + struct flow_offload_action fl_act = {}; + int err = 0; + + err = offload_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL); + if (err) + return err; + + return tcf_action_offload_cmd(&fl_act, NULL); +} + static void tcf_action_cleanup(struct tc_action *p) { + tcf_action_offload_del(p); if (p->ops->cleanup) p->ops->cleanup(p); @@ -1061,6 +1147,11 @@ err_out: return ERR_PTR(err); } +static bool tc_act_bind(u32 flags) +{ + return !!(flags & TCA_ACT_FLAGS_BIND); +} + /* Returns numbers of initialized actions or negative error. */ int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, @@ -1103,6 +1194,8 @@ int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, sz += tcf_action_fill_size(act); /* Start from index 0 */ actions[i - 1] = act; + if (!tc_act_bind(flags)) + tcf_action_offload_add(act, extack); } /* We have to commit them all together, because if any error happened in diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 4428852a03d7..e0f515b774ca 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -705,7 +705,9 @@ static int tcf_csum_offload_act_setup(struct tc_action *act, void *entry_data, entry->csum_flags = tcf_csum_update_flags(act); *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + fl_action->id = FLOW_ACTION_CSUM; } return 0; diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index dc64f31e5191..1c537913a189 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1505,7 +1505,9 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, entry->ct.flow_table = tcf_ct_ft(act); *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + fl_action->id = FLOW_ACTION_CT; } return 0; diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index f77be22069f4..bde6a6c01e64 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -272,7 +272,18 @@ static int tcf_gact_offload_act_setup(struct tc_action *act, void *entry_data, } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + if (is_tcf_gact_ok(act)) + fl_action->id = FLOW_ACTION_ACCEPT; + else if (is_tcf_gact_shot(act)) + fl_action->id = FLOW_ACTION_DROP; + else if (is_tcf_gact_trap(act)) + fl_action->id = FLOW_ACTION_TRAP; + else if (is_tcf_gact_goto_chain(act)) + fl_action->id = FLOW_ACTION_GOTO; + else + return -EOPNOTSUPP; } return 0; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index 1d8297497692..d56e73843a4b 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -637,7 +637,9 @@ static int tcf_gate_offload_act_setup(struct tc_action *act, void *entry_data, return err; *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + fl_action->id = FLOW_ACTION_GATE; } return 0; diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index 8eecf55be0a2..39acd1d18609 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -482,7 +482,18 @@ static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data, } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + if (is_tcf_mirred_egress_redirect(act)) + fl_action->id = FLOW_ACTION_REDIRECT; + else if (is_tcf_mirred_egress_mirror(act)) + fl_action->id = FLOW_ACTION_MIRRED; + else if (is_tcf_mirred_ingress_redirect(act)) + fl_action->id = FLOW_ACTION_REDIRECT_INGRESS; + else if (is_tcf_mirred_ingress_mirror(act)) + fl_action->id = FLOW_ACTION_MIRRED_INGRESS; + else + return -EOPNOTSUPP; } return 0; diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index a4615e1331e0..b9ff3459fdab 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -415,7 +415,21 @@ static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data, } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + switch (tcf_mpls_action(act)) { + case TCA_MPLS_ACT_PUSH: + fl_action->id = FLOW_ACTION_MPLS_PUSH; + break; + case TCA_MPLS_ACT_POP: + fl_action->id = FLOW_ACTION_MPLS_POP; + break; + case TCA_MPLS_ACT_MODIFY: + fl_action->id = FLOW_ACTION_MPLS_MANGLE; + break; + default: + return -EOPNOTSUPP; + } } return 0; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index abb6d16a20b2..0923aa2b8f8a 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -421,7 +421,9 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, entry->police.mtu = tcf_police_tcfp_mtu(act); *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + fl_action->id = FLOW_ACTION_POLICE; } return 0; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 07e56903211e..9a22cdda6bbd 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -303,7 +303,9 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, tcf_offload_sample_get_group(entry, act); *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + fl_action->id = FLOW_ACTION_SAMPLE; } return 0; diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index c380f9e6cc95..ceba11b198bb 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -347,7 +347,16 @@ static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + if (is_tcf_skbedit_mark(act)) + fl_action->id = FLOW_ACTION_MARK; + else if (is_tcf_skbedit_ptype(act)) + fl_action->id = FLOW_ACTION_PTYPE; + else if (is_tcf_skbedit_priority(act)) + fl_action->id = FLOW_ACTION_PRIORITY; + else + return -EOPNOTSUPP; } return 0; diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index e96a65a5323e..23aba03d26a8 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -827,7 +827,14 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + if (is_tcf_tunnel_set(act)) + fl_action->id = FLOW_ACTION_TUNNEL_ENCAP; + else if (is_tcf_tunnel_release(act)) + fl_action->id = FLOW_ACTION_TUNNEL_DECAP; + else + return -EOPNOTSUPP; } return 0; diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c index 0300792084f0..756e2dcde1cd 100644 --- a/net/sched/act_vlan.c +++ b/net/sched/act_vlan.c @@ -395,7 +395,21 @@ static int tcf_vlan_offload_act_setup(struct tc_action *act, void *entry_data, } *index_inc = 1; } else { - return -EOPNOTSUPP; + struct flow_offload_action *fl_action = entry_data; + + switch (tcf_vlan_action(act)) { + case TCA_VLAN_ACT_PUSH: + fl_action->id = FLOW_ACTION_VLAN_PUSH; + break; + case TCA_VLAN_ACT_POP: + fl_action->id = FLOW_ACTION_VLAN_POP; + break; + case TCA_VLAN_ACT_MODIFY: + fl_action->id = FLOW_ACTION_VLAN_MANGLE; + break; + default: + return -EOPNOTSUPP; + } } return 0; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 53f263c9a725..353e1eed48be 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3488,8 +3488,8 @@ static int tc_setup_offload_act(struct tc_action *act, #endif } -int tc_setup_offload_action(struct flow_action *flow_action, - const struct tcf_exts *exts) +int tc_setup_action(struct flow_action *flow_action, + struct tc_action *actions[]) { int i, j, index, err = 0; struct tc_action *act; @@ -3498,11 +3498,11 @@ int tc_setup_offload_action(struct flow_action *flow_action, BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE); BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED); - if (!exts) + if (!actions) return 0; j = 0; - tcf_exts_for_each_action(i, act, exts) { + tcf_act_for_each_action(i, act, actions) { struct flow_action_entry *entry; entry = &flow_action->entries[j]; @@ -3531,6 +3531,19 @@ err_out_locked: spin_unlock_bh(&act->tcfa_lock); goto err_out; } + +int tc_setup_offload_action(struct flow_action *flow_action, + const struct tcf_exts *exts) +{ +#ifdef CONFIG_NET_CLS_ACT + if (!exts) + return 0; + + return tc_setup_action(flow_action, exts->actions); +#else + return 0; +#endif +} EXPORT_SYMBOL(tc_setup_offload_action); unsigned int tcf_exts_num_actions(struct tcf_exts *exts) -- cgit v1.2.3-59-g8ed1b From 7d4203c13435c0bdae61bf16bbd0408d5b958ade Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Thu, 25 Nov 2021 18:15:37 +0100 Subject: mm: add virt_to_folio() and folio_address() These two wrappers around their respective struct page variants will be useful in the following patches. Signed-off-by: Vlastimil Babka Acked-by: Johannes Weiner Reviewed-by: Roman Gushchin --- include/linux/mm.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index a7e4a9e7d807..4a6cf22483da 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -863,6 +863,13 @@ static inline struct page *virt_to_head_page(const void *x) return compound_head(page); } +static inline struct folio *virt_to_folio(const void *x) +{ + struct page *page = virt_to_page(x); + + return page_folio(page); +} + void __put_page(struct page *page); void put_pages_list(struct list_head *pages); @@ -1753,6 +1760,11 @@ void page_address_init(void); #define page_address_init() do { } while(0) #endif +static inline void *folio_address(const struct folio *folio) +{ + return page_address(&folio->page); +} + extern void *page_rmapping(struct page *page); extern struct anon_vma *page_anon_vma(struct page *page); extern pgoff_t __page_file_index(struct page *page); -- cgit v1.2.3-59-g8ed1b From d5c383f2c98ac58c210b266cdaf7b86bc32d1ad1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:30:56 +0000 Subject: iommu/iova: Squash entry_dtor abstraction All flush queues are driven by iommu-dma now, so there is no need to abstract entry_dtor or its data any more. Squash the now-canonical implementation directly into the IOVA code to get it out of the way. Reviewed-by: John Garry Reviewed-by: Christoph Hellwig Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/2260f8de00ab5e0f9d2a1cf8978e6ae7cd4f182c.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 17 ++--------------- drivers/iommu/iova.c | 28 +++++++++++++++------------- include/linux/iova.h | 26 +++----------------------- 3 files changed, 20 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 84dee53fe892..6691f3cd768f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -64,18 +64,6 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); -static void iommu_dma_entry_dtor(unsigned long data) -{ - struct page *freelist = (struct page *)data; - - while (freelist) { - unsigned long p = (unsigned long)page_address(freelist); - - freelist = freelist->freelist; - free_page(p); - } -} - static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) { if (cookie->type == IOMMU_DMA_IOVA_COOKIE) @@ -324,8 +312,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain) if (cookie->fq_domain) return 0; - ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all, - iommu_dma_entry_dtor); + ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all); if (ret) { pr_warn("iova flush queue initialization failed\n"); return ret; @@ -471,7 +458,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, else if (gather && gather->queued) queue_iova(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad), - (unsigned long)gather->freelist); + gather->freelist); else free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 670211e41771..541857ca4fd5 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -91,11 +91,9 @@ static void free_iova_flush_queue(struct iova_domain *iovad) iovad->fq = NULL; iovad->flush_cb = NULL; - iovad->entry_dtor = NULL; } -int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, iova_entry_dtor entry_dtor) +int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) { struct iova_fq __percpu *queue; int cpu; @@ -108,7 +106,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, return -ENOMEM; iovad->flush_cb = flush_cb; - iovad->entry_dtor = entry_dtor; for_each_possible_cpu(cpu) { struct iova_fq *fq; @@ -547,6 +544,16 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) } EXPORT_SYMBOL_GPL(free_iova_fast); +static void fq_entry_dtor(struct page *freelist) +{ + while (freelist) { + unsigned long p = (unsigned long)page_address(freelist); + + freelist = freelist->freelist; + free_page(p); + } +} + #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -579,9 +586,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) if (fq->entries[idx].counter >= counter) break; - if (iovad->entry_dtor) - iovad->entry_dtor(fq->entries[idx].data); - + fq_entry_dtor(fq->entries[idx].freelist); free_iova_fast(iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -606,15 +611,12 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) * bother to free iovas, just call the entry_dtor on all remaining * entries. */ - if (!iovad->entry_dtor) - return; - for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); int idx; fq_ring_for_each(idx, fq) - iovad->entry_dtor(fq->entries[idx].data); + fq_entry_dtor(fq->entries[idx].freelist); } } @@ -639,7 +641,7 @@ static void fq_flush_timeout(struct timer_list *t) void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - unsigned long data) + struct page *freelist) { struct iova_fq *fq; unsigned long flags; @@ -673,7 +675,7 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].data = data; + fq->entries[idx].freelist = freelist; fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); spin_unlock_irqrestore(&fq->lock, flags); diff --git a/include/linux/iova.h b/include/linux/iova.h index 71d8a2de6635..e746d8e41449 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -40,9 +40,6 @@ struct iova_domain; /* Call-Back from IOVA code into IOMMU drivers */ typedef void (* iova_flush_cb)(struct iova_domain *domain); -/* Destructor for per-entry data */ -typedef void (* iova_entry_dtor)(unsigned long data); - /* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256 @@ -53,7 +50,7 @@ typedef void (* iova_entry_dtor)(unsigned long data); struct iova_fq_entry { unsigned long iova_pfn; unsigned long pages; - unsigned long data; + struct page *freelist; u64 counter; /* Flush counter when this entrie was added */ }; @@ -88,9 +85,6 @@ struct iova_domain { iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU TLBs */ - iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for - iova entry */ - struct timer_list fq_timer; /* Timer to regularily empty the flush-queues */ atomic_t fq_timer_on; /* 1 when timer is active, 0 @@ -146,15 +140,14 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - unsigned long data); + struct page *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, iova_entry_dtor entry_dtor); +int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else @@ -189,12 +182,6 @@ static inline void free_iova_fast(struct iova_domain *iovad, { } -static inline void queue_iova(struct iova_domain *iovad, - unsigned long pfn, unsigned long pages, - unsigned long data) -{ -} - static inline unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, @@ -216,13 +203,6 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } -static inline int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, - iova_entry_dtor entry_dtor) -{ - return -ENODEV; -} - static inline struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) { -- cgit v1.2.3-59-g8ed1b From 649ad9835a3783bcb6c69368fa939e0010abb2c6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:30:57 +0000 Subject: iommu/iova: Squash flush_cb abstraction Once again, with iommu-dma now being the only flush queue user, we no longer need the extra level of indirection through flush_cb. Squash that and let the flush queue code call the domain method directly. This does mean temporarily having to carry an additional copy of the IOMMU domain pointer around instead, but only until a later patch untangles it again. Reviewed-by: John Garry Reviewed-by: Christoph Hellwig Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/e3f9b4acdd6640012ef4fbc819ac868d727b64a9.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 13 +------------ drivers/iommu/iova.c | 11 +++++------ include/linux/iova.h | 11 +++-------- 3 files changed, 9 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 6691f3cd768f..c63d93581a4e 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -282,17 +282,6 @@ static int iova_reserve_iommu_regions(struct device *dev, return ret; } -static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) -{ - struct iommu_dma_cookie *cookie; - struct iommu_domain *domain; - - cookie = container_of(iovad, struct iommu_dma_cookie, iovad); - domain = cookie->fq_domain; - - domain->ops->flush_iotlb_all(domain); -} - static bool dev_is_untrusted(struct device *dev) { return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; @@ -312,7 +301,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain) if (cookie->fq_domain) return 0; - ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all); + ret = init_iova_flush_queue(&cookie->iovad, domain); if (ret) { pr_warn("iova flush queue initialization failed\n"); return ret; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 541857ca4fd5..bbf642940988 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -63,7 +63,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->flush_cb = NULL; + iovad->fq_domain = NULL; iovad->fq = NULL; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); @@ -90,10 +90,10 @@ static void free_iova_flush_queue(struct iova_domain *iovad) free_percpu(iovad->fq); iovad->fq = NULL; - iovad->flush_cb = NULL; + iovad->fq_domain = NULL; } -int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) { struct iova_fq __percpu *queue; int cpu; @@ -105,8 +105,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) if (!queue) return -ENOMEM; - iovad->flush_cb = flush_cb; - for_each_possible_cpu(cpu) { struct iova_fq *fq; @@ -117,6 +115,7 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) spin_lock_init(&fq->lock); } + iovad->fq_domain = fq_domain; iovad->fq = queue; timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); @@ -598,7 +597,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) static void iova_domain_flush(struct iova_domain *iovad) { atomic64_inc(&iovad->fq_flush_start_cnt); - iovad->flush_cb(iovad); + iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); atomic64_inc(&iovad->fq_flush_finish_cnt); } diff --git a/include/linux/iova.h b/include/linux/iova.h index e746d8e41449..99be4fcea4f3 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -14,6 +14,7 @@ #include #include #include +#include /* iova structure */ struct iova { @@ -35,11 +36,6 @@ struct iova_rcache { struct iova_cpu_rcache __percpu *cpu_rcaches; }; -struct iova_domain; - -/* Call-Back from IOVA code into IOMMU drivers */ -typedef void (* iova_flush_cb)(struct iova_domain *domain); - /* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256 @@ -82,8 +78,7 @@ struct iova_domain { struct iova anchor; /* rbtree lookup anchor */ struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ - iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU - TLBs */ + struct iommu_domain *fq_domain; struct timer_list fq_timer; /* Timer to regularily empty the flush-queues */ @@ -147,7 +142,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb); +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else -- cgit v1.2.3-59-g8ed1b From 87f60cc65d24939353b40aa1d9297fea080cdf8d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Dec 2021 15:31:00 +0000 Subject: iommu/vt-d: Use put_pages_list page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the Intel IOMMU and IOVA code over to using free_pages_list(). Signed-off-by: Matthew Wilcox (Oracle) [rm: split from original patch, cosmetic tweaks, fix fq entries] Signed-off-by: Robin Murphy Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/2115b560d9a0ce7cd4b948bd51a2b7bde8fdfd59.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/intel/iommu.c | 89 ++++++++++++++++----------------------------- drivers/iommu/iova.c | 26 +++++-------- include/linux/iommu.h | 3 +- include/linux/iova.h | 4 +- 5 files changed, 45 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c63d93581a4e..ccacd0f71443 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -447,7 +447,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, else if (gather && gather->queued) queue_iova(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad), - gather->freelist); + &gather->freelist); else free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b6a8f3282411..17b3d97111f3 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1303,35 +1303,30 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, know the hardware page-walk will no longer touch them. The 'pte' argument is the *parent* PTE, pointing to the page that is to be freed. */ -static struct page *dma_pte_list_pagetables(struct dmar_domain *domain, - int level, struct dma_pte *pte, - struct page *freelist) +static void dma_pte_list_pagetables(struct dmar_domain *domain, + int level, struct dma_pte *pte, + struct list_head *freelist) { struct page *pg; pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT); - pg->freelist = freelist; - freelist = pg; + list_add_tail(&pg->lru, freelist); if (level == 1) - return freelist; + return; pte = page_address(pg); do { if (dma_pte_present(pte) && !dma_pte_superpage(pte)) - freelist = dma_pte_list_pagetables(domain, level - 1, - pte, freelist); + dma_pte_list_pagetables(domain, level - 1, pte, freelist); pte++; } while (!first_pte_in_page(pte)); - - return freelist; } -static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, - struct dma_pte *pte, unsigned long pfn, - unsigned long start_pfn, - unsigned long last_pfn, - struct page *freelist) +static void dma_pte_clear_level(struct dmar_domain *domain, int level, + struct dma_pte *pte, unsigned long pfn, + unsigned long start_pfn, unsigned long last_pfn, + struct list_head *freelist) { struct dma_pte *first_pte = NULL, *last_pte = NULL; @@ -1350,7 +1345,7 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, /* These suborbinate page tables are going away entirely. Don't bother to clear them; we're just going to *free* them. */ if (level > 1 && !dma_pte_superpage(pte)) - freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist); + dma_pte_list_pagetables(domain, level - 1, pte, freelist); dma_clear_pte(pte); if (!first_pte) @@ -1358,10 +1353,10 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, last_pte = pte; } else if (level > 1) { /* Recurse down into a level that isn't *entirely* obsolete */ - freelist = dma_pte_clear_level(domain, level - 1, - phys_to_virt(dma_pte_addr(pte)), - level_pfn, start_pfn, last_pfn, - freelist); + dma_pte_clear_level(domain, level - 1, + phys_to_virt(dma_pte_addr(pte)), + level_pfn, start_pfn, last_pfn, + freelist); } next: pfn = level_pfn + level_size(level); @@ -1370,47 +1365,28 @@ next: if (first_pte) domain_flush_cache(domain, first_pte, (void *)++last_pte - (void *)first_pte); - - return freelist; } /* We can't just free the pages because the IOMMU may still be walking the page tables, and may have cached the intermediate levels. The pages can only be freed after the IOTLB flush has been done. */ -static struct page *domain_unmap(struct dmar_domain *domain, - unsigned long start_pfn, - unsigned long last_pfn, - struct page *freelist) +static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn, + unsigned long last_pfn, struct list_head *freelist) { BUG_ON(!domain_pfn_supported(domain, start_pfn)); BUG_ON(!domain_pfn_supported(domain, last_pfn)); BUG_ON(start_pfn > last_pfn); /* we don't need lock here; nobody else touches the iova range */ - freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw), - domain->pgd, 0, start_pfn, last_pfn, - freelist); + dma_pte_clear_level(domain, agaw_to_level(domain->agaw), + domain->pgd, 0, start_pfn, last_pfn, freelist); /* free pgd */ if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { struct page *pgd_page = virt_to_page(domain->pgd); - pgd_page->freelist = freelist; - freelist = pgd_page; - + list_add_tail(&pgd_page->lru, freelist); domain->pgd = NULL; } - - return freelist; -} - -static void dma_free_pagelist(struct page *freelist) -{ - struct page *pg; - - while ((pg = freelist)) { - freelist = pg->freelist; - free_pgtable_page(page_address(pg)); - } } /* iommu handling */ @@ -2095,11 +2071,10 @@ static void domain_exit(struct dmar_domain *domain) domain_remove_dev_info(domain); if (domain->pgd) { - struct page *freelist; + LIST_HEAD(freelist); - freelist = domain_unmap(domain, 0, - DOMAIN_MAX_PFN(domain->gaw), NULL); - dma_free_pagelist(freelist); + domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist); + put_pages_list(&freelist); } free_domain_mem(domain); @@ -4192,19 +4167,17 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; - struct page *freelist; + LIST_HEAD(freelist); - freelist = domain_unmap(si_domain, - start_vpfn, last_vpfn, - NULL); + domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist); rcu_read_lock(); for_each_active_iommu(iommu, drhd) iommu_flush_iotlb_psi(iommu, si_domain, start_vpfn, mhp->nr_pages, - !freelist, 0); + list_empty(&freelist), 0); rcu_read_unlock(); - dma_free_pagelist(freelist); + put_pages_list(&freelist); } break; } @@ -5211,8 +5184,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, start_pfn = iova >> VTD_PAGE_SHIFT; last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT; - gather->freelist = domain_unmap(dmar_domain, start_pfn, - last_pfn, gather->freelist); + domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; @@ -5248,9 +5220,10 @@ static void intel_iommu_tlb_sync(struct iommu_domain *domain, for_each_domain_iommu(iommu_id, dmar_domain) iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain, - start_pfn, nrpages, !gather->freelist, 0); + start_pfn, nrpages, + list_empty(&gather->freelist), 0); - dma_free_pagelist(gather->freelist); + put_pages_list(&gather->freelist); } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index bbf642940988..962614c7afbb 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -96,7 +96,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad) int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) { struct iova_fq __percpu *queue; - int cpu; + int i, cpu; atomic64_set(&iovad->fq_flush_start_cnt, 0); atomic64_set(&iovad->fq_flush_finish_cnt, 0); @@ -113,6 +113,9 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom fq->tail = 0; spin_lock_init(&fq->lock); + + for (i = 0; i < IOVA_FQ_SIZE; i++) + INIT_LIST_HEAD(&fq->entries[i].freelist); } iovad->fq_domain = fq_domain; @@ -543,16 +546,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) } EXPORT_SYMBOL_GPL(free_iova_fast); -static void fq_entry_dtor(struct page *freelist) -{ - while (freelist) { - unsigned long p = (unsigned long)page_address(freelist); - - freelist = freelist->freelist; - free_page(p); - } -} - #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -585,7 +578,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) if (fq->entries[idx].counter >= counter) break; - fq_entry_dtor(fq->entries[idx].freelist); + put_pages_list(&fq->entries[idx].freelist); free_iova_fast(iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -607,15 +600,14 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) /* * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just call the entry_dtor on all remaining - * entries. + * bother to free iovas, just free any remaining pagetable pages. */ for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); int idx; fq_ring_for_each(idx, fq) - fq_entry_dtor(fq->entries[idx].freelist); + put_pages_list(&fq->entries[idx].freelist); } } @@ -640,7 +632,7 @@ static void fq_flush_timeout(struct timer_list *t) void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - struct page *freelist) + struct list_head *freelist) { struct iova_fq *fq; unsigned long flags; @@ -674,8 +666,8 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].freelist = freelist; fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); + list_splice(freelist, &fq->entries[idx].freelist); spin_unlock_irqrestore(&fq->lock, flags); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d2f3435e7d17..de0c57a567c8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -186,7 +186,7 @@ struct iommu_iotlb_gather { unsigned long start; unsigned long end; size_t pgsize; - struct page *freelist; + struct list_head freelist; bool queued; }; @@ -399,6 +399,7 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather) { .start = ULONG_MAX, + .freelist = LIST_HEAD_INIT(gather->freelist), }; } diff --git a/include/linux/iova.h b/include/linux/iova.h index 99be4fcea4f3..072a09c06e8a 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -46,7 +46,7 @@ struct iova_rcache { struct iova_fq_entry { unsigned long iova_pfn; unsigned long pages; - struct page *freelist; + struct list_head freelist; u64 counter; /* Flush counter when this entrie was added */ }; @@ -135,7 +135,7 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - struct page *freelist); + struct list_head *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, -- cgit v1.2.3-59-g8ed1b From a17e3026bc4da9135ca9a42ec0b1fa67f95172e3 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:31:03 +0000 Subject: iommu: Move flush queue data into iommu_dma_cookie Complete the move into iommu-dma by refactoring the flush queues themselves to belong to the DMA cookie rather than the IOVA domain. The refactoring may as well extend to some minor cosmetic aspects too, to help us stay one step ahead of the style police. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/24304722005bc6f144e2a1fdd865d1465722fc2e.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 171 +++++++++++++++++++++++++--------------------- drivers/iommu/iova.c | 2 - include/linux/iova.h | 44 +----------- 3 files changed, 95 insertions(+), 122 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 96057bec4164..d85d54f2b549 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -9,9 +9,12 @@ */ #include +#include +#include #include -#include +#include #include +#include #include #include #include @@ -20,11 +23,10 @@ #include #include #include -#include #include +#include +#include #include -#include -#include struct iommu_dma_msi_page { struct list_head list; @@ -41,7 +43,19 @@ struct iommu_dma_cookie { enum iommu_dma_cookie_type type; union { /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ - struct iova_domain iovad; + struct { + struct iova_domain iovad; + + struct iova_fq __percpu *fq; /* Flush queue */ + /* Number of TLB flushes that have been started */ + atomic64_t fq_flush_start_cnt; + /* Number of TLB flushes that have been finished */ + atomic64_t fq_flush_finish_cnt; + /* Timer to regularily empty the flush queues */ + struct timer_list fq_timer; + /* 1 when timer is active, 0 when not */ + atomic_t fq_timer_on; + }; /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ dma_addr_t msi_iova; }; @@ -64,6 +78,27 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); +/* Number of entries per flush queue */ +#define IOVA_FQ_SIZE 256 + +/* Timeout (in ms) after which entries are flushed from the queue */ +#define IOVA_FQ_TIMEOUT 10 + +/* Flush queue entry for deferred flushing */ +struct iova_fq_entry { + unsigned long iova_pfn; + unsigned long pages; + struct list_head freelist; + u64 counter; /* Flush counter when this entry was added */ +}; + +/* Per-CPU flush queue structure */ +struct iova_fq { + struct iova_fq_entry entries[IOVA_FQ_SIZE]; + unsigned int head, tail; + spinlock_t lock; +}; + #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -73,9 +108,9 @@ static inline bool fq_full(struct iova_fq *fq) return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head); } -static inline unsigned fq_ring_add(struct iova_fq *fq) +static inline unsigned int fq_ring_add(struct iova_fq *fq) { - unsigned idx = fq->tail; + unsigned int idx = fq->tail; assert_spin_locked(&fq->lock); @@ -84,10 +119,10 @@ static inline unsigned fq_ring_add(struct iova_fq *fq) return idx; } -static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) +static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq) { - u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt); - unsigned idx; + u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt); + unsigned int idx; assert_spin_locked(&fq->lock); @@ -97,7 +132,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) break; put_pages_list(&fq->entries[idx].freelist); - free_iova_fast(iovad, + free_iova_fast(&cookie->iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -105,50 +140,50 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) } } -static void iova_domain_flush(struct iova_domain *iovad) +static void fq_flush_iotlb(struct iommu_dma_cookie *cookie) { - atomic64_inc(&iovad->fq_flush_start_cnt); - iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); - atomic64_inc(&iovad->fq_flush_finish_cnt); + atomic64_inc(&cookie->fq_flush_start_cnt); + cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain); + atomic64_inc(&cookie->fq_flush_finish_cnt); } static void fq_flush_timeout(struct timer_list *t) { - struct iova_domain *iovad = from_timer(iovad, t, fq_timer); + struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer); int cpu; - atomic_set(&iovad->fq_timer_on, 0); - iova_domain_flush(iovad); + atomic_set(&cookie->fq_timer_on, 0); + fq_flush_iotlb(cookie); for_each_possible_cpu(cpu) { unsigned long flags; struct iova_fq *fq; - fq = per_cpu_ptr(iovad->fq, cpu); + fq = per_cpu_ptr(cookie->fq, cpu); spin_lock_irqsave(&fq->lock, flags); - fq_ring_free(iovad, fq); + fq_ring_free(cookie, fq); spin_unlock_irqrestore(&fq->lock, flags); } } -void queue_iova(struct iova_domain *iovad, +static void queue_iova(struct iommu_dma_cookie *cookie, unsigned long pfn, unsigned long pages, struct list_head *freelist) { struct iova_fq *fq; unsigned long flags; - unsigned idx; + unsigned int idx; /* * Order against the IOMMU driver's pagetable update from unmapping - * @pte, to guarantee that iova_domain_flush() observes that if called + * @pte, to guarantee that fq_flush_iotlb() observes that if called * from a different CPU before we release the lock below. Full barrier * so it also pairs with iommu_dma_init_fq() to avoid seeing partially * written fq state here. */ smp_mb(); - fq = raw_cpu_ptr(iovad->fq); + fq = raw_cpu_ptr(cookie->fq); spin_lock_irqsave(&fq->lock, flags); /* @@ -156,65 +191,66 @@ void queue_iova(struct iova_domain *iovad, * flushed out on another CPU. This makes the fq_full() check below less * likely to be true. */ - fq_ring_free(iovad, fq); + fq_ring_free(cookie, fq); if (fq_full(fq)) { - iova_domain_flush(iovad); - fq_ring_free(iovad, fq); + fq_flush_iotlb(cookie); + fq_ring_free(cookie, fq); } idx = fq_ring_add(fq); fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); + fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt); list_splice(freelist, &fq->entries[idx].freelist); spin_unlock_irqrestore(&fq->lock, flags); /* Avoid false sharing as much as possible. */ - if (!atomic_read(&iovad->fq_timer_on) && - !atomic_xchg(&iovad->fq_timer_on, 1)) - mod_timer(&iovad->fq_timer, + if (!atomic_read(&cookie->fq_timer_on) && + !atomic_xchg(&cookie->fq_timer_on, 1)) + mod_timer(&cookie->fq_timer, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); } -static void free_iova_flush_queue(struct iova_domain *iovad) +static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie) { int cpu, idx; - if (!iovad->fq) + if (!cookie->fq) return; - del_timer_sync(&iovad->fq_timer); - /* - * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just free any remaining pagetable pages. - */ + del_timer_sync(&cookie->fq_timer); + /* The IOVAs will be torn down separately, so just free our queued pages */ for_each_possible_cpu(cpu) { - struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); + struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu); fq_ring_for_each(idx, fq) put_pages_list(&fq->entries[idx].freelist); } - free_percpu(iovad->fq); - - iovad->fq = NULL; - iovad->fq_domain = NULL; + free_percpu(cookie->fq); } -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) +/* sysfs updates are serialised by the mutex of the group owning @domain */ +int iommu_dma_init_fq(struct iommu_domain *domain) { + struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_fq __percpu *queue; int i, cpu; - atomic64_set(&iovad->fq_flush_start_cnt, 0); - atomic64_set(&iovad->fq_flush_finish_cnt, 0); + if (cookie->fq_domain) + return 0; + + atomic64_set(&cookie->fq_flush_start_cnt, 0); + atomic64_set(&cookie->fq_flush_finish_cnt, 0); queue = alloc_percpu(struct iova_fq); - if (!queue) + if (!queue) { + pr_warn("iova flush queue initialization failed\n"); return -ENOMEM; + } for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(queue, cpu); @@ -228,12 +264,16 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom INIT_LIST_HEAD(&fq->entries[i].freelist); } - iovad->fq_domain = fq_domain; - iovad->fq = queue; - - timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); - atomic_set(&iovad->fq_timer_on, 0); + cookie->fq = queue; + timer_setup(&cookie->fq_timer, fq_flush_timeout, 0); + atomic_set(&cookie->fq_timer_on, 0); + /* + * Prevent incomplete fq state being observable. Pairs with path from + * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() + */ + smp_wmb(); + WRITE_ONCE(cookie->fq_domain, domain); return 0; } @@ -318,7 +358,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain) return; if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { - free_iova_flush_queue(&cookie->iovad); + iommu_dma_free_fq(cookie); put_iova_domain(&cookie->iovad); } @@ -467,29 +507,6 @@ static bool dev_use_swiotlb(struct device *dev) return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev); } -/* sysfs updates are serialised by the mutex of the group owning @domain */ -int iommu_dma_init_fq(struct iommu_domain *domain) -{ - struct iommu_dma_cookie *cookie = domain->iova_cookie; - int ret; - - if (cookie->fq_domain) - return 0; - - ret = init_iova_flush_queue(&cookie->iovad, domain); - if (ret) { - pr_warn("iova flush queue initialization failed\n"); - return ret; - } - /* - * Prevent incomplete iovad->fq being observable. Pairs with path from - * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() - */ - smp_wmb(); - WRITE_ONCE(cookie->fq_domain, domain); - return 0; -} - /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() @@ -620,7 +637,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, if (cookie->type == IOMMU_DMA_MSI_COOKIE) cookie->msi_iova -= size; else if (gather && gather->queued) - queue_iova(iovad, iova_pfn(iovad, iova), + queue_iova(cookie, iova_pfn(iovad, iova), size >> iova_shift(iovad), &gather->freelist); else diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 081e5c0cf940..b28c9435b898 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -61,8 +61,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->fq_domain = NULL; - iovad->fq = NULL; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); rb_insert_color(&iovad->anchor.node, &iovad->rbroot); diff --git a/include/linux/iova.h b/include/linux/iova.h index 072a09c06e8a..0abd48c5e622 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -12,9 +12,6 @@ #include #include #include -#include -#include -#include /* iova structure */ struct iova { @@ -36,27 +33,6 @@ struct iova_rcache { struct iova_cpu_rcache __percpu *cpu_rcaches; }; -/* Number of entries per Flush Queue */ -#define IOVA_FQ_SIZE 256 - -/* Timeout (in ms) after which entries are flushed from the Flush-Queue */ -#define IOVA_FQ_TIMEOUT 10 - -/* Flush Queue entry for defered flushing */ -struct iova_fq_entry { - unsigned long iova_pfn; - unsigned long pages; - struct list_head freelist; - u64 counter; /* Flush counter when this entrie was added */ -}; - -/* Per-CPU Flush Queue structure */ -struct iova_fq { - struct iova_fq_entry entries[IOVA_FQ_SIZE]; - unsigned head, tail; - spinlock_t lock; -}; - /* holds all the iova translations for a domain */ struct iova_domain { spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */ @@ -67,23 +43,9 @@ struct iova_domain { unsigned long start_pfn; /* Lower limit for this domain */ unsigned long dma_32bit_pfn; unsigned long max32_alloc_size; /* Size of last failed allocation */ - struct iova_fq __percpu *fq; /* Flush Queue */ - - atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that - have been started */ - - atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that - have been finished */ - struct iova anchor; /* rbtree lookup anchor */ - struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ - - struct iommu_domain *fq_domain; - struct timer_list fq_timer; /* Timer to regularily empty the - flush-queues */ - atomic_t fq_timer_on; /* 1 when timer is active, 0 - when not */ + struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ struct hlist_node cpuhp_dead; }; @@ -133,16 +95,12 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, bool size_aligned); void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); -void queue_iova(struct iova_domain *iovad, - unsigned long pfn, unsigned long pages, - struct list_head *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else -- cgit v1.2.3-59-g8ed1b From 5bc9a9dd75351023793d8aa4116ead005d659729 Mon Sep 17 00:00:00 2001 From: Emmanuel Grumbach Date: Sun, 19 Dec 2021 21:51:24 +0200 Subject: rfkill: allow to get the software rfkill state iwlwifi needs to be able to differentiate between the software rfkill state and the hardware rfkill state. The reason for this is that iwlwifi needs to notify any change in the software rfkill state even when it doesn't own the device (which means even when the hardware rfkill is asserted). In order to be able to know the software rfkill when the host does not own the device, iwlwifi needs to be able to ask the state of the software rfkill ignoring the state of the hardware rfkill. Signed-off-by: Emmanuel Grumbach Link: https://lore.kernel.org/r/20211219195124.125689-1-emmanuel.grumbach@intel.com Signed-off-by: Johannes Berg --- include/linux/rfkill.h | 7 +++++++ net/rfkill/core.c | 12 ++++++++++++ 2 files changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rfkill.h b/include/linux/rfkill.h index 231e06b74b50..c35f3962dc4f 100644 --- a/include/linux/rfkill.h +++ b/include/linux/rfkill.h @@ -229,6 +229,13 @@ void rfkill_set_states(struct rfkill *rfkill, bool sw, bool hw); */ bool rfkill_blocked(struct rfkill *rfkill); +/** + * rfkill_soft_blocked - Query soft rfkill block state + * + * @rfkill: rfkill struct to query + */ +bool rfkill_soft_blocked(struct rfkill *rfkill); + /** * rfkill_find_type - Helper for finding rfkill type by name * @name: the name of the type diff --git a/net/rfkill/core.c b/net/rfkill/core.c index ac15a944573f..5b1927d66f0d 100644 --- a/net/rfkill/core.c +++ b/net/rfkill/core.c @@ -946,6 +946,18 @@ bool rfkill_blocked(struct rfkill *rfkill) } EXPORT_SYMBOL(rfkill_blocked); +bool rfkill_soft_blocked(struct rfkill *rfkill) +{ + unsigned long flags; + u32 state; + + spin_lock_irqsave(&rfkill->lock, flags); + state = rfkill->state; + spin_unlock_irqrestore(&rfkill->lock, flags); + + return !!(state & RFKILL_BLOCK_SW); +} +EXPORT_SYMBOL(rfkill_soft_blocked); struct rfkill * __must_check rfkill_alloc(const char *name, struct device *parent, -- cgit v1.2.3-59-g8ed1b From aade40b62745cf0b4e8a17d43652c5faff354e6b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 20 Dec 2021 13:34:48 +0100 Subject: iommu/iova: Temporarily include dma-mapping.h from iova.h Some users of iova.h still expect that dma-mapping.h is also included. Re-add the include until these users are updated to fix compile failures in the iommu tree. Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20211220123448.19996-1-joro@8bytes.org Signed-off-by: Joerg Roedel --- include/linux/iova.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/iova.h b/include/linux/iova.h index 0abd48c5e622..cea79cb9f26c 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -12,6 +12,7 @@ #include #include #include +#include /* iova structure */ struct iova { -- cgit v1.2.3-59-g8ed1b From 59f37b7370ef56e6faf25d0e18bc597a0af40bb8 Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Sat, 4 Dec 2021 21:57:55 +0200 Subject: tty: serial: samsung: Remove USI initialization USI control is now extracted to the dedicated USI driver. Remove USI related code from serial driver to avoid conflicts and code duplication. Signed-off-by: Sam Protsenko Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20211204195757.8600-4-semen.protsenko@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/samsung_tty.c | 36 ++++-------------------------------- include/linux/serial_s3c.h | 9 --------- 2 files changed, 4 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index ca084c10d0bb..f986a9253dc8 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -65,7 +65,6 @@ enum s3c24xx_port_type { struct s3c24xx_uart_info { char *name; enum s3c24xx_port_type type; - bool has_usi; unsigned int port_type; unsigned int fifosize; unsigned long rx_fifomask; @@ -1357,28 +1356,6 @@ static int apple_s5l_serial_startup(struct uart_port *port) return ret; } -static void exynos_usi_init(struct uart_port *port) -{ - struct s3c24xx_uart_port *ourport = to_ourport(port); - struct s3c24xx_uart_info *info = ourport->info; - unsigned int val; - - if (!info->has_usi) - return; - - /* Clear the software reset of USI block (it's set at startup) */ - val = rd_regl(port, USI_CON); - val &= ~USI_CON_RESET_MASK; - wr_regl(port, USI_CON, val); - udelay(1); - - /* Continuously provide the clock to USI IP w/o gating (for Rx mode) */ - val = rd_regl(port, USI_OPTION); - val &= ~USI_OPTION_HWACG_MASK; - val |= USI_OPTION_HWACG_CLKREQ_ON; - wr_regl(port, USI_OPTION, val); -} - /* power power management control */ static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, @@ -1405,8 +1382,6 @@ static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, if (!IS_ERR(ourport->baudclk)) clk_prepare_enable(ourport->baudclk); - - exynos_usi_init(port); break; default: dev_err(port->dev, "s3c24xx_serial: unknown pm %d\n", level); @@ -2130,8 +2105,6 @@ static int s3c24xx_serial_init_port(struct s3c24xx_uart_port *ourport, if (ret) pr_warn("uart: failed to enable baudclk\n"); - exynos_usi_init(port); - /* Keep all interrupts masked and cleared */ switch (ourport->info->type) { case TYPE_S3C6400: @@ -2780,11 +2753,10 @@ static struct s3c24xx_serial_drv_data s5pv210_serial_drv_data = { #endif #if defined(CONFIG_ARCH_EXYNOS) -#define EXYNOS_COMMON_SERIAL_DRV_DATA(_has_usi) \ +#define EXYNOS_COMMON_SERIAL_DRV_DATA() \ .info = &(struct s3c24xx_uart_info) { \ .name = "Samsung Exynos UART", \ .type = TYPE_S3C6400, \ - .has_usi = _has_usi, \ .port_type = PORT_S3C6400, \ .has_divslot = 1, \ .rx_fifomask = S5PV210_UFSTAT_RXMASK, \ @@ -2805,17 +2777,17 @@ static struct s3c24xx_serial_drv_data s5pv210_serial_drv_data = { } \ static struct s3c24xx_serial_drv_data exynos4210_serial_drv_data = { - EXYNOS_COMMON_SERIAL_DRV_DATA(false), + EXYNOS_COMMON_SERIAL_DRV_DATA(), .fifosize = { 256, 64, 16, 16 }, }; static struct s3c24xx_serial_drv_data exynos5433_serial_drv_data = { - EXYNOS_COMMON_SERIAL_DRV_DATA(false), + EXYNOS_COMMON_SERIAL_DRV_DATA(), .fifosize = { 64, 256, 16, 256 }, }; static struct s3c24xx_serial_drv_data exynos850_serial_drv_data = { - EXYNOS_COMMON_SERIAL_DRV_DATA(true), + EXYNOS_COMMON_SERIAL_DRV_DATA(), .fifosize = { 256, 64, 64, 64 }, }; diff --git a/include/linux/serial_s3c.h b/include/linux/serial_s3c.h index cf0de4a86640..f6c3323fc4c5 100644 --- a/include/linux/serial_s3c.h +++ b/include/linux/serial_s3c.h @@ -27,15 +27,6 @@ #define S3C2410_UERSTAT (0x14) #define S3C2410_UFSTAT (0x18) #define S3C2410_UMSTAT (0x1C) -#define USI_CON (0xC4) -#define USI_OPTION (0xC8) - -#define USI_CON_RESET (1<<0) -#define USI_CON_RESET_MASK (1<<0) - -#define USI_OPTION_HWACG_CLKREQ_ON (1<<1) -#define USI_OPTION_HWACG_CLKSTOP_ON (1<<2) -#define USI_OPTION_HWACG_MASK (3<<1) #define S3C2410_LCON_CFGMASK ((0xF<<3)|(0x3)) -- cgit v1.2.3-59-g8ed1b From 1a5e91d8375fc8369207cc0b9894a324f2bbf1d9 Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Mon, 13 Dec 2021 02:14:02 -0500 Subject: swiotlb: Add swiotlb bounce buffer remap function for HV IVM In Isolation VM with AMD SEV, bounce buffer needs to be accessed via extra address space which is above shared_gpa_boundary (E.G 39 bit address line) reported by Hyper-V CPUID ISOLATION_CONFIG. The access physical address will be original physical address + shared_gpa_boundary. The shared_gpa_boundary in the AMD SEV SNP spec is called virtual top of memory(vTOM). Memory addresses below vTOM are automatically treated as private while memory above vTOM is treated as shared. Expose swiotlb_unencrypted_base for platforms to set unencrypted memory base offset and platform calls swiotlb_update_mem_attributes() to remap swiotlb mem to unencrypted address space. memremap() can not be called in the early stage and so put remapping code into swiotlb_update_mem_attributes(). Store remap address and use it to copy data from/to swiotlb bounce buffer. Signed-off-by: Tianyu Lan Acked-by: Christoph Hellwig Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20211213071407.314309-2-ltykernel@gmail.com Signed-off-by: Wei Liu --- include/linux/swiotlb.h | 6 ++++++ kernel/dma/swiotlb.c | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 569272871375..f6c3638255d5 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -73,6 +73,9 @@ extern enum swiotlb_force swiotlb_force; * @end: The end address of the swiotlb memory pool. Used to do a quick * range check to see if the memory was in fact allocated by this * API. + * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool + * may be remapped in the memory encrypted case and store virtual + * address for bounce buffer operation. * @nslabs: The number of IO TLB blocks (in groups of 64) between @start and * @end. For default swiotlb, this is command line adjustable via * setup_io_tlb_npages. @@ -92,6 +95,7 @@ extern enum swiotlb_force swiotlb_force; struct io_tlb_mem { phys_addr_t start; phys_addr_t end; + void *vaddr; unsigned long nslabs; unsigned long used; unsigned int index; @@ -186,4 +190,6 @@ static inline bool is_swiotlb_for_alloc(struct device *dev) } #endif /* CONFIG_DMA_RESTRICTED_POOL */ +extern phys_addr_t swiotlb_unencrypted_base; + #endif /* __LINUX_SWIOTLB_H */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 8e840fbbed7c..b36c1cdd0c4f 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -50,6 +50,7 @@ #include #include +#include #include #include #include @@ -72,6 +73,8 @@ enum swiotlb_force swiotlb_force; struct io_tlb_mem io_tlb_default_mem; +phys_addr_t swiotlb_unencrypted_base; + /* * Max segment that we can provide which (if pages are contingous) will * not be bounced (unless SWIOTLB_FORCE is set). @@ -155,6 +158,27 @@ static inline unsigned long nr_slots(u64 val) return DIV_ROUND_UP(val, IO_TLB_SIZE); } +/* + * Remap swioltb memory in the unencrypted physical address space + * when swiotlb_unencrypted_base is set. (e.g. for Hyper-V AMD SEV-SNP + * Isolation VMs). + */ +static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) +{ + void *vaddr = NULL; + + if (swiotlb_unencrypted_base) { + phys_addr_t paddr = mem->start + swiotlb_unencrypted_base; + + vaddr = memremap(paddr, bytes, MEMREMAP_WB); + if (!vaddr) + pr_err("Failed to map the unencrypted memory %pa size %lx.\n", + &paddr, bytes); + } + + return vaddr; +} + /* * Early SWIOTLB allocation may be too early to allow an architecture to * perform the desired operations. This function allows the architecture to @@ -172,7 +196,12 @@ void __init swiotlb_update_mem_attributes(void) vaddr = phys_to_virt(mem->start); bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); - memset(vaddr, 0, bytes); + + mem->vaddr = swiotlb_mem_remap(mem, bytes); + if (!mem->vaddr) + mem->vaddr = vaddr; + + memset(mem->vaddr, 0, bytes); } static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, @@ -196,7 +225,17 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, mem->slots[i].orig_addr = INVALID_PHYS_ADDR; mem->slots[i].alloc_size = 0; } + + /* + * If swiotlb_unencrypted_base is set, the bounce buffer memory will + * be remapped and cleared in swiotlb_update_mem_attributes. + */ + if (swiotlb_unencrypted_base) + return; + memset(vaddr, 0, bytes); + mem->vaddr = vaddr; + return; } int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) @@ -371,7 +410,7 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size phys_addr_t orig_addr = mem->slots[index].orig_addr; size_t alloc_size = mem->slots[index].alloc_size; unsigned long pfn = PFN_DOWN(orig_addr); - unsigned char *vaddr = phys_to_virt(tlb_addr); + unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; unsigned int tlb_offset, orig_addr_offset; if (orig_addr == INVALID_PHYS_ADDR) -- cgit v1.2.3-59-g8ed1b From 743b237c3a7b0f5b44aa704aae8a1058877b6322 Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Mon, 13 Dec 2021 02:14:05 -0500 Subject: scsi: storvsc: Add Isolation VM support for storvsc driver In Isolation VM, all shared memory with host needs to mark visible to host via hvcall. vmbus_establish_gpadl() has already done it for storvsc rx/tx ring buffer. The page buffer used by vmbus_sendpacket_ mpb_desc() still needs to be handled. Use DMA API(scsi_dma_map/unmap) to map these memory during sending/receiving packet and return swiotlb bounce buffer dma address. In Isolation VM, swiotlb bounce buffer is marked to be visible to host and the swiotlb force mode is enabled. Set device's dma min align mask to HV_HYP_PAGE_SIZE - 1 in order to keep the original data offset in the bounce buffer. Signed-off-by: Tianyu Lan Reviewed-by: Long Li Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20211213071407.314309-5-ltykernel@gmail.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 4 ++++ drivers/scsi/storvsc_drv.c | 37 +++++++++++++++++++++---------------- include/linux/hyperv.h | 1 + 3 files changed, 26 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 392c1ac4f819..ae6ec503399a 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "hyperv_vmbus.h" @@ -2078,6 +2079,7 @@ struct hv_device *vmbus_device_create(const guid_t *type, return child_device_obj; } +static u64 vmbus_dma_mask = DMA_BIT_MASK(64); /* * vmbus_device_register - Register the child device */ @@ -2118,6 +2120,8 @@ int vmbus_device_register(struct hv_device *child_device_obj) } hv_debug_add_dev_dir(child_device_obj); + child_device_obj->device.dma_mask = &vmbus_dma_mask; + child_device_obj->device.dma_parms = &child_device_obj->dma_parms; return 0; err_kset_unregister: diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 20595c0ba0ae..ae293600d799 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -21,6 +21,8 @@ #include #include #include +#include + #include #include #include @@ -1336,6 +1338,7 @@ static void storvsc_on_channel_callback(void *context) continue; } request = (struct storvsc_cmd_request *)scsi_cmd_priv(scmnd); + scsi_dma_unmap(scmnd); } storvsc_on_receive(stor_device, packet, request); @@ -1749,7 +1752,6 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) struct hv_host_device *host_dev = shost_priv(host); struct hv_device *dev = host_dev->dev; struct storvsc_cmd_request *cmd_request = scsi_cmd_priv(scmnd); - int i; struct scatterlist *sgl; unsigned int sg_count; struct vmscsi_request *vm_srb; @@ -1831,10 +1833,11 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) payload_sz = sizeof(cmd_request->mpb); if (sg_count) { - unsigned int hvpgoff, hvpfns_to_add; unsigned long offset_in_hvpg = offset_in_hvpage(sgl->offset); unsigned int hvpg_count = HVPFN_UP(offset_in_hvpg + length); - u64 hvpfn; + struct scatterlist *sg; + unsigned long hvpfn, hvpfns_to_add; + int j, i = 0; if (hvpg_count > MAX_PAGE_BUFFER_COUNT) { @@ -1848,21 +1851,22 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) payload->range.len = length; payload->range.offset = offset_in_hvpg; + sg_count = scsi_dma_map(scmnd); + if (sg_count < 0) + return SCSI_MLQUEUE_DEVICE_BUSY; - for (i = 0; sgl != NULL; sgl = sg_next(sgl)) { + for_each_sg(sgl, sg, sg_count, j) { /* - * Init values for the current sgl entry. hvpgoff - * and hvpfns_to_add are in units of Hyper-V size - * pages. Handling the PAGE_SIZE != HV_HYP_PAGE_SIZE - * case also handles values of sgl->offset that are - * larger than PAGE_SIZE. Such offsets are handled - * even on other than the first sgl entry, provided - * they are a multiple of PAGE_SIZE. + * Init values for the current sgl entry. hvpfns_to_add + * is in units of Hyper-V size pages. Handling the + * PAGE_SIZE != HV_HYP_PAGE_SIZE case also handles + * values of sgl->offset that are larger than PAGE_SIZE. + * Such offsets are handled even on other than the first + * sgl entry, provided they are a multiple of PAGE_SIZE. */ - hvpgoff = HVPFN_DOWN(sgl->offset); - hvpfn = page_to_hvpfn(sg_page(sgl)) + hvpgoff; - hvpfns_to_add = HVPFN_UP(sgl->offset + sgl->length) - - hvpgoff; + hvpfn = HVPFN_DOWN(sg_dma_address(sg)); + hvpfns_to_add = HVPFN_UP(sg_dma_address(sg) + + sg_dma_len(sg)) - hvpfn; /* * Fill the next portion of the PFN array with @@ -1872,7 +1876,7 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd) * the PFN array is filled. */ while (hvpfns_to_add--) - payload->range.pfn_array[i++] = hvpfn++; + payload->range.pfn_array[i++] = hvpfn++; } } @@ -2016,6 +2020,7 @@ static int storvsc_probe(struct hv_device *device, stor_device->vmscsi_size_delta = sizeof(struct vmscsi_win8_extension); spin_lock_init(&stor_device->lock); hv_set_drvdata(device, stor_device); + dma_set_min_align_mask(&device->device, HV_HYP_PAGE_SIZE - 1); stor_device->port_number = host->host_no; ret = storvsc_connect_to_vsp(device, storvsc_ringbuffer_size, is_fc); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index b823311eac79..650a0574b746 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1261,6 +1261,7 @@ struct hv_device { struct vmbus_channel *channel; struct kset *channels_kset; + struct device_dma_parameters dma_parms; /* place holder to keep track of the dir for hv device in debugfs */ struct dentry *debug_dir; -- cgit v1.2.3-59-g8ed1b From 846da38de0e8224f2f94b885125cf1fd2d7b0d39 Mon Sep 17 00:00:00 2001 From: Tianyu Lan Date: Mon, 13 Dec 2021 02:14:06 -0500 Subject: net: netvsc: Add Isolation VM support for netvsc driver In Isolation VM, all shared memory with host needs to mark visible to host via hvcall. vmbus_establish_gpadl() has already done it for netvsc rx/tx ring buffer. The page buffer used by vmbus_sendpacket_ pagebuffer() stills need to be handled. Use DMA API to map/umap these memory during sending/receiving packet and Hyper-V swiotlb bounce buffer dma address will be returned. The swiotlb bounce buffer has been masked to be visible to host during boot up. rx/tx ring buffer is allocated via vzalloc() and they need to be mapped into unencrypted address space(above vTOM) before sharing with host and accessing. Add hv_map/unmap_memory() to map/umap rx /tx ring buffer. Signed-off-by: Tianyu Lan Reviewed-by: Haiyang Zhang Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20211213071407.314309-6-ltykernel@gmail.com Signed-off-by: Wei Liu --- arch/x86/hyperv/ivm.c | 28 ++++++++ drivers/hv/hv_common.c | 11 +++ drivers/net/hyperv/hyperv_net.h | 5 ++ drivers/net/hyperv/netvsc.c | 136 +++++++++++++++++++++++++++++++++++++- drivers/net/hyperv/netvsc_drv.c | 1 + drivers/net/hyperv/rndis_filter.c | 2 + include/asm-generic/mshyperv.h | 2 + include/linux/hyperv.h | 5 ++ 8 files changed, 187 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 69c7a57f3307..2b994117581e 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -287,3 +287,31 @@ int hv_set_mem_host_visibility(unsigned long kbuffer, int pagecount, bool visibl kfree(pfn_array); return ret; } + +/* + * hv_map_memory - map memory to extra space in the AMD SEV-SNP Isolation VM. + */ +void *hv_map_memory(void *addr, unsigned long size) +{ + unsigned long *pfns = kcalloc(size / PAGE_SIZE, + sizeof(unsigned long), GFP_KERNEL); + void *vaddr; + int i; + + if (!pfns) + return NULL; + + for (i = 0; i < size / PAGE_SIZE; i++) + pfns[i] = vmalloc_to_pfn(addr + i * PAGE_SIZE) + + (ms_hyperv.shared_gpa_boundary >> PAGE_SHIFT); + + vaddr = vmap_pfn(pfns, size / PAGE_SIZE, PAGE_KERNEL_IO); + kfree(pfns); + + return vaddr; +} + +void hv_unmap_memory(void *addr) +{ + vunmap(addr); +} diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 7be173a99f27..3c5cb1f70319 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -295,3 +295,14 @@ u64 __weak hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_s return HV_STATUS_INVALID_PARAMETER; } EXPORT_SYMBOL_GPL(hv_ghcb_hypercall); + +void __weak *hv_map_memory(void *addr, unsigned long size) +{ + return NULL; +} +EXPORT_SYMBOL_GPL(hv_map_memory); + +void __weak hv_unmap_memory(void *addr) +{ +} +EXPORT_SYMBOL_GPL(hv_unmap_memory); diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h index 315278a7cf88..cf69da0e296c 100644 --- a/drivers/net/hyperv/hyperv_net.h +++ b/drivers/net/hyperv/hyperv_net.h @@ -164,6 +164,7 @@ struct hv_netvsc_packet { u32 total_bytes; u32 send_buf_index; u32 total_data_buflen; + struct hv_dma_range *dma_range; }; #define NETVSC_HASH_KEYLEN 40 @@ -1074,6 +1075,7 @@ struct netvsc_device { /* Receive buffer allocated by us but manages by NetVSP */ void *recv_buf; + void *recv_original_buf; u32 recv_buf_size; /* allocated bytes */ struct vmbus_gpadl recv_buf_gpadl_handle; u32 recv_section_cnt; @@ -1082,6 +1084,7 @@ struct netvsc_device { /* Send buffer allocated by us */ void *send_buf; + void *send_original_buf; u32 send_buf_size; struct vmbus_gpadl send_buf_gpadl_handle; u32 send_section_cnt; @@ -1731,4 +1734,6 @@ struct rndis_message { #define RETRY_US_HI 10000 #define RETRY_MAX 2000 /* >10 sec */ +void netvsc_dma_unmap(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet); #endif /* _HYPERV_NET_H */ diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 396bc1c204e6..ea2d867121d5 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -153,8 +153,21 @@ static void free_netvsc_device(struct rcu_head *head) int i; kfree(nvdev->extension); - vfree(nvdev->recv_buf); - vfree(nvdev->send_buf); + + if (nvdev->recv_original_buf) { + hv_unmap_memory(nvdev->recv_buf); + vfree(nvdev->recv_original_buf); + } else { + vfree(nvdev->recv_buf); + } + + if (nvdev->send_original_buf) { + hv_unmap_memory(nvdev->send_buf); + vfree(nvdev->send_original_buf); + } else { + vfree(nvdev->send_buf); + } + kfree(nvdev->send_section_map); for (i = 0; i < VRSS_CHANNEL_MAX; i++) { @@ -338,6 +351,7 @@ static int netvsc_init_buf(struct hv_device *device, unsigned int buf_size; size_t map_words; int i, ret = 0; + void *vaddr; /* Get receive buffer area. */ buf_size = device_info->recv_sections * device_info->recv_section_size; @@ -373,6 +387,17 @@ static int netvsc_init_buf(struct hv_device *device, goto cleanup; } + if (hv_isolation_type_snp()) { + vaddr = hv_map_memory(net_device->recv_buf, buf_size); + if (!vaddr) { + ret = -ENOMEM; + goto cleanup; + } + + net_device->recv_original_buf = net_device->recv_buf; + net_device->recv_buf = vaddr; + } + /* Notify the NetVsp of the gpadl handle */ init_packet = &net_device->channel_init_pkt; memset(init_packet, 0, sizeof(struct nvsp_message)); @@ -476,6 +501,17 @@ static int netvsc_init_buf(struct hv_device *device, goto cleanup; } + if (hv_isolation_type_snp()) { + vaddr = hv_map_memory(net_device->send_buf, buf_size); + if (!vaddr) { + ret = -ENOMEM; + goto cleanup; + } + + net_device->send_original_buf = net_device->send_buf; + net_device->send_buf = vaddr; + } + /* Notify the NetVsp of the gpadl handle */ init_packet = &net_device->channel_init_pkt; memset(init_packet, 0, sizeof(struct nvsp_message)); @@ -766,7 +802,7 @@ static void netvsc_send_tx_complete(struct net_device *ndev, /* Notify the layer above us */ if (likely(skb)) { - const struct hv_netvsc_packet *packet + struct hv_netvsc_packet *packet = (struct hv_netvsc_packet *)skb->cb; u32 send_index = packet->send_buf_index; struct netvsc_stats *tx_stats; @@ -782,6 +818,7 @@ static void netvsc_send_tx_complete(struct net_device *ndev, tx_stats->bytes += packet->total_bytes; u64_stats_update_end(&tx_stats->syncp); + netvsc_dma_unmap(ndev_ctx->device_ctx, packet); napi_consume_skb(skb, budget); } @@ -946,6 +983,88 @@ static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, memset(dest, 0, padding); } +void netvsc_dma_unmap(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet) +{ + u32 page_count = packet->cp_partial ? + packet->page_buf_cnt - packet->rmsg_pgcnt : + packet->page_buf_cnt; + int i; + + if (!hv_is_isolation_supported()) + return; + + if (!packet->dma_range) + return; + + for (i = 0; i < page_count; i++) + dma_unmap_single(&hv_dev->device, packet->dma_range[i].dma, + packet->dma_range[i].mapping_size, + DMA_TO_DEVICE); + + kfree(packet->dma_range); +} + +/* netvsc_dma_map - Map swiotlb bounce buffer with data page of + * packet sent by vmbus_sendpacket_pagebuffer() in the Isolation + * VM. + * + * In isolation VM, netvsc send buffer has been marked visible to + * host and so the data copied to send buffer doesn't need to use + * bounce buffer. The data pages handled by vmbus_sendpacket_pagebuffer() + * may not be copied to send buffer and so these pages need to be + * mapped with swiotlb bounce buffer. netvsc_dma_map() is to do + * that. The pfns in the struct hv_page_buffer need to be converted + * to bounce buffer's pfn. The loop here is necessary because the + * entries in the page buffer array are not necessarily full + * pages of data. Each entry in the array has a separate offset and + * len that may be non-zero, even for entries in the middle of the + * array. And the entries are not physically contiguous. So each + * entry must be individually mapped rather than as a contiguous unit. + * So not use dma_map_sg() here. + */ +static int netvsc_dma_map(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet, + struct hv_page_buffer *pb) +{ + u32 page_count = packet->cp_partial ? + packet->page_buf_cnt - packet->rmsg_pgcnt : + packet->page_buf_cnt; + dma_addr_t dma; + int i; + + if (!hv_is_isolation_supported()) + return 0; + + packet->dma_range = kcalloc(page_count, + sizeof(*packet->dma_range), + GFP_KERNEL); + if (!packet->dma_range) + return -ENOMEM; + + for (i = 0; i < page_count; i++) { + char *src = phys_to_virt((pb[i].pfn << HV_HYP_PAGE_SHIFT) + + pb[i].offset); + u32 len = pb[i].len; + + dma = dma_map_single(&hv_dev->device, src, len, + DMA_TO_DEVICE); + if (dma_mapping_error(&hv_dev->device, dma)) { + kfree(packet->dma_range); + return -ENOMEM; + } + + /* pb[].offset and pb[].len are not changed during dma mapping + * and so not reassign. + */ + packet->dma_range[i].dma = dma; + packet->dma_range[i].mapping_size = len; + pb[i].pfn = dma >> HV_HYP_PAGE_SHIFT; + } + + return 0; +} + static inline int netvsc_send_pkt( struct hv_device *device, struct hv_netvsc_packet *packet, @@ -986,14 +1105,24 @@ static inline int netvsc_send_pkt( trace_nvsp_send_pkt(ndev, out_channel, rpkt); + packet->dma_range = NULL; if (packet->page_buf_cnt) { if (packet->cp_partial) pb += packet->rmsg_pgcnt; + ret = netvsc_dma_map(ndev_ctx->device_ctx, packet, pb); + if (ret) { + ret = -EAGAIN; + goto exit; + } + ret = vmbus_sendpacket_pagebuffer(out_channel, pb, packet->page_buf_cnt, &nvmsg, sizeof(nvmsg), req_id); + + if (ret) + netvsc_dma_unmap(ndev_ctx->device_ctx, packet); } else { ret = vmbus_sendpacket(out_channel, &nvmsg, sizeof(nvmsg), @@ -1001,6 +1130,7 @@ static inline int netvsc_send_pkt( VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); } +exit: if (ret == 0) { atomic_inc_return(&nvchan->queue_sends); diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 7e66ae1d2a59..17958533bf30 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -2512,6 +2512,7 @@ static int netvsc_probe(struct hv_device *dev, net->netdev_ops = &device_ops; net->ethtool_ops = ðtool_ops; SET_NETDEV_DEV(net, &dev->device); + dma_set_min_align_mask(&dev->device, HV_HYP_PAGE_SIZE - 1); /* We always need headroom for rndis header */ net->needed_headroom = RNDIS_AND_PPI_SIZE; diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index f6c9c2a670f9..448fcc325ed7 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -361,6 +361,8 @@ static void rndis_filter_receive_response(struct net_device *ndev, } } + netvsc_dma_unmap(((struct net_device_context *) + netdev_priv(ndev))->device_ctx, &request->pkt); complete(&request->wait_event); } else { netdev_err(ndev, diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index 3e2248ac328e..94e73ba129c5 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -269,6 +269,8 @@ bool hv_isolation_type_snp(void); u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size); void hyperv_cleanup(void); bool hv_query_ext_cap(u64 cap_query); +void *hv_map_memory(void *addr, unsigned long size); +void hv_unmap_memory(void *addr); #else /* CONFIG_HYPERV */ static inline bool hv_is_hyperv_initialized(void) { return false; } static inline bool hv_is_hibernation_supported(void) { return false; } diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 650a0574b746..f565a8938836 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1584,6 +1584,11 @@ struct hyperv_service_callback { void (*callback)(void *context); }; +struct hv_dma_range { + dma_addr_t dma; + u32 mapping_size; +}; + #define MAX_SRV_VER 0x7ffffff extern bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, u32 buflen, const int *fw_version, int fw_vercnt, -- cgit v1.2.3-59-g8ed1b From 6fc61c39ee1adb5f4115d288c876772fcd8b6979 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Sun, 21 Nov 2021 01:20:46 +0100 Subject: soc: qcom: llcc: Add configuration data for SM8350 Add LLCC configuration data for SM8350 SoC. Signed-off-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20211121002050.36977-2-konrad.dybcio@somainline.org --- drivers/soc/qcom/llcc-qcom.c | 28 ++++++++++++++++++++++++++++ include/linux/soc/qcom/llcc-qcom.h | 3 +++ 2 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index 6bf2f1d1f2c5..ec52f29c8867 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -195,6 +195,28 @@ static const struct llcc_slice_config sm8250_data[] = { { LLCC_WRCACHE, 31, 256, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, }; +static const struct llcc_slice_config sm8350_data[] = { + { LLCC_CPUSS, 1, 3072, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 1 }, + { LLCC_VIDSC0, 2, 512, 3, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_AUDIO, 6, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 0, 0 }, + { LLCC_MDMHPGRW, 7, 1024, 3, 0, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_MODHW, 9, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_CMPT, 10, 3072, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_GPUHTW, 11, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_GPU, 12, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 1, 1, 0 }, + { LLCC_MMUHWT, 13, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 0, 1 }, + { LLCC_DISP, 16, 3072, 2, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_MDMPNG, 21, 1024, 0, 1, 0xf, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_AUDHW, 22, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_CVP, 28, 512, 3, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_MODPE, 29, 256, 1, 1, 0xf, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_APTCM, 30, 1024, 3, 1, 0x0, 0x1, 1, 0, 0, 0, 1, 0 }, + { LLCC_WRCACHE, 31, 512, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 0, 1 }, + { LLCC_CVPFW, 17, 512, 1, 0, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_CPUSS1, 3, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_CPUHWT, 5, 512, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 0, 1 }, +}; + static const struct qcom_llcc_config sc7180_cfg = { .sct_data = sc7180_data, .size = ARRAY_SIZE(sc7180_data), @@ -228,6 +250,11 @@ static const struct qcom_llcc_config sm8250_cfg = { .size = ARRAY_SIZE(sm8250_data), }; +static const struct qcom_llcc_config sm8350_cfg = { + .sct_data = sm8350_data, + .size = ARRAY_SIZE(sm8350_data), +}; + static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER; /** @@ -644,6 +671,7 @@ static const struct of_device_id qcom_llcc_of_match[] = { { .compatible = "qcom,sm6350-llcc", .data = &sm6350_cfg }, { .compatible = "qcom,sm8150-llcc", .data = &sm8150_cfg }, { .compatible = "qcom,sm8250-llcc", .data = &sm8250_cfg }, + { .compatible = "qcom,sm8350-llcc", .data = &sm8350_cfg }, { } }; diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index 437c9df13229..9e8fd92c96b7 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -33,6 +33,9 @@ #define LLCC_MODPE 29 #define LLCC_APTCM 30 #define LLCC_WRCACHE 31 +#define LLCC_CVPFW 32 +#define LLCC_CPUSS1 33 +#define LLCC_CPUHWT 36 /** * struct llcc_slice_desc - Cache slice descriptor -- cgit v1.2.3-59-g8ed1b From 80a5ca99c5c04be6777df225ab932142a9d60c3f Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 16 Dec 2021 11:33:00 +0800 Subject: rapidio: remove not used macro definition in rio_ids.h The definition of RIO_VID_FREESCALE, RIO_DID_MPC8560, RIO_DID_TSI500, RIO_DID_TSI576 and RIO_DID_TSI721 are not used for many years in the current code, so just remove them. Signed-off-by: Tiezhu Yang Link: https://lore.kernel.org/r/1639625581-22867-2-git-send-email-yangtiezhu@loongson.cn Signed-off-by: Greg Kroah-Hartman --- include/linux/rio_ids.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rio_ids.h b/include/linux/rio_ids.h index 4846f72759b2..e74d8840708a 100644 --- a/include/linux/rio_ids.h +++ b/include/linux/rio_ids.h @@ -9,15 +9,10 @@ #ifndef LINUX_RIO_IDS_H #define LINUX_RIO_IDS_H -#define RIO_VID_FREESCALE 0x0002 -#define RIO_DID_MPC8560 0x0003 - #define RIO_VID_TUNDRA 0x000d -#define RIO_DID_TSI500 0x0500 #define RIO_DID_TSI568 0x0568 #define RIO_DID_TSI572 0x0572 #define RIO_DID_TSI574 0x0574 -#define RIO_DID_TSI576 0x0578 /* Same ID as Tsi578 */ #define RIO_DID_TSI577 0x0577 #define RIO_DID_TSI578 0x0578 @@ -33,7 +28,6 @@ #define RIO_DID_IDTCPS1616 0x0379 #define RIO_DID_IDTVPS1616 0x0377 #define RIO_DID_IDTSPS1616 0x0378 -#define RIO_DID_TSI721 0x80ab #define RIO_DID_IDTRXS1632 0x80e5 #define RIO_DID_IDTRXS2448 0x80e6 -- cgit v1.2.3-59-g8ed1b From 612d4904191ff9aca01b1e087d8687b3a223cb33 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 16 Dec 2021 11:33:01 +0800 Subject: rapidio: remove not used code about RIO_VID_TUNDRA According to https://rapidio.org/vendor-id/, there is no 0x000d vendor id in the complete and current list of VendorIDs, it means that the related code is dead code now, so just remove them. Signed-off-by: Tiezhu Yang Link: https://lore.kernel.org/r/1639625581-22867-3-git-send-email-yangtiezhu@loongson.cn Signed-off-by: Greg Kroah-Hartman --- drivers/rapidio/switches/Kconfig | 11 -- drivers/rapidio/switches/Makefile | 2 - drivers/rapidio/switches/tsi568.c | 195 -------------------- drivers/rapidio/switches/tsi57x.c | 365 -------------------------------------- include/linux/rio_ids.h | 7 - 5 files changed, 580 deletions(-) delete mode 100644 drivers/rapidio/switches/tsi568.c delete mode 100644 drivers/rapidio/switches/tsi57x.c (limited to 'include/linux') diff --git a/drivers/rapidio/switches/Kconfig b/drivers/rapidio/switches/Kconfig index 3e18f9c51e29..02771ba3e54f 100644 --- a/drivers/rapidio/switches/Kconfig +++ b/drivers/rapidio/switches/Kconfig @@ -2,22 +2,11 @@ # # RapidIO switches configuration # -config RAPIDIO_TSI57X - tristate "IDT Tsi57x SRIO switches support" - help - Includes support for IDT Tsi57x family of serial RapidIO switches. - config RAPIDIO_CPS_XX tristate "IDT CPS-xx SRIO switches support" help Includes support for IDT CPS-16/12/10/8 serial RapidIO switches. -config RAPIDIO_TSI568 - tristate "Tsi568 SRIO switch support" - default n - help - Includes support for IDT Tsi568 serial RapidIO switch. - config RAPIDIO_CPS_GEN2 tristate "IDT CPS Gen.2 SRIO switch support" default n diff --git a/drivers/rapidio/switches/Makefile b/drivers/rapidio/switches/Makefile index 69e7de31e41c..ef1749a79c2b 100644 --- a/drivers/rapidio/switches/Makefile +++ b/drivers/rapidio/switches/Makefile @@ -3,8 +3,6 @@ # Makefile for RIO switches # -obj-$(CONFIG_RAPIDIO_TSI57X) += tsi57x.o obj-$(CONFIG_RAPIDIO_CPS_XX) += idtcps.o -obj-$(CONFIG_RAPIDIO_TSI568) += tsi568.o obj-$(CONFIG_RAPIDIO_CPS_GEN2) += idt_gen2.o obj-$(CONFIG_RAPIDIO_RXS_GEN3) += idt_gen3.o diff --git a/drivers/rapidio/switches/tsi568.c b/drivers/rapidio/switches/tsi568.c deleted file mode 100644 index 103b48a24980..000000000000 --- a/drivers/rapidio/switches/tsi568.c +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * RapidIO Tsi568 switch support - * - * Copyright 2009-2010 Integrated Device Technology, Inc. - * Alexandre Bounine - * - Added EM support - * - Modified switch operations initialization. - * - * Copyright 2005 MontaVista Software, Inc. - * Matt Porter - */ - -#include -#include -#include -#include -#include -#include "../rio.h" - -/* Global (broadcast) route registers */ -#define SPBC_ROUTE_CFG_DESTID 0x10070 -#define SPBC_ROUTE_CFG_PORT 0x10074 - -/* Per port route registers */ -#define SPP_ROUTE_CFG_DESTID(n) (0x11070 + 0x100*n) -#define SPP_ROUTE_CFG_PORT(n) (0x11074 + 0x100*n) - -#define TSI568_SP_MODE(n) (0x11004 + 0x100*n) -#define TSI568_SP_MODE_PW_DIS 0x08000000 - -static int -tsi568_route_add_entry(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table, u16 route_destid, u8 route_port) -{ - if (table == RIO_GLOBAL_TABLE) { - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_DESTID, route_destid); - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_PORT, route_port); - } else { - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), - route_destid); - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table), route_port); - } - - udelay(10); - - return 0; -} - -static int -tsi568_route_get_entry(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table, u16 route_destid, u8 *route_port) -{ - int ret = 0; - u32 result; - - if (table == RIO_GLOBAL_TABLE) { - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_DESTID, route_destid); - rio_mport_read_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_PORT, &result); - } else { - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), - route_destid); - rio_mport_read_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table), &result); - } - - *route_port = result; - if (*route_port > 15) - ret = -1; - - return ret; -} - -static int -tsi568_route_clr_table(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table) -{ - u32 route_idx; - u32 lut_size; - - lut_size = (mport->sys_size) ? 0x1ff : 0xff; - - if (table == RIO_GLOBAL_TABLE) { - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_DESTID, 0x80000000); - for (route_idx = 0; route_idx <= lut_size; route_idx++) - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_PORT, - RIO_INVALID_ROUTE); - } else { - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), - 0x80000000); - for (route_idx = 0; route_idx <= lut_size; route_idx++) - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table), - RIO_INVALID_ROUTE); - } - - return 0; -} - -static int -tsi568_em_init(struct rio_dev *rdev) -{ - u32 regval; - int portnum; - - pr_debug("TSI568 %s [%d:%d]\n", __func__, rdev->destid, rdev->hopcount); - - /* Make sure that Port-Writes are disabled (for all ports) */ - for (portnum = 0; - portnum < RIO_GET_TOTAL_PORTS(rdev->swpinfo); portnum++) { - rio_read_config_32(rdev, TSI568_SP_MODE(portnum), ®val); - rio_write_config_32(rdev, TSI568_SP_MODE(portnum), - regval | TSI568_SP_MODE_PW_DIS); - } - - return 0; -} - -static struct rio_switch_ops tsi568_switch_ops = { - .owner = THIS_MODULE, - .add_entry = tsi568_route_add_entry, - .get_entry = tsi568_route_get_entry, - .clr_table = tsi568_route_clr_table, - .set_domain = NULL, - .get_domain = NULL, - .em_init = tsi568_em_init, - .em_handle = NULL, -}; - -static int tsi568_probe(struct rio_dev *rdev, const struct rio_device_id *id) -{ - pr_debug("RIO: %s for %s\n", __func__, rio_name(rdev)); - - spin_lock(&rdev->rswitch->lock); - - if (rdev->rswitch->ops) { - spin_unlock(&rdev->rswitch->lock); - return -EINVAL; - } - - rdev->rswitch->ops = &tsi568_switch_ops; - spin_unlock(&rdev->rswitch->lock); - return 0; -} - -static void tsi568_remove(struct rio_dev *rdev) -{ - pr_debug("RIO: %s for %s\n", __func__, rio_name(rdev)); - spin_lock(&rdev->rswitch->lock); - if (rdev->rswitch->ops != &tsi568_switch_ops) { - spin_unlock(&rdev->rswitch->lock); - return; - } - rdev->rswitch->ops = NULL; - spin_unlock(&rdev->rswitch->lock); -} - -static const struct rio_device_id tsi568_id_table[] = { - {RIO_DEVICE(RIO_DID_TSI568, RIO_VID_TUNDRA)}, - { 0, } /* terminate list */ -}; - -static struct rio_driver tsi568_driver = { - .name = "tsi568", - .id_table = tsi568_id_table, - .probe = tsi568_probe, - .remove = tsi568_remove, -}; - -static int __init tsi568_init(void) -{ - return rio_register_driver(&tsi568_driver); -} - -static void __exit tsi568_exit(void) -{ - rio_unregister_driver(&tsi568_driver); -} - -device_initcall(tsi568_init); -module_exit(tsi568_exit); - -MODULE_DESCRIPTION("IDT Tsi568 Serial RapidIO switch driver"); -MODULE_AUTHOR("Integrated Device Technology, Inc."); -MODULE_LICENSE("GPL"); diff --git a/drivers/rapidio/switches/tsi57x.c b/drivers/rapidio/switches/tsi57x.c deleted file mode 100644 index 271762046f8c..000000000000 --- a/drivers/rapidio/switches/tsi57x.c +++ /dev/null @@ -1,365 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * RapidIO Tsi57x switch family support - * - * Copyright 2009-2010 Integrated Device Technology, Inc. - * Alexandre Bounine - * - Added EM support - * - Modified switch operations initialization. - * - * Copyright 2005 MontaVista Software, Inc. - * Matt Porter - */ - -#include -#include -#include -#include -#include -#include "../rio.h" - -/* Global (broadcast) route registers */ -#define SPBC_ROUTE_CFG_DESTID 0x10070 -#define SPBC_ROUTE_CFG_PORT 0x10074 - -/* Per port route registers */ -#define SPP_ROUTE_CFG_DESTID(n) (0x11070 + 0x100*n) -#define SPP_ROUTE_CFG_PORT(n) (0x11074 + 0x100*n) - -#define TSI578_SP_MODE(n) (0x11004 + n*0x100) -#define TSI578_SP_MODE_GLBL 0x10004 -#define TSI578_SP_MODE_PW_DIS 0x08000000 -#define TSI578_SP_MODE_LUT_512 0x01000000 - -#define TSI578_SP_CTL_INDEP(n) (0x13004 + n*0x100) -#define TSI578_SP_LUT_PEINF(n) (0x13010 + n*0x100) -#define TSI578_SP_CS_TX(n) (0x13014 + n*0x100) -#define TSI578_SP_INT_STATUS(n) (0x13018 + n*0x100) - -#define TSI578_GLBL_ROUTE_BASE 0x10078 - -static int -tsi57x_route_add_entry(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table, u16 route_destid, u8 route_port) -{ - if (table == RIO_GLOBAL_TABLE) { - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_DESTID, route_destid); - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_PORT, route_port); - } else { - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), route_destid); - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table), route_port); - } - - udelay(10); - - return 0; -} - -static int -tsi57x_route_get_entry(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table, u16 route_destid, u8 *route_port) -{ - int ret = 0; - u32 result; - - if (table == RIO_GLOBAL_TABLE) { - /* Use local RT of the ingress port to avoid possible - race condition */ - rio_mport_read_config_32(mport, destid, hopcount, - RIO_SWP_INFO_CAR, &result); - table = (result & RIO_SWP_INFO_PORT_NUM_MASK); - } - - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), route_destid); - rio_mport_read_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table), &result); - - *route_port = (u8)result; - if (*route_port > 15) - ret = -1; - - return ret; -} - -static int -tsi57x_route_clr_table(struct rio_mport *mport, u16 destid, u8 hopcount, - u16 table) -{ - u32 route_idx; - u32 lut_size; - - lut_size = (mport->sys_size) ? 0x1ff : 0xff; - - if (table == RIO_GLOBAL_TABLE) { - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_DESTID, 0x80000000); - for (route_idx = 0; route_idx <= lut_size; route_idx++) - rio_mport_write_config_32(mport, destid, hopcount, - SPBC_ROUTE_CFG_PORT, - RIO_INVALID_ROUTE); - } else { - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_DESTID(table), 0x80000000); - for (route_idx = 0; route_idx <= lut_size; route_idx++) - rio_mport_write_config_32(mport, destid, hopcount, - SPP_ROUTE_CFG_PORT(table) , RIO_INVALID_ROUTE); - } - - return 0; -} - -static int -tsi57x_set_domain(struct rio_mport *mport, u16 destid, u8 hopcount, - u8 sw_domain) -{ - u32 regval; - - /* - * Switch domain configuration operates only at global level - */ - - /* Turn off flat (LUT_512) mode */ - rio_mport_read_config_32(mport, destid, hopcount, - TSI578_SP_MODE_GLBL, ®val); - rio_mport_write_config_32(mport, destid, hopcount, TSI578_SP_MODE_GLBL, - regval & ~TSI578_SP_MODE_LUT_512); - /* Set switch domain base */ - rio_mport_write_config_32(mport, destid, hopcount, - TSI578_GLBL_ROUTE_BASE, - (u32)(sw_domain << 24)); - return 0; -} - -static int -tsi57x_get_domain(struct rio_mport *mport, u16 destid, u8 hopcount, - u8 *sw_domain) -{ - u32 regval; - - /* - * Switch domain configuration operates only at global level - */ - rio_mport_read_config_32(mport, destid, hopcount, - TSI578_GLBL_ROUTE_BASE, ®val); - - *sw_domain = (u8)(regval >> 24); - - return 0; -} - -static int -tsi57x_em_init(struct rio_dev *rdev) -{ - u32 regval; - int portnum; - - pr_debug("TSI578 %s [%d:%d]\n", __func__, rdev->destid, rdev->hopcount); - - for (portnum = 0; - portnum < RIO_GET_TOTAL_PORTS(rdev->swpinfo); portnum++) { - /* Make sure that Port-Writes are enabled (for all ports) */ - rio_read_config_32(rdev, - TSI578_SP_MODE(portnum), ®val); - rio_write_config_32(rdev, - TSI578_SP_MODE(portnum), - regval & ~TSI578_SP_MODE_PW_DIS); - - /* Clear all pending interrupts */ - rio_read_config_32(rdev, - RIO_DEV_PORT_N_ERR_STS_CSR(rdev, portnum), - ®val); - rio_write_config_32(rdev, - RIO_DEV_PORT_N_ERR_STS_CSR(rdev, portnum), - regval & 0x07120214); - - rio_read_config_32(rdev, - TSI578_SP_INT_STATUS(portnum), ®val); - rio_write_config_32(rdev, - TSI578_SP_INT_STATUS(portnum), - regval & 0x000700bd); - - /* Enable all interrupts to allow ports to send a port-write */ - rio_read_config_32(rdev, - TSI578_SP_CTL_INDEP(portnum), ®val); - rio_write_config_32(rdev, - TSI578_SP_CTL_INDEP(portnum), - regval | 0x000b0000); - - /* Skip next (odd) port if the current port is in x4 mode */ - rio_read_config_32(rdev, - RIO_DEV_PORT_N_CTL_CSR(rdev, portnum), - ®val); - if ((regval & RIO_PORT_N_CTL_PWIDTH) == RIO_PORT_N_CTL_PWIDTH_4) - portnum++; - } - - /* set TVAL = ~50us */ - rio_write_config_32(rdev, - rdev->phys_efptr + RIO_PORT_LINKTO_CTL_CSR, 0x9a << 8); - - return 0; -} - -static int -tsi57x_em_handler(struct rio_dev *rdev, u8 portnum) -{ - struct rio_mport *mport = rdev->net->hport; - u32 intstat, err_status; - int sendcount, checkcount; - u8 route_port; - u32 regval; - - rio_read_config_32(rdev, - RIO_DEV_PORT_N_ERR_STS_CSR(rdev, portnum), - &err_status); - - if ((err_status & RIO_PORT_N_ERR_STS_PORT_OK) && - (err_status & (RIO_PORT_N_ERR_STS_OUT_ES | - RIO_PORT_N_ERR_STS_INP_ES))) { - /* Remove any queued packets by locking/unlocking port */ - rio_read_config_32(rdev, - RIO_DEV_PORT_N_CTL_CSR(rdev, portnum), - ®val); - if (!(regval & RIO_PORT_N_CTL_LOCKOUT)) { - rio_write_config_32(rdev, - RIO_DEV_PORT_N_CTL_CSR(rdev, portnum), - regval | RIO_PORT_N_CTL_LOCKOUT); - udelay(50); - rio_write_config_32(rdev, - RIO_DEV_PORT_N_CTL_CSR(rdev, portnum), - regval); - } - - /* Read from link maintenance response register to clear - * valid bit - */ - rio_read_config_32(rdev, - RIO_DEV_PORT_N_MNT_RSP_CSR(rdev, portnum), - ®val); - - /* Send a Packet-Not-Accepted/Link-Request-Input-Status control - * symbol to recover from IES/OES - */ - sendcount = 3; - while (sendcount) { - rio_write_config_32(rdev, - TSI578_SP_CS_TX(portnum), 0x40fc8000); - checkcount = 3; - while (checkcount--) { - udelay(50); - rio_read_config_32(rdev, - RIO_DEV_PORT_N_MNT_RSP_CSR(rdev, - portnum), - ®val); - if (regval & RIO_PORT_N_MNT_RSP_RVAL) - goto exit_es; - } - - sendcount--; - } - } - -exit_es: - /* Clear implementation specific error status bits */ - rio_read_config_32(rdev, TSI578_SP_INT_STATUS(portnum), &intstat); - pr_debug("TSI578[%x:%x] SP%d_INT_STATUS=0x%08x\n", - rdev->destid, rdev->hopcount, portnum, intstat); - - if (intstat & 0x10000) { - rio_read_config_32(rdev, - TSI578_SP_LUT_PEINF(portnum), ®val); - regval = (mport->sys_size) ? (regval >> 16) : (regval >> 24); - route_port = rdev->rswitch->route_table[regval]; - pr_debug("RIO: TSI578[%s] P%d LUT Parity Error (destID=%d)\n", - rio_name(rdev), portnum, regval); - tsi57x_route_add_entry(mport, rdev->destid, rdev->hopcount, - RIO_GLOBAL_TABLE, regval, route_port); - } - - rio_write_config_32(rdev, TSI578_SP_INT_STATUS(portnum), - intstat & 0x000700bd); - - return 0; -} - -static struct rio_switch_ops tsi57x_switch_ops = { - .owner = THIS_MODULE, - .add_entry = tsi57x_route_add_entry, - .get_entry = tsi57x_route_get_entry, - .clr_table = tsi57x_route_clr_table, - .set_domain = tsi57x_set_domain, - .get_domain = tsi57x_get_domain, - .em_init = tsi57x_em_init, - .em_handle = tsi57x_em_handler, -}; - -static int tsi57x_probe(struct rio_dev *rdev, const struct rio_device_id *id) -{ - pr_debug("RIO: %s for %s\n", __func__, rio_name(rdev)); - - spin_lock(&rdev->rswitch->lock); - - if (rdev->rswitch->ops) { - spin_unlock(&rdev->rswitch->lock); - return -EINVAL; - } - rdev->rswitch->ops = &tsi57x_switch_ops; - - if (rdev->do_enum) { - /* Ensure that default routing is disabled on startup */ - rio_write_config_32(rdev, RIO_STD_RTE_DEFAULT_PORT, - RIO_INVALID_ROUTE); - } - - spin_unlock(&rdev->rswitch->lock); - return 0; -} - -static void tsi57x_remove(struct rio_dev *rdev) -{ - pr_debug("RIO: %s for %s\n", __func__, rio_name(rdev)); - spin_lock(&rdev->rswitch->lock); - if (rdev->rswitch->ops != &tsi57x_switch_ops) { - spin_unlock(&rdev->rswitch->lock); - return; - } - rdev->rswitch->ops = NULL; - spin_unlock(&rdev->rswitch->lock); -} - -static const struct rio_device_id tsi57x_id_table[] = { - {RIO_DEVICE(RIO_DID_TSI572, RIO_VID_TUNDRA)}, - {RIO_DEVICE(RIO_DID_TSI574, RIO_VID_TUNDRA)}, - {RIO_DEVICE(RIO_DID_TSI577, RIO_VID_TUNDRA)}, - {RIO_DEVICE(RIO_DID_TSI578, RIO_VID_TUNDRA)}, - { 0, } /* terminate list */ -}; - -static struct rio_driver tsi57x_driver = { - .name = "tsi57x", - .id_table = tsi57x_id_table, - .probe = tsi57x_probe, - .remove = tsi57x_remove, -}; - -static int __init tsi57x_init(void) -{ - return rio_register_driver(&tsi57x_driver); -} - -static void __exit tsi57x_exit(void) -{ - rio_unregister_driver(&tsi57x_driver); -} - -device_initcall(tsi57x_init); -module_exit(tsi57x_exit); - -MODULE_DESCRIPTION("IDT Tsi57x Serial RapidIO switch family driver"); -MODULE_AUTHOR("Integrated Device Technology, Inc."); -MODULE_LICENSE("GPL"); diff --git a/include/linux/rio_ids.h b/include/linux/rio_ids.h index e74d8840708a..c7e2f21dd5c1 100644 --- a/include/linux/rio_ids.h +++ b/include/linux/rio_ids.h @@ -9,13 +9,6 @@ #ifndef LINUX_RIO_IDS_H #define LINUX_RIO_IDS_H -#define RIO_VID_TUNDRA 0x000d -#define RIO_DID_TSI568 0x0568 -#define RIO_DID_TSI572 0x0572 -#define RIO_DID_TSI574 0x0574 -#define RIO_DID_TSI577 0x0577 -#define RIO_DID_TSI578 0x0578 - #define RIO_VID_IDT 0x0038 #define RIO_DID_IDT70K200 0x0310 #define RIO_DID_IDTCPS8 0x035c -- cgit v1.2.3-59-g8ed1b From 0032ca576a79946492194ae4860b462d32815c66 Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Tue, 21 Dec 2021 17:16:46 +0900 Subject: counter: Add the necessary colons and indents to the comments of counter_compi Since commit aaec1a0f76ec ("counter: Internalize sysfs interface code") introduce a warning as: linux-next/Documentation/driver-api/generic-counter:234: ./include/linux/counter.h:43: WARNING: Unexpected indentation. linux-next/Documentation/driver-api/generic-counter:234: ./include/linux/counter.h:45: WARNING: Block quote ends without a blank line; unexpected unindent. Add the necessary colons and indents. Fixes: aaec1a0f76ec ("counter: Internalize sysfs interface code") Signed-off-by: Yanteng Si Signed-off-by: William Breathitt Gray Link: https://lore.kernel.org/r/26011e814d6eca02c7ebdbb92f171a49928a7e89.1640072891.git.vilhelm.gray@gmail.com Signed-off-by: Greg Kroah-Hartman --- include/linux/counter.h | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/counter.h b/include/linux/counter.h index b7d0a00a61cf..dfbde2808998 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -38,64 +38,64 @@ enum counter_comp_type { * @type: Counter component data type * @name: device-specific component name * @priv: component-relevant data - * @action_read Synapse action mode read callback. The read value of the + * @action_read: Synapse action mode read callback. The read value of the * respective Synapse action mode should be passed back via * the action parameter. - * @device_u8_read Device u8 component read callback. The read value of the + * @device_u8_read: Device u8 component read callback. The read value of the * respective Device u8 component should be passed back via * the val parameter. - * @count_u8_read Count u8 component read callback. The read value of the + * @count_u8_read: Count u8 component read callback. The read value of the * respective Count u8 component should be passed back via * the val parameter. - * @signal_u8_read Signal u8 component read callback. The read value of the + * @signal_u8_read: Signal u8 component read callback. The read value of the * respective Signal u8 component should be passed back via * the val parameter. - * @device_u32_read Device u32 component read callback. The read value of + * @device_u32_read: Device u32 component read callback. The read value of * the respective Device u32 component should be passed * back via the val parameter. - * @count_u32_read Count u32 component read callback. The read value of the + * @count_u32_read: Count u32 component read callback. The read value of the * respective Count u32 component should be passed back via * the val parameter. - * @signal_u32_read Signal u32 component read callback. The read value of + * @signal_u32_read: Signal u32 component read callback. The read value of * the respective Signal u32 component should be passed * back via the val parameter. - * @device_u64_read Device u64 component read callback. The read value of + * @device_u64_read: Device u64 component read callback. The read value of * the respective Device u64 component should be passed * back via the val parameter. - * @count_u64_read Count u64 component read callback. The read value of the + * @count_u64_read: Count u64 component read callback. The read value of the * respective Count u64 component should be passed back via * the val parameter. - * @signal_u64_read Signal u64 component read callback. The read value of + * @signal_u64_read: Signal u64 component read callback. The read value of * the respective Signal u64 component should be passed * back via the val parameter. - * @action_write Synapse action mode write callback. The write value of + * @action_write: Synapse action mode write callback. The write value of * the respective Synapse action mode is passed via the * action parameter. - * @device_u8_write Device u8 component write callback. The write value of + * @device_u8_write: Device u8 component write callback. The write value of * the respective Device u8 component is passed via the val * parameter. - * @count_u8_write Count u8 component write callback. The write value of + * @count_u8_write: Count u8 component write callback. The write value of * the respective Count u8 component is passed via the val * parameter. - * @signal_u8_write Signal u8 component write callback. The write value of + * @signal_u8_write: Signal u8 component write callback. The write value of * the respective Signal u8 component is passed via the val * parameter. - * @device_u32_write Device u32 component write callback. The write value of + * @device_u32_write: Device u32 component write callback. The write value of * the respective Device u32 component is passed via the * val parameter. - * @count_u32_write Count u32 component write callback. The write value of + * @count_u32_write: Count u32 component write callback. The write value of * the respective Count u32 component is passed via the val * parameter. - * @signal_u32_write Signal u32 component write callback. The write value of + * @signal_u32_write: Signal u32 component write callback. The write value of * the respective Signal u32 component is passed via the * val parameter. - * @device_u64_write Device u64 component write callback. The write value of + * @device_u64_write: Device u64 component write callback. The write value of * the respective Device u64 component is passed via the * val parameter. - * @count_u64_write Count u64 component write callback. The write value of + * @count_u64_write: Count u64 component write callback. The write value of * the respective Count u64 component is passed via the val * parameter. - * @signal_u64_write Signal u64 component write callback. The write value of + * @signal_u64_write: Signal u64 component write callback. The write value of * the respective Signal u64 component is passed via the * val parameter. */ -- cgit v1.2.3-59-g8ed1b From 79f1c7304295bbbc611bc53cfd5425b777b3e840 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 9 Dec 2021 14:30:08 +0200 Subject: kernfs: Replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. Acked-by: Tejun Heo Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20211209123008.3391-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/kernfs.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 9f650986a81b..861c4f0f8a29 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -6,7 +6,6 @@ #ifndef __LINUX_KERNFS_H #define __LINUX_KERNFS_H -#include #include #include #include @@ -14,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -23,6 +24,7 @@ struct dentry; struct iattr; struct seq_file; struct vm_area_struct; +struct vm_operations_struct; struct super_block; struct file_system_type; struct poll_table_struct; -- cgit v1.2.3-59-g8ed1b From eca6e2d4a4a4b824f055eeaaa24f1c2327fb91a2 Mon Sep 17 00:00:00 2001 From: Anand Ashok Dumbre Date: Fri, 3 Dec 2021 21:23:54 +0000 Subject: device property: Add fwnode_iomap() This patch introduces a new helper routine - fwnode_iomap(), which allows to map the memory mapped IO for a given device node. This implementation does not cover the ACPI case and may be expanded in the future. The main purpose here is to be able to develop resource provider agnostic drivers. Suggested-by: Andy Shevchenko Signed-off-by: Anand Ashok Dumbre Reviewed-by: Andy Shevchenko Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20211203212358.31444-2-anand.ashok.dumbre@xilinx.com Signed-off-by: Jonathan Cameron --- drivers/base/property.c | 16 ++++++++++++++++ include/linux/property.h | 2 ++ 2 files changed, 18 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index f1f35b48ab8b..ed4470410030 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -958,6 +958,22 @@ int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index) } EXPORT_SYMBOL(fwnode_irq_get); +/** + * fwnode_iomap - Maps the memory mapped IO for a given fwnode + * @fwnode: Pointer to the firmware node + * @index: Index of the IO range + * + * Returns a pointer to the mapped memory. + */ +void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index) +{ + if (IS_ENABLED(CONFIG_OF_ADDRESS) && is_of_node(fwnode)) + return of_iomap(to_of_node(fwnode), index); + + return NULL; +} +EXPORT_SYMBOL(fwnode_iomap); + /** * fwnode_graph_get_next_endpoint - Get next endpoint firmware node * @fwnode: Pointer to the parent firmware node diff --git a/include/linux/property.h b/include/linux/property.h index 88fa726a76df..6670d5a1ec2a 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -122,6 +122,8 @@ void fwnode_handle_put(struct fwnode_handle *fwnode); int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index); +void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index); + unsigned int device_get_child_node_count(struct device *dev); static inline bool device_property_read_bool(struct device *dev, -- cgit v1.2.3-59-g8ed1b From 1b0b6cc8030d08d2a24e9e5f85dc36c5a58200ba Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 24 Nov 2021 00:27:01 +0100 Subject: power: supply: add charge_behaviour attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This a revised version of "[RFC] add standardized attributes for force_discharge and inhibit_charge" [0], incorporating discussion results. The biggest change is the switch from two boolean attributes to a single enum attribute. [0] https://lore.kernel.org/platform-driver-x86/21569a89-8303-8573-05fb-c2fec29983d1@gmail.com/ Signed-off-by: Thomas Weißschuh Acked-by: Sebastian Reichel Link: https://lore.kernel.org/r/20211123232704.25394-2-linux@weissschuh.net Signed-off-by: Hans de Goede --- Documentation/ABI/testing/sysfs-class-power | 14 ++++++++++++++ include/linux/power_supply.h | 7 +++++++ 2 files changed, 21 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power index f7904efc4cfa..cece094764f8 100644 --- a/Documentation/ABI/testing/sysfs-class-power +++ b/Documentation/ABI/testing/sysfs-class-power @@ -455,6 +455,20 @@ Description: "Unknown", "Charging", "Discharging", "Not charging", "Full" +What: /sys/class/power_supply//charge_behaviour +Date: November 2021 +Contact: linux-pm@vger.kernel.org +Description: + Represents the charging behaviour. + + Access: Read, Write + + Valid values: + ================ ==================================== + auto: Charge normally, respect thresholds + inhibit-charge: Do not charge while AC is attached + force-discharge: Force discharge while AC is attached + What: /sys/class/power_supply//technology Date: May 2007 Contact: linux-pm@vger.kernel.org diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 9ca1f120a211..70c333e86293 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -132,6 +132,7 @@ enum power_supply_property { POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX, POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, /* in percents! */ POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, /* in percents! */ + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT, POWER_SUPPLY_PROP_INPUT_POWER_LIMIT, @@ -202,6 +203,12 @@ enum power_supply_usb_type { POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID, /* Apple Charging Method */ }; +enum power_supply_charge_behaviour { + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO = 0, + POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE, + POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE, +}; + enum power_supply_notifier_events { PSY_EVENT_PROP_CHANGED, }; -- cgit v1.2.3-59-g8ed1b From 539b9c94ac83563842a27e8cc3de5164b15c4de0 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 24 Nov 2021 00:27:02 +0100 Subject: power: supply: add helpers for charge_behaviour sysfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These helper functions can be used by drivers to implement their own sysfs-attributes. This is useful for ACPI-drivers extending the default ACPI-battery with their own charge_behaviour attributes. Signed-off-by: Thomas Weißschuh Acked-by: Sebastian Reichel Link: https://lore.kernel.org/r/20211123232704.25394-3-linux@weissschuh.net Signed-off-by: Hans de Goede --- drivers/power/supply/power_supply_sysfs.c | 55 +++++++++++++++++++++++++++++++ include/linux/power_supply.h | 9 +++++ 2 files changed, 64 insertions(+) (limited to 'include/linux') diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index c3d7cbcd4fad..5e3b8c15ddbe 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -133,6 +133,12 @@ static const char * const POWER_SUPPLY_SCOPE_TEXT[] = { [POWER_SUPPLY_SCOPE_DEVICE] = "Device", }; +static const char * const POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT[] = { + [POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO] = "auto", + [POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE] = "inhibit-charge", + [POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE] = "force-discharge", +}; + static struct power_supply_attr power_supply_attrs[] = { /* Properties of type `int' */ POWER_SUPPLY_ENUM_ATTR(STATUS), @@ -484,3 +490,52 @@ out: return ret; } + +ssize_t power_supply_charge_behaviour_show(struct device *dev, + unsigned int available_behaviours, + enum power_supply_charge_behaviour current_behaviour, + char *buf) +{ + bool match = false, available, active; + ssize_t count = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT); i++) { + available = available_behaviours & BIT(i); + active = i == current_behaviour; + + if (available && active) { + count += sysfs_emit_at(buf, count, "[%s] ", + POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT[i]); + match = true; + } else if (available) { + count += sysfs_emit_at(buf, count, "%s ", + POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT[i]); + } + } + + if (!match) { + dev_warn(dev, "driver reporting unsupported charge behaviour\n"); + return -EINVAL; + } + + if (count) + buf[count - 1] = '\n'; + + return count; +} +EXPORT_SYMBOL_GPL(power_supply_charge_behaviour_show); + +int power_supply_charge_behaviour_parse(unsigned int available_behaviours, const char *buf) +{ + int i = sysfs_match_string(POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT, buf); + + if (i < 0) + return i; + + if (available_behaviours & BIT(i)) + return i; + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(power_supply_charge_behaviour_parse); diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 70c333e86293..71f0379c2af8 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -546,4 +546,13 @@ static inline void power_supply_remove_hwmon_sysfs(struct power_supply *psy) {} #endif +#ifdef CONFIG_SYSFS +ssize_t power_supply_charge_behaviour_show(struct device *dev, + unsigned int available_behaviours, + enum power_supply_charge_behaviour behaviour, + char *buf); + +int power_supply_charge_behaviour_parse(unsigned int available_behaviours, const char *buf); +#endif + #endif /* __LINUX_POWER_SUPPLY_H__ */ -- cgit v1.2.3-59-g8ed1b From 37ae5a0f5287a52cf51242e76ccf198d02ffe495 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Sat, 18 Dec 2021 18:41:56 +0900 Subject: block: use "unsigned long" for blk_validate_block_size(). Since lo_simple_ioctl(LOOP_SET_BLOCK_SIZE) and ioctl(NBD_SET_BLKSIZE) pass user-controlled "unsigned long arg" to blk_validate_block_size(), "unsigned long" should be used for validation. Signed-off-by: Tetsuo Handa Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/9ecbf057-4375-c2db-ab53-e4cc0dff953d@i-love.sakura.ne.jp Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c80cfaefc0a8..bb5fb7282e6e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -45,7 +45,7 @@ struct blk_crypto_profile; */ #define BLKCG_MAX_POLS 6 -static inline int blk_validate_block_size(unsigned int bsize) +static inline int blk_validate_block_size(unsigned long bsize) { if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize)) return -EINVAL; -- cgit v1.2.3-59-g8ed1b From f857acfc457ea63fa5b862d77f055665d863acfe Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Wed, 17 Nov 2021 14:53:48 -0700 Subject: lib/scatterlist: cleanup macros into static inline functions Convert the sg_is_chain(), sg_is_last() and sg_chain_ptr() macros into static inline functions. There's no reason for these to be macros and static inline are generally preferred these days. Also introduce the SG_PAGE_LINK_MASK define so the P2PDMA work, which is adding another bit to this mask, can do so more easily. Suggested-by: Jason Gunthorpe Signed-off-by: Logan Gunthorpe Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- include/linux/scatterlist.h | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 266754a55327..7ff9d6386c12 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -69,10 +69,27 @@ struct sg_append_table { * a valid sg entry, or whether it points to the start of a new scatterlist. * Those low bits are there for everyone! (thanks mason :-) */ -#define sg_is_chain(sg) ((sg)->page_link & SG_CHAIN) -#define sg_is_last(sg) ((sg)->page_link & SG_END) -#define sg_chain_ptr(sg) \ - ((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END))) +#define SG_PAGE_LINK_MASK (SG_CHAIN | SG_END) + +static inline unsigned int __sg_flags(struct scatterlist *sg) +{ + return sg->page_link & SG_PAGE_LINK_MASK; +} + +static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg) +{ + return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK); +} + +static inline bool sg_is_chain(struct scatterlist *sg) +{ + return __sg_flags(sg) & SG_CHAIN; +} + +static inline bool sg_is_last(struct scatterlist *sg) +{ + return __sg_flags(sg) & SG_END; +} /** * sg_assign_page - Assign a given page to an SG entry @@ -92,7 +109,7 @@ static inline void sg_assign_page(struct scatterlist *sg, struct page *page) * In order for the low bit stealing approach to work, pages * must be aligned at a 32-bit boundary as a minimum. */ - BUG_ON((unsigned long) page & (SG_CHAIN | SG_END)); + BUG_ON((unsigned long)page & SG_PAGE_LINK_MASK); #ifdef CONFIG_DEBUG_SG BUG_ON(sg_is_chain(sg)); #endif @@ -126,7 +143,7 @@ static inline struct page *sg_page(struct scatterlist *sg) #ifdef CONFIG_DEBUG_SG BUG_ON(sg_is_chain(sg)); #endif - return (struct page *)((sg)->page_link & ~(SG_CHAIN | SG_END)); + return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK); } /** -- cgit v1.2.3-59-g8ed1b From 365481e42a8a95c55e43e8cc236138718e762e7b Mon Sep 17 00:00:00 2001 From: "David E. Box" Date: Tue, 7 Dec 2021 17:50:11 -0800 Subject: driver core: auxiliary bus: Add driver data helpers Adds get/set driver data helpers for auxiliary devices. Reviewed-by: Mark Gross Reviewed-by: Andy Shevchenko Signed-off-by: David E. Box Link: https://lore.kernel.org/r/20211208015015.891275-3-david.e.box@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/auxiliary_bus.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index e6d8b5c16226..de21d9d24a95 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -188,6 +188,16 @@ struct auxiliary_driver { const struct auxiliary_device_id *id_table; }; +static inline void *auxiliary_get_drvdata(struct auxiliary_device *auxdev) +{ + return dev_get_drvdata(&auxdev->dev); +} + +static inline void auxiliary_set_drvdata(struct auxiliary_device *auxdev, void *data) +{ + dev_set_drvdata(&auxdev->dev, data); +} + static inline struct auxiliary_device *to_auxiliary_dev(struct device *dev) { return container_of(dev, struct auxiliary_device, dev); -- cgit v1.2.3-59-g8ed1b From b398123bff3bcbc1facb0f29bf6e7b9f1bc55931 Mon Sep 17 00:00:00 2001 From: Pingfan Liu Date: Wed, 15 Dec 2021 10:13:48 +0800 Subject: efi: apply memblock cap after memblock_add() On arm64, during kdump kernel saves vmcore, it runs into the following bug: ... [ 15.148919] usercopy: Kernel memory exposure attempt detected from SLUB object 'kmem_cache_node' (offset 0, size 4096)! [ 15.159707] ------------[ cut here ]------------ [ 15.164311] kernel BUG at mm/usercopy.c:99! [ 15.168482] Internal error: Oops - BUG: 0 [#1] SMP [ 15.173261] Modules linked in: xfs libcrc32c crct10dif_ce ghash_ce sha2_ce sha256_arm64 sha1_ce sbsa_gwdt ast i2c_algo_bit drm_vram_helper drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cec drm_ttm_helper ttm drm nvme nvme_core xgene_hwmon i2c_designware_platform i2c_designware_core dm_mirror dm_region_hash dm_log dm_mod overlay squashfs zstd_decompress loop [ 15.206186] CPU: 0 PID: 542 Comm: cp Not tainted 5.16.0-rc4 #1 [ 15.212006] Hardware name: GIGABYTE R272-P30-JG/MP32-AR0-JG, BIOS F12 (SCP: 1.5.20210426) 05/13/2021 [ 15.221125] pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 15.228073] pc : usercopy_abort+0x9c/0xa0 [ 15.232074] lr : usercopy_abort+0x9c/0xa0 [ 15.236070] sp : ffff8000121abba0 [ 15.239371] x29: ffff8000121abbb0 x28: 0000000000003000 x27: 0000000000000000 [ 15.246494] x26: 0000000080000400 x25: 0000ffff885c7000 x24: 0000000000000000 [ 15.253617] x23: 000007ff80400000 x22: ffff07ff80401000 x21: 0000000000000001 [ 15.260739] x20: 0000000000001000 x19: ffff07ff80400000 x18: ffffffffffffffff [ 15.267861] x17: 656a626f2042554c x16: 53206d6f72662064 x15: 6574636574656420 [ 15.274983] x14: 74706d6574746120 x13: 2129363930342065 x12: 7a6973202c302074 [ 15.282105] x11: ffffc8b041d1b148 x10: 00000000ffff8000 x9 : ffffc8b04012812c [ 15.289228] x8 : 00000000ffff7fff x7 : ffffc8b041d1b148 x6 : 0000000000000000 [ 15.296349] x5 : 0000000000000000 x4 : 0000000000007fff x3 : 0000000000000000 [ 15.303471] x2 : 0000000000000000 x1 : ffff07ff8c064800 x0 : 000000000000006b [ 15.310593] Call trace: [ 15.313027] usercopy_abort+0x9c/0xa0 [ 15.316677] __check_heap_object+0xd4/0xf0 [ 15.320762] __check_object_size.part.0+0x160/0x1e0 [ 15.325628] __check_object_size+0x2c/0x40 [ 15.329711] copy_oldmem_page+0x7c/0x140 [ 15.333623] read_from_oldmem.part.0+0xfc/0x1c0 [ 15.338142] __read_vmcore.constprop.0+0x23c/0x350 [ 15.342920] read_vmcore+0x28/0x34 [ 15.346309] proc_reg_read+0xb4/0xf0 [ 15.349871] vfs_read+0xb8/0x1f0 [ 15.353088] ksys_read+0x74/0x100 [ 15.356390] __arm64_sys_read+0x28/0x34 ... This bug introduced by commit b261dba2fdb2 ("arm64: kdump: Remove custom linux,usable-memory-range handling"), which moves memblock_cap_memory_range() to fdt, but it breaches the rules that memblock_cap_memory_range() should come after memblock_add() etc as said in commit e888fa7bb882 ("memblock: Check memory add/cap ordering"). As a consequence, the virtual address set up by copy_oldmem_page() does not bail out from the test of virt_addr_valid() in check_heap_object(), and finally hits the BUG_ON(). Since memblock allocator has no idea about when the memblock is fully populated, while efi_init() is aware, so tackling this issue by calling the interface early_init_dt_check_for_usable_mem_range() exposed by of/fdt. Fixes: b261dba2fdb2 ("arm64: kdump: Remove custom linux,usable-memory-range handling") Signed-off-by: Pingfan Liu Cc: Rob Herring Cc: Zhen Lei Cc: Catalin Marinas Cc: Will Deacon Cc: Andrew Morton Cc: Mike Rapoport Cc: Geert Uytterhoeven Cc: Frank Rowand Cc: Ard Biesheuvel Cc: Nick Terrell Cc: linux-arm-kernel@lists.infradead.org To: devicetree@vger.kernel.org To: linux-efi@vger.kernel.org Acked-by: Ard Biesheuvel Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20211215021348.8766-1-kernelfans@gmail.com --- drivers/firmware/efi/efi-init.c | 5 +++++ drivers/of/fdt.c | 2 +- include/linux/of_fdt.h | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c index b19ce1a83f91..b2c829e95bd1 100644 --- a/drivers/firmware/efi/efi-init.c +++ b/drivers/firmware/efi/efi-init.c @@ -235,6 +235,11 @@ void __init efi_init(void) } reserve_regions(); + /* + * For memblock manipulation, the cap should come after the memblock_add(). + * And now, memblock is fully populated, it is time to do capping. + */ + early_init_dt_check_for_usable_mem_range(); efi_esrt_init(); efi_mokvar_table_init(); diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 5a238a933eb2..65af475dfa95 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -971,7 +971,7 @@ static unsigned long chosen_node_offset = -FDT_ERR_NOTFOUND; * early_init_dt_check_for_usable_mem_range - Decode usable memory range * location from flat tree */ -static void __init early_init_dt_check_for_usable_mem_range(void) +void __init early_init_dt_check_for_usable_mem_range(void) { const __be32 *prop; int len; diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index cf48983d3c86..ad09beb6d13c 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -62,6 +62,7 @@ extern int early_init_dt_scan_chosen(unsigned long node, const char *uname, int depth, void *data); extern int early_init_dt_scan_memory(unsigned long node, const char *uname, int depth, void *data); +extern void early_init_dt_check_for_usable_mem_range(void); extern int early_init_dt_scan_chosen_stdout(void); extern void early_init_fdt_scan_reserved_mem(void); extern void early_init_fdt_reserve_self(void); @@ -86,6 +87,7 @@ extern void unflatten_and_copy_device_tree(void); extern void early_init_devtree(void *); extern void early_get_first_memblock_info(void *, phys_addr_t *); #else /* CONFIG_OF_EARLY_FLATTREE */ +static inline void early_init_dt_check_for_usable_mem_range(void) {} static inline int early_init_dt_scan_chosen_stdout(void) { return -ENODEV; } static inline void early_init_fdt_scan_reserved_mem(void) {} static inline void early_init_fdt_reserve_self(void) {} -- cgit v1.2.3-59-g8ed1b From f2f8115fe8b390af27d013411045bd712a812103 Mon Sep 17 00:00:00 2001 From: Roger Quadros Date: Tue, 21 Dec 2021 15:17:56 +0200 Subject: memory: omap-gpmc: Use a compatible match table when checking for NAND controller As more compatibles can be added to the GPMC NAND controller driver use a compatible match table. Signed-off-by: Roger Quadros Acked-by: Miquel Raynal Link: https://lore.kernel.org/r/20211221131757.2030-4-rogerq@kernel.org [krzysztof: remove "is_nand" variable] Signed-off-by: Krzysztof Kozlowski --- drivers/memory/omap-gpmc.c | 2 +- drivers/mtd/nand/raw/omap2.c | 5 +---- include/linux/platform_data/mtd-nand-omap2.h | 9 ++++++++- 3 files changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c index 5e2ba39b6450..ed11887c1b7c 100644 --- a/drivers/memory/omap-gpmc.c +++ b/drivers/memory/omap-gpmc.c @@ -2183,7 +2183,7 @@ static int gpmc_probe_generic_child(struct platform_device *pdev, } } - if (of_device_is_compatible(child, "ti,omap2-nand")) { + if (of_match_node(omap_nand_ids, child)) { /* NAND specific setup */ val = 8; of_property_read_u32(child, "nand-bus-width", &val); diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c index b26d4947af02..e6dd8b4cf0d2 100644 --- a/drivers/mtd/nand/raw/omap2.c +++ b/drivers/mtd/nand/raw/omap2.c @@ -2352,10 +2352,7 @@ static int omap_nand_remove(struct platform_device *pdev) return ret; } -static const struct of_device_id omap_nand_ids[] = { - { .compatible = "ti,omap2-nand", }, - {}, -}; +/* omap_nand_ids defined in linux/platform_data/mtd-nand-omap2.h */ MODULE_DEVICE_TABLE(of, omap_nand_ids); static struct platform_driver omap_nand_driver = { diff --git a/include/linux/platform_data/mtd-nand-omap2.h b/include/linux/platform_data/mtd-nand-omap2.h index de6ada739121..92f011805ad4 100644 --- a/include/linux/platform_data/mtd-nand-omap2.h +++ b/include/linux/platform_data/mtd-nand-omap2.h @@ -7,6 +7,7 @@ #define _MTD_NAND_OMAP2_H #include +#include #define GPMC_BCH_NUM_REMAINDER 8 @@ -61,4 +62,10 @@ struct gpmc_nand_regs { void __iomem *gpmc_bch_result5[GPMC_BCH_NUM_REMAINDER]; void __iomem *gpmc_bch_result6[GPMC_BCH_NUM_REMAINDER]; }; -#endif + +static const struct of_device_id omap_nand_ids[] = { + { .compatible = "ti,omap2-nand", }, + {}, +}; + +#endif /* _MTD_NAND_OMAP2_H */ -- cgit v1.2.3-59-g8ed1b From 6fd3c510ee4b37f2f9fe3d3cafbfa459e15c5e11 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 22 Dec 2021 13:15:32 -0800 Subject: bio.h: fix kernel-doc warnings Fix all kernel-doc warnings in : include/linux/bio.h:136: warning: Function parameter or member 'nbytes' not described in 'bio_advance' include/linux/bio.h:136: warning: Excess function parameter 'bytes' description in 'bio_advance' include/linux/bio.h:391: warning: No description found for return value of 'bio_next_split' Signed-off-by: Randy Dunlap Cc: Kent Overstreet Cc: Jens Axboe Cc: linux-block@vger.kernel.org Link: https://lore.kernel.org/r/20211222211532.24060-1-rdunlap@infradead.org Signed-off-by: Jens Axboe --- include/linux/bio.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index fe6bdfbbef66..0a41efe02208 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -124,7 +124,7 @@ void __bio_advance(struct bio *, unsigned bytes); /** * bio_advance - increment/complete a bio by some number of bytes * @bio: bio to advance - * @bytes: number of bytes to complete + * @nbytes: number of bytes to complete * * This updates bi_sector, bi_size and bi_idx; if the number of bytes to * complete doesn't align with a bvec boundary, then bv_len and bv_offset will @@ -332,7 +332,7 @@ extern struct bio *bio_split(struct bio *bio, int sectors, * @gfp: gfp mask * @bs: bio set to allocate from * - * Returns a bio representing the next @sectors of @bio - if the bio is smaller + * Return: a bio representing the next @sectors of @bio - if the bio is smaller * than @sectors, returns the original bio unchanged. */ static inline struct bio *bio_next_split(struct bio *bio, int sectors, -- cgit v1.2.3-59-g8ed1b From a16c7246368db8935652c805bc446928d0e1c0aa Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 22 Dec 2021 13:52:39 -0800 Subject: block: remove unnecessary trailing '\' While harmless, the blank line is certainly not intended to be part of the rq_list_for_each() macro. Remove it. Signed-off-by: Keith Busch Link: https://lore.kernel.org/r/20211222215239.1768164-1-kbusch@kernel.org Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index bb5fb7282e6e..22746b2d6825 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1363,7 +1363,7 @@ struct io_comp_batch { }) #define rq_list_for_each(listptr, pos) \ - for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) \ + for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) #define rq_list_next(rq) (rq)->rq_next #define rq_list_empty(list) ((list) == (struct request *) NULL) -- cgit v1.2.3-59-g8ed1b From 6e1fcab00a23f7fe9f4fe9704905a790efa1eeab Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Mon, 20 Dec 2021 19:21:26 +0800 Subject: scsi: block: pm: Always set request queue runtime active in blk_post_runtime_resume() John Garry reported a deadlock that occurs when trying to access a runtime-suspended SATA device. For obscure reasons, the rescan procedure causes the link to be hard-reset, which disconnects the device. The rescan tries to carry out a runtime resume when accessing the device. scsi_rescan_device() holds the SCSI device lock and won't release it until it can put commands onto the device's block queue. This can't happen until the queue is successfully runtime-resumed or the device is unregistered. But the runtime resume fails because the device is disconnected, and __scsi_remove_device() can't do the unregistration because it can't get the device lock. The best way to resolve this deadlock appears to be to allow the block queue to start running again even after an unsuccessful runtime resume. The idea is that the driver or the SCSI error handler will need to be able to use the queue to resolve the runtime resume failure. This patch removes the err argument to blk_post_runtime_resume() and makes the routine act as though the resume was successful always. This fixes the deadlock. Link: https://lore.kernel.org/r/1639999298-244569-4-git-send-email-chenxiang66@hisilicon.com Fixes: e27829dc92e5 ("scsi: serialize ->rescan against ->remove") Reported-and-tested-by: John Garry Reviewed-by: Bart Van Assche Signed-off-by: Alan Stern Signed-off-by: Xiang Chen Signed-off-by: Martin K. Petersen --- block/blk-pm.c | 22 +++++++--------------- drivers/scsi/scsi_pm.c | 2 +- include/linux/blk-pm.h | 2 +- 3 files changed, 9 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/block/blk-pm.c b/block/blk-pm.c index 17bd020268d4..2dad62cc1572 100644 --- a/block/blk-pm.c +++ b/block/blk-pm.c @@ -163,27 +163,19 @@ EXPORT_SYMBOL(blk_pre_runtime_resume); /** * blk_post_runtime_resume - Post runtime resume processing * @q: the queue of the device - * @err: return value of the device's runtime_resume function * * Description: - * Update the queue's runtime status according to the return value of the - * device's runtime_resume function. If the resume was successful, call - * blk_set_runtime_active() to do the real work of restarting the queue. + * For historical reasons, this routine merely calls blk_set_runtime_active() + * to do the real work of restarting the queue. It does this regardless of + * whether the device's runtime-resume succeeded; even if it failed the + * driver or error handler will need to communicate with the device. * * This function should be called near the end of the device's * runtime_resume callback. */ -void blk_post_runtime_resume(struct request_queue *q, int err) +void blk_post_runtime_resume(struct request_queue *q) { - if (!q->dev) - return; - if (!err) { - blk_set_runtime_active(q); - } else { - spin_lock_irq(&q->queue_lock); - q->rpm_status = RPM_SUSPENDED; - spin_unlock_irq(&q->queue_lock); - } + blk_set_runtime_active(q); } EXPORT_SYMBOL(blk_post_runtime_resume); @@ -201,7 +193,7 @@ EXPORT_SYMBOL(blk_post_runtime_resume); * runtime PM status and re-enable peeking requests from the queue. It * should be called before first request is added to the queue. * - * This function is also called by blk_post_runtime_resume() for successful + * This function is also called by blk_post_runtime_resume() for * runtime resumes. It does everything necessary to restart the queue. */ void blk_set_runtime_active(struct request_queue *q) diff --git a/drivers/scsi/scsi_pm.c b/drivers/scsi/scsi_pm.c index 0e841e8761c5..d581613d87c7 100644 --- a/drivers/scsi/scsi_pm.c +++ b/drivers/scsi/scsi_pm.c @@ -180,7 +180,7 @@ static int sdev_runtime_resume(struct device *dev) blk_pre_runtime_resume(sdev->request_queue); if (pm && pm->runtime_resume) err = pm->runtime_resume(dev); - blk_post_runtime_resume(sdev->request_queue, err); + blk_post_runtime_resume(sdev->request_queue); return err; } diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h index b80c65aba249..2580e05a8ab6 100644 --- a/include/linux/blk-pm.h +++ b/include/linux/blk-pm.h @@ -14,7 +14,7 @@ extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); extern int blk_pre_runtime_suspend(struct request_queue *q); extern void blk_post_runtime_suspend(struct request_queue *q, int err); extern void blk_pre_runtime_resume(struct request_queue *q); -extern void blk_post_runtime_resume(struct request_queue *q, int err); +extern void blk_post_runtime_resume(struct request_queue *q); extern void blk_set_runtime_active(struct request_queue *q); #else static inline void blk_pm_runtime_init(struct request_queue *q, -- cgit v1.2.3-59-g8ed1b From 30be4551f9e26292599e666985119a5b559a2e4a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 22 Dec 2021 18:32:56 +0200 Subject: wwan: Replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. Signed-off-by: Andy Shevchenko Reviewed-by: Sergey Ryazanov Signed-off-by: David S. Miller --- include/linux/wwan.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/wwan.h b/include/linux/wwan.h index e143c88bf4b0..afb3334ec8c5 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -4,12 +4,9 @@ #ifndef __WWAN_H #define __WWAN_H -#include -#include #include -#include -#include #include +#include /** * enum wwan_port_type - WWAN port types @@ -37,6 +34,10 @@ enum wwan_port_type { WWAN_PORT_UNKNOWN, }; +struct device; +struct file; +struct netlink_ext_ack; +struct sk_buff; struct wwan_port; /** struct wwan_port_ops - The WWAN port operations -- cgit v1.2.3-59-g8ed1b From 7da1d1ddd1f02e5de7497a0c849256912652fb6c Mon Sep 17 00:00:00 2001 From: Nick Child Date: Thu, 16 Dec 2021 17:00:35 -0500 Subject: cuda/pmu: Make find_via_cuda/pmu init functions Make `find_via_cuda` and `find_via_pmu` initialization functions. Previously, their definitions in `drivers/macintosh/via-cuda.h` include the `__init` attribute but their alternative definitions in `arch/powerpc/powermac/sectup./c` and prototypes in `include/linux/ cuda.h` and `include/linux/pmu.h` do not use the `__init` macro. Since, only initialization functions call `find_via_cuda` and `find_via_pmu` it is safe to label these functions with `__init`. Signed-off-by: Nick Child Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20211216220035.605465-21-nick.child@ibm.com --- arch/powerpc/platforms/powermac/setup.c | 4 ++-- include/linux/cuda.h | 2 +- include/linux/pmu.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index f7661b81db18..974d4b49867b 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -166,7 +166,7 @@ static void pmac_show_cpuinfo(struct seq_file *m) } #ifndef CONFIG_ADB_CUDA -int find_via_cuda(void) +int __init find_via_cuda(void) { struct device_node *dn = of_find_node_by_name(NULL, "via-cuda"); @@ -180,7 +180,7 @@ int find_via_cuda(void) #endif #ifndef CONFIG_ADB_PMU -int find_via_pmu(void) +int __init find_via_pmu(void) { struct device_node *dn = of_find_node_by_name(NULL, "via-pmu"); diff --git a/include/linux/cuda.h b/include/linux/cuda.h index 45bfe9d61271..daf3e6f98444 100644 --- a/include/linux/cuda.h +++ b/include/linux/cuda.h @@ -12,7 +12,7 @@ #include -extern int find_via_cuda(void); +extern int __init find_via_cuda(void); extern int cuda_request(struct adb_request *req, void (*done)(struct adb_request *), int nbytes, ...); extern void cuda_poll(void); diff --git a/include/linux/pmu.h b/include/linux/pmu.h index 52453a24a24f..c677442d007c 100644 --- a/include/linux/pmu.h +++ b/include/linux/pmu.h @@ -13,7 +13,7 @@ #include -extern int find_via_pmu(void); +extern int __init find_via_pmu(void); extern int pmu_request(struct adb_request *req, void (*done)(struct adb_request *), int nbytes, ...); -- cgit v1.2.3-59-g8ed1b From 66b354064a35b6379963cba27b5d37a278fc9bd9 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 23 Nov 2021 11:16:00 +0100 Subject: powercap/drivers/dtpm: Remove unused function definition The dtpm.h header file is exporting a function which is not implemented neither needed. Remove it. Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20211123101601.2433340-1-daniel.lezcano@linaro.org --- include/linux/dtpm.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dtpm.h b/include/linux/dtpm.h index 2890f6370eb9..d37e5d06a357 100644 --- a/include/linux/dtpm.h +++ b/include/linux/dtpm.h @@ -70,6 +70,4 @@ void dtpm_unregister(struct dtpm *dtpm); int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent); -int dtpm_register_cpu(struct dtpm *parent); - #endif -- cgit v1.2.3-59-g8ed1b From dd123e62bdedcd3a486e48e883ec63138ec2c14c Mon Sep 17 00:00:00 2001 From: Henning Schild Date: Mon, 13 Dec 2021 13:04:59 +0100 Subject: platform/x86: simatic-ipc: add main driver for Siemens devices This mainly implements detection of these devices and will allow secondary drivers to work on such machines. The identification is DMI-based with a vendor specific way to tell them apart in a reliable way. Drivers for LEDs and Watchdogs will follow to make use of that platform detection. There is also some code to allow secondary drivers to find GPIO memory, that needs to be in place because the pinctrl drivers do not come up. Signed-off-by: Henning Schild Link: https://lore.kernel.org/r/20211213120502.20661-2-henning.schild@siemens.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/Kconfig | 12 ++ drivers/platform/x86/Makefile | 3 + drivers/platform/x86/simatic-ipc.c | 176 +++++++++++++++++++++ include/linux/platform_data/x86/simatic-ipc-base.h | 29 ++++ include/linux/platform_data/x86/simatic-ipc.h | 72 +++++++++ 5 files changed, 292 insertions(+) create mode 100644 drivers/platform/x86/simatic-ipc.c create mode 100644 include/linux/platform_data/x86/simatic-ipc-base.h create mode 100644 include/linux/platform_data/x86/simatic-ipc.h (limited to 'include/linux') diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 961f33bea1f1..afa0f9b0141d 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -1088,6 +1088,18 @@ config INTEL_SCU_IPC_UTIL low level access for debug work and updating the firmware. Say N unless you will be doing this on an Intel MID platform. +config SIEMENS_SIMATIC_IPC + tristate "Siemens Simatic IPC Class driver" + depends on PCI + help + This Simatic IPC class driver is the central of several drivers. It + is mainly used for system identification, after which drivers in other + classes will take care of driving specifics of those machines. + i.e. LEDs and watchdog. + + To compile this driver as a module, choose M here: the module + will be called simatic-ipc. + endif # X86_PLATFORM_DEVICES config PMC_ATOM diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 18b11769073b..d477aad34fab 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -124,3 +124,6 @@ obj-$(CONFIG_INTEL_SCU_PLATFORM) += intel_scu_pltdrv.o obj-$(CONFIG_INTEL_SCU_WDT) += intel_scu_wdt.o obj-$(CONFIG_INTEL_SCU_IPC_UTIL) += intel_scu_ipcutil.o obj-$(CONFIG_PMC_ATOM) += pmc_atom.o + +# Siemens Simatic Industrial PCs +obj-$(CONFIG_SIEMENS_SIMATIC_IPC) += simatic-ipc.o diff --git a/drivers/platform/x86/simatic-ipc.c b/drivers/platform/x86/simatic-ipc.c new file mode 100644 index 000000000000..b599cda5ba3c --- /dev/null +++ b/drivers/platform/x86/simatic-ipc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Siemens SIMATIC IPC platform driver + * + * Copyright (c) Siemens AG, 2018-2021 + * + * Authors: + * Henning Schild + * Jan Kiszka + * Gerd Haeussler + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +static struct platform_device *ipc_led_platform_device; +static struct platform_device *ipc_wdt_platform_device; + +static const struct dmi_system_id simatic_ipc_whitelist[] = { + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "SIEMENS AG"), + }, + }, + {} +}; + +static struct simatic_ipc_platform platform_data; + +static struct { + u32 station_id; + u8 led_mode; + u8 wdt_mode; +} device_modes[] = { + {SIMATIC_IPC_IPC127E, SIMATIC_IPC_DEVICE_127E, SIMATIC_IPC_DEVICE_NONE}, + {SIMATIC_IPC_IPC227D, SIMATIC_IPC_DEVICE_227D, SIMATIC_IPC_DEVICE_NONE}, + {SIMATIC_IPC_IPC227E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_227E}, + {SIMATIC_IPC_IPC277E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_227E}, + {SIMATIC_IPC_IPC427D, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_NONE}, + {SIMATIC_IPC_IPC427E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_427E}, + {SIMATIC_IPC_IPC477E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_427E}, +}; + +static int register_platform_devices(u32 station_id) +{ + u8 ledmode = SIMATIC_IPC_DEVICE_NONE; + u8 wdtmode = SIMATIC_IPC_DEVICE_NONE; + int i; + + platform_data.devmode = SIMATIC_IPC_DEVICE_NONE; + + for (i = 0; i < ARRAY_SIZE(device_modes); i++) { + if (device_modes[i].station_id == station_id) { + ledmode = device_modes[i].led_mode; + wdtmode = device_modes[i].wdt_mode; + break; + } + } + + if (ledmode != SIMATIC_IPC_DEVICE_NONE) { + platform_data.devmode = ledmode; + ipc_led_platform_device = + platform_device_register_data(NULL, + KBUILD_MODNAME "_leds", PLATFORM_DEVID_NONE, + &platform_data, + sizeof(struct simatic_ipc_platform)); + if (IS_ERR(ipc_led_platform_device)) + return PTR_ERR(ipc_led_platform_device); + + pr_debug("device=%s created\n", + ipc_led_platform_device->name); + } + + if (wdtmode != SIMATIC_IPC_DEVICE_NONE) { + platform_data.devmode = wdtmode; + ipc_wdt_platform_device = + platform_device_register_data(NULL, + KBUILD_MODNAME "_wdt", PLATFORM_DEVID_NONE, + &platform_data, + sizeof(struct simatic_ipc_platform)); + if (IS_ERR(ipc_wdt_platform_device)) + return PTR_ERR(ipc_wdt_platform_device); + + pr_debug("device=%s created\n", + ipc_wdt_platform_device->name); + } + + if (ledmode == SIMATIC_IPC_DEVICE_NONE && + wdtmode == SIMATIC_IPC_DEVICE_NONE) { + pr_warn("unsupported IPC detected, station id=%08x\n", + station_id); + return -EINVAL; + } + + return 0; +} + +/* FIXME: this should eventually be done with generic P2SB discovery code + * the individual drivers for watchdogs and LEDs access memory that implements + * GPIO, but pinctrl will not come up because of missing ACPI entries + * + * While there is no conflict a cleaner solution would be to somehow bring up + * pinctrl even with these ACPI entries missing, and base the drivers on pinctrl. + * After which the following function could be dropped, together with the code + * poking the memory. + */ +/* + * Get membase address from PCI, used in leds and wdt module. Here we read + * the bar0. The final address calculation is done in the appropriate modules + */ +u32 simatic_ipc_get_membase0(unsigned int p2sb) +{ + struct pci_bus *bus; + u32 bar0 = 0; + /* + * The GPIO memory is in bar0 of the hidden P2SB device. + * Unhide the device to have a quick look at it, before we hide it + * again. + * Also grab the pci rescan lock so that device does not get discovered + * and remapped while it is visible. + * This code is inspired by drivers/mfd/lpc_ich.c + */ + bus = pci_find_bus(0, 0); + pci_lock_rescan_remove(); + pci_bus_write_config_byte(bus, p2sb, 0xE1, 0x0); + pci_bus_read_config_dword(bus, p2sb, PCI_BASE_ADDRESS_0, &bar0); + + bar0 &= ~0xf; + pci_bus_write_config_byte(bus, p2sb, 0xE1, 0x1); + pci_unlock_rescan_remove(); + + return bar0; +} +EXPORT_SYMBOL(simatic_ipc_get_membase0); + +static int __init simatic_ipc_init_module(void) +{ + const struct dmi_system_id *match; + u32 station_id; + int err; + + match = dmi_first_match(simatic_ipc_whitelist); + if (!match) + return 0; + + err = dmi_walk(simatic_ipc_find_dmi_entry_helper, &station_id); + + if (err || station_id == SIMATIC_IPC_INVALID_STATION_ID) { + pr_warn("DMI entry %d not found\n", SIMATIC_IPC_DMI_ENTRY_OEM); + return 0; + } + + return register_platform_devices(station_id); +} + +static void __exit simatic_ipc_exit_module(void) +{ + platform_device_unregister(ipc_led_platform_device); + ipc_led_platform_device = NULL; + + platform_device_unregister(ipc_wdt_platform_device); + ipc_wdt_platform_device = NULL; +} + +module_init(simatic_ipc_init_module); +module_exit(simatic_ipc_exit_module); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Gerd Haeussler "); +MODULE_ALIAS("dmi:*:svnSIEMENSAG:*"); diff --git a/include/linux/platform_data/x86/simatic-ipc-base.h b/include/linux/platform_data/x86/simatic-ipc-base.h new file mode 100644 index 000000000000..62d2bc774067 --- /dev/null +++ b/include/linux/platform_data/x86/simatic-ipc-base.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Siemens SIMATIC IPC drivers + * + * Copyright (c) Siemens AG, 2018-2021 + * + * Authors: + * Henning Schild + * Gerd Haeussler + */ + +#ifndef __PLATFORM_DATA_X86_SIMATIC_IPC_BASE_H +#define __PLATFORM_DATA_X86_SIMATIC_IPC_BASE_H + +#include + +#define SIMATIC_IPC_DEVICE_NONE 0 +#define SIMATIC_IPC_DEVICE_227D 1 +#define SIMATIC_IPC_DEVICE_427E 2 +#define SIMATIC_IPC_DEVICE_127E 3 +#define SIMATIC_IPC_DEVICE_227E 4 + +struct simatic_ipc_platform { + u8 devmode; +}; + +u32 simatic_ipc_get_membase0(unsigned int p2sb); + +#endif /* __PLATFORM_DATA_X86_SIMATIC_IPC_BASE_H */ diff --git a/include/linux/platform_data/x86/simatic-ipc.h b/include/linux/platform_data/x86/simatic-ipc.h new file mode 100644 index 000000000000..f3b76b39776b --- /dev/null +++ b/include/linux/platform_data/x86/simatic-ipc.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Siemens SIMATIC IPC drivers + * + * Copyright (c) Siemens AG, 2018-2021 + * + * Authors: + * Henning Schild + * Gerd Haeussler + */ + +#ifndef __PLATFORM_DATA_X86_SIMATIC_IPC_H +#define __PLATFORM_DATA_X86_SIMATIC_IPC_H + +#include +#include + +#define SIMATIC_IPC_DMI_ENTRY_OEM 129 +/* binary type */ +#define SIMATIC_IPC_DMI_TYPE 0xff +#define SIMATIC_IPC_DMI_GROUP 0x05 +#define SIMATIC_IPC_DMI_ENTRY 0x02 +#define SIMATIC_IPC_DMI_TID 0x02 + +enum simatic_ipc_station_ids { + SIMATIC_IPC_INVALID_STATION_ID = 0, + SIMATIC_IPC_IPC227D = 0x00000501, + SIMATIC_IPC_IPC427D = 0x00000701, + SIMATIC_IPC_IPC227E = 0x00000901, + SIMATIC_IPC_IPC277E = 0x00000902, + SIMATIC_IPC_IPC427E = 0x00000A01, + SIMATIC_IPC_IPC477E = 0x00000A02, + SIMATIC_IPC_IPC127E = 0x00000D01, +}; + +static inline u32 simatic_ipc_get_station_id(u8 *data, int max_len) +{ + struct { + u8 type; /* type (0xff = binary) */ + u8 len; /* len of data entry */ + u8 group; + u8 entry; + u8 tid; + __le32 station_id; /* station id (LE) */ + } __packed * data_entry = (void *)data + sizeof(struct dmi_header); + + while ((u8 *)data_entry < data + max_len) { + if (data_entry->type == SIMATIC_IPC_DMI_TYPE && + data_entry->len == sizeof(*data_entry) && + data_entry->group == SIMATIC_IPC_DMI_GROUP && + data_entry->entry == SIMATIC_IPC_DMI_ENTRY && + data_entry->tid == SIMATIC_IPC_DMI_TID) { + return le32_to_cpu(data_entry->station_id); + } + data_entry = (void *)((u8 *)(data_entry) + data_entry->len); + } + + return SIMATIC_IPC_INVALID_STATION_ID; +} + +static inline void +simatic_ipc_find_dmi_entry_helper(const struct dmi_header *dh, void *_data) +{ + u32 *id = _data; + + if (dh->type != SIMATIC_IPC_DMI_ENTRY_OEM) + return; + + *id = simatic_ipc_get_station_id((u8 *)dh, dh->length); +} + +#endif /* __PLATFORM_DATA_X86_SIMATIC_IPC_H */ -- cgit v1.2.3-59-g8ed1b From b86947b52f0d0e5b6e6f0510933ca13aad266e47 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 24 Dec 2021 10:10:29 +0800 Subject: ASoC/soundwire: intel: simplify callbacks for params/hw_free We don't really need to pass a substream to the callback, we only need the direction. No functionality change, only simplification to enable improve suspend with paused streams. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Signed-off-by: Bard Liao Acked-By: Vinod Koul Link: https://lore.kernel.org/r/20211224021034.26635-3-yung-chuan.liao@linux.intel.com Signed-off-by: Mark Brown --- drivers/soundwire/intel.c | 14 +++++++------- include/linux/soundwire/sdw_intel.h | 4 ++-- sound/soc/sof/intel/hda.c | 6 ++---- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 78037ffdb09b..25c5f5b9f058 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -711,7 +711,7 @@ intel_pdi_alh_configure(struct sdw_intel *sdw, struct sdw_cdns_pdi *pdi) } static int intel_params_stream(struct sdw_intel *sdw, - struct snd_pcm_substream *substream, + int stream, struct snd_soc_dai *dai, struct snd_pcm_hw_params *hw_params, int link_id, int alh_stream_id) @@ -719,7 +719,7 @@ static int intel_params_stream(struct sdw_intel *sdw, struct sdw_intel_link_res *res = sdw->link_res; struct sdw_intel_stream_params_data params_data; - params_data.substream = substream; + params_data.stream = stream; /* direction */ params_data.dai = dai; params_data.hw_params = hw_params; params_data.link_id = link_id; @@ -732,14 +732,14 @@ static int intel_params_stream(struct sdw_intel *sdw, } static int intel_free_stream(struct sdw_intel *sdw, - struct snd_pcm_substream *substream, + int stream, struct snd_soc_dai *dai, int link_id) { struct sdw_intel_link_res *res = sdw->link_res; struct sdw_intel_stream_free_data free_data; - free_data.substream = substream; + free_data.stream = stream; /* direction */ free_data.dai = dai; free_data.link_id = link_id; @@ -876,7 +876,7 @@ static int intel_hw_params(struct snd_pcm_substream *substream, dma->hw_params = params; /* Inform DSP about PDI stream number */ - ret = intel_params_stream(sdw, substream, dai, params, + ret = intel_params_stream(sdw, substream->stream, dai, params, sdw->instance, pdi->intel_alh_id); if (ret) @@ -953,7 +953,7 @@ static int intel_prepare(struct snd_pcm_substream *substream, sdw_cdns_config_stream(cdns, ch, dir, dma->pdi); /* Inform DSP about PDI stream number */ - ret = intel_params_stream(sdw, substream, dai, + ret = intel_params_stream(sdw, substream->stream, dai, dma->hw_params, sdw->instance, dma->pdi->intel_alh_id); @@ -987,7 +987,7 @@ intel_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) return ret; } - ret = intel_free_stream(sdw, substream, dai, sdw->instance); + ret = intel_free_stream(sdw, substream->stream, dai, sdw->instance); if (ret < 0) { dev_err(dai->dev, "intel_free_stream: failed %d\n", ret); return ret; diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index 8a463b8fc12a..67e0d3e750b5 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -92,7 +92,7 @@ * firmware. */ struct sdw_intel_stream_params_data { - struct snd_pcm_substream *substream; + int stream; struct snd_soc_dai *dai; struct snd_pcm_hw_params *hw_params; int link_id; @@ -105,7 +105,7 @@ struct sdw_intel_stream_params_data { * firmware. */ struct sdw_intel_stream_free_data { - struct snd_pcm_substream *substream; + int stream; struct snd_soc_dai *dai; int link_id; }; diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 99255028d3fe..c8fb082209ce 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -184,12 +184,11 @@ static int sdw_dai_config_ipc(struct snd_sof_dev *sdev, static int sdw_params_stream(struct device *dev, struct sdw_intel_stream_params_data *params_data) { - struct snd_pcm_substream *substream = params_data->substream; struct snd_sof_dev *sdev = dev_get_drvdata(dev); struct snd_soc_dai *d = params_data->dai; struct snd_soc_dapm_widget *w; - w = snd_soc_dai_get_widget(d, substream->stream); + w = snd_soc_dai_get_widget(d, params_data->stream); return sdw_dai_config_ipc(sdev, w, params_data->link_id, params_data->alh_stream_id, d->id, true); @@ -198,12 +197,11 @@ static int sdw_params_stream(struct device *dev, static int sdw_free_stream(struct device *dev, struct sdw_intel_stream_free_data *free_data) { - struct snd_pcm_substream *substream = free_data->substream; struct snd_sof_dev *sdev = dev_get_drvdata(dev); struct snd_soc_dai *d = free_data->dai; struct snd_soc_dapm_widget *w; - w = snd_soc_dai_get_widget(d, substream->stream); + w = snd_soc_dai_get_widget(d, free_data->stream); /* send invalid stream_id */ return sdw_dai_config_ipc(sdev, w, free_data->link_id, 0xFFFF, d->id, false); -- cgit v1.2.3-59-g8ed1b From 54bf7fa3efd08eea03e4bac04e188ee3db6173a7 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Mon, 13 Dec 2021 17:11:45 +0100 Subject: ima: Fix undefined arch_ima_get_secureboot() and co Currently arch_ima_get_secureboot() and arch_get_ima_policy() are defined only when CONFIG_IMA is set, and this makes any code calling those functions without CONFIG_IMA fail. Move the declaration and the dummy definition of those functions outside ifdef-CONFIG_IMA block for fixing the undefined symbols. Signed-off-by: Takashi Iwai [zohar@linux.ibm.com: removed in-tree/out-of-tree comment in patch description] Reviewed-by: Petr Vorel Signed-off-by: Mimi Zohar --- include/linux/ima.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ima.h b/include/linux/ima.h index b6ab66a546ae..426b1744215e 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -50,21 +50,6 @@ static inline void ima_appraise_parse_cmdline(void) {} extern void ima_add_kexec_buffer(struct kimage *image); #endif -#ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT -extern bool arch_ima_get_secureboot(void); -extern const char * const *arch_get_ima_policy(void); -#else -static inline bool arch_ima_get_secureboot(void) -{ - return false; -} - -static inline const char * const *arch_get_ima_policy(void) -{ - return NULL; -} -#endif - #else static inline enum hash_algo ima_get_current_hash_algo(void) { @@ -155,6 +140,21 @@ static inline int ima_measure_critical_data(const char *event_label, #endif /* CONFIG_IMA */ +#ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT +extern bool arch_ima_get_secureboot(void); +extern const char * const *arch_get_ima_policy(void); +#else +static inline bool arch_ima_get_secureboot(void) +{ + return false; +} + +static inline const char * const *arch_get_ima_policy(void) +{ + return NULL; +} +#endif + #ifndef CONFIG_IMA_KEXEC struct kimage; -- cgit v1.2.3-59-g8ed1b From 4fb0abfee424b05f0ec6d2d09e38f04ee2b82a8a Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 8 Nov 2021 15:51:21 -0600 Subject: x86/amd_nb: Add AMD Family 19h Models (10h-1Fh) and (A0h-AFh) PCI IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the new PCI Device IDs to support new generation of AMD 19h family of processors. Signed-off-by: Yazen Ghannam Signed-off-by: Babu Moger Acked-by: Krzysztof Wilczyński Acked-by: Borislav Petkov Acked-by: Bjorn Helgaas # pci_ids.h Link: https://lore.kernel.org/r/163640828133.955062.18349019796157170473.stgit@bmoger-ubuntu Signed-off-by: Guenter Roeck --- arch/x86/kernel/amd_nb.c | 5 +++++ include/linux/pci_ids.h | 1 + 2 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index c92c9c774c0e..f3e885f3dd0f 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -19,12 +19,14 @@ #define PCI_DEVICE_ID_AMD_17H_M10H_ROOT 0x15d0 #define PCI_DEVICE_ID_AMD_17H_M30H_ROOT 0x1480 #define PCI_DEVICE_ID_AMD_17H_M60H_ROOT 0x1630 +#define PCI_DEVICE_ID_AMD_19H_M10H_ROOT 0x14a4 #define PCI_DEVICE_ID_AMD_17H_DF_F4 0x1464 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4 0x15ec #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4 0x1494 #define PCI_DEVICE_ID_AMD_17H_M60H_DF_F4 0x144c #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4 0x1444 #define PCI_DEVICE_ID_AMD_19H_DF_F4 0x1654 +#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F4 0x14b1 #define PCI_DEVICE_ID_AMD_19H_M40H_ROOT 0x14b5 #define PCI_DEVICE_ID_AMD_19H_M40H_DF_F4 0x167d #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4 0x166e @@ -39,6 +41,7 @@ static const struct pci_device_id amd_root_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_ROOT) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_ROOT) }, {} }; @@ -61,6 +64,7 @@ static const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F3) }, {} @@ -78,6 +82,7 @@ static const struct pci_device_id amd_nb_link_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M10H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M40H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) }, diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 011f2f1ea5bb..b5248f27910e 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -555,6 +555,7 @@ #define PCI_DEVICE_ID_AMD_17H_M60H_DF_F3 0x144b #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F3 0x1443 #define PCI_DEVICE_ID_AMD_19H_DF_F3 0x1653 +#define PCI_DEVICE_ID_AMD_19H_M10H_DF_F3 0x14b0 #define PCI_DEVICE_ID_AMD_19H_M40H_DF_F3 0x167c #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F3 0x166d #define PCI_DEVICE_ID_AMD_CNB17H_F3 0x1703 -- cgit v1.2.3-59-g8ed1b From 11a24ca7e34d968991a7d437b950d1924396bd81 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 25 Nov 2021 03:08:38 +0100 Subject: hwmon: (ntc_thermistor) Merge platform data into driver Platform data is supposed to be used with "board files", device descriptions in C. Since the introduction of the NTC driver in 2011, no such platforms have been submitted to the Linux kernel, and their use is strongly discouraged in favor of Device Tree, ACPI or as last resort software firmware nodes. Drop the external header and copy the platform data into the driver file. Cc: Peter Rosin Cc: Chris Lesiak Signed-off-by: Linus Walleij Link: https://lore.kernel.org/r/20211125020841.3616359-2-linus.walleij@linaro.org Signed-off-by: Guenter Roeck --- drivers/hwmon/ntc_thermistor.c | 41 ++++++++++++++++++++--- include/linux/platform_data/ntc_thermistor.h | 50 ---------------------------- 2 files changed, 36 insertions(+), 55 deletions(-) delete mode 100644 include/linux/platform_data/ntc_thermistor.h (limited to 'include/linux') diff --git a/drivers/hwmon/ntc_thermistor.c b/drivers/hwmon/ntc_thermistor.c index cf26c44f2b88..034ef55d0706 100644 --- a/drivers/hwmon/ntc_thermistor.c +++ b/drivers/hwmon/ntc_thermistor.c @@ -14,12 +14,45 @@ #include #include #include +#include +#include -#include +enum ntc_thermistor_type { + TYPE_B57330V2103, + TYPE_B57891S0103, + TYPE_NCPXXWB473, + TYPE_NCPXXWF104, + TYPE_NCPXXWL333, + TYPE_NCPXXXH103, +}; -#include +struct ntc_thermistor_platform_data { + /* + * One (not both) of read_uV and read_ohm should be provided and only + * one of the two should be provided. + * Both functions should return negative value for an error case. + * + * pullup_uV, pullup_ohm, pulldown_ohm, and connect are required to use + * read_uV() + * + * How to setup pullup_ohm, pulldown_ohm, and connect is + * described at Documentation/hwmon/ntc_thermistor.rst + * + * pullup/down_ohm: 0 for infinite / not-connected + * + * chan: iio_channel pointer to communicate with the ADC which the + * thermistor is using for conversion of the analog values. + */ + int (*read_uv)(struct ntc_thermistor_platform_data *); + unsigned int pullup_uv; -#include + unsigned int pullup_ohm; + unsigned int pulldown_ohm; + enum { NTC_CONNECTED_POSITIVE, NTC_CONNECTED_GROUND } connect; + struct iio_channel *chan; + + int (*read_ohm)(void); +}; struct ntc_compensation { int temp_c; @@ -651,8 +684,6 @@ static int ntc_thermistor_probe(struct platform_device *pdev) pdata = ntc_thermistor_parse_dt(dev); if (IS_ERR(pdata)) return PTR_ERR(pdata); - else if (pdata == NULL) - pdata = dev_get_platdata(dev); if (!pdata) { dev_err(dev, "No platform init data supplied.\n"); diff --git a/include/linux/platform_data/ntc_thermistor.h b/include/linux/platform_data/ntc_thermistor.h deleted file mode 100644 index b324d03e580c..000000000000 --- a/include/linux/platform_data/ntc_thermistor.h +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * ntc_thermistor.h - NTC Thermistors - * - * Copyright (C) 2010 Samsung Electronics - * MyungJoo Ham - */ -#ifndef _LINUX_NTC_H -#define _LINUX_NTC_H - -struct iio_channel; - -enum ntc_thermistor_type { - TYPE_B57330V2103, - TYPE_B57891S0103, - TYPE_NCPXXWB473, - TYPE_NCPXXWF104, - TYPE_NCPXXWL333, - TYPE_NCPXXXH103, -}; - -struct ntc_thermistor_platform_data { - /* - * One (not both) of read_uV and read_ohm should be provided and only - * one of the two should be provided. - * Both functions should return negative value for an error case. - * - * pullup_uV, pullup_ohm, pulldown_ohm, and connect are required to use - * read_uV() - * - * How to setup pullup_ohm, pulldown_ohm, and connect is - * described at Documentation/hwmon/ntc_thermistor.rst - * - * pullup/down_ohm: 0 for infinite / not-connected - * - * chan: iio_channel pointer to communicate with the ADC which the - * thermistor is using for conversion of the analog values. - */ - int (*read_uv)(struct ntc_thermistor_platform_data *); - unsigned int pullup_uv; - - unsigned int pullup_ohm; - unsigned int pulldown_ohm; - enum { NTC_CONNECTED_POSITIVE, NTC_CONNECTED_GROUND } connect; - struct iio_channel *chan; - - int (*read_ohm)(void); -}; - -#endif /* _LINUX_NTC_H */ -- cgit v1.2.3-59-g8ed1b From 130d168866a11829b844ffdb19b9aefe384f754c Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 16 Dec 2021 16:42:57 +0100 Subject: hwmon: prefix kernel-doc comments for structs with struct The command ./scripts/kernel-doc -none include/linux/hwmon.h warns: include/linux/hwmon.h:406: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Channel information include/linux/hwmon.h:425: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Chip configuration Address those kernel-doc warnings by prefixing kernel-doc descriptions for structs with the keyword 'struct'. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20211216154257.26758-1-lukas.bulwahn@gmail.com Signed-off-by: Guenter Roeck --- include/linux/hwmon.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 1e8d6ea8992e..fad1f1df26df 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -403,7 +403,7 @@ struct hwmon_ops { }; /** - * Channel information + * struct hwmon_channel_info - Channel information * @type: Channel type. * @config: Pointer to NULL-terminated list of channel parameters. * Use for per-channel attributes. @@ -422,7 +422,7 @@ struct hwmon_channel_info { }) /** - * Chip configuration + * struct hwmon_chip_info - Chip configuration * @ops: Pointer to hwmon operations. * @info: Null-terminated list of channel information. */ -- cgit v1.2.3-59-g8ed1b From ee6d3dd4ed48ab24b74bab3c3977b8218518247d Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Fri, 24 Dec 2021 23:13:45 +0000 Subject: driver core: make kobj_type constant. This way instances of kobj_type (which contain function pointers) can be stored in .rodata, which means that they cannot be [easily/accidentally] modified at runtime. Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20211224231345.777370-1-wedsonaf@google.com Signed-off-by: Greg Kroah-Hartman --- Documentation/core-api/kobject.rst | 4 ++-- drivers/base/bus.c | 2 +- drivers/base/core.c | 2 +- include/linux/kobject.h | 8 ++++---- kernel/params.c | 2 +- lib/kobject.c | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/Documentation/core-api/kobject.rst b/Documentation/core-api/kobject.rst index 2739f8b72575..d3b5bf9f643a 100644 --- a/Documentation/core-api/kobject.rst +++ b/Documentation/core-api/kobject.rst @@ -118,7 +118,7 @@ Initialization of kobjects Code which creates a kobject must, of course, initialize that object. Some of the internal fields are setup with a (mandatory) call to kobject_init():: - void kobject_init(struct kobject *kobj, struct kobj_type *ktype); + void kobject_init(struct kobject *kobj, const struct kobj_type *ktype); The ktype is required for a kobject to be created properly, as every kobject must have an associated kobj_type. After calling kobject_init(), to @@ -156,7 +156,7 @@ kobject_name():: There is a helper function to both initialize and add the kobject to the kernel at the same time, called surprisingly enough kobject_init_and_add():: - int kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, + int kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...); The arguments are the same as the individual kobject_init() and diff --git a/drivers/base/bus.c b/drivers/base/bus.c index bdc98c5713d5..a64454f5f8c0 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -165,7 +165,7 @@ static struct kobj_type bus_ktype = { static int bus_uevent_filter(struct kset *kset, struct kobject *kobj) { - struct kobj_type *ktype = get_ktype(kobj); + const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &bus_ktype) return 1; diff --git a/drivers/base/core.c b/drivers/base/core.c index fd034d742447..d712ea11066b 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2263,7 +2263,7 @@ static struct kobj_type device_ktype = { static int dev_uevent_filter(struct kset *kset, struct kobject *kobj) { - struct kobj_type *ktype = get_ktype(kobj); + const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &device_ktype) { struct device *dev = kobj_to_dev(kobj); diff --git a/include/linux/kobject.h b/include/linux/kobject.h index c740062b4b1a..683172b2e094 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -66,7 +66,7 @@ struct kobject { struct list_head entry; struct kobject *parent; struct kset *kset; - struct kobj_type *ktype; + const struct kobj_type *ktype; struct kernfs_node *sd; /* sysfs directory entry */ struct kref kref; #ifdef CONFIG_DEBUG_KOBJECT_RELEASE @@ -90,13 +90,13 @@ static inline const char *kobject_name(const struct kobject *kobj) return kobj->name; } -extern void kobject_init(struct kobject *kobj, struct kobj_type *ktype); +extern void kobject_init(struct kobject *kobj, const struct kobj_type *ktype); extern __printf(3, 4) __must_check int kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...); extern __printf(4, 5) __must_check int kobject_init_and_add(struct kobject *kobj, - struct kobj_type *ktype, struct kobject *parent, + const struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...); extern void kobject_del(struct kobject *kobj); @@ -217,7 +217,7 @@ static inline void kset_put(struct kset *k) kobject_put(&k->kobj); } -static inline struct kobj_type *get_ktype(struct kobject *kobj) +static inline const struct kobj_type *get_ktype(struct kobject *kobj) { return kobj->ktype; } diff --git a/kernel/params.c b/kernel/params.c index 8299bd764e42..9b90e3c4d3c0 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -928,7 +928,7 @@ static const struct sysfs_ops module_sysfs_ops = { static int uevent_filter(struct kset *kset, struct kobject *kobj) { - struct kobj_type *ktype = get_ktype(kobj); + const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &module_ktype) return 1; diff --git a/lib/kobject.c b/lib/kobject.c index 4a56f519139d..56fa037501b5 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -65,7 +65,7 @@ void kobject_get_ownership(struct kobject *kobj, kuid_t *uid, kgid_t *gid) */ static int populate_dir(struct kobject *kobj) { - struct kobj_type *t = get_ktype(kobj); + const struct kobj_type *t = get_ktype(kobj); struct attribute *attr; int error = 0; int i; @@ -346,7 +346,7 @@ EXPORT_SYMBOL(kobject_set_name); * to kobject_put(), not by a call to kfree directly to ensure that all of * the memory is cleaned up properly. */ -void kobject_init(struct kobject *kobj, struct kobj_type *ktype) +void kobject_init(struct kobject *kobj, const struct kobj_type *ktype) { char *err_str; @@ -461,7 +461,7 @@ EXPORT_SYMBOL(kobject_add); * same type of error handling after a call to kobject_add() and kobject * lifetime rules are the same here. */ -int kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, +int kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...) { va_list args; @@ -679,7 +679,7 @@ EXPORT_SYMBOL(kobject_get_unless_zero); static void kobject_cleanup(struct kobject *kobj) { struct kobject *parent = kobj->parent; - struct kobj_type *t = get_ktype(kobj); + const struct kobj_type *t = get_ktype(kobj); const char *name = kobj->name; pr_debug("kobject: '%s' (%p): %s, parent %p\n", -- cgit v1.2.3-59-g8ed1b From 1882de7fc56c2b0ea91dd9fd9922d434fc3feb15 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 22 Dec 2021 12:31:03 +0800 Subject: efi: Introduce EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER and corresponding structures Platform Firmware Runtime Update image starts with UEFI headers, and the headers are defined in UEFI specification, but some of them have not been defined in the kernel yet. For example, the header layout of a capsule file looks like this: EFI_CAPSULE_HEADER EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER EFI_FIRMWARE_IMAGE_AUTHENTICATION These structures would be used by the Platform Firmware Runtime Update driver to parse the format of capsule file to verify if the corresponding version number is valid. In this way, if the user provides an invalid capsule image, the kernel could be used as a guard to reject it, without switching to the Management Mode (which might be costly). EFI_CAPSULE_HEADER has been defined in the kernel, but the other structures have not been defined yet, so do that. Besides, EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER and EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER are required to be packed in the uefi specification. For this reason, use the __packed attribute to indicate to the compiler that the entire structure can appear misaligned in memory (as suggested by Ard) in case one of them follows the other directly in a capsule header. Acked-by: Ard Biesheuvel Signed-off-by: Chen Yu Signed-off-by: Rafael J. Wysocki --- include/linux/efi.h | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index dbd39b20e034..80e970f7e6f8 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -148,6 +148,52 @@ typedef struct { u32 imagesize; } efi_capsule_header_t; +/* EFI_FIRMWARE_MANAGEMENT_CAPSULE_HEADER */ +struct efi_manage_capsule_header { + u32 ver; + u16 emb_drv_cnt; + u16 payload_cnt; + /* + * Variable-size array of the size given by the sum of + * emb_drv_cnt and payload_cnt. + */ + u64 offset_list[]; +} __packed; + +/* EFI_FIRMWARE_MANAGEMENT_CAPSULE_IMAGE_HEADER */ +struct efi_manage_capsule_image_header { + u32 ver; + efi_guid_t image_type_id; + u8 image_index; + u8 reserved_bytes[3]; + u32 image_size; + u32 vendor_code_size; + /* hw_ins was introduced in version 2 */ + u64 hw_ins; + /* capsule_support was introduced in version 3 */ + u64 capsule_support; +} __packed; + +/* WIN_CERTIFICATE */ +struct win_cert { + u32 len; + u16 rev; + u16 cert_type; +}; + +/* WIN_CERTIFICATE_UEFI_GUID */ +struct win_cert_uefi_guid { + struct win_cert hdr; + efi_guid_t cert_type; + u8 cert_data[]; +}; + +/* EFI_FIRMWARE_IMAGE_AUTHENTICATION */ +struct efi_image_auth { + u64 mon_count; + struct win_cert_uefi_guid auth_info; +}; + /* * EFI capsule flags */ -- cgit v1.2.3-59-g8ed1b From cf6299b6101903c31bddb0065804b2121ed510c7 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 27 Dec 2021 17:39:24 +0100 Subject: kobject: remove kset from struct kset_uevent_ops callbacks There is no need to pass the pointer to the kset in the struct kset_uevent_ops callbacks as no one uses it, so just remove that pointer entirely. Reviewed-by: Rafael J. Wysocki Reviewed-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20211227163924.3970661-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- Documentation/core-api/kobject.rst | 7 +++---- Documentation/translations/zh_CN/core-api/kobject.rst | 7 +++---- drivers/base/bus.c | 2 +- drivers/base/core.c | 11 +++++------ drivers/dma-buf/dma-buf-sysfs-stats.c | 2 +- fs/dlm/lockspace.c | 3 +-- fs/gfs2/sys.c | 3 +-- include/linux/kobject.h | 7 +++---- kernel/params.c | 2 +- lib/kobject_uevent.c | 6 +++--- 10 files changed, 22 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/Documentation/core-api/kobject.rst b/Documentation/core-api/kobject.rst index d3b5bf9f643a..3d6e3107315d 100644 --- a/Documentation/core-api/kobject.rst +++ b/Documentation/core-api/kobject.rst @@ -373,10 +373,9 @@ If a kset wishes to control the uevent operations of the kobjects associated with it, it can use the struct kset_uevent_ops to handle it:: struct kset_uevent_ops { - int (* const filter)(struct kset *kset, struct kobject *kobj); - const char *(* const name)(struct kset *kset, struct kobject *kobj); - int (* const uevent)(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env); + int (* const filter)(struct kobject *kobj); + const char *(* const name)(struct kobject *kobj); + int (* const uevent)(struct kobject *kobj, struct kobj_uevent_env *env); }; diff --git a/Documentation/translations/zh_CN/core-api/kobject.rst b/Documentation/translations/zh_CN/core-api/kobject.rst index b7c37794cc7f..95634083dca0 100644 --- a/Documentation/translations/zh_CN/core-api/kobject.rst +++ b/Documentation/translations/zh_CN/core-api/kobject.rst @@ -325,10 +325,9 @@ ksets 结构体kset_uevent_ops来处理它:: struct kset_uevent_ops { - int (* const filter)(struct kset *kset, struct kobject *kobj); - const char *(* const name)(struct kset *kset, struct kobject *kobj); - int (* const uevent)(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env); + int (* const filter)(struct kobject *kobj); + const char *(* const name)(struct kobject *kobj); + int (* const uevent)(struct kobject *kobj, struct kobj_uevent_env *env); }; diff --git a/drivers/base/bus.c b/drivers/base/bus.c index a64454f5f8c0..97936ec49bde 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -163,7 +163,7 @@ static struct kobj_type bus_ktype = { .release = bus_release, }; -static int bus_uevent_filter(struct kset *kset, struct kobject *kobj) +static int bus_uevent_filter(struct kobject *kobj) { const struct kobj_type *ktype = get_ktype(kobj); diff --git a/drivers/base/core.c b/drivers/base/core.c index d712ea11066b..60d703ebd123 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2261,7 +2261,7 @@ static struct kobj_type device_ktype = { }; -static int dev_uevent_filter(struct kset *kset, struct kobject *kobj) +static int dev_uevent_filter(struct kobject *kobj) { const struct kobj_type *ktype = get_ktype(kobj); @@ -2275,7 +2275,7 @@ static int dev_uevent_filter(struct kset *kset, struct kobject *kobj) return 0; } -static const char *dev_uevent_name(struct kset *kset, struct kobject *kobj) +static const char *dev_uevent_name(struct kobject *kobj) { struct device *dev = kobj_to_dev(kobj); @@ -2286,8 +2286,7 @@ static const char *dev_uevent_name(struct kset *kset, struct kobject *kobj) return NULL; } -static int dev_uevent(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env) +static int dev_uevent(struct kobject *kobj, struct kobj_uevent_env *env) { struct device *dev = kobj_to_dev(kobj); int retval = 0; @@ -2382,7 +2381,7 @@ static ssize_t uevent_show(struct device *dev, struct device_attribute *attr, /* respect filter */ if (kset->uevent_ops && kset->uevent_ops->filter) - if (!kset->uevent_ops->filter(kset, &dev->kobj)) + if (!kset->uevent_ops->filter(&dev->kobj)) goto out; env = kzalloc(sizeof(struct kobj_uevent_env), GFP_KERNEL); @@ -2390,7 +2389,7 @@ static ssize_t uevent_show(struct device *dev, struct device_attribute *attr, return -ENOMEM; /* let the kset specific function add its keys */ - retval = kset->uevent_ops->uevent(kset, &dev->kobj, env); + retval = kset->uevent_ops->uevent(&dev->kobj, env); if (retval) goto out; diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.c b/drivers/dma-buf/dma-buf-sysfs-stats.c index 053baadcada9..2bba0babcb62 100644 --- a/drivers/dma-buf/dma-buf-sysfs-stats.c +++ b/drivers/dma-buf/dma-buf-sysfs-stats.c @@ -132,7 +132,7 @@ void dma_buf_stats_teardown(struct dma_buf *dmabuf) /* Statistics files do not need to send uevents. */ -static int dmabuf_sysfs_uevent_filter(struct kset *kset, struct kobject *kobj) +static int dmabuf_sysfs_uevent_filter(struct kobject *kobj) { return 0; } diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index 10eddfa6c3d7..0bbb346cb892 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -216,8 +216,7 @@ static int do_uevent(struct dlm_ls *ls, int in) return ls->ls_uevent_result; } -static int dlm_uevent(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env) +static int dlm_uevent(struct kobject *kobj, struct kobj_uevent_env *env) { struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); diff --git a/fs/gfs2/sys.c b/fs/gfs2/sys.c index c0a34d9ddee4..a6002b2d146d 100644 --- a/fs/gfs2/sys.c +++ b/fs/gfs2/sys.c @@ -767,8 +767,7 @@ void gfs2_sys_fs_del(struct gfs2_sbd *sdp) wait_for_completion(&sdp->sd_kobj_unregister); } -static int gfs2_uevent(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env) +static int gfs2_uevent(struct kobject *kobj, struct kobj_uevent_env *env) { struct gfs2_sbd *sdp = container_of(kobj, struct gfs2_sbd, sd_kobj); struct super_block *s = sdp->sd_vfs; diff --git a/include/linux/kobject.h b/include/linux/kobject.h index 683172b2e094..ad90b49824dc 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -153,10 +153,9 @@ struct kobj_uevent_env { }; struct kset_uevent_ops { - int (* const filter)(struct kset *kset, struct kobject *kobj); - const char *(* const name)(struct kset *kset, struct kobject *kobj); - int (* const uevent)(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env); + int (* const filter)(struct kobject *kobj); + const char *(* const name)(struct kobject *kobj); + int (* const uevent)(struct kobject *kobj, struct kobj_uevent_env *env); }; struct kobj_attribute { diff --git a/kernel/params.c b/kernel/params.c index 9b90e3c4d3c0..5b92310425c5 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -926,7 +926,7 @@ static const struct sysfs_ops module_sysfs_ops = { .store = module_attr_store, }; -static int uevent_filter(struct kset *kset, struct kobject *kobj) +static int uevent_filter(struct kobject *kobj) { const struct kobj_type *ktype = get_ktype(kobj); diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index c87d5b6a8a55..7c44b7ae4c5c 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -501,7 +501,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, } /* skip the event, if the filter returns zero. */ if (uevent_ops && uevent_ops->filter) - if (!uevent_ops->filter(kset, kobj)) { + if (!uevent_ops->filter(kobj)) { pr_debug("kobject: '%s' (%p): %s: filter function " "caused the event to drop!\n", kobject_name(kobj), kobj, __func__); @@ -510,7 +510,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, /* originating subsystem */ if (uevent_ops && uevent_ops->name) - subsystem = uevent_ops->name(kset, kobj); + subsystem = uevent_ops->name(kobj); else subsystem = kobject_name(&kset->kobj); if (!subsystem) { @@ -554,7 +554,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, /* let the kset specific function add its stuff */ if (uevent_ops && uevent_ops->uevent) { - retval = uevent_ops->uevent(kset, kobj, env); + retval = uevent_ops->uevent(kobj, env); if (retval) { pr_debug("kobject: '%s' (%p): %s: uevent() returned " "%d\n", kobject_name(kobj), kobj, -- cgit v1.2.3-59-g8ed1b From d6b9c679bbac1d1d2fcac64391b4cadb91763a6f Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 12 Nov 2021 14:46:32 -0800 Subject: watchdog: bcm7038_wdt: Support platform data configuration The BCM7038 watchdog driver needs to be able to obtain a specific clock name on BCM63xx platforms which is the "periph" clock ticking at 50MHz. make it possible to specify the clock name to obtain via platform data. Signed-off-by: Florian Fainelli Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20211112224636.395101-4-f.fainelli@gmail.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/bcm7038_wdt.c | 8 +++++++- include/linux/platform_data/bcm7038_wdt.h | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 include/linux/platform_data/bcm7038_wdt.h (limited to 'include/linux') diff --git a/drivers/watchdog/bcm7038_wdt.c b/drivers/watchdog/bcm7038_wdt.c index acaaa0005d5b..506cd7ef9c77 100644 --- a/drivers/watchdog/bcm7038_wdt.c +++ b/drivers/watchdog/bcm7038_wdt.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -133,8 +134,10 @@ static void bcm7038_clk_disable_unprepare(void *data) static int bcm7038_wdt_probe(struct platform_device *pdev) { + struct bcm7038_wdt_platform_data *pdata = pdev->dev.platform_data; struct device *dev = &pdev->dev; struct bcm7038_watchdog *wdt; + const char *clk_name = NULL; int err; wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL); @@ -147,7 +150,10 @@ static int bcm7038_wdt_probe(struct platform_device *pdev) if (IS_ERR(wdt->base)) return PTR_ERR(wdt->base); - wdt->clk = devm_clk_get(dev, NULL); + if (pdata && pdata->clk_name) + clk_name = pdata->clk_name; + + wdt->clk = devm_clk_get(dev, clk_name); /* If unable to get clock, use default frequency */ if (!IS_ERR(wdt->clk)) { err = clk_prepare_enable(wdt->clk); diff --git a/include/linux/platform_data/bcm7038_wdt.h b/include/linux/platform_data/bcm7038_wdt.h new file mode 100644 index 000000000000..e18cfd9ec8f9 --- /dev/null +++ b/include/linux/platform_data/bcm7038_wdt.h @@ -0,0 +1,8 @@ +#ifndef __BCM7038_WDT_PDATA_H +#define __BCM7038_WDT_PDATA_H + +struct bcm7038_wdt_platform_data { + const char *clk_name; +}; + +#endif /* __BCM7038_WDT_PDATA_H */ -- cgit v1.2.3-59-g8ed1b From b92e301633f0f454aa1cfedac2e096bb9649b367 Mon Sep 17 00:00:00 2001 From: Jonathan Neuschäfer Date: Sat, 18 Dec 2021 16:25:53 +0100 Subject: mfd: ntxec: Change return type of ntxec_reg8 from __be16 to u16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register values in NTXEC are big-endian on the I2C bus, but the regmap subsystem handles the conversion between CPU-endian and big-endian data internally. ntxec_reg8 should thus return u16, not __be16. Reported-by: kernel test robot Signed-off-by: Jonathan Neuschäfer Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20211218152553.744615-1-j.neuschaefer@gmx.net --- include/linux/mfd/ntxec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mfd/ntxec.h b/include/linux/mfd/ntxec.h index 26ab3b8eb612..cc6f07bfa2b3 100644 --- a/include/linux/mfd/ntxec.h +++ b/include/linux/mfd/ntxec.h @@ -26,7 +26,7 @@ struct ntxec { * This convenience function converts an 8-bit value to 16-bit for use in the * second kind of register. */ -static inline __be16 ntxec_reg8(u8 value) +static inline u16 ntxec_reg8(u8 value) { return value << 8; } -- cgit v1.2.3-59-g8ed1b From b6459415b384cb829f0b2a4268f211c789f6cf0b Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 28 Dec 2021 16:49:13 -0800 Subject: net: Don't include filter.h from net/sock.h sock.h is pretty heavily used (5k objects rebuilt on x86 after it's touched). We can drop the include of filter.h from it and add a forward declaration of struct sk_filter instead. This decreases the number of rebuilt objects when bpf.h is touched from ~5k to ~1k. There's a lot of missing includes this was masking. Primarily in networking tho, this time. Signed-off-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov Acked-by: Marc Kleine-Budde Acked-by: Florian Fainelli Acked-by: Nikolay Aleksandrov Acked-by: Stefano Garzarella Link: https://lore.kernel.org/bpf/20211229004913.513372-1-kuba@kernel.org --- drivers/bluetooth/btqca.c | 1 + drivers/infiniband/core/cache.c | 1 + drivers/infiniband/hw/irdma/ctrl.c | 2 ++ drivers/infiniband/hw/irdma/uda.c | 2 ++ drivers/infiniband/hw/mlx5/doorbell.c | 1 + drivers/infiniband/hw/mlx5/qp.c | 1 + drivers/net/amt.c | 1 + drivers/net/appletalk/ipddp.c | 1 + drivers/net/bonding/bond_main.c | 1 + drivers/net/can/usb/peak_usb/pcan_usb.c | 1 + drivers/net/dsa/microchip/ksz8795.c | 1 + drivers/net/dsa/xrs700x/xrs700x.c | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c | 1 + drivers/net/ethernet/huawei/hinic/hinic_tx.c | 1 + drivers/net/ethernet/intel/ice/ice_devlink.c | 2 ++ drivers/net/ethernet/intel/ice/ice_txrx_lib.c | 2 ++ drivers/net/ethernet/intel/igc/igc_xdp.c | 1 + drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 1 + drivers/net/ethernet/mellanox/mlx5/core/en/qos.c | 1 + drivers/net/ethernet/sfc/efx.c | 1 + drivers/net/ethernet/sfc/efx_channels.c | 1 + drivers/net/ethernet/sfc/efx_common.c | 1 + drivers/net/hamradio/hdlcdrv.c | 1 + drivers/net/hamradio/scc.c | 1 + drivers/net/loopback.c | 1 + drivers/net/vrf.c | 1 + drivers/net/wireless/ath/ath11k/debugfs.c | 2 ++ drivers/net/wireless/realtek/rtw89/debug.c | 2 ++ fs/nfs/dir.c | 1 + fs/nfs/fs_context.c | 1 + fs/select.c | 1 + include/linux/bpf_local_storage.h | 1 + include/linux/dsa/loop.h | 1 + include/net/ipv6.h | 2 ++ include/net/route.h | 1 + include/net/sock.h | 2 +- include/net/xdp_sock.h | 1 + kernel/sysctl.c | 1 + net/bluetooth/bnep/sock.c | 1 + net/bluetooth/eir.h | 2 ++ net/bluetooth/hidp/sock.c | 1 + net/bluetooth/l2cap_sock.c | 1 + net/bridge/br_ioctl.c | 1 + net/caif/caif_socket.c | 1 + net/core/devlink.c | 1 + net/core/flow_dissector.c | 1 + net/core/lwt_bpf.c | 1 + net/core/sock_diag.c | 1 + net/core/sysctl_net_core.c | 1 + net/decnet/dn_nsp_in.c | 1 + net/dsa/dsa_priv.h | 1 + net/ethtool/ioctl.c | 1 + net/ipv4/nexthop.c | 1 + net/ipv6/ip6_fib.c | 1 + net/ipv6/seg6_local.c | 1 + net/iucv/af_iucv.c | 1 + net/kcm/kcmsock.c | 1 + net/netfilter/nfnetlink_hook.c | 1 + net/netfilter/nft_reject_netdev.c | 1 + net/netlink/af_netlink.c | 2 ++ net/packet/af_packet.c | 1 + net/rose/rose_in.c | 1 + net/sched/sch_frag.c | 1 + net/smc/smc_ib.c | 2 ++ net/smc/smc_ism.c | 1 + net/unix/af_unix.c | 1 + net/vmw_vsock/af_vsock.c | 1 + net/xdp/xskmap.c | 1 + net/xfrm/xfrm_state.c | 1 + net/xfrm/xfrm_user.c | 1 + 70 files changed, 80 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index be04d74037d2..f7bf311d7910 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 0c98dd3dee67..b79f816a7203 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -33,6 +33,7 @@ * SOFTWARE. */ +#include #include #include #include diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 7264f8c2f7d5..3141a9c85de5 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB /* Copyright (c) 2015 - 2021 Intel Corporation */ +#include + #include "osdep.h" #include "status.h" #include "hmc.h" diff --git a/drivers/infiniband/hw/irdma/uda.c b/drivers/infiniband/hw/irdma/uda.c index f5b1b6150cdc..7a9988ddbd01 100644 --- a/drivers/infiniband/hw/irdma/uda.c +++ b/drivers/infiniband/hw/irdma/uda.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB /* Copyright (c) 2016 - 2021 Intel Corporation */ +#include + #include "osdep.h" #include "status.h" #include "hmc.h" diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c index 6398e2f48579..e32111117a5e 100644 --- a/drivers/infiniband/hw/mlx5/doorbell.c +++ b/drivers/infiniband/hw/mlx5/doorbell.c @@ -32,6 +32,7 @@ #include #include +#include #include #include "mlx5_ib.h" diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index e5abbcfc1d57..29475cf8c7c3 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -30,6 +30,7 @@ * SOFTWARE. */ +#include #include #include #include diff --git a/drivers/net/amt.c b/drivers/net/amt.c index b732ee9a50ef..a1c7a8acd9c8 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/appletalk/ipddp.c b/drivers/net/appletalk/ipddp.c index 5566daefbff4..d558535390f9 100644 --- a/drivers/net/appletalk/ipddp.c +++ b/drivers/net/appletalk/ipddp.c @@ -23,6 +23,7 @@ * of the GNU General Public License, incorporated herein by reference. */ +#include #include #include #include diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 0f39ad2af81c..d483f1102a9e 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c index 876218752766..ac6772fe9746 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb.c @@ -8,6 +8,7 @@ * * Many thanks to Klaus Hitschler */ +#include #include #include #include diff --git a/drivers/net/dsa/microchip/ksz8795.c b/drivers/net/dsa/microchip/ksz8795.c index 013e9c02be71..991b9c6b6ce7 100644 --- a/drivers/net/dsa/microchip/ksz8795.c +++ b/drivers/net/dsa/microchip/ksz8795.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index 35fa19ddaf19..0730352cdd57 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c index 951c4c569a9b..4da31b1b84f9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c @@ -9,6 +9,7 @@ #include #include +#include #include #include "bnxt_hsi.h" #include "bnxt.h" diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c index a984a7a6dd2e..8d59babbf476 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c @@ -4,6 +4,7 @@ * Copyright(c) 2017 Huawei Technologies Co., Ltd */ +#include #include #include #include diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index 1cfe918db8b9..716ec8616ff0 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2020, Intel Corporation. */ +#include + #include "ice.h" #include "ice_lib.h" #include "ice_devlink.h" diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index 1dd7e84f41f8..9520b140bdbf 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2019, Intel Corporation. */ +#include + #include "ice_txrx_lib.h" #include "ice_eswitch.h" #include "ice_lib.h" diff --git a/drivers/net/ethernet/intel/igc/igc_xdp.c b/drivers/net/ethernet/intel/igc/igc_xdp.c index a8cf5374be47..aeeb34e64610 100644 --- a/drivers/net/ethernet/intel/igc/igc_xdp.c +++ b/drivers/net/ethernet/intel/igc/igc_xdp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2020, Intel Corporation. */ +#include #include #include "igc.h" diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index f1c10f2bda78..40acfe12adc9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 50977f01a050..00449df98a5e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ +#include #include "en.h" #include "params.h" diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index a8c252e2b252..302dc835ac3d 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -5,6 +5,7 @@ * Copyright 2005-2013 Solarflare Communications Inc. */ +#include #include #include #include diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 3dbea028b325..b015d1f2e204 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -10,6 +10,7 @@ #include "net_driver.h" #include +#include #include "efx_channels.h" #include "efx.h" #include "efx_common.h" diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index f187631b2c5c..af37c990217e 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -9,6 +9,7 @@ */ #include "net_driver.h" +#include #include #include #include diff --git a/drivers/net/hamradio/hdlcdrv.c b/drivers/net/hamradio/hdlcdrv.c index b0edb91bb10a..8297411e87ea 100644 --- a/drivers/net/hamradio/hdlcdrv.c +++ b/drivers/net/hamradio/hdlcdrv.c @@ -30,6 +30,7 @@ /*****************************************************************************/ #include +#include #include #include #include diff --git a/drivers/net/hamradio/scc.c b/drivers/net/hamradio/scc.c index 3d59dac063ac..f90830d3dfa6 100644 --- a/drivers/net/hamradio/scc.c +++ b/drivers/net/hamradio/scc.c @@ -148,6 +148,7 @@ /* ----------------------------------------------------------------------- */ +#include #include #include #include diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index a1c77cc00416..ed0edf5884ef 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include /* For the statistics structure. */ diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index b4c64226b7ca..e0b1ab99a359 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c index dba055d085be..eb8b4f20c95e 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.c +++ b/drivers/net/wireless/ath/ath11k/debugfs.c @@ -3,6 +3,8 @@ * Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. */ +#include + #include "debugfs.h" #include "core.h" diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 1e85808aaf4b..be761157ea22 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -2,6 +2,8 @@ /* Copyright(c) 2019-2020 Realtek Corporation */ +#include + #include "coex.h" #include "debug.h" #include "fw.h" diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 731d31015b6a..347793626f19 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -18,6 +18,7 @@ * 6 Jun 1999 Cache readdir lookups in the page cache. -DaveM */ +#include #include #include #include diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 0d444a90f513..ea17fa1f31ec 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -10,6 +10,7 @@ * Split from fs/nfs/super.c by David Howells */ +#include #include #include #include diff --git a/fs/select.c b/fs/select.c index 945896d0ac9e..02cd8cb5e69f 100644 --- a/fs/select.c +++ b/fs/select.c @@ -15,6 +15,7 @@ * of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian). */ +#include #include #include #include diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 24496bc28e7b..a2b625960ffe 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -8,6 +8,7 @@ #define _BPF_LOCAL_STORAGE_H #include +#include #include #include #include diff --git a/include/linux/dsa/loop.h b/include/linux/dsa/loop.h index 5a3470bcc8a7..b8fef35591aa 100644 --- a/include/linux/dsa/loop.h +++ b/include/linux/dsa/loop.h @@ -2,6 +2,7 @@ #ifndef DSA_LOOP_H #define DSA_LOOP_H +#include #include #include #include diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 53ac7707ca70..3afcb128e064 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -21,6 +21,8 @@ #include #include +struct ip_tunnel_info; + #define SIN6_LEN_RFC2133 24 #define IPV6_MAXPLEN 65535 diff --git a/include/net/route.h b/include/net/route.h index 2e6c0e153e3a..4c858dcf1aa8 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -43,6 +43,7 @@ #define RT_CONN_FLAGS(sk) (RT_TOS(inet_sk(sk)->tos) | sock_flag(sk, SOCK_LOCALROUTE)) #define RT_CONN_FLAGS_TOS(sk,tos) (RT_TOS(tos) | sock_flag(sk, SOCK_LOCALROUTE)) +struct ip_tunnel_info; struct fib_nh; struct fib_info; struct uncached_list; diff --git a/include/net/sock.h b/include/net/sock.h index 37f878564d25..40f6406b9ca5 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -249,6 +248,7 @@ struct sock_common { }; struct bpf_local_storage; +struct sk_filter; /** * struct sock - network layer representation of sockets diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index fff069d2ed1b..3057e1a4a11c 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -6,6 +6,7 @@ #ifndef _LINUX_XDP_SOCK_H #define _LINUX_XDP_SOCK_H +#include #include #include #include diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 083be6af29d7..d7ed1dffa426 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/net/bluetooth/bnep/sock.c b/net/bluetooth/bnep/sock.c index d515571b2afb..57d509d77cb4 100644 --- a/net/bluetooth/bnep/sock.c +++ b/net/bluetooth/bnep/sock.c @@ -24,6 +24,7 @@ SOFTWARE IS DISCLAIMED. */ +#include #include #include diff --git a/net/bluetooth/eir.h b/net/bluetooth/eir.h index 724662f8f8b1..05e2e917fc25 100644 --- a/net/bluetooth/eir.h +++ b/net/bluetooth/eir.h @@ -5,6 +5,8 @@ * Copyright (C) 2021 Intel Corporation */ +#include + void eir_create(struct hci_dev *hdev, u8 *data); u8 eir_create_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr); diff --git a/net/bluetooth/hidp/sock.c b/net/bluetooth/hidp/sock.c index 595fb3c9d6c3..369ed92dac99 100644 --- a/net/bluetooth/hidp/sock.c +++ b/net/bluetooth/hidp/sock.c @@ -20,6 +20,7 @@ SOFTWARE IS DISCLAIMED. */ +#include #include #include diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 4574c5cb1b59..dc50737b785b 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -29,6 +29,7 @@ #include #include +#include #include #include diff --git a/net/bridge/br_ioctl.c b/net/bridge/br_ioctl.c index db4ab2c2ce18..9b54d7d0bfc4 100644 --- a/net/bridge/br_ioctl.c +++ b/net/bridge/br_ioctl.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include diff --git a/net/caif/caif_socket.c b/net/caif/caif_socket.c index e12fd3cad619..2b8892d502f7 100644 --- a/net/caif/caif_socket.c +++ b/net/caif/caif_socket.c @@ -6,6 +6,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ":%s(): " fmt, __func__ +#include #include #include #include diff --git a/net/core/devlink.c b/net/core/devlink.c index 0a9349a02cad..492a26d3c3f1 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -7,6 +7,7 @@ * Copyright (c) 2016 Jiri Pirko */ +#include #include #include #include diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 257976cb55ce..de1109f2cfcf 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index 2f7940bcf715..349480ef68a5 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -2,6 +2,7 @@ /* Copyright (c) 2016 Thomas Graf */ +#include #include #include #include diff --git a/net/core/sock_diag.c b/net/core/sock_diag.c index c9c45b935f99..f7cf74cdd3db 100644 --- a/net/core/sock_diag.c +++ b/net/core/sock_diag.c @@ -1,5 +1,6 @@ /* License: GPL */ +#include #include #include #include diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 5f88526ad61c..7b4d485aac7a 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -6,6 +6,7 @@ * Added /proc/sys/net/core directory entry (empty =) ). [MS] */ +#include #include #include #include diff --git a/net/decnet/dn_nsp_in.c b/net/decnet/dn_nsp_in.c index 7ab788f41a3f..c59be5b04479 100644 --- a/net/decnet/dn_nsp_in.c +++ b/net/decnet/dn_nsp_in.c @@ -38,6 +38,7 @@ *******************************************************************************/ #include +#include #include #include #include diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index 38ce5129a33d..0194a969c9b5 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -8,6 +8,7 @@ #define __DSA_PRIV_H #include +#include #include #include #include diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 9a113d893521..b2cdba1b4aae 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 1319d093cdda..eeafeccebb8d 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 0371d2c14145..463c37dea449 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -15,6 +15,7 @@ #define pr_fmt(fmt) "IPv6: " fmt +#include #include #include #include diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index 2dc40b3f373e..a5eea182149d 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -7,6 +7,7 @@ * eBPF support: Mathieu Xhonneux */ +#include #include #include #include diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index 49ecbe8d176a..a1760add5bf1 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -13,6 +13,7 @@ #define KMSG_COMPONENT "af_iucv" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include #include #include #include diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 11a715d76a4f..71899e5a5a11 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c index d5c719c9e36c..71e29adac48b 100644 --- a/net/netfilter/nfnetlink_hook.c +++ b/net/netfilter/nfnetlink_hook.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include diff --git a/net/netfilter/nft_reject_netdev.c b/net/netfilter/nft_reject_netdev.c index d89f68754f42..61cd8c4ac385 100644 --- a/net/netfilter/nft_reject_netdev.c +++ b/net/netfilter/nft_reject_netdev.c @@ -4,6 +4,7 @@ * Copyright (c) 2020 Jose M. Guisado */ +#include #include #include #include diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 4be2d97ff93e..7b344035bfe3 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -20,8 +20,10 @@ #include +#include #include #include +#include #include #include #include diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index a1ffdb48cc47..3ca4f890371a 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -49,6 +49,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include diff --git a/net/rose/rose_in.c b/net/rose/rose_in.c index 6af786d66b03..4d67f36dce1b 100644 --- a/net/rose/rose_in.c +++ b/net/rose/rose_in.c @@ -9,6 +9,7 @@ * diagrams as the code is not obvious and probably very easy to break. */ #include +#include #include #include #include diff --git a/net/sched/sch_frag.c b/net/sched/sch_frag.c index 8c06381391d6..cd85a69820b1 100644 --- a/net/sched/sch_frag.c +++ b/net/sched/sch_frag.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +#include #include #include #include diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index d93055ec17ae..905604c378ad 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -12,6 +12,8 @@ * Author(s): Ursula Braun */ +#include +#include #include #include #include diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c index fd28cc498b98..a2084ecdb97e 100644 --- a/net/smc/smc_ism.c +++ b/net/smc/smc_ism.c @@ -6,6 +6,7 @@ * Copyright IBM Corp. 2018 */ +#include #include #include #include diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 4d6e33bbd446..c19569819866 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -89,6 +89,7 @@ #include #include #include +#include #include #include #include diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index ed0df839c38c..3235261f138d 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -85,6 +85,7 @@ * TCP_LISTEN - listening */ +#include #include #include #include diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c index 2e48d0e094d9..65b53fb3de13 100644 --- a/net/xdp/xskmap.c +++ b/net/xdp/xskmap.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index a2f4001221d1..0407272a990c 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -14,6 +14,7 @@ * */ +#include #include #include #include diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 7c36cc1f3d79..e3e26f4da6c2 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -11,6 +11,7 @@ * */ +#include #include #include #include -- cgit v1.2.3-59-g8ed1b From 0fe4b381a59ebc53522fce579b281a67a9e1bee6 Mon Sep 17 00:00:00 2001 From: KP Singh Date: Fri, 24 Dec 2021 15:29:15 +0000 Subject: bpf: Allow bpf_local_storage to be used by sleepable programs Other maps like hashmaps are already available to sleepable programs. Sleepable BPF programs run under trace RCU. Allow task, sk and inode storage to be used from sleepable programs. This allows sleepable and non-sleepable programs to provide shareable annotations on kernel objects. Sleepable programs run in trace RCU where as non-sleepable programs run in a normal RCU critical section i.e. __bpf_prog_enter{_sleepable} and __bpf_prog_exit{_sleepable}) (rcu_read_lock or rcu_read_lock_trace). In order to make the local storage maps accessible to both sleepable and non-sleepable programs, one needs to call both call_rcu_tasks_trace and call_rcu to wait for both trace and classical RCU grace periods to expire before freeing memory. Paul's work on call_rcu_tasks_trace allows us to have per CPU queueing for call_rcu_tasks_trace. This behaviour can be achieved by setting rcupdate.rcu_task_enqueue_lim= boot parameter. In light of these new performance changes and to keep the local storage code simple, avoid adding a new flag for sleepable maps / local storage to select the RCU synchronization (trace / classical). Also, update the dereferencing of the pointers to use rcu_derference_check (with either the trace or normal RCU locks held) with a common bpf_rcu_lock_held helper method. Signed-off-by: KP Singh Signed-off-by: Alexei Starovoitov Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20211224152916.1550677-2-kpsingh@kernel.org --- include/linux/bpf_local_storage.h | 5 ++++ kernel/bpf/bpf_inode_storage.c | 6 ++++- kernel/bpf/bpf_local_storage.c | 50 +++++++++++++++++++++++++++++---------- kernel/bpf/bpf_task_storage.c | 6 ++++- kernel/bpf/verifier.c | 3 +++ net/core/bpf_sk_storage.c | 8 ++++++- 6 files changed, 62 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index a2b625960ffe..37b3906af8b1 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -17,6 +17,9 @@ #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 +#define bpf_rcu_lock_held() \ + (rcu_read_lock_held() || rcu_read_lock_trace_held() || \ + rcu_read_lock_bh_held()) struct bpf_local_storage_map_bucket { struct hlist_head list; raw_spinlock_t lock; @@ -162,4 +165,6 @@ struct bpf_local_storage_data * bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, void *value, u64 map_flags); +void bpf_local_storage_free_rcu(struct rcu_head *rcu); + #endif /* _BPF_LOCAL_STORAGE_H */ diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 96ceed0e0fb5..e29d9e3d853e 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -17,6 +17,7 @@ #include #include #include +#include DEFINE_BPF_STORAGE_CACHE(inode_cache); @@ -44,7 +45,8 @@ static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, if (!bsb) return NULL; - inode_storage = rcu_dereference(bsb->storage); + inode_storage = + rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); if (!inode_storage) return NULL; @@ -172,6 +174,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, { struct bpf_local_storage_data *sdata; + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) return (unsigned long)NULL; @@ -204,6 +207,7 @@ BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, BPF_CALL_2(bpf_inode_storage_delete, struct bpf_map *, map, struct inode *, inode) { + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (!inode) return -EINVAL; diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index b305270b7a4b..71de2a89869c 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_CLONE) @@ -81,6 +84,22 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return NULL; } +void bpf_local_storage_free_rcu(struct rcu_head *rcu) +{ + struct bpf_local_storage *local_storage; + + local_storage = container_of(rcu, struct bpf_local_storage, rcu); + kfree_rcu(local_storage, rcu); +} + +static void bpf_selem_free_rcu(struct rcu_head *rcu) +{ + struct bpf_local_storage_elem *selem; + + selem = container_of(rcu, struct bpf_local_storage_elem, rcu); + kfree_rcu(selem, rcu); +} + /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be * dereferenced for its smap->elem_size and smap->cache_idx. @@ -93,7 +112,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, bool free_local_storage; void *owner; - smap = rcu_dereference(SDATA(selem)->smap); + smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); owner = local_storage->owner; /* All uncharging on the owner must be done first. @@ -118,12 +137,12 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, * * Although the unlock will be done under * rcu_read_lock(), it is more intutivie to - * read if kfree_rcu(local_storage, rcu) is done + * read if the freeing of the storage is done * after the raw_spin_unlock_bh(&local_storage->lock). * * Hence, a "bool free_local_storage" is returned - * to the caller which then calls the kfree_rcu() - * after unlock. + * to the caller which then calls then frees the storage after + * all the RCU grace periods have expired. */ } hlist_del_init_rcu(&selem->snode); @@ -131,8 +150,7 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, SDATA(selem)) RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL); - kfree_rcu(selem, rcu); - + call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu); return free_local_storage; } @@ -146,7 +164,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) /* selem has already been unlinked from sk */ return; - local_storage = rcu_dereference(selem->local_storage); + local_storage = rcu_dereference_check(selem->local_storage, + bpf_rcu_lock_held()); raw_spin_lock_irqsave(&local_storage->lock, flags); if (likely(selem_linked_to_storage(selem))) free_local_storage = bpf_selem_unlink_storage_nolock( @@ -154,7 +173,8 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) raw_spin_unlock_irqrestore(&local_storage->lock, flags); if (free_local_storage) - kfree_rcu(local_storage, rcu); + call_rcu_tasks_trace(&local_storage->rcu, + bpf_local_storage_free_rcu); } void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, @@ -174,7 +194,7 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) /* selem has already be unlinked from smap */ return; - smap = rcu_dereference(SDATA(selem)->smap); + smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); b = select_bucket(smap, selem); raw_spin_lock_irqsave(&b->lock, flags); if (likely(selem_linked_to_map(selem))) @@ -213,12 +233,14 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem; /* Fast path (cache hit) */ - sdata = rcu_dereference(local_storage->cache[smap->cache_idx]); + sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx], + bpf_rcu_lock_held()); if (sdata && rcu_access_pointer(sdata->smap) == smap) return sdata; /* Slow path (cache miss) */ - hlist_for_each_entry_rcu(selem, &local_storage->list, snode) + hlist_for_each_entry_rcu(selem, &local_storage->list, snode, + rcu_read_lock_trace_held()) if (rcu_access_pointer(SDATA(selem)->smap) == smap) break; @@ -306,7 +328,8 @@ int bpf_local_storage_alloc(void *owner, * bucket->list, first_selem can be freed immediately * (instead of kfree_rcu) because * bpf_local_storage_map_free() does a - * synchronize_rcu() before walking the bucket->list. + * synchronize_rcu_mult (waiting for both sleepable and + * normal programs) before walking the bucket->list. * Hence, no one is accessing selem from the * bucket->list under rcu_read_lock(). */ @@ -342,7 +365,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, !map_value_has_spin_lock(&smap->map))) return ERR_PTR(-EINVAL); - local_storage = rcu_dereference(*owner_storage(smap, owner)); + local_storage = rcu_dereference_check(*owner_storage(smap, owner), + bpf_rcu_lock_held()); if (!local_storage || hlist_empty(&local_storage->list)) { /* Very first elem for the owner */ err = check_flags(NULL, map_flags); diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c index bb69aea1a777..5da7bed0f5f6 100644 --- a/kernel/bpf/bpf_task_storage.c +++ b/kernel/bpf/bpf_task_storage.c @@ -17,6 +17,7 @@ #include #include #include +#include DEFINE_BPF_STORAGE_CACHE(task_cache); @@ -59,7 +60,8 @@ task_storage_lookup(struct task_struct *task, struct bpf_map *map, struct bpf_local_storage *task_storage; struct bpf_local_storage_map *smap; - task_storage = rcu_dereference(task->bpf_storage); + task_storage = + rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held()); if (!task_storage) return NULL; @@ -229,6 +231,7 @@ BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, { struct bpf_local_storage_data *sdata; + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) return (unsigned long)NULL; @@ -260,6 +263,7 @@ BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *, { int ret; + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (!task) return -EINVAL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ca5cd0de804c..133599dfe2a2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11874,6 +11874,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, } break; case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_INODE_STORAGE: + case BPF_MAP_TYPE_SK_STORAGE: + case BPF_MAP_TYPE_TASK_STORAGE: break; default: verbose(env, diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index ea61dfe19c86..d9c37fd10809 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -13,6 +13,7 @@ #include #include #include +#include DEFINE_BPF_STORAGE_CACHE(sk_cache); @@ -22,7 +23,8 @@ bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit) struct bpf_local_storage *sk_storage; struct bpf_local_storage_map *smap; - sk_storage = rcu_dereference(sk->sk_bpf_storage); + sk_storage = + rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held()); if (!sk_storage) return NULL; @@ -258,6 +260,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, { struct bpf_local_storage_data *sdata; + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE) return (unsigned long)NULL; @@ -288,6 +291,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) { + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (!sk || !sk_fullsock(sk)) return -EINVAL; @@ -416,6 +420,7 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog) BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, void *, value, u64, flags) { + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (in_hardirq() || in_nmi()) return (unsigned long)NULL; @@ -425,6 +430,7 @@ BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map, struct sock *, sk) { + WARN_ON_ONCE(!bpf_rcu_lock_held()); if (in_hardirq() || in_nmi()) return -EPERM; -- cgit v1.2.3-59-g8ed1b From aebb51ec3db2a871d74b4afad3f9914812acf120 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 29 Dec 2021 17:27:42 -0800 Subject: bpf: Invert the dependency between bpf-netns.h and netns/bpf.h netns/bpf.h gets included by netdevice.h (thru net_namespace.h) which in turn gets included in a lot of places. We should keep netns/bpf.h as light-weight as possible. bpf-netns.h seems to contain more implementation details than deserves to be included in a netns header. It needs to pull in uapi/bpf.h to get various enum types. Move enum netns_bpf_attach_type to netns/bpf.h and invert the dependency. This makes netns/bpf.h fit the mold of a struct definition header more clearly, and drops the number of objects rebuilt when uapi/bpf.h is touched from 7.7k to 1.1k. Signed-off-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20211230012742.770642-3-kuba@kernel.org --- include/linux/bpf-netns.h | 8 +------- include/net/netns/bpf.h | 9 ++++++++- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-netns.h b/include/linux/bpf-netns.h index 722f799c1a2e..413cfa5e4b07 100644 --- a/include/linux/bpf-netns.h +++ b/include/linux/bpf-netns.h @@ -3,15 +3,9 @@ #define _BPF_NETNS_H #include +#include #include -enum netns_bpf_attach_type { - NETNS_BPF_INVALID = -1, - NETNS_BPF_FLOW_DISSECTOR = 0, - NETNS_BPF_SK_LOOKUP, - MAX_NETNS_BPF_ATTACH_TYPE -}; - static inline enum netns_bpf_attach_type to_netns_bpf_attach_type(enum bpf_attach_type attach_type) { diff --git a/include/net/netns/bpf.h b/include/net/netns/bpf.h index 0ca6a1b87185..2c01a278d1eb 100644 --- a/include/net/netns/bpf.h +++ b/include/net/netns/bpf.h @@ -6,11 +6,18 @@ #ifndef __NETNS_BPF_H__ #define __NETNS_BPF_H__ -#include +#include struct bpf_prog; struct bpf_prog_array; +enum netns_bpf_attach_type { + NETNS_BPF_INVALID = -1, + NETNS_BPF_FLOW_DISSECTOR = 0, + NETNS_BPF_SK_LOOKUP, + MAX_NETNS_BPF_ATTACH_TYPE +}; + struct netns_bpf { /* Array of programs to run compiled from progs or links */ struct bpf_prog_array __rcu *run_array[MAX_NETNS_BPF_ATTACH_TYPE]; -- cgit v1.2.3-59-g8ed1b From 730b49aac426e1e8016d3c2dd6b407e500423821 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Thu, 23 Dec 2021 11:24:22 +0300 Subject: usb: typec: port-mapper: Convert to the component framework Instead of trying to keep track of the connections to the USB Type-C connectors separately, letting the component framework take care of that. From now on every USB Type-C connector will register itself as "aggregate" - component master - and anything that can be connected to it inside the system can then simply register itself as a generic component. The matching of the components and the connector shall rely on ACPI _PLD initially. Before registering itself as the aggregate, the connector will find all other ACPI devices that have matching _PLD crc hash with it (matching value in the pld_crc member of struct acpi_device), and add a component match entry for each one of them. Because only ACPI is supported for now, the driver shall only be build when ACPI is supported. This removes the need for the custom API that the driver exposed. The components and the connector can therefore exist completely independently of each other. The order in which they are registered, as well as are they modules or not, is now irrelevant. Acked-by: Rafael J. Wysocki Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20211223082422.45637-1-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/Makefile | 3 +- drivers/usb/typec/class.c | 2 - drivers/usb/typec/class.h | 10 +- drivers/usb/typec/port-mapper.c | 279 ++++++---------------------------------- include/linux/usb/typec.h | 12 -- 5 files changed, 46 insertions(+), 260 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index a0adb8947a30..57870a2bd787 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_TYPEC) += typec.o -typec-y := class.o mux.o bus.o port-mapper.o +typec-y := class.o mux.o bus.o +typec-$(CONFIG_ACPI) += port-mapper.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index aeef453aa658..45a6f0c807cb 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -2039,8 +2039,6 @@ struct typec_port *typec_register_port(struct device *parent, ida_init(&port->mode_ids); mutex_init(&port->port_type_lock); - mutex_init(&port->port_list_lock); - INIT_LIST_HEAD(&port->port_list); port->id = id; port->ops = cap->ops; diff --git a/drivers/usb/typec/class.h b/drivers/usb/typec/class.h index aef03eb7e152..0f1bd6d19d67 100644 --- a/drivers/usb/typec/class.h +++ b/drivers/usb/typec/class.h @@ -54,11 +54,6 @@ struct typec_port { const struct typec_capability *cap; const struct typec_operations *ops; - - struct list_head port_list; - struct mutex port_list_lock; /* Port list lock */ - - void *pld; }; #define to_typec_port(_dev_) container_of(_dev_, struct typec_port, dev) @@ -79,7 +74,12 @@ extern const struct device_type typec_port_dev_type; extern struct class typec_mux_class; extern struct class typec_class; +#if defined(CONFIG_ACPI) int typec_link_ports(struct typec_port *connector); void typec_unlink_ports(struct typec_port *connector); +#else +static inline int typec_link_ports(struct typec_port *connector) { return 0; } +static inline void typec_unlink_ports(struct typec_port *connector) { } +#endif #endif /* __USB_TYPEC_CLASS__ */ diff --git a/drivers/usb/typec/port-mapper.c b/drivers/usb/typec/port-mapper.c index 9b0991bdf391..07d307418b47 100644 --- a/drivers/usb/typec/port-mapper.c +++ b/drivers/usb/typec/port-mapper.c @@ -7,273 +7,72 @@ */ #include -#include -#include +#include #include "class.h" -struct port_node { - struct list_head list; - struct device *dev; - void *pld; -}; - -static int acpi_pld_match(const struct acpi_pld_info *pld1, - const struct acpi_pld_info *pld2) -{ - if (!pld1 || !pld2) - return 0; - - /* - * To speed things up, first checking only the group_position. It seems - * to often have the first unique value in the _PLD. - */ - if (pld1->group_position == pld2->group_position) - return !memcmp(pld1, pld2, sizeof(struct acpi_pld_info)); - - return 0; -} - -static void *get_pld(struct device *dev) +static int typec_aggregate_bind(struct device *dev) { -#ifdef CONFIG_ACPI - struct acpi_pld_info *pld; - acpi_status status; - - if (!has_acpi_companion(dev)) - return NULL; - - status = acpi_get_physical_device_location(ACPI_HANDLE(dev), &pld); - if (ACPI_FAILURE(status)) - return NULL; - - return pld; -#else - return NULL; -#endif -} - -static void free_pld(void *pld) -{ -#ifdef CONFIG_ACPI - ACPI_FREE(pld); -#endif + return component_bind_all(dev, NULL); } -static int __link_port(struct typec_port *con, struct port_node *node) +static void typec_aggregate_unbind(struct device *dev) { - int ret; - - ret = sysfs_create_link(&node->dev->kobj, &con->dev.kobj, "connector"); - if (ret) - return ret; - - ret = sysfs_create_link(&con->dev.kobj, &node->dev->kobj, - dev_name(node->dev)); - if (ret) { - sysfs_remove_link(&node->dev->kobj, "connector"); - return ret; - } - - list_add_tail(&node->list, &con->port_list); - - return 0; + component_unbind_all(dev, NULL); } -static int link_port(struct typec_port *con, struct port_node *node) -{ - int ret; - - mutex_lock(&con->port_list_lock); - ret = __link_port(con, node); - mutex_unlock(&con->port_list_lock); - - return ret; -} - -static void __unlink_port(struct typec_port *con, struct port_node *node) -{ - sysfs_remove_link(&con->dev.kobj, dev_name(node->dev)); - sysfs_remove_link(&node->dev->kobj, "connector"); - list_del(&node->list); -} - -static void unlink_port(struct typec_port *con, struct port_node *node) -{ - mutex_lock(&con->port_list_lock); - __unlink_port(con, node); - mutex_unlock(&con->port_list_lock); -} - -static struct port_node *create_port_node(struct device *port) -{ - struct port_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->dev = get_device(port); - node->pld = get_pld(port); - - return node; -} - -static void remove_port_node(struct port_node *node) -{ - put_device(node->dev); - free_pld(node->pld); - kfree(node); -} - -static int connector_match(struct device *dev, const void *data) -{ - const struct port_node *node = data; - - if (!is_typec_port(dev)) - return 0; - - return acpi_pld_match(to_typec_port(dev)->pld, node->pld); -} - -static struct device *find_connector(struct port_node *node) -{ - if (!node->pld) - return NULL; - - return class_find_device(&typec_class, NULL, node, connector_match); -} - -/** - * typec_link_port - Link a port to its connector - * @port: The port device - * - * Find the connector of @port and create symlink named "connector" for it. - * Returns 0 on success, or errno in case of a failure. - * - * NOTE. The function increments the reference count of @port on success. - */ -int typec_link_port(struct device *port) -{ - struct device *connector; - struct port_node *node; - int ret; - - node = create_port_node(port); - if (IS_ERR(node)) - return PTR_ERR(node); - - connector = find_connector(node); - if (!connector) { - ret = 0; - goto remove_node; - } - - ret = link_port(to_typec_port(connector), node); - if (ret) - goto put_connector; - - return 0; - -put_connector: - put_device(connector); -remove_node: - remove_port_node(node); - - return ret; -} -EXPORT_SYMBOL_GPL(typec_link_port); - -static int port_match_and_unlink(struct device *connector, void *port) -{ - struct port_node *node; - struct port_node *tmp; - int ret = 0; - - if (!is_typec_port(connector)) - return 0; - - mutex_lock(&to_typec_port(connector)->port_list_lock); - list_for_each_entry_safe(node, tmp, &to_typec_port(connector)->port_list, list) { - ret = node->dev == port; - if (ret) { - unlink_port(to_typec_port(connector), node); - remove_port_node(node); - put_device(connector); - break; - } - } - mutex_unlock(&to_typec_port(connector)->port_list_lock); +static const struct component_master_ops typec_aggregate_ops = { + .bind = typec_aggregate_bind, + .unbind = typec_aggregate_unbind, +}; - return ret; -} +struct each_port_arg { + struct typec_port *port; + struct component_match *match; +}; -/** - * typec_unlink_port - Unlink port from its connector - * @port: The port device - * - * Removes the symlink "connector" and decrements the reference count of @port. - */ -void typec_unlink_port(struct device *port) +static int typec_port_compare(struct device *dev, void *fwnode) { - class_for_each_device(&typec_class, NULL, port, port_match_and_unlink); + return device_match_fwnode(dev, fwnode); } -EXPORT_SYMBOL_GPL(typec_unlink_port); -static int each_port(struct device *port, void *connector) +static int typec_port_match(struct device *dev, void *data) { - struct port_node *node; - int ret; - - node = create_port_node(port); - if (IS_ERR(node)) - return PTR_ERR(node); + struct acpi_device *adev = to_acpi_device(dev); + struct each_port_arg *arg = data; + struct acpi_device *con_adev; - if (!connector_match(connector, node)) { - remove_port_node(node); + con_adev = ACPI_COMPANION(&arg->port->dev); + if (con_adev == adev) return 0; - } - - ret = link_port(to_typec_port(connector), node); - if (ret) { - remove_port_node(node->pld); - return ret; - } - - get_device(connector); + if (con_adev->pld_crc == adev->pld_crc) + component_match_add(&arg->port->dev, &arg->match, typec_port_compare, + acpi_fwnode_handle(adev)); return 0; } int typec_link_ports(struct typec_port *con) { - int ret = 0; + struct each_port_arg arg = { .port = con, .match = NULL }; - con->pld = get_pld(&con->dev); - if (!con->pld) - return 0; + bus_for_each_dev(&acpi_bus_type, NULL, &arg, typec_port_match); - ret = usb_for_each_port(&con->dev, each_port); - if (ret) - typec_unlink_ports(con); - - return ret; + /* + * REVISIT: Now each connector can have only a single component master. + * So far only the USB ports connected to the USB Type-C connector share + * the _PLD with it, but if there one day is something else (like maybe + * the DisplayPort ACPI device object) that also shares the _PLD with + * the connector, every one of those needs to have its own component + * master, because each different type of component needs to be bind to + * the connector independently of the other components. That requires + * improvements to the component framework. Right now you can only have + * one master per device. + */ + return component_master_add_with_match(&con->dev, &typec_aggregate_ops, arg.match); } void typec_unlink_ports(struct typec_port *con) { - struct port_node *node; - struct port_node *tmp; - - mutex_lock(&con->port_list_lock); - - list_for_each_entry_safe(node, tmp, &con->port_list, list) { - __unlink_port(con, node); - remove_port_node(node); - put_device(&con->dev); - } - - mutex_unlock(&con->port_list_lock); - - free_pld(con->pld); + component_master_del(&con->dev, &typec_aggregate_ops); } diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index e2e44bb1dad8..7ba45a97eeae 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -305,16 +305,4 @@ void typec_partner_set_svdm_version(struct typec_partner *partner, enum usb_pd_svdm_ver svdm_version); int typec_get_negotiated_svdm_version(struct typec_port *port); -#if IS_REACHABLE(CONFIG_TYPEC) -int typec_link_port(struct device *port); -void typec_unlink_port(struct device *port); -#else -static inline int typec_link_port(struct device *port) -{ - return 0; -} - -static inline void typec_unlink_port(struct device *port) { } -#endif - #endif /* __LINUX_USB_TYPEC_H */ -- cgit v1.2.3-59-g8ed1b From 510a0bdb2bfcff8d7be822c72adc3add7a97d559 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Thu, 23 Dec 2021 11:24:32 +0300 Subject: usb: Remove usb_for_each_port() There are no more users for the function. Reviewed-by: Andy Shevchenko Acked-by: Rafael J. Wysocki Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20211223082432.45653-1-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/usb.c | 46 ---------------------------------------------- include/linux/usb.h | 9 --------- 2 files changed, 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c index 62368c4ed37a..2ce3667ec6fa 100644 --- a/drivers/usb/core/usb.c +++ b/drivers/usb/core/usb.c @@ -398,52 +398,6 @@ int usb_for_each_dev(void *data, int (*fn)(struct usb_device *, void *)) } EXPORT_SYMBOL_GPL(usb_for_each_dev); -struct each_hub_arg { - void *data; - int (*fn)(struct device *, void *); -}; - -static int __each_hub(struct usb_device *hdev, void *data) -{ - struct each_hub_arg *arg = (struct each_hub_arg *)data; - struct usb_hub *hub; - int ret = 0; - int i; - - hub = usb_hub_to_struct_hub(hdev); - if (!hub) - return 0; - - mutex_lock(&usb_port_peer_mutex); - - for (i = 0; i < hdev->maxchild; i++) { - ret = arg->fn(&hub->ports[i]->dev, arg->data); - if (ret) - break; - } - - mutex_unlock(&usb_port_peer_mutex); - - return ret; -} - -/** - * usb_for_each_port - interate over all USB ports in the system - * @data: data pointer that will be handed to the callback function - * @fn: callback function to be called for each USB port - * - * Iterate over all USB ports and call @fn for each, passing it @data. If it - * returns anything other than 0, we break the iteration prematurely and return - * that value. - */ -int usb_for_each_port(void *data, int (*fn)(struct device *, void *)) -{ - struct each_hub_arg arg = {data, fn}; - - return usb_for_each_dev(&arg, __each_hub); -} -EXPORT_SYMBOL_GPL(usb_for_each_port); - /** * usb_release_dev - free a usb device structure when all users of it are finished. * @dev: device that's been disconnected diff --git a/include/linux/usb.h b/include/linux/usb.h index 7ccaa76a9a96..200b7b79acb5 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -875,15 +875,6 @@ extern struct usb_host_interface *usb_find_alt_setting( unsigned int iface_num, unsigned int alt_num); -#if IS_REACHABLE(CONFIG_USB) -int usb_for_each_port(void *data, int (*fn)(struct device *, void *)); -#else -static inline int usb_for_each_port(void *data, int (*fn)(struct device *, void *)) -{ - return 0; -} -#endif - /* port claiming functions */ int usb_hub_claim_port(struct usb_device *hdev, unsigned port1, struct usb_dev_state *owner); -- cgit v1.2.3-59-g8ed1b From b4a29b94804c4774f22555651296b838df6ec0e4 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Tue, 28 Dec 2021 18:22:00 +0100 Subject: serial: 8250: Move Alpha-specific quirk out of the core struct uart_8250_port contains mcr_mask and mcr_force members whose sole purpose is to work around an Alpha-specific quirk. This code doesn't belong in the core where it is executed by everyone else, so move it to a proper ->set_mctrl callback which is used on the affected Alpha machine only. The quirk was introduced in January 1995: https://git.kernel.org/pub/scm/linux/kernel/git/history/history.git/diff/drivers/char/serial.c?h=1.1.83 The members in struct uart_8250_port were added in 2002: https://git.kernel.org/history/history/c/4524aad27854 The quirk applies to non-PCI Alphas and arch/alpha/Kconfig specifies "select FORCE_PCI if !ALPHA_JENSEN". So apparently the only affected machine is the EISA-based Jensen that Linus was working on back then: https://lore.kernel.org/all/CAHk-=wj1JWZ3sCrGz16nxEj7=0O+srMg6Ah3iPTDXSPKEws_SA@mail.gmail.com/ Up until now the quirk is not applied unless CONFIG_PCI is disabled. If users forget to do that or run a generic Alpha kernel, the serial ports aren't usable on Jensen. Avoid by confining the quirk to CONFIG_ALPHA_JENSEN instead of !CONFIG_PCI. On generic Alpha kernels, auto-detect at runtime whether the quirk needs to be applied. Cc: Russell King Cc: Ulrich Teichert Cc: Linus Torvalds Signed-off-by: Lukas Wunner Link: https://lore.kernel.org/r/b83d069cb516549b8a5420e097bb6bdd806f36fc.1640695609.git.lukas@wunner.de Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250.h | 12 ++---------- drivers/tty/serial/8250/8250_alpha.c | 21 +++++++++++++++++++++ drivers/tty/serial/8250/8250_core.c | 9 ++++----- drivers/tty/serial/8250/8250_port.c | 2 +- drivers/tty/serial/8250/Makefile | 2 ++ drivers/tty/serial/sunsu.c | 3 ++- include/linux/serial_8250.h | 2 -- 7 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 drivers/tty/serial/8250/8250_alpha.c (limited to 'include/linux') diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 6473361525d1..db784ace25d8 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -241,16 +241,8 @@ static inline int serial8250_in_MCR(struct uart_8250_port *up) return mctrl; } -#if defined(__alpha__) && !defined(CONFIG_PCI) -/* - * Digital did something really horribly wrong with the OUT1 and OUT2 - * lines on at least some ALPHA's. The failure mode is that if either - * is cleared, the machine locks up with endless interrupts. - */ -#define ALPHA_KLUDGE_MCR (UART_MCR_OUT2 | UART_MCR_OUT1) -#else -#define ALPHA_KLUDGE_MCR 0 -#endif +bool alpha_jensen(void); +void alpha_jensen_set_mctrl(struct uart_port *port, unsigned int mctrl); #ifdef CONFIG_SERIAL_8250_PNP int serial8250_pnp_init(void); diff --git a/drivers/tty/serial/8250/8250_alpha.c b/drivers/tty/serial/8250/8250_alpha.c new file mode 100644 index 000000000000..58e70328aa4d --- /dev/null +++ b/drivers/tty/serial/8250/8250_alpha.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include "8250.h" + +bool alpha_jensen(void) +{ + return !strcmp(alpha_mv.vector_name, "Jensen"); +} + +void alpha_jensen_set_mctrl(struct uart_port *port, unsigned int mctrl) +{ + /* + * Digital did something really horribly wrong with the OUT1 and OUT2 + * lines on Alpha Jensen. The failure mode is that if either is + * cleared, the machine locks up with endless interrupts. + */ + mctrl |= TIOCM_OUT1 | TIOCM_OUT2; + + serial8250_do_set_mctrl(port, mctrl); +} diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 1ce193daea7f..01d30f6ed8fb 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -509,11 +509,10 @@ static void __init serial8250_isa_init_ports(void) up->ops = &univ8250_driver_ops; - /* - * ALPHA_KLUDGE_MCR needs to be killed. - */ - up->mcr_mask = ~ALPHA_KLUDGE_MCR; - up->mcr_force = ALPHA_KLUDGE_MCR; + if (IS_ENABLED(CONFIG_ALPHA_JENSEN) || + (IS_ENABLED(CONFIG_ALPHA_GENERIC) && alpha_jensen())) + port->set_mctrl = alpha_jensen_set_mctrl; + serial8250_set_defaults(up); } diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 7e7e67dea0ad..2abb3de11a48 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -2026,7 +2026,7 @@ void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl) mcr = serial8250_TIOCM_to_MCR(mctrl); - mcr = (mcr & up->mcr_mask) | up->mcr_force | up->mcr; + mcr |= up->mcr; serial8250_out_MCR(up, mcr); } diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile index 4e4913e0e4d2..bee908f99ea0 100644 --- a/drivers/tty/serial/8250/Makefile +++ b/drivers/tty/serial/8250/Makefile @@ -5,6 +5,8 @@ obj-$(CONFIG_SERIAL_8250) += 8250.o 8250_base.o 8250-y := 8250_core.o +8250-$(CONFIG_ALPHA_GENERIC) += 8250_alpha.o +8250-$(CONFIG_ALPHA_JENSEN) += 8250_alpha.o 8250-$(CONFIG_SERIAL_8250_PNP) += 8250_pnp.o 8250_base-y := 8250_port.o 8250_base-$(CONFIG_SERIAL_8250_DMA) += 8250_dma.o diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c index 425a016f9db7..98b2f4fb9a99 100644 --- a/drivers/tty/serial/sunsu.c +++ b/drivers/tty/serial/sunsu.c @@ -127,7 +127,8 @@ static void serial_out(struct uart_sunsu_port *up, int offset, int value) * gate outputs a logical one. Since we use level triggered interrupts * we have lockup and watchdog reset. We cannot mask IRQ because * keyboard shares IRQ with us (Word has it as Bob Smelik's design). - * This problem is similar to what Alpha people suffer, see serial.c. + * This problem is similar to what Alpha people suffer, see + * 8250_alpha.c. */ if (offset == UART_MCR) value |= UART_MCR_OUT2; diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 5db211f43b29..ff84a3ed10ea 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -104,8 +104,6 @@ struct uart_8250_port { unsigned char ier; unsigned char lcr; unsigned char mcr; - unsigned char mcr_mask; /* mask of user bits */ - unsigned char mcr_force; /* mask of forced bits */ unsigned char cur_iotype; /* Running I/O type */ unsigned int rpm_tx_active; unsigned char canary; /* non-zero during system sleep -- cgit v1.2.3-59-g8ed1b From d8e9a406a931f687945703a4bac45042eb81ce92 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Sun, 12 Dec 2021 22:21:28 +0900 Subject: serdev: BREAK/FRAME/PARITY/OVERRUN notification prototype V2 Allow serdev device drivers get notified by hardware errors such as BREAK, FRAME, PARITY and OVERRUN. With this patch, in the event of an error detected in the UART device driver the serdev_device_driver will get the newly introduced ->error() callback invoked if serdev_device_set_error_mask() has previously been used to enable the type of error. The errors are taken straight from the TTY layer and fed into the serdev_device_driver after filtering out only enabled errors. Without this patch the hardware errors never reach the serdev_device_driver. Signed-off-by: Magnus Damm Link: https://lore.kernel.org/r/163931528842.27756.3665040315954968747.sendpatchset@octo Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serdev/core.c | 11 +++++++++ drivers/tty/serdev/serdev-ttyport.c | 49 +++++++++++++++++++++++++++++++++++++ include/linux/serdev.h | 22 +++++++++++++++++ 3 files changed, 82 insertions(+) (limited to 'include/linux') diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c index f1324fe99378..5c25ae20e508 100644 --- a/drivers/tty/serdev/core.c +++ b/drivers/tty/serdev/core.c @@ -349,6 +349,17 @@ unsigned int serdev_device_set_baudrate(struct serdev_device *serdev, unsigned i } EXPORT_SYMBOL_GPL(serdev_device_set_baudrate); +void serdev_device_set_error_mask(struct serdev_device *serdev, unsigned long mask) +{ + struct serdev_controller *ctrl = serdev->ctrl; + + if (!ctrl || !ctrl->ops->set_error_mask) + return; + + ctrl->ops->set_error_mask(ctrl, mask); +} +EXPORT_SYMBOL_GPL(serdev_device_set_error_mask); + void serdev_device_set_flow_control(struct serdev_device *serdev, bool enable) { struct serdev_controller *ctrl = serdev->ctrl; diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c index d367803e2044..239a1d5c66df 100644 --- a/drivers/tty/serdev/serdev-ttyport.c +++ b/drivers/tty/serdev/serdev-ttyport.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2016-2017 Linaro Ltd., Rob Herring */ +#include #include #include #include @@ -9,6 +10,10 @@ #include #define SERPORT_ACTIVE 1 +#define SERPORT_NOTIFY_BREAK 2 +#define SERPORT_NOTIFY_FRAME 3 +#define SERPORT_NOTIFY_PARITY 4 +#define SERPORT_NOTIFY_OVERRUN 5 struct serport { struct tty_port *port; @@ -27,11 +32,39 @@ static int ttyport_receive_buf(struct tty_port *port, const unsigned char *cp, { struct serdev_controller *ctrl = port->client_data; struct serport *serport = serdev_controller_get_drvdata(ctrl); + unsigned long errors = 0; + unsigned int i; int ret; if (!test_bit(SERPORT_ACTIVE, &serport->flags)) return 0; + for (i = 0; fp && i < count; i++) { + switch (fp[i]) { + case TTY_BREAK: + if (test_bit(SERPORT_NOTIFY_BREAK, &serport->flags)) + __set_bit(SERDEV_ERROR_BREAK, &errors); + break; + + case TTY_FRAME: + if (test_bit(SERPORT_NOTIFY_FRAME, &serport->flags)) + __set_bit(SERDEV_ERROR_FRAME, &errors); + break; + + case TTY_PARITY: + if (test_bit(SERPORT_NOTIFY_PARITY, &serport->flags)) + __set_bit(SERDEV_ERROR_PARITY, &errors); + break; + + case TTY_OVERRUN: + if (test_bit(SERPORT_NOTIFY_OVERRUN, &serport->flags)) + __set_bit(SERDEV_ERROR_OVERRUN, &errors); + break; + } + } + if (errors) + serdev_controller_error(ctrl, errors); + ret = serdev_controller_receive_buf(ctrl, cp, count); dev_WARN_ONCE(&ctrl->dev, ret < 0 || ret > count, @@ -180,6 +213,21 @@ static unsigned int ttyport_set_baudrate(struct serdev_controller *ctrl, unsigne return ktermios.c_ospeed; } +static void ttyport_set_error_mask(struct serdev_controller *ctrl, + unsigned long m) +{ + struct serport *sp = serdev_controller_get_drvdata(ctrl); + + assign_bit(SERPORT_NOTIFY_BREAK, &sp->flags, + m & BIT(SERDEV_ERROR_BREAK)); + assign_bit(SERPORT_NOTIFY_FRAME, &sp->flags, + m & BIT(SERDEV_ERROR_FRAME)); + assign_bit(SERPORT_NOTIFY_PARITY, &sp->flags, + m & BIT(SERDEV_ERROR_PARITY)); + assign_bit(SERPORT_NOTIFY_OVERRUN, &sp->flags, + m & BIT(SERDEV_ERROR_OVERRUN)); +} + static void ttyport_set_flow_control(struct serdev_controller *ctrl, bool enable) { struct serport *serport = serdev_controller_get_drvdata(ctrl); @@ -253,6 +301,7 @@ static const struct serdev_controller_ops ctrl_ops = { .write_room = ttyport_write_room, .open = ttyport_open, .close = ttyport_close, + .set_error_mask = ttyport_set_error_mask, .set_flow_control = ttyport_set_flow_control, .set_parity = ttyport_set_parity, .set_baudrate = ttyport_set_baudrate, diff --git a/include/linux/serdev.h b/include/linux/serdev.h index 3368c261ab62..0d0b22fc7e37 100644 --- a/include/linux/serdev.h +++ b/include/linux/serdev.h @@ -19,12 +19,15 @@ struct serdev_device; /** * struct serdev_device_ops - Callback operations for a serdev device + * @error: Function called with errors received from device; + * may sleep. * @receive_buf: Function called with data received from device; * returns number of bytes accepted; may sleep. * @write_wakeup: Function called when ready to transmit more data; must * not sleep. */ struct serdev_device_ops { + void (*error)(struct serdev_device *, unsigned long); int (*receive_buf)(struct serdev_device *, const unsigned char *, size_t); void (*write_wakeup)(struct serdev_device *); }; @@ -76,6 +79,11 @@ enum serdev_parity { SERDEV_PARITY_ODD, }; +#define SERDEV_ERROR_BREAK 0 +#define SERDEV_ERROR_FRAME 1 +#define SERDEV_ERROR_PARITY 2 +#define SERDEV_ERROR_OVERRUN 3 + /* * serdev controller structures */ @@ -85,6 +93,7 @@ struct serdev_controller_ops { int (*write_room)(struct serdev_controller *); int (*open)(struct serdev_controller *); void (*close)(struct serdev_controller *); + void (*set_error_mask)(struct serdev_controller *, unsigned long); void (*set_flow_control)(struct serdev_controller *, bool); int (*set_parity)(struct serdev_controller *, enum serdev_parity); unsigned int (*set_baudrate)(struct serdev_controller *, unsigned int); @@ -190,12 +199,24 @@ static inline int serdev_controller_receive_buf(struct serdev_controller *ctrl, return serdev->ops->receive_buf(serdev, data, count); } +static inline void serdev_controller_error(struct serdev_controller *ctrl, + unsigned long errors) +{ + struct serdev_device *serdev = ctrl->serdev; + + if (!serdev || !serdev->ops->error) + return; + + serdev->ops->error(serdev, errors); +} + #if IS_ENABLED(CONFIG_SERIAL_DEV_BUS) int serdev_device_open(struct serdev_device *); void serdev_device_close(struct serdev_device *); int devm_serdev_device_open(struct device *, struct serdev_device *); unsigned int serdev_device_set_baudrate(struct serdev_device *, unsigned int); +void serdev_device_set_error_mask(struct serdev_device *, unsigned long); void serdev_device_set_flow_control(struct serdev_device *, bool); int serdev_device_write_buf(struct serdev_device *, const unsigned char *, size_t); void serdev_device_wait_until_sent(struct serdev_device *, long); @@ -238,6 +259,7 @@ static inline unsigned int serdev_device_set_baudrate(struct serdev_device *sdev { return 0; } +static inline void serdev_device_set_error_mask(struct serdev_device *sdev, unsigned long mask) {} static inline void serdev_device_set_flow_control(struct serdev_device *sdev, bool enable) {} static inline int serdev_device_write_buf(struct serdev_device *serdev, const unsigned char *buf, -- cgit v1.2.3-59-g8ed1b From 5207fb2f311b0c45a9abfa1c84b7a7b657ffa550 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 30 Dec 2021 16:02:41 +0100 Subject: counter: Provide a wrapper to access device private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now this just wraps accessing struct counter_device::priv. However this is about to change and converting drivers to this helper individually makes fixing device lifetime issues result in easier to review patches. Reviewed-by: Jonathan Cameron Acked-by: William Breathitt Gray Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20211230150300.72196-5-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/counter/counter-core.c | 12 ++++++++++++ include/linux/counter.h | 2 ++ 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c index f053a43c6c04..00c41f28c101 100644 --- a/drivers/counter/counter-core.c +++ b/drivers/counter/counter-core.c @@ -45,6 +45,18 @@ static struct bus_type counter_bus_type = { static dev_t counter_devt; +/** + * counter_priv - access counter device private data + * @counter: counter device + * + * Get the counter device private data + */ +void *counter_priv(const struct counter_device *const counter) +{ + return counter->priv; +} +EXPORT_SYMBOL_GPL(counter_priv); + /** * counter_register - register Counter to the system * @counter: pointer to Counter to register diff --git a/include/linux/counter.h b/include/linux/counter.h index dfbde2808998..627f1757f6bb 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -329,6 +329,8 @@ struct counter_device { struct mutex ops_exist_lock; }; +void *counter_priv(const struct counter_device *const counter); + int counter_register(struct counter_device *const counter); void counter_unregister(struct counter_device *const counter); int devm_counter_register(struct device *dev, -- cgit v1.2.3-59-g8ed1b From c18e2760308e30f007fa24b558b87c39d7e86ff1 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 30 Dec 2021 16:02:50 +0100 Subject: counter: Provide alternative counter registration functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current implementation gets device lifetime tracking wrong. The problem is that allocation of struct counter_device is controlled by the individual drivers but this structure contains a struct device that might have to live longer than a driver is bound. As a result a command sequence like: { sleep 5; echo bang; } > /dev/counter0 & sleep 1; echo 40000000.timer:counter > /sys/bus/platform/drivers/stm32-timer-counter/unbind can keep a reference to the struct device and unbinding results in freeing the memory occupied by this device resulting in an oops. This commit provides two new functions (plus some helpers): - counter_alloc() to allocate a struct counter_device that is automatically freed once the embedded struct device is released - counter_add() to register such a device. Note that this commit doesn't fix any issues, all drivers have to be converted to these new functions to correct the lifetime problems. Reviewed-by: Jonathan Cameron Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20211230150300.72196-14-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/counter/counter-core.c | 168 ++++++++++++++++++++++++++++++++++++++++- include/linux/counter.h | 15 ++++ 2 files changed, 181 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c index 00c41f28c101..b3fa15bbcbdb 100644 --- a/drivers/counter/counter-core.c +++ b/drivers/counter/counter-core.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,16 @@ /* Provides a unique ID for each counter device */ static DEFINE_IDA(counter_ida); +struct counter_device_allochelper { + struct counter_device counter; + + /* + * This is cache line aligned to ensure private data behaves like if it + * were kmalloced separately. + */ + unsigned long privdata[] ____cacheline_aligned; +}; + static void counter_device_release(struct device *dev) { struct counter_device *const counter = @@ -31,6 +42,9 @@ static void counter_device_release(struct device *dev) counter_chrdev_remove(counter); ida_free(&counter_ida, dev->id); + + if (!counter->legacy_device) + kfree(container_of(counter, struct counter_device_allochelper, counter)); } static struct device_type counter_device_type = { @@ -53,7 +67,14 @@ static dev_t counter_devt; */ void *counter_priv(const struct counter_device *const counter) { - return counter->priv; + if (counter->legacy_device) { + return counter->priv; + } else { + struct counter_device_allochelper *ch = + container_of(counter, struct counter_device_allochelper, counter); + + return &ch->privdata; + } } EXPORT_SYMBOL_GPL(counter_priv); @@ -74,6 +95,8 @@ int counter_register(struct counter_device *const counter) int id; int err; + counter->legacy_device = true; + /* Acquire unique ID */ id = ida_alloc(&counter_ida, GFP_KERNEL); if (id < 0) @@ -114,6 +137,95 @@ err_free_id: } EXPORT_SYMBOL_GPL(counter_register); +/** + * counter_alloc - allocate a counter_device + * @sizeof_priv: size of the driver private data + * + * This is part one of counter registration. The structure is allocated + * dynamically to ensure the right lifetime for the embedded struct device. + * + * If this succeeds, call counter_put() to get rid of the counter_device again. + */ +struct counter_device *counter_alloc(size_t sizeof_priv) +{ + struct counter_device_allochelper *ch; + struct counter_device *counter; + struct device *dev; + int err; + + ch = kzalloc(sizeof(*ch) + sizeof_priv, GFP_KERNEL); + if (!ch) { + err = -ENOMEM; + goto err_alloc_ch; + } + + counter = &ch->counter; + dev = &counter->dev; + + /* Acquire unique ID */ + err = ida_alloc(&counter_ida, GFP_KERNEL); + if (err < 0) + goto err_ida_alloc; + dev->id = err; + + mutex_init(&counter->ops_exist_lock); + dev->type = &counter_device_type; + dev->bus = &counter_bus_type; + dev->devt = MKDEV(MAJOR(counter_devt), dev->id); + + err = counter_chrdev_add(counter); + if (err < 0) + goto err_chrdev_add; + + device_initialize(dev); + + return counter; + +err_chrdev_add: + + ida_free(&counter_ida, dev->id); +err_ida_alloc: + + kfree(ch); +err_alloc_ch: + + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(counter_alloc); + +void counter_put(struct counter_device *counter) +{ + put_device(&counter->dev); +} +EXPORT_SYMBOL_GPL(counter_put); + +/** + * counter_add - complete registration of a counter + * @counter: the counter to add + * + * This is part two of counter registration. + * + * If this succeeds, call counter_unregister() to get rid of the counter_device again. + */ +int counter_add(struct counter_device *counter) +{ + int err; + struct device *dev = &counter->dev; + + if (counter->parent) { + dev->parent = counter->parent; + dev->of_node = counter->parent->of_node; + } + + err = counter_sysfs_add(counter); + if (err < 0) + return err; + + /* implies device_add(dev) */ + return cdev_device_add(&counter->chrdev, dev); +} +EXPORT_SYMBOL_GPL(counter_add); + /** * counter_unregister - unregister Counter from the system * @counter: pointer to Counter to unregister @@ -134,7 +246,8 @@ void counter_unregister(struct counter_device *const counter) mutex_unlock(&counter->ops_exist_lock); - put_device(&counter->dev); + if (counter->legacy_device) + put_device(&counter->dev); } EXPORT_SYMBOL_GPL(counter_unregister); @@ -168,6 +281,57 @@ int devm_counter_register(struct device *dev, } EXPORT_SYMBOL_GPL(devm_counter_register); +static void devm_counter_put(void *counter) +{ + counter_put(counter); +} + +/** + * devm_counter_alloc - allocate a counter_device + * @dev: the device to register the release callback for + * @sizeof_priv: size of the driver private data + * + * This is the device managed version of counter_add(). It registers a cleanup + * callback to care for calling counter_put(). + */ +struct counter_device *devm_counter_alloc(struct device *dev, size_t sizeof_priv) +{ + struct counter_device *counter; + int err; + + counter = counter_alloc(sizeof_priv); + if (IS_ERR(counter)) + return counter; + + err = devm_add_action_or_reset(dev, devm_counter_put, counter); + if (err < 0) + return ERR_PTR(err); + + return counter; +} +EXPORT_SYMBOL_GPL(devm_counter_alloc); + +/** + * devm_counter_add - complete registration of a counter + * @dev: the device to register the release callback for + * @counter: the counter to add + * + * This is the device managed version of counter_add(). It registers a cleanup + * callback to care for calling counter_unregister(). + */ +int devm_counter_add(struct device *dev, + struct counter_device *const counter) +{ + int err; + + err = counter_add(counter); + if (err < 0) + return err; + + return devm_add_action_or_reset(dev, devm_counter_release, counter); +} +EXPORT_SYMBOL_GPL(devm_counter_add); + #define COUNTER_DEV_MAX 256 static int __init counter_init(void) diff --git a/include/linux/counter.h b/include/linux/counter.h index 627f1757f6bb..ed8d5820f0d1 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -327,14 +327,29 @@ struct counter_device { spinlock_t events_in_lock; struct mutex events_out_lock; struct mutex ops_exist_lock; + + /* + * This can go away once all drivers are converted to + * counter_alloc()/counter_add(). + */ + bool legacy_device; }; void *counter_priv(const struct counter_device *const counter); int counter_register(struct counter_device *const counter); + +struct counter_device *counter_alloc(size_t sizeof_priv); +void counter_put(struct counter_device *const counter); +int counter_add(struct counter_device *const counter); + void counter_unregister(struct counter_device *const counter); int devm_counter_register(struct device *dev, struct counter_device *const counter); +struct counter_device *devm_counter_alloc(struct device *dev, + size_t sizeof_priv); +int devm_counter_add(struct device *dev, + struct counter_device *const counter); void counter_push_event(struct counter_device *const counter, const u8 event, const u8 channel); -- cgit v1.2.3-59-g8ed1b From f2ee4759fb700b32a1bd830960fe86bf6bdfd0ab Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 30 Dec 2021 16:03:00 +0100 Subject: counter: remove old and now unused registration API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usage of counter_register() yields issues in device lifetime tracking. All drivers were converted to the new API, so the old one can go away. Reviewed-by: Jonathan Cameron Acked-by: William Breathitt Gray Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20211230150300.72196-24-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/counter/counter-core.c | 100 ++--------------------------------------- include/linux/counter.h | 12 ----- 2 files changed, 4 insertions(+), 108 deletions(-) (limited to 'include/linux') diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c index b3fa15bbcbdb..7e0957eea094 100644 --- a/drivers/counter/counter-core.c +++ b/drivers/counter/counter-core.c @@ -43,8 +43,7 @@ static void counter_device_release(struct device *dev) counter_chrdev_remove(counter); ida_free(&counter_ida, dev->id); - if (!counter->legacy_device) - kfree(container_of(counter, struct counter_device_allochelper, counter)); + kfree(container_of(counter, struct counter_device_allochelper, counter)); } static struct device_type counter_device_type = { @@ -67,76 +66,13 @@ static dev_t counter_devt; */ void *counter_priv(const struct counter_device *const counter) { - if (counter->legacy_device) { - return counter->priv; - } else { - struct counter_device_allochelper *ch = - container_of(counter, struct counter_device_allochelper, counter); + struct counter_device_allochelper *ch = + container_of(counter, struct counter_device_allochelper, counter); - return &ch->privdata; - } + return &ch->privdata; } EXPORT_SYMBOL_GPL(counter_priv); -/** - * counter_register - register Counter to the system - * @counter: pointer to Counter to register - * - * This function registers a Counter to the system. A sysfs "counter" directory - * will be created and populated with sysfs attributes correlating with the - * Counter Signals, Synapses, and Counts respectively. - * - * RETURNS: - * 0 on success, negative error number on failure. - */ -int counter_register(struct counter_device *const counter) -{ - struct device *const dev = &counter->dev; - int id; - int err; - - counter->legacy_device = true; - - /* Acquire unique ID */ - id = ida_alloc(&counter_ida, GFP_KERNEL); - if (id < 0) - return id; - - mutex_init(&counter->ops_exist_lock); - - /* Configure device structure for Counter */ - dev->id = id; - dev->type = &counter_device_type; - dev->bus = &counter_bus_type; - dev->devt = MKDEV(MAJOR(counter_devt), id); - if (counter->parent) { - dev->parent = counter->parent; - dev->of_node = counter->parent->of_node; - } - device_initialize(dev); - - err = counter_sysfs_add(counter); - if (err < 0) - goto err_free_id; - - err = counter_chrdev_add(counter); - if (err < 0) - goto err_free_id; - - err = cdev_device_add(&counter->chrdev, dev); - if (err < 0) - goto err_remove_chrdev; - - return 0; - -err_remove_chrdev: - counter_chrdev_remove(counter); -err_free_id: - put_device(dev); - return err; -} -EXPORT_SYMBOL_GPL(counter_register); - /** * counter_alloc - allocate a counter_device * @sizeof_priv: size of the driver private data @@ -245,9 +181,6 @@ void counter_unregister(struct counter_device *const counter) wake_up(&counter->events_wait); mutex_unlock(&counter->ops_exist_lock); - - if (counter->legacy_device) - put_device(&counter->dev); } EXPORT_SYMBOL_GPL(counter_unregister); @@ -256,31 +189,6 @@ static void devm_counter_release(void *counter) counter_unregister(counter); } -/** - * devm_counter_register - Resource-managed counter_register - * @dev: device to allocate counter_device for - * @counter: pointer to Counter to register - * - * Managed counter_register. The Counter registered with this function is - * automatically unregistered on driver detach. This function calls - * counter_register internally. Refer to that function for more information. - * - * RETURNS: - * 0 on success, negative error number on failure. - */ -int devm_counter_register(struct device *dev, - struct counter_device *const counter) -{ - int err; - - err = counter_register(counter); - if (err < 0) - return err; - - return devm_add_action_or_reset(dev, devm_counter_release, counter); -} -EXPORT_SYMBOL_GPL(devm_counter_register); - static void devm_counter_put(void *counter) { counter_put(counter); diff --git a/include/linux/counter.h b/include/linux/counter.h index ed8d5820f0d1..1fe17f5adb09 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -314,8 +314,6 @@ struct counter_device { struct counter_comp *ext; size_t num_ext; - void *priv; - struct device dev; struct cdev chrdev; struct list_head events_list; @@ -327,25 +325,15 @@ struct counter_device { spinlock_t events_in_lock; struct mutex events_out_lock; struct mutex ops_exist_lock; - - /* - * This can go away once all drivers are converted to - * counter_alloc()/counter_add(). - */ - bool legacy_device; }; void *counter_priv(const struct counter_device *const counter); -int counter_register(struct counter_device *const counter); - struct counter_device *counter_alloc(size_t sizeof_priv); void counter_put(struct counter_device *const counter); int counter_add(struct counter_device *const counter); void counter_unregister(struct counter_device *const counter); -int devm_counter_register(struct device *dev, - struct counter_device *const counter); struct counter_device *devm_counter_alloc(struct device *dev, size_t sizeof_priv); int devm_counter_add(struct device *dev, -- cgit v1.2.3-59-g8ed1b From d6c6d0bb2cb3af91c9c1546af7cdf4770d0df443 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 29 Dec 2021 12:36:20 +0100 Subject: net: remove references to CONFIG_IRDA in network header files Commit d64c2a76123f ("staging: irda: remove the irda network stack and drivers") removes the config IRDA. Remove the remaining references to this non-existing config in the network header files. Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20211229113620.19368-1-lukas.bulwahn@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/atalk.h | 2 +- include/linux/netdevice.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atalk.h b/include/linux/atalk.h index f6034ba774be..a55bfc6567d0 100644 --- a/include/linux/atalk.h +++ b/include/linux/atalk.h @@ -113,7 +113,7 @@ extern int aarp_proto_init(void); /* Inter module exports */ /* Give a device find its atif control structure */ -#if IS_ENABLED(CONFIG_IRDA) || IS_ENABLED(CONFIG_ATALK) +#if IS_ENABLED(CONFIG_ATALK) static inline struct atalk_iface *atalk_find_dev(struct net_device *dev) { return dev->atalk_ptr; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 2684bdb6defa..6f99c8f51b60 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2098,7 +2098,7 @@ struct net_device { #if IS_ENABLED(CONFIG_TIPC) struct tipc_bearer __rcu *tipc_ptr; #endif -#if IS_ENABLED(CONFIG_IRDA) || IS_ENABLED(CONFIG_ATALK) +#if IS_ENABLED(CONFIG_ATALK) void *atalk_ptr; #endif struct in_device __rcu *ip_ptr; -- cgit v1.2.3-59-g8ed1b From c3fb0e280b4cdbf382f59eb6b276e4c6047bc219 Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Thu, 18 Nov 2021 02:32:37 +0200 Subject: net/mlx5: DR, Fix lower case macro prefix "mlx5_" to "MLX5_" Macros prefix should be capital letters - fix the prefix in mlx5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED. Signed-off-by: Yevgeny Kliteynik --- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c | 2 +- include/linux/mlx5/mlx5_ifc.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index 1d8febed0d76..0d7575b64ca4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -152,7 +152,7 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, caps->flex_parser_id_mpls_over_gre = MLX5_CAP_GEN(mdev, flex_parser_id_outer_first_mpls_over_gre); - if (caps->flex_protocols & mlx5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED) + if (caps->flex_protocols & MLX5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED) caps->flex_parser_id_mpls_over_udp = MLX5_CAP_GEN(mdev, flex_parser_id_outer_first_mpls_over_udp_label); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c index 3d0cdc36a91a..613074d50212 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c @@ -359,7 +359,7 @@ static bool dr_mask_is_tnl_mpls_over_gre(struct mlx5dr_match_param *mask, static int dr_matcher_supp_tnl_mpls_over_udp(struct mlx5dr_cmd_caps *caps) { - return caps->flex_protocols & mlx5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED; + return caps->flex_protocols & MLX5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED; } static bool dr_mask_is_tnl_mpls_over_udp(struct mlx5dr_match_param *mask, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index e9db12aae8f9..18b816b41545 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1291,7 +1291,7 @@ enum { enum { MLX5_FLEX_PARSER_GENEVE_ENABLED = 1 << 3, MLX5_FLEX_PARSER_MPLS_OVER_GRE_ENABLED = 1 << 4, - mlx5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED = 1 << 5, + MLX5_FLEX_PARSER_MPLS_OVER_UDP_ENABLED = 1 << 5, MLX5_FLEX_PARSER_VXLAN_GPE_ENABLED = 1 << 7, MLX5_FLEX_PARSER_ICMP_V4_ENABLED = 1 << 8, MLX5_FLEX_PARSER_ICMP_V6_ENABLED = 1 << 9, -- cgit v1.2.3-59-g8ed1b From 0f2a6c3b9219bdf497750258cd2ad513f0056b42 Mon Sep 17 00:00:00 2001 From: Muhammad Sammar Date: Sun, 5 Sep 2021 15:16:21 +0300 Subject: net/mlx5: Add misc5 flow table match parameters Add support for misc5 match parameter as per HW spec, this will allow matching on tunnel_header fields. Signed-off-by: Muhammad Sammar Signed-off-by: Yevgeny Kliteynik --- drivers/net/ethernet/mellanox/mlx5/core/fs_core.h | 2 +- include/linux/mlx5/device.h | 1 + include/linux/mlx5/mlx5_ifc.h | 25 ++++++++++++++++++++++- include/uapi/rdma/mlx5_user_ioctl_cmds.h | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 7711db245c63..5469b08d635f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -203,7 +203,7 @@ struct mlx5_ft_underlay_qp { u32 qpn; }; -#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_c00 +#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_e00 /* Calculate the fte_match_param length and without the reserved length. * Make sure the reserved field is the last. */ diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 9c25edfd59a6..604b85dd770a 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -1117,6 +1117,7 @@ enum { MLX5_MATCH_MISC_PARAMETERS_2 = 1 << 3, MLX5_MATCH_MISC_PARAMETERS_3 = 1 << 4, MLX5_MATCH_MISC_PARAMETERS_4 = 1 << 5, + MLX5_MATCH_MISC_PARAMETERS_5 = 1 << 6, }; enum { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 18b816b41545..c74c5e147cb9 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -670,6 +670,26 @@ struct mlx5_ifc_fte_match_set_misc4_bits { u8 reserved_at_100[0x100]; }; +struct mlx5_ifc_fte_match_set_misc5_bits { + u8 macsec_tag_0[0x20]; + + u8 macsec_tag_1[0x20]; + + u8 macsec_tag_2[0x20]; + + u8 macsec_tag_3[0x20]; + + u8 tunnel_header_0[0x20]; + + u8 tunnel_header_1[0x20]; + + u8 tunnel_header_2[0x20]; + + u8 tunnel_header_3[0x20]; + + u8 reserved_at_100[0x100]; +}; + struct mlx5_ifc_cmd_pas_bits { u8 pa_h[0x20]; @@ -1839,7 +1859,9 @@ struct mlx5_ifc_fte_match_param_bits { struct mlx5_ifc_fte_match_set_misc4_bits misc_parameters_4; - u8 reserved_at_c00[0x400]; + struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5; + + u8 reserved_at_e00[0x200]; }; enum { @@ -5977,6 +5999,7 @@ enum { MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2 = 0x3, MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_3 = 0x4, MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_4 = 0x5, + MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_5 = 0x6, }; struct mlx5_ifc_query_flow_group_out_bits { diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index ca2372864b70..e539c84d63f1 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -252,7 +252,7 @@ enum mlx5_ib_device_query_context_attrs { MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX = (1U << UVERBS_ID_NS_SHIFT), }; -#define MLX5_IB_DW_MATCH_PARAM 0x90 +#define MLX5_IB_DW_MATCH_PARAM 0xA0 struct mlx5_ib_match_params { __u32 match_params[MLX5_IB_DW_MATCH_PARAM]; -- cgit v1.2.3-59-g8ed1b From f59464e257bdbd4df6df9a4505d7858a0baf6cf7 Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Mon, 8 Nov 2021 02:42:50 +0200 Subject: net/mlx5: DR, Add support for matching on geneve_tlv_option_0_exist field Match on geneve_tlv_option_0_exist field on devices that support STEv1. Signed-off-by: Muhammad Sammar Signed-off-by: Yevgeny Kliteynik --- .../ethernet/mellanox/mlx5/core/steering/dr_cmd.c | 7 ++++++ .../mellanox/mlx5/core/steering/dr_matcher.c | 17 +++++++++++++ .../ethernet/mellanox/mlx5/core/steering/dr_ste.c | 17 +++++++++++++ .../ethernet/mellanox/mlx5/core/steering/dr_ste.h | 1 + .../mellanox/mlx5/core/steering/dr_ste_v1.c | 28 ++++++++++++++++++++++ .../mellanox/mlx5/core/steering/dr_types.h | 9 ++++++- .../mellanox/mlx5/core/steering/mlx5_ifc_dr.h | 8 +++++++ include/linux/mlx5/mlx5_ifc.h | 6 +++-- 8 files changed, 90 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index e1a79eb66f3c..4dd619d238cc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -132,6 +132,13 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, caps->isolate_vl_tc = MLX5_CAP_GEN(mdev, isolate_vl_tc_new); + /* geneve_tlv_option_0_exist is the indication of + * STE support for lookup type flex_parser_ok + */ + caps->flex_parser_ok_bits_supp = + MLX5_CAP_FLOWTABLE(mdev, + flow_table_properties_nic_receive.ft_field_support.geneve_tlv_option_0_exist); + if (caps->flex_protocols & MLX5_FLEX_PARSER_ICMP_V4_ENABLED) { caps->flex_parser_id_icmp_dw0 = MLX5_CAP_GEN(mdev, flex_parser_id_icmp_dw0); caps->flex_parser_id_icmp_dw1 = MLX5_CAP_GEN(mdev, flex_parser_id_icmp_dw1); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c index b3e7a611f99e..d9f66faa619e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c @@ -140,6 +140,19 @@ static bool dr_mask_is_tnl_geneve_tlv_opt(struct mlx5dr_match_misc3 *misc3) return misc3->geneve_tlv_option_0_data; } +static bool +dr_matcher_supp_flex_parser_ok(struct mlx5dr_cmd_caps *caps) +{ + return caps->flex_parser_ok_bits_supp; +} + +static bool dr_mask_is_tnl_geneve_tlv_opt_exist_set(struct mlx5dr_match_misc *misc, + struct mlx5dr_domain *dmn) +{ + return dr_matcher_supp_flex_parser_ok(&dmn->info.caps) && + misc->geneve_tlv_option_0_exist; +} + static bool dr_matcher_supp_tnl_geneve(struct mlx5dr_cmd_caps *caps) { @@ -521,6 +534,10 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, mlx5dr_ste_build_tnl_geneve_tlv_opt(ste_ctx, &sb[idx++], &mask, &dmn->info.caps, inner, rx); + if (dr_mask_is_tnl_geneve_tlv_opt_exist_set(&mask.misc, dmn)) + mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(ste_ctx, &sb[idx++], + &mask, &dmn->info.caps, + inner, rx); } else if (dr_mask_is_tnl_gtpu_any(&mask, dmn)) { if (dr_mask_is_tnl_gtpu_flex_parser_0(&mask, dmn)) mlx5dr_ste_build_tnl_gtpu_flex_parser_0(ste_ctx, &sb[idx++], diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c index 67094dba233c..7e61742e58a0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c @@ -719,6 +719,8 @@ static void dr_ste_copy_mask_misc(char *mask, struct mlx5dr_match_misc *spec, bo spec->vxlan_vni = IFC_GET_CLR(fte_match_set_misc, mask, vxlan_vni, clr); spec->geneve_vni = IFC_GET_CLR(fte_match_set_misc, mask, geneve_vni, clr); + spec->geneve_tlv_option_0_exist = + IFC_GET_CLR(fte_match_set_misc, mask, geneve_tlv_option_0_exist, clr); spec->geneve_oam = IFC_GET_CLR(fte_match_set_misc, mask, geneve_oam, clr); spec->outer_ipv6_flow_label = @@ -1214,6 +1216,21 @@ void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, ste_ctx->build_tnl_geneve_tlv_opt_init(sb, mask); } +void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, + struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask, + struct mlx5dr_cmd_caps *caps, + bool inner, bool rx) +{ + if (!ste_ctx->build_tnl_geneve_tlv_opt_exist_init) + return; + + sb->rx = rx; + sb->caps = caps; + sb->inner = inner; + ste_ctx->build_tnl_geneve_tlv_opt_exist_init(sb, mask); +} + void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, struct mlx5dr_ste_build *sb, struct mlx5dr_match_param *mask, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h index e6c25bdf0da0..ca8fa32b8680 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h @@ -135,6 +135,7 @@ struct mlx5dr_ste_ctx { void DR_STE_CTX_BUILDER(tnl_vxlan_gpe); void DR_STE_CTX_BUILDER(tnl_geneve); void DR_STE_CTX_BUILDER(tnl_geneve_tlv_opt); + void DR_STE_CTX_BUILDER(tnl_geneve_tlv_opt_exist); void DR_STE_CTX_BUILDER(register_0); void DR_STE_CTX_BUILDER(register_1); void DR_STE_CTX_BUILDER(src_gvmi_qpn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c index 9c72be2c2b6b..6ca06800f1d9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c @@ -47,6 +47,7 @@ enum { DR_STE_V1_LU_TYPE_ETHL3_IPV4_MISC_I = 0x000f, DR_STE_V1_LU_TYPE_STEERING_REGISTERS_0 = 0x010f, DR_STE_V1_LU_TYPE_STEERING_REGISTERS_1 = 0x0110, + DR_STE_V1_LU_TYPE_FLEX_PARSER_OK = 0x0011, DR_STE_V1_LU_TYPE_FLEX_PARSER_0 = 0x0111, DR_STE_V1_LU_TYPE_FLEX_PARSER_1 = 0x0112, DR_STE_V1_LU_TYPE_ETHL4_MISC_O = 0x0113, @@ -1942,6 +1943,32 @@ dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_init(struct mlx5dr_ste_build *sb, sb->ste_build_tag_func = &dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_tag; } +static int +dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_exist_tag(struct mlx5dr_match_param *value, + struct mlx5dr_ste_build *sb, + uint8_t *tag) +{ + u8 parser_id = sb->caps->flex_parser_id_geneve_tlv_option_0; + struct mlx5dr_match_misc *misc = &value->misc; + + if (misc->geneve_tlv_option_0_exist) { + MLX5_SET(ste_flex_parser_ok, tag, flex_parsers_ok, 1 << parser_id); + misc->geneve_tlv_option_0_exist = 0; + } + + return 0; +} + +static void +dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_exist_init(struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask) +{ + sb->lu_type = DR_STE_V1_LU_TYPE_FLEX_PARSER_OK; + dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_exist_tag(mask, sb, sb->bit_mask); + sb->byte_mask = mlx5dr_ste_conv_bit_to_byte_mask(sb->bit_mask); + sb->ste_build_tag_func = &dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_exist_tag; +} + static int dr_ste_v1_build_flex_parser_tnl_gtpu_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, u8 *tag) @@ -2041,6 +2068,7 @@ struct mlx5dr_ste_ctx ste_ctx_v1 = { .build_tnl_vxlan_gpe_init = &dr_ste_v1_build_flex_parser_tnl_vxlan_gpe_init, .build_tnl_geneve_init = &dr_ste_v1_build_flex_parser_tnl_geneve_init, .build_tnl_geneve_tlv_opt_init = &dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_init, + .build_tnl_geneve_tlv_opt_exist_init = &dr_ste_v1_build_flex_parser_tnl_geneve_tlv_opt_exist_init, .build_register_0_init = &dr_ste_v1_build_register_0_init, .build_register_1_init = &dr_ste_v1_build_register_1_init, .build_src_gvmi_qpn_init = &dr_ste_v1_build_src_gvmi_qpn_init, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 5805e2554a59..21a9b07ba327 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -442,6 +442,11 @@ void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx, struct mlx5dr_match_param *mask, struct mlx5dr_cmd_caps *caps, bool inner, bool rx); +void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx, + struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask, + struct mlx5dr_cmd_caps *caps, + bool inner, bool rx); void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx, struct mlx5dr_ste_build *sb, struct mlx5dr_match_param *mask, @@ -666,7 +671,8 @@ struct mlx5dr_match_misc { u32 reserved_auto3:8; u32 geneve_vni:24; /* GENEVE VNI field (outer) */ - u32 reserved_auto4:7; + u32 reserved_auto4:6; + u32 geneve_tlv_option_0_exist:1; u32 geneve_oam:1; /* GENEVE OAM field (outer) */ u32 reserved_auto5:12; @@ -842,6 +848,7 @@ struct mlx5dr_cmd_caps { u8 flex_parser_id_gtpu_teid; u8 flex_parser_id_gtpu_dw_2; u8 flex_parser_id_gtpu_first_ext_dw_0; + u8 flex_parser_ok_bits_supp; u8 max_ft_level; u16 roce_min_src_udp; u8 sw_format_ver; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h index d0e20bda2622..9604b2091358 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h @@ -447,6 +447,14 @@ struct mlx5_ifc_ste_flex_parser_1_bits { u8 flex_parser_4[0x20]; }; +struct mlx5_ifc_ste_flex_parser_ok_bits { + u8 flex_parser_3[0x20]; + u8 flex_parser_2[0x20]; + u8 flex_parsers_ok[0x8]; + u8 reserved_at_48[0x18]; + u8 flex_parser_0[0x20]; +}; + struct mlx5_ifc_ste_flex_parser_tnl_bits { u8 flex_parser_tunneling_header_63_32[0x20]; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index c74c5e147cb9..deaa0f71213f 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -372,7 +372,8 @@ struct mlx5_ifc_flow_table_fields_supported_bits { u8 reserved_at_37[0x9]; u8 geneve_tlv_option_0_data[0x1]; - u8 reserved_at_41[0x4]; + u8 geneve_tlv_option_0_exist[0x1]; + u8 reserved_at_42[0x3]; u8 outer_first_mpls_over_udp[0x4]; u8 outer_first_mpls_over_gre[0x4]; u8 inner_first_mpls[0x4]; @@ -551,7 +552,8 @@ struct mlx5_ifc_fte_match_set_misc_bits { u8 bth_opcode[0x8]; u8 geneve_vni[0x18]; - u8 reserved_at_d8[0x7]; + u8 reserved_at_d8[0x6]; + u8 geneve_tlv_option_0_exist[0x1]; u8 geneve_oam[0x1]; u8 reserved_at_e0[0xc]; -- cgit v1.2.3-59-g8ed1b From 4ff725e1d4adfd313bc9767523fc8d6e90d50f9c Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Tue, 23 Nov 2021 02:11:12 +0200 Subject: net/mlx5: DR, Ignore modify TTL if device doesn't support it When modifying TTL, packet's csum has to be recalculated. Due to HW issue in ConnectX-5, csum recalculation for modify TTL is supported through a work-around that is specifically enabled by configuration. If the work-around isn't enabled, ignore the modify TTL action rather than adding an unsupported action. Signed-off-by: Yevgeny Kliteynik --- .../mellanox/mlx5/core/steering/dr_action.c | 21 ++++++++++++++++++--- include/linux/mlx5/mlx5_ifc.h | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index f0faf04536d3..c61a5e83c78c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -1560,6 +1560,12 @@ dr_action_modify_check_is_ttl_modify(const void *sw_action) return sw_field == MLX5_ACTION_IN_FIELD_OUT_IP_TTL; } +static bool dr_action_modify_ttl_ignore(struct mlx5dr_domain *dmn) +{ + return !mlx5dr_ste_supp_ttl_cs_recalc(&dmn->info.caps) && + !MLX5_CAP_ESW_FLOWTABLE(dmn->mdev, fdb_ipv4_ttl_modify); +} + static int dr_actions_convert_modify_header(struct mlx5dr_action *action, u32 max_hw_actions, u32 num_sw_actions, @@ -1591,8 +1597,13 @@ static int dr_actions_convert_modify_header(struct mlx5dr_action *action, if (ret) return ret; - if (!(*modify_ttl)) - *modify_ttl = dr_action_modify_check_is_ttl_modify(sw_action); + if (!(*modify_ttl) && + dr_action_modify_check_is_ttl_modify(sw_action)) { + if (dr_action_modify_ttl_ignore(dmn)) + continue; + + *modify_ttl = true; + } /* Convert SW action to HW action */ ret = dr_action_modify_sw_to_hw(dmn, @@ -1631,7 +1642,7 @@ static int dr_actions_convert_modify_header(struct mlx5dr_action *action, * modify actions doesn't exceeds the limit */ hw_idx++; - if ((num_sw_actions + hw_idx - i) >= max_hw_actions) { + if (hw_idx >= max_hw_actions) { mlx5dr_dbg(dmn, "Modify header action number exceeds HW limit\n"); return -EINVAL; } @@ -1642,6 +1653,10 @@ static int dr_actions_convert_modify_header(struct mlx5dr_action *action, hw_idx++; } + /* if the resulting HW actions list is empty, add NOP action */ + if (!hw_idx) + hw_idx++; + *num_hw_actions = hw_idx; return 0; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index deaa0f71213f..598ac3bcc901 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -833,7 +833,7 @@ struct mlx5_ifc_flow_table_eswitch_cap_bits { u8 fdb_to_vport_reg_c_id[0x8]; u8 reserved_at_8[0xd]; u8 fdb_modify_header_fwd_to_table[0x1]; - u8 reserved_at_16[0x1]; + u8 fdb_ipv4_ttl_modify[0x1]; u8 flow_source[0x1]; u8 reserved_at_18[0x2]; u8 multi_fdb_encap[0x1]; -- cgit v1.2.3-59-g8ed1b From 99a507a8ea28542ec196e2dd80096708e2482735 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 31 Dec 2021 13:42:30 +0100 Subject: Revert "serdev: BREAK/FRAME/PARITY/OVERRUN notification prototype V2" This reverts commit d8e9a406a931f687945703a4bac45042eb81ce92. It needs some future changes as pointed out by Johan and is not ready to be merged just yet. Reported-by: Johan Hovold Cc: Magnus Damm Link: https://lore.kernel.org/r/Yc7oZ/1tu95Z4wPS@hovoldconsulting.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serdev/core.c | 11 --------- drivers/tty/serdev/serdev-ttyport.c | 49 ------------------------------------- include/linux/serdev.h | 22 ----------------- 3 files changed, 82 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c index 5c25ae20e508..f1324fe99378 100644 --- a/drivers/tty/serdev/core.c +++ b/drivers/tty/serdev/core.c @@ -349,17 +349,6 @@ unsigned int serdev_device_set_baudrate(struct serdev_device *serdev, unsigned i } EXPORT_SYMBOL_GPL(serdev_device_set_baudrate); -void serdev_device_set_error_mask(struct serdev_device *serdev, unsigned long mask) -{ - struct serdev_controller *ctrl = serdev->ctrl; - - if (!ctrl || !ctrl->ops->set_error_mask) - return; - - ctrl->ops->set_error_mask(ctrl, mask); -} -EXPORT_SYMBOL_GPL(serdev_device_set_error_mask); - void serdev_device_set_flow_control(struct serdev_device *serdev, bool enable) { struct serdev_controller *ctrl = serdev->ctrl; diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c index 239a1d5c66df..d367803e2044 100644 --- a/drivers/tty/serdev/serdev-ttyport.c +++ b/drivers/tty/serdev/serdev-ttyport.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2016-2017 Linaro Ltd., Rob Herring */ -#include #include #include #include @@ -10,10 +9,6 @@ #include #define SERPORT_ACTIVE 1 -#define SERPORT_NOTIFY_BREAK 2 -#define SERPORT_NOTIFY_FRAME 3 -#define SERPORT_NOTIFY_PARITY 4 -#define SERPORT_NOTIFY_OVERRUN 5 struct serport { struct tty_port *port; @@ -32,39 +27,11 @@ static int ttyport_receive_buf(struct tty_port *port, const unsigned char *cp, { struct serdev_controller *ctrl = port->client_data; struct serport *serport = serdev_controller_get_drvdata(ctrl); - unsigned long errors = 0; - unsigned int i; int ret; if (!test_bit(SERPORT_ACTIVE, &serport->flags)) return 0; - for (i = 0; fp && i < count; i++) { - switch (fp[i]) { - case TTY_BREAK: - if (test_bit(SERPORT_NOTIFY_BREAK, &serport->flags)) - __set_bit(SERDEV_ERROR_BREAK, &errors); - break; - - case TTY_FRAME: - if (test_bit(SERPORT_NOTIFY_FRAME, &serport->flags)) - __set_bit(SERDEV_ERROR_FRAME, &errors); - break; - - case TTY_PARITY: - if (test_bit(SERPORT_NOTIFY_PARITY, &serport->flags)) - __set_bit(SERDEV_ERROR_PARITY, &errors); - break; - - case TTY_OVERRUN: - if (test_bit(SERPORT_NOTIFY_OVERRUN, &serport->flags)) - __set_bit(SERDEV_ERROR_OVERRUN, &errors); - break; - } - } - if (errors) - serdev_controller_error(ctrl, errors); - ret = serdev_controller_receive_buf(ctrl, cp, count); dev_WARN_ONCE(&ctrl->dev, ret < 0 || ret > count, @@ -213,21 +180,6 @@ static unsigned int ttyport_set_baudrate(struct serdev_controller *ctrl, unsigne return ktermios.c_ospeed; } -static void ttyport_set_error_mask(struct serdev_controller *ctrl, - unsigned long m) -{ - struct serport *sp = serdev_controller_get_drvdata(ctrl); - - assign_bit(SERPORT_NOTIFY_BREAK, &sp->flags, - m & BIT(SERDEV_ERROR_BREAK)); - assign_bit(SERPORT_NOTIFY_FRAME, &sp->flags, - m & BIT(SERDEV_ERROR_FRAME)); - assign_bit(SERPORT_NOTIFY_PARITY, &sp->flags, - m & BIT(SERDEV_ERROR_PARITY)); - assign_bit(SERPORT_NOTIFY_OVERRUN, &sp->flags, - m & BIT(SERDEV_ERROR_OVERRUN)); -} - static void ttyport_set_flow_control(struct serdev_controller *ctrl, bool enable) { struct serport *serport = serdev_controller_get_drvdata(ctrl); @@ -301,7 +253,6 @@ static const struct serdev_controller_ops ctrl_ops = { .write_room = ttyport_write_room, .open = ttyport_open, .close = ttyport_close, - .set_error_mask = ttyport_set_error_mask, .set_flow_control = ttyport_set_flow_control, .set_parity = ttyport_set_parity, .set_baudrate = ttyport_set_baudrate, diff --git a/include/linux/serdev.h b/include/linux/serdev.h index 0d0b22fc7e37..3368c261ab62 100644 --- a/include/linux/serdev.h +++ b/include/linux/serdev.h @@ -19,15 +19,12 @@ struct serdev_device; /** * struct serdev_device_ops - Callback operations for a serdev device - * @error: Function called with errors received from device; - * may sleep. * @receive_buf: Function called with data received from device; * returns number of bytes accepted; may sleep. * @write_wakeup: Function called when ready to transmit more data; must * not sleep. */ struct serdev_device_ops { - void (*error)(struct serdev_device *, unsigned long); int (*receive_buf)(struct serdev_device *, const unsigned char *, size_t); void (*write_wakeup)(struct serdev_device *); }; @@ -79,11 +76,6 @@ enum serdev_parity { SERDEV_PARITY_ODD, }; -#define SERDEV_ERROR_BREAK 0 -#define SERDEV_ERROR_FRAME 1 -#define SERDEV_ERROR_PARITY 2 -#define SERDEV_ERROR_OVERRUN 3 - /* * serdev controller structures */ @@ -93,7 +85,6 @@ struct serdev_controller_ops { int (*write_room)(struct serdev_controller *); int (*open)(struct serdev_controller *); void (*close)(struct serdev_controller *); - void (*set_error_mask)(struct serdev_controller *, unsigned long); void (*set_flow_control)(struct serdev_controller *, bool); int (*set_parity)(struct serdev_controller *, enum serdev_parity); unsigned int (*set_baudrate)(struct serdev_controller *, unsigned int); @@ -199,24 +190,12 @@ static inline int serdev_controller_receive_buf(struct serdev_controller *ctrl, return serdev->ops->receive_buf(serdev, data, count); } -static inline void serdev_controller_error(struct serdev_controller *ctrl, - unsigned long errors) -{ - struct serdev_device *serdev = ctrl->serdev; - - if (!serdev || !serdev->ops->error) - return; - - serdev->ops->error(serdev, errors); -} - #if IS_ENABLED(CONFIG_SERIAL_DEV_BUS) int serdev_device_open(struct serdev_device *); void serdev_device_close(struct serdev_device *); int devm_serdev_device_open(struct device *, struct serdev_device *); unsigned int serdev_device_set_baudrate(struct serdev_device *, unsigned int); -void serdev_device_set_error_mask(struct serdev_device *, unsigned long); void serdev_device_set_flow_control(struct serdev_device *, bool); int serdev_device_write_buf(struct serdev_device *, const unsigned char *, size_t); void serdev_device_wait_until_sent(struct serdev_device *, long); @@ -259,7 +238,6 @@ static inline unsigned int serdev_device_set_baudrate(struct serdev_device *sdev { return 0; } -static inline void serdev_device_set_error_mask(struct serdev_device *sdev, unsigned long mask) {} static inline void serdev_device_set_flow_control(struct serdev_device *sdev, bool enable) {} static inline int serdev_device_write_buf(struct serdev_device *serdev, const unsigned char *buf, -- cgit v1.2.3-59-g8ed1b From e7026f15564fbe0c8b091f218203111f77b84eda Mon Sep 17 00:00:00 2001 From: Colin Foster Date: Tue, 28 Dec 2021 21:03:06 -0800 Subject: net: phy: lynx: refactor Lynx PCS module to use generic phylink_pcs Remove references to lynx_pcs structures so drivers like the Felix DSA can reference alternate PCS drivers. Signed-off-by: Colin Foster Signed-off-by: David S. Miller --- drivers/net/dsa/ocelot/felix.c | 3 +-- drivers/net/dsa/ocelot/felix.h | 2 +- drivers/net/dsa/ocelot/felix_vsc9959.c | 20 +++++++++++--------- drivers/net/dsa/ocelot/seville_vsc9953.c | 22 ++++++++++++---------- drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c | 13 ++++++++----- drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h | 3 +-- drivers/net/ethernet/freescale/enetc/enetc_pf.c | 16 ++++++++++------ drivers/net/ethernet/freescale/enetc/enetc_pf.h | 4 ++-- drivers/net/pcs/pcs-lynx.c | 24 ++++++++++++++++++++---- include/linux/pcs-lynx.h | 9 +++------ 10 files changed, 69 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index f4fc403fbc1e..bb2a43070ea8 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include "felix.h" @@ -832,7 +831,7 @@ static void felix_phylink_mac_config(struct dsa_switch *ds, int port, struct dsa_port *dp = dsa_to_port(ds, port); if (felix->pcs && felix->pcs[port]) - phylink_set_pcs(dp->pl, &felix->pcs[port]->pcs); + phylink_set_pcs(dp->pl, felix->pcs[port]); } static void felix_phylink_mac_link_down(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index 515bddc012c0..9395ac119d33 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -62,7 +62,7 @@ struct felix { const struct felix_info *info; struct ocelot ocelot; struct mii_bus *imdio; - struct lynx_pcs **pcs; + struct phylink_pcs **pcs; resource_size_t switch_base; resource_size_t imdio_base; enum dsa_tag_protocol tag_proto; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 4ffd303c64ea..93ad1d65e212 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -1039,7 +1039,7 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) int rc; felix->pcs = devm_kcalloc(dev, felix->info->num_ports, - sizeof(struct lynx_pcs *), + sizeof(struct phylink_pcs *), GFP_KERNEL); if (!felix->pcs) { dev_err(dev, "failed to allocate array for PCS PHYs\n"); @@ -1088,8 +1088,8 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) for (port = 0; port < felix->info->num_ports; port++) { struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct phylink_pcs *phylink_pcs; struct mdio_device *pcs; - struct lynx_pcs *lynx; if (dsa_is_unused_port(felix->ds, port)) continue; @@ -1101,13 +1101,13 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) if (IS_ERR(pcs)) continue; - lynx = lynx_pcs_create(pcs); - if (!lynx) { + phylink_pcs = lynx_pcs_create(pcs); + if (!phylink_pcs) { mdio_device_free(pcs); continue; } - felix->pcs[port] = lynx; + felix->pcs[port] = phylink_pcs; dev_info(dev, "Found PCS at internal MDIO address %d\n", port); } @@ -1121,13 +1121,15 @@ static void vsc9959_mdio_bus_free(struct ocelot *ocelot) int port; for (port = 0; port < ocelot->num_phys_ports; port++) { - struct lynx_pcs *pcs = felix->pcs[port]; + struct phylink_pcs *phylink_pcs = felix->pcs[port]; + struct mdio_device *mdio_device; - if (!pcs) + if (!phylink_pcs) continue; - mdio_device_free(pcs->mdio); - lynx_pcs_destroy(pcs); + mdio_device = lynx_get_mdio_device(phylink_pcs); + mdio_device_free(mdio_device); + lynx_pcs_destroy(phylink_pcs); } mdiobus_unregister(felix->imdio); } diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index e110550e3507..d34d0f737c16 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -1012,7 +1012,7 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) int rc; felix->pcs = devm_kcalloc(dev, felix->info->num_ports, - sizeof(struct phy_device *), + sizeof(struct phylink_pcs *), GFP_KERNEL); if (!felix->pcs) { dev_err(dev, "failed to allocate array for PCS PHYs\n"); @@ -1039,9 +1039,9 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) for (port = 0; port < felix->info->num_ports; port++) { struct ocelot_port *ocelot_port = ocelot->ports[port]; - int addr = port + 4; + struct phylink_pcs *phylink_pcs; struct mdio_device *pcs; - struct lynx_pcs *lynx; + int addr = port + 4; if (dsa_is_unused_port(felix->ds, port)) continue; @@ -1053,13 +1053,13 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) if (IS_ERR(pcs)) continue; - lynx = lynx_pcs_create(pcs); - if (!lynx) { + phylink_pcs = lynx_pcs_create(pcs); + if (!phylink_pcs) { mdio_device_free(pcs); continue; } - felix->pcs[port] = lynx; + felix->pcs[port] = phylink_pcs; dev_info(dev, "Found PCS at internal MDIO address %d\n", addr); } @@ -1073,13 +1073,15 @@ static void vsc9953_mdio_bus_free(struct ocelot *ocelot) int port; for (port = 0; port < ocelot->num_phys_ports; port++) { - struct lynx_pcs *pcs = felix->pcs[port]; + struct phylink_pcs *phylink_pcs = felix->pcs[port]; + struct mdio_device *mdio_device; - if (!pcs) + if (!phylink_pcs) continue; - mdio_device_free(pcs->mdio); - lynx_pcs_destroy(pcs); + mdio_device = lynx_get_mdio_device(phylink_pcs); + mdio_device_free(mdio_device); + lynx_pcs_destroy(phylink_pcs); } mdiobus_unregister(felix->imdio); } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index 34b2a73c347f..7f509f427e3d 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -2,6 +2,7 @@ /* Copyright 2019 NXP */ #include +#include #include #include "dpaa2-eth.h" @@ -204,11 +205,13 @@ static int dpaa2_pcs_create(struct dpaa2_mac *mac, static void dpaa2_pcs_destroy(struct dpaa2_mac *mac) { - struct lynx_pcs *pcs = mac->pcs; + struct phylink_pcs *phylink_pcs = mac->pcs; - if (pcs) { - struct device *dev = &pcs->mdio->dev; - lynx_pcs_destroy(pcs); + if (phylink_pcs) { + struct mdio_device *mdio = lynx_get_mdio_device(phylink_pcs); + struct device *dev = &mdio->dev; + + lynx_pcs_destroy(phylink_pcs); put_device(dev); mac->pcs = NULL; } @@ -292,7 +295,7 @@ int dpaa2_mac_connect(struct dpaa2_mac *mac) mac->phylink = phylink; if (mac->pcs) - phylink_set_pcs(mac->phylink, &mac->pcs->pcs); + phylink_set_pcs(mac->phylink, mac->pcs); err = phylink_fwnode_phy_connect(mac->phylink, dpmac_node, 0); if (err) { diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h index 7842cbb2207a..1331a8477fe4 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h @@ -7,7 +7,6 @@ #include #include #include -#include #include "dpmac.h" #include "dpmac-cmd.h" @@ -23,7 +22,7 @@ struct dpaa2_mac { struct phylink *phylink; phy_interface_t if_mode; enum dpmac_link_type if_link_type; - struct lynx_pcs *pcs; + struct phylink_pcs *pcs; struct fwnode_handle *fw_node; }; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index fe6a544f37f0..e36d2d2ba03d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "enetc_ierb.h" #include "enetc_pf.h" @@ -828,7 +829,7 @@ static int enetc_imdio_create(struct enetc_pf *pf) { struct device *dev = &pf->si->pdev->dev; struct enetc_mdio_priv *mdio_priv; - struct lynx_pcs *pcs_lynx; + struct phylink_pcs *phylink_pcs; struct mdio_device *pcs; struct mii_bus *bus; int err; @@ -860,8 +861,8 @@ static int enetc_imdio_create(struct enetc_pf *pf) goto unregister_mdiobus; } - pcs_lynx = lynx_pcs_create(pcs); - if (!pcs_lynx) { + phylink_pcs = lynx_pcs_create(pcs); + if (!phylink_pcs) { mdio_device_free(pcs); err = -ENOMEM; dev_err(dev, "cannot create lynx pcs (%d)\n", err); @@ -869,7 +870,7 @@ static int enetc_imdio_create(struct enetc_pf *pf) } pf->imdio = bus; - pf->pcs = pcs_lynx; + pf->pcs = phylink_pcs; return 0; @@ -882,8 +883,11 @@ free_mdio_bus: static void enetc_imdio_remove(struct enetc_pf *pf) { + struct mdio_device *mdio_device; + if (pf->pcs) { - mdio_device_free(pf->pcs->mdio); + mdio_device = lynx_get_mdio_device(pf->pcs); + mdio_device_free(mdio_device); lynx_pcs_destroy(pf->pcs); } if (pf->imdio) { @@ -941,7 +945,7 @@ static void enetc_pl_mac_config(struct phylink_config *config, priv = netdev_priv(pf->si->ndev); if (pf->pcs) - phylink_set_pcs(priv->phylink, &pf->pcs->pcs); + phylink_set_pcs(priv->phylink, pf->pcs); } static void enetc_force_rgmii_mac(struct enetc_hw *hw, int speed, int duplex) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.h b/drivers/net/ethernet/freescale/enetc/enetc_pf.h index 263946c51e37..c26bd66e4597 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.h @@ -2,7 +2,7 @@ /* Copyright 2017-2019 NXP */ #include "enetc.h" -#include +#include #define ENETC_PF_NUM_RINGS 8 @@ -46,7 +46,7 @@ struct enetc_pf { struct mii_bus *mdio; /* saved for cleanup */ struct mii_bus *imdio; - struct lynx_pcs *pcs; + struct phylink_pcs *pcs; phy_interface_t if_mode; struct phylink_config phylink_config; diff --git a/drivers/net/pcs/pcs-lynx.c b/drivers/net/pcs/pcs-lynx.c index af36cd647bf5..7ff7f86ad430 100644 --- a/drivers/net/pcs/pcs-lynx.c +++ b/drivers/net/pcs/pcs-lynx.c @@ -22,6 +22,11 @@ #define IF_MODE_SPEED_MSK GENMASK(3, 2) #define IF_MODE_HALF_DUPLEX BIT(4) +struct lynx_pcs { + struct phylink_pcs pcs; + struct mdio_device *mdio; +}; + enum sgmii_speed { SGMII_SPEED_10 = 0, SGMII_SPEED_100 = 1, @@ -30,6 +35,15 @@ enum sgmii_speed { }; #define phylink_pcs_to_lynx(pl_pcs) container_of((pl_pcs), struct lynx_pcs, pcs) +#define lynx_to_phylink_pcs(lynx) (&(lynx)->pcs) + +struct mdio_device *lynx_get_mdio_device(struct phylink_pcs *pcs) +{ + struct lynx_pcs *lynx = phylink_pcs_to_lynx(pcs); + + return lynx->mdio; +} +EXPORT_SYMBOL(lynx_get_mdio_device); static void lynx_pcs_get_state_usxgmii(struct mdio_device *pcs, struct phylink_link_state *state) @@ -329,7 +343,7 @@ static const struct phylink_pcs_ops lynx_pcs_phylink_ops = { .pcs_link_up = lynx_pcs_link_up, }; -struct lynx_pcs *lynx_pcs_create(struct mdio_device *mdio) +struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio) { struct lynx_pcs *lynx_pcs; @@ -341,13 +355,15 @@ struct lynx_pcs *lynx_pcs_create(struct mdio_device *mdio) lynx_pcs->pcs.ops = &lynx_pcs_phylink_ops; lynx_pcs->pcs.poll = true; - return lynx_pcs; + return lynx_to_phylink_pcs(lynx_pcs); } EXPORT_SYMBOL(lynx_pcs_create); -void lynx_pcs_destroy(struct lynx_pcs *pcs) +void lynx_pcs_destroy(struct phylink_pcs *pcs) { - kfree(pcs); + struct lynx_pcs *lynx = phylink_pcs_to_lynx(pcs); + + kfree(lynx); } EXPORT_SYMBOL(lynx_pcs_destroy); diff --git a/include/linux/pcs-lynx.h b/include/linux/pcs-lynx.h index a6440d6ebe95..5712cc2ce775 100644 --- a/include/linux/pcs-lynx.h +++ b/include/linux/pcs-lynx.h @@ -9,13 +9,10 @@ #include #include -struct lynx_pcs { - struct phylink_pcs pcs; - struct mdio_device *mdio; -}; +struct mdio_device *lynx_get_mdio_device(struct phylink_pcs *pcs); -struct lynx_pcs *lynx_pcs_create(struct mdio_device *mdio); +struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio); -void lynx_pcs_destroy(struct lynx_pcs *pcs); +void lynx_pcs_destroy(struct phylink_pcs *pcs); #endif /* __LINUX_PCS_LYNX_H */ -- cgit v1.2.3-59-g8ed1b From ece014141cd4b49f2d763f28b19e417b84460560 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 3 May 2021 07:29:47 -0400 Subject: mm/doc: Add documentation for folio_test_uptodate Move the PG_uptodate documentation to be documentation for folio_test_uptodate() and expand on it a little. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/page-flags.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index b5f14d581113..b3d353d537e2 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -68,9 +68,6 @@ * might lose their PG_swapbacked flag when they simply can be dropped (e.g. as * a result of MADV_FREE). * - * PG_uptodate tells whether the page's contents is valid. When a read - * completes, the page becomes uptodate, unless a disk I/O error happened. - * * PG_referenced, PG_reclaim are used for page reclaim for anonymous and * file-backed pagecache (see mm/vmscan.c). * @@ -615,6 +612,16 @@ TESTPAGEFLAG_FALSE(Ksm, ksm) u64 stable_page_flags(struct page *page); +/** + * folio_test_uptodate - Is this folio up to date? + * @folio: The folio. + * + * The uptodate flag is set on a folio when every byte in the folio is + * at least as new as the corresponding bytes on storage. Anonymous + * and CoW folios are always uptodate. If the folio is not uptodate, + * some of the bytes in it may be; see the is_partially_uptodate() + * address_space operation. + */ static inline bool folio_test_uptodate(struct folio *folio) { bool ret = test_bit(PG_uptodate, folio_flags(folio, 0)); -- cgit v1.2.3-59-g8ed1b From 10331795fb7991a39ebd0330fdb074cbd81fef48 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 6 Dec 2021 15:24:51 -0500 Subject: pagevec: Add folio_batch The folio_batch is the same as the pagevec, except that it is typed to contain folios and not pages. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- include/linux/pagevec.h | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 7f3f19065a9f..c3fa616d7ae7 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -15,8 +15,10 @@ #define PAGEVEC_SIZE 15 struct page; +struct folio; struct address_space; +/* Layout must match folio_batch */ struct pagevec { unsigned char nr; bool percpu_pvec_drained; @@ -81,4 +83,71 @@ static inline void pagevec_release(struct pagevec *pvec) __pagevec_release(pvec); } +/** + * struct folio_batch - A collection of folios. + * + * The folio_batch is used to amortise the cost of retrieving and + * operating on a set of folios. The order of folios in the batch may be + * significant (eg delete_from_page_cache_batch()). Some users of the + * folio_batch store "exceptional" entries in it which can be removed + * by calling folio_batch_remove_exceptionals(). + */ +struct folio_batch { + unsigned char nr; + bool percpu_pvec_drained; + struct folio *folios[PAGEVEC_SIZE]; +}; + +/* Layout must match pagevec */ +static_assert(sizeof(struct pagevec) == sizeof(struct folio_batch)); +static_assert(offsetof(struct pagevec, pages) == + offsetof(struct folio_batch, folios)); + +/** + * folio_batch_init() - Initialise a batch of folios + * @fbatch: The folio batch. + * + * A freshly initialised folio_batch contains zero folios. + */ +static inline void folio_batch_init(struct folio_batch *fbatch) +{ + fbatch->nr = 0; +} + +static inline unsigned int folio_batch_count(struct folio_batch *fbatch) +{ + return fbatch->nr; +} + +static inline unsigned int fbatch_space(struct folio_batch *fbatch) +{ + return PAGEVEC_SIZE - fbatch->nr; +} + +/** + * folio_batch_add() - Add a folio to a batch. + * @fbatch: The folio batch. + * @folio: The folio to add. + * + * The folio is added to the end of the batch. + * The batch must have previously been initialised using folio_batch_init(). + * + * Return: The number of slots still available. + */ +static inline unsigned folio_batch_add(struct folio_batch *fbatch, + struct folio *folio) +{ + fbatch->folios[fbatch->nr++] = folio; + return fbatch_space(fbatch); +} + +static inline void folio_batch_release(struct folio_batch *fbatch) +{ + pagevec_release((struct pagevec *)fbatch); +} + +static inline void folio_batch_remove_exceptionals(struct folio_batch *fbatch) +{ + pagevec_remove_exceptionals((struct pagevec *)fbatch); +} #endif /* _LINUX_PAGEVEC_H */ -- cgit v1.2.3-59-g8ed1b From e66d70c034dbdfe1a48863f0865ac86aaf2fef1a Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Mon, 13 Dec 2021 10:51:41 +0530 Subject: dmaengine: xilinx_dpdma: use correct SDPX tag for header file Commit 188c310bdd5d ("dmaengine: xilinx_dpdma: stop using slave_id field") add the header file with incorrect format for SPDX tag, fix that WARNING: Improper SPDX comment style for 'include/linux/dma/xilinx_dpdma.h', please use '/*' instead #1: FILE: include/linux/dma/xilinx_dpdma.h:1: +// SPDX-License-Identifier: GPL-2.0 WARNING: Missing or malformed SPDX-License-Identifier tag in line 1 #1: FILE: include/linux/dma/xilinx_dpdma.h:1: +// SPDX-License-Identifier: GPL-2.0 Fixes: 188c310bdd5d ("dmaengine: xilinx_dpdma: stop using slave_id field") Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20211213052141.850807-1-vkoul@kernel.org Signed-off-by: Vinod Koul --- include/linux/dma/xilinx_dpdma.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma/xilinx_dpdma.h b/include/linux/dma/xilinx_dpdma.h index 83a1377f03f8..02a4adf8921b 100644 --- a/include/linux/dma/xilinx_dpdma.h +++ b/include/linux/dma/xilinx_dpdma.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __LINUX_DMA_XILINX_DPDMA_H #define __LINUX_DMA_XILINX_DPDMA_H -- cgit v1.2.3-59-g8ed1b From 25fd330370ac40653671f323acc7fb6db27ef6fe Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 15 Dec 2021 02:01:18 +0100 Subject: power: supply_core: Pass pointer to battery info The function to retrieve battery info (from the device tree) assumes we have a static info struct that gets populated by calling into power_supply_get_battery_info(). This is awkward since I want to support tables of static battery info by just assigning a pointer to all info based on e.g. a compatible value in the device tree. We also have a mixture of static and dynamically allocated variables here. Bite the bullet and let power_supply_get_battery_info() allocate also the memory used for the very top level struct power_supply_battery_info container. Pass pointers around and lifecycle this with the psy device just like the stuff we allocate inside it. Change all current users over. As part of the change, initializers need to be added to some previously uninitialized fields in struct power_supply_battery_info. Reviewed-By: Matti Vaittinen Signed-off-by: Linus Walleij Signed-off-by: Sebastian Reichel --- drivers/power/supply/ab8500-bm.h | 2 +- drivers/power/supply/ab8500_bmdata.c | 7 +++--- drivers/power/supply/ab8500_btemp.c | 10 ++++++--- drivers/power/supply/ab8500_chargalg.c | 16 +++++++------- drivers/power/supply/ab8500_fg.c | 18 ++++++++------- drivers/power/supply/axp20x_battery.c | 6 ++--- drivers/power/supply/bd99954-charger.c | 24 +++++++++++--------- drivers/power/supply/bq24190_charger.c | 6 ++--- drivers/power/supply/bq2515x_charger.c | 8 +++---- drivers/power/supply/bq256xx_charger.c | 24 ++++++++++---------- drivers/power/supply/bq25980_charger.c | 6 ++--- drivers/power/supply/bq27xxx_battery.c | 38 ++++++++++++++++---------------- drivers/power/supply/cw2015_battery.c | 20 +++++++++++------ drivers/power/supply/ingenic-battery.c | 14 ++++++------ drivers/power/supply/power_supply_core.c | 19 ++++++++++++++-- drivers/power/supply/sc2731_charger.c | 8 +++---- drivers/power/supply/sc27xx_fuel_gauge.c | 22 +++++++++--------- drivers/power/supply/smb347-charger.c | 34 ++++++++++++++-------------- include/linux/power_supply.h | 2 +- 19 files changed, 157 insertions(+), 127 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/ab8500-bm.h b/drivers/power/supply/ab8500-bm.h index 57e1a8e27e51..56a5aaf9a27a 100644 --- a/drivers/power/supply/ab8500-bm.h +++ b/drivers/power/supply/ab8500-bm.h @@ -439,7 +439,7 @@ struct ab8500_bm_charger_parameters { * @fg_params fuel gauge parameters */ struct ab8500_bm_data { - struct power_supply_battery_info bi; + struct power_supply_battery_info *bi; int temp_now; int temp_interval_chg; int temp_interval_nochg; diff --git a/drivers/power/supply/ab8500_bmdata.c b/drivers/power/supply/ab8500_bmdata.c index 62953f9cb85a..7ae95f537580 100644 --- a/drivers/power/supply/ab8500_bmdata.c +++ b/drivers/power/supply/ab8500_bmdata.c @@ -167,15 +167,16 @@ struct ab8500_bm_data ab8500_bm_data = { int ab8500_bm_of_probe(struct power_supply *psy, struct ab8500_bm_data *bm) { - struct power_supply_battery_info *bi = &bm->bi; + struct power_supply_battery_info *bi; struct device *dev = &psy->dev; int ret; - ret = power_supply_get_battery_info(psy, bi); + ret = power_supply_get_battery_info(psy, &bm->bi); if (ret) { dev_err(dev, "cannot retrieve battery info\n"); return ret; } + bi = bm->bi; /* Fill in defaults for any data missing from the device tree */ if (bi->charge_full_design_uah < 0) @@ -240,5 +241,5 @@ int ab8500_bm_of_probe(struct power_supply *psy, void ab8500_bm_of_remove(struct power_supply *psy, struct ab8500_bm_data *bm) { - power_supply_put_battery_info(psy, &bm->bi); + power_supply_put_battery_info(psy, bm->bi); } diff --git a/drivers/power/supply/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c index 20253b8a7fe9..cc33c5187fbb 100644 --- a/drivers/power/supply/ab8500_btemp.c +++ b/drivers/power/supply/ab8500_btemp.c @@ -451,12 +451,13 @@ static int ab8500_btemp_res_to_temp(struct ab8500_btemp *di, */ static int ab8500_btemp_measure_temp(struct ab8500_btemp *di) { + struct power_supply_battery_info *bi = di->bm->bi; int temp, ret; static int prev; int rbat, rntc, vntc; if ((di->bm->adc_therm == AB8500_ADC_THERM_BATCTRL) && - (di->bm->bi.technology == POWER_SUPPLY_TECHNOLOGY_UNKNOWN)) { + (bi && (bi->technology == POWER_SUPPLY_TECHNOLOGY_UNKNOWN))) { rbat = ab8500_btemp_get_batctrl_res(di); if (rbat < 0) { @@ -540,7 +541,7 @@ static int ab8500_btemp_id(struct ab8500_btemp *di) * that need it. */ if ((di->bm->adc_therm == AB8500_ADC_THERM_BATCTRL) && - (di->bm->bi.technology == POWER_SUPPLY_TECHNOLOGY_LIPO) && + (di->bm->bi && (di->bm->bi->technology == POWER_SUPPLY_TECHNOLOGY_LIPO)) && (res <= 53407) && (res >= 12500)) { dev_dbg(di->dev, "Set BATCTRL current source to 20uA\n"); di->curr_source = BTEMP_BATCTRL_CURR_SRC_20UA; @@ -807,7 +808,10 @@ static int ab8500_btemp_get_property(struct power_supply *psy, val->intval = 1; break; case POWER_SUPPLY_PROP_TECHNOLOGY: - val->intval = di->bm->bi.technology; + if (di->bm->bi) + val->intval = di->bm->bi->technology; + else + val->intval = POWER_SUPPLY_TECHNOLOGY_UNKNOWN; break; case POWER_SUPPLY_PROP_TEMP: val->intval = ab8500_btemp_get_temp(di); diff --git a/drivers/power/supply/ab8500_chargalg.c b/drivers/power/supply/ab8500_chargalg.c index 86d740ce3a63..c4a2fe07126c 100644 --- a/drivers/power/supply/ab8500_chargalg.c +++ b/drivers/power/supply/ab8500_chargalg.c @@ -352,7 +352,7 @@ static void ab8500_chargalg_state_to(struct ab8500_chargalg *di, static int ab8500_chargalg_check_charger_enable(struct ab8500_chargalg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; switch (di->charge_state) { case STATE_NORMAL: @@ -731,7 +731,7 @@ static void ab8500_chargalg_start_charging(struct ab8500_chargalg *di, */ static void ab8500_chargalg_check_temp(struct ab8500_chargalg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; if (di->batt_data.temp > (bi->temp_alert_min + di->t_hyst_norm) && di->batt_data.temp < (bi->temp_alert_max - di->t_hyst_norm)) { @@ -802,10 +802,10 @@ static void ab8500_chargalg_end_of_charge(struct ab8500_chargalg *di) if (di->charge_status == POWER_SUPPLY_STATUS_CHARGING && di->charge_state == STATE_NORMAL && !di->maintenance_chg && (di->batt_data.volt_uv >= - di->bm->bi.overvoltage_limit_uv || + di->bm->bi->overvoltage_limit_uv || di->events.usb_cv_active || di->events.ac_cv_active) && di->batt_data.avg_curr_ua < - di->bm->bi.charge_term_current_ua && + di->bm->bi->charge_term_current_ua && di->batt_data.avg_curr_ua > 0) { if (++di->eoc_cnt >= EOC_COND_CNT) { di->eoc_cnt = 0; @@ -827,7 +827,7 @@ static void ab8500_chargalg_end_of_charge(struct ab8500_chargalg *di) static void init_maxim_chg_curr(struct ab8500_chargalg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; di->ccm.original_iset_ua = bi->constant_charge_current_max_ua; di->ccm.current_iset_ua = bi->constant_charge_current_max_ua; @@ -920,7 +920,7 @@ static enum maxim_ret ab8500_chargalg_chg_curr_maxim(struct ab8500_chargalg *di) static void handle_maxim_chg_curr(struct ab8500_chargalg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; enum maxim_ret ret; int result; @@ -1299,7 +1299,7 @@ static void ab8500_chargalg_external_power_changed(struct power_supply *psy) */ static void ab8500_chargalg_algorithm(struct ab8500_chargalg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; int charger_status; int ret; int curr_step_lvl_ua; @@ -1723,7 +1723,7 @@ static int ab8500_chargalg_get_property(struct power_supply *psy, if (di->events.batt_ovv) { val->intval = POWER_SUPPLY_HEALTH_OVERVOLTAGE; } else if (di->events.btemp_underover) { - if (di->batt_data.temp <= di->bm->bi.temp_min) + if (di->batt_data.temp <= di->bm->bi->temp_min) val->intval = POWER_SUPPLY_HEALTH_COLD; else val->intval = POWER_SUPPLY_HEALTH_OVERHEAT; diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index eb3e5c4ca44f..b0919a6a6587 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -852,7 +852,7 @@ static int ab8500_fg_bat_voltage(struct ab8500_fg *di) */ static int ab8500_fg_volt_to_capacity(struct ab8500_fg *di, int voltage_uv) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; /* Multiply by 10 because the capacity is tracked in per mille */ return power_supply_batinfo_ocv2cap(bi, voltage_uv, di->bat_temp) * 10; @@ -881,7 +881,7 @@ static int ab8500_fg_uncomp_volt_to_capacity(struct ab8500_fg *di) */ static int ab8500_fg_battery_resistance(struct ab8500_fg *di) { - struct power_supply_battery_info *bi = &di->bm->bi; + struct power_supply_battery_info *bi = di->bm->bi; int resistance_percent = 0; int resistance; @@ -2140,11 +2140,13 @@ static int ab8500_fg_get_ext_psy_data(struct device *dev, void *data) struct power_supply *ext = dev_get_drvdata(dev); const char **supplicants = (const char **)ext->supplied_to; struct ab8500_fg *di; + struct power_supply_battery_info *bi; union power_supply_propval ret; int j; psy = (struct power_supply *)data; di = power_supply_get_drvdata(psy); + bi = di->bm->bi; /* * For all psy where the name of your driver @@ -2207,8 +2209,8 @@ static int ab8500_fg_get_ext_psy_data(struct device *dev, void *data) switch (ext->desc->type) { case POWER_SUPPLY_TYPE_BATTERY: if (!di->flags.batt_id_received && - (di->bm->bi.technology != - POWER_SUPPLY_TECHNOLOGY_UNKNOWN)) { + (bi && (bi->technology != + POWER_SUPPLY_TECHNOLOGY_UNKNOWN))) { const struct ab8500_battery_type *b; b = di->bm->bat_type; @@ -2216,13 +2218,13 @@ static int ab8500_fg_get_ext_psy_data(struct device *dev, void *data) di->flags.batt_id_received = true; di->bat_cap.max_mah_design = - di->bm->bi.charge_full_design_uah; + di->bm->bi->charge_full_design_uah; di->bat_cap.max_mah = di->bat_cap.max_mah_design; di->vbat_nom_uv = - di->bm->bi.voltage_max_design_uv; + di->bm->bi->voltage_max_design_uv; } if (ret.intval) @@ -2992,9 +2994,9 @@ static int ab8500_fg_bind(struct device *dev, struct device *master, return -ENOMEM; } - di->bat_cap.max_mah_design = di->bm->bi.charge_full_design_uah; + di->bat_cap.max_mah_design = di->bm->bi->charge_full_design_uah; di->bat_cap.max_mah = di->bat_cap.max_mah_design; - di->vbat_nom_uv = di->bm->bi.voltage_max_design_uv; + di->vbat_nom_uv = di->bm->bi->voltage_max_design_uv; /* Start the coulomb counter */ ab8500_fg_coulomb_counter(di, true); diff --git a/drivers/power/supply/axp20x_battery.c b/drivers/power/supply/axp20x_battery.c index 18a9db0df4b1..5d197141f476 100644 --- a/drivers/power/supply/axp20x_battery.c +++ b/drivers/power/supply/axp20x_battery.c @@ -561,7 +561,7 @@ static int axp20x_power_probe(struct platform_device *pdev) { struct axp20x_batt_ps *axp20x_batt; struct power_supply_config psy_cfg = {}; - struct power_supply_battery_info info; + struct power_supply_battery_info *info; struct device *dev = &pdev->dev; if (!of_device_is_available(pdev->dev.of_node)) @@ -615,8 +615,8 @@ static int axp20x_power_probe(struct platform_device *pdev) } if (!power_supply_get_battery_info(axp20x_batt->batt, &info)) { - int vmin = info.voltage_min_design_uv; - int ccc = info.constant_charge_current_max_ua; + int vmin = info->voltage_min_design_uv; + int ccc = info->constant_charge_current_max_ua; if (vmin > 0 && axp20x_set_voltage_min_design(axp20x_batt, vmin)) diff --git a/drivers/power/supply/bd99954-charger.c b/drivers/power/supply/bd99954-charger.c index ffd8bfa08179..96e93e1b8094 100644 --- a/drivers/power/supply/bd99954-charger.c +++ b/drivers/power/supply/bd99954-charger.c @@ -882,7 +882,7 @@ struct dt_init { static int bd9995x_fw_probe(struct bd9995x_device *bd) { int ret; - struct power_supply_battery_info info; + struct power_supply_battery_info *info; u32 property; int i; int regval; @@ -891,49 +891,41 @@ static int bd9995x_fw_probe(struct bd9995x_device *bd) struct battery_init battery_inits[] = { { .name = "trickle-charging current", - .info_data = &info.tricklecharge_current_ua, .range = &charging_current_ranges[0], .ranges = 2, .data = &init->itrich_set, }, { .name = "pre-charging current", - .info_data = &info.precharge_current_ua, .range = &charging_current_ranges[0], .ranges = 2, .data = &init->iprech_set, }, { .name = "pre-to-trickle charge voltage threshold", - .info_data = &info.precharge_voltage_max_uv, .range = &trickle_to_pre_threshold_ranges[0], .ranges = 2, .data = &init->vprechg_th_set, }, { .name = "charging termination current", - .info_data = &info.charge_term_current_ua, .range = &charging_current_ranges[0], .ranges = 2, .data = &init->iterm_set, }, { .name = "charging re-start voltage", - .info_data = &info.charge_restart_voltage_uv, .range = &charge_voltage_regulation_ranges[0], .ranges = 2, .data = &init->vrechg_set, }, { .name = "battery overvoltage limit", - .info_data = &info.overvoltage_limit_uv, .range = &charge_voltage_regulation_ranges[0], .ranges = 2, .data = &init->vbatovp_set, }, { .name = "fast-charging max current", - .info_data = &info.constant_charge_current_max_ua, .range = &fast_charge_current_ranges[0], .ranges = 1, .data = &init->ichg_set, }, { .name = "fast-charging voltage", - .info_data = &info.constant_charge_voltage_max_uv, .range = &charge_voltage_regulation_ranges[0], .ranges = 2, .data = &init->vfastchg_reg_set1, @@ -966,6 +958,16 @@ static int bd9995x_fw_probe(struct bd9995x_device *bd) if (ret < 0) return ret; + /* Put pointers to the generic battery info */ + battery_inits[0].info_data = &info->tricklecharge_current_ua; + battery_inits[1].info_data = &info->precharge_current_ua; + battery_inits[2].info_data = &info->precharge_voltage_max_uv; + battery_inits[3].info_data = &info->charge_term_current_ua; + battery_inits[4].info_data = &info->charge_restart_voltage_uv; + battery_inits[5].info_data = &info->overvoltage_limit_uv; + battery_inits[6].info_data = &info->constant_charge_current_max_ua; + battery_inits[7].info_data = &info->constant_charge_voltage_max_uv; + for (i = 0; i < ARRAY_SIZE(battery_inits); i++) { int val = *battery_inits[i].info_data; const struct linear_range *range = battery_inits[i].range; @@ -980,7 +982,7 @@ static int bd9995x_fw_probe(struct bd9995x_device *bd) dev_err(bd->dev, "Unsupported value for %s\n", battery_inits[i].name); - power_supply_put_battery_info(bd->charger, &info); + power_supply_put_battery_info(bd->charger, info); return -EINVAL; } if (!found) { @@ -991,7 +993,7 @@ static int bd9995x_fw_probe(struct bd9995x_device *bd) *(battery_inits[i].data) = regval; } - power_supply_put_battery_info(bd->charger, &info); + power_supply_put_battery_info(bd->charger, info); for (i = 0; i < ARRAY_SIZE(props); i++) { ret = device_property_read_u32(bd->dev, props[i].prop, diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c index 35ff0c8fe96f..06c34b09349c 100644 --- a/drivers/power/supply/bq24190_charger.c +++ b/drivers/power/supply/bq24190_charger.c @@ -1670,7 +1670,7 @@ static int bq24190_hw_init(struct bq24190_dev_info *bdi) static int bq24190_get_config(struct bq24190_dev_info *bdi) { const char * const s = "ti,system-minimum-microvolt"; - struct power_supply_battery_info info = {}; + struct power_supply_battery_info *info; int v; if (device_property_read_u32(bdi->dev, s, &v) == 0) { @@ -1684,7 +1684,7 @@ static int bq24190_get_config(struct bq24190_dev_info *bdi) if (bdi->dev->of_node && !power_supply_get_battery_info(bdi->charger, &info)) { - v = info.precharge_current_ua / 1000; + v = info->precharge_current_ua / 1000; if (v >= BQ24190_REG_PCTCC_IPRECHG_MIN && v <= BQ24190_REG_PCTCC_IPRECHG_MAX) bdi->iprechg = v; @@ -1692,7 +1692,7 @@ static int bq24190_get_config(struct bq24190_dev_info *bdi) dev_warn(bdi->dev, "invalid value for battery:precharge-current-microamp: %d\n", v); - v = info.charge_term_current_ua / 1000; + v = info->charge_term_current_ua / 1000; if (v >= BQ24190_REG_PCTCC_ITERM_MIN && v <= BQ24190_REG_PCTCC_ITERM_MAX) bdi->iterm = v; diff --git a/drivers/power/supply/bq2515x_charger.c b/drivers/power/supply/bq2515x_charger.c index 374b112f712a..4f76ad9c2f18 100644 --- a/drivers/power/supply/bq2515x_charger.c +++ b/drivers/power/supply/bq2515x_charger.c @@ -945,7 +945,7 @@ static int bq2515x_power_supply_register(struct bq2515x_device *bq2515x, static int bq2515x_hw_init(struct bq2515x_device *bq2515x) { int ret; - struct power_supply_battery_info bat_info = { }; + struct power_supply_battery_info *bat_info; ret = bq2515x_disable_watchdog_timers(bq2515x); if (ret) @@ -969,13 +969,13 @@ static int bq2515x_hw_init(struct bq2515x_device *bq2515x) } else { bq2515x->init_data.ichg = - bat_info.constant_charge_current_max_ua; + bat_info->constant_charge_current_max_ua; bq2515x->init_data.vbatreg = - bat_info.constant_charge_voltage_max_uv; + bat_info->constant_charge_voltage_max_uv; bq2515x->init_data.iprechg = - bat_info.precharge_current_ua; + bat_info->precharge_current_ua; } ret = bq2515x_set_const_charge_current(bq2515x, diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index f501ecd49202..b274942dc46a 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -1504,7 +1504,7 @@ static int bq256xx_power_supply_init(struct bq256xx_device *bq, static int bq256xx_hw_init(struct bq256xx_device *bq) { - struct power_supply_battery_info bat_info = { }; + struct power_supply_battery_info *bat_info; int wd_reg_val = BQ256XX_WATCHDOG_DIS; int ret = 0; int i; @@ -1526,16 +1526,16 @@ static int bq256xx_hw_init(struct bq256xx_device *bq) if (ret) { dev_warn(bq->dev, "battery info missing, default values will be applied\n"); - bat_info.constant_charge_current_max_ua = + bat_info->constant_charge_current_max_ua = bq->chip_info->bq256xx_def_ichg; - bat_info.constant_charge_voltage_max_uv = + bat_info->constant_charge_voltage_max_uv = bq->chip_info->bq256xx_def_vbatreg; - bat_info.precharge_current_ua = + bat_info->precharge_current_ua = bq->chip_info->bq256xx_def_iprechg; - bat_info.charge_term_current_ua = + bat_info->charge_term_current_ua = bq->chip_info->bq256xx_def_iterm; bq->init_data.ichg_max = @@ -1545,10 +1545,10 @@ static int bq256xx_hw_init(struct bq256xx_device *bq) bq->chip_info->bq256xx_max_vbatreg; } else { bq->init_data.ichg_max = - bat_info.constant_charge_current_max_ua; + bat_info->constant_charge_current_max_ua; bq->init_data.vbatreg_max = - bat_info.constant_charge_voltage_max_uv; + bat_info->constant_charge_voltage_max_uv; } ret = bq->chip_info->bq256xx_set_vindpm(bq, bq->init_data.vindpm); @@ -1560,26 +1560,26 @@ static int bq256xx_hw_init(struct bq256xx_device *bq) return ret; ret = bq->chip_info->bq256xx_set_ichg(bq, - bat_info.constant_charge_current_max_ua); + bat_info->constant_charge_current_max_ua); if (ret) return ret; ret = bq->chip_info->bq256xx_set_iprechg(bq, - bat_info.precharge_current_ua); + bat_info->precharge_current_ua); if (ret) return ret; ret = bq->chip_info->bq256xx_set_vbatreg(bq, - bat_info.constant_charge_voltage_max_uv); + bat_info->constant_charge_voltage_max_uv); if (ret) return ret; ret = bq->chip_info->bq256xx_set_iterm(bq, - bat_info.charge_term_current_ua); + bat_info->charge_term_current_ua); if (ret) return ret; - power_supply_put_battery_info(bq->charger, &bat_info); + power_supply_put_battery_info(bq->charger, bat_info); return 0; } diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 0008c229fd9c..9daa6d14db4d 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -1079,7 +1079,7 @@ static int bq25980_power_supply_init(struct bq25980_device *bq, static int bq25980_hw_init(struct bq25980_device *bq) { - struct power_supply_battery_info bat_info = { }; + struct power_supply_battery_info *bat_info; int wd_reg_val = BQ25980_WATCHDOG_DIS; int wd_max_val = BQ25980_NUM_WD_VAL - 1; int ret = 0; @@ -1112,8 +1112,8 @@ static int bq25980_hw_init(struct bq25980_device *bq) return -EINVAL; } - bq->init_data.ichg_max = bat_info.constant_charge_current_max_ua; - bq->init_data.vreg_max = bat_info.constant_charge_voltage_max_uv; + bq->init_data.ichg_max = bat_info->constant_charge_current_max_ua; + bq->init_data.vreg_max = bat_info->constant_charge_voltage_max_uv; if (bq->state.bypass) { ret = regmap_update_bits(bq->regmap, BQ25980_CHRGR_CTRL_2, diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 7e5e24b585d8..72e727cd31e8 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1474,7 +1474,7 @@ static void bq27xxx_battery_set_config(struct bq27xxx_device_info *di, static void bq27xxx_battery_settings(struct bq27xxx_device_info *di) { - struct power_supply_battery_info info = {}; + struct power_supply_battery_info *info; unsigned int min, max; if (power_supply_get_battery_info(di->bat, &info) < 0) @@ -1485,43 +1485,43 @@ static void bq27xxx_battery_settings(struct bq27xxx_device_info *di) return; } - if (info.energy_full_design_uwh != info.charge_full_design_uah) { - if (info.energy_full_design_uwh == -EINVAL) + if (info->energy_full_design_uwh != info->charge_full_design_uah) { + if (info->energy_full_design_uwh == -EINVAL) dev_warn(di->dev, "missing battery:energy-full-design-microwatt-hours\n"); - else if (info.charge_full_design_uah == -EINVAL) + else if (info->charge_full_design_uah == -EINVAL) dev_warn(di->dev, "missing battery:charge-full-design-microamp-hours\n"); } /* assume min == 0 */ max = di->dm_regs[BQ27XXX_DM_DESIGN_ENERGY].max; - if (info.energy_full_design_uwh > max * 1000) { + if (info->energy_full_design_uwh > max * 1000) { dev_err(di->dev, "invalid battery:energy-full-design-microwatt-hours %d\n", - info.energy_full_design_uwh); - info.energy_full_design_uwh = -EINVAL; + info->energy_full_design_uwh); + info->energy_full_design_uwh = -EINVAL; } /* assume min == 0 */ max = di->dm_regs[BQ27XXX_DM_DESIGN_CAPACITY].max; - if (info.charge_full_design_uah > max * 1000) { + if (info->charge_full_design_uah > max * 1000) { dev_err(di->dev, "invalid battery:charge-full-design-microamp-hours %d\n", - info.charge_full_design_uah); - info.charge_full_design_uah = -EINVAL; + info->charge_full_design_uah); + info->charge_full_design_uah = -EINVAL; } min = di->dm_regs[BQ27XXX_DM_TERMINATE_VOLTAGE].min; max = di->dm_regs[BQ27XXX_DM_TERMINATE_VOLTAGE].max; - if ((info.voltage_min_design_uv < min * 1000 || - info.voltage_min_design_uv > max * 1000) && - info.voltage_min_design_uv != -EINVAL) { + if ((info->voltage_min_design_uv < min * 1000 || + info->voltage_min_design_uv > max * 1000) && + info->voltage_min_design_uv != -EINVAL) { dev_err(di->dev, "invalid battery:voltage-min-design-microvolt %d\n", - info.voltage_min_design_uv); - info.voltage_min_design_uv = -EINVAL; + info->voltage_min_design_uv); + info->voltage_min_design_uv = -EINVAL; } - if ((info.energy_full_design_uwh != -EINVAL && - info.charge_full_design_uah != -EINVAL) || - info.voltage_min_design_uv != -EINVAL) - bq27xxx_battery_set_config(di, &info); + if ((info->energy_full_design_uwh != -EINVAL && + info->charge_full_design_uah != -EINVAL) || + info->voltage_min_design_uv != -EINVAL) + bq27xxx_battery_set_config(di, info); } /* diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c index 091868e9e9e8..0c87ad0dbf71 100644 --- a/drivers/power/supply/cw2015_battery.c +++ b/drivers/power/supply/cw2015_battery.c @@ -61,7 +61,7 @@ struct cw_battery { struct delayed_work battery_delay_work; struct regmap *regmap; struct power_supply *rk_bat; - struct power_supply_battery_info battery; + struct power_supply_battery_info *battery; u8 *bat_profile; bool charger_attached; @@ -505,22 +505,22 @@ static int cw_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_CHARGE_FULL: case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: - if (cw_bat->battery.charge_full_design_uah > 0) - val->intval = cw_bat->battery.charge_full_design_uah; + if (cw_bat->battery->charge_full_design_uah > 0) + val->intval = cw_bat->battery->charge_full_design_uah; else val->intval = 0; break; case POWER_SUPPLY_PROP_CHARGE_NOW: - val->intval = cw_bat->battery.charge_full_design_uah; + val->intval = cw_bat->battery->charge_full_design_uah; val->intval = val->intval * cw_bat->soc / 100; break; case POWER_SUPPLY_PROP_CURRENT_NOW: if (cw_battery_valid_time_to_empty(cw_bat) && - cw_bat->battery.charge_full_design_uah > 0) { + cw_bat->battery->charge_full_design_uah > 0) { /* calculate remaining capacity */ - val->intval = cw_bat->battery.charge_full_design_uah; + val->intval = cw_bat->battery->charge_full_design_uah; val->intval = val->intval * cw_bat->soc / 100; /* estimate current based on time to empty */ @@ -687,6 +687,12 @@ static int cw_bat_probe(struct i2c_client *client) ret = power_supply_get_battery_info(cw_bat->rk_bat, &cw_bat->battery); if (ret) { + /* Allocate an empty battery */ + cw_bat->battery = devm_kzalloc(&client->dev, + sizeof(cw_bat->battery), + GFP_KERNEL); + if (!cw_bat->battery) + return -ENOMEM; dev_warn(cw_bat->dev, "No monitored battery, some properties will be missing\n"); } @@ -724,7 +730,7 @@ static int cw_bat_remove(struct i2c_client *client) struct cw_battery *cw_bat = i2c_get_clientdata(client); cancel_delayed_work_sync(&cw_bat->battery_delay_work); - power_supply_put_battery_info(cw_bat->rk_bat, &cw_bat->battery); + power_supply_put_battery_info(cw_bat->rk_bat, cw_bat->battery); return 0; } diff --git a/drivers/power/supply/ingenic-battery.c b/drivers/power/supply/ingenic-battery.c index 8b18219ebe90..2e7fdfde47ec 100644 --- a/drivers/power/supply/ingenic-battery.c +++ b/drivers/power/supply/ingenic-battery.c @@ -18,7 +18,7 @@ struct ingenic_battery { struct iio_channel *channel; struct power_supply_desc desc; struct power_supply *battery; - struct power_supply_battery_info info; + struct power_supply_battery_info *info; }; static int ingenic_battery_get_property(struct power_supply *psy, @@ -26,7 +26,7 @@ static int ingenic_battery_get_property(struct power_supply *psy, union power_supply_propval *val) { struct ingenic_battery *bat = power_supply_get_drvdata(psy); - struct power_supply_battery_info *info = &bat->info; + struct power_supply_battery_info *info = bat->info; int ret; switch (psp) { @@ -80,7 +80,7 @@ static int ingenic_battery_set_scale(struct ingenic_battery *bat) if (ret != IIO_AVAIL_LIST || scale_type != IIO_VAL_FRACTIONAL_LOG2) return -EINVAL; - max_mV = bat->info.voltage_max_design_uv / 1000; + max_mV = bat->info->voltage_max_design_uv / 1000; for (i = 0; i < scale_len; i += 2) { u64 scale_mV = (max_raw * scale_raw[i]) >> scale_raw[i + 1]; @@ -156,13 +156,13 @@ static int ingenic_battery_probe(struct platform_device *pdev) dev_err(dev, "Unable to get battery info: %d\n", ret); return ret; } - if (bat->info.voltage_min_design_uv < 0) { + if (bat->info->voltage_min_design_uv < 0) { dev_err(dev, "Unable to get voltage min design\n"); - return bat->info.voltage_min_design_uv; + return bat->info->voltage_min_design_uv; } - if (bat->info.voltage_max_design_uv < 0) { + if (bat->info->voltage_max_design_uv < 0) { dev_err(dev, "Unable to get voltage max design\n"); - return bat->info.voltage_max_design_uv; + return bat->info->voltage_max_design_uv; } return ingenic_battery_set_scale(bat); diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index 2907b84ceea9..fffb87dca5a0 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -564,14 +564,19 @@ EXPORT_SYMBOL_GPL(devm_power_supply_get_by_phandle); #endif /* CONFIG_OF */ int power_supply_get_battery_info(struct power_supply *psy, - struct power_supply_battery_info *info) + struct power_supply_battery_info **info_out) { struct power_supply_resistance_temp_table *resist_table; + struct power_supply_battery_info *info; struct device_node *battery_np; const char *value; int err, len, index; const __be32 *list; + info = devm_kmalloc(&psy->dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + info->technology = POWER_SUPPLY_TECHNOLOGY_UNKNOWN; info->energy_full_design_uwh = -EINVAL; info->charge_full_design_uah = -EINVAL; @@ -581,6 +586,10 @@ int power_supply_get_battery_info(struct power_supply *psy, info->charge_term_current_ua = -EINVAL; info->constant_charge_current_max_ua = -EINVAL; info->constant_charge_voltage_max_uv = -EINVAL; + info->tricklecharge_current_ua = -EINVAL; + info->precharge_voltage_max_uv = -EINVAL; + info->charge_restart_voltage_uv = -EINVAL; + info->overvoltage_limit_uv = -EINVAL; info->temp_ambient_alert_min = INT_MIN; info->temp_ambient_alert_max = INT_MAX; info->temp_alert_min = INT_MIN; @@ -728,7 +737,7 @@ int power_supply_get_battery_info(struct power_supply *psy, list = of_get_property(battery_np, "resistance-temp-table", &len); if (!list || !len) - goto out_put_node; + goto out_ret_pointer; info->resist_table_size = len / (2 * sizeof(__be32)); resist_table = info->resist_table = devm_kcalloc(&psy->dev, @@ -746,6 +755,10 @@ int power_supply_get_battery_info(struct power_supply *psy, resist_table[index].resistance = be32_to_cpu(*list++); } +out_ret_pointer: + /* Finally return the whole thing */ + *info_out = info; + out_put_node: of_node_put(battery_np); return err; @@ -764,6 +777,8 @@ void power_supply_put_battery_info(struct power_supply *psy, if (info->resist_table) devm_kfree(&psy->dev, info->resist_table); + + devm_kfree(&psy->dev, info); } EXPORT_SYMBOL_GPL(power_supply_put_battery_info); diff --git a/drivers/power/supply/sc2731_charger.c b/drivers/power/supply/sc2731_charger.c index 288b79836c13..9ac17cf7a126 100644 --- a/drivers/power/supply/sc2731_charger.c +++ b/drivers/power/supply/sc2731_charger.c @@ -368,7 +368,7 @@ static int sc2731_charger_usb_change(struct notifier_block *nb, static int sc2731_charger_hw_init(struct sc2731_charger_info *info) { - struct power_supply_battery_info bat_info = { }; + struct power_supply_battery_info *bat_info; u32 term_currrent, term_voltage, cur_val, vol_val; int ret; @@ -390,7 +390,7 @@ static int sc2731_charger_hw_init(struct sc2731_charger_info *info) cur_val = 0x2; vol_val = 0x1; } else { - term_currrent = bat_info.charge_term_current_ua / 1000; + term_currrent = bat_info->charge_term_current_ua / 1000; if (term_currrent <= 90) cur_val = 0; @@ -399,7 +399,7 @@ static int sc2731_charger_hw_init(struct sc2731_charger_info *info) else cur_val = ((term_currrent - 90) / 25) + 1; - term_voltage = bat_info.constant_charge_voltage_max_uv / 1000; + term_voltage = bat_info->constant_charge_voltage_max_uv / 1000; if (term_voltage > 4500) term_voltage = 4500; @@ -409,7 +409,7 @@ static int sc2731_charger_hw_init(struct sc2731_charger_info *info) else vol_val = 0; - power_supply_put_battery_info(info->psy_usb, &bat_info); + power_supply_put_battery_info(info->psy_usb, bat_info); } /* Set charge termination current */ diff --git a/drivers/power/supply/sc27xx_fuel_gauge.c b/drivers/power/supply/sc27xx_fuel_gauge.c index ae45069bd5e1..632977f84b95 100644 --- a/drivers/power/supply/sc27xx_fuel_gauge.c +++ b/drivers/power/supply/sc27xx_fuel_gauge.c @@ -998,7 +998,7 @@ static int sc27xx_fgu_calibration(struct sc27xx_fgu_data *data) static int sc27xx_fgu_hw_init(struct sc27xx_fgu_data *data) { - struct power_supply_battery_info info = { }; + struct power_supply_battery_info *info; struct power_supply_battery_ocv_table *table; int ret, delta_clbcnt, alarm_adc; @@ -1008,16 +1008,16 @@ static int sc27xx_fgu_hw_init(struct sc27xx_fgu_data *data) return ret; } - data->total_cap = info.charge_full_design_uah / 1000; - data->max_volt = info.constant_charge_voltage_max_uv / 1000; - data->internal_resist = info.factory_internal_resistance_uohm / 1000; - data->min_volt = info.voltage_min_design_uv; + data->total_cap = info->charge_full_design_uah / 1000; + data->max_volt = info->constant_charge_voltage_max_uv / 1000; + data->internal_resist = info->factory_internal_resistance_uohm / 1000; + data->min_volt = info->voltage_min_design_uv; /* * For SC27XX fuel gauge device, we only use one ocv-capacity * table in normal temperature 20 Celsius. */ - table = power_supply_find_ocv2cap_table(&info, 20, &data->table_len); + table = power_supply_find_ocv2cap_table(info, 20, &data->table_len); if (!table) return -EINVAL; @@ -1025,7 +1025,7 @@ static int sc27xx_fgu_hw_init(struct sc27xx_fgu_data *data) data->table_len * sizeof(*table), GFP_KERNEL); if (!data->cap_table) { - power_supply_put_battery_info(data->battery, &info); + power_supply_put_battery_info(data->battery, info); return -ENOMEM; } @@ -1035,19 +1035,19 @@ static int sc27xx_fgu_hw_init(struct sc27xx_fgu_data *data) if (!data->alarm_cap) data->alarm_cap += 1; - data->resist_table_len = info.resist_table_size; + data->resist_table_len = info->resist_table_size; if (data->resist_table_len > 0) { - data->resist_table = devm_kmemdup(data->dev, info.resist_table, + data->resist_table = devm_kmemdup(data->dev, info->resist_table, data->resist_table_len * sizeof(struct power_supply_resistance_temp_table), GFP_KERNEL); if (!data->resist_table) { - power_supply_put_battery_info(data->battery, &info); + power_supply_put_battery_info(data->battery, info); return -ENOMEM; } } - power_supply_put_battery_info(data->battery, &info); + power_supply_put_battery_info(data->battery, info); ret = sc27xx_fgu_calibration(data); if (ret) diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c index 753944e774c4..d56e469043bb 100644 --- a/drivers/power/supply/smb347-charger.c +++ b/drivers/power/supply/smb347-charger.c @@ -1281,7 +1281,7 @@ static void smb347_dt_parse_dev_info(struct smb347_charger *smb) static int smb347_get_battery_info(struct smb347_charger *smb) { - struct power_supply_battery_info info = {}; + struct power_supply_battery_info *info; struct power_supply *supply; int err; @@ -1296,29 +1296,29 @@ static int smb347_get_battery_info(struct smb347_charger *smb) if (err) return err; - if (info.constant_charge_current_max_ua != -EINVAL) - smb->max_charge_current = info.constant_charge_current_max_ua; + if (info->constant_charge_current_max_ua != -EINVAL) + smb->max_charge_current = info->constant_charge_current_max_ua; - if (info.constant_charge_voltage_max_uv != -EINVAL) - smb->max_charge_voltage = info.constant_charge_voltage_max_uv; + if (info->constant_charge_voltage_max_uv != -EINVAL) + smb->max_charge_voltage = info->constant_charge_voltage_max_uv; - if (info.precharge_current_ua != -EINVAL) - smb->pre_charge_current = info.precharge_current_ua; + if (info->precharge_current_ua != -EINVAL) + smb->pre_charge_current = info->precharge_current_ua; - if (info.charge_term_current_ua != -EINVAL) - smb->termination_current = info.charge_term_current_ua; + if (info->charge_term_current_ua != -EINVAL) + smb->termination_current = info->charge_term_current_ua; - if (info.temp_alert_min != INT_MIN) - smb->soft_cold_temp_limit = info.temp_alert_min; + if (info->temp_alert_min != INT_MIN) + smb->soft_cold_temp_limit = info->temp_alert_min; - if (info.temp_alert_max != INT_MAX) - smb->soft_hot_temp_limit = info.temp_alert_max; + if (info->temp_alert_max != INT_MAX) + smb->soft_hot_temp_limit = info->temp_alert_max; - if (info.temp_min != INT_MIN) - smb->hard_cold_temp_limit = info.temp_min; + if (info->temp_min != INT_MIN) + smb->hard_cold_temp_limit = info->temp_min; - if (info.temp_max != INT_MAX) - smb->hard_hot_temp_limit = info.temp_max; + if (info->temp_max != INT_MAX) + smb->hard_hot_temp_limit = info->temp_max; /* Suspend when battery temperature is outside hard limits */ if (smb->hard_cold_temp_limit != SMB3XX_TEMP_USE_DEFAULT || diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index f6e94eae4f28..86b4d5c4dab9 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -575,7 +575,7 @@ devm_power_supply_get_by_phandle(struct device *dev, const char *property) #endif /* CONFIG_OF */ extern int power_supply_get_battery_info(struct power_supply *psy, - struct power_supply_battery_info *info); + struct power_supply_battery_info **info_out); extern void power_supply_put_battery_info(struct power_supply *psy, struct power_supply_battery_info *info); extern int power_supply_ocv2cap_simple(struct power_supply_battery_ocv_table *table, -- cgit v1.2.3-59-g8ed1b From d5dbcca70182501bed99de85c224cef04c38ed92 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 3 Jan 2022 17:24:08 +0100 Subject: pktcdvd: convert to use attribute groups There is no need to create kobject children of the pktcdvd device just to display a subdirectory name. Instead, use a named attribute group which removes the extra kobjects and also fixes the userspace race where the device is created yet tools like libudev can not see the attributes as they think the subdirectories are some other sort of device. Cc: linux-block@vger.kernel.org Cc: Jens Axboe Signed-off-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20220103162408.742003-1-gregkh@linuxfoundation.org Signed-off-by: Jens Axboe --- drivers/block/pktcdvd.c | 275 +++++++++++++++++++++++------------------------- include/linux/pktcdvd.h | 10 -- 2 files changed, 134 insertions(+), 151 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 713b7dcf39f9..2b6b70a39e76 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -113,57 +113,10 @@ static sector_t get_zone(sector_t sector, struct pktcdvd_device *pd) return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1); } -/* - * create and register a pktcdvd kernel object. - */ -static struct pktcdvd_kobj* pkt_kobj_create(struct pktcdvd_device *pd, - const char* name, - struct kobject* parent, - struct kobj_type* ktype) -{ - struct pktcdvd_kobj *p; - int error; - - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return NULL; - p->pd = pd; - error = kobject_init_and_add(&p->kobj, ktype, parent, "%s", name); - if (error) { - kobject_put(&p->kobj); - return NULL; - } - kobject_uevent(&p->kobj, KOBJ_ADD); - return p; -} -/* - * remove a pktcdvd kernel object. - */ -static void pkt_kobj_remove(struct pktcdvd_kobj *p) -{ - if (p) - kobject_put(&p->kobj); -} -/* - * default release function for pktcdvd kernel objects. - */ -static void pkt_kobj_release(struct kobject *kobj) -{ - kfree(to_pktcdvdkobj(kobj)); -} - - /********************************************************** - * * sysfs interface for pktcdvd * by (C) 2006 Thomas Maier - * - **********************************************************/ - -#define DEF_ATTR(_obj,_name,_mode) \ - static struct attribute _obj = { .name = _name, .mode = _mode } - -/********************************************************** + /sys/class/pktcdvd/pktcdvd[0-7]/ stat/reset stat/packets_started @@ -176,75 +129,94 @@ static void pkt_kobj_release(struct kobject *kobj) write_queue/congestion_on **********************************************************/ -DEF_ATTR(kobj_pkt_attr_st1, "reset", 0200); -DEF_ATTR(kobj_pkt_attr_st2, "packets_started", 0444); -DEF_ATTR(kobj_pkt_attr_st3, "packets_finished", 0444); -DEF_ATTR(kobj_pkt_attr_st4, "kb_written", 0444); -DEF_ATTR(kobj_pkt_attr_st5, "kb_read", 0444); -DEF_ATTR(kobj_pkt_attr_st6, "kb_read_gather", 0444); - -static struct attribute *kobj_pkt_attrs_stat[] = { - &kobj_pkt_attr_st1, - &kobj_pkt_attr_st2, - &kobj_pkt_attr_st3, - &kobj_pkt_attr_st4, - &kobj_pkt_attr_st5, - &kobj_pkt_attr_st6, - NULL -}; +static ssize_t packets_started_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); -DEF_ATTR(kobj_pkt_attr_wq1, "size", 0444); -DEF_ATTR(kobj_pkt_attr_wq2, "congestion_off", 0644); -DEF_ATTR(kobj_pkt_attr_wq3, "congestion_on", 0644); + return sysfs_emit(buf, "%lu\n", pd->stats.pkt_started); +} +static DEVICE_ATTR_RO(packets_started); -static struct attribute *kobj_pkt_attrs_wqueue[] = { - &kobj_pkt_attr_wq1, - &kobj_pkt_attr_wq2, - &kobj_pkt_attr_wq3, - NULL -}; +static ssize_t packets_finished_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); -static ssize_t kobj_pkt_show(struct kobject *kobj, - struct attribute *attr, char *data) + return sysfs_emit(buf, "%lu\n", pd->stats.pkt_ended); +} +static DEVICE_ATTR_RO(packets_finished); + +static ssize_t kb_written_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd; - int n = 0; - int v; - if (strcmp(attr->name, "packets_started") == 0) { - n = sprintf(data, "%lu\n", pd->stats.pkt_started); + struct pktcdvd_device *pd = dev_get_drvdata(dev); - } else if (strcmp(attr->name, "packets_finished") == 0) { - n = sprintf(data, "%lu\n", pd->stats.pkt_ended); + return sysfs_emit(buf, "%lu\n", pd->stats.secs_w >> 1); +} +static DEVICE_ATTR_RO(kb_written); - } else if (strcmp(attr->name, "kb_written") == 0) { - n = sprintf(data, "%lu\n", pd->stats.secs_w >> 1); +static ssize_t kb_read_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); - } else if (strcmp(attr->name, "kb_read") == 0) { - n = sprintf(data, "%lu\n", pd->stats.secs_r >> 1); + return sysfs_emit(buf, "%lu\n", pd->stats.secs_r >> 1); +} +static DEVICE_ATTR_RO(kb_read); - } else if (strcmp(attr->name, "kb_read_gather") == 0) { - n = sprintf(data, "%lu\n", pd->stats.secs_rg >> 1); +static ssize_t kb_read_gather_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); - } else if (strcmp(attr->name, "size") == 0) { - spin_lock(&pd->lock); - v = pd->bio_queue_size; - spin_unlock(&pd->lock); - n = sprintf(data, "%d\n", v); + return sysfs_emit(buf, "%lu\n", pd->stats.secs_rg >> 1); +} +static DEVICE_ATTR_RO(kb_read_gather); - } else if (strcmp(attr->name, "congestion_off") == 0) { - spin_lock(&pd->lock); - v = pd->write_congestion_off; - spin_unlock(&pd->lock); - n = sprintf(data, "%d\n", v); +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); - } else if (strcmp(attr->name, "congestion_on") == 0) { - spin_lock(&pd->lock); - v = pd->write_congestion_on; - spin_unlock(&pd->lock); - n = sprintf(data, "%d\n", v); + if (len > 0) { + pd->stats.pkt_started = 0; + pd->stats.pkt_ended = 0; + pd->stats.secs_w = 0; + pd->stats.secs_rg = 0; + pd->stats.secs_r = 0; } + return len; +} +static DEVICE_ATTR_WO(reset); + +static struct attribute *pkt_stat_attrs[] = { + &dev_attr_packets_finished.attr, + &dev_attr_packets_started.attr, + &dev_attr_kb_read.attr, + &dev_attr_kb_written.attr, + &dev_attr_kb_read_gather.attr, + &dev_attr_reset.attr, + NULL, +}; + +static const struct attribute_group pkt_stat_group = { + .name = "stat", + .attrs = pkt_stat_attrs, +}; + +static ssize_t size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); + int n; + + spin_lock(&pd->lock); + n = sysfs_emit(buf, "%d\n", pd->bio_queue_size); + spin_unlock(&pd->lock); return n; } +static DEVICE_ATTR_RO(size); static void init_write_congestion_marks(int* lo, int* hi) { @@ -263,30 +235,56 @@ static void init_write_congestion_marks(int* lo, int* hi) } } -static ssize_t kobj_pkt_store(struct kobject *kobj, - struct attribute *attr, - const char *data, size_t len) +static ssize_t congestion_off_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd; - int val; + struct pktcdvd_device *pd = dev_get_drvdata(dev); + int n; - if (strcmp(attr->name, "reset") == 0 && len > 0) { - pd->stats.pkt_started = 0; - pd->stats.pkt_ended = 0; - pd->stats.secs_w = 0; - pd->stats.secs_rg = 0; - pd->stats.secs_r = 0; + spin_lock(&pd->lock); + n = sysfs_emit(buf, "%d\n", pd->write_congestion_off); + spin_unlock(&pd->lock); + return n; +} + +static ssize_t congestion_off_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); + int val; - } else if (strcmp(attr->name, "congestion_off") == 0 - && sscanf(data, "%d", &val) == 1) { + if (sscanf(buf, "%d", &val) == 1) { spin_lock(&pd->lock); pd->write_congestion_off = val; init_write_congestion_marks(&pd->write_congestion_off, &pd->write_congestion_on); spin_unlock(&pd->lock); + } + return len; +} +static DEVICE_ATTR_RW(congestion_off); + +static ssize_t congestion_on_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); + int n; - } else if (strcmp(attr->name, "congestion_on") == 0 - && sscanf(data, "%d", &val) == 1) { + spin_lock(&pd->lock); + n = sysfs_emit(buf, "%d\n", pd->write_congestion_on); + spin_unlock(&pd->lock); + return n; +} + +static ssize_t congestion_on_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct pktcdvd_device *pd = dev_get_drvdata(dev); + int val; + + if (sscanf(buf, "%d", &val) == 1) { spin_lock(&pd->lock); pd->write_congestion_on = val; init_write_congestion_marks(&pd->write_congestion_off, @@ -295,44 +293,39 @@ static ssize_t kobj_pkt_store(struct kobject *kobj, } return len; } +static DEVICE_ATTR_RW(congestion_on); -static const struct sysfs_ops kobj_pkt_ops = { - .show = kobj_pkt_show, - .store = kobj_pkt_store +static struct attribute *pkt_wq_attrs[] = { + &dev_attr_congestion_on.attr, + &dev_attr_congestion_off.attr, + &dev_attr_size.attr, + NULL, }; -static struct kobj_type kobj_pkt_type_stat = { - .release = pkt_kobj_release, - .sysfs_ops = &kobj_pkt_ops, - .default_attrs = kobj_pkt_attrs_stat + +static const struct attribute_group pkt_wq_group = { + .name = "write_queue", + .attrs = pkt_wq_attrs, }; -static struct kobj_type kobj_pkt_type_wqueue = { - .release = pkt_kobj_release, - .sysfs_ops = &kobj_pkt_ops, - .default_attrs = kobj_pkt_attrs_wqueue + +static const struct attribute_group *pkt_groups[] = { + &pkt_stat_group, + &pkt_wq_group, + NULL, }; static void pkt_sysfs_dev_new(struct pktcdvd_device *pd) { if (class_pktcdvd) { - pd->dev = device_create(class_pktcdvd, NULL, MKDEV(0, 0), NULL, - "%s", pd->name); + pd->dev = device_create_with_groups(class_pktcdvd, NULL, + MKDEV(0, 0), pd, pkt_groups, + "%s", pd->name); if (IS_ERR(pd->dev)) pd->dev = NULL; } - if (pd->dev) { - pd->kobj_stat = pkt_kobj_create(pd, "stat", - &pd->dev->kobj, - &kobj_pkt_type_stat); - pd->kobj_wqueue = pkt_kobj_create(pd, "write_queue", - &pd->dev->kobj, - &kobj_pkt_type_wqueue); - } } static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd) { - pkt_kobj_remove(pd->kobj_stat); - pkt_kobj_remove(pd->kobj_wqueue); if (class_pktcdvd) device_unregister(pd->dev); } diff --git a/include/linux/pktcdvd.h b/include/linux/pktcdvd.h index c391e694aa26..f9c5ac80d59b 100644 --- a/include/linux/pktcdvd.h +++ b/include/linux/pktcdvd.h @@ -152,14 +152,6 @@ struct packet_stacked_data }; #define PSD_POOL_SIZE 64 -struct pktcdvd_kobj -{ - struct kobject kobj; - struct pktcdvd_device *pd; -}; -#define to_pktcdvdkobj(_k) \ - ((struct pktcdvd_kobj*)container_of(_k,struct pktcdvd_kobj,kobj)) - struct pktcdvd_device { struct block_device *bdev; /* dev attached */ @@ -197,8 +189,6 @@ struct pktcdvd_device int write_congestion_on; struct device *dev; /* sysfs pktcdvd[0-7] dev */ - struct pktcdvd_kobj *kobj_stat; /* sysfs pktcdvd[0-7]/stat/ */ - struct pktcdvd_kobj *kobj_wqueue; /* sysfs pktcdvd[0-7]/write_queue/ */ struct dentry *dfs_d_root; /* debugfs: devname directory */ struct dentry *dfs_f_info; /* debugfs: info file */ -- cgit v1.2.3-59-g8ed1b From 6c952a0dc9c3ced98c4c8aa7cd11c25c59157f1f Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:20:26 +0100 Subject: ata: libata: Add ata_port_classify() helper Add an ata_port_classify() helper to print out the results from the device classification and remove the debugging statements from ata_dev_classify(). Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libahci.c | 2 +- drivers/ata/libata-core.c | 21 +++++---------------- drivers/ata/libata-sff.c | 2 +- drivers/ata/libata-transport.c | 30 ++++++++++++++++++++++++++++++ drivers/ata/sata_fsl.c | 2 +- drivers/ata/sata_inic162x.c | 2 +- drivers/ata/sata_sil24.c | 2 +- include/linux/libata.h | 2 ++ 8 files changed, 42 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 94edbc89a48c..b7b460560a92 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -1300,7 +1300,7 @@ unsigned int ahci_dev_classify(struct ata_port *ap) tf.lbal = (tmp >> 8) & 0xff; tf.nsect = (tmp) & 0xff; - return ata_dev_classify(&tf); + return ata_port_classify(ap, &tf); } EXPORT_SYMBOL_GPL(ahci_dev_classify); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 72f56c32fe83..28645ac04d9f 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1007,32 +1007,21 @@ unsigned int ata_dev_classify(const struct ata_taskfile *tf) * SEMB signature. This is worked around in * ata_dev_read_id(). */ - if ((tf->lbam == 0) && (tf->lbah == 0)) { - DPRINTK("found ATA device by sig\n"); + if (tf->lbam == 0 && tf->lbah == 0) return ATA_DEV_ATA; - } - if ((tf->lbam == 0x14) && (tf->lbah == 0xeb)) { - DPRINTK("found ATAPI device by sig\n"); + if (tf->lbam == 0x14 && tf->lbah == 0xeb) return ATA_DEV_ATAPI; - } - if ((tf->lbam == 0x69) && (tf->lbah == 0x96)) { - DPRINTK("found PMP device by sig\n"); + if (tf->lbam == 0x69 && tf->lbah == 0x96) return ATA_DEV_PMP; - } - if ((tf->lbam == 0x3c) && (tf->lbah == 0xc3)) { - DPRINTK("found SEMB device by sig (could be ATA device)\n"); + if (tf->lbam == 0x3c && tf->lbah == 0xc3) return ATA_DEV_SEMB; - } - if ((tf->lbam == 0xcd) && (tf->lbah == 0xab)) { - DPRINTK("found ZAC device by sig\n"); + if (tf->lbam == 0xcd && tf->lbah == 0xab) return ATA_DEV_ZAC; - } - DPRINTK("unknown device\n"); return ATA_DEV_UNKNOWN; } EXPORT_SYMBOL_GPL(ata_dev_classify); diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index 39c026f3948c..a119fabe0919 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -1853,7 +1853,7 @@ unsigned int ata_sff_dev_classify(struct ata_device *dev, int present, return ATA_DEV_NONE; /* determine if device is ATA or ATAPI */ - class = ata_dev_classify(&tf); + class = ata_port_classify(ap, &tf); if (class == ATA_DEV_UNKNOWN) { /* If the device failed diagnostic, it's likely to diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index 4162d625fc92..ca129854a88c 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -321,6 +321,36 @@ int ata_tport_add(struct device *parent, return error; } +/** + * ata_port_classify - determine device type based on ATA-spec signature + * @ap: ATA port device on which the classification should be run + * @tf: ATA taskfile register set for device to be identified + * + * A wrapper around ata_dev_classify() to provide additional logging + * + * RETURNS: + * Device type, %ATA_DEV_ATA, %ATA_DEV_ATAPI, %ATA_DEV_PMP, + * %ATA_DEV_ZAC, or %ATA_DEV_UNKNOWN the event of failure. + */ +unsigned int ata_port_classify(struct ata_port *ap, + const struct ata_taskfile *tf) +{ + int i; + unsigned int class = ata_dev_classify(tf); + + /* Start with index '1' to skip the 'unknown' entry */ + for (i = 1; i < ARRAY_SIZE(ata_class_names); i++) { + if (ata_class_names[i].value == class) { + ata_port_dbg(ap, "found %s device by sig\n", + ata_class_names[i].name); + return class; + } + } + + ata_port_info(ap, "found unknown device (class %u)\n", class); + return class; +} +EXPORT_SYMBOL_GPL(ata_port_classify); /* * ATA link attributes diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c index ec52511ae60f..7504d9fbff2a 100644 --- a/drivers/ata/sata_fsl.c +++ b/drivers/ata/sata_fsl.c @@ -814,7 +814,7 @@ static unsigned int sata_fsl_dev_classify(struct ata_port *ap) tf.lbal = (temp >> 8) & 0xff; tf.nsect = temp & 0xff; - return ata_dev_classify(&tf); + return ata_port_classify(ap, &tf); } static int sata_fsl_hardreset(struct ata_link *link, unsigned int *class, diff --git a/drivers/ata/sata_inic162x.c b/drivers/ata/sata_inic162x.c index e517bd8822a5..b6239dae524a 100644 --- a/drivers/ata/sata_inic162x.c +++ b/drivers/ata/sata_inic162x.c @@ -657,7 +657,7 @@ static int inic_hardreset(struct ata_link *link, unsigned int *class, } inic_tf_read(ap, &tf); - *class = ata_dev_classify(&tf); + *class = ata_port_classify(ap, &tf); } return 0; diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c index f99ec6f7d7c0..7e9c1945dc81 100644 --- a/drivers/ata/sata_sil24.c +++ b/drivers/ata/sata_sil24.c @@ -680,7 +680,7 @@ static int sil24_softreset(struct ata_link *link, unsigned int *class, } sil24_read_tf(ap, 0, &tf); - *class = ata_dev_classify(&tf); + *class = ata_port_classify(ap, &tf); DPRINTK("EXIT, class=%u\n", *class); return 0; diff --git a/include/linux/libata.h b/include/linux/libata.h index 2a8404b26083..235fdbeb19ea 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -1160,6 +1160,8 @@ extern enum ata_completion_errors ata_noop_qc_prep(struct ata_queued_cmd *qc); extern void ata_sg_init(struct ata_queued_cmd *qc, struct scatterlist *sg, unsigned int n_elem); extern unsigned int ata_dev_classify(const struct ata_taskfile *tf); +extern unsigned int ata_port_classify(struct ata_port *ap, + const struct ata_taskfile *tf); extern void ata_dev_disable(struct ata_device *adev); extern void ata_id_string(const u16 *id, unsigned char *s, unsigned int ofs, unsigned int len); -- cgit v1.2.3-59-g8ed1b From f083266487690124481eac0869da850406fb3ed3 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 29 Aug 2021 09:18:53 +0200 Subject: headers/uninline: Uninline single-use function: kobject_has_children() This was the only usage of in , so we'll able to decouple the two after this change. Signed-off-by: Ingo Molnar Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 17 +++++++++++++++++ include/linux/kobject.h | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 60d703ebd123..603941b00988 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -3028,6 +3028,23 @@ static inline struct kobject *get_glue_dir(struct device *dev) return dev->kobj.parent; } +/** + * kobject_has_children - Returns whether a kobject has children. + * @kobj: the object to test + * + * This will return whether a kobject has other kobjects as children. + * + * It does NOT account for the presence of attribute files, only sub + * directories. It also assumes there is no concurrent addition or + * removal of such children, and thus relies on external locking. + */ +static inline bool kobject_has_children(struct kobject *kobj) +{ + WARN_ON_ONCE(kref_read(&kobj->kref) == 0); + + return kobj->sd && kobj->sd->dir.subdirs; +} + /* * make sure cleaning up dir as the last step, we need to make * sure .release handler of kobject is run with holding the diff --git a/include/linux/kobject.h b/include/linux/kobject.h index ad90b49824dc..c7b47399b36a 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -117,23 +117,6 @@ extern void kobject_get_ownership(struct kobject *kobj, kuid_t *uid, kgid_t *gid); extern char *kobject_get_path(struct kobject *kobj, gfp_t flag); -/** - * kobject_has_children - Returns whether a kobject has children. - * @kobj: the object to test - * - * This will return whether a kobject has other kobjects as children. - * - * It does NOT account for the presence of attribute files, only sub - * directories. It also assumes there is no concurrent addition or - * removal of such children, and thus relies on external locking. - */ -static inline bool kobject_has_children(struct kobject *kobj) -{ - WARN_ON_ONCE(kref_read(&kobj->kref) == 0); - - return kobj->sd && kobj->sd->dir.subdirs; -} - struct kobj_type { void (*release)(struct kobject *kobj); const struct sysfs_ops *sysfs_ops; -- cgit v1.2.3-59-g8ed1b From d9c19d32d86fa54934b632c4314beb067bf98378 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 18 Oct 2021 10:39:06 -0400 Subject: iov_iter: Add copy_folio_to_iter() This wrapper around copy_page_to_iter() works because copy_page_to_iter() handles compound pages correctly. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- include/linux/uio.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 6350354f97e9..43321dbebba8 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -7,6 +7,7 @@ #include #include +#include #include struct page; @@ -146,6 +147,12 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i); size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i); size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i); +static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset, + size_t bytes, struct iov_iter *i) +{ + return copy_page_to_iter(&folio->page, offset, bytes, i); +} + static __always_inline __must_check size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { -- cgit v1.2.3-59-g8ed1b From 5bf34d7c7ffe773c3b3c1b6ebf39e0f34a2436ec Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 28 Nov 2021 14:24:43 -0500 Subject: mm: Add folio_test_pmd_mappable() Add a predicate to determine if the folio might be mapped by a PMD entry. If CONFIG_TRANSPARENT_HUGEPAGE is disabled, we know it can't be, even if it's large enough. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/huge_mm.h | 14 ++++++++++++++ include/linux/mm.h | 42 +++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index f280f33ff223..e4c18ba8d3bf 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -274,6 +274,15 @@ static inline int thp_nr_pages(struct page *page) return 1; } +/** + * folio_test_pmd_mappable - Can we map this folio with a PMD? + * @folio: The folio to test + */ +static inline bool folio_test_pmd_mappable(struct folio *folio) +{ + return folio_order(folio) >= HPAGE_PMD_ORDER; +} + struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap); struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, @@ -339,6 +348,11 @@ static inline int thp_nr_pages(struct page *page) return 1; } +static inline bool folio_test_pmd_mappable(struct folio *folio) +{ + return false; +} + static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) { return false; diff --git a/include/linux/mm.h b/include/linux/mm.h index a7e4a9e7d807..72ca04f16711 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -714,6 +714,27 @@ int vma_is_stack_for_current(struct vm_area_struct *vma); struct mmu_gather; struct inode; +static inline unsigned int compound_order(struct page *page) +{ + if (!PageHead(page)) + return 0; + return page[1].compound_order; +} + +/** + * folio_order - The allocation order of a folio. + * @folio: The folio. + * + * A folio is composed of 2^order pages. See get_order() for the definition + * of order. + * + * Return: The order of the folio. + */ +static inline unsigned int folio_order(struct folio *folio) +{ + return compound_order(&folio->page); +} + #include /* @@ -906,27 +927,6 @@ static inline void destroy_compound_page(struct page *page) compound_page_dtors[page[1].compound_dtor](page); } -static inline unsigned int compound_order(struct page *page) -{ - if (!PageHead(page)) - return 0; - return page[1].compound_order; -} - -/** - * folio_order - The allocation order of a folio. - * @folio: The folio. - * - * A folio is composed of 2^order pages. See get_order() for the definition - * of order. - * - * Return: The order of the folio. - */ -static inline unsigned int folio_order(struct folio *folio) -{ - return compound_order(&folio->page); -} - static inline bool hpage_pincount_available(struct page *page) { /* -- cgit v1.2.3-59-g8ed1b From 9f2b04a25a41b1f41b3cead4f56854a4192ec5b0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 16 Aug 2021 23:36:31 -0400 Subject: filemap: Add folio_put_wait_locked() Convert all three callers of put_and_wait_on_page_locked() to folio_put_wait_locked(). This shrinks the kernel overall by 19 bytes. filemap_update_page() shrinks by 19 bytes while __migration_entry_wait() is unchanged. folio_put_wait_locked() is 14 bytes smaller than put_and_wait_on_page_locked(), but pmd_migration_entry_wait() grows by 14 bytes. It removes the assumption from pmd_migration_entry_wait() that pages cannot be larger than a PMD (which is true today, but may be interesting to explore in the future). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagemap.h | 2 +- mm/filemap.c | 27 +++++++++++++++------------ mm/migrate.c | 21 ++++++++++----------- 3 files changed, 26 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 605246452305..841f7ba62d7d 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -868,7 +868,7 @@ static inline int wait_on_page_locked_killable(struct page *page) return folio_wait_locked_killable(page_folio(page)); } -int put_and_wait_on_page_locked(struct page *page, int state); +int folio_put_wait_locked(struct folio *folio, int state); void wait_on_page_writeback(struct page *page); void folio_wait_writeback(struct folio *folio); int folio_wait_writeback_killable(struct folio *folio); diff --git a/mm/filemap.c b/mm/filemap.c index 39c4c46c6133..5dd3c6e39c9f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1259,10 +1259,10 @@ enum behavior { * __folio_lock() waiting on then setting PG_locked. */ SHARED, /* Hold ref to page and check the bit when woken, like - * wait_on_page_writeback() waiting on PG_writeback. + * folio_wait_writeback() waiting on PG_writeback. */ DROP, /* Drop ref to page before wait, no check when woken, - * like put_and_wait_on_page_locked() on PG_locked. + * like folio_put_wait_locked() on PG_locked. */ }; @@ -1439,22 +1439,21 @@ int folio_wait_bit_killable(struct folio *folio, int bit_nr) EXPORT_SYMBOL(folio_wait_bit_killable); /** - * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked - * @page: The page to wait for. + * folio_put_wait_locked - Drop a reference and wait for it to be unlocked + * @folio: The folio to wait for. * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). * - * The caller should hold a reference on @page. They expect the page to + * The caller should hold a reference on @folio. They expect the page to * become unlocked relatively soon, but do not wish to hold up migration - * (for example) by holding the reference while waiting for the page to + * (for example) by holding the reference while waiting for the folio to * come unlocked. After this function returns, the caller should not - * dereference @page. + * dereference @folio. * - * Return: 0 if the page was unlocked or -EINTR if interrupted by a signal. + * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. */ -int put_and_wait_on_page_locked(struct page *page, int state) +int folio_put_wait_locked(struct folio *folio, int state) { - return folio_wait_bit_common(page_folio(page), PG_locked, state, - DROP); + return folio_wait_bit_common(folio, PG_locked, state, DROP); } /** @@ -2447,7 +2446,11 @@ static int filemap_update_page(struct kiocb *iocb, goto unlock_mapping; if (!(iocb->ki_flags & IOCB_WAITQ)) { filemap_invalidate_unlock_shared(mapping); - put_and_wait_on_page_locked(&folio->page, TASK_KILLABLE); + /* + * This is where we usually end up waiting for a + * previously submitted readahead to finish. + */ + folio_put_wait_locked(folio, TASK_KILLABLE); return AOP_TRUNCATED_PAGE; } error = __folio_lock_async(folio, iocb->ki_waitq); diff --git a/mm/migrate.c b/mm/migrate.c index cf25b00f03c8..311638177536 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -291,7 +291,7 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, { pte_t pte; swp_entry_t entry; - struct page *page; + struct folio *folio; spin_lock(ptl); pte = *ptep; @@ -302,18 +302,17 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, if (!is_migration_entry(entry)) goto out; - page = pfn_swap_entry_to_page(entry); - page = compound_head(page); + folio = page_folio(pfn_swap_entry_to_page(entry)); /* * Once page cache replacement of page migration started, page_count - * is zero; but we must not call put_and_wait_on_page_locked() without - * a ref. Use get_page_unless_zero(), and just fault again if it fails. + * is zero; but we must not call folio_put_wait_locked() without + * a ref. Use folio_try_get(), and just fault again if it fails. */ - if (!get_page_unless_zero(page)) + if (!folio_try_get(folio)) goto out; pte_unmap_unlock(ptep, ptl); - put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); + folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); return; out: pte_unmap_unlock(ptep, ptl); @@ -338,16 +337,16 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; - struct page *page; + struct folio *folio; ptl = pmd_lock(mm, pmd); if (!is_pmd_migration_entry(*pmd)) goto unlock; - page = pfn_swap_entry_to_page(pmd_to_swp_entry(*pmd)); - if (!get_page_unless_zero(page)) + folio = page_folio(pfn_swap_entry_to_page(pmd_to_swp_entry(*pmd))); + if (!folio_try_get(folio)) goto unlock; spin_unlock(ptl); - put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); + folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); return; unlock: spin_unlock(ptl); -- cgit v1.2.3-59-g8ed1b From 621db4880d305bc37b343b1671e03b7eb5d61389 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 8 May 2021 20:04:05 -0400 Subject: filemap: Add filemap_unaccount_folio() Replace unaccount_page_cache_page() with filemap_unaccount_folio(). The bug handling path could be a bit more robust (eg taking into account the mapcounts of tail pages), but it's really never supposed to happen. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagemap.h | 5 ---- mm/filemap.c | 70 ++++++++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 841f7ba62d7d..077b6f378666 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -884,11 +884,6 @@ static inline void __set_page_dirty(struct page *page, } void folio_account_cleaned(struct folio *folio, struct address_space *mapping, struct bdi_writeback *wb); -static inline void account_page_cleaned(struct page *page, - struct address_space *mapping, struct bdi_writeback *wb) -{ - return folio_account_cleaned(page_folio(page), mapping, wb); -} void __folio_cancel_dirty(struct folio *folio); static inline void folio_cancel_dirty(struct folio *folio) { diff --git a/mm/filemap.c b/mm/filemap.c index 38fb26e16b85..600b8c921a67 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -145,74 +145,74 @@ static void page_cache_delete(struct address_space *mapping, mapping->nrpages -= nr; } -static void unaccount_page_cache_page(struct address_space *mapping, - struct page *page) +static void filemap_unaccount_folio(struct address_space *mapping, + struct folio *folio) { - int nr; + long nr; /* * if we're uptodate, flush out into the cleancache, otherwise * invalidate any existing cleancache entries. We can't leave * stale data around in the cleancache once our page is gone */ - if (PageUptodate(page) && PageMappedToDisk(page)) - cleancache_put_page(page); + if (folio_test_uptodate(folio) && folio_test_mappedtodisk(folio)) + cleancache_put_page(&folio->page); else - cleancache_invalidate_page(mapping, page); + cleancache_invalidate_page(mapping, &folio->page); - VM_BUG_ON_PAGE(PageTail(page), page); - VM_BUG_ON_PAGE(page_mapped(page), page); - if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) { + VM_BUG_ON_FOLIO(folio_mapped(folio), folio); + if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { int mapcount; pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", - current->comm, page_to_pfn(page)); - dump_page(page, "still mapped when deleted"); + current->comm, folio_pfn(folio)); + dump_page(&folio->page, "still mapped when deleted"); dump_stack(); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); - mapcount = page_mapcount(page); + mapcount = page_mapcount(&folio->page); if (mapping_exiting(mapping) && - page_count(page) >= mapcount + 2) { + folio_ref_count(folio) >= mapcount + 2) { /* * All vmas have already been torn down, so it's - * a good bet that actually the page is unmapped, + * a good bet that actually the folio is unmapped, * and we'd prefer not to leak it: if we're wrong, * some other bad page check should catch it later. */ - page_mapcount_reset(page); - page_ref_sub(page, mapcount); + page_mapcount_reset(&folio->page); + folio_ref_sub(folio, mapcount); } } - /* hugetlb pages do not participate in page cache accounting. */ - if (PageHuge(page)) + /* hugetlb folios do not participate in page cache accounting. */ + if (folio_test_hugetlb(folio)) return; - nr = thp_nr_pages(page); + nr = folio_nr_pages(folio); - __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr); - if (PageSwapBacked(page)) { - __mod_lruvec_page_state(page, NR_SHMEM, -nr); - if (PageTransHuge(page)) - __mod_lruvec_page_state(page, NR_SHMEM_THPS, -nr); - } else if (PageTransHuge(page)) { - __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); + __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); + if (folio_test_swapbacked(folio)) { + __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); + if (folio_test_pmd_mappable(folio)) + __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); + } else if (folio_test_pmd_mappable(folio)) { + __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); } /* - * At this point page must be either written or cleaned by - * truncate. Dirty page here signals a bug and loss of + * At this point folio must be either written or cleaned by + * truncate. Dirty folio here signals a bug and loss of * unwritten data. * - * This fixes dirty accounting after removing the page entirely - * but leaves PageDirty set: it has no effect for truncated - * page and anyway will be cleared before returning page into + * This fixes dirty accounting after removing the folio entirely + * but leaves the dirty flag set: it has no effect for truncated + * folio and anyway will be cleared before returning folio to * buddy allocator. */ - if (WARN_ON_ONCE(PageDirty(page))) - account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); + if (WARN_ON_ONCE(folio_test_dirty(folio))) + folio_account_cleaned(folio, mapping, + inode_to_wb(mapping->host)); } /* @@ -227,7 +227,7 @@ void __delete_from_page_cache(struct page *page, void *shadow) trace_mm_filemap_delete_from_page_cache(page); - unaccount_page_cache_page(mapping, page); + filemap_unaccount_folio(mapping, folio); page_cache_delete(mapping, folio, shadow); } @@ -348,7 +348,7 @@ void delete_from_page_cache_batch(struct address_space *mapping, for (i = 0; i < pagevec_count(pvec); i++) { trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); - unaccount_page_cache_page(mapping, pvec->pages[i]); + filemap_unaccount_folio(mapping, page_folio(pvec->pages[i])); } page_cache_delete_batch(mapping, pvec); xa_unlock_irq(&mapping->i_pages); -- cgit v1.2.3-59-g8ed1b From 452e9e6992fe058a650c81d01a9982e3faf10278 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 9 May 2021 09:33:42 -0400 Subject: filemap: Add filemap_remove_folio and __filemap_remove_folio Reimplement __delete_from_page_cache() as a wrapper around __filemap_remove_folio() and delete_from_page_cache() as a wrapper around filemap_remove_folio(). Remove the EXPORT_SYMBOL as delete_from_page_cache() was not used by any in-tree modules. Convert page_cache_free_page() into filemap_free_folio(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagemap.h | 9 +++++++-- mm/filemap.c | 43 ++++++++++++++++++++----------------------- mm/folio-compat.c | 5 +++++ 3 files changed, 32 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 077b6f378666..3f26b191ede3 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -930,8 +930,13 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp); int filemap_add_folio(struct address_space *mapping, struct folio *folio, pgoff_t index, gfp_t gfp); -extern void delete_from_page_cache(struct page *page); -extern void __delete_from_page_cache(struct page *page, void *shadow); +void filemap_remove_folio(struct folio *folio); +void delete_from_page_cache(struct page *page); +void __filemap_remove_folio(struct folio *folio, void *shadow); +static inline void __delete_from_page_cache(struct page *page, void *shadow) +{ + __filemap_remove_folio(page_folio(page), shadow); +} void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, struct pagevec *pvec); diff --git a/mm/filemap.c b/mm/filemap.c index bcdc8bb4d2c8..4fe845b30f33 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -220,58 +220,55 @@ static void filemap_unaccount_folio(struct address_space *mapping, * sure the page is locked and that nobody else uses it - or that usage * is safe. The caller must hold the i_pages lock. */ -void __delete_from_page_cache(struct page *page, void *shadow) +void __filemap_remove_folio(struct folio *folio, void *shadow) { - struct folio *folio = page_folio(page); - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; trace_mm_filemap_delete_from_page_cache(folio); - filemap_unaccount_folio(mapping, folio); page_cache_delete(mapping, folio, shadow); } -static void page_cache_free_page(struct address_space *mapping, - struct page *page) +static void filemap_free_folio(struct address_space *mapping, + struct folio *folio) { void (*freepage)(struct page *); freepage = mapping->a_ops->freepage; if (freepage) - freepage(page); + freepage(&folio->page); - if (PageTransHuge(page) && !PageHuge(page)) { - page_ref_sub(page, thp_nr_pages(page)); - VM_BUG_ON_PAGE(page_count(page) <= 0, page); + if (folio_test_large(folio) && !folio_test_hugetlb(folio)) { + folio_ref_sub(folio, folio_nr_pages(folio)); + VM_BUG_ON_FOLIO(folio_ref_count(folio) <= 0, folio); } else { - put_page(page); + folio_put(folio); } } /** - * delete_from_page_cache - delete page from page cache - * @page: the page which the kernel is trying to remove from page cache + * filemap_remove_folio - Remove folio from page cache. + * @folio: The folio. * - * This must be called only on pages that have been verified to be in the page - * cache and locked. It will never put the page into the free list, the caller - * has a reference on the page. + * This must be called only on folios that are locked and have been + * verified to be in the page cache. It will never put the folio into + * the free list because the caller has a reference on the page. */ -void delete_from_page_cache(struct page *page) +void filemap_remove_folio(struct folio *folio) { - struct address_space *mapping = page_mapping(page); + struct address_space *mapping = folio->mapping; - BUG_ON(!PageLocked(page)); + BUG_ON(!folio_test_locked(folio)); spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); - __delete_from_page_cache(page, NULL); + __filemap_remove_folio(folio, NULL); xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_add_lru(mapping->host); spin_unlock(&mapping->host->i_lock); - page_cache_free_page(mapping, page); + filemap_free_folio(mapping, folio); } -EXPORT_SYMBOL(delete_from_page_cache); /* * page_cache_delete_batch - delete several pages from page cache @@ -358,7 +355,7 @@ void delete_from_page_cache_batch(struct address_space *mapping, spin_unlock(&mapping->host->i_lock); for (i = 0; i < pagevec_count(pvec); i++) - page_cache_free_page(mapping, pvec->pages[i]); + filemap_free_folio(mapping, page_folio(pvec->pages[i])); } int filemap_check_errors(struct address_space *mapping) diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 5b6ae1da314e..749a695b4217 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -140,3 +140,8 @@ struct page *grab_cache_page_write_begin(struct address_space *mapping, mapping_gfp_mask(mapping)); } EXPORT_SYMBOL(grab_cache_page_write_begin); + +void delete_from_page_cache(struct page *page) +{ + return filemap_remove_folio(page_folio(page)); +} -- cgit v1.2.3-59-g8ed1b From bb2e98b613a3c76c904dfa82eb4b86773817598b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 28 Nov 2021 16:14:50 -0500 Subject: filemap: Remove thp_contains() This function is now unused, so delete it. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagemap.h | 9 --------- 1 file changed, 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 3f26b191ede3..8c2cad7f0c36 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -512,15 +512,6 @@ static inline struct page *grab_cache_page_nowait(struct address_space *mapping, mapping_gfp_mask(mapping)); } -/* Does this page contain this index? */ -static inline bool thp_contains(struct page *head, pgoff_t index) -{ - /* HugeTLBfs indexes the page cache in units of hpage_size */ - if (PageHuge(head)) - return head->index == index; - return page_index(head) == (index & ~(thp_nr_pages(head) - 1UL)); -} - #define swapcache_index(folio) __page_file_index(&(folio)->page) /** -- cgit v1.2.3-59-g8ed1b From 7836d9990079ed611199819ccf487061b748193a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 27 May 2021 12:30:54 -0400 Subject: readahead: Convert page_cache_async_ra() to take a folio Using the folio here avoids checking whether it's a tail page. This patch mostly just enables some of the following patches. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagemap.h | 4 ++-- mm/readahead.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 8c2cad7f0c36..30302be6977f 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -993,7 +993,7 @@ struct readahead_control { void page_cache_ra_unbounded(struct readahead_control *, unsigned long nr_to_read, unsigned long lookahead_count); void page_cache_sync_ra(struct readahead_control *, unsigned long req_count); -void page_cache_async_ra(struct readahead_control *, struct page *, +void page_cache_async_ra(struct readahead_control *, struct folio *, unsigned long req_count); void readahead_expand(struct readahead_control *ractl, loff_t new_start, size_t new_len); @@ -1040,7 +1040,7 @@ void page_cache_async_readahead(struct address_space *mapping, struct page *page, pgoff_t index, unsigned long req_count) { DEFINE_READAHEAD(ractl, file, ra, mapping, index); - page_cache_async_ra(&ractl, page, req_count); + page_cache_async_ra(&ractl, page_folio(page), req_count); } static inline struct folio *__readahead_folio(struct readahead_control *ractl) diff --git a/mm/readahead.c b/mm/readahead.c index 6ae5693de28c..e48e78641772 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -581,7 +581,7 @@ void page_cache_sync_ra(struct readahead_control *ractl, EXPORT_SYMBOL_GPL(page_cache_sync_ra); void page_cache_async_ra(struct readahead_control *ractl, - struct page *page, unsigned long req_count) + struct folio *folio, unsigned long req_count) { /* no read-ahead */ if (!ractl->ra->ra_pages) @@ -590,10 +590,10 @@ void page_cache_async_ra(struct readahead_control *ractl, /* * Same bit is used for PG_readahead and PG_reclaim. */ - if (PageWriteback(page)) + if (folio_test_writeback(folio)) return; - ClearPageReadahead(page); + folio_clear_readahead(folio); /* * Defer asynchronous read-ahead on IO congestion. -- cgit v1.2.3-59-g8ed1b From 539a3322f208db478db88c4a76239476defce6b1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 16 Dec 2020 11:45:30 -0500 Subject: filemap: Add read_cache_folio and read_mapping_folio Reimplement read_cache_page() as a wrapper around read_cache_folio(). Saves over 400 bytes of text from do_read_cache_folio() which more than makes up for the extra 100 bytes of text added to the various wrapper functions. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- include/linux/pagemap.h | 12 +++++- mm/filemap.c | 97 ++++++++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 30302be6977f..7bef50ea5435 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -629,8 +629,10 @@ static inline struct page *grab_cache_page(struct address_space *mapping, return find_or_create_page(mapping, index, mapping_gfp_mask(mapping)); } -extern struct page * read_cache_page(struct address_space *mapping, - pgoff_t index, filler_t *filler, void *data); +struct folio *read_cache_folio(struct address_space *, pgoff_t index, + filler_t *filler, void *data); +struct page *read_cache_page(struct address_space *, pgoff_t index, + filler_t *filler, void *data); extern struct page * read_cache_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern int read_cache_pages(struct address_space *mapping, @@ -642,6 +644,12 @@ static inline struct page *read_mapping_page(struct address_space *mapping, return read_cache_page(mapping, index, NULL, data); } +static inline struct folio *read_mapping_folio(struct address_space *mapping, + pgoff_t index, void *data) +{ + return read_cache_folio(mapping, index, NULL, data); +} + /* * Get index of the page within radix-tree (but not for hugetlb pages). * (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE) diff --git a/mm/filemap.c b/mm/filemap.c index fc0f1d9904d2..f98e084ffb31 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3418,35 +3418,20 @@ EXPORT_SYMBOL(filemap_page_mkwrite); EXPORT_SYMBOL(generic_file_mmap); EXPORT_SYMBOL(generic_file_readonly_mmap); -static struct page *wait_on_page_read(struct page *page) +static struct folio *do_read_cache_folio(struct address_space *mapping, + pgoff_t index, filler_t filler, void *data, gfp_t gfp) { - if (!IS_ERR(page)) { - wait_on_page_locked(page); - if (!PageUptodate(page)) { - put_page(page); - page = ERR_PTR(-EIO); - } - } - return page; -} - -static struct page *do_read_cache_page(struct address_space *mapping, - pgoff_t index, - int (*filler)(void *, struct page *), - void *data, - gfp_t gfp) -{ - struct page *page; + struct folio *folio; int err; repeat: - page = find_get_page(mapping, index); - if (!page) { - page = __page_cache_alloc(gfp); - if (!page) + folio = filemap_get_folio(mapping, index); + if (!folio) { + folio = filemap_alloc_folio(gfp, 0); + if (!folio) return ERR_PTR(-ENOMEM); - err = add_to_page_cache_lru(page, mapping, index, gfp); + err = filemap_add_folio(mapping, folio, index, gfp); if (unlikely(err)) { - put_page(page); + folio_put(folio); if (err == -EEXIST) goto repeat; /* Presumably ENOMEM for xarray node */ @@ -3455,21 +3440,24 @@ repeat: filler: if (filler) - err = filler(data, page); + err = filler(data, &folio->page); else - err = mapping->a_ops->readpage(data, page); + err = mapping->a_ops->readpage(data, &folio->page); if (err < 0) { - put_page(page); + folio_put(folio); return ERR_PTR(err); } - page = wait_on_page_read(page); - if (IS_ERR(page)) - return page; + folio_wait_locked(folio); + if (!folio_test_uptodate(folio)) { + folio_put(folio); + return ERR_PTR(-EIO); + } + goto out; } - if (PageUptodate(page)) + if (folio_test_uptodate(folio)) goto out; /* @@ -3503,23 +3491,23 @@ filler: * avoid spurious serialisations and wakeups when multiple processes * wait on the same page for IO to complete. */ - wait_on_page_locked(page); - if (PageUptodate(page)) + folio_wait_locked(folio); + if (folio_test_uptodate(folio)) goto out; /* Distinguish between all the cases under the safety of the lock */ - lock_page(page); + folio_lock(folio); /* Case c or d, restart the operation */ - if (!page->mapping) { - unlock_page(page); - put_page(page); + if (!folio->mapping) { + folio_unlock(folio); + folio_put(folio); goto repeat; } /* Someone else locked and filled the page in a very small window */ - if (PageUptodate(page)) { - unlock_page(page); + if (folio_test_uptodate(folio)) { + folio_unlock(folio); goto out; } @@ -3529,16 +3517,16 @@ filler: * Clear page error before actual read, PG_error will be * set again if read page fails. */ - ClearPageError(page); + folio_clear_error(folio); goto filler; out: - mark_page_accessed(page); - return page; + folio_mark_accessed(folio); + return folio; } /** - * read_cache_page - read into page cache, fill it if needed + * read_cache_folio - read into page cache, fill it if needed * @mapping: the page's address_space * @index: the page index * @filler: function to perform the read @@ -3553,10 +3541,27 @@ out: * * Return: up to date page on success, ERR_PTR() on failure. */ +struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, + filler_t filler, void *data) +{ + return do_read_cache_folio(mapping, index, filler, data, + mapping_gfp_mask(mapping)); +} +EXPORT_SYMBOL(read_cache_folio); + +static struct page *do_read_cache_page(struct address_space *mapping, + pgoff_t index, filler_t *filler, void *data, gfp_t gfp) +{ + struct folio *folio; + + folio = do_read_cache_folio(mapping, index, filler, data, gfp); + if (IS_ERR(folio)) + return &folio->page; + return folio_file_page(folio, index); +} + struct page *read_cache_page(struct address_space *mapping, - pgoff_t index, - int (*filler)(void *, struct page *), - void *data) + pgoff_t index, filler_t *filler, void *data) { return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping)); -- cgit v1.2.3-59-g8ed1b From 82c50f8b443359ec99348cd9b1289f55cd47779d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 28 Jul 2021 15:14:48 -0400 Subject: filemap: Add filemap_release_folio() Reimplement try_to_release_page() as a wrapper around filemap_release_folio(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/mm.h | 1 - include/linux/pagemap.h | 2 ++ mm/filemap.c | 39 +++++++++++++++++++-------------------- mm/folio-compat.c | 6 ++++++ 4 files changed, 27 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 72ca04f16711..145f045b0ddc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1970,7 +1970,6 @@ int get_kernel_pages(const struct kvec *iov, int nr_pages, int write, struct page **pages); struct page *get_dump_page(unsigned long addr); -extern int try_to_release_page(struct page * page, gfp_t gfp_mask); extern void do_invalidatepage(struct page *page, unsigned int offset, unsigned int length); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 7bef50ea5435..eb6e58e106c8 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -939,6 +939,8 @@ static inline void __delete_from_page_cache(struct page *page, void *shadow) void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, struct pagevec *pvec); +int try_to_release_page(struct page *page, gfp_t gfp); +bool filemap_release_folio(struct folio *folio, gfp_t gfp); loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end, int whence); diff --git a/mm/filemap.c b/mm/filemap.c index bbe982e64e62..4c39e09a2f51 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3889,33 +3889,32 @@ ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) EXPORT_SYMBOL(generic_file_write_iter); /** - * try_to_release_page() - release old fs-specific metadata on a page + * filemap_release_folio() - Release fs-specific metadata on a folio. + * @folio: The folio which the kernel is trying to free. + * @gfp: Memory allocation flags (and I/O mode). * - * @page: the page which the kernel is trying to free - * @gfp_mask: memory allocation flags (and I/O mode) + * The address_space is trying to release any data attached to a folio + * (presumably at folio->private). * - * The address_space is to try to release any data against the page - * (presumably at page->private). + * This will also be called if the private_2 flag is set on a page, + * indicating that the folio has other metadata associated with it. * - * This may also be called if PG_fscache is set on a page, indicating that the - * page is known to the local caching routines. + * The @gfp argument specifies whether I/O may be performed to release + * this page (__GFP_IO), and whether the call may block + * (__GFP_RECLAIM & __GFP_FS). * - * The @gfp_mask argument specifies whether I/O may be performed to release - * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS). - * - * Return: %1 if the release was successful, otherwise return zero. + * Return: %true if the release was successful, otherwise %false. */ -int try_to_release_page(struct page *page, gfp_t gfp_mask) +bool filemap_release_folio(struct folio *folio, gfp_t gfp) { - struct address_space * const mapping = page->mapping; + struct address_space * const mapping = folio->mapping; - BUG_ON(!PageLocked(page)); - if (PageWriteback(page)) - return 0; + BUG_ON(!folio_test_locked(folio)); + if (folio_test_writeback(folio)) + return false; if (mapping && mapping->a_ops->releasepage) - return mapping->a_ops->releasepage(page, gfp_mask); - return try_to_free_buffers(page); + return mapping->a_ops->releasepage(&folio->page, gfp); + return try_to_free_buffers(&folio->page); } - -EXPORT_SYMBOL(try_to_release_page); +EXPORT_SYMBOL(filemap_release_folio); diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 749a695b4217..749555a232a8 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -145,3 +145,9 @@ void delete_from_page_cache(struct page *page) { return filemap_remove_folio(page_folio(page)); } + +int try_to_release_page(struct page *page, gfp_t gfp) +{ + return filemap_release_folio(page_folio(page), gfp); +} +EXPORT_SYMBOL(try_to_release_page); -- cgit v1.2.3-59-g8ed1b From 77e2a04745ff8e391ad402e2d2d1157a5d3a7ebc Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 4 Jan 2022 19:51:08 +0000 Subject: ACPI: PCC: Implement OperationRegion handler for the PCC Type 3 subtype PCC OpRegion provides a mechanism to communicate with the platform directly from the AML. PCCT provides the list of PCC channel available in the platform, a subset or all of them can be used in PCC Opregion. This patch registers the PCC OpRegion handler before ACPI tables are loaded. This relies on the special context data passed to identify and set up the PCC channel before the OpRegion handler is executed for the first time. Typical PCC Opregion declaration looks like this: OperationRegion (PFRM, PCC, 2, 0x74) Field (PFRM, ByteAcc, NoLock, Preserve) { SIGN, 32, FLGS, 32, LEN, 32, CMD, 32, DATA, 800 } It contains four named double words followed by 100 bytes of buffer names DATA. ASL can fill out the buffer something like: /* Create global or local buffer */ Name (BUFF, Buffer (0x0C){}) /* Create double word fields over the buffer */ CreateDWordField (BUFF, 0x0, WD0) CreateDWordField (BUFF, 0x04, WD1) CreateDWordField (BUFF, 0x08, WD2) /* Fill the named fields */ WD0 = 0x50434300 SIGN = BUFF WD0 = 1 FLGS = BUFF WD0 = 0x10 LEN = BUFF /* Fill the payload in the DATA buffer */ WD0 = 0 WD1 = 0x08 WD2 = 0 DATA = BUFF /* Write to CMD field to trigger handler */ WD0 = 0x4404 CMD = BUFF This buffer is received by acpi_pcc_opregion_space_handler. This handler will fetch the complete buffer via internal_pcc_buffer. The setup handler will receive the special PCC context data which will contain the PCC channel index which used to set up the channel. The buffer pointer and length is saved in region context which is then used in the handler. (kernel test robot: Build failure with CONFIG_ACPI_DEBUGGER) Link: https://lore.kernel.org/r/202201041539.feAV0l27-lkp@intel.com Reported-by: kernel test robot Signed-off-by: Sudeep Holla Signed-off-by: Rafael J. Wysocki --- drivers/acpi/Kconfig | 17 +++++++ drivers/acpi/Makefile | 1 + drivers/acpi/acpi_pcc.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/acpi/bus.c | 1 + include/linux/acpi.h | 6 +++ 5 files changed, 145 insertions(+) create mode 100644 drivers/acpi/acpi_pcc.c (limited to 'include/linux') diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index cdbdf68bd98f..60b5424bd318 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -524,6 +524,23 @@ config ACPI_PPTT bool endif +config ACPI_PCC + bool "ACPI PCC Address Space" + depends on PCC + default y + help + The PCC Address Space also referred as PCC Operation Region pertains + to the region of PCC subspace that succeeds the PCC signature. + + The PCC Operation Region works in conjunction with the PCC Table + (Platform Communications Channel Table). PCC subspaces that are + marked for use as PCC Operation Regions must not be used as PCC + subspaces for the standard ACPI features such as CPPC, RASF, PDTT and + MPST. These standard features must always use the PCC Table instead. + + Enable this feature if you want to set up and install the PCC Address + Space handler to handle PCC OpRegion in the firmware. + source "drivers/acpi/pmic/Kconfig" config ACPI_VIOT diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 3018714e87d9..08c2d985c57c 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -67,6 +67,7 @@ acpi-$(CONFIG_ACPI_LPIT) += acpi_lpit.o acpi-$(CONFIG_ACPI_GENERIC_GSI) += irq.o acpi-$(CONFIG_ACPI_WATCHDOG) += acpi_watchdog.o acpi-$(CONFIG_ACPI_PRMT) += prmt.o +acpi-$(CONFIG_ACPI_PCC) += acpi_pcc.o # Address translation acpi-$(CONFIG_ACPI_ADXL) += acpi_adxl.o diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c new file mode 100644 index 000000000000..41e3ebd204ff --- /dev/null +++ b/drivers/acpi/acpi_pcc.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Author: Sudeep Holla + * Copyright 2021 Arm Limited + * + * The PCC Address Space also referred as PCC Operation Region pertains to the + * region of PCC subspace that succeeds the PCC signature. The PCC Operation + * Region works in conjunction with the PCC Table(Platform Communications + * Channel Table). PCC subspaces that are marked for use as PCC Operation + * Regions must not be used as PCC subspaces for the standard ACPI features + * such as CPPC, RASF, PDTT and MPST. These standard features must always use + * the PCC Table instead. + * + * This driver sets up the PCC Address Space and installs an handler to enable + * handling of PCC OpRegion in the firmware. + * + */ +#include +#include +#include +#include +#include + +#include + +struct pcc_data { + struct pcc_mbox_chan *pcc_chan; + void __iomem *pcc_comm_addr; + struct completion done; + struct mbox_client cl; + struct acpi_pcc_info ctx; +}; + +struct acpi_pcc_info pcc_ctx; + +static void pcc_rx_callback(struct mbox_client *cl, void *m) +{ + struct pcc_data *data = container_of(cl, struct pcc_data, cl); + + complete(&data->done); +} + +static acpi_status +acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function, + void *handler_context, void **region_context) +{ + struct pcc_data *data; + struct acpi_pcc_info *ctx = handler_context; + struct pcc_mbox_chan *pcc_chan; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return AE_NO_MEMORY; + + data->cl.rx_callback = pcc_rx_callback; + data->cl.knows_txdone = true; + data->ctx.length = ctx->length; + data->ctx.subspace_id = ctx->subspace_id; + data->ctx.internal_buffer = ctx->internal_buffer; + + init_completion(&data->done); + data->pcc_chan = pcc_mbox_request_channel(&data->cl, ctx->subspace_id); + if (IS_ERR(data->pcc_chan)) { + pr_err("Failed to find PCC channel for subspace %d\n", + ctx->subspace_id); + return AE_NOT_FOUND; + } + + pcc_chan = data->pcc_chan; + data->pcc_comm_addr = acpi_os_ioremap(pcc_chan->shmem_base_addr, + pcc_chan->shmem_size); + if (!data->pcc_comm_addr) { + pr_err("Failed to ioremap PCC comm region mem for %d\n", + ctx->subspace_id); + return AE_NO_MEMORY; + } + + *region_context = data; + return AE_OK; +} + +static acpi_status +acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr, + u32 bits, acpi_integer *value, + void *handler_context, void *region_context) +{ + int ret; + struct pcc_data *data = region_context; + + reinit_completion(&data->done); + + /* Write to Shared Memory */ + memcpy_toio(data->pcc_comm_addr, (void *)value, data->ctx.length); + + ret = mbox_send_message(data->pcc_chan->mchan, NULL); + if (ret < 0) + return AE_ERROR; + + if (data->pcc_chan->mchan->mbox->txdone_irq) + wait_for_completion(&data->done); + + mbox_client_txdone(data->pcc_chan->mchan, ret); + + memcpy_fromio(value, data->pcc_comm_addr, data->ctx.length); + + return AE_OK; +} + +void __init acpi_init_pcc(void) +{ + acpi_status status; + + status = acpi_install_address_space_handler(ACPI_ROOT_OBJECT, + ACPI_ADR_SPACE_PLATFORM_COMM, + &acpi_pcc_address_space_handler, + &acpi_pcc_address_space_setup, + &pcc_ctx); + if (ACPI_FAILURE(status)) + pr_alert("OperationRegion handler could not be installed\n"); +} diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index fa923a929224..b64014b4203e 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1320,6 +1320,7 @@ static int __init acpi_init(void) pr_debug("%s: kset create error\n", __func__); init_prmt(); + acpi_init_pcc(); result = acpi_bus_init(); if (result) { kobject_put(acpi_kobj); diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b28f8790192a..93eaba2485e3 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1389,6 +1389,12 @@ static inline int find_acpi_cpu_cache_topology(unsigned int cpu, int level) } #endif +#ifdef CONFIG_ACPI_PCC +void acpi_init_pcc(void); +#else +static inline void acpi_init_pcc(void) { } +#endif + #ifdef CONFIG_ACPI extern void acpi_device_notify(struct device *dev); extern void acpi_device_notify_remove(struct device *dev); -- cgit v1.2.3-59-g8ed1b From 742bef476ca5352b16063161fb73a56629a6d995 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:20:35 +0100 Subject: ata: libata: move ata_{port,link,dev}_dbg to standard pr_XXX() macros Use standard pr_{debug,info,notice,warn,err} macros instead of the hand-crafted printk helpers. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-acpi.c | 48 ++++++++++++++++-------------- drivers/ata/libata-core.c | 61 --------------------------------------- drivers/ata/pata_ixp4xx_cf.c | 6 ++-- include/linux/libata.h | 69 ++++++++++++++++++++++++-------------------- 4 files changed, 67 insertions(+), 117 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-acpi.c b/drivers/ata/libata-acpi.c index 7a7d6642edcc..7007377880ce 100644 --- a/drivers/ata/libata-acpi.c +++ b/drivers/ata/libata-acpi.c @@ -650,9 +650,7 @@ static int ata_acpi_run_tf(struct ata_device *dev, struct ata_taskfile *pptf = NULL; struct ata_taskfile tf, ptf, rtf; unsigned int err_mask; - const char *level; const char *descr; - char msg[60]; int rc; if ((gtf->tf[0] == 0) && (gtf->tf[1] == 0) && (gtf->tf[2] == 0) @@ -666,6 +664,10 @@ static int ata_acpi_run_tf(struct ata_device *dev, pptf = &ptf; } + descr = ata_get_cmd_descript(tf.command); + if (!descr) + descr = "unknown"; + if (!ata_acpi_filter_tf(dev, &tf, pptf)) { rtf = tf; err_mask = ata_exec_internal(dev, &rtf, NULL, @@ -673,40 +675,42 @@ static int ata_acpi_run_tf(struct ata_device *dev, switch (err_mask) { case 0: - level = KERN_DEBUG; - snprintf(msg, sizeof(msg), "succeeded"); + ata_dev_dbg(dev, + "ACPI cmd %02x/%02x:%02x:%02x:%02x:%02x:%02x" + "(%s) succeeded\n", + tf.command, tf.feature, tf.nsect, tf.lbal, + tf.lbam, tf.lbah, tf.device, descr); rc = 1; break; case AC_ERR_DEV: - level = KERN_INFO; - snprintf(msg, sizeof(msg), - "rejected by device (Stat=0x%02x Err=0x%02x)", - rtf.command, rtf.feature); + ata_dev_info(dev, + "ACPI cmd %02x/%02x:%02x:%02x:%02x:%02x:%02x" + "(%s) rejected by device (Stat=0x%02x Err=0x%02x)", + tf.command, tf.feature, tf.nsect, tf.lbal, + tf.lbam, tf.lbah, tf.device, descr, + rtf.command, rtf.feature); rc = 0; break; default: - level = KERN_ERR; - snprintf(msg, sizeof(msg), - "failed (Emask=0x%x Stat=0x%02x Err=0x%02x)", - err_mask, rtf.command, rtf.feature); + ata_dev_err(dev, + "ACPI cmd %02x/%02x:%02x:%02x:%02x:%02x:%02x" + "(%s) failed (Emask=0x%x Stat=0x%02x Err=0x%02x)", + tf.command, tf.feature, tf.nsect, tf.lbal, + tf.lbam, tf.lbah, tf.device, descr, + err_mask, rtf.command, rtf.feature); rc = -EIO; break; } } else { - level = KERN_INFO; - snprintf(msg, sizeof(msg), "filtered out"); + ata_dev_info(dev, + "ACPI cmd %02x/%02x:%02x:%02x:%02x:%02x:%02x" + "(%s) filtered out\n", + tf.command, tf.feature, tf.nsect, tf.lbal, + tf.lbam, tf.lbah, tf.device, descr); rc = 0; } - descr = ata_get_cmd_descript(tf.command); - - ata_dev_printk(dev, level, - "ACPI cmd %02x/%02x:%02x:%02x:%02x:%02x:%02x (%s) %s\n", - tf.command, tf.feature, tf.nsect, tf.lbal, - tf.lbam, tf.lbah, tf.device, - (descr ? descr : "unknown"), msg); - return rc; } diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 3db4fd2029ce..d19984e5dfbc 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -6510,67 +6510,6 @@ const struct ata_port_info ata_dummy_port_info = { }; EXPORT_SYMBOL_GPL(ata_dummy_port_info); -/* - * Utility print functions - */ -void ata_port_printk(const struct ata_port *ap, const char *level, - const char *fmt, ...) -{ - struct va_format vaf; - va_list args; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - printk("%sata%u: %pV", level, ap->print_id, &vaf); - - va_end(args); -} -EXPORT_SYMBOL(ata_port_printk); - -void ata_link_printk(const struct ata_link *link, const char *level, - const char *fmt, ...) -{ - struct va_format vaf; - va_list args; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - if (sata_pmp_attached(link->ap) || link->ap->slave_link) - printk("%sata%u.%02u: %pV", - level, link->ap->print_id, link->pmp, &vaf); - else - printk("%sata%u: %pV", - level, link->ap->print_id, &vaf); - - va_end(args); -} -EXPORT_SYMBOL(ata_link_printk); - -void ata_dev_printk(const struct ata_device *dev, const char *level, - const char *fmt, ...) -{ - struct va_format vaf; - va_list args; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - printk("%sata%u.%02u: %pV", - level, dev->link->ap->print_id, dev->link->pmp + dev->devno, - &vaf); - - va_end(args); -} -EXPORT_SYMBOL(ata_dev_printk); - void ata_print_version(const struct device *dev, const char *version) { dev_printk(KERN_DEBUG, dev, "version %s\n", version); diff --git a/drivers/ata/pata_ixp4xx_cf.c b/drivers/ata/pata_ixp4xx_cf.c index 99c63087c8ae..17b557c91e1c 100644 --- a/drivers/ata/pata_ixp4xx_cf.c +++ b/drivers/ata/pata_ixp4xx_cf.c @@ -114,7 +114,7 @@ static void ixp4xx_set_piomode(struct ata_port *ap, struct ata_device *adev) { struct ixp4xx_pata *ixpp = ap->host->private_data; - ata_dev_printk(adev, KERN_INFO, "configured for PIO%d 8bit\n", + ata_dev_info(adev, "configured for PIO%d 8bit\n", adev->pio_mode - XFER_PIO_0); ixp4xx_set_8bit_timing(ixpp, adev->pio_mode); } @@ -132,8 +132,8 @@ static unsigned int ixp4xx_mmio_data_xfer(struct ata_queued_cmd *qc, struct ixp4xx_pata *ixpp = ap->host->private_data; unsigned long flags; - ata_dev_printk(adev, KERN_DEBUG, "%s %d bytes\n", (rw == READ) ? "READ" : "WRITE", - buflen); + ata_dev_dbg(adev, "%s %d bytes\n", (rw == READ) ? "READ" : "WRITE", + buflen); spin_lock_irqsave(ap->lock, flags); /* set the expansion bus in 16bit mode and restore diff --git a/include/linux/libata.h b/include/linux/libata.h index 235fdbeb19ea..39cdde0b9491 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -1489,51 +1489,61 @@ static inline int sata_srst_pmp(struct ata_link *link) return link->pmp; } -/* - * printk helpers - */ -__printf(3, 4) -void ata_port_printk(const struct ata_port *ap, const char *level, - const char *fmt, ...); -__printf(3, 4) -void ata_link_printk(const struct ata_link *link, const char *level, - const char *fmt, ...); -__printf(3, 4) -void ata_dev_printk(const struct ata_device *dev, const char *level, - const char *fmt, ...); +#define ata_port_printk(level, ap, fmt, ...) \ + pr_ ## level ("ata%u: " fmt, (ap)->print_id, ##__VA_ARGS__) #define ata_port_err(ap, fmt, ...) \ - ata_port_printk(ap, KERN_ERR, fmt, ##__VA_ARGS__) + ata_port_printk(err, ap, fmt, ##__VA_ARGS__) #define ata_port_warn(ap, fmt, ...) \ - ata_port_printk(ap, KERN_WARNING, fmt, ##__VA_ARGS__) + ata_port_printk(warn, ap, fmt, ##__VA_ARGS__) #define ata_port_notice(ap, fmt, ...) \ - ata_port_printk(ap, KERN_NOTICE, fmt, ##__VA_ARGS__) + ata_port_printk(notice, ap, fmt, ##__VA_ARGS__) #define ata_port_info(ap, fmt, ...) \ - ata_port_printk(ap, KERN_INFO, fmt, ##__VA_ARGS__) + ata_port_printk(info, ap, fmt, ##__VA_ARGS__) #define ata_port_dbg(ap, fmt, ...) \ - ata_port_printk(ap, KERN_DEBUG, fmt, ##__VA_ARGS__) + ata_port_printk(debug, ap, fmt, ##__VA_ARGS__) + +#define ata_link_printk(level, link, fmt, ...) \ +do { \ + if (sata_pmp_attached((link)->ap) || \ + (link)->ap->slave_link) \ + pr_ ## level ("ata%u.%02u: " fmt, \ + (link)->ap->print_id, \ + (link)->pmp, \ + ##__VA_ARGS__); \ + else \ + pr_ ## level ("ata%u: " fmt, \ + (link)->ap->print_id, \ + ##__VA_ARGS__); \ +} while (0) #define ata_link_err(link, fmt, ...) \ - ata_link_printk(link, KERN_ERR, fmt, ##__VA_ARGS__) + ata_link_printk(err, link, fmt, ##__VA_ARGS__) #define ata_link_warn(link, fmt, ...) \ - ata_link_printk(link, KERN_WARNING, fmt, ##__VA_ARGS__) + ata_link_printk(warn, link, fmt, ##__VA_ARGS__) #define ata_link_notice(link, fmt, ...) \ - ata_link_printk(link, KERN_NOTICE, fmt, ##__VA_ARGS__) + ata_link_printk(notice, link, fmt, ##__VA_ARGS__) #define ata_link_info(link, fmt, ...) \ - ata_link_printk(link, KERN_INFO, fmt, ##__VA_ARGS__) + ata_link_printk(info, link, fmt, ##__VA_ARGS__) #define ata_link_dbg(link, fmt, ...) \ - ata_link_printk(link, KERN_DEBUG, fmt, ##__VA_ARGS__) + ata_link_printk(debug, link, fmt, ##__VA_ARGS__) + +#define ata_dev_printk(level, dev, fmt, ...) \ + pr_ ## level("ata%u.%02u: " fmt, \ + (dev)->link->ap->print_id, \ + (dev)->link->pmp + (dev)->devno, \ + ##__VA_ARGS__) #define ata_dev_err(dev, fmt, ...) \ - ata_dev_printk(dev, KERN_ERR, fmt, ##__VA_ARGS__) + ata_dev_printk(err, dev, fmt, ##__VA_ARGS__) #define ata_dev_warn(dev, fmt, ...) \ - ata_dev_printk(dev, KERN_WARNING, fmt, ##__VA_ARGS__) + ata_dev_printk(warn, dev, fmt, ##__VA_ARGS__) #define ata_dev_notice(dev, fmt, ...) \ - ata_dev_printk(dev, KERN_NOTICE, fmt, ##__VA_ARGS__) + ata_dev_printk(notice, dev, fmt, ##__VA_ARGS__) #define ata_dev_info(dev, fmt, ...) \ - ata_dev_printk(dev, KERN_INFO, fmt, ##__VA_ARGS__) + ata_dev_printk(info, dev, fmt, ##__VA_ARGS__) #define ata_dev_dbg(dev, fmt, ...) \ - ata_dev_printk(dev, KERN_DEBUG, fmt, ##__VA_ARGS__) + ata_dev_printk(debug, dev, fmt, ##__VA_ARGS__) void ata_print_version(const struct device *dev, const char *version); @@ -2067,11 +2077,8 @@ static inline u8 ata_wait_idle(struct ata_port *ap) { u8 status = ata_sff_busy_wait(ap, ATA_BUSY | ATA_DRQ, 1000); -#ifdef ATA_DEBUG if (status != 0xff && (status & (ATA_BUSY | ATA_DRQ))) - ata_port_printk(ap, KERN_DEBUG, "abnormal Status 0x%X\n", - status); -#endif + ata_port_dbg(ap, "abnormal Status 0x%X\n", status); return status; } -- cgit v1.2.3-59-g8ed1b From d97c75edd806669c9f4b56c0ddae37725c0b708c Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:01 +0100 Subject: ata: libata: drop ata_msg_error() and ata_msg_intr() Unused. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 6 +++--- include/linux/libata.h | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 447a46bdc820..165210576288 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -5345,11 +5345,11 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x00FF; + ap->msg_enable = 0x003F; #elif defined(ATA_DEBUG) - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_CTL | ATA_MSG_WARN | ATA_MSG_ERR; + ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_CTL | ATA_MSG_WARN; #else - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_ERR | ATA_MSG_WARN; + ap->msg_enable = ATA_MSG_DRV | ATA_MSG_WARN; #endif mutex_init(&ap->scsi_scan_mutex); diff --git a/include/linux/libata.h b/include/linux/libata.h index 39cdde0b9491..4f0a85f4e69a 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -78,8 +78,6 @@ enum { ATA_MSG_WARN = 0x0008, ATA_MSG_MALLOC = 0x0010, ATA_MSG_CTL = 0x0020, - ATA_MSG_INTR = 0x0040, - ATA_MSG_ERR = 0x0080, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) @@ -88,8 +86,6 @@ enum { #define ata_msg_warn(p) ((p)->msg_enable & ATA_MSG_WARN) #define ata_msg_malloc(p) ((p)->msg_enable & ATA_MSG_MALLOC) #define ata_msg_ctl(p) ((p)->msg_enable & ATA_MSG_CTL) -#define ata_msg_intr(p) ((p)->msg_enable & ATA_MSG_INTR) -#define ata_msg_err(p) ((p)->msg_enable & ATA_MSG_ERR) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 5cef96b4207e01c9cdb7752acaa178056fe94632 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:02 +0100 Subject: ata: libata: drop ata_msg_ctl() The one caller have been converted to dynamic debugging. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 7 ++----- include/linux/libata.h | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 165210576288..5b50a6d0d6eb 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1763,9 +1763,6 @@ int ata_dev_read_id(struct ata_device *dev, unsigned int *p_class, int may_fallback = 1, tried_spinup = 0; int rc; - if (ata_msg_ctl(ap)) - ata_dev_dbg(dev, "%s: ENTER\n", __func__); - retry: ata_tf_init(dev, &tf); @@ -5345,9 +5342,9 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x003F; + ap->msg_enable = 0x001F; #elif defined(ATA_DEBUG) - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_CTL | ATA_MSG_WARN; + ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_WARN; #else ap->msg_enable = ATA_MSG_DRV | ATA_MSG_WARN; #endif diff --git a/include/linux/libata.h b/include/linux/libata.h index 4f0a85f4e69a..e384cce62963 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -77,7 +77,6 @@ enum { ATA_MSG_PROBE = 0x0004, ATA_MSG_WARN = 0x0008, ATA_MSG_MALLOC = 0x0010, - ATA_MSG_CTL = 0x0020, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) @@ -85,7 +84,6 @@ enum { #define ata_msg_probe(p) ((p)->msg_enable & ATA_MSG_PROBE) #define ata_msg_warn(p) ((p)->msg_enable & ATA_MSG_WARN) #define ata_msg_malloc(p) ((p)->msg_enable & ATA_MSG_MALLOC) -#define ata_msg_ctl(p) ((p)->msg_enable & ATA_MSG_CTL) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 2f784b923d50cdef1f6bd24d7c18614321b0833a Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:03 +0100 Subject: ata: libata: drop ata_msg_malloc() Unused. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 2 +- include/linux/libata.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 5b50a6d0d6eb..3accab132492 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -5342,7 +5342,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x001F; + ap->msg_enable = 0x000F; #elif defined(ATA_DEBUG) ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_WARN; #else diff --git a/include/linux/libata.h b/include/linux/libata.h index e384cce62963..5651bbf4902b 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -76,14 +76,12 @@ enum { ATA_MSG_INFO = 0x0002, ATA_MSG_PROBE = 0x0004, ATA_MSG_WARN = 0x0008, - ATA_MSG_MALLOC = 0x0010, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) #define ata_msg_info(p) ((p)->msg_enable & ATA_MSG_INFO) #define ata_msg_probe(p) ((p)->msg_enable & ATA_MSG_PROBE) #define ata_msg_warn(p) ((p)->msg_enable & ATA_MSG_WARN) -#define ata_msg_malloc(p) ((p)->msg_enable & ATA_MSG_MALLOC) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 16d424672716dc886fb58ec4a47a408db4781cc0 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:04 +0100 Subject: ata: libata: drop ata_msg_warn() The WARN level was always enabled, so drop ata_msg_warn(). Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 19 ++++++++----------- include/linux/libata.h | 2 -- 2 files changed, 8 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 3accab132492..b5334e0a8603 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1571,9 +1571,8 @@ unsigned ata_exec_internal_sg(struct ata_device *dev, else ata_qc_complete(qc); - if (ata_msg_warn(ap)) - ata_dev_warn(dev, "qc timeout (cmd 0x%x)\n", - command); + ata_dev_warn(dev, "qc timeout (cmd 0x%x)\n", + command); } spin_unlock_irqrestore(ap->lock, flags); @@ -1932,9 +1931,8 @@ retry: return 0; err_out: - if (ata_msg_warn(ap)) - ata_dev_warn(dev, "failed to IDENTIFY (%s, err_mask=0x%x)\n", - reason, err_mask); + ata_dev_warn(dev, "failed to IDENTIFY (%s, err_mask=0x%x)\n", + reason, err_mask); return rc; } @@ -2683,8 +2681,7 @@ int ata_dev_configure(struct ata_device *dev) rc = atapi_cdb_len(id); if ((rc < 12) || (rc > ATAPI_CDB_LEN)) { - if (ata_msg_warn(ap)) - ata_dev_warn(dev, "unsupported CDB len\n"); + ata_dev_warn(dev, "unsupported CDB len %d\n", rc); rc = -EINVAL; goto err_out_nosup; } @@ -5342,11 +5339,11 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x000F; + ap->msg_enable = 0x0007; #elif defined(ATA_DEBUG) - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO | ATA_MSG_WARN; + ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO; #else - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_WARN; + ap->msg_enable = ATA_MSG_DRV; #endif mutex_init(&ap->scsi_scan_mutex); diff --git a/include/linux/libata.h b/include/linux/libata.h index 5651bbf4902b..0e5ed2ff94be 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -75,13 +75,11 @@ enum { ATA_MSG_DRV = 0x0001, ATA_MSG_INFO = 0x0002, ATA_MSG_PROBE = 0x0004, - ATA_MSG_WARN = 0x0008, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) #define ata_msg_info(p) ((p)->msg_enable & ATA_MSG_INFO) #define ata_msg_probe(p) ((p)->msg_enable & ATA_MSG_PROBE) -#define ata_msg_warn(p) ((p)->msg_enable & ATA_MSG_WARN) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 17a1e1be2fc7dc99945b41df0485037dcb6044d0 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:05 +0100 Subject: ata: libata: drop ata_msg_probe() All callsites have been converted to dynamic debugging. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-acpi.c | 23 +++++++---------------- drivers/ata/libata-core.c | 20 +++++++------------- drivers/ata/libata-sff.c | 4 ---- include/linux/libata.h | 2 -- 4 files changed, 14 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-acpi.c b/drivers/ata/libata-acpi.c index 9e1e62b9cf63..8cfa8c96bb13 100644 --- a/drivers/ata/libata-acpi.c +++ b/drivers/ata/libata-acpi.c @@ -402,7 +402,6 @@ EXPORT_SYMBOL_GPL(ata_acpi_stm); */ static int ata_dev_get_GTF(struct ata_device *dev, struct ata_acpi_gtf **gtf) { - struct ata_port *ap = dev->link->ap; acpi_status status; struct acpi_buffer output; union acpi_object *out_obj; @@ -418,10 +417,6 @@ static int ata_dev_get_GTF(struct ata_device *dev, struct ata_acpi_gtf **gtf) output.length = ACPI_ALLOCATE_BUFFER; output.pointer = NULL; /* ACPI-CA sets this; save/free it later */ - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: ENTER: port#: %d\n", - __func__, ap->port_no); - /* _GTF has no input parameters */ status = acpi_evaluate_object(ata_dev_acpi_handle(dev), "_GTF", NULL, &output); @@ -437,11 +432,9 @@ static int ata_dev_get_GTF(struct ata_device *dev, struct ata_acpi_gtf **gtf) } if (!output.length || !output.pointer) { - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: Run _GTF: length or ptr is NULL (0x%llx, 0x%p)\n", - __func__, - (unsigned long long)output.length, - output.pointer); + ata_dev_dbg(dev, "Run _GTF: length or ptr is NULL (0x%llx, 0x%p)\n", + (unsigned long long)output.length, + output.pointer); rc = -EINVAL; goto out_free; } @@ -464,9 +457,8 @@ static int ata_dev_get_GTF(struct ata_device *dev, struct ata_acpi_gtf **gtf) rc = out_obj->buffer.length / REGS_PER_GTF; if (gtf) { *gtf = (void *)out_obj->buffer.pointer; - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: returning gtf=%p, gtf_count=%d\n", - __func__, *gtf, rc); + ata_dev_dbg(dev, "returning gtf=%p, gtf_count=%d\n", + *gtf, rc); } return rc; @@ -778,9 +770,8 @@ static int ata_acpi_push_id(struct ata_device *dev) struct acpi_object_list input; union acpi_object in_params[1]; - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: ix = %d, port#: %d\n", - __func__, dev->devno, ap->port_no); + ata_dev_dbg(dev, "%s: ix = %d, port#: %d\n", + __func__, dev->devno, ap->port_no); /* Give the drive Identify data to the drive via the _SDD method */ /* _SDD: set up input parameters */ diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index b5334e0a8603..623a6f272c7e 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2535,9 +2535,6 @@ int ata_dev_configure(struct ata_device *dev) return 0; } - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: ENTER\n", __func__); - /* set horkage */ dev->horkage |= ata_dev_blacklisted(dev); ata_force_horkage(dev); @@ -2585,13 +2582,12 @@ int ata_dev_configure(struct ata_device *dev) return rc; /* print device capabilities */ - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, - "%s: cfg 49:%04x 82:%04x 83:%04x 84:%04x " - "85:%04x 86:%04x 87:%04x 88:%04x\n", - __func__, - id[49], id[82], id[83], id[84], - id[85], id[86], id[87], id[88]); + ata_dev_dbg(dev, + "%s: cfg 49:%04x 82:%04x 83:%04x 84:%04x " + "85:%04x 86:%04x 87:%04x 88:%04x\n", + __func__, + id[49], id[82], id[83], id[84], + id[85], id[86], id[87], id[88]); /* initialize to-be-configured parameters */ dev->flags &= ~ATA_DFLAG_CFG_MASK; @@ -2791,8 +2787,6 @@ int ata_dev_configure(struct ata_device *dev) return 0; err_out_nosup: - if (ata_msg_probe(ap)) - ata_dev_dbg(dev, "%s: EXIT, err\n", __func__); return rc; } @@ -5339,7 +5333,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x0007; + ap->msg_enable = 0x0003; #elif defined(ATA_DEBUG) ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO; #else diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index d5dbeb68b2bf..01f1673f3297 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -330,10 +330,6 @@ EXPORT_SYMBOL_GPL(ata_sff_dev_select); static void ata_dev_select(struct ata_port *ap, unsigned int device, unsigned int wait, unsigned int can_sleep) { - if (ata_msg_probe(ap)) - ata_port_info(ap, "ata_dev_select: ENTER, device %u, wait %u\n", - device, wait); - if (wait) ata_wait_idle(ap); diff --git a/include/linux/libata.h b/include/linux/libata.h index 0e5ed2ff94be..455d7e77e562 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -74,12 +74,10 @@ enum { ATA_MSG_DRV = 0x0001, ATA_MSG_INFO = 0x0002, - ATA_MSG_PROBE = 0x0004, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) #define ata_msg_info(p) ((p)->msg_enable & ATA_MSG_INFO) -#define ata_msg_probe(p) ((p)->msg_enable & ATA_MSG_PROBE) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 96c810f216cb6da15bfa8fe8ef3bf73ca91c5dd8 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:06 +0100 Subject: ata: libata: drop ata_msg_info() Convert the sole caller to ata_dev_dbg() and remove the definition. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 10 +++------- include/linux/libata.h | 2 -- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 623a6f272c7e..80ca94eb3ce0 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2530,8 +2530,8 @@ int ata_dev_configure(struct ata_device *dev) char modelbuf[ATA_ID_PROD_LEN+1]; int rc; - if (!ata_dev_enabled(dev) && ata_msg_info(ap)) { - ata_dev_info(dev, "%s: ENTER/EXIT -- nodev\n", __func__); + if (!ata_dev_enabled(dev)) { + ata_dev_dbg(dev, "no device\n"); return 0; } @@ -5333,11 +5333,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) #if defined(ATA_VERBOSE_DEBUG) /* turn on all debugging levels */ - ap->msg_enable = 0x0003; -#elif defined(ATA_DEBUG) - ap->msg_enable = ATA_MSG_DRV | ATA_MSG_INFO; -#else - ap->msg_enable = ATA_MSG_DRV; + ap->msg_enable = 0x0001; #endif mutex_init(&ap->scsi_scan_mutex); diff --git a/include/linux/libata.h b/include/linux/libata.h index 455d7e77e562..524d09b1dc82 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -73,11 +73,9 @@ enum { ATA_MSG_DRV = 0x0001, - ATA_MSG_INFO = 0x0002, }; #define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) -#define ata_msg_info(p) ((p)->msg_enable & ATA_MSG_INFO) static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { -- cgit v1.2.3-59-g8ed1b From 1c95a27c1e544f723f6e0e5a4384098f92996ec0 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:07 +0100 Subject: ata: libata: drop ata_msg_drv() Callers are already protected by ata_dev_print_info(), so no need to have an additional configuration parameter here. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-core.c | 19 ++++++------------- drivers/ata/libata-eh.c | 3 +-- include/linux/libata.h | 6 ------ 3 files changed, 7 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 80ca94eb3ce0..9c2947905d1e 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2354,7 +2354,6 @@ static void ata_dev_config_trusted(struct ata_device *dev) static int ata_dev_config_lba(struct ata_device *dev) { - struct ata_port *ap = dev->link->ap; const u16 *id = dev->id; const char *lba_desc; char ncq_desc[24]; @@ -2376,7 +2375,7 @@ static int ata_dev_config_lba(struct ata_device *dev) ret = ata_dev_config_ncq(dev, ncq_desc, sizeof(ncq_desc)); /* print device info to dmesg */ - if (ata_msg_drv(ap) && ata_dev_print_info(dev)) + if (ata_dev_print_info(dev)) ata_dev_info(dev, "%llu sectors, multi %u: %s %s\n", (unsigned long long)dev->n_sectors, @@ -2387,7 +2386,6 @@ static int ata_dev_config_lba(struct ata_device *dev) static void ata_dev_config_chs(struct ata_device *dev) { - struct ata_port *ap = dev->link->ap; const u16 *id = dev->id; if (ata_id_current_chs_valid(id)) { @@ -2403,7 +2401,7 @@ static void ata_dev_config_chs(struct ata_device *dev) } /* print device info to dmesg */ - if (ata_msg_drv(ap) && ata_dev_print_info(dev)) + if (ata_dev_print_info(dev)) ata_dev_info(dev, "%llu sectors, multi %u, CHS %u/%u/%u\n", (unsigned long long)dev->n_sectors, @@ -2644,7 +2642,7 @@ int ata_dev_configure(struct ata_device *dev) } /* print device info to dmesg */ - if (ata_msg_drv(ap) && print_info) + if (print_info) ata_dev_info(dev, "%s: %s, %s, max %s\n", revbuf, modelbuf, fwrevbuf, ata_mode_string(xfer_mask)); @@ -2664,7 +2662,7 @@ int ata_dev_configure(struct ata_device *dev) ata_dev_config_cpr(dev); dev->cdb_len = 32; - if (ata_msg_drv(ap) && print_info) + if (print_info) ata_dev_print_features(dev); } @@ -2721,7 +2719,7 @@ int ata_dev_configure(struct ata_device *dev) } /* print device info to dmesg */ - if (ata_msg_drv(ap) && print_info) + if (print_info) ata_dev_info(dev, "ATAPI: %s, %s, max %s%s%s%s\n", modelbuf, fwrevbuf, @@ -2738,7 +2736,7 @@ int ata_dev_configure(struct ata_device *dev) /* Limit PATA drive on SATA cable bridge transfers to udma5, 200 sectors */ if (ata_dev_knobble(dev)) { - if (ata_msg_drv(ap) && print_info) + if (print_info) ata_dev_info(dev, "applying bridge limits\n"); dev->udma_mask &= ATA_UDMA5; dev->max_sectors = ATA_MAX_SECTORS; @@ -5331,11 +5329,6 @@ struct ata_port *ata_port_alloc(struct ata_host *host) ap->host = host; ap->dev = host->dev; -#if defined(ATA_VERBOSE_DEBUG) - /* turn on all debugging levels */ - ap->msg_enable = 0x0001; -#endif - mutex_init(&ap->scsi_scan_mutex); INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug); INIT_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 8bf52a6239aa..7951fd946bf9 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -1214,8 +1214,7 @@ void ata_dev_disable(struct ata_device *dev) if (!ata_dev_enabled(dev)) return; - if (ata_msg_drv(dev->link->ap)) - ata_dev_warn(dev, "disabled\n"); + ata_dev_warn(dev, "disable device\n"); ata_acpi_on_disable(dev); ata_down_xfermask_limit(dev, ATA_DNXFER_FORCE_PIO0 | ATA_DNXFER_QUIET); dev->class++; diff --git a/include/linux/libata.h b/include/linux/libata.h index 524d09b1dc82..65172609a005 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -71,12 +71,6 @@ /* NEW: debug levels */ #define HAVE_LIBATA_MSG 1 -enum { - ATA_MSG_DRV = 0x0001, -}; - -#define ata_msg_drv(p) ((p)->msg_enable & ATA_MSG_DRV) - static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) { if (dval < 0 || dval >= (sizeof(u32) * 8)) -- cgit v1.2.3-59-g8ed1b From db45905e74e6ae035305719bc683eca40f526669 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:08 +0100 Subject: ata: libata: remove 'new' ata message handling Remove the remaining bits for the 'new' ata message handling. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- include/linux/libata.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/libata.h b/include/linux/libata.h index 65172609a005..145c0132b75e 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -68,18 +68,6 @@ } \ }) -/* NEW: debug levels */ -#define HAVE_LIBATA_MSG 1 - -static inline u32 ata_msg_init(int dval, int default_msg_enable_bits) -{ - if (dval < 0 || dval >= (sizeof(u32) * 8)) - return default_msg_enable_bits; /* should be 0x1 - only driver info msgs */ - if (!dval) - return 0; - return (1 << dval) - 1; -} - /* defines only for the constants which don't work well as enums */ #define ATA_TAG_POISON 0xfafbfcfdU @@ -864,7 +852,6 @@ struct ata_port { unsigned int hsm_task_state; - u32 msg_enable; struct list_head eh_done_q; wait_queue_head_t eh_wait_q; int eh_tries; -- cgit v1.2.3-59-g8ed1b From 870bb833c0acb29d8471eac5c2d2e6274826dbb6 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Tue, 21 Dec 2021 08:21:09 +0100 Subject: ata: libata: remove debug compilation switches Unused now, so remove and drop any references to them. Signed-off-by: Hannes Reinecke Signed-off-by: Damien Le Moal --- drivers/ata/libata-sff.c | 1 - drivers/ata/pata_ep93xx.c | 1 - drivers/ata/sata_rcar.c | 1 - include/linux/libata.h | 16 ---------------- 4 files changed, 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index 01f1673f3297..75217828dfe3 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -2091,7 +2091,6 @@ void ata_sff_drain_fifo(struct ata_queued_cmd *qc) && count < 65536; count += 2) ioread16(ap->ioaddr.data_addr); - /* Can become DEBUG later */ if (count) ata_port_dbg(ap, "drained %d bytes to clear DRQ\n", count); diff --git a/drivers/ata/pata_ep93xx.c b/drivers/ata/pata_ep93xx.c index 46208ececbb6..b78f71c70f27 100644 --- a/drivers/ata/pata_ep93xx.c +++ b/drivers/ata/pata_ep93xx.c @@ -855,7 +855,6 @@ static void ep93xx_pata_drain_fifo(struct ata_queued_cmd *qc) && count < 65536; count += 2) ep93xx_pata_read_reg(drv_data, IDECTRL_ADDR_DATA); - /* Can become DEBUG later */ if (count) ata_port_dbg(ap, "drained %d bytes to clear DRQ.\n", count); diff --git a/drivers/ata/sata_rcar.c b/drivers/ata/sata_rcar.c index 91b39a6aa9f7..3d96b6faa3f0 100644 --- a/drivers/ata/sata_rcar.c +++ b/drivers/ata/sata_rcar.c @@ -479,7 +479,6 @@ static void sata_rcar_drain_fifo(struct ata_queued_cmd *qc) count < 65536; count += 2) ioread32(ap->ioaddr.data_addr); - /* Can become DEBUG later */ if (count) ata_port_dbg(ap, "drained %d bytes to clear DRQ\n", count); } diff --git a/include/linux/libata.h b/include/linux/libata.h index 145c0132b75e..c258f69106f4 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -39,25 +39,9 @@ * compile-time options: to be removed as soon as all the drivers are * converted to the new debugging mechanism */ -#undef ATA_DEBUG /* debugging output */ -#undef ATA_VERBOSE_DEBUG /* yet more debugging output */ #undef ATA_IRQ_TRAP /* define to ack screaming irqs */ -#undef ATA_NDEBUG /* define to disable quick runtime checks */ -/* note: prints function name for you */ -#ifdef ATA_DEBUG -#define DPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args) -#ifdef ATA_VERBOSE_DEBUG -#define VPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args) -#else -#define VPRINTK(fmt, args...) -#endif /* ATA_VERBOSE_DEBUG */ -#else -#define DPRINTK(fmt, args...) -#define VPRINTK(fmt, args...) -#endif /* ATA_DEBUG */ - #define ata_print_version_once(dev, version) \ ({ \ static bool __print_once; \ -- cgit v1.2.3-59-g8ed1b From cc4b08c31b5c51352f258032cc65e884b3e61e6a Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Tue, 7 Dec 2021 21:15:31 +0900 Subject: can: do not increase tx_bytes statistics for RTR frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actual payload length of the CAN Remote Transmission Request (RTR) frames is always 0, i.e. no payload is transmitted on the wire. However, those RTR frames still use the DLC to indicate the length of the requested frame. As such, net_device_stats::tx_bytes should not be increased when sending RTR frames. The function can_get_echo_skb() already returns the correct length, even for RTR frames (c.f. [1]). However, for historical reasons, the drivers do not use can_get_echo_skb()'s return value and instead, most of them store a temporary length (or dlc) in some local structure or array. Using the return value of can_get_echo_skb() solves the issue. After doing this, such length/dlc fields become unused and so this patch does the adequate cleaning when needed. This patch fixes all the CAN drivers. Finally, can_get_echo_skb() is decorated with the __must_check attribute in order to force future drivers to correctly use its return value (else the compiler would emit a warning). [1] commit ed3320cec279 ("can: dev: __can_get_echo_skb(): fix real payload length return value for RTR frames") Link: https://lore.kernel.org/all/20211207121531.42941-6-mailhol.vincent@wanadoo.fr Cc: Nicolas Ferre Cc: Alexandre Belloni Cc: Ludovic Desroches Cc: Maxime Ripard Cc: Chen-Yu Tsai Cc: Jernej Skrabec Cc: Yasushi SHOJI Cc: Oliver Hartkopp Cc: Stephane Grosjean Cc: Andreas Larsson Tested-by: Jimmy Assarsson # kvaser Signed-off-by: Vincent Mailhol Acked-by: Stefan Mätje # esd_usb2 Tested-by: Stefan Mätje # esd_usb2 [mkl: add conversion for grcan] Signed-off-by: Marc Kleine-Budde --- drivers/net/can/at91_can.c | 8 ++--- drivers/net/can/c_can/c_can.h | 1 - drivers/net/can/c_can/c_can_main.c | 7 ++-- drivers/net/can/cc770/cc770.c | 8 ++--- drivers/net/can/grcan.c | 17 ++------- drivers/net/can/janz-ican3.c | 3 +- drivers/net/can/mscan/mscan.c | 4 +-- drivers/net/can/pch_can.c | 9 ++--- drivers/net/can/peak_canfd/peak_canfd.c | 3 +- drivers/net/can/rcar/rcar_can.c | 11 +++--- drivers/net/can/rcar/rcar_canfd.c | 6 +--- drivers/net/can/sja1000/sja1000.c | 4 +-- drivers/net/can/slcan.c | 3 +- drivers/net/can/softing/softing_main.c | 8 ++--- drivers/net/can/spi/hi311x.c | 24 ++++++------- drivers/net/can/spi/mcp251x.c | 25 +++++++------- drivers/net/can/sun4i_can.c | 5 +-- drivers/net/can/usb/ems_usb.c | 7 ++-- drivers/net/can/usb/esd_usb2.c | 6 ++-- drivers/net/can/usb/gs_usb.c | 7 ++-- drivers/net/can/usb/kvaser_usb/kvaser_usb.h | 5 ++- drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c | 42 +++++++++++------------ drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c | 13 +++---- drivers/net/can/usb/mcba_usb.c | 12 +++---- drivers/net/can/usb/peak_usb/pcan_usb_core.c | 20 +++++------ drivers/net/can/usb/peak_usb/pcan_usb_core.h | 1 - drivers/net/can/usb/ucan.c | 10 ++---- drivers/net/can/usb/usb_8dev.c | 6 +--- drivers/net/can/vcan.c | 7 ++-- drivers/net/can/vxcan.c | 2 +- include/linux/can/skb.h | 5 +-- 32 files changed, 116 insertions(+), 175 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index 97f1d08b4133..a00655ccda02 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -448,7 +448,6 @@ static void at91_chip_stop(struct net_device *dev, enum can_state state) static netdev_tx_t at91_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct at91_priv *priv = netdev_priv(dev); - struct net_device_stats *stats = &dev->stats; struct can_frame *cf = (struct can_frame *)skb->data; unsigned int mb, prio; u32 reg_mid, reg_mcr; @@ -480,8 +479,6 @@ static netdev_tx_t at91_start_xmit(struct sk_buff *skb, struct net_device *dev) /* This triggers transmission */ at91_write(priv, AT91_MCR(mb), reg_mcr); - stats->tx_bytes += cf->len; - /* _NOTE_: subtract AT91_MB_TX_FIRST offset from mb! */ can_put_echo_skb(skb, dev, mb - get_mb_tx_first(priv), 0); @@ -852,7 +849,10 @@ static void at91_irq_tx(struct net_device *dev, u32 reg_sr) if (likely(reg_msr & AT91_MSR_MRDY && ~reg_msr & AT91_MSR_MABT)) { /* _NOTE_: subtract AT91_MB_TX_FIRST offset from mb! */ - can_get_echo_skb(dev, mb - get_mb_tx_first(priv), NULL); + dev->stats.tx_bytes += + can_get_echo_skb(dev, + mb - get_mb_tx_first(priv), + NULL); dev->stats.tx_packets++; can_led_event(dev, CAN_LED_EVENT_TX); } diff --git a/drivers/net/can/c_can/c_can.h b/drivers/net/can/c_can/c_can.h index 08b6efa7a1a7..bd2f6dc01194 100644 --- a/drivers/net/can/c_can/c_can.h +++ b/drivers/net/can/c_can/c_can.h @@ -211,7 +211,6 @@ struct c_can_priv { struct c_can_raminit raminit_sys; /* RAMINIT via syscon regmap */ void (*raminit)(const struct c_can_priv *priv, bool enable); u32 comm_rcv_high; - u32 dlc[]; }; struct net_device *alloc_c_can_dev(int msg_obj_num); diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c index 29e91804d81c..faa217f26771 100644 --- a/drivers/net/can/c_can/c_can_main.c +++ b/drivers/net/can/c_can/c_can_main.c @@ -477,7 +477,6 @@ static netdev_tx_t c_can_start_xmit(struct sk_buff *skb, * transmit as we might race against do_tx(). */ c_can_setup_tx_object(dev, IF_TX, frame, idx); - priv->dlc[idx] = frame->len; can_put_echo_skb(skb, dev, idx, 0); obj = idx + priv->msg_obj_tx_first; c_can_object_put(dev, IF_TX, obj, cmd); @@ -742,8 +741,7 @@ static void c_can_do_tx(struct net_device *dev) * NAPI. We are not transmitting. */ c_can_inval_tx_object(dev, IF_NAPI, obj); - can_get_echo_skb(dev, idx, NULL); - bytes += priv->dlc[idx]; + bytes += can_get_echo_skb(dev, idx, NULL); pkts++; } @@ -1227,8 +1225,7 @@ struct net_device *alloc_c_can_dev(int msg_obj_num) struct c_can_priv *priv; int msg_obj_tx_num = msg_obj_num / 2; - dev = alloc_candev(struct_size(priv, dlc, msg_obj_tx_num), - msg_obj_tx_num); + dev = alloc_candev(sizeof(*priv), msg_obj_tx_num); if (!dev) return NULL; diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c index 994073c0a2f6..bb7224cfc6ab 100644 --- a/drivers/net/can/cc770/cc770.c +++ b/drivers/net/can/cc770/cc770.c @@ -664,7 +664,6 @@ static void cc770_tx_interrupt(struct net_device *dev, unsigned int o) struct cc770_priv *priv = netdev_priv(dev); struct net_device_stats *stats = &dev->stats; unsigned int mo = obj2msgobj(o); - struct can_frame *cf; u8 ctrl1; ctrl1 = cc770_read_reg(priv, msgobj[mo].ctrl1); @@ -696,12 +695,9 @@ static void cc770_tx_interrupt(struct net_device *dev, unsigned int o) return; } - cf = (struct can_frame *)priv->tx_skb->data; - stats->tx_bytes += cf->len; - stats->tx_packets++; - can_put_echo_skb(priv->tx_skb, dev, 0, 0); - can_get_echo_skb(dev, 0, NULL); + stats->tx_bytes += can_get_echo_skb(dev, 0, NULL); + stats->tx_packets++; priv->tx_skb = NULL; netif_wake_queue(dev); diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index 1b8ef97e4139..d0c5a7a60daf 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -255,7 +255,6 @@ struct grcan_priv { struct grcan_dma dma; struct sk_buff **echo_skb; /* We allocate this on our own */ - u8 *txdlc; /* Length of queued frames */ /* The echo skb pointer, pointing into echo_skb and indicating which * frames can be echoed back. See the "Notes on the tx cyclic buffer @@ -515,9 +514,7 @@ static int catch_up_echo_skb(struct net_device *dev, int budget, bool echo) if (echo) { /* Normal echo of messages */ stats->tx_packets++; - stats->tx_bytes += priv->txdlc[i]; - priv->txdlc[i] = 0; - can_get_echo_skb(dev, i, NULL); + stats->tx_bytes += can_get_echo_skb(dev, i, NULL); } else { /* For cleanup of untransmitted messages */ can_free_echo_skb(dev, i, NULL); @@ -1062,16 +1059,10 @@ static int grcan_open(struct net_device *dev) priv->can.echo_skb_max = dma->tx.size; priv->can.echo_skb = priv->echo_skb; - priv->txdlc = kcalloc(dma->tx.size, sizeof(*priv->txdlc), GFP_KERNEL); - if (!priv->txdlc) { - err = -ENOMEM; - goto exit_free_echo_skb; - } - /* Get can device up */ err = open_candev(dev); if (err) - goto exit_free_txdlc; + goto exit_free_echo_skb; err = request_irq(dev->irq, grcan_interrupt, IRQF_SHARED, dev->name, dev); @@ -1093,8 +1084,6 @@ static int grcan_open(struct net_device *dev) exit_close_candev: close_candev(dev); -exit_free_txdlc: - kfree(priv->txdlc); exit_free_echo_skb: kfree(priv->echo_skb); exit_free_dma_buffers: @@ -1129,7 +1118,6 @@ static int grcan_close(struct net_device *dev) priv->can.echo_skb_max = 0; priv->can.echo_skb = NULL; kfree(priv->echo_skb); - kfree(priv->txdlc); return 0; } @@ -1447,7 +1435,6 @@ static netdev_tx_t grcan_start_xmit(struct sk_buff *skb, * can_put_echo_skb would be an error unless other measures are * taken. */ - priv->txdlc[slotindex] = cf->len; /* Store dlc for statistics */ can_put_echo_skb(skb, dev, slotindex, 0); /* Make sure everything is written before allowing hardware to diff --git a/drivers/net/can/janz-ican3.c b/drivers/net/can/janz-ican3.c index 5c589aa9dff8..5b677af5f2a4 100644 --- a/drivers/net/can/janz-ican3.c +++ b/drivers/net/can/janz-ican3.c @@ -1294,7 +1294,8 @@ static unsigned int ican3_get_echo_skb(struct ican3_dev *mod) } cf = (struct can_frame *)skb->data; - dlc = cf->len; + if (!(cf->can_id & CAN_RTR_FLAG)) + dlc = cf->len; /* check flag whether this packet has to be looped back */ if (skb->pkt_type != PACKET_LOOPBACK) { diff --git a/drivers/net/can/mscan/mscan.c b/drivers/net/can/mscan/mscan.c index 59b8284d00e5..5b5802fac772 100644 --- a/drivers/net/can/mscan/mscan.c +++ b/drivers/net/can/mscan/mscan.c @@ -448,9 +448,9 @@ static irqreturn_t mscan_isr(int irq, void *dev_id) continue; out_8(®s->cantbsel, mask); - stats->tx_bytes += in_8(®s->tx.dlr); + stats->tx_bytes += can_get_echo_skb(dev, entry->id, + NULL); stats->tx_packets++; - can_get_echo_skb(dev, entry->id, NULL); priv->tx_active &= ~mask; list_del(pos); } diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c index b46f9cfb9e0a..888bef03de09 100644 --- a/drivers/net/can/pch_can.c +++ b/drivers/net/can/pch_can.c @@ -707,16 +707,13 @@ static void pch_can_tx_complete(struct net_device *ndev, u32 int_stat) { struct pch_can_priv *priv = netdev_priv(ndev); struct net_device_stats *stats = &(priv->ndev->stats); - u32 dlc; - can_get_echo_skb(ndev, int_stat - PCH_RX_OBJ_END - 1, NULL); + stats->tx_bytes += can_get_echo_skb(ndev, int_stat - PCH_RX_OBJ_END - 1, + NULL); + stats->tx_packets++; iowrite32(PCH_CMASK_RX_TX_GET | PCH_CMASK_CLRINTPND, &priv->regs->ifregs[1].cmask); pch_can_rw_msg_obj(&priv->regs->ifregs[1].creq, int_stat); - dlc = can_cc_dlc2len(ioread32(&priv->regs->ifregs[1].mcont) & - PCH_IF_MCONT_DLC); - stats->tx_bytes += dlc; - stats->tx_packets++; if (int_stat == PCH_TX_OBJ_END) netif_wake_queue(ndev); } diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c index 216609198eac..b2dea360813d 100644 --- a/drivers/net/can/peak_canfd/peak_canfd.c +++ b/drivers/net/can/peak_canfd/peak_canfd.c @@ -266,10 +266,9 @@ static int pucan_handle_can_rx(struct peak_canfd_priv *priv, unsigned long flags; spin_lock_irqsave(&priv->echo_lock, flags); - can_get_echo_skb(priv->ndev, msg->client, NULL); /* count bytes of the echo instead of skb */ - stats->tx_bytes += cf_len; + stats->tx_bytes += can_get_echo_skb(priv->ndev, msg->client, NULL); stats->tx_packets++; /* restart tx queue (a slot is free) */ diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index 62bbd58bfef8..33e37395379d 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -94,7 +94,6 @@ struct rcar_can_priv { struct rcar_can_regs __iomem *regs; struct clk *clk; struct clk *can_clk; - u8 tx_dlc[RCAR_CAN_FIFO_DEPTH]; u32 tx_head; u32 tx_tail; u8 clock_select; @@ -379,10 +378,11 @@ static void rcar_can_tx_done(struct net_device *ndev) if (priv->tx_head - priv->tx_tail <= unsent) break; stats->tx_packets++; - stats->tx_bytes += priv->tx_dlc[priv->tx_tail % - RCAR_CAN_FIFO_DEPTH]; - priv->tx_dlc[priv->tx_tail % RCAR_CAN_FIFO_DEPTH] = 0; - can_get_echo_skb(ndev, priv->tx_tail % RCAR_CAN_FIFO_DEPTH, NULL); + stats->tx_bytes += + can_get_echo_skb(ndev, + priv->tx_tail % RCAR_CAN_FIFO_DEPTH, + NULL); + priv->tx_tail++; netif_wake_queue(ndev); } @@ -612,7 +612,6 @@ static netdev_tx_t rcar_can_start_xmit(struct sk_buff *skb, writeb(cf->len, &priv->regs->mb[RCAR_CAN_TX_FIFO_MBX].dlc); - priv->tx_dlc[priv->tx_head % RCAR_CAN_FIFO_DEPTH] = cf->len; can_put_echo_skb(skb, ndev, priv->tx_head % RCAR_CAN_FIFO_DEPTH, 0); priv->tx_head++; /* Start Tx: write 0xff to the TFPCR register to increment diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index b1eded2f2c5d..d0e795f60338 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -502,7 +502,6 @@ struct rcar_canfd_channel { struct rcar_canfd_global *gpriv; /* Controller reference */ void __iomem *base; /* Register base address */ struct napi_struct napi; - u8 tx_len[RCANFD_FIFO_DEPTH]; /* For net stats */ u32 tx_head; /* Incremented on xmit */ u32 tx_tail; /* Incremented on xmit done */ u32 channel; /* Channel number */ @@ -1049,9 +1048,7 @@ static void rcar_canfd_tx_done(struct net_device *ndev) sent = priv->tx_tail % RCANFD_FIFO_DEPTH; stats->tx_packets++; - stats->tx_bytes += priv->tx_len[sent]; - priv->tx_len[sent] = 0; - can_get_echo_skb(ndev, sent, NULL); + stats->tx_bytes += can_get_echo_skb(ndev, sent, NULL); spin_lock_irqsave(&priv->tx_lock, flags); priv->tx_tail++; @@ -1461,7 +1458,6 @@ static netdev_tx_t rcar_canfd_start_xmit(struct sk_buff *skb, RCANFD_C_CFDF(ch, RCANFD_CFFIFO_IDX, 0)); } - priv->tx_len[priv->tx_head % RCANFD_FIFO_DEPTH] = cf->len; can_put_echo_skb(skb, ndev, priv->tx_head % RCANFD_FIFO_DEPTH, 0); spin_lock_irqsave(&priv->tx_lock, flags); diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index 4bf44d449987..966316479485 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -527,10 +527,8 @@ irqreturn_t sja1000_interrupt(int irq, void *dev_id) can_free_echo_skb(dev, 0, NULL); } else { /* transmission complete */ - stats->tx_bytes += - priv->read_reg(priv, SJA1000_FI) & 0xf; + stats->tx_bytes += can_get_echo_skb(dev, 0, NULL); stats->tx_packets++; - can_get_echo_skb(dev, 0, NULL); } netif_wake_queue(dev); can_led_event(dev, CAN_LED_EVENT_TX); diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c index 5cf03458e948..d4c7ce998a34 100644 --- a/drivers/net/can/slcan.c +++ b/drivers/net/can/slcan.c @@ -290,6 +290,8 @@ static void slc_encaps(struct slcan *sl, struct can_frame *cf) if (!(cf->can_id & CAN_RTR_FLAG)) { for (i = 0; i < cf->len; i++) pos = hex_byte_pack_upper(pos, cf->data[i]); + + sl->dev->stats.tx_bytes += cf->len; } *pos++ = '\r'; @@ -306,7 +308,6 @@ static void slc_encaps(struct slcan *sl, struct can_frame *cf) actual = sl->tty->ops->write(sl->tty, sl->xbuff, pos - sl->xbuff); sl->xleft = (pos - sl->xbuff) - actual; sl->xhead = sl->xbuff + actual; - sl->dev->stats.tx_bytes += cf->len; } /* Write out any remaining transmit buffer. Scheduled when tty is writable */ diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index cfc1325aad10..d74e895bddf7 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -282,7 +282,10 @@ static int softing_handle_1(struct softing *card) skb = priv->can.echo_skb[priv->tx.echo_get]; if (skb) skb->tstamp = ktime; - can_get_echo_skb(netdev, priv->tx.echo_get, NULL); + ++netdev->stats.tx_packets; + netdev->stats.tx_bytes += + can_get_echo_skb(netdev, priv->tx.echo_get, + NULL); ++priv->tx.echo_get; if (priv->tx.echo_get >= TX_ECHO_SKB_MAX) priv->tx.echo_get = 0; @@ -290,9 +293,6 @@ static int softing_handle_1(struct softing *card) --priv->tx.pending; if (card->tx.pending) --card->tx.pending; - ++netdev->stats.tx_packets; - if (!(msg.can_id & CAN_RTR_FLAG)) - netdev->stats.tx_bytes += msg.len; } else { int ret; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index 506625082b84..cfcc14fe3e42 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -153,7 +153,6 @@ struct hi3110_priv { u8 *spi_rx_buf; struct sk_buff *tx_skb; - int tx_len; struct workqueue_struct *wq; struct work_struct tx_work; @@ -166,6 +165,8 @@ struct hi3110_priv { #define HI3110_AFTER_SUSPEND_POWER 4 #define HI3110_AFTER_SUSPEND_RESTART 8 int restart_tx; + bool tx_busy; + struct regulator *power; struct regulator *transceiver; struct clk *clk; @@ -175,13 +176,13 @@ static void hi3110_clean(struct net_device *net) { struct hi3110_priv *priv = netdev_priv(net); - if (priv->tx_skb || priv->tx_len) + if (priv->tx_skb || priv->tx_busy) net->stats.tx_errors++; dev_kfree_skb(priv->tx_skb); - if (priv->tx_len) + if (priv->tx_busy) can_free_echo_skb(priv->net, 0, NULL); priv->tx_skb = NULL; - priv->tx_len = 0; + priv->tx_busy = false; } /* Note about handling of error return of hi3110_spi_trans: accessing @@ -369,7 +370,7 @@ static netdev_tx_t hi3110_hard_start_xmit(struct sk_buff *skb, struct hi3110_priv *priv = netdev_priv(net); struct spi_device *spi = priv->spi; - if (priv->tx_skb || priv->tx_len) { + if (priv->tx_skb || priv->tx_busy) { dev_err(&spi->dev, "hard_xmit called while tx busy\n"); return NETDEV_TX_BUSY; } @@ -586,7 +587,7 @@ static void hi3110_tx_work_handler(struct work_struct *ws) } else { frame = (struct can_frame *)priv->tx_skb->data; hi3110_hw_tx(spi, frame); - priv->tx_len = 1 + frame->len; + priv->tx_busy = true; can_put_echo_skb(priv->tx_skb, net, 0, 0); priv->tx_skb = NULL; } @@ -721,14 +722,11 @@ static irqreturn_t hi3110_can_ist(int irq, void *dev_id) } } - if (priv->tx_len && statf & HI3110_STAT_TXMTY) { + if (priv->tx_busy && statf & HI3110_STAT_TXMTY) { net->stats.tx_packets++; - net->stats.tx_bytes += priv->tx_len - 1; + net->stats.tx_bytes += can_get_echo_skb(net, 0, NULL); can_led_event(net, CAN_LED_EVENT_TX); - if (priv->tx_len) { - can_get_echo_skb(net, 0, NULL); - priv->tx_len = 0; - } + priv->tx_busy = false; netif_wake_queue(net); } @@ -755,7 +753,7 @@ static int hi3110_open(struct net_device *net) priv->force_quit = 0; priv->tx_skb = NULL; - priv->tx_len = 0; + priv->tx_busy = false; ret = request_threaded_irq(spi->irq, NULL, hi3110_can_ist, flags, DEVICE_NAME, priv); diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index 0da965da32cb..025e07cb7439 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -237,7 +237,6 @@ struct mcp251x_priv { u8 *spi_rx_buf; struct sk_buff *tx_skb; - int tx_len; struct workqueue_struct *wq; struct work_struct tx_work; @@ -250,6 +249,8 @@ struct mcp251x_priv { #define AFTER_SUSPEND_POWER 4 #define AFTER_SUSPEND_RESTART 8 int restart_tx; + bool tx_busy; + struct regulator *power; struct regulator *transceiver; struct clk *clk; @@ -272,13 +273,13 @@ static void mcp251x_clean(struct net_device *net) { struct mcp251x_priv *priv = netdev_priv(net); - if (priv->tx_skb || priv->tx_len) + if (priv->tx_skb || priv->tx_busy) net->stats.tx_errors++; dev_kfree_skb(priv->tx_skb); - if (priv->tx_len) + if (priv->tx_busy) can_free_echo_skb(priv->net, 0, NULL); priv->tx_skb = NULL; - priv->tx_len = 0; + priv->tx_busy = false; } /* Note about handling of error return of mcp251x_spi_trans: accessing @@ -785,7 +786,7 @@ static netdev_tx_t mcp251x_hard_start_xmit(struct sk_buff *skb, struct mcp251x_priv *priv = netdev_priv(net); struct spi_device *spi = priv->spi; - if (priv->tx_skb || priv->tx_len) { + if (priv->tx_skb || priv->tx_busy) { dev_warn(&spi->dev, "hard_xmit called while tx busy\n"); return NETDEV_TX_BUSY; } @@ -1010,7 +1011,7 @@ static void mcp251x_tx_work_handler(struct work_struct *ws) if (frame->len > CAN_FRAME_MAX_DATA_LEN) frame->len = CAN_FRAME_MAX_DATA_LEN; mcp251x_hw_tx(spi, frame, 0); - priv->tx_len = 1 + frame->len; + priv->tx_busy = true; can_put_echo_skb(priv->tx_skb, net, 0, 0); priv->tx_skb = NULL; } @@ -1176,12 +1177,12 @@ static irqreturn_t mcp251x_can_ist(int irq, void *dev_id) break; if (intf & CANINTF_TX) { - net->stats.tx_packets++; - net->stats.tx_bytes += priv->tx_len - 1; can_led_event(net, CAN_LED_EVENT_TX); - if (priv->tx_len) { - can_get_echo_skb(net, 0, NULL); - priv->tx_len = 0; + if (priv->tx_busy) { + net->stats.tx_packets++; + net->stats.tx_bytes += can_get_echo_skb(net, 0, + NULL); + priv->tx_busy = false; } netif_wake_queue(net); } @@ -1208,7 +1209,7 @@ static int mcp251x_open(struct net_device *net) priv->force_quit = 0; priv->tx_skb = NULL; - priv->tx_len = 0; + priv->tx_busy = false; if (!dev_fwnode(&spi->dev)) flags = IRQF_TRIGGER_FALLING; diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index e18868a43ce8..25d6d81ab4f4 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -661,11 +661,8 @@ static irqreturn_t sun4i_can_interrupt(int irq, void *dev_id) if (isrc & SUN4I_INT_TBUF_VLD) { /* transmission complete interrupt */ - stats->tx_bytes += - readl(priv->base + - SUN4I_REG_RBUF_RBACK_START_ADDR) & 0xf; + stats->tx_bytes += can_get_echo_skb(dev, 0, NULL); stats->tx_packets++; - can_get_echo_skb(dev, 0, NULL); netif_wake_queue(dev); can_led_event(dev, CAN_LED_EVENT_TX); } diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index c9b6adf5c1ec..7bedceffdfa3 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -230,7 +230,6 @@ struct ems_tx_urb_context { struct ems_usb *dev; u32 echo_index; - u8 dlc; }; struct ems_usb { @@ -517,9 +516,8 @@ static void ems_usb_write_bulk_callback(struct urb *urb) /* transmission complete interrupt */ netdev->stats.tx_packets++; - netdev->stats.tx_bytes += context->dlc; - - can_get_echo_skb(netdev, context->echo_index, NULL); + netdev->stats.tx_bytes += can_get_echo_skb(netdev, context->echo_index, + NULL); /* Release context */ context->echo_index = MAX_TX_URBS; @@ -805,7 +803,6 @@ static netdev_tx_t ems_usb_start_xmit(struct sk_buff *skb, struct net_device *ne context->dev = dev; context->echo_index = i; - context->dlc = cf->len; usb_fill_bulk_urb(urb, dev->udev, usb_sndbulkpipe(dev->udev, 2), buf, size, ems_usb_write_bulk_callback, context); diff --git a/drivers/net/can/usb/esd_usb2.c b/drivers/net/can/usb/esd_usb2.c index 9ac7ee44b6e3..286daaaea0b8 100644 --- a/drivers/net/can/usb/esd_usb2.c +++ b/drivers/net/can/usb/esd_usb2.c @@ -183,7 +183,6 @@ struct esd_usb2_net_priv; struct esd_tx_urb_context { struct esd_usb2_net_priv *priv; u32 echo_index; - int len; /* CAN payload length */ }; struct esd_usb2 { @@ -357,8 +356,8 @@ static void esd_usb2_tx_done_msg(struct esd_usb2_net_priv *priv, if (!msg->msg.txdone.status) { stats->tx_packets++; - stats->tx_bytes += context->len; - can_get_echo_skb(netdev, context->echo_index, NULL); + stats->tx_bytes += can_get_echo_skb(netdev, context->echo_index, + NULL); } else { stats->tx_errors++; can_free_echo_skb(netdev, context->echo_index, NULL); @@ -783,7 +782,6 @@ static netdev_tx_t esd_usb2_start_xmit(struct sk_buff *skb, context->priv = priv; context->echo_index = i; - context->len = cf->len; /* hnd must not be 0 - MSB is stripped in txdone handling */ msg->msg.tx.hnd = 0x80000000 | i; /* returned in TX done message */ diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 1b400de00f51..03b012963c20 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -357,9 +357,6 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) goto resubmit_urb; } - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += hf->can_dlc; - txc = gs_get_tx_context(dev, hf->echo_id); /* bad devices send bad echo_ids. */ @@ -370,7 +367,9 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) goto resubmit_urb; } - can_get_echo_skb(netdev, hf->echo_id, NULL); + netdev->stats.tx_packets++; + netdev->stats.tx_bytes += can_get_echo_skb(netdev, hf->echo_id, + NULL); gs_free_tx_context(txc); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb.h b/drivers/net/can/usb/kvaser_usb/kvaser_usb.h index 390b6bde883c..3a49257f9fa6 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb.h +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb.h @@ -77,7 +77,6 @@ struct kvaser_usb_dev_card_data { struct kvaser_usb_tx_urb_context { struct kvaser_usb_net_priv *priv; u32 echo_index; - int dlc; }; struct kvaser_usb { @@ -162,8 +161,8 @@ struct kvaser_usb_dev_ops { void (*dev_read_bulk_callback)(struct kvaser_usb *dev, void *buf, int len); void *(*dev_frame_to_cmd)(const struct kvaser_usb_net_priv *priv, - const struct sk_buff *skb, int *frame_len, - int *cmd_len, u16 transid); + const struct sk_buff *skb, int *cmd_len, + u16 transid); }; struct kvaser_usb_dev_cfg { diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index 3e682ef43f8e..c4b4d3d0a387 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -565,7 +565,7 @@ static netdev_tx_t kvaser_usb_start_xmit(struct sk_buff *skb, goto freeurb; } - buf = dev->ops->dev_frame_to_cmd(priv, skb, &context->dlc, &cmd_len, + buf = dev->ops->dev_frame_to_cmd(priv, skb, &cmd_len, context->echo_index); if (!buf) { stats->tx_dropped++; diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index bc902da9c6eb..a26823c5b62a 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -1114,6 +1114,7 @@ static void kvaser_usb_hydra_tx_acknowledge(const struct kvaser_usb *dev, struct kvaser_usb_tx_urb_context *context; struct kvaser_usb_net_priv *priv; unsigned long irq_flags; + unsigned int len; bool one_shot_fail = false; bool is_err_frame = false; u16 transid = kvaser_usb_hydra_get_cmd_transid(cmd); @@ -1140,21 +1141,22 @@ static void kvaser_usb_hydra_tx_acknowledge(const struct kvaser_usb *dev, } context = &priv->tx_contexts[transid % dev->max_tx_urbs]; - if (!one_shot_fail && !is_err_frame) { - struct net_device_stats *stats = &priv->netdev->stats; - - stats->tx_packets++; - stats->tx_bytes += can_fd_dlc2len(context->dlc); - } spin_lock_irqsave(&priv->tx_contexts_lock, irq_flags); - can_get_echo_skb(priv->netdev, context->echo_index, NULL); + len = can_get_echo_skb(priv->netdev, context->echo_index, NULL); context->echo_index = dev->max_tx_urbs; --priv->active_tx_contexts; netif_wake_queue(priv->netdev); spin_unlock_irqrestore(&priv->tx_contexts_lock, irq_flags); + + if (!one_shot_fail && !is_err_frame) { + struct net_device_stats *stats = &priv->netdev->stats; + + stats->tx_packets++; + stats->tx_bytes += len; + } } static void kvaser_usb_hydra_rx_msg_std(const struct kvaser_usb *dev, @@ -1373,8 +1375,8 @@ static void kvaser_usb_hydra_handle_cmd(const struct kvaser_usb *dev, static void * kvaser_usb_hydra_frame_to_cmd_ext(const struct kvaser_usb_net_priv *priv, - const struct sk_buff *skb, int *frame_len, - int *cmd_len, u16 transid) + const struct sk_buff *skb, int *cmd_len, + u16 transid) { struct kvaser_usb *dev = priv->dev; struct kvaser_cmd_ext *cmd; @@ -1386,8 +1388,6 @@ kvaser_usb_hydra_frame_to_cmd_ext(const struct kvaser_usb_net_priv *priv, u32 kcan_id; u32 kcan_header; - *frame_len = nbr_of_bytes; - cmd = kcalloc(1, sizeof(struct kvaser_cmd_ext), GFP_ATOMIC); if (!cmd) return NULL; @@ -1453,8 +1453,8 @@ kvaser_usb_hydra_frame_to_cmd_ext(const struct kvaser_usb_net_priv *priv, static void * kvaser_usb_hydra_frame_to_cmd_std(const struct kvaser_usb_net_priv *priv, - const struct sk_buff *skb, int *frame_len, - int *cmd_len, u16 transid) + const struct sk_buff *skb, int *cmd_len, + u16 transid) { struct kvaser_usb *dev = priv->dev; struct kvaser_cmd *cmd; @@ -1462,8 +1462,6 @@ kvaser_usb_hydra_frame_to_cmd_std(const struct kvaser_usb_net_priv *priv, u32 flags; u32 id; - *frame_len = cf->len; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_ATOMIC); if (!cmd) return NULL; @@ -1497,7 +1495,7 @@ kvaser_usb_hydra_frame_to_cmd_std(const struct kvaser_usb_net_priv *priv, cmd->tx_can.id = cpu_to_le32(id); cmd->tx_can.flags = flags; - memcpy(cmd->tx_can.data, cf->data, *frame_len); + memcpy(cmd->tx_can.data, cf->data, cf->len); return cmd; } @@ -2005,17 +2003,17 @@ static void kvaser_usb_hydra_read_bulk_callback(struct kvaser_usb *dev, static void * kvaser_usb_hydra_frame_to_cmd(const struct kvaser_usb_net_priv *priv, - const struct sk_buff *skb, int *frame_len, - int *cmd_len, u16 transid) + const struct sk_buff *skb, int *cmd_len, + u16 transid) { void *buf; if (priv->dev->card_data.capabilities & KVASER_USB_HYDRA_CAP_EXT_CMD) - buf = kvaser_usb_hydra_frame_to_cmd_ext(priv, skb, frame_len, - cmd_len, transid); + buf = kvaser_usb_hydra_frame_to_cmd_ext(priv, skb, cmd_len, + transid); else - buf = kvaser_usb_hydra_frame_to_cmd_std(priv, skb, frame_len, - cmd_len, transid); + buf = kvaser_usb_hydra_frame_to_cmd_std(priv, skb, cmd_len, + transid); return buf; } diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index 891271af7616..c805b999c543 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -389,16 +389,14 @@ static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_dev_cfg_32mhz = { static void * kvaser_usb_leaf_frame_to_cmd(const struct kvaser_usb_net_priv *priv, - const struct sk_buff *skb, int *frame_len, - int *cmd_len, u16 transid) + const struct sk_buff *skb, int *cmd_len, + u16 transid) { struct kvaser_usb *dev = priv->dev; struct kvaser_cmd *cmd; u8 *cmd_tx_can_flags = NULL; /* GCC */ struct can_frame *cf = (struct can_frame *)skb->data; - *frame_len = cf->len; - cmd = kmalloc(sizeof(*cmd), GFP_ATOMIC); if (cmd) { cmd->u.tx_can.tid = transid & 0xff; @@ -654,12 +652,11 @@ static void kvaser_usb_leaf_tx_acknowledge(const struct kvaser_usb *dev, priv->can.state = CAN_STATE_ERROR_ACTIVE; } - stats->tx_packets++; - stats->tx_bytes += context->dlc; - spin_lock_irqsave(&priv->tx_contexts_lock, flags); - can_get_echo_skb(priv->netdev, context->echo_index, NULL); + stats->tx_packets++; + stats->tx_bytes += can_get_echo_skb(priv->netdev, + context->echo_index, NULL); context->echo_index = dev->max_tx_urbs; --priv->active_tx_contexts; netif_wake_queue(priv->netdev); diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index 4d20ea860ea8..77bddff86252 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -64,7 +64,6 @@ struct mcba_usb_ctx { struct mcba_priv *priv; u32 ndx; - u8 dlc; bool can; }; @@ -184,13 +183,10 @@ static inline struct mcba_usb_ctx *mcba_usb_get_free_ctx(struct mcba_priv *priv, ctx = &priv->tx_context[i]; ctx->ndx = i; - if (cf) { + if (cf) ctx->can = true; - ctx->dlc = cf->len; - } else { + else ctx->can = false; - ctx->dlc = 0; - } atomic_dec(&priv->free_ctx_cnt); break; @@ -236,10 +232,10 @@ static void mcba_usb_write_bulk_callback(struct urb *urb) return; netdev->stats.tx_packets++; - netdev->stats.tx_bytes += ctx->dlc; + netdev->stats.tx_bytes += can_get_echo_skb(netdev, ctx->ndx, + NULL); can_led_event(netdev, CAN_LED_EVENT_TX); - can_get_echo_skb(netdev, ctx->ndx, NULL); } if (urb->status) diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 6107fef9f4a0..b850ff8fe4bd 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -291,6 +291,7 @@ static void peak_usb_write_bulk_callback(struct urb *urb) struct peak_tx_urb_context *context = urb->context; struct peak_usb_device *dev; struct net_device *netdev; + int tx_bytes; BUG_ON(!context); @@ -305,10 +306,6 @@ static void peak_usb_write_bulk_callback(struct urb *urb) /* check tx status */ switch (urb->status) { case 0: - /* transmission complete */ - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += context->data_len; - /* prevent tx timeout */ netif_trans_update(netdev); break; @@ -327,12 +324,17 @@ static void peak_usb_write_bulk_callback(struct urb *urb) } /* should always release echo skb and corresponding context */ - can_get_echo_skb(netdev, context->echo_index, NULL); + tx_bytes = can_get_echo_skb(netdev, context->echo_index, NULL); context->echo_index = PCAN_USB_MAX_TX_URBS; - /* do wakeup tx queue in case of success only */ - if (!urb->status) + if (!urb->status) { + /* transmission complete */ + netdev->stats.tx_packets++; + netdev->stats.tx_bytes += tx_bytes; + + /* do wakeup tx queue in case of success only */ netif_wake_queue(netdev); + } } /* @@ -344,7 +346,6 @@ static netdev_tx_t peak_usb_ndo_start_xmit(struct sk_buff *skb, struct peak_usb_device *dev = netdev_priv(netdev); struct peak_tx_urb_context *context = NULL; struct net_device_stats *stats = &netdev->stats; - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct urb *urb; u8 *obuf; int i, err; @@ -378,9 +379,6 @@ static netdev_tx_t peak_usb_ndo_start_xmit(struct sk_buff *skb, context->echo_index = i; - /* Note: this works with CANFD frames too */ - context->data_len = cfd->len; - usb_anchor_urb(urb, &dev->tx_submitted); can_put_echo_skb(skb, netdev, context->echo_index, 0); diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.h b/drivers/net/can/usb/peak_usb/pcan_usb_core.h index daa19f57e742..f60af573a2e0 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.h +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.h @@ -99,7 +99,6 @@ struct peak_time_ref { struct peak_tx_urb_context { struct peak_usb_device *dev; u32 echo_index; - u8 data_len; struct urb *urb; }; diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 388899019955..c7c41d1fd038 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -259,7 +259,6 @@ struct ucan_priv; /* Context Information for transmission URBs */ struct ucan_urb_context { struct ucan_priv *up; - u8 dlc; bool allocated; }; @@ -637,7 +636,7 @@ static void ucan_tx_complete_msg(struct ucan_priv *up, { unsigned long flags; u16 count, i; - u8 echo_index, dlc; + u8 echo_index; u16 len = le16_to_cpu(m->len); struct ucan_urb_context *context; @@ -661,7 +660,6 @@ static void ucan_tx_complete_msg(struct ucan_priv *up, /* gather information from the context */ context = &up->context_array[echo_index]; - dlc = READ_ONCE(context->dlc); /* Release context and restart queue if necessary. * Also check if the context was allocated @@ -674,8 +672,8 @@ static void ucan_tx_complete_msg(struct ucan_priv *up, UCAN_TX_COMPLETE_SUCCESS) { /* update statistics */ up->netdev->stats.tx_packets++; - up->netdev->stats.tx_bytes += dlc; - can_get_echo_skb(up->netdev, echo_index, NULL); + up->netdev->stats.tx_bytes += + can_get_echo_skb(up->netdev, echo_index, NULL); } else { up->netdev->stats.tx_dropped++; can_free_echo_skb(up->netdev, echo_index, NULL); @@ -1089,8 +1087,6 @@ static struct urb *ucan_prepare_tx_urb(struct ucan_priv *up, } m->len = cpu_to_le16(mlen); - context->dlc = cf->len; - m->subtype = echo_index; /* build the urb */ diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index d0ef1478cf0a..431af1ec1e3c 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -114,7 +114,6 @@ struct usb_8dev_tx_urb_context { struct usb_8dev_priv *priv; u32 echo_index; - u8 dlc; }; /* Structure to hold all of our device specific stuff */ @@ -581,9 +580,7 @@ static void usb_8dev_write_bulk_callback(struct urb *urb) urb->status); netdev->stats.tx_packets++; - netdev->stats.tx_bytes += context->dlc; - - can_get_echo_skb(netdev, context->echo_index, NULL); + netdev->stats.tx_bytes += can_get_echo_skb(netdev, context->echo_index, NULL); can_led_event(netdev, CAN_LED_EVENT_TX); @@ -654,7 +651,6 @@ static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb, context->priv = priv; context->echo_index = i; - context->dlc = cf->len; usb_fill_bulk_urb(urb, priv->udev, usb_sndbulkpipe(priv->udev, USB_8DEV_ENDP_DATA_TX), diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c index 067705e2850b..c42f18845b02 100644 --- a/drivers/net/can/vcan.c +++ b/drivers/net/can/vcan.c @@ -87,13 +87,14 @@ static netdev_tx_t vcan_tx(struct sk_buff *skb, struct net_device *dev) { struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct net_device_stats *stats = &dev->stats; - int loop; + int loop, len; if (can_dropped_invalid_skb(dev, skb)) return NETDEV_TX_OK; + len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len; stats->tx_packets++; - stats->tx_bytes += cfd->len; + stats->tx_bytes += len; /* set flag whether this packet has to be looped back */ loop = skb->pkt_type == PACKET_LOOPBACK; @@ -105,7 +106,7 @@ static netdev_tx_t vcan_tx(struct sk_buff *skb, struct net_device *dev) * CAN core already did the echo for us */ stats->rx_packets++; - stats->rx_bytes += cfd->len; + stats->rx_bytes += len; } consume_skb(skb); return NETDEV_TX_OK; diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c index 8861a7d875e7..47ccc15a3486 100644 --- a/drivers/net/can/vxcan.c +++ b/drivers/net/can/vxcan.c @@ -62,7 +62,7 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *skb, struct net_device *dev) skb->dev = peer; skb->ip_summed = CHECKSUM_UNNECESSARY; - len = cfd->len; + len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len; if (netif_rx_ni(skb) == NET_RX_SUCCESS) { srcstats->tx_packets++; srcstats->tx_bytes += len; diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index d311bc369a39..fdb22b00674a 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -21,8 +21,9 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, unsigned int idx, unsigned int frame_len); struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, unsigned int *frame_len_ptr); -unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx, - unsigned int *frame_len_ptr); +unsigned int __must_check can_get_echo_skb(struct net_device *dev, + unsigned int idx, + unsigned int *frame_len_ptr); void can_free_echo_skb(struct net_device *dev, unsigned int idx, unsigned int *frame_len_ptr); struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); -- cgit v1.2.3-59-g8ed1b From c9e1d8ed304cc6106c3241add170193995953325 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Tue, 14 Dec 2021 01:02:23 +0900 Subject: can: dev: replace can_priv::ctrlmode_static by can_get_static_ctrlmode() The statically enabled features of a CAN controller can be retrieved using below formula: | u32 ctrlmode_static = priv->ctrlmode & ~priv->ctrlmode_supported; As such, there is no need to store this information. This patch remove the field ctrlmode_static of struct can_priv and provides, in replacement, the inline function can_get_static_ctrlmode() which returns the same value. Link: https://lore.kernel.org/all/20211213160226.56219-2-mailhol.vincent@wanadoo.fr Signed-off-by: Vincent Mailhol Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/dev.c | 5 +++-- drivers/net/can/dev/netlink.c | 2 +- include/linux/can/dev.h | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 4845ae6456e1..c192f25f9695 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -296,6 +296,7 @@ EXPORT_SYMBOL_GPL(free_candev); int can_change_mtu(struct net_device *dev, int new_mtu) { struct can_priv *priv = netdev_priv(dev); + u32 ctrlmode_static = can_get_static_ctrlmode(priv); /* Do not allow changing the MTU while running */ if (dev->flags & IFF_UP) @@ -305,7 +306,7 @@ int can_change_mtu(struct net_device *dev, int new_mtu) switch (new_mtu) { case CAN_MTU: /* 'CANFD-only' controllers can not switch to CAN_MTU */ - if (priv->ctrlmode_static & CAN_CTRLMODE_FD) + if (ctrlmode_static & CAN_CTRLMODE_FD) return -EINVAL; priv->ctrlmode &= ~CAN_CTRLMODE_FD; @@ -314,7 +315,7 @@ int can_change_mtu(struct net_device *dev, int new_mtu) case CANFD_MTU: /* check for potential CANFD ability */ if (!(priv->ctrlmode_supported & CAN_CTRLMODE_FD) && - !(priv->ctrlmode_static & CAN_CTRLMODE_FD)) + !(ctrlmode_static & CAN_CTRLMODE_FD)) return -EINVAL; priv->ctrlmode |= CAN_CTRLMODE_FD; diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c index 95cca4e5251f..26c336808be5 100644 --- a/drivers/net/can/dev/netlink.c +++ b/drivers/net/can/dev/netlink.c @@ -211,7 +211,7 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[], if (dev->flags & IFF_UP) return -EBUSY; cm = nla_data(data[IFLA_CAN_CTRLMODE]); - ctrlstatic = priv->ctrlmode_static; + ctrlstatic = can_get_static_ctrlmode(priv); maskedflags = cm->flags & cm->mask; /* check whether provided bits are allowed to be passed */ diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 45f19d9db5ca..92e2d69462f0 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -69,7 +69,6 @@ struct can_priv { /* CAN controller features - see include/uapi/linux/can/netlink.h */ u32 ctrlmode; /* current options setting */ u32 ctrlmode_supported; /* options that can be modified by netlink */ - u32 ctrlmode_static; /* static enabled options for driver/hardware */ int restart_ms; struct delayed_work restart_work; @@ -139,13 +138,17 @@ static inline void can_set_static_ctrlmode(struct net_device *dev, /* alloc_candev() succeeded => netdev_priv() is valid at this point */ priv->ctrlmode = static_mode; - priv->ctrlmode_static = static_mode; /* override MTU which was set by default in can_setup()? */ if (static_mode & CAN_CTRLMODE_FD) dev->mtu = CANFD_MTU; } +static inline u32 can_get_static_ctrlmode(struct can_priv *priv) +{ + return priv->ctrlmode & ~priv->ctrlmode_supported; +} + void can_setup(struct net_device *dev); struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, -- cgit v1.2.3-59-g8ed1b From 7d4a101c0bd3c6e5c6e45c705a54f7bc8f6c128d Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Tue, 14 Dec 2021 01:02:24 +0900 Subject: can: dev: add sanity check in can_set_static_ctrlmode() Previous patch removed can_priv::ctrlmode_static to replace it with can_get_static_ctrlmode(). A condition sine qua non for this to work is that the controller static modes should never be set in can_priv::ctrlmode_supported (c.f. the comment on can_priv::ctrlmode_supported which states that it is for "options that can be *modified* by netlink"). Also, this condition is already correctly fulfilled by all existing drivers which rely on the ctrlmode_static feature. Nonetheless, we added an extra safeguard in can_set_static_ctrlmode() to return an error value and to warn the developer who would be adventurous enough to set to static a given feature that is already set to supported. The drivers which rely on the static controller mode are then updated to check the return value of can_set_static_ctrlmode(). Link: https://lore.kernel.org/all/20211213160226.56219-3-mailhol.vincent@wanadoo.fr Signed-off-by: Vincent Mailhol Signed-off-by: Marc Kleine-Budde --- drivers/net/can/m_can/m_can.c | 10 +++++++--- drivers/net/can/rcar/rcar_canfd.c | 4 +++- include/linux/can/dev.h | 11 +++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index ae2349420983..5b47cd867783 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -1456,7 +1456,7 @@ static bool m_can_niso_supported(struct m_can_classdev *cdev) static int m_can_dev_setup(struct m_can_classdev *cdev) { struct net_device *dev = cdev->net; - int m_can_version; + int m_can_version, err; m_can_version = m_can_check_core_release(cdev); /* return if unsupported version */ @@ -1486,7 +1486,9 @@ static int m_can_dev_setup(struct m_can_classdev *cdev) switch (cdev->version) { case 30: /* CAN_CTRLMODE_FD_NON_ISO is fixed with M_CAN IP v3.0.x */ - can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO); + err = can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO); + if (err) + return err; cdev->can.bittiming_const = cdev->bit_timing ? cdev->bit_timing : &m_can_bittiming_const_30X; @@ -1496,7 +1498,9 @@ static int m_can_dev_setup(struct m_can_classdev *cdev) break; case 31: /* CAN_CTRLMODE_FD_NON_ISO is fixed with M_CAN IP v3.1.x */ - can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO); + err = can_set_static_ctrlmode(dev, CAN_CTRLMODE_FD_NON_ISO); + if (err) + return err; cdev->can.bittiming_const = cdev->bit_timing ? cdev->bit_timing : &m_can_bittiming_const_31X; diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index d0e795f60338..82cc71d5ee10 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1699,7 +1699,9 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, &rcar_canfd_data_bittiming_const; /* Controller starts in CAN FD only mode */ - can_set_static_ctrlmode(ndev, CAN_CTRLMODE_FD); + err = can_set_static_ctrlmode(ndev, CAN_CTRLMODE_FD); + if (err) + goto fail; priv->can.ctrlmode_supported = CAN_CTRLMODE_BERR_REPORTING; } else { /* Controller starts in Classical CAN only mode */ diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 92e2d69462f0..fff3f70df697 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -131,17 +131,24 @@ static inline s32 can_get_relative_tdco(const struct can_priv *priv) } /* helper to define static CAN controller features at device creation time */ -static inline void can_set_static_ctrlmode(struct net_device *dev, - u32 static_mode) +static inline int __must_check can_set_static_ctrlmode(struct net_device *dev, + u32 static_mode) { struct can_priv *priv = netdev_priv(dev); /* alloc_candev() succeeded => netdev_priv() is valid at this point */ + if (priv->ctrlmode_supported & static_mode) { + netdev_warn(dev, + "Controller features can not be supported and static at the same time\n"); + return -EINVAL; + } priv->ctrlmode = static_mode; /* override MTU which was set by default in can_setup()? */ if (static_mode & CAN_CTRLMODE_FD) dev->mtu = CANFD_MTU; + + return 0; } static inline u32 can_get_static_ctrlmode(struct can_priv *priv) -- cgit v1.2.3-59-g8ed1b From 5fe1be81efd28bf2afcac218f23118dca6b1b648 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Tue, 14 Dec 2021 01:02:25 +0900 Subject: can: dev: reorder struct can_priv members for better packing Save eight bytes of holes on x86-64 architectures by reordering the members of struct can_priv. Before: | $ pahole -C can_priv drivers/net/can/dev/dev.o | struct can_priv { | struct net_device * dev; /* 0 8 */ | struct can_device_stats can_stats; /* 8 24 */ | const struct can_bittiming_const * bittiming_const; /* 32 8 */ | const struct can_bittiming_const * data_bittiming_const; /* 40 8 */ | struct can_bittiming bittiming; /* 48 32 */ | /* --- cacheline 1 boundary (64 bytes) was 16 bytes ago --- */ | struct can_bittiming data_bittiming; /* 80 32 */ | const struct can_tdc_const * tdc_const; /* 112 8 */ | struct can_tdc tdc; /* 120 12 */ | /* --- cacheline 2 boundary (128 bytes) was 4 bytes ago --- */ | unsigned int bitrate_const_cnt; /* 132 4 */ | const u32 * bitrate_const; /* 136 8 */ | const u32 * data_bitrate_const; /* 144 8 */ | unsigned int data_bitrate_const_cnt; /* 152 4 */ | u32 bitrate_max; /* 156 4 */ | struct can_clock clock; /* 160 4 */ | unsigned int termination_const_cnt; /* 164 4 */ | const u16 * termination_const; /* 168 8 */ | u16 termination; /* 176 2 */ | | /* XXX 6 bytes hole, try to pack */ | | struct gpio_desc * termination_gpio; /* 184 8 */ | /* --- cacheline 3 boundary (192 bytes) --- */ | u16 termination_gpio_ohms[2]; /* 192 4 */ | enum can_state state; /* 196 4 */ | u32 ctrlmode; /* 200 4 */ | u32 ctrlmode_supported; /* 204 4 */ | int restart_ms; /* 208 4 */ | | /* XXX 4 bytes hole, try to pack */ | | struct delayed_work restart_work; /* 216 88 */ | | /* XXX last struct has 4 bytes of padding */ | | /* --- cacheline 4 boundary (256 bytes) was 48 bytes ago --- */ | int (*do_set_bittiming)(struct net_device *); /* 304 8 */ | int (*do_set_data_bittiming)(struct net_device *); /* 312 8 */ | /* --- cacheline 5 boundary (320 bytes) --- */ | int (*do_set_mode)(struct net_device *, enum can_mode); /* 320 8 */ | int (*do_set_termination)(struct net_device *, u16); /* 328 8 */ | int (*do_get_state)(const struct net_device *, enum can_state *); /* 336 8 */ | int (*do_get_berr_counter)(const struct net_device *, struct can_berr_counter *); /* 344 8 */ | unsigned int echo_skb_max; /* 352 4 */ | | /* XXX 4 bytes hole, try to pack */ | | struct sk_buff * * echo_skb; /* 360 8 */ | | /* size: 368, cachelines: 6, members: 32 */ | /* sum members: 354, holes: 3, sum holes: 14 */ | /* paddings: 1, sum paddings: 4 */ | /* last cacheline: 48 bytes */ | }; After: | $ pahole -C can_priv drivers/net/can/dev/dev.o | struct can_priv { | struct net_device * dev; /* 0 8 */ | struct can_device_stats can_stats; /* 8 24 */ | const struct can_bittiming_const * bittiming_const; /* 32 8 */ | const struct can_bittiming_const * data_bittiming_const; /* 40 8 */ | struct can_bittiming bittiming; /* 48 32 */ | /* --- cacheline 1 boundary (64 bytes) was 16 bytes ago --- */ | struct can_bittiming data_bittiming; /* 80 32 */ | const struct can_tdc_const * tdc_const; /* 112 8 */ | struct can_tdc tdc; /* 120 12 */ | /* --- cacheline 2 boundary (128 bytes) was 4 bytes ago --- */ | unsigned int bitrate_const_cnt; /* 132 4 */ | const u32 * bitrate_const; /* 136 8 */ | const u32 * data_bitrate_const; /* 144 8 */ | unsigned int data_bitrate_const_cnt; /* 152 4 */ | u32 bitrate_max; /* 156 4 */ | struct can_clock clock; /* 160 4 */ | unsigned int termination_const_cnt; /* 164 4 */ | const u16 * termination_const; /* 168 8 */ | u16 termination; /* 176 2 */ | | /* XXX 6 bytes hole, try to pack */ | | struct gpio_desc * termination_gpio; /* 184 8 */ | /* --- cacheline 3 boundary (192 bytes) --- */ | u16 termination_gpio_ohms[2]; /* 192 4 */ | unsigned int echo_skb_max; /* 196 4 */ | struct sk_buff * * echo_skb; /* 200 8 */ | enum can_state state; /* 208 4 */ | u32 ctrlmode; /* 212 4 */ | u32 ctrlmode_supported; /* 216 4 */ | int restart_ms; /* 220 4 */ | struct delayed_work restart_work; /* 224 88 */ | | /* XXX last struct has 4 bytes of padding */ | | /* --- cacheline 4 boundary (256 bytes) was 56 bytes ago --- */ | int (*do_set_bittiming)(struct net_device *); /* 312 8 */ | /* --- cacheline 5 boundary (320 bytes) --- */ | int (*do_set_data_bittiming)(struct net_device *); /* 320 8 */ | int (*do_set_mode)(struct net_device *, enum can_mode); /* 328 8 */ | int (*do_set_termination)(struct net_device *, u16); /* 336 8 */ | int (*do_get_state)(const struct net_device *, enum can_state *); /* 344 8 */ | int (*do_get_berr_counter)(const struct net_device *, struct can_berr_counter *); /* 352 8 */ | | /* size: 360, cachelines: 6, members: 32 */ | /* sum members: 354, holes: 1, sum holes: 6 */ | /* paddings: 1, sum paddings: 4 */ | /* last cacheline: 40 bytes */ | }; Link: https://lore.kernel.org/all/20211213160226.56219-4-mailhol.vincent@wanadoo.fr Signed-off-by: Vincent Mailhol Signed-off-by: Marc Kleine-Budde --- include/linux/can/dev.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index fff3f70df697..c2ea47f30046 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -64,6 +64,9 @@ struct can_priv { struct gpio_desc *termination_gpio; u16 termination_gpio_ohms[CAN_TERMINATION_GPIO_MAX]; + unsigned int echo_skb_max; + struct sk_buff **echo_skb; + enum can_state state; /* CAN controller features - see include/uapi/linux/can/netlink.h */ @@ -83,9 +86,6 @@ struct can_priv { struct can_berr_counter *bec); int (*do_get_auto_tdcv)(const struct net_device *dev, u32 *tdcv); - unsigned int echo_skb_max; - struct sk_buff **echo_skb; - #ifdef CONFIG_CAN_LEDS struct led_trigger *tx_led_trig; char tx_led_trig_name[CAN_LED_NAME_SZ]; -- cgit v1.2.3-59-g8ed1b From c6af53f038aa32cec12e8a305ba07c7ef168f1b0 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Tue, 4 Jan 2022 12:07:00 +0000 Subject: net: mdio: add helpers to extract clause 45 regad and devad fields Add a couple of helpers and definitions to extract the clause 45 regad and devad fields from the regnum passed into MDIO drivers. Tested-by: Daniel Golle Reviewed-by: Andrew Lunn Signed-off-by: Russell King (Oracle) Signed-off-by: Daniel Golle Signed-off-by: David S. Miller --- include/linux/mdio.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 9f3587a61e14..ecac96d52e01 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -7,6 +7,7 @@ #define __LINUX_MDIO_H__ #include +#include #include /* Or MII_ADDR_C45 into regnum for read/write on mii_bus to enable the 21 bit @@ -14,6 +15,7 @@ */ #define MII_ADDR_C45 (1<<30) #define MII_DEVADDR_C45_SHIFT 16 +#define MII_DEVADDR_C45_MASK GENMASK(20, 16) #define MII_REGADDR_C45_MASK GENMASK(15, 0) struct gpio_desc; @@ -381,6 +383,16 @@ static inline u32 mdiobus_c45_addr(int devad, u16 regnum) return MII_ADDR_C45 | devad << MII_DEVADDR_C45_SHIFT | regnum; } +static inline u16 mdiobus_c45_regad(u32 regnum) +{ + return FIELD_GET(MII_REGADDR_C45_MASK, regnum); +} + +static inline u16 mdiobus_c45_devad(u32 regnum) +{ + return FIELD_GET(MII_DEVADDR_C45_MASK, regnum); +} + static inline int __mdiobus_c45_read(struct mii_bus *bus, int prtad, int devad, u16 regnum) { -- cgit v1.2.3-59-g8ed1b From 01ec4a2e8f01f027a0f06cad237c935da8d643bf Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sat, 3 Jul 2021 00:23:39 +0200 Subject: headers/deps: USB: Optimize dependencies, remove The header is used over 1,400 times in a typical distro build, but few of its users actually need the full header. -------------------------------------------------------------------- | Combined, preprocessed C code size of header, without line markers, | with comments stripped: ------------------------- before: | #include | LOC: 7,078 | headers: 172 after: | #include | LOC: 812 | headers: 38 Remove it and add it to the places that need it. Signed-off-by: Ingo Molnar Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/ch9.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/usb/ch9.h b/include/linux/usb/ch9.h index 1cffa34740b0..969e7dba6358 100644 --- a/include/linux/usb/ch9.h +++ b/include/linux/usb/ch9.h @@ -33,7 +33,6 @@ #ifndef __LINUX_USB_CH9_H #define __LINUX_USB_CH9_H -#include #include /* USB 3.2 SuperSpeed Plus phy signaling rate generation and lane count */ @@ -45,6 +44,8 @@ enum usb_ssp_rate { USB_SSP_GEN_2x2, }; +struct device; + extern const char *usb_ep_type_string(int ep_type); extern const char *usb_speed_string(enum usb_device_speed speed); extern enum usb_device_speed usb_get_maximum_speed(struct device *dev); -- cgit v1.2.3-59-g8ed1b From edce22e19bfa86efa2522d041d6367f2f099e8ed Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 5 Jan 2022 09:05:15 -0800 Subject: block: move rq_list macros to blk-mq.h Move the request list macros to the header file that defines that struct they operate on. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220105170518.3181469-2-kbusch@kernel.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- include/linux/blk-mq.h | 29 +++++++++++++++++++++++++++++ include/linux/blkdev.h | 29 ----------------------------- 3 files changed, 30 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/fs/io_uring.c b/fs/io_uring.c index c4f217613f56..42bbbd34f45e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -57,7 +57,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 550996cf419c..bf64b94cd64e 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -216,6 +216,35 @@ static inline unsigned short req_get_ioprio(struct request *req) #define rq_dma_dir(rq) \ (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE) +#define rq_list_add(listptr, rq) do { \ + (rq)->rq_next = *(listptr); \ + *(listptr) = rq; \ +} while (0) + +#define rq_list_pop(listptr) \ +({ \ + struct request *__req = NULL; \ + if ((listptr) && *(listptr)) { \ + __req = *(listptr); \ + *(listptr) = __req->rq_next; \ + } \ + __req; \ +}) + +#define rq_list_peek(listptr) \ +({ \ + struct request *__req = NULL; \ + if ((listptr) && *(listptr)) \ + __req = *(listptr); \ + __req; \ +}) + +#define rq_list_for_each(listptr, pos) \ + for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) + +#define rq_list_next(rq) (rq)->rq_next +#define rq_list_empty(list) ((list) == (struct request *) NULL) + enum blk_eh_timer_return { BLK_EH_DONE, /* drivers has completed the command */ BLK_EH_RESET_TIMER, /* reset timer and try again */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 22746b2d6825..9c95df26fc26 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1339,33 +1339,4 @@ struct io_comp_batch { #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } -#define rq_list_add(listptr, rq) do { \ - (rq)->rq_next = *(listptr); \ - *(listptr) = rq; \ -} while (0) - -#define rq_list_pop(listptr) \ -({ \ - struct request *__req = NULL; \ - if ((listptr) && *(listptr)) { \ - __req = *(listptr); \ - *(listptr) = __req->rq_next; \ - } \ - __req; \ -}) - -#define rq_list_peek(listptr) \ -({ \ - struct request *__req = NULL; \ - if ((listptr) && *(listptr)) \ - __req = *(listptr); \ - __req; \ -}) - -#define rq_list_for_each(listptr, pos) \ - for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) - -#define rq_list_next(rq) (rq)->rq_next -#define rq_list_empty(list) ((list) == (struct request *) NULL) - #endif /* _LINUX_BLKDEV_H */ -- cgit v1.2.3-59-g8ed1b From 3764fd05e1f89530e2ee5cbff0b638f2b1141b90 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 5 Jan 2022 09:05:16 -0800 Subject: block: introduce rq_list_for_each_safe macro While iterating a list, a particular request may need to be removed for special handling. Provide an iterator that can safely handle that. Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://lore.kernel.org/r/20220105170518.3181469-3-kbusch@kernel.org Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index bf64b94cd64e..1467f0fa2142 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -242,6 +242,10 @@ static inline unsigned short req_get_ioprio(struct request *req) #define rq_list_for_each(listptr, pos) \ for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos)) +#define rq_list_for_each_safe(listptr, pos, nxt) \ + for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \ + pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL) + #define rq_list_next(rq) (rq)->rq_next #define rq_list_empty(list) ((list) == (struct request *) NULL) -- cgit v1.2.3-59-g8ed1b From d2528be7a8b09af9796a270debd14101a72bb552 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 5 Jan 2022 09:05:17 -0800 Subject: block: introduce rq_list_move When iterating a list, a particular request may need to be moved for special handling. Provide a helper function to achieve that so drivers don't need to reimplement rqlist manipulation. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220105170518.3181469-4-kbusch@kernel.org Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 1467f0fa2142..f40a05ecca4a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -249,6 +249,23 @@ static inline unsigned short req_get_ioprio(struct request *req) #define rq_list_next(rq) (rq)->rq_next #define rq_list_empty(list) ((list) == (struct request *) NULL) +/** + * rq_list_move() - move a struct request from one list to another + * @src: The source list @rq is currently in + * @dst: The destination list that @rq will be appended to + * @rq: The request to move + * @prev: The request preceding @rq in @src (NULL if @rq is the head) + */ +static void inline rq_list_move(struct request **src, struct request **dst, + struct request *rq, struct request *prev) +{ + if (prev) + prev->rq_next = rq->rq_next; + else + *src = rq->rq_next; + rq_list_add(dst, rq); +} + enum blk_eh_timer_return { BLK_EH_DONE, /* drivers has completed the command */ BLK_EH_RESET_TIMER, /* reset timer and try again */ -- cgit v1.2.3-59-g8ed1b From d53ad5d8b218a885e95080d4d3d556b16b91b1b9 Mon Sep 17 00:00:00 2001 From: Toke Høiland-Jørgensen Date: Mon, 3 Jan 2022 16:08:09 +0100 Subject: xdp: Move conversion to xdp_frame out of map functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All map redirect functions except XSK maps convert xdp_buff to xdp_frame before enqueueing it. So move this conversion of out the map functions and into xdp_do_redirect(). This removes a bit of duplicated code, but more importantly it makes it possible to support caller-allocated xdp_frame structures, which will be added in a subsequent commit. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220103150812.87914-5-toke@redhat.com --- include/linux/bpf.h | 20 ++++++++++---------- kernel/bpf/cpumap.c | 8 +------- kernel/bpf/devmap.c | 32 +++++++++++--------------------- net/core/filter.c | 24 +++++++++++++++++------- 4 files changed, 39 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 26753139d5b4..6e947cd91152 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1669,17 +1669,17 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); struct btf *bpf_get_btf_vmlinux(void); /* Map specifics */ -struct xdp_buff; +struct xdp_frame; struct sk_buff; struct bpf_dtab_netdev; struct bpf_cpu_map_entry; void __dev_flush(void); -int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, +int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx); -int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, +int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, struct net_device *dev_rx); -int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, +int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress); int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, struct bpf_prog *xdp_prog); @@ -1688,7 +1688,7 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, bool exclude_ingress); void __cpu_map_flush(void); -int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, +int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, struct net_device *dev_rx); int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, struct sk_buff *skb); @@ -1866,26 +1866,26 @@ static inline void __dev_flush(void) { } -struct xdp_buff; +struct xdp_frame; struct bpf_dtab_netdev; struct bpf_cpu_map_entry; static inline -int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, +int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; } static inline -int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, +int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; } static inline -int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, +int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress) { return 0; @@ -1913,7 +1913,7 @@ static inline void __cpu_map_flush(void) } static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, - struct xdp_buff *xdp, + struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 0421061d95f1..b3e6b9422238 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -746,15 +746,9 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) list_add(&bq->flush_node, flush_list); } -int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, +int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, struct net_device *dev_rx) { - struct xdp_frame *xdpf; - - xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) - return -EOVERFLOW; - /* Info needed when constructing SKB on remote CPU */ xdpf->dev_rx = dev_rx; diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 6feea293ff10..fe019dbdb3f0 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -467,24 +467,19 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, bq->q[bq->count++] = xdpf; } -static inline int __xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, +static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_prog *xdp_prog) { - struct xdp_frame *xdpf; int err; if (!dev->netdev_ops->ndo_xdp_xmit) return -EOPNOTSUPP; - err = xdp_ok_fwd_dev(dev, xdp->data_end - xdp->data); + err = xdp_ok_fwd_dev(dev, xdpf->len); if (unlikely(err)) return err; - xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) - return -EOVERFLOW; - bq_enqueue(dev, xdpf, dev_rx, xdp_prog); return 0; } @@ -520,27 +515,27 @@ static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev return act; } -int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, +int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx) { - return __xdp_enqueue(dev, xdp, dev_rx, NULL); + return __xdp_enqueue(dev, xdpf, dev_rx, NULL); } -int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, +int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, struct net_device *dev_rx) { struct net_device *dev = dst->dev; - return __xdp_enqueue(dev, xdp, dev_rx, dst->xdp_prog); + return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog); } -static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_buff *xdp) +static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf) { if (!obj || !obj->dev->netdev_ops->ndo_xdp_xmit) return false; - if (xdp_ok_fwd_dev(obj->dev, xdp->data_end - xdp->data)) + if (xdp_ok_fwd_dev(obj->dev, xdpf->len)) return false; return true; @@ -586,14 +581,13 @@ static int get_upper_ifindexes(struct net_device *dev, int *indexes) return n; } -int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, +int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress) { struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); struct bpf_dtab_netdev *dst, *last_dst = NULL; int excluded_devices[1+MAX_NEST_DEV]; struct hlist_head *head; - struct xdp_frame *xdpf; int num_excluded = 0; unsigned int i; int err; @@ -603,15 +597,11 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, excluded_devices[num_excluded++] = dev_rx->ifindex; } - xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) - return -EOVERFLOW; - if (map->map_type == BPF_MAP_TYPE_DEVMAP) { for (i = 0; i < map->max_entries; i++) { dst = rcu_dereference_check(dtab->netdev_map[i], rcu_read_lock_bh_held()); - if (!is_valid_dst(dst, xdp)) + if (!is_valid_dst(dst, xdpf)) continue; if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex)) @@ -634,7 +624,7 @@ int dev_map_enqueue_multi(struct xdp_buff *xdp, struct net_device *dev_rx, head = dev_map_index_hash(dtab, i); hlist_for_each_entry_rcu(dst, head, index_hlist, lockdep_is_held(&dtab->index_lock)) { - if (!is_valid_dst(dst, xdp)) + if (!is_valid_dst(dst, xdpf)) continue; if (is_ifindex_excluded(excluded_devices, num_excluded, diff --git a/net/core/filter.c b/net/core/filter.c index cac2be559ab0..e2b83056246c 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3964,12 +3964,24 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, enum bpf_map_type map_type = ri->map_type; void *fwd = ri->tgt_value; u32 map_id = ri->map_id; + struct xdp_frame *xdpf; struct bpf_map *map; int err; ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ ri->map_type = BPF_MAP_TYPE_UNSPEC; + if (map_type == BPF_MAP_TYPE_XSKMAP) { + err = __xsk_map_redirect(fwd, xdp); + goto out; + } + + xdpf = xdp_convert_buff_to_frame(xdp); + if (unlikely(!xdpf)) { + err = -EOVERFLOW; + goto err; + } + switch (map_type) { case BPF_MAP_TYPE_DEVMAP: fallthrough; @@ -3977,17 +3989,14 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, map = READ_ONCE(ri->map); if (unlikely(map)) { WRITE_ONCE(ri->map, NULL); - err = dev_map_enqueue_multi(xdp, dev, map, + err = dev_map_enqueue_multi(xdpf, dev, map, ri->flags & BPF_F_EXCLUDE_INGRESS); } else { - err = dev_map_enqueue(fwd, xdp, dev); + err = dev_map_enqueue(fwd, xdpf, dev); } break; case BPF_MAP_TYPE_CPUMAP: - err = cpu_map_enqueue(fwd, xdp, dev); - break; - case BPF_MAP_TYPE_XSKMAP: - err = __xsk_map_redirect(fwd, xdp); + err = cpu_map_enqueue(fwd, xdpf, dev); break; case BPF_MAP_TYPE_UNSPEC: if (map_id == INT_MAX) { @@ -3996,7 +4005,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, err = -EINVAL; break; } - err = dev_xdp_enqueue(fwd, xdp, dev); + err = dev_xdp_enqueue(fwd, xdpf, dev); break; } fallthrough; @@ -4004,6 +4013,7 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, err = -EBADRQC; } +out: if (unlikely(err)) goto err; -- cgit v1.2.3-59-g8ed1b From 1372d34ccf6dd480332b2bcb2fd59a2b9a0df415 Mon Sep 17 00:00:00 2001 From: Toke Høiland-Jørgensen Date: Mon, 3 Jan 2022 16:08:10 +0100 Subject: xdp: Add xdp_do_redirect_frame() for pre-computed xdp_frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an xdp_do_redirect_frame() variant which supports pre-computed xdp_frame structures. This will be used in bpf_prog_run() to avoid having to write to the xdp_frame structure when the XDP program doesn't modify the frame boundaries. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20220103150812.87914-6-toke@redhat.com --- include/linux/filter.h | 4 ++++ net/core/filter.c | 65 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 60eec80fa1d4..71fa57b88bfc 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1019,6 +1019,10 @@ int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, struct bpf_prog *prog); +int xdp_do_redirect_frame(struct net_device *dev, + struct xdp_buff *xdp, + struct xdp_frame *xdpf, + struct bpf_prog *prog); void xdp_do_flush(void); /* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as diff --git a/net/core/filter.c b/net/core/filter.c index e2b83056246c..4603b7cd3cd1 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3957,26 +3957,44 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) } EXPORT_SYMBOL_GPL(xdp_master_redirect); -int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, - struct bpf_prog *xdp_prog) +static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri, + struct net_device *dev, + struct xdp_buff *xdp, + struct bpf_prog *xdp_prog) { - struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); enum bpf_map_type map_type = ri->map_type; void *fwd = ri->tgt_value; u32 map_id = ri->map_id; - struct xdp_frame *xdpf; - struct bpf_map *map; int err; ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ ri->map_type = BPF_MAP_TYPE_UNSPEC; - if (map_type == BPF_MAP_TYPE_XSKMAP) { - err = __xsk_map_redirect(fwd, xdp); - goto out; - } + err = __xsk_map_redirect(fwd, xdp); + if (unlikely(err)) + goto err; + + _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index); + return 0; +err: + _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err); + return err; +} + +static __always_inline int __xdp_do_redirect_frame(struct bpf_redirect_info *ri, + struct net_device *dev, + struct xdp_frame *xdpf, + struct bpf_prog *xdp_prog) +{ + enum bpf_map_type map_type = ri->map_type; + void *fwd = ri->tgt_value; + u32 map_id = ri->map_id; + struct bpf_map *map; + int err; + + ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */ + ri->map_type = BPF_MAP_TYPE_UNSPEC; - xdpf = xdp_convert_buff_to_frame(xdp); if (unlikely(!xdpf)) { err = -EOVERFLOW; goto err; @@ -4013,7 +4031,6 @@ int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, err = -EBADRQC; } -out: if (unlikely(err)) goto err; @@ -4023,8 +4040,34 @@ err: _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err); return err; } + +int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp, + struct bpf_prog *xdp_prog) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + enum bpf_map_type map_type = ri->map_type; + + if (map_type == BPF_MAP_TYPE_XSKMAP) + return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog); + + return __xdp_do_redirect_frame(ri, dev, xdp_convert_buff_to_frame(xdp), + xdp_prog); +} EXPORT_SYMBOL_GPL(xdp_do_redirect); +int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp, + struct xdp_frame *xdpf, struct bpf_prog *xdp_prog) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + enum bpf_map_type map_type = ri->map_type; + + if (map_type == BPF_MAP_TYPE_XSKMAP) + return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog); + + return __xdp_do_redirect_frame(ri, dev, xdpf, xdp_prog); +} +EXPORT_SYMBOL_GPL(xdp_do_redirect_frame); + static int xdp_do_generic_redirect_map(struct net_device *dev, struct sk_buff *skb, struct xdp_buff *xdp, -- cgit v1.2.3-59-g8ed1b From ae16d059f8c9409eba0c412639def0494765b761 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Tue, 26 Oct 2021 18:22:44 +0200 Subject: mm/slub: Make object_err() static There are no callers outside of mm/slub.c anymore. Move freelist_corrupted() that calls object_err() to avoid a need for forward declaration. Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin --- include/linux/slub_def.h | 3 --- mm/slub.c | 30 +++++++++++++++--------------- 2 files changed, 15 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 0fa751b946fa..1ef68d4de9c0 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -156,9 +156,6 @@ static inline void sysfs_slab_release(struct kmem_cache *s) } #endif -void object_err(struct kmem_cache *s, struct page *page, - u8 *object, char *reason); - void *fixup_red_left(struct kmem_cache *s, void *p); static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, diff --git a/mm/slub.c b/mm/slub.c index abe7db581d68..2ccb1c71fc36 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -822,20 +822,6 @@ static void slab_fix(struct kmem_cache *s, char *fmt, ...) va_end(args); } -static bool freelist_corrupted(struct kmem_cache *s, struct page *page, - void **freelist, void *nextfree) -{ - if ((s->flags & SLAB_CONSISTENCY_CHECKS) && - !check_valid_pointer(s, page, nextfree) && freelist) { - object_err(s, page, *freelist, "Freechain corrupt"); - *freelist = NULL; - slab_fix(s, "Isolate corrupted freechain"); - return true; - } - - return false; -} - static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) { unsigned int off; /* Offset of last byte */ @@ -875,7 +861,7 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) dump_stack(); } -void object_err(struct kmem_cache *s, struct page *page, +static void object_err(struct kmem_cache *s, struct page *page, u8 *object, char *reason) { if (slab_add_kunit_errors()) @@ -886,6 +872,20 @@ void object_err(struct kmem_cache *s, struct page *page, add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); } +static bool freelist_corrupted(struct kmem_cache *s, struct page *page, + void **freelist, void *nextfree) +{ + if ((s->flags & SLAB_CONSISTENCY_CHECKS) && + !check_valid_pointer(s, page, nextfree) && freelist) { + object_err(s, page, *freelist, "Freechain corrupt"); + *freelist = NULL; + slab_fix(s, "Isolate corrupted freechain"); + return true; + } + + return false; +} + static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page, const char *fmt, ...) { -- cgit v1.2.3-59-g8ed1b From d122019bf061cccc4583eb9ad40bf58c2fe517be Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Oct 2021 14:45:51 +0100 Subject: mm: Split slab into its own type Make struct slab independent of struct page. It still uses the underlying memory in struct page for storing slab-specific data, but slab and slub can now be weaned off using struct page directly. Some of the wrapper functions (slab_address() and slab_order()) still need to cast to struct folio, but this is a significant disentanglement. [ vbabka@suse.cz: Rebase on folios, use folio instead of page where possible. Do not duplicate flags field in struct slab, instead make the related accessors go through slab_folio(). For testing pfmemalloc use the folio_*_active flag accessors directly so the PageSlabPfmemalloc wrappers can be removed later. Make folio_slab() expect only folio_test_slab() == true folios and virt_to_slab() return NULL when folio_test_slab() == false. Move struct slab to mm/slab.h. Don't represent with struct slab pages that are not true slab pages, but just a compound page obtained directly rom page allocator (with large kmalloc() for SLUB and SLOB). ] Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka Acked-by: Johannes Weiner Reviewed-by: Roman Gushchin --- include/linux/mm_types.h | 10 +-- mm/slab.h | 167 +++++++++++++++++++++++++++++++++++++++++++++++ mm/slub.c | 8 +-- 3 files changed, 176 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c3a6e6209600..1ae3537c7920 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -56,11 +56,11 @@ struct mem_cgroup; * in each subpage, but you may need to restore some of their values * afterwards. * - * SLUB uses cmpxchg_double() to atomically update its freelist and - * counters. That requires that freelist & counters be adjacent and - * double-word aligned. We align all struct pages to double-word - * boundaries, and ensure that 'freelist' is aligned within the - * struct. + * SLUB uses cmpxchg_double() to atomically update its freelist and counters. + * That requires that freelist & counters in struct slab be adjacent and + * double-word aligned. Because struct slab currently just reinterprets the + * bits of struct page, we align all struct pages to double-word boundaries, + * and ensure that 'freelist' is aligned within struct slab. */ #ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE #define _struct_page_alignment __aligned(2 * sizeof(unsigned long)) diff --git a/mm/slab.h b/mm/slab.h index 56ad7eea3ddf..0e67a8cb7f80 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -5,6 +5,173 @@ * Internal slab definitions */ +/* Reuses the bits in struct page */ +struct slab { + unsigned long __page_flags; + union { + struct list_head slab_list; + struct { /* Partial pages */ + struct slab *next; +#ifdef CONFIG_64BIT + int slabs; /* Nr of slabs left */ +#else + short int slabs; +#endif + }; + struct rcu_head rcu_head; + }; + struct kmem_cache *slab_cache; /* not slob */ + /* Double-word boundary */ + void *freelist; /* first free object */ + union { + void *s_mem; /* slab: first object */ + unsigned long counters; /* SLUB */ + struct { /* SLUB */ + unsigned inuse:16; + unsigned objects:15; + unsigned frozen:1; + }; + }; + + union { + unsigned int active; /* SLAB */ + int units; /* SLOB */ + }; + atomic_t __page_refcount; +#ifdef CONFIG_MEMCG + unsigned long memcg_data; +#endif +}; + +#define SLAB_MATCH(pg, sl) \ + static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) +SLAB_MATCH(flags, __page_flags); +SLAB_MATCH(compound_head, slab_list); /* Ensure bit 0 is clear */ +SLAB_MATCH(slab_list, slab_list); +SLAB_MATCH(rcu_head, rcu_head); +SLAB_MATCH(slab_cache, slab_cache); +SLAB_MATCH(s_mem, s_mem); +SLAB_MATCH(active, active); +SLAB_MATCH(_refcount, __page_refcount); +#ifdef CONFIG_MEMCG +SLAB_MATCH(memcg_data, memcg_data); +#endif +#undef SLAB_MATCH +static_assert(sizeof(struct slab) <= sizeof(struct page)); + +/** + * folio_slab - Converts from folio to slab. + * @folio: The folio. + * + * Currently struct slab is a different representation of a folio where + * folio_test_slab() is true. + * + * Return: The slab which contains this folio. + */ +#define folio_slab(folio) (_Generic((folio), \ + const struct folio *: (const struct slab *)(folio), \ + struct folio *: (struct slab *)(folio))) + +/** + * slab_folio - The folio allocated for a slab + * @slab: The slab. + * + * Slabs are allocated as folios that contain the individual objects and are + * using some fields in the first struct page of the folio - those fields are + * now accessed by struct slab. It is occasionally necessary to convert back to + * a folio in order to communicate with the rest of the mm. Please use this + * helper function instead of casting yourself, as the implementation may change + * in the future. + */ +#define slab_folio(s) (_Generic((s), \ + const struct slab *: (const struct folio *)s, \ + struct slab *: (struct folio *)s)) + +/** + * page_slab - Converts from first struct page to slab. + * @p: The first (either head of compound or single) page of slab. + * + * A temporary wrapper to convert struct page to struct slab in situations where + * we know the page is the compound head, or single order-0 page. + * + * Long-term ideally everything would work with struct slab directly or go + * through folio to struct slab. + * + * Return: The slab which contains this page + */ +#define page_slab(p) (_Generic((p), \ + const struct page *: (const struct slab *)(p), \ + struct page *: (struct slab *)(p))) + +/** + * slab_page - The first struct page allocated for a slab + * @slab: The slab. + * + * A convenience wrapper for converting slab to the first struct page of the + * underlying folio, to communicate with code not yet converted to folio or + * struct slab. + */ +#define slab_page(s) folio_page(slab_folio(s), 0) + +/* + * If network-based swap is enabled, sl*b must keep track of whether pages + * were allocated from pfmemalloc reserves. + */ +static inline bool slab_test_pfmemalloc(const struct slab *slab) +{ + return folio_test_active((struct folio *)slab_folio(slab)); +} + +static inline void slab_set_pfmemalloc(struct slab *slab) +{ + folio_set_active(slab_folio(slab)); +} + +static inline void slab_clear_pfmemalloc(struct slab *slab) +{ + folio_clear_active(slab_folio(slab)); +} + +static inline void __slab_clear_pfmemalloc(struct slab *slab) +{ + __folio_clear_active(slab_folio(slab)); +} + +static inline void *slab_address(const struct slab *slab) +{ + return folio_address(slab_folio(slab)); +} + +static inline int slab_nid(const struct slab *slab) +{ + return folio_nid(slab_folio(slab)); +} + +static inline pg_data_t *slab_pgdat(const struct slab *slab) +{ + return folio_pgdat(slab_folio(slab)); +} + +static inline struct slab *virt_to_slab(const void *addr) +{ + struct folio *folio = virt_to_folio(addr); + + if (!folio_test_slab(folio)) + return NULL; + + return folio_slab(folio); +} + +static inline int slab_order(const struct slab *slab) +{ + return folio_order((struct folio *)slab_folio(slab)); +} + +static inline size_t slab_size(const struct slab *slab) +{ + return PAGE_SIZE << slab_order(slab); +} + #ifdef CONFIG_SLOB /* * Common fields provided in kmem_cache by all slab allocators diff --git a/mm/slub.c b/mm/slub.c index 2ccb1c71fc36..a211d96011ba 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3787,7 +3787,7 @@ static unsigned int slub_min_objects; * requested a higher minimum order then we start with that one instead of * the smallest order which will fit the object. */ -static inline unsigned int slab_order(unsigned int size, +static inline unsigned int calc_slab_order(unsigned int size, unsigned int min_objects, unsigned int max_order, unsigned int fract_leftover) { @@ -3851,7 +3851,7 @@ static inline int calculate_order(unsigned int size) fraction = 16; while (fraction >= 4) { - order = slab_order(size, min_objects, + order = calc_slab_order(size, min_objects, slub_max_order, fraction); if (order <= slub_max_order) return order; @@ -3864,14 +3864,14 @@ static inline int calculate_order(unsigned int size) * We were unable to place multiple objects in a slab. Now * lets see if we can place a single object there. */ - order = slab_order(size, 1, slub_max_order, 1); + order = calc_slab_order(size, 1, slub_max_order, 1); if (order <= slub_max_order) return order; /* * Doh this slab cannot be placed using slub_max_order. */ - order = slab_order(size, 1, MAX_ORDER, 1); + order = calc_slab_order(size, 1, MAX_ORDER, 1); if (order < MAX_ORDER) return order; return -ENOSYS; -- cgit v1.2.3-59-g8ed1b From 0b3eb091d5759479d44cb793fad2c51ea06bdcec Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Oct 2021 14:45:56 +0100 Subject: mm: Convert check_heap_object() to use struct slab Ensure that we're not seeing a tail page inside __check_heap_object() by converting to a slab instead of a page. Take the opportunity to mark the slab as const since we're not modifying it. Also move the declaration of __check_heap_object() to mm/slab.h so it's not available to the wider kernel. [ vbabka@suse.cz: in check_heap_object() only convert to struct slab for actual PageSlab pages; use folio as intermediate step instead of page ] Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin --- include/linux/slab.h | 8 -------- mm/slab.c | 14 +++++++------- mm/slab.h | 11 +++++++++++ mm/slub.c | 10 +++++----- mm/usercopy.c | 13 +++++++------ 5 files changed, 30 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 181045148b06..367366f1d1ff 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -189,14 +189,6 @@ bool kmem_valid_obj(void *object); void kmem_dump_obj(void *object); #endif -#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR -void __check_heap_object(const void *ptr, unsigned long n, struct page *page, - bool to_user); -#else -static inline void __check_heap_object(const void *ptr, unsigned long n, - struct page *page, bool to_user) { } -#endif - /* * Some archs want to perform DMA into kmalloc caches and need a guaranteed * alignment larger than the alignment of a 64-bit integer. diff --git a/mm/slab.c b/mm/slab.c index 44bc1fcd1393..38fcd3f496df 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -372,8 +372,8 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp) static int slab_max_order = SLAB_MAX_ORDER_LO; static bool slab_max_order_set __initdata; -static inline void *index_to_obj(struct kmem_cache *cache, struct page *page, - unsigned int idx) +static inline void *index_to_obj(struct kmem_cache *cache, + const struct page *page, unsigned int idx) { return page->s_mem + cache->size * idx; } @@ -4166,8 +4166,8 @@ ssize_t slabinfo_write(struct file *file, const char __user *buffer, * Returns NULL if check passes, otherwise const char * to name of cache * to indicate an error. */ -void __check_heap_object(const void *ptr, unsigned long n, struct page *page, - bool to_user) +void __check_heap_object(const void *ptr, unsigned long n, + const struct slab *slab, bool to_user) { struct kmem_cache *cachep; unsigned int objnr; @@ -4176,15 +4176,15 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page, ptr = kasan_reset_tag(ptr); /* Find and validate object. */ - cachep = page->slab_cache; - objnr = obj_to_index(cachep, page, (void *)ptr); + cachep = slab->slab_cache; + objnr = obj_to_index(cachep, slab_page(slab), (void *)ptr); BUG_ON(objnr >= cachep->num); /* Find offset within object. */ if (is_kfence_address(ptr)) offset = ptr - kfence_object_start(ptr); else - offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep); + offset = ptr - index_to_obj(cachep, slab_page(slab), objnr) - obj_offset(cachep); /* Allow address range falling entirely within usercopy region. */ if (offset >= cachep->useroffset && diff --git a/mm/slab.h b/mm/slab.h index 9ae9f6c3d1cb..039babfde2fe 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -812,4 +812,15 @@ struct kmem_obj_info { void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab); #endif +#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR +void __check_heap_object(const void *ptr, unsigned long n, + const struct slab *slab, bool to_user); +#else +static inline +void __check_heap_object(const void *ptr, unsigned long n, + const struct slab *slab, bool to_user) +{ +} +#endif + #endif /* MM_SLAB_H */ diff --git a/mm/slub.c b/mm/slub.c index 8e9667815f81..8b82188849ae 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4485,8 +4485,8 @@ EXPORT_SYMBOL(__kmalloc_node); * Returns NULL if check passes, otherwise const char * to name of cache * to indicate an error. */ -void __check_heap_object(const void *ptr, unsigned long n, struct page *page, - bool to_user) +void __check_heap_object(const void *ptr, unsigned long n, + const struct slab *slab, bool to_user) { struct kmem_cache *s; unsigned int offset; @@ -4495,10 +4495,10 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page, ptr = kasan_reset_tag(ptr); /* Find object and usable object size. */ - s = page->slab_cache; + s = slab->slab_cache; /* Reject impossible pointers. */ - if (ptr < page_address(page)) + if (ptr < slab_address(slab)) usercopy_abort("SLUB object not in SLUB page?!", NULL, to_user, 0, n); @@ -4506,7 +4506,7 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page, if (is_kfence) offset = ptr - kfence_object_start(ptr); else - offset = (ptr - page_address(page)) % s->size; + offset = (ptr - slab_address(slab)) % s->size; /* Adjust for redzone and reject if within the redzone. */ if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) { diff --git a/mm/usercopy.c b/mm/usercopy.c index b3de3c4eefba..d0d268135d96 100644 --- a/mm/usercopy.c +++ b/mm/usercopy.c @@ -20,6 +20,7 @@ #include #include #include +#include "slab.h" /* * Checks if a given pointer and length is contained by the current @@ -223,7 +224,7 @@ static inline void check_page_span(const void *ptr, unsigned long n, static inline void check_heap_object(const void *ptr, unsigned long n, bool to_user) { - struct page *page; + struct folio *folio; if (!virt_addr_valid(ptr)) return; @@ -231,16 +232,16 @@ static inline void check_heap_object(const void *ptr, unsigned long n, /* * When CONFIG_HIGHMEM=y, kmap_to_page() will give either the * highmem page or fallback to virt_to_page(). The following - * is effectively a highmem-aware virt_to_head_page(). + * is effectively a highmem-aware virt_to_slab(). */ - page = compound_head(kmap_to_page((void *)ptr)); + folio = page_folio(kmap_to_page((void *)ptr)); - if (PageSlab(page)) { + if (folio_test_slab(folio)) { /* Check slab allocator for flags and size. */ - __check_heap_object(ptr, n, page, to_user); + __check_heap_object(ptr, n, folio_slab(folio), to_user); } else { /* Verify object does not incorrectly span multiple pages. */ - check_page_span(ptr, n, page, to_user); + check_page_span(ptr, n, folio_page(folio, 0), to_user); } } -- cgit v1.2.3-59-g8ed1b From bb192ed9aa7191a5d65548f82c42b6750d65f569 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 3 Nov 2021 15:39:59 +0100 Subject: mm/slub: Convert most struct page to struct slab by spatch The majority of conversion from struct page to struct slab in SLUB internals can be delegated to a coccinelle semantic patch. This includes renaming of variables with 'page' in name to 'slab', and similar. Big thanks to Julia Lawall and Luis Chamberlain for help with coccinelle. // Options: --include-headers --no-includes --smpl-spacing include/linux/slub_def.h mm/slub.c // Note: needs coccinelle 1.1.1 to avoid breaking whitespace, and ocaml for the // embedded script // build list of functions to exclude from applying the next rule @initialize:ocaml@ @@ let ok_function p = not (List.mem (List.hd p).current_element ["nearest_obj";"obj_to_index";"objs_per_slab_page";"__slab_lock";"__slab_unlock";"free_nonslab_page";"kmalloc_large_node"]) // convert the type from struct page to struct page in all functions except the // list from previous rule // this also affects struct kmem_cache_cpu, but that's ok @@ position p : script:ocaml() { ok_function p }; @@ - struct page@p + struct slab // in struct kmem_cache_cpu, change the name from page to slab // the type was already converted by the previous rule @@ @@ struct kmem_cache_cpu { ... -struct slab *page; +struct slab *slab; ... } // there are many places that use c->page which is now c->slab after the // previous rule @@ struct kmem_cache_cpu *c; @@ -c->page +c->slab @@ @@ struct kmem_cache { ... - unsigned int cpu_partial_pages; + unsigned int cpu_partial_slabs; ... } @@ struct kmem_cache *s; @@ - s->cpu_partial_pages + s->cpu_partial_slabs @@ @@ static void - setup_page_debug( + setup_slab_debug( ...) {...} @@ @@ - setup_page_debug( + setup_slab_debug( ...); // for all functions (with exceptions), change any "struct slab *page" // parameter to "struct slab *slab" in the signature, and generally all // occurences of "page" to "slab" in the body - with some special cases. @@ identifier fn !~ "free_nonslab_page|obj_to_index|objs_per_slab_page|nearest_obj"; @@ fn(..., - struct slab *page + struct slab *slab ,...) { <... - page + slab ...> } // similar to previous but the param is called partial_page @@ identifier fn; @@ fn(..., - struct slab *partial_page + struct slab *partial_slab ,...) { <... - partial_page + partial_slab ...> } // similar to previous but for functions that take pointer to struct page ptr @@ identifier fn; @@ fn(..., - struct slab **ret_page + struct slab **ret_slab ,...) { <... - ret_page + ret_slab ...> } // functions converted by previous rules that were temporarily called using // slab_page(E) so we want to remove the wrapper now that they accept struct // slab ptr directly @@ identifier fn =~ "slab_free|do_slab_free"; expression E; @@ fn(..., - slab_page(E) + E ,...) // similar to previous but for another pattern @@ identifier fn =~ "slab_pad_check|check_object"; @@ fn(..., - folio_page(folio, 0) + slab ,...) // functions that were returning struct page ptr and now will return struct // slab ptr, including slab_page() wrapper removal @@ identifier fn =~ "allocate_slab|new_slab"; expression E; @@ static -struct slab * +struct slab * fn(...) { <... - slab_page(E) + E ...> } // rename any former struct page * declarations @@ @@ struct slab * ( - page + slab | - partial_page + partial_slab | - oldpage + oldslab ) ; // this has to be separate from previous rule as page and page2 appear at the // same line @@ @@ struct slab * -page2 +slab2 ; // similar but with initial assignment @@ expression E; @@ struct slab * ( - page + slab | - flush_page + flush_slab | - discard_page + slab_to_discard | - page_to_unfreeze + slab_to_unfreeze ) = E; // convert most of struct page to struct slab usage inside functions (with // exceptions), including specific variable renames @@ identifier fn !~ "nearest_obj|obj_to_index|objs_per_slab_page|__slab_(un)*lock|__free_slab|free_nonslab_page|kmalloc_large_node"; expression E; @@ fn(...) { <... ( - int pages; + int slabs; | - int pages = E; + int slabs = E; | - page + slab | - flush_page + flush_slab | - partial_page + partial_slab | - oldpage->pages + oldslab->slabs | - oldpage + oldslab | - unsigned int nr_pages; + unsigned int nr_slabs; | - nr_pages + nr_slabs | - unsigned int partial_pages = E; + unsigned int partial_slabs = E; | - partial_pages + partial_slabs ) ...> } // this has to be split out from the previous rule so that lines containing // multiple matching changes will be fully converted @@ identifier fn !~ "nearest_obj|obj_to_index|objs_per_slab_page|__slab_(un)*lock|__free_slab|free_nonslab_page|kmalloc_large_node"; @@ fn(...) { <... ( - slab->pages + slab->slabs | - pages + slabs | - page2 + slab2 | - discard_page + slab_to_discard | - page_to_unfreeze + slab_to_unfreeze ) ...> } // after we simply changed all occurences of page to slab, some usages need // adjustment for slab-specific functions, or use slab_page() wrapper @@ identifier fn !~ "nearest_obj|obj_to_index|objs_per_slab_page|__slab_(un)*lock|__free_slab|free_nonslab_page|kmalloc_large_node"; @@ fn(...) { <... ( - page_slab(slab) + slab | - kasan_poison_slab(slab) + kasan_poison_slab(slab_page(slab)) | - page_address(slab) + slab_address(slab) | - page_size(slab) + slab_size(slab) | - PageSlab(slab) + folio_test_slab(slab_folio(slab)) | - page_to_nid(slab) + slab_nid(slab) | - compound_order(slab) + slab_order(slab) ) ...> } Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Julia Lawall Cc: Luis Chamberlain --- include/linux/slub_def.h | 6 +- mm/slub.c | 872 +++++++++++++++++++++++------------------------ 2 files changed, 439 insertions(+), 439 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 1ef68d4de9c0..00d99afe1c0e 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -48,9 +48,9 @@ enum stat_item { struct kmem_cache_cpu { void **freelist; /* Pointer to next available object */ unsigned long tid; /* Globally unique transaction id */ - struct page *page; /* The slab from which we are allocating */ + struct slab *slab; /* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL - struct page *partial; /* Partially allocated frozen slabs */ + struct slab *partial; /* Partially allocated frozen slabs */ #endif local_lock_t lock; /* Protects the fields above */ #ifdef CONFIG_SLUB_STATS @@ -100,7 +100,7 @@ struct kmem_cache { /* Number of per cpu partial objects to keep around */ unsigned int cpu_partial; /* Number of per cpu partial pages to keep around */ - unsigned int cpu_partial_pages; + unsigned int cpu_partial_slabs; #endif struct kmem_cache_order_objects oo; diff --git a/mm/slub.c b/mm/slub.c index c369806084e1..e89208f3197a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -417,7 +417,7 @@ static inline unsigned int oo_objects(struct kmem_cache_order_objects x) #ifdef CONFIG_SLUB_CPU_PARTIAL static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) { - unsigned int nr_pages; + unsigned int nr_slabs; s->cpu_partial = nr_objects; @@ -427,8 +427,8 @@ static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) * growth of the list. For simplicity we assume that the pages will * be half-full. */ - nr_pages = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); - s->cpu_partial_pages = nr_pages; + nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); + s->cpu_partial_slabs = nr_slabs; } #else static inline void @@ -456,16 +456,16 @@ static __always_inline void __slab_unlock(struct slab *slab) __bit_spin_unlock(PG_locked, &page->flags); } -static __always_inline void slab_lock(struct page *page, unsigned long *flags) +static __always_inline void slab_lock(struct slab *slab, unsigned long *flags) { if (IS_ENABLED(CONFIG_PREEMPT_RT)) local_irq_save(*flags); - __slab_lock(page_slab(page)); + __slab_lock(slab); } -static __always_inline void slab_unlock(struct page *page, unsigned long *flags) +static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags) { - __slab_unlock(page_slab(page)); + __slab_unlock(slab); if (IS_ENABLED(CONFIG_PREEMPT_RT)) local_irq_restore(*flags); } @@ -475,7 +475,7 @@ static __always_inline void slab_unlock(struct page *page, unsigned long *flags) * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different * so we disable interrupts as part of slab_[un]lock(). */ -static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, +static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab, void *freelist_old, unsigned long counters_old, void *freelist_new, unsigned long counters_new, const char *n) @@ -485,7 +485,7 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) if (s->flags & __CMPXCHG_DOUBLE) { - if (cmpxchg_double(&page->freelist, &page->counters, + if (cmpxchg_double(&slab->freelist, &slab->counters, freelist_old, counters_old, freelist_new, counters_new)) return true; @@ -495,15 +495,15 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page /* init to 0 to prevent spurious warnings */ unsigned long flags = 0; - slab_lock(page, &flags); - if (page->freelist == freelist_old && - page->counters == counters_old) { - page->freelist = freelist_new; - page->counters = counters_new; - slab_unlock(page, &flags); + slab_lock(slab, &flags); + if (slab->freelist == freelist_old && + slab->counters == counters_old) { + slab->freelist = freelist_new; + slab->counters = counters_new; + slab_unlock(slab, &flags); return true; } - slab_unlock(page, &flags); + slab_unlock(slab, &flags); } cpu_relax(); @@ -516,7 +516,7 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page return false; } -static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, +static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab, void *freelist_old, unsigned long counters_old, void *freelist_new, unsigned long counters_new, const char *n) @@ -524,7 +524,7 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) if (s->flags & __CMPXCHG_DOUBLE) { - if (cmpxchg_double(&page->freelist, &page->counters, + if (cmpxchg_double(&slab->freelist, &slab->counters, freelist_old, counters_old, freelist_new, counters_new)) return true; @@ -534,16 +534,16 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, unsigned long flags; local_irq_save(flags); - __slab_lock(page_slab(page)); - if (page->freelist == freelist_old && - page->counters == counters_old) { - page->freelist = freelist_new; - page->counters = counters_new; - __slab_unlock(page_slab(page)); + __slab_lock(slab); + if (slab->freelist == freelist_old && + slab->counters == counters_old) { + slab->freelist = freelist_new; + slab->counters = counters_new; + __slab_unlock(slab); local_irq_restore(flags); return true; } - __slab_unlock(page_slab(page)); + __slab_unlock(slab); local_irq_restore(flags); } @@ -562,14 +562,14 @@ static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; static DEFINE_RAW_SPINLOCK(object_map_lock); static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, - struct page *page) + struct slab *slab) { - void *addr = page_address(page); + void *addr = slab_address(slab); void *p; - bitmap_zero(obj_map, page->objects); + bitmap_zero(obj_map, slab->objects); - for (p = page->freelist; p; p = get_freepointer(s, p)) + for (p = slab->freelist; p; p = get_freepointer(s, p)) set_bit(__obj_to_index(s, addr, p), obj_map); } @@ -599,14 +599,14 @@ static inline bool slab_add_kunit_errors(void) { return false; } * Node listlock must be held to guarantee that the page does * not vanish from under us. */ -static unsigned long *get_map(struct kmem_cache *s, struct page *page) +static unsigned long *get_map(struct kmem_cache *s, struct slab *slab) __acquires(&object_map_lock) { VM_BUG_ON(!irqs_disabled()); raw_spin_lock(&object_map_lock); - __fill_map(object_map, s, page); + __fill_map(object_map, s, slab); return object_map; } @@ -667,17 +667,17 @@ static inline void metadata_access_disable(void) /* Verify that a pointer has an address that is valid within a slab page */ static inline int check_valid_pointer(struct kmem_cache *s, - struct page *page, void *object) + struct slab *slab, void *object) { void *base; if (!object) return 1; - base = page_address(page); + base = slab_address(slab); object = kasan_reset_tag(object); object = restore_red_left(s, object); - if (object < base || object >= base + page->objects * s->size || + if (object < base || object >= base + slab->objects * s->size || (object - base) % s->size) { return 0; } @@ -827,14 +827,14 @@ static void slab_fix(struct kmem_cache *s, char *fmt, ...) va_end(args); } -static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) +static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) { unsigned int off; /* Offset of last byte */ - u8 *addr = page_address(page); + u8 *addr = slab_address(slab); print_tracking(s, p); - print_slab_info(page_slab(page)); + print_slab_info(slab); pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n", p, p - addr, get_freepointer(s, p)); @@ -866,23 +866,23 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) dump_stack(); } -static void object_err(struct kmem_cache *s, struct page *page, +static void object_err(struct kmem_cache *s, struct slab *slab, u8 *object, char *reason) { if (slab_add_kunit_errors()) return; slab_bug(s, "%s", reason); - print_trailer(s, page, object); + print_trailer(s, slab, object); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); } -static bool freelist_corrupted(struct kmem_cache *s, struct page *page, +static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, void **freelist, void *nextfree) { if ((s->flags & SLAB_CONSISTENCY_CHECKS) && - !check_valid_pointer(s, page, nextfree) && freelist) { - object_err(s, page, *freelist, "Freechain corrupt"); + !check_valid_pointer(s, slab, nextfree) && freelist) { + object_err(s, slab, *freelist, "Freechain corrupt"); *freelist = NULL; slab_fix(s, "Isolate corrupted freechain"); return true; @@ -891,7 +891,7 @@ static bool freelist_corrupted(struct kmem_cache *s, struct page *page, return false; } -static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page, +static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, const char *fmt, ...) { va_list args; @@ -904,7 +904,7 @@ static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page, vsnprintf(buf, sizeof(buf), fmt, args); va_end(args); slab_bug(s, "%s", buf); - print_slab_info(page_slab(page)); + print_slab_info(slab); dump_stack(); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); } @@ -932,13 +932,13 @@ static void restore_bytes(struct kmem_cache *s, char *message, u8 data, memset(from, data, to - from); } -static int check_bytes_and_report(struct kmem_cache *s, struct page *page, +static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab, u8 *object, char *what, u8 *start, unsigned int value, unsigned int bytes) { u8 *fault; u8 *end; - u8 *addr = page_address(page); + u8 *addr = slab_address(slab); metadata_access_enable(); fault = memchr_inv(kasan_reset_tag(start), value, bytes); @@ -957,7 +957,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", fault, end - 1, fault - addr, fault[0], value); - print_trailer(s, page, object); + print_trailer(s, slab, object); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); skip_bug_print: @@ -1003,7 +1003,7 @@ skip_bug_print: * may be used with merged slabcaches. */ -static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) +static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) { unsigned long off = get_info_end(s); /* The end of info */ @@ -1016,12 +1016,12 @@ static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) if (size_from_object(s) == off) return 1; - return check_bytes_and_report(s, page, p, "Object padding", + return check_bytes_and_report(s, slab, p, "Object padding", p + off, POISON_INUSE, size_from_object(s) - off); } /* Check the pad bytes at the end of a slab page */ -static int slab_pad_check(struct kmem_cache *s, struct page *page) +static int slab_pad_check(struct kmem_cache *s, struct slab *slab) { u8 *start; u8 *fault; @@ -1033,8 +1033,8 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) if (!(s->flags & SLAB_POISON)) return 1; - start = page_address(page); - length = page_size(page); + start = slab_address(slab); + length = slab_size(slab); end = start + length; remainder = length % s->size; if (!remainder) @@ -1049,7 +1049,7 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) while (end > fault && end[-1] == POISON_INUSE) end--; - slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu", + slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu", fault, end - 1, fault - start); print_section(KERN_ERR, "Padding ", pad, remainder); @@ -1057,23 +1057,23 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) return 0; } -static int check_object(struct kmem_cache *s, struct page *page, +static int check_object(struct kmem_cache *s, struct slab *slab, void *object, u8 val) { u8 *p = object; u8 *endobject = object + s->object_size; if (s->flags & SLAB_RED_ZONE) { - if (!check_bytes_and_report(s, page, object, "Left Redzone", + if (!check_bytes_and_report(s, slab, object, "Left Redzone", object - s->red_left_pad, val, s->red_left_pad)) return 0; - if (!check_bytes_and_report(s, page, object, "Right Redzone", + if (!check_bytes_and_report(s, slab, object, "Right Redzone", endobject, val, s->inuse - s->object_size)) return 0; } else { if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { - check_bytes_and_report(s, page, p, "Alignment padding", + check_bytes_and_report(s, slab, p, "Alignment padding", endobject, POISON_INUSE, s->inuse - s->object_size); } @@ -1081,15 +1081,15 @@ static int check_object(struct kmem_cache *s, struct page *page, if (s->flags & SLAB_POISON) { if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && - (!check_bytes_and_report(s, page, p, "Poison", p, + (!check_bytes_and_report(s, slab, p, "Poison", p, POISON_FREE, s->object_size - 1) || - !check_bytes_and_report(s, page, p, "End Poison", + !check_bytes_and_report(s, slab, p, "End Poison", p + s->object_size - 1, POISON_END, 1))) return 0; /* * check_pad_bytes cleans up on its own. */ - check_pad_bytes(s, page, p); + check_pad_bytes(s, slab, p); } if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE) @@ -1100,8 +1100,8 @@ static int check_object(struct kmem_cache *s, struct page *page, return 1; /* Check free pointer validity */ - if (!check_valid_pointer(s, page, get_freepointer(s, p))) { - object_err(s, page, p, "Freepointer corrupt"); + if (!check_valid_pointer(s, slab, get_freepointer(s, p))) { + object_err(s, slab, p, "Freepointer corrupt"); /* * No choice but to zap it and thus lose the remainder * of the free objects in this slab. May cause @@ -1113,28 +1113,28 @@ static int check_object(struct kmem_cache *s, struct page *page, return 1; } -static int check_slab(struct kmem_cache *s, struct page *page) +static int check_slab(struct kmem_cache *s, struct slab *slab) { int maxobj; - if (!PageSlab(page)) { - slab_err(s, page, "Not a valid slab page"); + if (!folio_test_slab(slab_folio(slab))) { + slab_err(s, slab, "Not a valid slab page"); return 0; } - maxobj = order_objects(compound_order(page), s->size); - if (page->objects > maxobj) { - slab_err(s, page, "objects %u > max %u", - page->objects, maxobj); + maxobj = order_objects(slab_order(slab), s->size); + if (slab->objects > maxobj) { + slab_err(s, slab, "objects %u > max %u", + slab->objects, maxobj); return 0; } - if (page->inuse > page->objects) { - slab_err(s, page, "inuse %u > max %u", - page->inuse, page->objects); + if (slab->inuse > slab->objects) { + slab_err(s, slab, "inuse %u > max %u", + slab->inuse, slab->objects); return 0; } /* Slab_pad_check fixes things up after itself */ - slab_pad_check(s, page); + slab_pad_check(s, slab); return 1; } @@ -1142,26 +1142,26 @@ static int check_slab(struct kmem_cache *s, struct page *page) * Determine if a certain object on a page is on the freelist. Must hold the * slab lock to guarantee that the chains are in a consistent state. */ -static int on_freelist(struct kmem_cache *s, struct page *page, void *search) +static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search) { int nr = 0; void *fp; void *object = NULL; int max_objects; - fp = page->freelist; - while (fp && nr <= page->objects) { + fp = slab->freelist; + while (fp && nr <= slab->objects) { if (fp == search) return 1; - if (!check_valid_pointer(s, page, fp)) { + if (!check_valid_pointer(s, slab, fp)) { if (object) { - object_err(s, page, object, + object_err(s, slab, object, "Freechain corrupt"); set_freepointer(s, object, NULL); } else { - slab_err(s, page, "Freepointer corrupt"); - page->freelist = NULL; - page->inuse = page->objects; + slab_err(s, slab, "Freepointer corrupt"); + slab->freelist = NULL; + slab->inuse = slab->objects; slab_fix(s, "Freelist cleared"); return 0; } @@ -1172,34 +1172,34 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search) nr++; } - max_objects = order_objects(compound_order(page), s->size); + max_objects = order_objects(slab_order(slab), s->size); if (max_objects > MAX_OBJS_PER_PAGE) max_objects = MAX_OBJS_PER_PAGE; - if (page->objects != max_objects) { - slab_err(s, page, "Wrong number of objects. Found %d but should be %d", - page->objects, max_objects); - page->objects = max_objects; + if (slab->objects != max_objects) { + slab_err(s, slab, "Wrong number of objects. Found %d but should be %d", + slab->objects, max_objects); + slab->objects = max_objects; slab_fix(s, "Number of objects adjusted"); } - if (page->inuse != page->objects - nr) { - slab_err(s, page, "Wrong object count. Counter is %d but counted were %d", - page->inuse, page->objects - nr); - page->inuse = page->objects - nr; + if (slab->inuse != slab->objects - nr) { + slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d", + slab->inuse, slab->objects - nr); + slab->inuse = slab->objects - nr; slab_fix(s, "Object count adjusted"); } return search == NULL; } -static void trace(struct kmem_cache *s, struct page *page, void *object, +static void trace(struct kmem_cache *s, struct slab *slab, void *object, int alloc) { if (s->flags & SLAB_TRACE) { pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", s->name, alloc ? "alloc" : "free", - object, page->inuse, - page->freelist); + object, slab->inuse, + slab->freelist); if (!alloc) print_section(KERN_INFO, "Object ", (void *)object, @@ -1213,22 +1213,22 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, * Tracking of fully allocated slabs for debugging purposes. */ static void add_full(struct kmem_cache *s, - struct kmem_cache_node *n, struct page *page) + struct kmem_cache_node *n, struct slab *slab) { if (!(s->flags & SLAB_STORE_USER)) return; lockdep_assert_held(&n->list_lock); - list_add(&page->slab_list, &n->full); + list_add(&slab->slab_list, &n->full); } -static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) +static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) { if (!(s->flags & SLAB_STORE_USER)) return; lockdep_assert_held(&n->list_lock); - list_del(&page->slab_list); + list_del(&slab->slab_list); } /* Tracking of the number of slabs for debugging purposes */ @@ -1268,7 +1268,7 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) } /* Object debug checks for alloc/free paths */ -static void setup_object_debug(struct kmem_cache *s, struct page *page, +static void setup_object_debug(struct kmem_cache *s, struct slab *slab, void *object) { if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)) @@ -1279,89 +1279,89 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page, } static -void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) +void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) { if (!kmem_cache_debug_flags(s, SLAB_POISON)) return; metadata_access_enable(); - memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page)); + memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); metadata_access_disable(); } static inline int alloc_consistency_checks(struct kmem_cache *s, - struct page *page, void *object) + struct slab *slab, void *object) { - if (!check_slab(s, page)) + if (!check_slab(s, slab)) return 0; - if (!check_valid_pointer(s, page, object)) { - object_err(s, page, object, "Freelist Pointer check fails"); + if (!check_valid_pointer(s, slab, object)) { + object_err(s, slab, object, "Freelist Pointer check fails"); return 0; } - if (!check_object(s, page, object, SLUB_RED_INACTIVE)) + if (!check_object(s, slab, object, SLUB_RED_INACTIVE)) return 0; return 1; } static noinline int alloc_debug_processing(struct kmem_cache *s, - struct page *page, + struct slab *slab, void *object, unsigned long addr) { if (s->flags & SLAB_CONSISTENCY_CHECKS) { - if (!alloc_consistency_checks(s, page, object)) + if (!alloc_consistency_checks(s, slab, object)) goto bad; } /* Success perform special debug activities for allocs */ if (s->flags & SLAB_STORE_USER) set_track(s, object, TRACK_ALLOC, addr); - trace(s, page, object, 1); + trace(s, slab, object, 1); init_object(s, object, SLUB_RED_ACTIVE); return 1; bad: - if (PageSlab(page)) { + if (folio_test_slab(slab_folio(slab))) { /* * If this is a slab page then lets do the best we can * to avoid issues in the future. Marking all objects * as used avoids touching the remaining objects. */ slab_fix(s, "Marking all objects used"); - page->inuse = page->objects; - page->freelist = NULL; + slab->inuse = slab->objects; + slab->freelist = NULL; } return 0; } static inline int free_consistency_checks(struct kmem_cache *s, - struct page *page, void *object, unsigned long addr) + struct slab *slab, void *object, unsigned long addr) { - if (!check_valid_pointer(s, page, object)) { - slab_err(s, page, "Invalid object pointer 0x%p", object); + if (!check_valid_pointer(s, slab, object)) { + slab_err(s, slab, "Invalid object pointer 0x%p", object); return 0; } - if (on_freelist(s, page, object)) { - object_err(s, page, object, "Object already free"); + if (on_freelist(s, slab, object)) { + object_err(s, slab, object, "Object already free"); return 0; } - if (!check_object(s, page, object, SLUB_RED_ACTIVE)) + if (!check_object(s, slab, object, SLUB_RED_ACTIVE)) return 0; - if (unlikely(s != page->slab_cache)) { - if (!PageSlab(page)) { - slab_err(s, page, "Attempt to free object(0x%p) outside of slab", + if (unlikely(s != slab->slab_cache)) { + if (!folio_test_slab(slab_folio(slab))) { + slab_err(s, slab, "Attempt to free object(0x%p) outside of slab", object); - } else if (!page->slab_cache) { + } else if (!slab->slab_cache) { pr_err("SLUB : no slab for object 0x%p.\n", object); dump_stack(); } else - object_err(s, page, object, + object_err(s, slab, object, "page slab pointer corrupt."); return 0; } @@ -1370,21 +1370,21 @@ static inline int free_consistency_checks(struct kmem_cache *s, /* Supports checking bulk free of a constructed freelist */ static noinline int free_debug_processing( - struct kmem_cache *s, struct page *page, + struct kmem_cache *s, struct slab *slab, void *head, void *tail, int bulk_cnt, unsigned long addr) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct kmem_cache_node *n = get_node(s, slab_nid(slab)); void *object = head; int cnt = 0; unsigned long flags, flags2; int ret = 0; spin_lock_irqsave(&n->list_lock, flags); - slab_lock(page, &flags2); + slab_lock(slab, &flags2); if (s->flags & SLAB_CONSISTENCY_CHECKS) { - if (!check_slab(s, page)) + if (!check_slab(s, slab)) goto out; } @@ -1392,13 +1392,13 @@ next_object: cnt++; if (s->flags & SLAB_CONSISTENCY_CHECKS) { - if (!free_consistency_checks(s, page, object, addr)) + if (!free_consistency_checks(s, slab, object, addr)) goto out; } if (s->flags & SLAB_STORE_USER) set_track(s, object, TRACK_FREE, addr); - trace(s, page, object, 0); + trace(s, slab, object, 0); /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ init_object(s, object, SLUB_RED_INACTIVE); @@ -1411,10 +1411,10 @@ next_object: out: if (cnt != bulk_cnt) - slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n", + slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n", bulk_cnt, cnt); - slab_unlock(page, &flags2); + slab_unlock(slab, &flags2); spin_unlock_irqrestore(&n->list_lock, flags); if (!ret) slab_fix(s, "Object at 0x%p not freed", object); @@ -1629,26 +1629,26 @@ slab_flags_t kmem_cache_flags(unsigned int object_size, } #else /* !CONFIG_SLUB_DEBUG */ static inline void setup_object_debug(struct kmem_cache *s, - struct page *page, void *object) {} + struct slab *slab, void *object) {} static inline -void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {} +void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} static inline int alloc_debug_processing(struct kmem_cache *s, - struct page *page, void *object, unsigned long addr) { return 0; } + struct slab *slab, void *object, unsigned long addr) { return 0; } static inline int free_debug_processing( - struct kmem_cache *s, struct page *page, + struct kmem_cache *s, struct slab *slab, void *head, void *tail, int bulk_cnt, unsigned long addr) { return 0; } -static inline int slab_pad_check(struct kmem_cache *s, struct page *page) +static inline int slab_pad_check(struct kmem_cache *s, struct slab *slab) { return 1; } -static inline int check_object(struct kmem_cache *s, struct page *page, +static inline int check_object(struct kmem_cache *s, struct slab *slab, void *object, u8 val) { return 1; } static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, - struct page *page) {} + struct slab *slab) {} static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, - struct page *page) {} + struct slab *slab) {} slab_flags_t kmem_cache_flags(unsigned int object_size, slab_flags_t flags, const char *name) { @@ -1667,7 +1667,7 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) {} -static bool freelist_corrupted(struct kmem_cache *s, struct page *page, +static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, void **freelist, void *nextfree) { return false; @@ -1772,10 +1772,10 @@ static inline bool slab_free_freelist_hook(struct kmem_cache *s, return *head != NULL; } -static void *setup_object(struct kmem_cache *s, struct page *page, +static void *setup_object(struct kmem_cache *s, struct slab *slab, void *object) { - setup_object_debug(s, page, object); + setup_object_debug(s, slab, object); object = kasan_init_slab_obj(s, object); if (unlikely(s->ctor)) { kasan_unpoison_object_data(s, object); @@ -1853,7 +1853,7 @@ static void __init init_freelist_randomization(void) } /* Get the next entry on the pre-computed freelist randomized */ -static void *next_freelist_entry(struct kmem_cache *s, struct page *page, +static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab, unsigned long *pos, void *start, unsigned long page_limit, unsigned long freelist_count) @@ -1875,32 +1875,32 @@ static void *next_freelist_entry(struct kmem_cache *s, struct page *page, } /* Shuffle the single linked freelist based on a random pre-computed sequence */ -static bool shuffle_freelist(struct kmem_cache *s, struct page *page) +static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) { void *start; void *cur; void *next; unsigned long idx, pos, page_limit, freelist_count; - if (page->objects < 2 || !s->random_seq) + if (slab->objects < 2 || !s->random_seq) return false; freelist_count = oo_objects(s->oo); pos = get_random_int() % freelist_count; - page_limit = page->objects * s->size; - start = fixup_red_left(s, page_address(page)); + page_limit = slab->objects * s->size; + start = fixup_red_left(s, slab_address(slab)); /* First entry is used as the base of the freelist */ - cur = next_freelist_entry(s, page, &pos, start, page_limit, + cur = next_freelist_entry(s, slab, &pos, start, page_limit, freelist_count); - cur = setup_object(s, page, cur); - page->freelist = cur; + cur = setup_object(s, slab, cur); + slab->freelist = cur; - for (idx = 1; idx < page->objects; idx++) { - next = next_freelist_entry(s, page, &pos, start, page_limit, + for (idx = 1; idx < slab->objects; idx++) { + next = next_freelist_entry(s, slab, &pos, start, page_limit, freelist_count); - next = setup_object(s, page, next); + next = setup_object(s, slab, next); set_freepointer(s, cur, next); cur = next; } @@ -1914,15 +1914,15 @@ static inline int init_cache_random_seq(struct kmem_cache *s) return 0; } static inline void init_freelist_randomization(void) { } -static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page) +static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) { return false; } #endif /* CONFIG_SLAB_FREELIST_RANDOM */ -static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) { - struct page *page; + struct slab *slab; struct kmem_cache_order_objects oo = s->oo; gfp_t alloc_gfp; void *start, *p, *next; @@ -1941,60 +1941,60 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL); - page = slab_page(alloc_slab_page(s, alloc_gfp, node, oo)); - if (unlikely(!page)) { + slab = alloc_slab_page(s, alloc_gfp, node, oo); + if (unlikely(!slab)) { oo = s->min; alloc_gfp = flags; /* * Allocation may have failed due to fragmentation. * Try a lower order alloc if possible */ - page = slab_page(alloc_slab_page(s, alloc_gfp, node, oo)); - if (unlikely(!page)) + slab = alloc_slab_page(s, alloc_gfp, node, oo); + if (unlikely(!slab)) goto out; stat(s, ORDER_FALLBACK); } - page->objects = oo_objects(oo); + slab->objects = oo_objects(oo); - account_slab(page_slab(page), oo_order(oo), s, flags); + account_slab(slab, oo_order(oo), s, flags); - page->slab_cache = s; + slab->slab_cache = s; - kasan_poison_slab(page); + kasan_poison_slab(slab_page(slab)); - start = page_address(page); + start = slab_address(slab); - setup_page_debug(s, page, start); + setup_slab_debug(s, slab, start); - shuffle = shuffle_freelist(s, page); + shuffle = shuffle_freelist(s, slab); if (!shuffle) { start = fixup_red_left(s, start); - start = setup_object(s, page, start); - page->freelist = start; - for (idx = 0, p = start; idx < page->objects - 1; idx++) { + start = setup_object(s, slab, start); + slab->freelist = start; + for (idx = 0, p = start; idx < slab->objects - 1; idx++) { next = p + s->size; - next = setup_object(s, page, next); + next = setup_object(s, slab, next); set_freepointer(s, p, next); p = next; } set_freepointer(s, p, NULL); } - page->inuse = page->objects; - page->frozen = 1; + slab->inuse = slab->objects; + slab->frozen = 1; out: - if (!page) + if (!slab) return NULL; - inc_slabs_node(s, page_to_nid(page), page->objects); + inc_slabs_node(s, slab_nid(slab), slab->objects); - return page; + return slab; } -static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) { if (unlikely(flags & GFP_SLAB_BUG_MASK)) flags = kmalloc_fix_flags(flags); @@ -2014,9 +2014,9 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) { void *p; - slab_pad_check(s, folio_page(folio, 0)); + slab_pad_check(s, slab); for_each_object(p, s, slab_address(slab), slab->objects) - check_object(s, folio_page(folio, 0), p, SLUB_RED_INACTIVE); + check_object(s, slab, p, SLUB_RED_INACTIVE); } __slab_clear_pfmemalloc(slab); @@ -2030,50 +2030,50 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) static void rcu_free_slab(struct rcu_head *h) { - struct page *page = container_of(h, struct page, rcu_head); + struct slab *slab = container_of(h, struct slab, rcu_head); - __free_slab(page->slab_cache, page_slab(page)); + __free_slab(slab->slab_cache, slab); } -static void free_slab(struct kmem_cache *s, struct page *page) +static void free_slab(struct kmem_cache *s, struct slab *slab) { if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) { - call_rcu(&page->rcu_head, rcu_free_slab); + call_rcu(&slab->rcu_head, rcu_free_slab); } else - __free_slab(s, page_slab(page)); + __free_slab(s, slab); } -static void discard_slab(struct kmem_cache *s, struct page *page) +static void discard_slab(struct kmem_cache *s, struct slab *slab) { - dec_slabs_node(s, page_to_nid(page), page->objects); - free_slab(s, page); + dec_slabs_node(s, slab_nid(slab), slab->objects); + free_slab(s, slab); } /* * Management of partially allocated slabs. */ static inline void -__add_partial(struct kmem_cache_node *n, struct page *page, int tail) +__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail) { n->nr_partial++; if (tail == DEACTIVATE_TO_TAIL) - list_add_tail(&page->slab_list, &n->partial); + list_add_tail(&slab->slab_list, &n->partial); else - list_add(&page->slab_list, &n->partial); + list_add(&slab->slab_list, &n->partial); } static inline void add_partial(struct kmem_cache_node *n, - struct page *page, int tail) + struct slab *slab, int tail) { lockdep_assert_held(&n->list_lock); - __add_partial(n, page, tail); + __add_partial(n, slab, tail); } static inline void remove_partial(struct kmem_cache_node *n, - struct page *page) + struct slab *slab) { lockdep_assert_held(&n->list_lock); - list_del(&page->slab_list); + list_del(&slab->slab_list); n->nr_partial--; } @@ -2084,12 +2084,12 @@ static inline void remove_partial(struct kmem_cache_node *n, * Returns a list of objects or NULL if it fails. */ static inline void *acquire_slab(struct kmem_cache *s, - struct kmem_cache_node *n, struct page *page, + struct kmem_cache_node *n, struct slab *slab, int mode) { void *freelist; unsigned long counters; - struct page new; + struct slab new; lockdep_assert_held(&n->list_lock); @@ -2098,11 +2098,11 @@ static inline void *acquire_slab(struct kmem_cache *s, * The old freelist is the list of objects for the * per cpu allocation list. */ - freelist = page->freelist; - counters = page->counters; + freelist = slab->freelist; + counters = slab->counters; new.counters = counters; if (mode) { - new.inuse = page->objects; + new.inuse = slab->objects; new.freelist = NULL; } else { new.freelist = freelist; @@ -2111,21 +2111,21 @@ static inline void *acquire_slab(struct kmem_cache *s, VM_BUG_ON(new.frozen); new.frozen = 1; - if (!__cmpxchg_double_slab(s, page, + if (!__cmpxchg_double_slab(s, slab, freelist, counters, new.freelist, new.counters, "acquire_slab")) return NULL; - remove_partial(n, page); + remove_partial(n, slab); WARN_ON(!freelist); return freelist; } #ifdef CONFIG_SLUB_CPU_PARTIAL -static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); +static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain); #else -static inline void put_cpu_partial(struct kmem_cache *s, struct page *page, +static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) { } #endif static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); @@ -2134,12 +2134,12 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); * Try to allocate a partial slab from a specific node. */ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, - struct page **ret_page, gfp_t gfpflags) + struct slab **ret_slab, gfp_t gfpflags) { - struct page *page, *page2; + struct slab *slab, *slab2; void *object = NULL; unsigned long flags; - unsigned int partial_pages = 0; + unsigned int partial_slabs = 0; /* * Racy check. If we mistakenly see no partial slabs then we @@ -2151,28 +2151,28 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, return NULL; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry_safe(page, page2, &n->partial, slab_list) { + list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { void *t; - if (!pfmemalloc_match(page_slab(page), gfpflags)) + if (!pfmemalloc_match(slab, gfpflags)) continue; - t = acquire_slab(s, n, page, object == NULL); + t = acquire_slab(s, n, slab, object == NULL); if (!t) break; if (!object) { - *ret_page = page; + *ret_slab = slab; stat(s, ALLOC_FROM_PARTIAL); object = t; } else { - put_cpu_partial(s, page, 0); + put_cpu_partial(s, slab, 0); stat(s, CPU_PARTIAL_NODE); - partial_pages++; + partial_slabs++; } #ifdef CONFIG_SLUB_CPU_PARTIAL if (!kmem_cache_has_cpu_partial(s) - || partial_pages > s->cpu_partial_pages / 2) + || partial_slabs > s->cpu_partial_slabs / 2) break; #else break; @@ -2187,7 +2187,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, * Get a page from somewhere. Search in increasing NUMA distances. */ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, - struct page **ret_page) + struct slab **ret_slab) { #ifdef CONFIG_NUMA struct zonelist *zonelist; @@ -2229,7 +2229,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, if (n && cpuset_zone_allowed(zone, flags) && n->nr_partial > s->min_partial) { - object = get_partial_node(s, n, ret_page, flags); + object = get_partial_node(s, n, ret_slab, flags); if (object) { /* * Don't check read_mems_allowed_retry() @@ -2251,7 +2251,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, * Get a partial page, lock it and return it. */ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, - struct page **ret_page) + struct slab **ret_slab) { void *object; int searchnode = node; @@ -2259,11 +2259,11 @@ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, if (node == NUMA_NO_NODE) searchnode = numa_mem_id(); - object = get_partial_node(s, get_node(s, searchnode), ret_page, flags); + object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags); if (object || node != NUMA_NO_NODE) return object; - return get_any_partial(s, flags, ret_page); + return get_any_partial(s, flags, ret_slab); } #ifdef CONFIG_PREEMPTION @@ -2345,20 +2345,20 @@ static void init_kmem_cache_cpus(struct kmem_cache *s) * Assumes the slab has been already safely taken away from kmem_cache_cpu * by the caller. */ -static void deactivate_slab(struct kmem_cache *s, struct page *page, +static void deactivate_slab(struct kmem_cache *s, struct slab *slab, void *freelist) { enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct kmem_cache_node *n = get_node(s, slab_nid(slab)); int lock = 0, free_delta = 0; enum slab_modes l = M_NONE, m = M_NONE; void *nextfree, *freelist_iter, *freelist_tail; int tail = DEACTIVATE_TO_HEAD; unsigned long flags = 0; - struct page new; - struct page old; + struct slab new; + struct slab old; - if (page->freelist) { + if (slab->freelist) { stat(s, DEACTIVATE_REMOTE_FREES); tail = DEACTIVATE_TO_TAIL; } @@ -2377,7 +2377,7 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, * 'freelist_iter' is already corrupted. So isolate all objects * starting at 'freelist_iter' by skipping them. */ - if (freelist_corrupted(s, page, &freelist_iter, nextfree)) + if (freelist_corrupted(s, slab, &freelist_iter, nextfree)) break; freelist_tail = freelist_iter; @@ -2404,8 +2404,8 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, */ redo: - old.freelist = READ_ONCE(page->freelist); - old.counters = READ_ONCE(page->counters); + old.freelist = READ_ONCE(slab->freelist); + old.counters = READ_ONCE(slab->counters); VM_BUG_ON(!old.frozen); /* Determine target state of the slab */ @@ -2447,18 +2447,18 @@ redo: if (l != m) { if (l == M_PARTIAL) - remove_partial(n, page); + remove_partial(n, slab); else if (l == M_FULL) - remove_full(s, n, page); + remove_full(s, n, slab); if (m == M_PARTIAL) - add_partial(n, page, tail); + add_partial(n, slab, tail); else if (m == M_FULL) - add_full(s, n, page); + add_full(s, n, slab); } l = m; - if (!cmpxchg_double_slab(s, page, + if (!cmpxchg_double_slab(s, slab, old.freelist, old.counters, new.freelist, new.counters, "unfreezing slab")) @@ -2473,26 +2473,26 @@ redo: stat(s, DEACTIVATE_FULL); else if (m == M_FREE) { stat(s, DEACTIVATE_EMPTY); - discard_slab(s, page); + discard_slab(s, slab); stat(s, FREE_SLAB); } } #ifdef CONFIG_SLUB_CPU_PARTIAL -static void __unfreeze_partials(struct kmem_cache *s, struct page *partial_page) +static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab) { struct kmem_cache_node *n = NULL, *n2 = NULL; - struct page *page, *discard_page = NULL; + struct slab *slab, *slab_to_discard = NULL; unsigned long flags = 0; - while (partial_page) { - struct page new; - struct page old; + while (partial_slab) { + struct slab new; + struct slab old; - page = partial_page; - partial_page = page->next; + slab = partial_slab; + partial_slab = slab->next; - n2 = get_node(s, page_to_nid(page)); + n2 = get_node(s, slab_nid(slab)); if (n != n2) { if (n) spin_unlock_irqrestore(&n->list_lock, flags); @@ -2503,8 +2503,8 @@ static void __unfreeze_partials(struct kmem_cache *s, struct page *partial_page) do { - old.freelist = page->freelist; - old.counters = page->counters; + old.freelist = slab->freelist; + old.counters = slab->counters; VM_BUG_ON(!old.frozen); new.counters = old.counters; @@ -2512,16 +2512,16 @@ static void __unfreeze_partials(struct kmem_cache *s, struct page *partial_page) new.frozen = 0; - } while (!__cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, slab, old.freelist, old.counters, new.freelist, new.counters, "unfreezing slab")); if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { - page->next = discard_page; - discard_page = page; + slab->next = slab_to_discard; + slab_to_discard = slab; } else { - add_partial(n, page, DEACTIVATE_TO_TAIL); + add_partial(n, slab, DEACTIVATE_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } } @@ -2529,12 +2529,12 @@ static void __unfreeze_partials(struct kmem_cache *s, struct page *partial_page) if (n) spin_unlock_irqrestore(&n->list_lock, flags); - while (discard_page) { - page = discard_page; - discard_page = discard_page->next; + while (slab_to_discard) { + slab = slab_to_discard; + slab_to_discard = slab_to_discard->next; stat(s, DEACTIVATE_EMPTY); - discard_slab(s, page); + discard_slab(s, slab); stat(s, FREE_SLAB); } } @@ -2544,28 +2544,28 @@ static void __unfreeze_partials(struct kmem_cache *s, struct page *partial_page) */ static void unfreeze_partials(struct kmem_cache *s) { - struct page *partial_page; + struct slab *partial_slab; unsigned long flags; local_lock_irqsave(&s->cpu_slab->lock, flags); - partial_page = this_cpu_read(s->cpu_slab->partial); + partial_slab = this_cpu_read(s->cpu_slab->partial); this_cpu_write(s->cpu_slab->partial, NULL); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - if (partial_page) - __unfreeze_partials(s, partial_page); + if (partial_slab) + __unfreeze_partials(s, partial_slab); } static void unfreeze_partials_cpu(struct kmem_cache *s, struct kmem_cache_cpu *c) { - struct page *partial_page; + struct slab *partial_slab; - partial_page = slub_percpu_partial(c); + partial_slab = slub_percpu_partial(c); c->partial = NULL; - if (partial_page) - __unfreeze_partials(s, partial_page); + if (partial_slab) + __unfreeze_partials(s, partial_slab); } /* @@ -2575,42 +2575,42 @@ static void unfreeze_partials_cpu(struct kmem_cache *s, * If we did not find a slot then simply move all the partials to the * per node partial list. */ -static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) +static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) { - struct page *oldpage; - struct page *page_to_unfreeze = NULL; + struct slab *oldslab; + struct slab *slab_to_unfreeze = NULL; unsigned long flags; - int pages = 0; + int slabs = 0; local_lock_irqsave(&s->cpu_slab->lock, flags); - oldpage = this_cpu_read(s->cpu_slab->partial); + oldslab = this_cpu_read(s->cpu_slab->partial); - if (oldpage) { - if (drain && oldpage->pages >= s->cpu_partial_pages) { + if (oldslab) { + if (drain && oldslab->slabs >= s->cpu_partial_slabs) { /* * Partial array is full. Move the existing set to the * per node partial list. Postpone the actual unfreezing * outside of the critical section. */ - page_to_unfreeze = oldpage; - oldpage = NULL; + slab_to_unfreeze = oldslab; + oldslab = NULL; } else { - pages = oldpage->pages; + slabs = oldslab->slabs; } } - pages++; + slabs++; - page->pages = pages; - page->next = oldpage; + slab->slabs = slabs; + slab->next = oldslab; - this_cpu_write(s->cpu_slab->partial, page); + this_cpu_write(s->cpu_slab->partial, slab); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - if (page_to_unfreeze) { - __unfreeze_partials(s, page_to_unfreeze); + if (slab_to_unfreeze) { + __unfreeze_partials(s, slab_to_unfreeze); stat(s, CPU_PARTIAL_DRAIN); } } @@ -2626,22 +2626,22 @@ static inline void unfreeze_partials_cpu(struct kmem_cache *s, static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) { unsigned long flags; - struct page *page; + struct slab *slab; void *freelist; local_lock_irqsave(&s->cpu_slab->lock, flags); - page = c->page; + slab = c->slab; freelist = c->freelist; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - if (page) { - deactivate_slab(s, page, freelist); + if (slab) { + deactivate_slab(s, slab, freelist); stat(s, CPUSLAB_FLUSH); } } @@ -2650,14 +2650,14 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); void *freelist = c->freelist; - struct page *page = c->page; + struct slab *slab = c->slab; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); - if (page) { - deactivate_slab(s, page, freelist); + if (slab) { + deactivate_slab(s, slab, freelist); stat(s, CPUSLAB_FLUSH); } @@ -2686,7 +2686,7 @@ static void flush_cpu_slab(struct work_struct *w) s = sfw->s; c = this_cpu_ptr(s->cpu_slab); - if (c->page) + if (c->slab) flush_slab(s, c); unfreeze_partials(s); @@ -2696,7 +2696,7 @@ static bool has_cpu_slab(int cpu, struct kmem_cache *s) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - return c->page || slub_percpu_partial(c); + return c->slab || slub_percpu_partial(c); } static DEFINE_MUTEX(flush_lock); @@ -2758,19 +2758,19 @@ static int slub_cpu_dead(unsigned int cpu) * Check if the objects in a per cpu structure fit numa * locality expectations. */ -static inline int node_match(struct page *page, int node) +static inline int node_match(struct slab *slab, int node) { #ifdef CONFIG_NUMA - if (node != NUMA_NO_NODE && page_to_nid(page) != node) + if (node != NUMA_NO_NODE && slab_nid(slab) != node) return 0; #endif return 1; } #ifdef CONFIG_SLUB_DEBUG -static int count_free(struct page *page) +static int count_free(struct slab *slab) { - return page->objects - page->inuse; + return slab->objects - slab->inuse; } static inline unsigned long node_nr_objs(struct kmem_cache_node *n) @@ -2781,15 +2781,15 @@ static inline unsigned long node_nr_objs(struct kmem_cache_node *n) #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) static unsigned long count_partial(struct kmem_cache_node *n, - int (*get_count)(struct page *)) + int (*get_count)(struct slab *)) { unsigned long flags; unsigned long x = 0; - struct page *page; + struct slab *slab; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, slab_list) - x += get_count(page); + list_for_each_entry(slab, &n->partial, slab_list) + x += get_count(slab); spin_unlock_irqrestore(&n->list_lock, flags); return x; } @@ -2848,25 +2848,25 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) * * If this function returns NULL then the page has been unfrozen. */ -static inline void *get_freelist(struct kmem_cache *s, struct page *page) +static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) { - struct page new; + struct slab new; unsigned long counters; void *freelist; lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); do { - freelist = page->freelist; - counters = page->counters; + freelist = slab->freelist; + counters = slab->counters; new.counters = counters; VM_BUG_ON(!new.frozen); - new.inuse = page->objects; + new.inuse = slab->objects; new.frozen = freelist != NULL; - } while (!__cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, slab, freelist, counters, NULL, new.counters, "get_freelist")); @@ -2897,15 +2897,15 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { void *freelist; - struct page *page; + struct slab *slab; unsigned long flags; stat(s, ALLOC_SLOWPATH); reread_page: - page = READ_ONCE(c->page); - if (!page) { + slab = READ_ONCE(c->slab); + if (!slab) { /* * if the node is not online or has no normal memory, just * ignore the node constraint @@ -2917,7 +2917,7 @@ reread_page: } redo: - if (unlikely(!node_match(page, node))) { + if (unlikely(!node_match(slab, node))) { /* * same as above but node_match() being false already * implies node != NUMA_NO_NODE @@ -2936,12 +2936,12 @@ redo: * PFMEMALLOC but right now, we are losing the pfmemalloc * information when the page leaves the per-cpu allocator */ - if (unlikely(!pfmemalloc_match(page_slab(page), gfpflags))) + if (unlikely(!pfmemalloc_match(slab, gfpflags))) goto deactivate_slab; /* must check again c->page in case we got preempted and it changed */ local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(page != c->page)) { + if (unlikely(slab != c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); goto reread_page; } @@ -2949,10 +2949,10 @@ redo: if (freelist) goto load_freelist; - freelist = get_freelist(s, page); + freelist = get_freelist(s, slab); if (!freelist) { - c->page = NULL; + c->slab = NULL; local_unlock_irqrestore(&s->cpu_slab->lock, flags); stat(s, DEACTIVATE_BYPASS); goto new_slab; @@ -2969,7 +2969,7 @@ load_freelist: * page is pointing to the page from which the objects are obtained. * That page must be frozen for per cpu allocations to work. */ - VM_BUG_ON(!c->page->frozen); + VM_BUG_ON(!c->slab->frozen); c->freelist = get_freepointer(s, freelist); c->tid = next_tid(c->tid); local_unlock_irqrestore(&s->cpu_slab->lock, flags); @@ -2978,21 +2978,21 @@ load_freelist: deactivate_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); - if (page != c->page) { + if (slab != c->slab) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); goto reread_page; } freelist = c->freelist; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; local_unlock_irqrestore(&s->cpu_slab->lock, flags); - deactivate_slab(s, page, freelist); + deactivate_slab(s, slab, freelist); new_slab: if (slub_percpu_partial(c)) { local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(c->page)) { + if (unlikely(c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); goto reread_page; } @@ -3002,8 +3002,8 @@ new_slab: goto new_objects; } - page = c->page = slub_percpu_partial(c); - slub_set_percpu_partial(c, page); + slab = c->slab = slub_percpu_partial(c); + slub_set_percpu_partial(c, slab); local_unlock_irqrestore(&s->cpu_slab->lock, flags); stat(s, CPU_PARTIAL_ALLOC); goto redo; @@ -3011,15 +3011,15 @@ new_slab: new_objects: - freelist = get_partial(s, gfpflags, node, &page); + freelist = get_partial(s, gfpflags, node, &slab); if (freelist) goto check_new_page; slub_put_cpu_ptr(s->cpu_slab); - page = new_slab(s, gfpflags, node); + slab = new_slab(s, gfpflags, node); c = slub_get_cpu_ptr(s->cpu_slab); - if (unlikely(!page)) { + if (unlikely(!slab)) { slab_out_of_memory(s, gfpflags, node); return NULL; } @@ -3028,15 +3028,15 @@ new_objects: * No other reference to the page yet so we can * muck around with it freely without cmpxchg */ - freelist = page->freelist; - page->freelist = NULL; + freelist = slab->freelist; + slab->freelist = NULL; stat(s, ALLOC_SLAB); check_new_page: if (kmem_cache_debug(s)) { - if (!alloc_debug_processing(s, page, freelist, addr)) { + if (!alloc_debug_processing(s, slab, freelist, addr)) { /* Slab failed checks. Next slab needed */ goto new_slab; } else { @@ -3048,7 +3048,7 @@ check_new_page: } } - if (unlikely(!pfmemalloc_match(page_slab(page), gfpflags))) + if (unlikely(!pfmemalloc_match(slab, gfpflags))) /* * For !pfmemalloc_match() case we don't load freelist so that * we don't make further mismatched allocations easier. @@ -3058,29 +3058,29 @@ check_new_page: retry_load_page: local_lock_irqsave(&s->cpu_slab->lock, flags); - if (unlikely(c->page)) { + if (unlikely(c->slab)) { void *flush_freelist = c->freelist; - struct page *flush_page = c->page; + struct slab *flush_slab = c->slab; - c->page = NULL; + c->slab = NULL; c->freelist = NULL; c->tid = next_tid(c->tid); local_unlock_irqrestore(&s->cpu_slab->lock, flags); - deactivate_slab(s, flush_page, flush_freelist); + deactivate_slab(s, flush_slab, flush_freelist); stat(s, CPUSLAB_FLUSH); goto retry_load_page; } - c->page = page; + c->slab = slab; goto load_freelist; return_single: - deactivate_slab(s, page, get_freepointer(s, freelist)); + deactivate_slab(s, slab, get_freepointer(s, freelist)); return freelist; } @@ -3137,7 +3137,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s, { void *object; struct kmem_cache_cpu *c; - struct page *page; + struct slab *slab; unsigned long tid; struct obj_cgroup *objcg = NULL; bool init = false; @@ -3184,7 +3184,7 @@ redo: */ object = c->freelist; - page = c->page; + slab = c->slab; /* * We cannot use the lockless fastpath on PREEMPT_RT because if a * slowpath has taken the local_lock_irqsave(), it is not protected @@ -3193,7 +3193,7 @@ redo: * there is a suitable cpu freelist. */ if (IS_ENABLED(CONFIG_PREEMPT_RT) || - unlikely(!object || !page || !node_match(page, node))) { + unlikely(!object || !slab || !node_match(slab, node))) { object = __slab_alloc(s, gfpflags, node, addr, c); } else { void *next_object = get_freepointer_safe(s, object); @@ -3298,14 +3298,14 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_trace); * lock and free the item. If there is no additional partial page * handling required then we can return immediately. */ -static void __slab_free(struct kmem_cache *s, struct page *page, +static void __slab_free(struct kmem_cache *s, struct slab *slab, void *head, void *tail, int cnt, unsigned long addr) { void *prior; int was_frozen; - struct page new; + struct slab new; unsigned long counters; struct kmem_cache_node *n = NULL; unsigned long flags; @@ -3316,7 +3316,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, return; if (kmem_cache_debug(s) && - !free_debug_processing(s, page, head, tail, cnt, addr)) + !free_debug_processing(s, slab, head, tail, cnt, addr)) return; do { @@ -3324,8 +3324,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page, spin_unlock_irqrestore(&n->list_lock, flags); n = NULL; } - prior = page->freelist; - counters = page->counters; + prior = slab->freelist; + counters = slab->counters; set_freepointer(s, tail, prior); new.counters = counters; was_frozen = new.frozen; @@ -3344,7 +3344,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, } else { /* Needs to be taken off a list */ - n = get_node(s, page_to_nid(page)); + n = get_node(s, slab_nid(slab)); /* * Speculatively acquire the list_lock. * If the cmpxchg does not succeed then we may @@ -3358,7 +3358,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, } } - } while (!cmpxchg_double_slab(s, page, + } while (!cmpxchg_double_slab(s, slab, prior, counters, head, new.counters, "__slab_free")); @@ -3376,7 +3376,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, * If we just froze the page then put it onto the * per cpu partial list. */ - put_cpu_partial(s, page, 1); + put_cpu_partial(s, slab, 1); stat(s, CPU_PARTIAL_FREE); } @@ -3391,8 +3391,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page, * then add it. */ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { - remove_full(s, n, page); - add_partial(n, page, DEACTIVATE_TO_TAIL); + remove_full(s, n, slab); + add_partial(n, slab, DEACTIVATE_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } spin_unlock_irqrestore(&n->list_lock, flags); @@ -3403,16 +3403,16 @@ slab_empty: /* * Slab on the partial list. */ - remove_partial(n, page); + remove_partial(n, slab); stat(s, FREE_REMOVE_PARTIAL); } else { /* Slab must be on the full list */ - remove_full(s, n, page); + remove_full(s, n, slab); } spin_unlock_irqrestore(&n->list_lock, flags); stat(s, FREE_SLAB); - discard_slab(s, page); + discard_slab(s, slab); } /* @@ -3431,7 +3431,7 @@ slab_empty: * count (cnt). Bulk free indicated by tail pointer being set. */ static __always_inline void do_slab_free(struct kmem_cache *s, - struct page *page, void *head, void *tail, + struct slab *slab, void *head, void *tail, int cnt, unsigned long addr) { void *tail_obj = tail ? : head; @@ -3454,7 +3454,7 @@ redo: /* Same with comment on barrier() in slab_alloc_node() */ barrier(); - if (likely(page == c->page)) { + if (likely(slab == c->slab)) { #ifndef CONFIG_PREEMPT_RT void **freelist = READ_ONCE(c->freelist); @@ -3480,7 +3480,7 @@ redo: local_lock(&s->cpu_slab->lock); c = this_cpu_ptr(s->cpu_slab); - if (unlikely(page != c->page)) { + if (unlikely(slab != c->slab)) { local_unlock(&s->cpu_slab->lock); goto redo; } @@ -3495,11 +3495,11 @@ redo: #endif stat(s, FREE_FASTPATH); } else - __slab_free(s, page, head, tail_obj, cnt, addr); + __slab_free(s, slab, head, tail_obj, cnt, addr); } -static __always_inline void slab_free(struct kmem_cache *s, struct page *page, +static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab, void *head, void *tail, int cnt, unsigned long addr) { @@ -3508,13 +3508,13 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page, * to remove objects, whose reuse must be delayed. */ if (slab_free_freelist_hook(s, &head, &tail, &cnt)) - do_slab_free(s, page, head, tail, cnt, addr); + do_slab_free(s, slab, head, tail, cnt, addr); } #ifdef CONFIG_KASAN_GENERIC void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) { - do_slab_free(cache, slab_page(virt_to_slab(x)), x, NULL, 1, addr); + do_slab_free(cache, virt_to_slab(x), x, NULL, 1, addr); } #endif @@ -3524,7 +3524,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) if (!s) return; trace_kmem_cache_free(_RET_IP_, x, s->name); - slab_free(s, slab_page(virt_to_slab(x)), x, NULL, 1, _RET_IP_); + slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_); } EXPORT_SYMBOL(kmem_cache_free); @@ -3654,7 +3654,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) if (!df.slab) continue; - slab_free(df.s, slab_page(df.slab), df.freelist, df.tail, df.cnt, _RET_IP_); + slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, _RET_IP_); } while (likely(size)); } EXPORT_SYMBOL(kmem_cache_free_bulk); @@ -3924,38 +3924,38 @@ static struct kmem_cache *kmem_cache_node; */ static void early_kmem_cache_node_alloc(int node) { - struct page *page; + struct slab *slab; struct kmem_cache_node *n; BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); - page = new_slab(kmem_cache_node, GFP_NOWAIT, node); + slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); - BUG_ON(!page); - if (page_to_nid(page) != node) { + BUG_ON(!slab); + if (slab_nid(slab) != node) { pr_err("SLUB: Unable to allocate memory from node %d\n", node); pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); } - n = page->freelist; + n = slab->freelist; BUG_ON(!n); #ifdef CONFIG_SLUB_DEBUG init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); init_tracking(kmem_cache_node, n); #endif n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false); - page->freelist = get_freepointer(kmem_cache_node, n); - page->inuse = 1; - page->frozen = 0; + slab->freelist = get_freepointer(kmem_cache_node, n); + slab->inuse = 1; + slab->frozen = 0; kmem_cache_node->node[node] = n; init_kmem_cache_node(n); - inc_slabs_node(kmem_cache_node, node, page->objects); + inc_slabs_node(kmem_cache_node, node, slab->objects); /* * No locks need to be taken here as it has just been * initialized and there is no concurrent access. */ - __add_partial(n, page, DEACTIVATE_TO_HEAD); + __add_partial(n, slab, DEACTIVATE_TO_HEAD); } static void free_kmem_cache_nodes(struct kmem_cache *s) @@ -4241,20 +4241,20 @@ error: return -EINVAL; } -static void list_slab_objects(struct kmem_cache *s, struct page *page, +static void list_slab_objects(struct kmem_cache *s, struct slab *slab, const char *text) { #ifdef CONFIG_SLUB_DEBUG - void *addr = page_address(page); + void *addr = slab_address(slab); unsigned long flags; unsigned long *map; void *p; - slab_err(s, page, text, s->name); - slab_lock(page, &flags); + slab_err(s, slab, text, s->name); + slab_lock(slab, &flags); - map = get_map(s, page); - for_each_object(p, s, addr, page->objects) { + map = get_map(s, slab); + for_each_object(p, s, addr, slab->objects) { if (!test_bit(__obj_to_index(s, addr, p), map)) { pr_err("Object 0x%p @offset=%tu\n", p, p - addr); @@ -4262,7 +4262,7 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page, } } put_map(map); - slab_unlock(page, &flags); + slab_unlock(slab, &flags); #endif } @@ -4274,23 +4274,23 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page, static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) { LIST_HEAD(discard); - struct page *page, *h; + struct slab *slab, *h; BUG_ON(irqs_disabled()); spin_lock_irq(&n->list_lock); - list_for_each_entry_safe(page, h, &n->partial, slab_list) { - if (!page->inuse) { - remove_partial(n, page); - list_add(&page->slab_list, &discard); + list_for_each_entry_safe(slab, h, &n->partial, slab_list) { + if (!slab->inuse) { + remove_partial(n, slab); + list_add(&slab->slab_list, &discard); } else { - list_slab_objects(s, page, + list_slab_objects(s, slab, "Objects remaining in %s on __kmem_cache_shutdown()"); } } spin_unlock_irq(&n->list_lock); - list_for_each_entry_safe(page, h, &discard, slab_list) - discard_slab(s, page); + list_for_each_entry_safe(slab, h, &discard, slab_list) + discard_slab(s, slab); } bool __kmem_cache_empty(struct kmem_cache *s) @@ -4560,7 +4560,7 @@ void kfree(const void *x) return; } slab = folio_slab(folio); - slab_free(slab->slab_cache, slab_page(slab), object, NULL, 1, _RET_IP_); + slab_free(slab->slab_cache, slab, object, NULL, 1, _RET_IP_); } EXPORT_SYMBOL(kfree); @@ -4580,8 +4580,8 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) int node; int i; struct kmem_cache_node *n; - struct page *page; - struct page *t; + struct slab *slab; + struct slab *t; struct list_head discard; struct list_head promote[SHRINK_PROMOTE_MAX]; unsigned long flags; @@ -4600,8 +4600,8 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) * Note that concurrent frees may occur while we hold the * list_lock. page->inuse here is the upper limit. */ - list_for_each_entry_safe(page, t, &n->partial, slab_list) { - int free = page->objects - page->inuse; + list_for_each_entry_safe(slab, t, &n->partial, slab_list) { + int free = slab->objects - slab->inuse; /* Do not reread page->inuse */ barrier(); @@ -4609,11 +4609,11 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) /* We do not keep full slabs on the list */ BUG_ON(free <= 0); - if (free == page->objects) { - list_move(&page->slab_list, &discard); + if (free == slab->objects) { + list_move(&slab->slab_list, &discard); n->nr_partial--; } else if (free <= SHRINK_PROMOTE_MAX) - list_move(&page->slab_list, promote + free - 1); + list_move(&slab->slab_list, promote + free - 1); } /* @@ -4626,8 +4626,8 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) spin_unlock_irqrestore(&n->list_lock, flags); /* Release empty slabs */ - list_for_each_entry_safe(page, t, &discard, slab_list) - discard_slab(s, page); + list_for_each_entry_safe(slab, t, &discard, slab_list) + discard_slab(s, slab); if (slabs_node(s, node)) ret = 1; @@ -4788,7 +4788,7 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) */ __flush_cpu_slab(s, smp_processor_id()); for_each_kmem_cache_node(s, node, n) { - struct page *p; + struct slab *p; list_for_each_entry(p, &n->partial, slab_list) p->slab_cache = s; @@ -4966,54 +4966,54 @@ EXPORT_SYMBOL(__kmalloc_node_track_caller); #endif #ifdef CONFIG_SYSFS -static int count_inuse(struct page *page) +static int count_inuse(struct slab *slab) { - return page->inuse; + return slab->inuse; } -static int count_total(struct page *page) +static int count_total(struct slab *slab) { - return page->objects; + return slab->objects; } #endif #ifdef CONFIG_SLUB_DEBUG -static void validate_slab(struct kmem_cache *s, struct page *page, +static void validate_slab(struct kmem_cache *s, struct slab *slab, unsigned long *obj_map) { void *p; - void *addr = page_address(page); + void *addr = slab_address(slab); unsigned long flags; - slab_lock(page, &flags); + slab_lock(slab, &flags); - if (!check_slab(s, page) || !on_freelist(s, page, NULL)) + if (!check_slab(s, slab) || !on_freelist(s, slab, NULL)) goto unlock; /* Now we know that a valid freelist exists */ - __fill_map(obj_map, s, page); - for_each_object(p, s, addr, page->objects) { + __fill_map(obj_map, s, slab); + for_each_object(p, s, addr, slab->objects) { u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ? SLUB_RED_INACTIVE : SLUB_RED_ACTIVE; - if (!check_object(s, page, p, val)) + if (!check_object(s, slab, p, val)) break; } unlock: - slab_unlock(page, &flags); + slab_unlock(slab, &flags); } static int validate_slab_node(struct kmem_cache *s, struct kmem_cache_node *n, unsigned long *obj_map) { unsigned long count = 0; - struct page *page; + struct slab *slab; unsigned long flags; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, slab_list) { - validate_slab(s, page, obj_map); + list_for_each_entry(slab, &n->partial, slab_list) { + validate_slab(s, slab, obj_map); count++; } if (count != n->nr_partial) { @@ -5025,8 +5025,8 @@ static int validate_slab_node(struct kmem_cache *s, if (!(s->flags & SLAB_STORE_USER)) goto out; - list_for_each_entry(page, &n->full, slab_list) { - validate_slab(s, page, obj_map); + list_for_each_entry(slab, &n->full, slab_list) { + validate_slab(s, slab, obj_map); count++; } if (count != atomic_long_read(&n->nr_slabs)) { @@ -5192,15 +5192,15 @@ static int add_location(struct loc_track *t, struct kmem_cache *s, } static void process_slab(struct loc_track *t, struct kmem_cache *s, - struct page *page, enum track_item alloc, + struct slab *slab, enum track_item alloc, unsigned long *obj_map) { - void *addr = page_address(page); + void *addr = slab_address(slab); void *p; - __fill_map(obj_map, s, page); + __fill_map(obj_map, s, slab); - for_each_object(p, s, addr, page->objects) + for_each_object(p, s, addr, slab->objects) if (!test_bit(__obj_to_index(s, addr, p), obj_map)) add_location(t, s, get_track(s, p, alloc)); } @@ -5242,32 +5242,32 @@ static ssize_t show_slab_objects(struct kmem_cache *s, struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); int node; - struct page *page; + struct slab *slab; - page = READ_ONCE(c->page); - if (!page) + slab = READ_ONCE(c->slab); + if (!slab) continue; - node = page_to_nid(page); + node = slab_nid(slab); if (flags & SO_TOTAL) - x = page->objects; + x = slab->objects; else if (flags & SO_OBJECTS) - x = page->inuse; + x = slab->inuse; else x = 1; total += x; nodes[node] += x; - page = slub_percpu_partial_read_once(c); - if (page) { - node = page_to_nid(page); + slab = slub_percpu_partial_read_once(c); + if (slab) { + node = slab_nid(slab); if (flags & SO_TOTAL) WARN_ON_ONCE(1); else if (flags & SO_OBJECTS) WARN_ON_ONCE(1); else - x = page->pages; + x = slab->slabs; total += x; nodes[node] += x; } @@ -5469,33 +5469,33 @@ SLAB_ATTR_RO(objects_partial); static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) { int objects = 0; - int pages = 0; + int slabs = 0; int cpu; int len = 0; for_each_online_cpu(cpu) { - struct page *page; + struct slab *slab; - page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); + slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); - if (page) - pages += page->pages; + if (slab) + slabs += slab->slabs; } /* Approximate half-full pages , see slub_set_cpu_partial() */ - objects = (pages * oo_objects(s->oo)) / 2; - len += sysfs_emit_at(buf, len, "%d(%d)", objects, pages); + objects = (slabs * oo_objects(s->oo)) / 2; + len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); #ifdef CONFIG_SMP for_each_online_cpu(cpu) { - struct page *page; + struct slab *slab; - page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); - if (page) { - pages = READ_ONCE(page->pages); - objects = (pages * oo_objects(s->oo)) / 2; + slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); + if (slab) { + slabs = READ_ONCE(slab->slabs); + objects = (slabs * oo_objects(s->oo)) / 2; len += sysfs_emit_at(buf, len, " C%d=%d(%d)", - cpu, objects, pages); + cpu, objects, slabs); } } #endif @@ -6163,16 +6163,16 @@ static int slab_debug_trace_open(struct inode *inode, struct file *filep) for_each_kmem_cache_node(s, node, n) { unsigned long flags; - struct page *page; + struct slab *slab; if (!atomic_long_read(&n->nr_slabs)) continue; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, slab_list) - process_slab(t, s, page, alloc, obj_map); - list_for_each_entry(page, &n->full, slab_list) - process_slab(t, s, page, alloc, obj_map); + list_for_each_entry(slab, &n->partial, slab_list) + process_slab(t, s, slab, alloc, obj_map); + list_for_each_entry(slab, &n->full, slab_list) + process_slab(t, s, slab, alloc, obj_map); spin_unlock_irqrestore(&n->list_lock, flags); } -- cgit v1.2.3-59-g8ed1b From c2092c12064a9728b2928979f88575cc1c2247fa Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 15 Nov 2021 16:55:15 +0100 Subject: mm/slub: Finish struct page to struct slab conversion Update comments mentioning pages to mention slabs where appropriate. Also some goto labels. Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin --- include/linux/slub_def.h | 2 +- mm/slub.c | 105 +++++++++++++++++++++++------------------------ 2 files changed, 53 insertions(+), 54 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 00d99afe1c0e..8a9c2876ca89 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -99,7 +99,7 @@ struct kmem_cache { #ifdef CONFIG_SLUB_CPU_PARTIAL /* Number of per cpu partial objects to keep around */ unsigned int cpu_partial; - /* Number of per cpu partial pages to keep around */ + /* Number of per cpu partial slabs to keep around */ unsigned int cpu_partial_slabs; #endif struct kmem_cache_order_objects oo; diff --git a/mm/slub.c b/mm/slub.c index e89208f3197a..cc64ba9d9963 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -48,7 +48,7 @@ * 1. slab_mutex (Global Mutex) * 2. node->list_lock (Spinlock) * 3. kmem_cache->cpu_slab->lock (Local lock) - * 4. slab_lock(page) (Only on some arches or for debugging) + * 4. slab_lock(slab) (Only on some arches or for debugging) * 5. object_map_lock (Only for debugging) * * slab_mutex @@ -64,19 +64,19 @@ * * The slab_lock is only used for debugging and on arches that do not * have the ability to do a cmpxchg_double. It only protects: - * A. page->freelist -> List of object free in a page - * B. page->inuse -> Number of objects in use - * C. page->objects -> Number of objects in page - * D. page->frozen -> frozen state + * A. slab->freelist -> List of free objects in a slab + * B. slab->inuse -> Number of objects in use + * C. slab->objects -> Number of objects in slab + * D. slab->frozen -> frozen state * * Frozen slabs * * If a slab is frozen then it is exempt from list management. It is not * on any list except per cpu partial list. The processor that froze the - * slab is the one who can perform list operations on the page. Other + * slab is the one who can perform list operations on the slab. Other * processors may put objects onto the freelist but the processor that * froze the slab is the only one that can retrieve the objects from the - * page's freelist. + * slab's freelist. * * list_lock * @@ -135,7 +135,7 @@ * minimal so we rely on the page allocators per cpu caches for * fast frees and allocs. * - * page->frozen The slab is frozen and exempt from list processing. + * slab->frozen The slab is frozen and exempt from list processing. * This means that the slab is dedicated to a purpose * such as satisfying allocations for a specific * processor. Objects may be freed in the slab while @@ -250,7 +250,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) #define OO_SHIFT 16 #define OO_MASK ((1 << OO_SHIFT) - 1) -#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ +#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */ /* Internal SLUB flags */ /* Poison object */ @@ -423,8 +423,8 @@ static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) /* * We take the number of objects but actually limit the number of - * pages on the per cpu partial list, in order to limit excessive - * growth of the list. For simplicity we assume that the pages will + * slabs on the per cpu partial list, in order to limit excessive + * growth of the list. For simplicity we assume that the slabs will * be half-full. */ nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); @@ -594,9 +594,9 @@ static inline bool slab_add_kunit_errors(void) { return false; } #endif /* - * Determine a map of object in use on a page. + * Determine a map of objects in use in a slab. * - * Node listlock must be held to guarantee that the page does + * Node listlock must be held to guarantee that the slab does * not vanish from under us. */ static unsigned long *get_map(struct kmem_cache *s, struct slab *slab) @@ -1139,7 +1139,7 @@ static int check_slab(struct kmem_cache *s, struct slab *slab) } /* - * Determine if a certain object on a page is on the freelist. Must hold the + * Determine if a certain object in a slab is on the freelist. Must hold the * slab lock to guarantee that the chains are in a consistent state. */ static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search) @@ -2184,7 +2184,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, } /* - * Get a page from somewhere. Search in increasing NUMA distances. + * Get a slab from somewhere. Search in increasing NUMA distances. */ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, struct slab **ret_slab) @@ -2248,7 +2248,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, } /* - * Get a partial page, lock it and return it. + * Get a partial slab, lock it and return it. */ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, struct slab **ret_slab) @@ -2340,7 +2340,7 @@ static void init_kmem_cache_cpus(struct kmem_cache *s) } /* - * Finishes removing the cpu slab. Merges cpu's freelist with page's freelist, + * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist, * unfreezes the slabs and puts it on the proper list. * Assumes the slab has been already safely taken away from kmem_cache_cpu * by the caller. @@ -2387,18 +2387,18 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab, } /* - * Stage two: Unfreeze the page while splicing the per-cpu - * freelist to the head of page's freelist. + * Stage two: Unfreeze the slab while splicing the per-cpu + * freelist to the head of slab's freelist. * - * Ensure that the page is unfrozen while the list presence + * Ensure that the slab is unfrozen while the list presence * reflects the actual number of objects during unfreeze. * * We setup the list membership and then perform a cmpxchg - * with the count. If there is a mismatch then the page - * is not unfrozen but the page is on the wrong list. + * with the count. If there is a mismatch then the slab + * is not unfrozen but the slab is on the wrong list. * * Then we restart the process which may have to remove - * the page from the list that we just put it on again + * the slab from the list that we just put it on again * because the number of objects in the slab may have * changed. */ @@ -2426,9 +2426,8 @@ redo: if (!lock) { lock = 1; /* - * Taking the spinlock removes the possibility - * that acquire_slab() will see a slab page that - * is frozen + * Taking the spinlock removes the possibility that + * acquire_slab() will see a slab that is frozen */ spin_lock_irqsave(&n->list_lock, flags); } @@ -2569,8 +2568,8 @@ static void unfreeze_partials_cpu(struct kmem_cache *s, } /* - * Put a page that was just frozen (in __slab_free|get_partial_node) into a - * partial page slot if available. + * Put a slab that was just frozen (in __slab_free|get_partial_node) into a + * partial slab slot if available. * * If we did not find a slot then simply move all the partials to the * per node partial list. @@ -2841,12 +2840,12 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) } /* - * Check the page->freelist of a page and either transfer the freelist to the - * per cpu freelist or deactivate the page. + * Check the slab->freelist and either transfer the freelist to the + * per cpu freelist or deactivate the slab. * - * The page is still frozen if the return value is not NULL. + * The slab is still frozen if the return value is not NULL. * - * If this function returns NULL then the page has been unfrozen. + * If this function returns NULL then the slab has been unfrozen. */ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) { @@ -2902,7 +2901,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, stat(s, ALLOC_SLOWPATH); -reread_page: +reread_slab: slab = READ_ONCE(c->slab); if (!slab) { @@ -2939,11 +2938,11 @@ redo: if (unlikely(!pfmemalloc_match(slab, gfpflags))) goto deactivate_slab; - /* must check again c->page in case we got preempted and it changed */ + /* must check again c->slab in case we got preempted and it changed */ local_lock_irqsave(&s->cpu_slab->lock, flags); if (unlikely(slab != c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } freelist = c->freelist; if (freelist) @@ -2966,8 +2965,8 @@ load_freelist: /* * freelist is pointing to the list of objects to be used. - * page is pointing to the page from which the objects are obtained. - * That page must be frozen for per cpu allocations to work. + * slab is pointing to the slab from which the objects are obtained. + * That slab must be frozen for per cpu allocations to work. */ VM_BUG_ON(!c->slab->frozen); c->freelist = get_freepointer(s, freelist); @@ -2980,7 +2979,7 @@ deactivate_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); if (slab != c->slab) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } freelist = c->freelist; c->slab = NULL; @@ -2994,7 +2993,7 @@ new_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); if (unlikely(c->slab)) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); - goto reread_page; + goto reread_slab; } if (unlikely(!slub_percpu_partial(c))) { local_unlock_irqrestore(&s->cpu_slab->lock, flags); @@ -3013,7 +3012,7 @@ new_objects: freelist = get_partial(s, gfpflags, node, &slab); if (freelist) - goto check_new_page; + goto check_new_slab; slub_put_cpu_ptr(s->cpu_slab); slab = new_slab(s, gfpflags, node); @@ -3025,7 +3024,7 @@ new_objects: } /* - * No other reference to the page yet so we can + * No other reference to the slab yet so we can * muck around with it freely without cmpxchg */ freelist = slab->freelist; @@ -3033,7 +3032,7 @@ new_objects: stat(s, ALLOC_SLAB); -check_new_page: +check_new_slab: if (kmem_cache_debug(s)) { if (!alloc_debug_processing(s, slab, freelist, addr)) { @@ -3055,7 +3054,7 @@ check_new_page: */ goto return_single; -retry_load_page: +retry_load_slab: local_lock_irqsave(&s->cpu_slab->lock, flags); if (unlikely(c->slab)) { @@ -3072,7 +3071,7 @@ retry_load_page: stat(s, CPUSLAB_FLUSH); - goto retry_load_page; + goto retry_load_slab; } c->slab = slab; @@ -3169,9 +3168,9 @@ redo: /* * Irqless object alloc/free algorithm used here depends on sequence * of fetching cpu_slab's data. tid should be fetched before anything - * on c to guarantee that object and page associated with previous tid + * on c to guarantee that object and slab associated with previous tid * won't be used with current tid. If we fetch tid first, object and - * page could be one associated with next tid and our alloc/free + * slab could be one associated with next tid and our alloc/free * request will be failed. In this case, we will retry. So, no problem. */ barrier(); @@ -3295,7 +3294,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_trace); * have a longer lifetime than the cpu slabs in most processing loads. * * So we still attempt to reduce cache line usage. Just take the slab - * lock and free the item. If there is no additional partial page + * lock and free the item. If there is no additional partial slab * handling required then we can return immediately. */ static void __slab_free(struct kmem_cache *s, struct slab *slab, @@ -3373,7 +3372,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, stat(s, FREE_FROZEN); } else if (new.frozen) { /* - * If we just froze the page then put it onto the + * If we just froze the slab then put it onto the * per cpu partial list. */ put_cpu_partial(s, slab, 1); @@ -3427,7 +3426,7 @@ slab_empty: * with all sorts of special processing. * * Bulk free of a freelist with several objects (all pointing to the - * same page) possible by specifying head and tail ptr, plus objects + * same slab) possible by specifying head and tail ptr, plus objects * count (cnt). Bulk free indicated by tail pointer being set. */ static __always_inline void do_slab_free(struct kmem_cache *s, @@ -4213,7 +4212,7 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) #endif /* - * The larger the object size is, the more pages we want on the partial + * The larger the object size is, the more slabs we want on the partial * list to avoid pounding the page allocator excessively. */ set_min_partial(s, ilog2(s->size) / 2); @@ -4598,12 +4597,12 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) * Build lists of slabs to discard or promote. * * Note that concurrent frees may occur while we hold the - * list_lock. page->inuse here is the upper limit. + * list_lock. slab->inuse here is the upper limit. */ list_for_each_entry_safe(slab, t, &n->partial, slab_list) { int free = slab->objects - slab->inuse; - /* Do not reread page->inuse */ + /* Do not reread slab->inuse */ barrier(); /* We do not keep full slabs on the list */ @@ -5482,7 +5481,7 @@ static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) slabs += slab->slabs; } - /* Approximate half-full pages , see slub_set_cpu_partial() */ + /* Approximate half-full slabs, see slub_set_cpu_partial() */ objects = (slabs * oo_objects(s->oo)) / 2; len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); -- cgit v1.2.3-59-g8ed1b From 40f3bf0cb04c91d33531b1b95788ad2f0e4062cf Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Tue, 2 Nov 2021 15:42:04 +0100 Subject: mm: Convert struct page to struct slab in functions used by other subsystems KASAN, KFENCE and memcg interact with SLAB or SLUB internals through functions nearest_obj(), obj_to_index() and objs_per_slab() that use struct page as parameter. This patch converts it to struct slab including all callers, through a coccinelle semantic patch. // Options: --include-headers --no-includes --smpl-spacing include/linux/slab_def.h include/linux/slub_def.h mm/slab.h mm/kasan/*.c mm/kfence/kfence_test.c mm/memcontrol.c mm/slab.c mm/slub.c // Note: needs coccinelle 1.1.1 to avoid breaking whitespace @@ @@ -objs_per_slab_page( +objs_per_slab( ... ) { ... } @@ @@ -objs_per_slab_page( +objs_per_slab( ... ) @@ identifier fn =~ "obj_to_index|objs_per_slab"; @@ fn(..., - const struct page *page + const struct slab *slab ,...) { <... ( - page_address(page) + slab_address(slab) | - page + slab ) ...> } @@ identifier fn =~ "nearest_obj"; @@ fn(..., - struct page *page + const struct slab *slab ,...) { <... ( - page_address(page) + slab_address(slab) | - page + slab ) ...> } @@ identifier fn =~ "nearest_obj|obj_to_index|objs_per_slab"; expression E; @@ fn(..., ( - slab_page(E) + E | - virt_to_page(E) + virt_to_slab(E) | - virt_to_head_page(E) + virt_to_slab(E) | - page + page_slab(page) ) ,...) Signed-off-by: Vlastimil Babka Reviewed-by: Andrey Konovalov Reviewed-by: Roman Gushchin Acked-by: Johannes Weiner Cc: Julia Lawall Cc: Luis Chamberlain Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Marco Elver Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Cc: Cc: --- include/linux/slab_def.h | 16 ++++++++-------- include/linux/slub_def.h | 18 +++++++++--------- mm/kasan/common.c | 4 ++-- mm/kasan/generic.c | 2 +- mm/kasan/report.c | 2 +- mm/kasan/report_tags.c | 2 +- mm/kfence/kfence_test.c | 4 ++-- mm/memcontrol.c | 4 ++-- mm/slab.c | 10 +++++----- mm/slab.h | 4 ++-- mm/slub.c | 2 +- 11 files changed, 34 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 3aa5e1e73ab6..e24c9aff6fed 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -87,11 +87,11 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, +static inline void *nearest_obj(struct kmem_cache *cache, const struct slab *slab, void *x) { - void *object = x - (x - page->s_mem) % cache->size; - void *last_object = page->s_mem + (cache->num - 1) * cache->size; + void *object = x - (x - slab->s_mem) % cache->size; + void *last_object = slab->s_mem + (cache->num - 1) * cache->size; if (unlikely(object > last_object)) return last_object; @@ -106,16 +106,16 @@ static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, * reciprocal_divide(offset, cache->reciprocal_buffer_size) */ static inline unsigned int obj_to_index(const struct kmem_cache *cache, - const struct page *page, void *obj) + const struct slab *slab, void *obj) { - u32 offset = (obj - page->s_mem); + u32 offset = (obj - slab->s_mem); return reciprocal_divide(offset, cache->reciprocal_buffer_size); } -static inline int objs_per_slab_page(const struct kmem_cache *cache, - const struct page *page) +static inline int objs_per_slab(const struct kmem_cache *cache, + const struct slab *slab) { - if (is_kfence_address(page_address(page))) + if (is_kfence_address(slab_address(slab))) return 1; return cache->num; } diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 8a9c2876ca89..33c5c0e3bd8d 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -158,11 +158,11 @@ static inline void sysfs_slab_release(struct kmem_cache *s) void *fixup_red_left(struct kmem_cache *s, void *p); -static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, +static inline void *nearest_obj(struct kmem_cache *cache, const struct slab *slab, void *x) { - void *object = x - (x - page_address(page)) % cache->size; - void *last_object = page_address(page) + - (page->objects - 1) * cache->size; + void *object = x - (x - slab_address(slab)) % cache->size; + void *last_object = slab_address(slab) + + (slab->objects - 1) * cache->size; void *result = (unlikely(object > last_object)) ? last_object : object; result = fixup_red_left(cache, result); @@ -178,16 +178,16 @@ static inline unsigned int __obj_to_index(const struct kmem_cache *cache, } static inline unsigned int obj_to_index(const struct kmem_cache *cache, - const struct page *page, void *obj) + const struct slab *slab, void *obj) { if (is_kfence_address(obj)) return 0; - return __obj_to_index(cache, page_address(page), obj); + return __obj_to_index(cache, slab_address(slab), obj); } -static inline int objs_per_slab_page(const struct kmem_cache *cache, - const struct page *page) +static inline int objs_per_slab(const struct kmem_cache *cache, + const struct slab *slab) { - return page->objects; + return slab->objects; } #endif /* _LINUX_SLUB_DEF_H */ diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 8428da2aaf17..6a1cd2d38bff 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -298,7 +298,7 @@ static inline u8 assign_tag(struct kmem_cache *cache, /* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */ #ifdef CONFIG_SLAB /* For SLAB assign tags based on the object index in the freelist. */ - return (u8)obj_to_index(cache, virt_to_head_page(object), (void *)object); + return (u8)obj_to_index(cache, virt_to_slab(object), (void *)object); #else /* * For SLUB assign a random tag during slab creation, otherwise reuse @@ -341,7 +341,7 @@ static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object, if (is_kfence_address(object)) return false; - if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) != + if (unlikely(nearest_obj(cache, virt_to_slab(object), object) != object)) { kasan_report_invalid_free(tagged_object, ip); return true; diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 84a038b07c6f..5d0b79416c4e 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -339,7 +339,7 @@ static void __kasan_record_aux_stack(void *addr, bool can_alloc) return; cache = page->slab_cache; - object = nearest_obj(cache, page, addr); + object = nearest_obj(cache, page_slab(page), addr); alloc_meta = kasan_get_alloc_meta(cache, object); if (!alloc_meta) return; diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 0bc10f452f7e..e00999dc6499 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -249,7 +249,7 @@ static void print_address_description(void *addr, u8 tag) if (page && PageSlab(page)) { struct kmem_cache *cache = page->slab_cache; - void *object = nearest_obj(cache, page, addr); + void *object = nearest_obj(cache, page_slab(page), addr); describe_object(cache, object, addr, tag); } diff --git a/mm/kasan/report_tags.c b/mm/kasan/report_tags.c index 8a319fc16dab..06c21dd77493 100644 --- a/mm/kasan/report_tags.c +++ b/mm/kasan/report_tags.c @@ -23,7 +23,7 @@ const char *kasan_get_bug_type(struct kasan_access_info *info) page = kasan_addr_to_page(addr); if (page && PageSlab(page)) { cache = page->slab_cache; - object = nearest_obj(cache, page, (void *)addr); + object = nearest_obj(cache, page_slab(page), (void *)addr); alloc_meta = kasan_get_alloc_meta(cache, object); if (alloc_meta) { diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c index 695030c1fff8..f7276711d7b9 100644 --- a/mm/kfence/kfence_test.c +++ b/mm/kfence/kfence_test.c @@ -291,8 +291,8 @@ static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocat * even for KFENCE objects; these are required so that * memcg accounting works correctly. */ - KUNIT_EXPECT_EQ(test, obj_to_index(s, page, alloc), 0U); - KUNIT_EXPECT_EQ(test, objs_per_slab_page(s, page), 1); + KUNIT_EXPECT_EQ(test, obj_to_index(s, page_slab(page), alloc), 0U); + KUNIT_EXPECT_EQ(test, objs_per_slab(s, page_slab(page)), 1); if (policy == ALLOCATE_ANY) return alloc; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2ed5f2a0879d..f7b789e692a0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2819,7 +2819,7 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, gfp_t gfp, bool new_page) { - unsigned int objects = objs_per_slab_page(s, page); + unsigned int objects = objs_per_slab(s, page_slab(page)); unsigned long memcg_data; void *vec; @@ -2881,7 +2881,7 @@ struct mem_cgroup *mem_cgroup_from_obj(void *p) struct obj_cgroup *objcg; unsigned int off; - off = obj_to_index(page->slab_cache, page, p); + off = obj_to_index(page->slab_cache, page_slab(page), p); objcg = page_objcgs(page)[off]; if (objcg) return obj_cgroup_memcg(objcg); diff --git a/mm/slab.c b/mm/slab.c index 547ed068a569..c13258116791 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1559,7 +1559,7 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) struct slab *slab = virt_to_slab(objp); unsigned int objnr; - objnr = obj_to_index(cachep, slab_page(slab), objp); + objnr = obj_to_index(cachep, slab, objp); if (objnr) { objp = index_to_obj(cachep, slab, objnr - 1); realobj = (char *)objp + obj_offset(cachep); @@ -2529,7 +2529,7 @@ static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slab) static void slab_put_obj(struct kmem_cache *cachep, struct slab *slab, void *objp) { - unsigned int objnr = obj_to_index(cachep, slab_page(slab), objp); + unsigned int objnr = obj_to_index(cachep, slab, objp); #if DEBUG unsigned int i; @@ -2716,7 +2716,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, if (cachep->flags & SLAB_STORE_USER) *dbg_userword(cachep, objp) = (void *)caller; - objnr = obj_to_index(cachep, slab_page(slab), objp); + objnr = obj_to_index(cachep, slab, objp); BUG_ON(objnr >= cachep->num); BUG_ON(objp != index_to_obj(cachep, slab, objnr)); @@ -3662,7 +3662,7 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) objp = object - obj_offset(cachep); kpp->kp_data_offset = obj_offset(cachep); slab = virt_to_slab(objp); - objnr = obj_to_index(cachep, slab_page(slab), objp); + objnr = obj_to_index(cachep, slab, objp); objp = index_to_obj(cachep, slab, objnr); kpp->kp_objp = objp; if (DEBUG && cachep->flags & SLAB_STORE_USER) @@ -4180,7 +4180,7 @@ void __check_heap_object(const void *ptr, unsigned long n, /* Find and validate object. */ cachep = slab->slab_cache; - objnr = obj_to_index(cachep, slab_page(slab), (void *)ptr); + objnr = obj_to_index(cachep, slab, (void *)ptr); BUG_ON(objnr >= cachep->num); /* Find offset within object. */ diff --git a/mm/slab.h b/mm/slab.h index 039babfde2fe..bca9181e96d7 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -483,7 +483,7 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, continue; } - off = obj_to_index(s, page, p[i]); + off = obj_to_index(s, page_slab(page), p[i]); obj_cgroup_get(objcg); page_objcgs(page)[off] = objcg; mod_objcg_state(objcg, page_pgdat(page), @@ -522,7 +522,7 @@ static inline void memcg_slab_free_hook(struct kmem_cache *s_orig, else s = s_orig; - off = obj_to_index(s, page, p[i]); + off = obj_to_index(s, page_slab(page), p[i]); objcg = objcgs[off]; if (!objcg) continue; diff --git a/mm/slub.c b/mm/slub.c index cc64ba9d9963..ddf21c7a381a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4342,7 +4342,7 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) #else objp = objp0; #endif - objnr = obj_to_index(s, slab_page(slab), objp); + objnr = obj_to_index(s, slab, objp); kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); objp = base + s->size * objnr; kpp->kp_objp = objp; -- cgit v1.2.3-59-g8ed1b From 4b5f8d9a895ada8e0abb58ccd35d9fe229e3a595 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Tue, 2 Nov 2021 22:42:04 +0100 Subject: mm/memcg: Convert slab objcgs from struct page to struct slab page->memcg_data is used with MEMCG_DATA_OBJCGS flag only for slab pages so convert all the related infrastructure to struct slab. Also use struct folio instead of struct page when resolving object pointers. This is not just mechanistic changing of types and names. Now in mem_cgroup_from_obj() we use folio_test_slab() to decide if we interpret the folio as a real slab instead of a large kmalloc, instead of relying on MEMCG_DATA_OBJCGS bit that used to be checked in page_objcgs_check(). Similarly in memcg_slab_free_hook() where we can encounter kmalloc_large() pages (here the folio slab flag check is implied by virt_to_slab()). As a result, page_objcgs_check() can be dropped instead of converted. To avoid include cycles, move the inline definition of slab_objcgs() from memcontrol.h to mm/slab.h. Signed-off-by: Vlastimil Babka Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Cc: --- include/linux/memcontrol.h | 48 ------------------------------ mm/memcontrol.c | 55 ++++++++++++++++++---------------- mm/slab.h | 73 +++++++++++++++++++++++++++++++--------------- 3 files changed, 80 insertions(+), 96 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 0c5c403f4be6..e34112f6a369 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -536,45 +536,6 @@ static inline bool folio_memcg_kmem(struct folio *folio) return folio->memcg_data & MEMCG_DATA_KMEM; } -/* - * page_objcgs - get the object cgroups vector associated with a page - * @page: a pointer to the page struct - * - * Returns a pointer to the object cgroups vector associated with the page, - * or NULL. This function assumes that the page is known to have an - * associated object cgroups vector. It's not safe to call this function - * against pages, which might have an associated memory cgroup: e.g. - * kernel stack pages. - */ -static inline struct obj_cgroup **page_objcgs(struct page *page) -{ - unsigned long memcg_data = READ_ONCE(page->memcg_data); - - VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), page); - VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page); - - return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); -} - -/* - * page_objcgs_check - get the object cgroups vector associated with a page - * @page: a pointer to the page struct - * - * Returns a pointer to the object cgroups vector associated with the page, - * or NULL. This function is safe to use if the page can be directly associated - * with a memory cgroup. - */ -static inline struct obj_cgroup **page_objcgs_check(struct page *page) -{ - unsigned long memcg_data = READ_ONCE(page->memcg_data); - - if (!memcg_data || !(memcg_data & MEMCG_DATA_OBJCGS)) - return NULL; - - VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page); - - return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); -} #else static inline bool folio_memcg_kmem(struct folio *folio) @@ -582,15 +543,6 @@ static inline bool folio_memcg_kmem(struct folio *folio) return false; } -static inline struct obj_cgroup **page_objcgs(struct page *page) -{ - return NULL; -} - -static inline struct obj_cgroup **page_objcgs_check(struct page *page) -{ - return NULL; -} #endif static inline bool PageMemcgKmem(struct page *page) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index f7b789e692a0..4a7b3ebf8e48 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2816,31 +2816,31 @@ static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, rcu_read_unlock(); } -int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, - gfp_t gfp, bool new_page) +int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab) { - unsigned int objects = objs_per_slab(s, page_slab(page)); + unsigned int objects = objs_per_slab(s, slab); unsigned long memcg_data; void *vec; gfp &= ~OBJCGS_CLEAR_MASK; vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, - page_to_nid(page)); + slab_nid(slab)); if (!vec) return -ENOMEM; memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; - if (new_page) { + if (new_slab) { /* - * If the slab page is brand new and nobody can yet access - * it's memcg_data, no synchronization is required and - * memcg_data can be simply assigned. + * If the slab is brand new and nobody can yet access its + * memcg_data, no synchronization is required and memcg_data can + * be simply assigned. */ - page->memcg_data = memcg_data; - } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) { + slab->memcg_data = memcg_data; + } else if (cmpxchg(&slab->memcg_data, 0, memcg_data)) { /* - * If the slab page is already in use, somebody can allocate - * and assign obj_cgroups in parallel. In this case the existing + * If the slab is already in use, somebody can allocate and + * assign obj_cgroups in parallel. In this case the existing * objcg vector should be reused. */ kfree(vec); @@ -2865,38 +2865,43 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, */ struct mem_cgroup *mem_cgroup_from_obj(void *p) { - struct page *page; + struct folio *folio; if (mem_cgroup_disabled()) return NULL; - page = virt_to_head_page(p); + folio = virt_to_folio(p); /* * Slab objects are accounted individually, not per-page. * Memcg membership data for each individual object is saved in - * the page->obj_cgroups. + * slab->memcg_data. */ - if (page_objcgs_check(page)) { - struct obj_cgroup *objcg; + if (folio_test_slab(folio)) { + struct obj_cgroup **objcgs; + struct slab *slab; unsigned int off; - off = obj_to_index(page->slab_cache, page_slab(page), p); - objcg = page_objcgs(page)[off]; - if (objcg) - return obj_cgroup_memcg(objcg); + slab = folio_slab(folio); + objcgs = slab_objcgs(slab); + if (!objcgs) + return NULL; + + off = obj_to_index(slab->slab_cache, slab, p); + if (objcgs[off]) + return obj_cgroup_memcg(objcgs[off]); return NULL; } /* - * page_memcg_check() is used here, because page_has_obj_cgroups() - * check above could fail because the object cgroups vector wasn't set - * at that moment, but it can be set concurrently. + * page_memcg_check() is used here, because in theory we can encounter + * a folio where the slab flag has been cleared already, but + * slab->memcg_data has not been freed yet * page_memcg_check(page) will guarantee that a proper memory * cgroup pointer or NULL will be returned. */ - return page_memcg_check(page); + return page_memcg_check(folio_page(folio, 0)); } __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void) diff --git a/mm/slab.h b/mm/slab.h index bca9181e96d7..b8d7ed42cca8 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -412,15 +412,33 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla } #ifdef CONFIG_MEMCG_KMEM -int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, - gfp_t gfp, bool new_page); +/* + * slab_objcgs - get the object cgroups vector associated with a slab + * @slab: a pointer to the slab struct + * + * Returns a pointer to the object cgroups vector associated with the slab, + * or NULL if no such vector has been associated yet. + */ +static inline struct obj_cgroup **slab_objcgs(struct slab *slab) +{ + unsigned long memcg_data = READ_ONCE(slab->memcg_data); + + VM_BUG_ON_PAGE(memcg_data && !(memcg_data & MEMCG_DATA_OBJCGS), + slab_page(slab)); + VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, slab_page(slab)); + + return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); +} + +int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, + gfp_t gfp, bool new_slab); void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr); -static inline void memcg_free_page_obj_cgroups(struct page *page) +static inline void memcg_free_slab_cgroups(struct slab *slab) { - kfree(page_objcgs(page)); - page->memcg_data = 0; + kfree(slab_objcgs(slab)); + slab->memcg_data = 0; } static inline size_t obj_full_size(struct kmem_cache *s) @@ -465,7 +483,7 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size, void **p) { - struct page *page; + struct slab *slab; unsigned long off; size_t i; @@ -474,19 +492,19 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, for (i = 0; i < size; i++) { if (likely(p[i])) { - page = virt_to_head_page(p[i]); + slab = virt_to_slab(p[i]); - if (!page_objcgs(page) && - memcg_alloc_page_obj_cgroups(page, s, flags, + if (!slab_objcgs(slab) && + memcg_alloc_slab_cgroups(slab, s, flags, false)) { obj_cgroup_uncharge(objcg, obj_full_size(s)); continue; } - off = obj_to_index(s, page_slab(page), p[i]); + off = obj_to_index(s, slab, p[i]); obj_cgroup_get(objcg); - page_objcgs(page)[off] = objcg; - mod_objcg_state(objcg, page_pgdat(page), + slab_objcgs(slab)[off] = objcg; + mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), obj_full_size(s)); } else { obj_cgroup_uncharge(objcg, obj_full_size(s)); @@ -501,7 +519,7 @@ static inline void memcg_slab_free_hook(struct kmem_cache *s_orig, struct kmem_cache *s; struct obj_cgroup **objcgs; struct obj_cgroup *objcg; - struct page *page; + struct slab *slab; unsigned int off; int i; @@ -512,43 +530,52 @@ static inline void memcg_slab_free_hook(struct kmem_cache *s_orig, if (unlikely(!p[i])) continue; - page = virt_to_head_page(p[i]); - objcgs = page_objcgs_check(page); + slab = virt_to_slab(p[i]); + /* we could be given a kmalloc_large() object, skip those */ + if (!slab) + continue; + + objcgs = slab_objcgs(slab); if (!objcgs) continue; if (!s_orig) - s = page->slab_cache; + s = slab->slab_cache; else s = s_orig; - off = obj_to_index(s, page_slab(page), p[i]); + off = obj_to_index(s, slab, p[i]); objcg = objcgs[off]; if (!objcg) continue; objcgs[off] = NULL; obj_cgroup_uncharge(objcg, obj_full_size(s)); - mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s), + mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s), -obj_full_size(s)); obj_cgroup_put(objcg); } } #else /* CONFIG_MEMCG_KMEM */ +static inline struct obj_cgroup **slab_objcgs(struct slab *slab) +{ + return NULL; +} + static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) { return NULL; } -static inline int memcg_alloc_page_obj_cgroups(struct page *page, +static inline int memcg_alloc_slab_cgroups(struct slab *slab, struct kmem_cache *s, gfp_t gfp, - bool new_page) + bool new_slab) { return 0; } -static inline void memcg_free_page_obj_cgroups(struct page *page) +static inline void memcg_free_slab_cgroups(struct slab *slab) { } @@ -587,7 +614,7 @@ static __always_inline void account_slab(struct slab *slab, int order, struct kmem_cache *s, gfp_t gfp) { if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT)) - memcg_alloc_page_obj_cgroups(slab_page(slab), s, gfp, true); + memcg_alloc_slab_cgroups(slab, s, gfp, true); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), PAGE_SIZE << order); @@ -597,7 +624,7 @@ static __always_inline void unaccount_slab(struct slab *slab, int order, struct kmem_cache *s) { if (memcg_kmem_enabled()) - memcg_free_page_obj_cgroups(slab_page(slab)); + memcg_free_slab_cgroups(slab); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), -(PAGE_SIZE << order)); -- cgit v1.2.3-59-g8ed1b From 6e48a966dfd18987fec9385566a67d36e2b5fc11 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Oct 2021 14:46:46 +0100 Subject: mm/kasan: Convert to struct folio and struct slab KASAN accesses some slab related struct page fields so we need to convert it to struct slab. Some places are a bit simplified thanks to kasan_addr_to_slab() encapsulating the PageSlab flag check through virt_to_slab(). When resolving object address to either a real slab or a large kmalloc, use struct folio as the intermediate type for testing the slab flag to avoid unnecessary implicit compound_head(). [ vbabka@suse.cz: use struct folio, adjust to differences in previous patches ] Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka Reviewed-by: Andrey Konovalov Reviewed-by: Roman Gushchin Tested-by: Hyeongogn Yoo <42.hyeyoo@gmail.com> Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: --- include/linux/kasan.h | 9 +++++---- mm/kasan/common.c | 23 +++++++++++++---------- mm/kasan/generic.c | 8 ++++---- mm/kasan/kasan.h | 1 + mm/kasan/quarantine.c | 2 +- mm/kasan/report.c | 13 +++++++++++-- mm/kasan/report_tags.c | 10 +++++----- mm/slab.c | 2 +- mm/slub.c | 2 +- 9 files changed, 42 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kasan.h b/include/linux/kasan.h index d8783b682669..fb78108d694e 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -9,6 +9,7 @@ struct kmem_cache; struct page; +struct slab; struct vm_struct; struct task_struct; @@ -193,11 +194,11 @@ static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache) return 0; } -void __kasan_poison_slab(struct page *page); -static __always_inline void kasan_poison_slab(struct page *page) +void __kasan_poison_slab(struct slab *slab); +static __always_inline void kasan_poison_slab(struct slab *slab) { if (kasan_enabled()) - __kasan_poison_slab(page); + __kasan_poison_slab(slab); } void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object); @@ -322,7 +323,7 @@ static inline void kasan_cache_create(struct kmem_cache *cache, slab_flags_t *flags) {} static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {} static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; } -static inline void kasan_poison_slab(struct page *page) {} +static inline void kasan_poison_slab(struct slab *slab) {} static inline void kasan_unpoison_object_data(struct kmem_cache *cache, void *object) {} static inline void kasan_poison_object_data(struct kmem_cache *cache, diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 6a1cd2d38bff..7c06db78a76c 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -247,8 +247,9 @@ struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, } #endif -void __kasan_poison_slab(struct page *page) +void __kasan_poison_slab(struct slab *slab) { + struct page *page = slab_page(slab); unsigned long i; for (i = 0; i < compound_nr(page); i++) @@ -401,9 +402,9 @@ void __kasan_kfree_large(void *ptr, unsigned long ip) void __kasan_slab_free_mempool(void *ptr, unsigned long ip) { - struct page *page; + struct folio *folio; - page = virt_to_head_page(ptr); + folio = virt_to_folio(ptr); /* * Even though this function is only called for kmem_cache_alloc and @@ -411,12 +412,14 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip) * !PageSlab() when the size provided to kmalloc is larger than * KMALLOC_MAX_SIZE, and kmalloc falls back onto page_alloc. */ - if (unlikely(!PageSlab(page))) { + if (unlikely(!folio_test_slab(folio))) { if (____kasan_kfree_large(ptr, ip)) return; - kasan_poison(ptr, page_size(page), KASAN_FREE_PAGE, false); + kasan_poison(ptr, folio_size(folio), KASAN_FREE_PAGE, false); } else { - ____kasan_slab_free(page->slab_cache, ptr, ip, false, false); + struct slab *slab = folio_slab(folio); + + ____kasan_slab_free(slab->slab_cache, ptr, ip, false, false); } } @@ -560,7 +563,7 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags) { - struct page *page; + struct slab *slab; if (unlikely(object == ZERO_SIZE_PTR)) return (void *)object; @@ -572,13 +575,13 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag */ kasan_unpoison(object, size, false); - page = virt_to_head_page(object); + slab = virt_to_slab(object); /* Piggy-back on kmalloc() instrumentation to poison the redzone. */ - if (unlikely(!PageSlab(page))) + if (unlikely(!slab)) return __kasan_kmalloc_large(object, size, flags); else - return ____kasan_kmalloc(page->slab_cache, object, size, flags); + return ____kasan_kmalloc(slab->slab_cache, object, size, flags); } bool __kasan_check_byte(const void *address, unsigned long ip) diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 5d0b79416c4e..a25ad4090615 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -330,16 +330,16 @@ DEFINE_ASAN_SET_SHADOW(f8); static void __kasan_record_aux_stack(void *addr, bool can_alloc) { - struct page *page = kasan_addr_to_page(addr); + struct slab *slab = kasan_addr_to_slab(addr); struct kmem_cache *cache; struct kasan_alloc_meta *alloc_meta; void *object; - if (is_kfence_address(addr) || !(page && PageSlab(page))) + if (is_kfence_address(addr) || !slab) return; - cache = page->slab_cache; - object = nearest_obj(cache, page_slab(page), addr); + cache = slab->slab_cache; + object = nearest_obj(cache, slab, addr); alloc_meta = kasan_get_alloc_meta(cache, object); if (!alloc_meta) return; diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index aebd8df86a1f..c17fa8d26ffe 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -265,6 +265,7 @@ bool kasan_report(unsigned long addr, size_t size, void kasan_report_invalid_free(void *object, unsigned long ip); struct page *kasan_addr_to_page(const void *addr); +struct slab *kasan_addr_to_slab(const void *addr); depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc); void kasan_set_track(struct kasan_track *track, gfp_t flags); diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index d8ccff4c1275..587da8995f2d 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -117,7 +117,7 @@ static unsigned long quarantine_batch_size; static struct kmem_cache *qlink_to_cache(struct qlist_node *qlink) { - return virt_to_head_page(qlink)->slab_cache; + return virt_to_slab(qlink)->slab_cache; } static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache) diff --git a/mm/kasan/report.c b/mm/kasan/report.c index e00999dc6499..3ad9624dcc56 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -150,6 +150,14 @@ struct page *kasan_addr_to_page(const void *addr) return NULL; } +struct slab *kasan_addr_to_slab(const void *addr) +{ + if ((addr >= (void *)PAGE_OFFSET) && + (addr < high_memory)) + return virt_to_slab(addr); + return NULL; +} + static void describe_object_addr(struct kmem_cache *cache, void *object, const void *addr) { @@ -248,8 +256,9 @@ static void print_address_description(void *addr, u8 tag) pr_err("\n"); if (page && PageSlab(page)) { - struct kmem_cache *cache = page->slab_cache; - void *object = nearest_obj(cache, page_slab(page), addr); + struct slab *slab = page_slab(page); + struct kmem_cache *cache = slab->slab_cache; + void *object = nearest_obj(cache, slab, addr); describe_object(cache, object, addr, tag); } diff --git a/mm/kasan/report_tags.c b/mm/kasan/report_tags.c index 06c21dd77493..1b41de88c53e 100644 --- a/mm/kasan/report_tags.c +++ b/mm/kasan/report_tags.c @@ -12,7 +12,7 @@ const char *kasan_get_bug_type(struct kasan_access_info *info) #ifdef CONFIG_KASAN_TAGS_IDENTIFY struct kasan_alloc_meta *alloc_meta; struct kmem_cache *cache; - struct page *page; + struct slab *slab; const void *addr; void *object; u8 tag; @@ -20,10 +20,10 @@ const char *kasan_get_bug_type(struct kasan_access_info *info) tag = get_tag(info->access_addr); addr = kasan_reset_tag(info->access_addr); - page = kasan_addr_to_page(addr); - if (page && PageSlab(page)) { - cache = page->slab_cache; - object = nearest_obj(cache, page_slab(page), (void *)addr); + slab = kasan_addr_to_slab(addr); + if (slab) { + cache = slab->slab_cache; + object = nearest_obj(cache, slab, (void *)addr); alloc_meta = kasan_get_alloc_meta(cache, object); if (alloc_meta) { diff --git a/mm/slab.c b/mm/slab.c index c13258116791..ddf5737c63d9 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2604,7 +2604,7 @@ static struct slab *cache_grow_begin(struct kmem_cache *cachep, * page_address() in the latter returns a non-tagged pointer, * as it should be for slab pages. */ - kasan_poison_slab(slab_page(slab)); + kasan_poison_slab(slab); /* Get slab management. */ freelist = alloc_slabmgmt(cachep, slab, offset, diff --git a/mm/slub.c b/mm/slub.c index ddf21c7a381a..d08ba1025aae 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1961,7 +1961,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->slab_cache = s; - kasan_poison_slab(slab_page(slab)); + kasan_poison_slab(slab); start = slab_address(slab); -- cgit v1.2.3-59-g8ed1b From c5e97ed154589524a1df4ae2be55c4cfdb0d0573 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Oct 2021 14:46:48 +0100 Subject: bootmem: Use page->index instead of page->freelist page->freelist is for the use of slab. Using page->index is the same set of bits as page->freelist, and by using an integer instead of a pointer, we can avoid casts. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka Acked-by: Johannes Weiner Cc: Dave Hansen Cc: Andy Lutomirski Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Cc: "H. Peter Anvin" --- arch/x86/mm/init_64.c | 2 +- include/linux/bootmem_info.h | 2 +- mm/bootmem_info.c | 7 +++---- mm/sparse.c | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 36098226a957..96d34ebb20a9 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -981,7 +981,7 @@ static void __meminit free_pagetable(struct page *page, int order) if (PageReserved(page)) { __ClearPageReserved(page); - magic = (unsigned long)page->freelist; + magic = page->index; if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) { while (nr_pages--) put_page_bootmem(page++); diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h index 2bc8b1f69c93..cc35d010fa94 100644 --- a/include/linux/bootmem_info.h +++ b/include/linux/bootmem_info.h @@ -30,7 +30,7 @@ void put_page_bootmem(struct page *page); */ static inline void free_bootmem_page(struct page *page) { - unsigned long magic = (unsigned long)page->freelist; + unsigned long magic = page->index; /* * The reserve_bootmem_region sets the reserved flag on bootmem diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c index f03f42f426f6..f18a631e7479 100644 --- a/mm/bootmem_info.c +++ b/mm/bootmem_info.c @@ -15,7 +15,7 @@ void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) { - page->freelist = (void *)type; + page->index = type; SetPagePrivate(page); set_page_private(page, info); page_ref_inc(page); @@ -23,14 +23,13 @@ void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) void put_page_bootmem(struct page *page) { - unsigned long type; + unsigned long type = page->index; - type = (unsigned long) page->freelist; BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); if (page_ref_dec_return(page) == 1) { - page->freelist = NULL; + page->index = 0; ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); diff --git a/mm/sparse.c b/mm/sparse.c index e5c84b0cf0c9..d21c6e5910d0 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -722,7 +722,7 @@ static void free_map_bootmem(struct page *memmap) >> PAGE_SHIFT; for (i = 0; i < nr_pages; i++, page++) { - magic = (unsigned long) page->freelist; + magic = page->index; BUG_ON(magic == NODE_INFO); -- cgit v1.2.3-59-g8ed1b From 007747a984ea5e895b7d8b056b24ebf431e1e71d Mon Sep 17 00:00:00 2001 From: Miroslav Lichvar Date: Wed, 5 Jan 2022 11:33:26 +0100 Subject: net: fix SOF_TIMESTAMPING_BIND_PHC to work with multiple sockets When multiple sockets using the SOF_TIMESTAMPING_BIND_PHC flag received a packet with a hardware timestamp (e.g. multiple PTP instances in different PTP domains using the UDPv4/v6 multicast or L2 transport), the timestamps received on some sockets were corrupted due to repeated conversion of the same timestamp (by the same or different vclocks). Fix ptp_convert_timestamp() to not modify the shared skb timestamp and return the converted timestamp as a ktime_t instead. If the conversion fails, return 0 to not confuse the application with timestamps corresponding to an unexpected PHC. Fixes: d7c088265588 ("net: socket: support hardware timestamp conversion to PHC bound") Signed-off-by: Miroslav Lichvar Cc: Yangbo Lu Cc: Richard Cochran Acked-by: Richard Cochran Signed-off-by: David S. Miller --- drivers/ptp/ptp_vclock.c | 10 +++++----- include/linux/ptp_clock_kernel.h | 12 +++++++----- net/socket.c | 9 ++++++--- 3 files changed, 18 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/ptp/ptp_vclock.c b/drivers/ptp/ptp_vclock.c index baee0379482b..ab1d233173e1 100644 --- a/drivers/ptp/ptp_vclock.c +++ b/drivers/ptp/ptp_vclock.c @@ -185,8 +185,8 @@ out: } EXPORT_SYMBOL(ptp_get_vclocks_index); -void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps, - int vclock_index) +ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps, + int vclock_index) { char name[PTP_CLOCK_NAME_LEN] = ""; struct ptp_vclock *vclock; @@ -198,12 +198,12 @@ void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps, snprintf(name, PTP_CLOCK_NAME_LEN, "ptp%d", vclock_index); dev = class_find_device_by_name(ptp_class, name); if (!dev) - return; + return 0; ptp = dev_get_drvdata(dev); if (!ptp->is_virtual_clock) { put_device(dev); - return; + return 0; } vclock = info_to_vclock(ptp->info); @@ -215,7 +215,7 @@ void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps, spin_unlock_irqrestore(&vclock->lock, flags); put_device(dev); - hwtstamps->hwtstamp = ns_to_ktime(ns); + return ns_to_ktime(ns); } EXPORT_SYMBOL(ptp_convert_timestamp); #endif diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index 2e5565067355..554454cb8693 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -351,15 +351,17 @@ int ptp_get_vclocks_index(int pclock_index, int **vclock_index); * * @hwtstamps: skb_shared_hwtstamps structure pointer * @vclock_index: phc index of ptp vclock. + * + * Returns converted timestamp, or 0 on error. */ -void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps, - int vclock_index); +ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps, + int vclock_index); #else static inline int ptp_get_vclocks_index(int pclock_index, int **vclock_index) { return 0; } -static inline void ptp_convert_timestamp(struct skb_shared_hwtstamps *hwtstamps, - int vclock_index) -{ } +static inline ktime_t ptp_convert_timestamp(const struct skb_shared_hwtstamps *hwtstamps, + int vclock_index) +{ return 0; } #endif diff --git a/net/socket.c b/net/socket.c index 5e644c858428..50cf75730fd7 100644 --- a/net/socket.c +++ b/net/socket.c @@ -830,6 +830,7 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk, int empty = 1, false_tstamp = 0; struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); + ktime_t hwtstamp; /* Race occurred between timestamp enabling and packet receiving. Fill in the current time for now. */ @@ -878,10 +879,12 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk, (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) && !skb_is_swtx_tstamp(skb, false_tstamp)) { if (sk->sk_tsflags & SOF_TIMESTAMPING_BIND_PHC) - ptp_convert_timestamp(shhwtstamps, sk->sk_bind_phc); + hwtstamp = ptp_convert_timestamp(shhwtstamps, + sk->sk_bind_phc); + else + hwtstamp = shhwtstamps->hwtstamp; - if (ktime_to_timespec64_cond(shhwtstamps->hwtstamp, - tss.ts + 2)) { + if (ktime_to_timespec64_cond(hwtstamp, tss.ts + 2)) { empty = 0; if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_PKTINFO) && -- cgit v1.2.3-59-g8ed1b From eac1b93c14d645ef147b049ace0d5230df755548 Mon Sep 17 00:00:00 2001 From: Coco Li Date: Wed, 5 Jan 2022 02:48:38 -0800 Subject: gro: add ability to control gro max packet size Eric Dumazet suggested to allow users to modify max GRO packet size. We have seen GRO being disabled by users of appliances (such as wifi access points) because of claimed bufferbloat issues, or some work arounds in sch_cake, to split GRO/GSO packets. Instead of disabling GRO completely, one can chose to limit the maximum packet size of GRO packets, depending on their latency constraints. This patch adds a per device gro_max_size attribute that can be changed with ip link command. ip link set dev eth0 gro_max_size 16000 Suggested-by: Eric Dumazet Signed-off-by: Coco Li Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/netdevice.h | 11 +++++++++++ include/uapi/linux/if_link.h | 1 + net/core/dev.c | 1 + net/core/gro.c | 6 +++++- net/core/rtnetlink.c | 22 ++++++++++++++++++++++ tools/include/uapi/linux/if_link.h | 1 + 6 files changed, 41 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 6f99c8f51b60..3213c7227b59 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1942,6 +1942,8 @@ enum netdev_ml_priv_type { * dev->addr_list_lock. * @unlink_list: As netif_addr_lock() can be called recursively, * keep a list of interfaces to be deleted. + * @gro_max_size: Maximum size of aggregated packet in generic + * receive offload (GRO) * * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. * @linkwatch_dev_tracker: refcount tracker used by linkwatch. @@ -2131,6 +2133,8 @@ struct net_device { struct bpf_prog __rcu *xdp_prog; unsigned long gro_flush_timeout; int napi_defer_hard_irqs; +#define GRO_MAX_SIZE 65536 + unsigned int gro_max_size; rx_handler_func_t __rcu *rx_handler; void __rcu *rx_handler_data; @@ -4806,6 +4810,13 @@ static inline void netif_set_gso_max_segs(struct net_device *dev, WRITE_ONCE(dev->gso_max_segs, segs); } +static inline void netif_set_gro_max_size(struct net_device *dev, + unsigned int size) +{ + /* This pairs with the READ_ONCE() in skb_gro_receive() */ + WRITE_ONCE(dev->gro_max_size, size); +} + static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol, int pulled_hlen, u16 mac_offset, int mac_len) diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index 4ac53b30b6dc..6218f93f5c1a 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -347,6 +347,7 @@ enum { */ IFLA_PARENT_DEV_NAME, IFLA_PARENT_DEV_BUS_NAME, + IFLA_GRO_MAX_SIZE, __IFLA_MAX }; diff --git a/net/core/dev.c b/net/core/dev.c index 6c8b226b5f2f..83a4089990a0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10180,6 +10180,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, dev->gso_max_size = GSO_MAX_SIZE; dev->gso_max_segs = GSO_MAX_SEGS; + dev->gro_max_size = GRO_MAX_SIZE; dev->upper_level = 1; dev->lower_level = 1; #ifdef CONFIG_LOCKDEP diff --git a/net/core/gro.c b/net/core/gro.c index 8ec8b44596da..a11b286d1495 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -132,10 +132,14 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) unsigned int headlen = skb_headlen(skb); unsigned int len = skb_gro_len(skb); unsigned int delta_truesize; + unsigned int gro_max_size; unsigned int new_truesize; struct sk_buff *lp; - if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) + /* pairs with WRITE_ONCE() in netif_set_gro_max_size() */ + gro_max_size = READ_ONCE(p->dev->gro_max_size); + + if (unlikely(p->len + len >= gro_max_size || NAPI_GRO_CB(skb)->flush)) return -E2BIG; lp = NAPI_GRO_CB(p)->last; diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index d6eba554b137..e476403231f0 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -1026,6 +1026,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + nla_total_size(4) /* IFLA_NUM_RX_QUEUES */ + nla_total_size(4) /* IFLA_GSO_MAX_SEGS */ + nla_total_size(4) /* IFLA_GSO_MAX_SIZE */ + + nla_total_size(4) /* IFLA_GRO_MAX_SIZE */ + nla_total_size(1) /* IFLA_OPERSTATE */ + nla_total_size(1) /* IFLA_LINKMODE */ + nla_total_size(4) /* IFLA_CARRIER_CHANGES */ @@ -1728,6 +1729,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, nla_put_u32(skb, IFLA_NUM_TX_QUEUES, dev->num_tx_queues) || nla_put_u32(skb, IFLA_GSO_MAX_SEGS, dev->gso_max_segs) || nla_put_u32(skb, IFLA_GSO_MAX_SIZE, dev->gso_max_size) || + nla_put_u32(skb, IFLA_GRO_MAX_SIZE, dev->gro_max_size) || #ifdef CONFIG_RPS nla_put_u32(skb, IFLA_NUM_RX_QUEUES, dev->num_rx_queues) || #endif @@ -1880,6 +1882,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_PROTO_DOWN_REASON] = { .type = NLA_NESTED }, [IFLA_NEW_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 1), [IFLA_PARENT_DEV_NAME] = { .type = NLA_NUL_STRING }, + [IFLA_GRO_MAX_SIZE] = { .type = NLA_U32 }, }; static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = { @@ -2299,6 +2302,14 @@ static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[], } } + if (tb[IFLA_GRO_MAX_SIZE]) { + u32 gro_max_size = nla_get_u32(tb[IFLA_GRO_MAX_SIZE]); + + if (gro_max_size > GRO_MAX_SIZE) { + NL_SET_ERR_MSG(extack, "too big gro_max_size"); + return -EINVAL; + } + } return 0; } @@ -2772,6 +2783,15 @@ static int do_setlink(const struct sk_buff *skb, } } + if (tb[IFLA_GRO_MAX_SIZE]) { + u32 gro_max_size = nla_get_u32(tb[IFLA_GRO_MAX_SIZE]); + + if (dev->gro_max_size ^ gro_max_size) { + netif_set_gro_max_size(dev, gro_max_size); + status |= DO_SETLINK_MODIFIED; + } + } + if (tb[IFLA_OPERSTATE]) set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE])); @@ -3222,6 +3242,8 @@ struct net_device *rtnl_create_link(struct net *net, const char *ifname, netif_set_gso_max_size(dev, nla_get_u32(tb[IFLA_GSO_MAX_SIZE])); if (tb[IFLA_GSO_MAX_SEGS]) netif_set_gso_max_segs(dev, nla_get_u32(tb[IFLA_GSO_MAX_SEGS])); + if (tb[IFLA_GRO_MAX_SIZE]) + netif_set_gro_max_size(dev, nla_get_u32(tb[IFLA_GRO_MAX_SIZE])); return dev; } diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index 4ac53b30b6dc..6218f93f5c1a 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -347,6 +347,7 @@ enum { */ IFLA_PARENT_DEV_NAME, IFLA_PARENT_DEV_BUS_NAME, + IFLA_GRO_MAX_SIZE, __IFLA_MAX }; -- cgit v1.2.3-59-g8ed1b From 3809fe479861194e310c23ed48b010c7c0f72d22 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 16 Dec 2021 10:21:57 +0100 Subject: HID: address kernel-doc warnings The command ./scripts/kernel-doc -none include/linux/hid.h reports: include/linux/hid.h:818: warning: cannot understand function prototype: 'struct hid_ll_driver ' include/linux/hid.h:1135: warning: expecting prototype for hid_may_wakeup(). Prototype was for hid_hw_may_wakeup() instead Address those kernel-doc warnings. Signed-off-by: Lukas Bulwahn Signed-off-by: Jiri Kosina --- include/linux/hid.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hid.h b/include/linux/hid.h index f453be385bd4..aaf89a8a836c 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -788,7 +788,7 @@ struct hid_driver { container_of(pdrv, struct hid_driver, driver) /** - * hid_ll_driver - low level driver callbacks + * struct hid_ll_driver - low level driver callbacks * @start: called on probe to start the device * @stop: called on remove * @open: called by input layer on open @@ -1158,7 +1158,7 @@ static inline int hid_hw_idle(struct hid_device *hdev, int report, int idle, } /** - * hid_may_wakeup - return if the hid device may act as a wakeup source during system-suspend + * hid_hw_may_wakeup - return if the hid device may act as a wakeup source during system-suspend * * @hdev: hid device */ -- cgit v1.2.3-59-g8ed1b From 36dacddbf0bdba86cd00f066b4d724157eeb63f1 Mon Sep 17 00:00:00 2001 From: Dirk Müller Date: Wed, 5 Jan 2022 17:38:47 +0100 Subject: lib/raid6: Use strict priority ranking for pq gen() benchmarking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On x86_64, currently 3 variants of AVX512, 3 variants of AVX2 and 3 variants of SSE2 are benchmarked on initialization, taking between 144-153 jiffies. Testing across a hardware pool of various generations of intel cpus I could not find a single case where SSE2 won over AVX2 or AVX512. There are cases where AVX2 wins over AVX512 however. Change "prefer" into an integer priority field (similar to how recov selection works) to have more than one ranking level available, which is backwards compatible with existing behavior. Give AVX2/512 variants higher priority over SSE2 in order to skip SSE testing when AVX is available. in a AVX2/x86_64/HZ=250 case this saves in the order of 200ms of initialization time. Signed-off-by: Dirk Müller Acked-by: Paul Menzel Signed-off-by: Song Liu --- include/linux/raid/pq.h | 2 +- lib/raid6/algos.c | 2 +- lib/raid6/avx2.c | 8 ++++---- lib/raid6/avx512.c | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 154e954b711d..d6e5a1feb947 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -81,7 +81,7 @@ struct raid6_calls { void (*xor_syndrome)(int, int, int, size_t, void **); int (*valid)(void); /* Returns 1 if this routine set is usable */ const char *name; /* Name of this routine set */ - int prefer; /* Has special performance attribute */ + int priority; /* Relative priority ranking if non-zero */ }; /* Selected algorithm */ diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 9b7e8a837b27..39b74221f4a7 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -151,7 +151,7 @@ static inline const struct raid6_calls *raid6_choose_gen( const struct raid6_calls *best; for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { - if (!best || (*algo)->prefer >= best->prefer) { + if (!best || (*algo)->priority >= best->priority) { if ((*algo)->valid && !(*algo)->valid()) continue; diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c index f299476e1d76..059024234dce 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid6/avx2.c @@ -132,7 +132,7 @@ const struct raid6_calls raid6_avx2x1 = { raid6_avx21_xor_syndrome, raid6_have_avx2, "avx2x1", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ }; /* @@ -262,7 +262,7 @@ const struct raid6_calls raid6_avx2x2 = { raid6_avx22_xor_syndrome, raid6_have_avx2, "avx2x2", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ }; #ifdef CONFIG_X86_64 @@ -465,6 +465,6 @@ const struct raid6_calls raid6_avx2x4 = { raid6_avx24_xor_syndrome, raid6_have_avx2, "avx2x4", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ }; -#endif +#endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/avx512.c b/lib/raid6/avx512.c index bb684d144ee2..9c3e822e1adf 100644 --- a/lib/raid6/avx512.c +++ b/lib/raid6/avx512.c @@ -162,7 +162,7 @@ const struct raid6_calls raid6_avx512x1 = { raid6_avx5121_xor_syndrome, raid6_have_avx512, "avx512x1", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ }; /* @@ -319,7 +319,7 @@ const struct raid6_calls raid6_avx512x2 = { raid6_avx5122_xor_syndrome, raid6_have_avx512, "avx512x2", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ }; #ifdef CONFIG_X86_64 @@ -557,7 +557,7 @@ const struct raid6_calls raid6_avx512x4 = { raid6_avx5124_xor_syndrome, raid6_have_avx512, "avx512x4", - 1 /* Has cache hints */ + .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ }; #endif -- cgit v1.2.3-59-g8ed1b From 07f910f9b7295b6a28b337fedb56e612684c5659 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 4 Oct 2021 14:46:50 +0100 Subject: mm: Remove slab from struct page All members of struct slab can now be removed from struct page. This shrinks the definition of struct page by 30 LOC, making it easier to understand. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka --- include/linux/mm_types.h | 28 ---------------------------- include/linux/page-flags.h | 37 ------------------------------------- mm/slab.h | 6 ------ 3 files changed, 71 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 1ae3537c7920..646f3ed4f6df 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -118,31 +118,6 @@ struct page { atomic_long_t pp_frag_count; }; }; - struct { /* slab, slob and slub */ - union { - struct list_head slab_list; - struct { /* Partial pages */ - struct page *next; -#ifdef CONFIG_64BIT - int pages; /* Nr of pages left */ -#else - short int pages; -#endif - }; - }; - struct kmem_cache *slab_cache; /* not slob */ - /* Double-word boundary */ - void *freelist; /* first free object */ - union { - void *s_mem; /* slab: first object */ - unsigned long counters; /* SLUB */ - struct { /* SLUB */ - unsigned inuse:16; - unsigned objects:15; - unsigned frozen:1; - }; - }; - }; struct { /* Tail pages of compound page */ unsigned long compound_head; /* Bit zero is set */ @@ -206,9 +181,6 @@ struct page { * which are currently stored here. */ unsigned int page_type; - - unsigned int active; /* SLAB */ - int units; /* SLOB */ }; /* Usage count. *DO NOT USE DIRECTLY*. See page_ref.h */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index b5f14d581113..1b08e33265fa 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -909,43 +909,6 @@ extern bool is_free_buddy_page(struct page *page); __PAGEFLAG(Isolated, isolated, PF_ANY); -/* - * If network-based swap is enabled, sl*b must keep track of whether pages - * were allocated from pfmemalloc reserves. - */ -static inline int PageSlabPfmemalloc(struct page *page) -{ - VM_BUG_ON_PAGE(!PageSlab(page), page); - return PageActive(page); -} - -/* - * A version of PageSlabPfmemalloc() for opportunistic checks where the page - * might have been freed under us and not be a PageSlab anymore. - */ -static inline int __PageSlabPfmemalloc(struct page *page) -{ - return PageActive(page); -} - -static inline void SetPageSlabPfmemalloc(struct page *page) -{ - VM_BUG_ON_PAGE(!PageSlab(page), page); - SetPageActive(page); -} - -static inline void __ClearPageSlabPfmemalloc(struct page *page) -{ - VM_BUG_ON_PAGE(!PageSlab(page), page); - __ClearPageActive(page); -} - -static inline void ClearPageSlabPfmemalloc(struct page *page) -{ - VM_BUG_ON_PAGE(!PageSlab(page), page); - ClearPageActive(page); -} - #ifdef CONFIG_MMU #define __PG_MLOCKED (1UL << PG_mlocked) #else diff --git a/mm/slab.h b/mm/slab.h index 95b9a74a2d51..207658b200ef 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -67,14 +67,8 @@ struct slab { static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl)) SLAB_MATCH(flags, __page_flags); SLAB_MATCH(compound_head, slab_list); /* Ensure bit 0 is clear */ -SLAB_MATCH(slab_list, slab_list); #ifndef CONFIG_SLOB SLAB_MATCH(rcu_head, rcu_head); -SLAB_MATCH(slab_cache, slab_cache); -#endif -#ifdef CONFIG_SLAB -SLAB_MATCH(s_mem, s_mem); -SLAB_MATCH(active, active); #endif SLAB_MATCH(_refcount, __page_refcount); #ifdef CONFIG_MEMCG -- cgit v1.2.3-59-g8ed1b From b5e7b59c3480f355910f9d2c6ece5857922a5e54 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 28 Sep 2021 09:47:57 +1000 Subject: NFS: change nfs_access_get_cached to only report the mask Currently the nfs_access_get_cached family of functions report a 'struct nfs_access_entry' as the result, with both .mask and .cred set. However the .cred is never used. This is probably good and there is no guarantee that it won't be freed before use. Change to only report the 'mask' - as this is all that is used or needed. Signed-off-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 20 +++++++++----------- fs/nfs/nfs4proc.c | 18 +++++++++--------- include/linux/nfs_fs.h | 4 ++-- 3 files changed, 20 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 731d31015b6a..8487a6d69116 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2673,7 +2673,7 @@ static struct nfs_access_entry *nfs_access_search_rbtree(struct inode *inode, co return NULL; } -static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res, bool may_block) +static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *cred, u32 *mask, bool may_block) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_access_entry *cache; @@ -2703,8 +2703,7 @@ static int nfs_access_get_cached_locked(struct inode *inode, const struct cred * spin_lock(&inode->i_lock); retry = false; } - res->cred = cache->cred; - res->mask = cache->mask; + *mask = cache->mask; list_move_tail(&cache->lru, &nfsi->access_cache_entry_lru); err = 0; out: @@ -2716,7 +2715,7 @@ out_zap: return -ENOENT; } -static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res) +static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cred, u32 *mask) { /* Only check the most recently returned cache entry, * but do it without locking. @@ -2738,22 +2737,21 @@ static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cre goto out; if (nfs_check_cache_invalid(inode, NFS_INO_INVALID_ACCESS)) goto out; - res->cred = cache->cred; - res->mask = cache->mask; + *mask = cache->mask; err = 0; out: rcu_read_unlock(); return err; } -int nfs_access_get_cached(struct inode *inode, const struct cred *cred, struct -nfs_access_entry *res, bool may_block) +int nfs_access_get_cached(struct inode *inode, const struct cred *cred, + u32 *mask, bool may_block) { int status; - status = nfs_access_get_cached_rcu(inode, cred, res); + status = nfs_access_get_cached_rcu(inode, cred, mask); if (status != 0) - status = nfs_access_get_cached_locked(inode, cred, res, + status = nfs_access_get_cached_locked(inode, cred, mask, may_block); return status; @@ -2874,7 +2872,7 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) trace_nfs_access_enter(inode); - status = nfs_access_get_cached(inode, cred, &cache, may_block); + status = nfs_access_get_cached(inode, cred, &cache.mask, may_block); if (status == 0) goto out_cached; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index ee3bc79f6ca3..322ff45ad15c 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7611,7 +7611,7 @@ static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler, const char *key, const void *buf, size_t buflen, int flags) { - struct nfs_access_entry cache; + u32 mask; int ret; if (!nfs_server_capable(inode, NFS_CAP_XATTR)) @@ -7626,8 +7626,8 @@ static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler, * do a cached access check for the XA* flags to possibly avoid * doing an RPC and getting EACCES back. */ - if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) { - if (!(cache.mask & NFS_ACCESS_XAWRITE)) + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XAWRITE)) return -EACCES; } @@ -7648,14 +7648,14 @@ static int nfs4_xattr_get_nfs4_user(const struct xattr_handler *handler, struct dentry *unused, struct inode *inode, const char *key, void *buf, size_t buflen) { - struct nfs_access_entry cache; + u32 mask; ssize_t ret; if (!nfs_server_capable(inode, NFS_CAP_XATTR)) return -EOPNOTSUPP; - if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) { - if (!(cache.mask & NFS_ACCESS_XAREAD)) + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XAREAD)) return -EACCES; } @@ -7680,13 +7680,13 @@ nfs4_listxattr_nfs4_user(struct inode *inode, char *list, size_t list_len) ssize_t ret, size; char *buf; size_t buflen; - struct nfs_access_entry cache; + u32 mask; if (!nfs_server_capable(inode, NFS_CAP_XATTR)) return 0; - if (!nfs_access_get_cached(inode, current_cred(), &cache, true)) { - if (!(cache.mask & NFS_ACCESS_XALIST)) + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XALIST)) return 0; } diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 05f249f20f55..f33559acbcc2 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -533,8 +533,8 @@ extern int nfs_instantiate(struct dentry *dentry, struct nfs_fh *fh, struct nfs_fattr *fattr); extern int nfs_may_open(struct inode *inode, const struct cred *cred, int openflags); extern void nfs_access_zap_cache(struct inode *inode); -extern int nfs_access_get_cached(struct inode *inode, const struct cred *cred, struct nfs_access_entry *res, - bool may_block); +extern int nfs_access_get_cached(struct inode *inode, const struct cred *cred, + u32 *mask, bool may_block); /* * linux/fs/nfs/symlink.c -- cgit v1.2.3-59-g8ed1b From 73fbb3fa647bdb5b60469af8101c741ece03a825 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 28 Sep 2021 09:47:57 +1000 Subject: NFS: pass cred explicitly for access tests Storing the 'struct cred *' in nfs_access_entry is problematic. An active 'cred' can keep a 'struct key *' active, and a quota is imposed on the number of such keys that a user can maintain. Cached 'nfs_access_entry' structs have indefinite lifetime, and having these keep 'struct key's alive imposes on that quota. So a future patch will remove the ->cred ref from nfs_access_entry. To prepare, change various functions to not assume there is a 'cred' in the nfs_access_entry, but to pass the cred around explicitly. Signed-off-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 17 ++++++++++------- fs/nfs/nfs3proc.c | 5 +++-- fs/nfs/nfs4proc.c | 12 +++++++----- include/linux/nfs_fs.h | 2 +- include/linux/nfs_xdr.h | 2 +- 5 files changed, 22 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 8487a6d69116..a34351ce79a2 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2758,7 +2758,9 @@ int nfs_access_get_cached(struct inode *inode, const struct cred *cred, } EXPORT_SYMBOL_GPL(nfs_access_get_cached); -static void nfs_access_add_rbtree(struct inode *inode, struct nfs_access_entry *set) +static void nfs_access_add_rbtree(struct inode *inode, + struct nfs_access_entry *set, + const struct cred *cred) { struct nfs_inode *nfsi = NFS_I(inode); struct rb_root *root_node = &nfsi->access_cache; @@ -2771,7 +2773,7 @@ static void nfs_access_add_rbtree(struct inode *inode, struct nfs_access_entry * while (*p != NULL) { parent = *p; entry = rb_entry(parent, struct nfs_access_entry, rb_node); - cmp = cred_fscmp(set->cred, entry->cred); + cmp = cred_fscmp(cred, entry->cred); if (cmp < 0) p = &parent->rb_left; @@ -2793,13 +2795,14 @@ found: nfs_access_free_entry(entry); } -void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set) +void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set, + const struct cred *cred) { struct nfs_access_entry *cache = kmalloc(sizeof(*cache), GFP_KERNEL); if (cache == NULL) return; RB_CLEAR_NODE(&cache->rb_node); - cache->cred = get_cred(set->cred); + cache->cred = get_cred(cred); cache->mask = set->mask; /* The above field assignments must be visible @@ -2807,7 +2810,7 @@ void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set) * use rcu_assign_pointer, so just force the memory barrier. */ smp_wmb(); - nfs_access_add_rbtree(inode, cache); + nfs_access_add_rbtree(inode, cache, cred); /* Update accounting */ smp_mb__before_atomic(); @@ -2893,7 +2896,7 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) else cache.mask |= NFS_ACCESS_EXECUTE; cache.cred = cred; - status = NFS_PROTO(inode)->access(inode, &cache); + status = NFS_PROTO(inode)->access(inode, &cache, cred); if (status != 0) { if (status == -ESTALE) { if (!S_ISDIR(inode->i_mode)) @@ -2903,7 +2906,7 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) } goto out; } - nfs_access_add_cache(inode, &cache); + nfs_access_add_cache(inode, &cache, cred); out_cached: cache_mask = nfs_access_calc_mask(cache.mask, inode->i_mode); if ((mask & ~cache_mask & (MAY_READ | MAY_WRITE | MAY_EXEC)) != 0) diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index 7100514d306b..1597eef40d54 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -220,7 +220,8 @@ static int nfs3_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle, task_flags); } -static int nfs3_proc_access(struct inode *inode, struct nfs_access_entry *entry) +static int nfs3_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) { struct nfs3_accessargs arg = { .fh = NFS_FH(inode), @@ -231,7 +232,7 @@ static int nfs3_proc_access(struct inode *inode, struct nfs_access_entry *entry) .rpc_proc = &nfs3_procedures[NFS3PROC_ACCESS], .rpc_argp = &arg, .rpc_resp = &res, - .rpc_cred = entry->cred, + .rpc_cred = cred, }; int status = -ENOMEM; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 322ff45ad15c..f02aa9877e6f 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2655,7 +2655,7 @@ static int nfs4_opendata_access(const struct cred *cred, cache.cred = cred; nfs_access_set_mask(&cache, opendata->o_res.access_result); - nfs_access_add_cache(state->inode, &cache); + nfs_access_add_cache(state->inode, &cache, cred); flags = NFS4_ACCESS_READ | NFS4_ACCESS_EXECUTE | NFS4_ACCESS_LOOKUP; if ((mask & ~cache.mask & flags) == 0) @@ -4441,7 +4441,8 @@ static int nfs4_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle, return err; } -static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry) +static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) { struct nfs_server *server = NFS_SERVER(inode); struct nfs4_accessargs args = { @@ -4455,7 +4456,7 @@ static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ACCESS], .rpc_argp = &args, .rpc_resp = &res, - .rpc_cred = entry->cred, + .rpc_cred = cred, }; int status = 0; @@ -4475,14 +4476,15 @@ static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry return status; } -static int nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry) +static int nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) { struct nfs4_exception exception = { .interruptible = true, }; int err; do { - err = _nfs4_proc_access(inode, entry); + err = _nfs4_proc_access(inode, entry, cred); trace_nfs4_access(inode, err); err = nfs4_handle_exception(NFS_SERVER(inode), err, &exception); diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index f33559acbcc2..c33b1b792bc9 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -396,7 +396,7 @@ extern int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fa extern int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr); extern int nfs_getattr(struct user_namespace *, const struct path *, struct kstat *, u32, unsigned int); -extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *); +extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *, const struct cred *); extern void nfs_access_set_mask(struct nfs_access_entry *, u32); extern int nfs_permission(struct user_namespace *, struct inode *, int); extern int nfs_open(struct inode *, struct file *); diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 967a0098f0a9..9102bdaa3faa 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1737,7 +1737,7 @@ struct nfs_rpc_ops { struct nfs_fh *, struct nfs_fattr *); int (*lookupp) (struct inode *, struct nfs_fh *, struct nfs_fattr *); - int (*access) (struct inode *, struct nfs_access_entry *); + int (*access) (struct inode *, struct nfs_access_entry *, const struct cred *); int (*readlink)(struct inode *, struct page *, unsigned int, unsigned int); int (*create) (struct inode *, struct dentry *, -- cgit v1.2.3-59-g8ed1b From 6238aec83f3fb12132f964937e5bbcf248fea8f9 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 28 Sep 2021 09:47:57 +1000 Subject: NFS: don't store 'struct cred *' in struct nfs_access_entry Storing the 'struct cred *' in nfs_access_entry is problematic. An active 'cred' can keep a 'struct key *' active, and a quota is imposed on the number of such keys that a user can maintain. Cached 'nfs_access_entry' structs have indefinite lifetime, and having these keep 'struct key's alive imposes on that quota. So remove the 'struct cred *' and replace it with the fields we need: kuid_t, kgid_t, and struct group_info * This makes the 'struct nfs_access_entry' 64 bits larger. New function "access_cmp" is introduced which is identical to cred_fscmp() except that the second arg is an 'nfs_access_entry', rather than a 'cred' Fixes: b68572e07c58 ("NFS: change access cache to use 'struct cred'.") Signed-off-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 50 ++++++++++++++++++++++++++++++++++++++++++++------ fs/nfs/nfs4proc.c | 1 - include/linux/nfs_fs.h | 4 +++- 3 files changed, 47 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index a34351ce79a2..7dee3fd10382 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2528,7 +2528,7 @@ MODULE_PARM_DESC(nfs_access_max_cachesize, "NFS access maximum total cache lengt static void nfs_access_free_entry(struct nfs_access_entry *entry) { - put_cred(entry->cred); + put_group_info(entry->group_info); kfree_rcu(entry, rcu_head); smp_mb__before_atomic(); atomic_long_dec(&nfs_access_nr_entries); @@ -2654,6 +2654,43 @@ void nfs_access_zap_cache(struct inode *inode) } EXPORT_SYMBOL_GPL(nfs_access_zap_cache); +static int access_cmp(const struct cred *a, const struct nfs_access_entry *b) +{ + struct group_info *ga, *gb; + int g; + + if (uid_lt(a->fsuid, b->fsuid)) + return -1; + if (uid_gt(a->fsuid, b->fsuid)) + return 1; + + if (gid_lt(a->fsgid, b->fsgid)) + return -1; + if (gid_gt(a->fsgid, b->fsgid)) + return 1; + + ga = a->group_info; + gb = b->group_info; + if (ga == gb) + return 0; + if (ga == NULL) + return -1; + if (gb == NULL) + return 1; + if (ga->ngroups < gb->ngroups) + return -1; + if (ga->ngroups > gb->ngroups) + return 1; + + for (g = 0; g < ga->ngroups; g++) { + if (gid_lt(ga->gid[g], gb->gid[g])) + return -1; + if (gid_gt(ga->gid[g], gb->gid[g])) + return 1; + } + return 0; +} + static struct nfs_access_entry *nfs_access_search_rbtree(struct inode *inode, const struct cred *cred) { struct rb_node *n = NFS_I(inode)->access_cache.rb_node; @@ -2661,7 +2698,7 @@ static struct nfs_access_entry *nfs_access_search_rbtree(struct inode *inode, co while (n != NULL) { struct nfs_access_entry *entry = rb_entry(n, struct nfs_access_entry, rb_node); - int cmp = cred_fscmp(cred, entry->cred); + int cmp = access_cmp(cred, entry); if (cmp < 0) n = n->rb_left; @@ -2731,7 +2768,7 @@ static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cre lh = rcu_dereference(list_tail_rcu(&nfsi->access_cache_entry_lru)); cache = list_entry(lh, struct nfs_access_entry, lru); if (lh == &nfsi->access_cache_entry_lru || - cred_fscmp(cred, cache->cred) != 0) + access_cmp(cred, cache) != 0) cache = NULL; if (cache == NULL) goto out; @@ -2773,7 +2810,7 @@ static void nfs_access_add_rbtree(struct inode *inode, while (*p != NULL) { parent = *p; entry = rb_entry(parent, struct nfs_access_entry, rb_node); - cmp = cred_fscmp(cred, entry->cred); + cmp = access_cmp(cred, entry); if (cmp < 0) p = &parent->rb_left; @@ -2802,7 +2839,9 @@ void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set, if (cache == NULL) return; RB_CLEAR_NODE(&cache->rb_node); - cache->cred = get_cred(cred); + cache->fsuid = cred->fsuid; + cache->fsgid = cred->fsgid; + cache->group_info = get_group_info(cred->group_info); cache->mask = set->mask; /* The above field assignments must be visible @@ -2895,7 +2934,6 @@ static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) cache.mask |= NFS_ACCESS_DELETE | NFS_ACCESS_LOOKUP; else cache.mask |= NFS_ACCESS_EXECUTE; - cache.cred = cred; status = NFS_PROTO(inode)->access(inode, &cache, cred); if (status != 0) { if (status == -ESTALE) { diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index f02aa9877e6f..5d9ff01ddf46 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -2653,7 +2653,6 @@ static int nfs4_opendata_access(const struct cred *cred, } else if ((fmode & FMODE_READ) && !opendata->file_created) mask = NFS4_ACCESS_READ; - cache.cred = cred; nfs_access_set_mask(&cache, opendata->o_res.access_result); nfs_access_add_cache(state->inode, &cache, cred); diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index c33b1b792bc9..450e393d4bf2 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -61,7 +61,9 @@ struct nfs_access_entry { struct rb_node rb_node; struct list_head lru; - const struct cred * cred; + kuid_t fsuid; + kgid_t fsgid; + struct group_info *group_info; __u32 mask; struct rcu_head rcu_head; }; -- cgit v1.2.3-59-g8ed1b From 1ab5be4ac5b1c9ce39ce1037c45b68d2ce6eede0 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Fri, 17 Dec 2021 15:36:54 -0500 Subject: NFSv4: Add some support for case insensitive filesystems Add capabilities to allow the NFS client to recognise when it is dealing with case insensitive and case preserving filesystems. Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 8 +++++++- fs/nfs/nfs4xdr.c | 40 ++++++++++++++++++++++++++++++++++++++++ include/linux/nfs_fs_sb.h | 2 ++ include/linux/nfs_xdr.h | 2 ++ 4 files changed, 51 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 5d9ff01ddf46..a6d7472058ad 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -3840,7 +3840,9 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f FATTR4_WORD0_FH_EXPIRE_TYPE | FATTR4_WORD0_LINK_SUPPORT | FATTR4_WORD0_SYMLINK_SUPPORT | - FATTR4_WORD0_ACLSUPPORT; + FATTR4_WORD0_ACLSUPPORT | + FATTR4_WORD0_CASE_INSENSITIVE | + FATTR4_WORD0_CASE_PRESERVING; if (minorversion) bitmask[2] = FATTR4_WORD2_SUPPATTR_EXCLCREAT; @@ -3869,6 +3871,10 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f server->caps |= NFS_CAP_HARDLINKS; if (res.has_symlinks != 0) server->caps |= NFS_CAP_SYMLINKS; + if (res.case_insensitive) + server->caps |= NFS_CAP_CASE_INSENSITIVE; + if (res.case_preserving) + server->caps |= NFS_CAP_CASE_PRESERVING; #ifdef CONFIG_NFS_V4_SECURITY_LABEL if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL) server->caps |= NFS_CAP_SECURITY_LABEL; diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index 801119b7a596..a9487c840052 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -3533,6 +3533,42 @@ static int decode_attr_aclsupport(struct xdr_stream *xdr, uint32_t *bitmap, uint return 0; } +static int decode_attr_case_insensitive(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_CASE_INSENSITIVE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_CASE_INSENSITIVE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_CASE_INSENSITIVE; + } + dprintk("%s: case_insensitive=%s\n", __func__, *res == 0 ? "false" : "true"); + return 0; +} + +static int decode_attr_case_preserving(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_CASE_PRESERVING - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_CASE_PRESERVING)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_CASE_PRESERVING; + } + dprintk("%s: case_preserving=%s\n", __func__, *res == 0 ? "false" : "true"); + return 0; +} + static int decode_attr_fileid(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *fileid) { __be32 *p; @@ -4413,6 +4449,10 @@ static int decode_server_caps(struct xdr_stream *xdr, struct nfs4_server_caps_re goto xdr_error; if ((status = decode_attr_aclsupport(xdr, bitmap, &res->acl_bitmask)) != 0) goto xdr_error; + if ((status = decode_attr_case_insensitive(xdr, bitmap, &res->case_insensitive)) != 0) + goto xdr_error; + if ((status = decode_attr_case_preserving(xdr, bitmap, &res->case_preserving)) != 0) + goto xdr_error; if ((status = decode_attr_exclcreat_supported(xdr, bitmap, res->exclcreat_bitmask)) != 0) goto xdr_error; diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 2a9acbfe00f0..f24fc67af42d 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -271,6 +271,8 @@ struct nfs_server { #define NFS_CAP_ACLS (1U << 3) #define NFS_CAP_ATOMIC_OPEN (1U << 4) #define NFS_CAP_LGOPEN (1U << 5) +#define NFS_CAP_CASE_INSENSITIVE (1U << 6) +#define NFS_CAP_CASE_PRESERVING (1U << 7) #define NFS_CAP_POSIX_LOCK (1U << 14) #define NFS_CAP_UIDGID_NOMAP (1U << 15) #define NFS_CAP_STATEID_NFSV41 (1U << 16) diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 9102bdaa3faa..ddff92396a3f 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1194,6 +1194,8 @@ struct nfs4_server_caps_res { u32 has_links; u32 has_symlinks; u32 fh_expire_type; + u32 case_insensitive; + u32 case_preserving; }; #define NFS4_PATHNAME_MAXCOMPONENTS 512 -- cgit v1.2.3-59-g8ed1b From 703f7066f40599c290babdb79dd61319264987e9 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 7 Dec 2021 13:17:33 +0100 Subject: random: remove unused irq_flags argument from add_interrupt_randomness() Since commit ee3e00e9e7101 ("random: use registers from interrupted code for CPU's w/o a cycle counter") the irq_flags argument is no longer used. Remove unused irq_flags. Cc: Borislav Petkov Cc: Dave Hansen Cc: Dexuan Cui Cc: H. Peter Anvin Cc: Haiyang Zhang Cc: Ingo Molnar Cc: K. Y. Srinivasan Cc: Stephen Hemminger Cc: Thomas Gleixner Cc: Wei Liu Cc: linux-hyperv@vger.kernel.org Cc: x86@kernel.org Signed-off-by: Sebastian Andrzej Siewior Acked-by: Wei Liu Signed-off-by: Jason A. Donenfeld --- arch/x86/kernel/cpu/mshyperv.c | 2 +- drivers/char/random.c | 4 ++-- drivers/hv/vmbus_drv.c | 2 +- include/linux/random.h | 2 +- kernel/irq/handle.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index ff55df60228f..2a0f83678911 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -79,7 +79,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) inc_irq_stat(hyperv_stimer0_count); if (hv_stimer0_handler) hv_stimer0_handler(); - add_interrupt_randomness(HYPERV_STIMER0_VECTOR, 0); + add_interrupt_randomness(HYPERV_STIMER0_VECTOR); ack_APIC_irq(); set_irq_regs(old_regs); diff --git a/drivers/char/random.c b/drivers/char/random.c index c81485e2f126..4b0753eb8bfb 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -200,7 +200,7 @@ * void add_device_randomness(const void *buf, unsigned int size); * void add_input_randomness(unsigned int type, unsigned int code, * unsigned int value); - * void add_interrupt_randomness(int irq, int irq_flags); + * void add_interrupt_randomness(int irq); * void add_disk_randomness(struct gendisk *disk); * void add_hwgenerator_randomness(const char *buffer, size_t count, * size_t entropy); @@ -1253,7 +1253,7 @@ static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs) return *ptr; } -void add_interrupt_randomness(int irq, int irq_flags) +void add_interrupt_randomness(int irq) { struct entropy_store *r; struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 392c1ac4f819..7ae04ccb1043 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1381,7 +1381,7 @@ static void vmbus_isr(void) tasklet_schedule(&hv_cpu->msg_dpc); } - add_interrupt_randomness(vmbus_interrupt, 0); + add_interrupt_randomness(vmbus_interrupt); } static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) diff --git a/include/linux/random.h b/include/linux/random.h index f45b8be3e3c4..c45b2693e51f 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -35,7 +35,7 @@ static inline void add_latent_entropy(void) {} extern void add_input_randomness(unsigned int type, unsigned int code, unsigned int value) __latent_entropy; -extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy; +extern void add_interrupt_randomness(int irq) __latent_entropy; extern void get_random_bytes(void *buf, int nbytes); extern int wait_for_random_bytes(void); diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 27182003b879..68c76c3caaf5 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -197,7 +197,7 @@ irqreturn_t handle_irq_event_percpu(struct irq_desc *desc) retval = __handle_irq_event_percpu(desc, &flags); - add_interrupt_randomness(desc->irq_data.irq, flags); + add_interrupt_randomness(desc->irq_data.irq); if (!irq_settings_no_debug(desc)) note_interrupt(desc, retval); -- cgit v1.2.3-59-g8ed1b From 79b60ca83b6fa63ef307d2edcc77ee6581da8971 Mon Sep 17 00:00:00 2001 From: Shay Drory Date: Sun, 12 Dec 2021 14:51:27 +0200 Subject: net/mlx5: Introduce API for bulk request and release of IRQs Currently IRQs are requested one by one. To balance spreading IRQs among cpus using such scheme requires remembering cpu mask for the cpus used for a given device. This complicates the IRQ allocation scheme in subsequent patch. Hence, prepare the code for bulk IRQs allocation. This enables spreading IRQs among cpus in subsequent patch. Signed-off-by: Shay Drory Reviewed-by: Parav Pandit Signed-off-by: Saeed Mahameed --- drivers/infiniband/hw/mlx5/odp.c | 6 -- drivers/net/ethernet/mellanox/mlx5/core/eq.c | 98 ++++++++++++++-------- drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h | 5 +- drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c | 69 +++++++++++++-- include/linux/mlx5/eq.h | 4 +- 5 files changed, 135 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 91eb615b89ee..86842cd580ba 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -1541,16 +1541,10 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; param = (struct mlx5_eq_param) { - .irq_index = MLX5_IRQ_EQ_CTRL, .nent = MLX5_IB_NUM_PF_EQE, }; param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; - if (!zalloc_cpumask_var(¶m.affinity, GFP_KERNEL)) { - err = -ENOMEM; - goto err_wq; - } eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); - free_cpumask_var(param.affinity); if (IS_ERR(eq->core)) { err = PTR_ERR(eq->core); goto err_wq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 1eb0326a489b..14547b6f2894 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -59,6 +59,8 @@ struct mlx5_eq_table { struct mutex lock; /* sync async eqs creations */ int num_comp_eqs; struct mlx5_irq_table *irq_table; + struct mlx5_irq **comp_irqs; + struct mlx5_irq *ctrl_irq; #ifdef CONFIG_RFS_ACCEL struct cpu_rmap *rmap; #endif @@ -266,8 +268,8 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; u8 log_eq_stride = ilog2(MLX5_EQE_SIZE); struct mlx5_priv *priv = &dev->priv; - u16 vecidx = param->irq_index; __be64 *pas; + u16 vecidx; void *eqc; int inlen; u32 *in; @@ -289,23 +291,16 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc); init_eq_buf(eq); - if (vecidx == MLX5_IRQ_EQ_CTRL) - eq->irq = mlx5_ctrl_irq_request(dev); - else - eq->irq = mlx5_irq_request(dev, vecidx, param->affinity); - if (IS_ERR(eq->irq)) { - err = PTR_ERR(eq->irq); - goto err_buf; - } - + eq->irq = param->irq; vecidx = mlx5_irq_get_index(eq->irq); + inlen = MLX5_ST_SZ_BYTES(create_eq_in) + MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages; in = kvzalloc(inlen, GFP_KERNEL); if (!in) { err = -ENOMEM; - goto err_irq; + goto err_buf; } pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas); @@ -349,8 +344,6 @@ err_eq: err_in: kvfree(in); -err_irq: - mlx5_irq_release(eq->irq); err_buf: mlx5_frag_buf_free(dev, &eq->frag_buf); return err; @@ -404,7 +397,6 @@ static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) if (err) mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", eq->eqn); - mlx5_irq_release(eq->irq); mlx5_frag_buf_free(dev, &eq->frag_buf); return err; @@ -597,11 +589,8 @@ setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, eq->irq_nb.notifier_call = mlx5_eq_async_int; spin_lock_init(&eq->lock); - if (!zalloc_cpumask_var(¶m->affinity, GFP_KERNEL)) - return -ENOMEM; err = create_async_eq(dev, &eq->core, param); - free_cpumask_var(param->affinity); if (err) { mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); return err; @@ -646,11 +635,18 @@ static int create_async_eqs(struct mlx5_core_dev *dev) struct mlx5_eq_param param = {}; int err; + /* All the async_eqs are using single IRQ, request one IRQ and share its + * index among all the async_eqs of this device. + */ + table->ctrl_irq = mlx5_ctrl_irq_request(dev); + if (IS_ERR(table->ctrl_irq)) + return PTR_ERR(table->ctrl_irq); + MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR); mlx5_eq_notifier_register(dev, &table->cq_err_nb); param = (struct mlx5_eq_param) { - .irq_index = MLX5_IRQ_EQ_CTRL, + .irq = table->ctrl_irq, .nent = MLX5_NUM_CMD_EQE, .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, }; @@ -663,7 +659,7 @@ static int create_async_eqs(struct mlx5_core_dev *dev) mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); param = (struct mlx5_eq_param) { - .irq_index = MLX5_IRQ_EQ_CTRL, + .irq = table->ctrl_irq, .nent = async_eq_depth_devlink_param_get(dev), }; @@ -673,7 +669,7 @@ static int create_async_eqs(struct mlx5_core_dev *dev) goto err2; param = (struct mlx5_eq_param) { - .irq_index = MLX5_IRQ_EQ_CTRL, + .irq = table->ctrl_irq, .nent = /* TODO: sriov max_vf + */ 1, .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, }; @@ -692,6 +688,7 @@ err2: err1: mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); + mlx5_ctrl_irq_release(table->ctrl_irq); return err; } @@ -706,6 +703,7 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev) cleanup_async_eq(dev, &table->cmd_eq, "cmd"); mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); + mlx5_ctrl_irq_release(table->ctrl_irq); } struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev) @@ -733,12 +731,10 @@ mlx5_eq_create_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL); int err; - if (!cpumask_available(param->affinity)) - return ERR_PTR(-EINVAL); - if (!eq) return ERR_PTR(-ENOMEM); + param->irq = dev->priv.eq_table->ctrl_irq; err = create_async_eq(dev, eq, param); if (err) { kvfree(eq); @@ -798,6 +794,45 @@ void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm) } EXPORT_SYMBOL(mlx5_eq_update_ci); +static void comp_irqs_release(struct mlx5_core_dev *dev) +{ + struct mlx5_eq_table *table = dev->priv.eq_table; + + mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs); + kfree(table->comp_irqs); +} + +static int comp_irqs_request(struct mlx5_core_dev *dev) +{ + struct mlx5_eq_table *table = dev->priv.eq_table; + int ncomp_eqs = table->num_comp_eqs; + u16 *cpus; + int ret; + int i; + + ncomp_eqs = table->num_comp_eqs; + table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL); + if (!table->comp_irqs) + return -ENOMEM; + + cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL); + if (!cpus) { + ret = -ENOMEM; + goto free_irqs; + } + for (i = 0; i < ncomp_eqs; i++) + cpus[i] = cpumask_local_spread(i, dev->priv.numa_node); + ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs); + kfree(cpus); + if (ret < 0) + goto free_irqs; + return ret; + +free_irqs: + kfree(table->comp_irqs); + return ret; +} + static void destroy_comp_eqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = dev->priv.eq_table; @@ -812,6 +847,7 @@ static void destroy_comp_eqs(struct mlx5_core_dev *dev) tasklet_disable(&eq->tasklet_ctx.task); kfree(eq); } + comp_irqs_release(dev); } static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev) @@ -838,12 +874,13 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) int err; int i; + ncomp_eqs = comp_irqs_request(dev); + if (ncomp_eqs < 0) + return ncomp_eqs; INIT_LIST_HEAD(&table->comp_eqs_list); - ncomp_eqs = table->num_comp_eqs; nent = comp_eq_depth_devlink_param_get(dev); for (i = 0; i < ncomp_eqs; i++) { struct mlx5_eq_param param = {}; - int vecidx = i; eq = kzalloc(sizeof(*eq), GFP_KERNEL); if (!eq) { @@ -858,18 +895,11 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) eq->irq_nb.notifier_call = mlx5_eq_comp_int; param = (struct mlx5_eq_param) { - .irq_index = vecidx, + .irq = table->comp_irqs[i], .nent = nent, }; - if (!zalloc_cpumask_var(¶m.affinity, GFP_KERNEL)) { - err = -ENOMEM; - goto clean_eq; - } - cpumask_set_cpu(cpumask_local_spread(i, dev->priv.numa_node), - param.affinity); err = create_map_eq(dev, &eq->core, ¶m); - free_cpumask_var(param.affinity); if (err) goto clean_eq; err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); @@ -883,7 +913,9 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) list_add_tail(&eq->list, &table->comp_eqs_list); } + table->num_comp_eqs = ncomp_eqs; return 0; + clean_eq: kfree(eq); clean: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h index db58f5e3f457..f2ad3c21802f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h @@ -23,9 +23,12 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int devfn, int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs); struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev); +void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq); struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx, struct cpumask *affinity); -void mlx5_irq_release(struct mlx5_irq *irq); +int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs, + struct mlx5_irq **irqs); +void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs); int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb); int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb); struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 496826a7a88b..2e19c3c222fe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -342,13 +342,27 @@ static struct mlx5_irq_pool *ctrl_irq_pool_get(struct mlx5_core_dev *dev) } /** - * mlx5_irq_release - release an IRQ back to the system. - * @irq: irq to be released. + * mlx5_irqs_release - release one or more IRQs back to the system. + * @irqs: IRQs to be released. + * @nirqs: number of IRQs to be released. */ -void mlx5_irq_release(struct mlx5_irq *irq) +static void mlx5_irqs_release(struct mlx5_irq **irqs, int nirqs) { - synchronize_irq(irq->irqn); - irq_put(irq); + int i; + + for (i = 0; i < nirqs; i++) { + synchronize_irq(irqs[i]->irqn); + irq_put(irqs[i]); + } +} + +/** + * mlx5_ctrl_irq_release - release a ctrl IRQ back to the system. + * @ctrl_irq: ctrl IRQ to be released. + */ +void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq) +{ + mlx5_irqs_release(&ctrl_irq, 1); } /** @@ -423,6 +437,51 @@ out: return irq; } +/** + * mlx5_irqs_release_vectors - release one or more IRQs back to the system. + * @irqs: IRQs to be released. + * @nirqs: number of IRQs to be released. + */ +void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs) +{ + mlx5_irqs_release(irqs, nirqs); +} + +/** + * mlx5_irqs_request_vectors - request one or more IRQs for mlx5 device. + * @dev: mlx5 device that is requesting the IRQs. + * @cpus: CPUs array for binding the IRQs + * @nirqs: number of IRQs to request. + * @irqs: an output array of IRQs pointers. + * + * Each IRQ is bound to at most 1 CPU. + * This function is requests nirqs IRQs, starting from @vecidx. + * + * This function returns the number of IRQs requested, (which might be smaller than + * @nirqs), if successful, or a negative error code in case of an error. + */ +int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs, + struct mlx5_irq **irqs) +{ + cpumask_var_t req_mask; + struct mlx5_irq *irq; + int i; + + if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL)) + return -ENOMEM; + for (i = 0; i < nirqs; i++) { + cpumask_set_cpu(cpus[i], req_mask); + irq = mlx5_irq_request(dev, i, req_mask); + if (IS_ERR(irq)) + break; + cpumask_clear(req_mask); + irqs[i] = irq; + } + + free_cpumask_var(req_mask); + return i ? i : PTR_ERR(irq); +} + static struct mlx5_irq_pool * irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name, u32 min_threshold, u32 max_threshold) diff --git a/include/linux/mlx5/eq.h b/include/linux/mlx5/eq.h index ea3ff5a8ced3..3705a382276b 100644 --- a/include/linux/mlx5/eq.h +++ b/include/linux/mlx5/eq.h @@ -9,13 +9,13 @@ #define MLX5_NUM_SPARE_EQE (0x80) struct mlx5_eq; +struct mlx5_irq; struct mlx5_core_dev; struct mlx5_eq_param { - u8 irq_index; int nent; u64 mask[4]; - cpumask_var_t affinity; + struct mlx5_irq *irq; }; struct mlx5_eq * -- cgit v1.2.3-59-g8ed1b From 3663f26b389b3951426971b44bb9312fdff0efec Mon Sep 17 00:00:00 2001 From: Ajit Kumar Pandey Date: Sun, 12 Dec 2021 23:35:24 +0530 Subject: drivers: acpi: acpi_apd: Remove unused device property "is-rv" Initially "is-rv" device property is added for 48MHz fixed clock support on Raven or RV architecture. It's unused now as we moved to pci device_id based selection to extend such support on other architectures. This change removed unused code from acpi driver. Signed-off-by: Ajit Kumar Pandey Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20211212180527.1641362-3-AjitKumar.Pandey@amd.com Signed-off-by: Stephen Boyd --- drivers/acpi/acpi_apd.c | 3 --- include/linux/platform_data/clk-fch.h | 1 - 2 files changed, 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 6e02448d15d9..6913e9712852 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -87,9 +87,6 @@ static int fch_misc_setup(struct apd_private_data *pdata) if (ret < 0) return -ENOENT; - if (!acpi_dev_get_property(adev, "is-rv", ACPI_TYPE_INTEGER, &obj)) - clk_data->is_rv = obj->integer.value; - list_for_each_entry(rentry, &resource_list, node) { clk_data->base = devm_ioremap(&adev->dev, rentry->res->start, resource_size(rentry->res)); diff --git a/include/linux/platform_data/clk-fch.h b/include/linux/platform_data/clk-fch.h index b9f682459f08..850ca776156d 100644 --- a/include/linux/platform_data/clk-fch.h +++ b/include/linux/platform_data/clk-fch.h @@ -12,7 +12,6 @@ struct fch_clk_data { void __iomem *base; - u32 is_rv; }; #endif /* __CLK_FCH_H */ -- cgit v1.2.3-59-g8ed1b From 7fdb98e8a768b3ccc05494d3ea4436047f512b9d Mon Sep 17 00:00:00 2001 From: Ajit Kumar Pandey Date: Sun, 12 Dec 2021 23:35:25 +0530 Subject: ACPI: APD: Add a fmw property clk-name Add a new device property to fetch clk-name from firmware. Signed-off-by: Ajit Kumar Pandey Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20211212180527.1641362-4-AjitKumar.Pandey@amd.com Signed-off-by: Stephen Boyd --- drivers/acpi/acpi_apd.c | 10 ++++++++++ include/linux/platform_data/clk-fch.h | 1 + 2 files changed, 11 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 6913e9712852..2b958b426b03 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -87,6 +87,16 @@ static int fch_misc_setup(struct apd_private_data *pdata) if (ret < 0) return -ENOENT; + if (!acpi_dev_get_property(adev, "clk-name", ACPI_TYPE_STRING, &obj)) { + clk_data->name = devm_kzalloc(&adev->dev, obj->string.length, + GFP_KERNEL); + + strcpy(clk_data->name, obj->string.pointer); + } else { + /* Set default name to mclk if entry missing in firmware */ + clk_data->name = "mclk"; + } + list_for_each_entry(rentry, &resource_list, node) { clk_data->base = devm_ioremap(&adev->dev, rentry->res->start, resource_size(rentry->res)); diff --git a/include/linux/platform_data/clk-fch.h b/include/linux/platform_data/clk-fch.h index 850ca776156d..11a2a23fd9b2 100644 --- a/include/linux/platform_data/clk-fch.h +++ b/include/linux/platform_data/clk-fch.h @@ -12,6 +12,7 @@ struct fch_clk_data { void __iomem *base; + char *name; }; #endif /* __CLK_FCH_H */ -- cgit v1.2.3-59-g8ed1b From 2cee6fbb7f01bcb25f11ef1439e89a29de4c0c1d Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 25 Oct 2021 21:53:44 +0100 Subject: fscache: Remove the contents of the fscache driver, pending rewrite Remove the code that comprises the fscache driver as it's going to be substantially rewritten, with the majority of the code being erased in the rewrite. A small piece of linux/fscache.h is left as that is #included by a bunch of network filesystems. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819578724.215744.18210619052245724238.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906884814.143852.6727245089843862889.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967077097.1823006.1377665951499979089.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021485548.640689.13876080567388696162.stgit@warthog.procyon.org.uk/ # v4 --- fs/9p/vfs_addr.c | 8 +- fs/Makefile | 1 - fs/afs/file.c | 8 +- fs/fscache/Kconfig | 39 -- fs/fscache/Makefile | 20 - fs/fscache/cache.c | 416 -------------- fs/fscache/cookie.c | 1071 ---------------------------------- fs/fscache/fsdef.c | 98 ---- fs/fscache/internal.h | 461 --------------- fs/fscache/io.c | 116 ---- fs/fscache/main.c | 230 -------- fs/fscache/netfs.c | 74 --- fs/fscache/object.c | 1125 ------------------------------------ fs/fscache/operation.c | 633 -------------------- fs/fscache/page.c | 1242 ---------------------------------------- fs/fscache/proc.c | 71 --- fs/fscache/stats.c | 283 --------- include/linux/fscache-cache.h | 548 +----------------- include/linux/fscache.h | 851 +-------------------------- include/trace/events/fscache.h | 523 ----------------- 20 files changed, 13 insertions(+), 7805 deletions(-) delete mode 100644 fs/fscache/Makefile delete mode 100644 fs/fscache/cache.c delete mode 100644 fs/fscache/cookie.c delete mode 100644 fs/fscache/fsdef.c delete mode 100644 fs/fscache/internal.h delete mode 100644 fs/fscache/io.c delete mode 100644 fs/fscache/main.c delete mode 100644 fs/fscache/netfs.c delete mode 100644 fs/fscache/object.c delete mode 100644 fs/fscache/operation.c delete mode 100644 fs/fscache/page.c delete mode 100644 fs/fscache/proc.c delete mode 100644 fs/fscache/stats.c delete mode 100644 include/trace/events/fscache.h (limited to 'include/linux') diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index fac918ccb305..4ea8f862b9e4 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -76,9 +76,7 @@ static void v9fs_req_cleanup(struct address_space *mapping, void *priv) */ static bool v9fs_is_cache_enabled(struct inode *inode) { - struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(inode)); - - return fscache_cookie_enabled(cookie) && !hlist_empty(&cookie->backing_objects); + return fscache_cookie_enabled(v9fs_inode_cookie(V9FS_I(inode))); } /** @@ -87,9 +85,13 @@ static bool v9fs_is_cache_enabled(struct inode *inode) */ static int v9fs_begin_cache_operation(struct netfs_read_request *rreq) { +#ifdef CONFIG_9P_FSCACHE struct fscache_cookie *cookie = v9fs_inode_cookie(V9FS_I(rreq->inode)); return fscache_begin_read_operation(rreq, cookie); +#else + return -ENOBUFS; +#endif } static const struct netfs_read_request_ops v9fs_req_ops = { diff --git a/fs/Makefile b/fs/Makefile index 290815f3fd31..23ddd0803d14 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -67,7 +67,6 @@ obj-$(CONFIG_DLM) += dlm/ # Do not add any filesystems before this line obj-$(CONFIG_NETFS_SUPPORT) += netfs/ -obj-$(CONFIG_FSCACHE) += fscache/ obj-$(CONFIG_REISERFS_FS) += reiserfs/ obj-$(CONFIG_EXT4_FS) += ext4/ # We place ext4 before ext2 so that clean ext3 root fs's do NOT mount using the diff --git a/fs/afs/file.c b/fs/afs/file.c index cb6ad61eec3b..97a51e1de55c 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -352,16 +352,18 @@ static void afs_init_rreq(struct netfs_read_request *rreq, struct file *file) static bool afs_is_cache_enabled(struct inode *inode) { - struct fscache_cookie *cookie = afs_vnode_cache(AFS_FS_I(inode)); - - return fscache_cookie_enabled(cookie) && !hlist_empty(&cookie->backing_objects); + return fscache_cookie_enabled(afs_vnode_cache(AFS_FS_I(inode))); } static int afs_begin_cache_operation(struct netfs_read_request *rreq) { +#ifdef CONFIG_AFS_FSCACHE struct afs_vnode *vnode = AFS_FS_I(rreq->inode); return fscache_begin_read_operation(rreq, afs_vnode_cache(vnode)); +#else + return -ENOBUFS; +#endif } static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len, diff --git a/fs/fscache/Kconfig b/fs/fscache/Kconfig index 76316c4a3fb7..6440484d9461 100644 --- a/fs/fscache/Kconfig +++ b/fs/fscache/Kconfig @@ -1,43 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -config FSCACHE - tristate "General filesystem local caching manager" - select NETFS_SUPPORT - help - This option enables a generic filesystem caching manager that can be - used by various network and other filesystems to cache data locally. - Different sorts of caches can be plugged in, depending on the - resources available. - - See Documentation/filesystems/caching/fscache.rst for more information. - -config FSCACHE_STATS - bool "Gather statistical information on local caching" - depends on FSCACHE && PROC_FS - select NETFS_STATS - help - This option causes statistical information to be gathered on local - caching and exported through file: - - /proc/fs/fscache/stats - - The gathering of statistics adds a certain amount of overhead to - execution as there are a quite a few stats gathered, and on a - multi-CPU system these may be on cachelines that keep bouncing - between CPUs. On the other hand, the stats are very useful for - debugging purposes. Saying 'Y' here is recommended. - - See Documentation/filesystems/caching/fscache.rst for more information. - -config FSCACHE_DEBUG - bool "Debug FS-Cache" - depends on FSCACHE - help - This permits debugging to be dynamically enabled in the local caching - management module. If this is set, the debugging output may be - enabled by setting bits in /sys/modules/fscache/parameter/debug. - - See Documentation/filesystems/caching/fscache.rst for more information. - config FSCACHE_OLD_API bool diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile deleted file mode 100644 index 03a871d689bb..000000000000 --- a/fs/fscache/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for general filesystem caching code -# - -fscache-y := \ - cache.o \ - cookie.o \ - fsdef.o \ - io.o \ - main.o \ - netfs.o \ - object.o \ - operation.o \ - page.o - -fscache-$(CONFIG_PROC_FS) += proc.o -fscache-$(CONFIG_FSCACHE_STATS) += stats.o - -obj-$(CONFIG_FSCACHE) := fscache.o diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c deleted file mode 100644 index bd4f44c1cce0..000000000000 --- a/fs/fscache/cache.c +++ /dev/null @@ -1,416 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache cache handling - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#include "internal.h" - -LIST_HEAD(fscache_cache_list); -DECLARE_RWSEM(fscache_addremove_sem); -DECLARE_WAIT_QUEUE_HEAD(fscache_cache_cleared_wq); -EXPORT_SYMBOL(fscache_cache_cleared_wq); - -static LIST_HEAD(fscache_cache_tag_list); - -/* - * look up a cache tag - */ -struct fscache_cache_tag *__fscache_lookup_cache_tag(const char *name) -{ - struct fscache_cache_tag *tag, *xtag; - - /* firstly check for the existence of the tag under read lock */ - down_read(&fscache_addremove_sem); - - list_for_each_entry(tag, &fscache_cache_tag_list, link) { - if (strcmp(tag->name, name) == 0) { - atomic_inc(&tag->usage); - up_read(&fscache_addremove_sem); - return tag; - } - } - - up_read(&fscache_addremove_sem); - - /* the tag does not exist - create a candidate */ - xtag = kzalloc(sizeof(*xtag) + strlen(name) + 1, GFP_KERNEL); - if (!xtag) - /* return a dummy tag if out of memory */ - return ERR_PTR(-ENOMEM); - - atomic_set(&xtag->usage, 1); - strcpy(xtag->name, name); - - /* write lock, search again and add if still not present */ - down_write(&fscache_addremove_sem); - - list_for_each_entry(tag, &fscache_cache_tag_list, link) { - if (strcmp(tag->name, name) == 0) { - atomic_inc(&tag->usage); - up_write(&fscache_addremove_sem); - kfree(xtag); - return tag; - } - } - - list_add_tail(&xtag->link, &fscache_cache_tag_list); - up_write(&fscache_addremove_sem); - return xtag; -} - -/* - * release a reference to a cache tag - */ -void __fscache_release_cache_tag(struct fscache_cache_tag *tag) -{ - if (tag != ERR_PTR(-ENOMEM)) { - down_write(&fscache_addremove_sem); - - if (atomic_dec_and_test(&tag->usage)) - list_del_init(&tag->link); - else - tag = NULL; - - up_write(&fscache_addremove_sem); - - kfree(tag); - } -} - -/* - * select a cache in which to store an object - * - the cache addremove semaphore must be at least read-locked by the caller - * - the object will never be an index - */ -struct fscache_cache *fscache_select_cache_for_object( - struct fscache_cookie *cookie) -{ - struct fscache_cache_tag *tag; - struct fscache_object *object; - struct fscache_cache *cache; - - _enter(""); - - if (list_empty(&fscache_cache_list)) { - _leave(" = NULL [no cache]"); - return NULL; - } - - /* we check the parent to determine the cache to use */ - spin_lock(&cookie->lock); - - /* the first in the parent's backing list should be the preferred - * cache */ - if (!hlist_empty(&cookie->backing_objects)) { - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - cache = object->cache; - if (fscache_object_is_dying(object) || - test_bit(FSCACHE_IOERROR, &cache->flags)) - cache = NULL; - - spin_unlock(&cookie->lock); - _leave(" = %s [parent]", cache ? cache->tag->name : "NULL"); - return cache; - } - - /* the parent is unbacked */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - /* cookie not an index and is unbacked */ - spin_unlock(&cookie->lock); - _leave(" = NULL [cookie ub,ni]"); - return NULL; - } - - spin_unlock(&cookie->lock); - - if (!cookie->def->select_cache) - goto no_preference; - - /* ask the netfs for its preference */ - tag = cookie->def->select_cache(cookie->parent->netfs_data, - cookie->netfs_data); - if (!tag) - goto no_preference; - - if (tag == ERR_PTR(-ENOMEM)) { - _leave(" = NULL [nomem tag]"); - return NULL; - } - - if (!tag->cache) { - _leave(" = NULL [unbacked tag]"); - return NULL; - } - - if (test_bit(FSCACHE_IOERROR, &tag->cache->flags)) - return NULL; - - _leave(" = %s [specific]", tag->name); - return tag->cache; - -no_preference: - /* netfs has no preference - just select first cache */ - cache = list_entry(fscache_cache_list.next, - struct fscache_cache, link); - _leave(" = %s [first]", cache->tag->name); - return cache; -} - -/** - * fscache_init_cache - Initialise a cache record - * @cache: The cache record to be initialised - * @ops: The cache operations to be installed in that record - * @idfmt: Format string to define identifier - * @...: sprintf-style arguments - * - * Initialise a record of a cache and fill in the name. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_init_cache(struct fscache_cache *cache, - const struct fscache_cache_ops *ops, - const char *idfmt, - ...) -{ - va_list va; - - memset(cache, 0, sizeof(*cache)); - - cache->ops = ops; - - va_start(va, idfmt); - vsnprintf(cache->identifier, sizeof(cache->identifier), idfmt, va); - va_end(va); - - INIT_WORK(&cache->op_gc, fscache_operation_gc); - INIT_LIST_HEAD(&cache->link); - INIT_LIST_HEAD(&cache->object_list); - INIT_LIST_HEAD(&cache->op_gc_list); - spin_lock_init(&cache->object_list_lock); - spin_lock_init(&cache->op_gc_list_lock); -} -EXPORT_SYMBOL(fscache_init_cache); - -/** - * fscache_add_cache - Declare a cache as being open for business - * @cache: The record describing the cache - * @ifsdef: The record of the cache object describing the top-level index - * @tagname: The tag describing this cache - * - * Add a cache to the system, making it available for netfs's to use. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -int fscache_add_cache(struct fscache_cache *cache, - struct fscache_object *ifsdef, - const char *tagname) -{ - struct fscache_cache_tag *tag; - - ASSERTCMP(ifsdef->cookie, ==, &fscache_fsdef_index); - BUG_ON(!cache->ops); - BUG_ON(!ifsdef); - - cache->flags = 0; - ifsdef->event_mask = - ((1 << NR_FSCACHE_OBJECT_EVENTS) - 1) & - ~(1 << FSCACHE_OBJECT_EV_CLEARED); - __set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &ifsdef->flags); - - if (!tagname) - tagname = cache->identifier; - - BUG_ON(!tagname[0]); - - _enter("{%s.%s},,%s", cache->ops->name, cache->identifier, tagname); - - /* we use the cache tag to uniquely identify caches */ - tag = __fscache_lookup_cache_tag(tagname); - if (IS_ERR(tag)) - goto nomem; - - if (test_and_set_bit(FSCACHE_TAG_RESERVED, &tag->flags)) - goto tag_in_use; - - cache->kobj = kobject_create_and_add(tagname, fscache_root); - if (!cache->kobj) - goto error; - - ifsdef->cache = cache; - cache->fsdef = ifsdef; - - down_write(&fscache_addremove_sem); - - tag->cache = cache; - cache->tag = tag; - - /* add the cache to the list */ - list_add(&cache->link, &fscache_cache_list); - - /* add the cache's netfs definition index object to the cache's - * list */ - spin_lock(&cache->object_list_lock); - list_add_tail(&ifsdef->cache_link, &cache->object_list); - spin_unlock(&cache->object_list_lock); - - /* add the cache's netfs definition index object to the top level index - * cookie as a known backing object */ - spin_lock(&fscache_fsdef_index.lock); - - hlist_add_head(&ifsdef->cookie_link, - &fscache_fsdef_index.backing_objects); - - refcount_inc(&fscache_fsdef_index.ref); - - /* done */ - spin_unlock(&fscache_fsdef_index.lock); - up_write(&fscache_addremove_sem); - - pr_notice("Cache \"%s\" added (type %s)\n", - cache->tag->name, cache->ops->name); - kobject_uevent(cache->kobj, KOBJ_ADD); - - _leave(" = 0 [%s]", cache->identifier); - return 0; - -tag_in_use: - pr_err("Cache tag '%s' already in use\n", tagname); - __fscache_release_cache_tag(tag); - _leave(" = -EXIST"); - return -EEXIST; - -error: - __fscache_release_cache_tag(tag); - _leave(" = -EINVAL"); - return -EINVAL; - -nomem: - _leave(" = -ENOMEM"); - return -ENOMEM; -} -EXPORT_SYMBOL(fscache_add_cache); - -/** - * fscache_io_error - Note a cache I/O error - * @cache: The record describing the cache - * - * Note that an I/O error occurred in a cache and that it should no longer be - * used for anything. This also reports the error into the kernel log. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_io_error(struct fscache_cache *cache) -{ - if (!test_and_set_bit(FSCACHE_IOERROR, &cache->flags)) - pr_err("Cache '%s' stopped due to I/O error\n", - cache->ops->name); -} -EXPORT_SYMBOL(fscache_io_error); - -/* - * request withdrawal of all the objects in a cache - * - all the objects being withdrawn are moved onto the supplied list - */ -static void fscache_withdraw_all_objects(struct fscache_cache *cache, - struct list_head *dying_objects) -{ - struct fscache_object *object; - - while (!list_empty(&cache->object_list)) { - spin_lock(&cache->object_list_lock); - - if (!list_empty(&cache->object_list)) { - object = list_entry(cache->object_list.next, - struct fscache_object, cache_link); - list_move_tail(&object->cache_link, dying_objects); - - _debug("withdraw %x", object->cookie->debug_id); - - /* This must be done under object_list_lock to prevent - * a race with fscache_drop_object(). - */ - fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL); - } - - spin_unlock(&cache->object_list_lock); - cond_resched(); - } -} - -/** - * fscache_withdraw_cache - Withdraw a cache from the active service - * @cache: The record describing the cache - * - * Withdraw a cache from service, unbinding all its cache objects from the - * netfs cookies they're currently representing. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_withdraw_cache(struct fscache_cache *cache) -{ - LIST_HEAD(dying_objects); - - _enter(""); - - pr_notice("Withdrawing cache \"%s\"\n", - cache->tag->name); - - /* make the cache unavailable for cookie acquisition */ - if (test_and_set_bit(FSCACHE_CACHE_WITHDRAWN, &cache->flags)) - BUG(); - - down_write(&fscache_addremove_sem); - list_del_init(&cache->link); - cache->tag->cache = NULL; - up_write(&fscache_addremove_sem); - - /* make sure all pages pinned by operations on behalf of the netfs are - * written to disk */ - fscache_stat(&fscache_n_cop_sync_cache); - cache->ops->sync_cache(cache); - fscache_stat_d(&fscache_n_cop_sync_cache); - - /* dissociate all the netfs pages backed by this cache from the block - * mappings in the cache */ - fscache_stat(&fscache_n_cop_dissociate_pages); - cache->ops->dissociate_pages(cache); - fscache_stat_d(&fscache_n_cop_dissociate_pages); - - /* we now have to destroy all the active objects pertaining to this - * cache - which we do by passing them off to thread pool to be - * disposed of */ - _debug("destroy"); - - fscache_withdraw_all_objects(cache, &dying_objects); - - /* wait for all extant objects to finish their outstanding operations - * and go away */ - _debug("wait for finish"); - wait_event(fscache_cache_cleared_wq, - atomic_read(&cache->object_count) == 0); - _debug("wait for clearance"); - wait_event(fscache_cache_cleared_wq, - list_empty(&cache->object_list)); - _debug("cleared"); - ASSERT(list_empty(&dying_objects)); - - kobject_put(cache->kobj); - - clear_bit(FSCACHE_TAG_RESERVED, &cache->tag->flags); - fscache_release_cache_tag(cache->tag); - cache->tag = NULL; - - _leave(""); -} -EXPORT_SYMBOL(fscache_withdraw_cache); diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c deleted file mode 100644 index cd42be646ed3..000000000000 --- a/fs/fscache/cookie.c +++ /dev/null @@ -1,1071 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* netfs cookie management - * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/netfs-api.rst for more information on - * the netfs API. - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include -#include -#include "internal.h" - -struct kmem_cache *fscache_cookie_jar; - -static atomic_t fscache_object_debug_id = ATOMIC_INIT(0); - -#define fscache_cookie_hash_shift 15 -static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; -static LIST_HEAD(fscache_cookies); -static DEFINE_RWLOCK(fscache_cookies_lock); - -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, - loff_t object_size); -static int fscache_alloc_object(struct fscache_cache *cache, - struct fscache_cookie *cookie); -static int fscache_attach_object(struct fscache_cookie *cookie, - struct fscache_object *object); - -static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) -{ - struct fscache_object *object; - struct hlist_node *o; - const u8 *k; - unsigned loop; - - pr_err("%c-cookie c=%08x [p=%08x fl=%lx nc=%u na=%u]\n", - prefix, - cookie->debug_id, - cookie->parent ? cookie->parent->debug_id : 0, - cookie->flags, - atomic_read(&cookie->n_children), - atomic_read(&cookie->n_active)); - pr_err("%c-cookie d=%p{%s} n=%p\n", - prefix, - cookie->def, - cookie->def ? cookie->def->name : "?", - cookie->netfs_data); - - o = READ_ONCE(cookie->backing_objects.first); - if (o) { - object = hlist_entry(o, struct fscache_object, cookie_link); - pr_err("%c-cookie o=%u\n", prefix, object->debug_id); - } - - pr_err("%c-key=[%u] '", prefix, cookie->key_len); - k = (cookie->key_len <= sizeof(cookie->inline_key)) ? - cookie->inline_key : cookie->key; - for (loop = 0; loop < cookie->key_len; loop++) - pr_cont("%02x", k[loop]); - pr_cont("'\n"); -} - -void fscache_free_cookie(struct fscache_cookie *cookie) -{ - if (cookie) { - BUG_ON(!hlist_empty(&cookie->backing_objects)); - write_lock(&fscache_cookies_lock); - list_del(&cookie->proc_link); - write_unlock(&fscache_cookies_lock); - if (cookie->aux_len > sizeof(cookie->inline_aux)) - kfree(cookie->aux); - if (cookie->key_len > sizeof(cookie->inline_key)) - kfree(cookie->key); - kmem_cache_free(fscache_cookie_jar, cookie); - } -} - -/* - * Set the index key in a cookie. The cookie struct has space for a 16-byte - * key plus length and hash, but if that's not big enough, it's instead a - * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then - * the key data. - */ -static int fscache_set_key(struct fscache_cookie *cookie, - const void *index_key, size_t index_key_len) -{ - u32 *buf; - int bufs; - - bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf)); - - if (index_key_len > sizeof(cookie->inline_key)) { - buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL); - if (!buf) - return -ENOMEM; - cookie->key = buf; - } else { - buf = (u32 *)cookie->inline_key; - } - - memcpy(buf, index_key, index_key_len); - cookie->key_hash = fscache_hash(0, buf, bufs); - return 0; -} - -static long fscache_compare_cookie(const struct fscache_cookie *a, - const struct fscache_cookie *b) -{ - const void *ka, *kb; - - if (a->key_hash != b->key_hash) - return (long)a->key_hash - (long)b->key_hash; - if (a->parent != b->parent) - return (long)a->parent - (long)b->parent; - if (a->key_len != b->key_len) - return (long)a->key_len - (long)b->key_len; - if (a->type != b->type) - return (long)a->type - (long)b->type; - - if (a->key_len <= sizeof(a->inline_key)) { - ka = &a->inline_key; - kb = &b->inline_key; - } else { - ka = a->key; - kb = b->key; - } - return memcmp(ka, kb, a->key_len); -} - -static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); - -/* - * Allocate a cookie. - */ -struct fscache_cookie *fscache_alloc_cookie( - struct fscache_cookie *parent, - const struct fscache_cookie_def *def, - const void *index_key, size_t index_key_len, - const void *aux_data, size_t aux_data_len, - void *netfs_data, - loff_t object_size) -{ - struct fscache_cookie *cookie; - - /* allocate and initialise a cookie */ - cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); - if (!cookie) - return NULL; - - cookie->key_len = index_key_len; - cookie->aux_len = aux_data_len; - - if (fscache_set_key(cookie, index_key, index_key_len) < 0) - goto nomem; - - if (cookie->aux_len <= sizeof(cookie->inline_aux)) { - memcpy(cookie->inline_aux, aux_data, cookie->aux_len); - } else { - cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); - if (!cookie->aux) - goto nomem; - } - - refcount_set(&cookie->ref, 1); - atomic_set(&cookie->n_children, 0); - cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); - - /* We keep the active count elevated until relinquishment to prevent an - * attempt to wake up every time the object operations queue quiesces. - */ - atomic_set(&cookie->n_active, 1); - - cookie->def = def; - cookie->parent = parent; - cookie->netfs_data = netfs_data; - cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); - cookie->type = def->type; - spin_lock_init(&cookie->lock); - spin_lock_init(&cookie->stores_lock); - INIT_HLIST_HEAD(&cookie->backing_objects); - - /* radix tree insertion won't use the preallocation pool unless it's - * told it may not wait */ - INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); - - write_lock(&fscache_cookies_lock); - list_add_tail(&cookie->proc_link, &fscache_cookies); - write_unlock(&fscache_cookies_lock); - return cookie; - -nomem: - fscache_free_cookie(cookie); - return NULL; -} - -/* - * Attempt to insert the new cookie into the hash. If there's a collision, we - * return the old cookie if it's not in use and an error otherwise. - */ -struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate) -{ - struct fscache_cookie *cursor; - struct hlist_bl_head *h; - struct hlist_bl_node *p; - unsigned int bucket; - - bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); - h = &fscache_cookie_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_for_each_entry(cursor, p, h, hash_link) { - if (fscache_compare_cookie(candidate, cursor) == 0) - goto collision; - } - - __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags); - fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent); - atomic_inc(&candidate->parent->n_children); - hlist_bl_add_head(&candidate->hash_link, h); - hlist_bl_unlock(h); - return candidate; - -collision: - if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) { - trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), - fscache_cookie_collision); - pr_err("Duplicate cookie detected\n"); - fscache_print_cookie(cursor, 'O'); - fscache_print_cookie(candidate, 'N'); - hlist_bl_unlock(h); - return NULL; - } - - fscache_cookie_get(cursor, fscache_cookie_get_reacquire); - hlist_bl_unlock(h); - return cursor; -} - -/* - * request a cookie to represent an object (index, datafile, xattr, etc) - * - parent specifies the parent object - * - the top level index cookie for each netfs is stored in the fscache_netfs - * struct upon registration - * - def points to the definition - * - the netfs_data will be passed to the functions pointed to in *def - * - all attached caches will be searched to see if they contain this object - * - index objects aren't stored on disk until there's a dependent file that - * needs storing - * - other objects are stored in a selected cache immediately, and all the - * indices forming the path to it are instantiated if necessary - * - we never let on to the netfs about errors - * - we may set a negative cookie pointer, but that's okay - */ -struct fscache_cookie *__fscache_acquire_cookie( - struct fscache_cookie *parent, - const struct fscache_cookie_def *def, - const void *index_key, size_t index_key_len, - const void *aux_data, size_t aux_data_len, - void *netfs_data, - loff_t object_size, - bool enable) -{ - struct fscache_cookie *candidate, *cookie; - - BUG_ON(!def); - - _enter("{%s},{%s},%p,%u", - parent ? (char *) parent->def->name : "", - def->name, netfs_data, enable); - - if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) - return NULL; - if (!aux_data || !aux_data_len) { - aux_data = NULL; - aux_data_len = 0; - } - - fscache_stat(&fscache_n_acquires); - - /* if there's no parent cookie, then we don't create one here either */ - if (!parent) { - fscache_stat(&fscache_n_acquires_null); - _leave(" [no parent]"); - return NULL; - } - - /* validate the definition */ - BUG_ON(!def->name[0]); - - BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX && - parent->type != FSCACHE_COOKIE_TYPE_INDEX); - - candidate = fscache_alloc_cookie(parent, def, - index_key, index_key_len, - aux_data, aux_data_len, - netfs_data, object_size); - if (!candidate) { - fscache_stat(&fscache_n_acquires_oom); - _leave(" [ENOMEM]"); - return NULL; - } - - cookie = fscache_hash_cookie(candidate); - if (!cookie) { - trace_fscache_cookie(candidate->debug_id, 1, - fscache_cookie_discard); - goto out; - } - - if (cookie == candidate) - candidate = NULL; - - switch (cookie->type) { - case FSCACHE_COOKIE_TYPE_INDEX: - fscache_stat(&fscache_n_cookie_index); - break; - case FSCACHE_COOKIE_TYPE_DATAFILE: - fscache_stat(&fscache_n_cookie_data); - break; - default: - fscache_stat(&fscache_n_cookie_special); - break; - } - - trace_fscache_acquire(cookie); - - if (enable) { - /* if the object is an index then we need do nothing more here - * - we create indices on disk when we need them as an index - * may exist in multiple caches */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } else { - atomic_dec(&parent->n_children); - fscache_cookie_put(cookie, - fscache_cookie_put_acquire_nobufs); - fscache_stat(&fscache_n_acquires_nobufs); - _leave(" = NULL"); - return NULL; - } - } else { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } - } - - fscache_stat(&fscache_n_acquires_ok); - -out: - fscache_free_cookie(candidate); - return cookie; -} -EXPORT_SYMBOL(__fscache_acquire_cookie); - -/* - * Enable a cookie to permit it to accept new operations. - */ -void __fscache_enable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - loff_t object_size, - bool (*can_enable)(void *data), - void *data) -{ - _enter("%x", cookie->debug_id); - - trace_fscache_enable(cookie); - - wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, - TASK_UNINTERRUPTIBLE); - - fscache_update_aux(cookie, aux_data); - - if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) - goto out_unlock; - - if (can_enable && !can_enable(data)) { - /* The netfs decided it didn't want to enable after all */ - } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { - /* Wait for outstanding disablement to complete */ - __fscache_wait_on_invalidate(cookie); - - if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } else { - set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); - } - -out_unlock: - clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); -} -EXPORT_SYMBOL(__fscache_enable_cookie); - -/* - * acquire a non-index cookie - * - this must make sure the index chain is instantiated and instantiate the - * object representation too - */ -static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, - loff_t object_size) -{ - struct fscache_object *object; - struct fscache_cache *cache; - int ret; - - _enter(""); - - set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - /* now we need to see whether the backing objects for this cookie yet - * exist, if not there'll be nothing to search */ - down_read(&fscache_addremove_sem); - - if (list_empty(&fscache_cache_list)) { - up_read(&fscache_addremove_sem); - _leave(" = 0 [no caches]"); - return 0; - } - - /* select a cache in which to store the object */ - cache = fscache_select_cache_for_object(cookie->parent); - if (!cache) { - up_read(&fscache_addremove_sem); - fscache_stat(&fscache_n_acquires_no_cache); - _leave(" = -ENOMEDIUM [no cache]"); - return -ENOMEDIUM; - } - - _debug("cache %s", cache->tag->name); - - set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - - /* ask the cache to allocate objects for this cookie and its parent - * chain */ - ret = fscache_alloc_object(cache, cookie); - if (ret < 0) { - up_read(&fscache_addremove_sem); - _leave(" = %d", ret); - return ret; - } - - spin_lock(&cookie->lock); - if (hlist_empty(&cookie->backing_objects)) { - spin_unlock(&cookie->lock); - goto unavailable; - } - - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - fscache_set_store_limit(object, object_size); - - /* initiate the process of looking up all the objects in the chain - * (done by fscache_initialise_object()) */ - fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD); - - spin_unlock(&cookie->lock); - - /* we may be required to wait for lookup to complete at this point */ - if (!fscache_defer_lookup) { - wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, - TASK_UNINTERRUPTIBLE); - if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags)) - goto unavailable; - } - - up_read(&fscache_addremove_sem); - _leave(" = 0 [deferred]"); - return 0; - -unavailable: - up_read(&fscache_addremove_sem); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} - -/* - * recursively allocate cache object records for a cookie/cache combination - * - caller must be holding the addremove sem - */ -static int fscache_alloc_object(struct fscache_cache *cache, - struct fscache_cookie *cookie) -{ - struct fscache_object *object; - int ret; - - _enter("%s,%x{%s}", cache->tag->name, cookie->debug_id, cookie->def->name); - - spin_lock(&cookie->lock); - hlist_for_each_entry(object, &cookie->backing_objects, - cookie_link) { - if (object->cache == cache) - goto object_already_extant; - } - spin_unlock(&cookie->lock); - - /* ask the cache to allocate an object (we may end up with duplicate - * objects at this stage, but we sort that out later) */ - fscache_stat(&fscache_n_cop_alloc_object); - object = cache->ops->alloc_object(cache, cookie); - fscache_stat_d(&fscache_n_cop_alloc_object); - if (IS_ERR(object)) { - fscache_stat(&fscache_n_object_no_alloc); - ret = PTR_ERR(object); - goto error; - } - - ASSERTCMP(object->cookie, ==, cookie); - fscache_stat(&fscache_n_object_alloc); - - object->debug_id = atomic_inc_return(&fscache_object_debug_id); - - _debug("ALLOC OBJ%x: %s {%lx}", - object->debug_id, cookie->def->name, object->events); - - ret = fscache_alloc_object(cache, cookie->parent); - if (ret < 0) - goto error_put; - - /* only attach if we managed to allocate all we needed, otherwise - * discard the object we just allocated and instead use the one - * attached to the cookie */ - if (fscache_attach_object(cookie, object) < 0) { - fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object, fscache_obj_put_attach_fail); - fscache_stat_d(&fscache_n_cop_put_object); - } - - _leave(" = 0"); - return 0; - -object_already_extant: - ret = -ENOBUFS; - if (fscache_object_is_dying(object) || - fscache_cache_is_broken(object)) { - spin_unlock(&cookie->lock); - goto error; - } - spin_unlock(&cookie->lock); - _leave(" = 0 [found]"); - return 0; - -error_put: - fscache_stat(&fscache_n_cop_put_object); - cache->ops->put_object(object, fscache_obj_put_alloc_fail); - fscache_stat_d(&fscache_n_cop_put_object); -error: - _leave(" = %d", ret); - return ret; -} - -/* - * attach a cache object to a cookie - */ -static int fscache_attach_object(struct fscache_cookie *cookie, - struct fscache_object *object) -{ - struct fscache_object *p; - struct fscache_cache *cache = object->cache; - int ret; - - _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id); - - ASSERTCMP(object->cookie, ==, cookie); - - spin_lock(&cookie->lock); - - /* there may be multiple initial creations of this object, but we only - * want one */ - ret = -EEXIST; - hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) { - if (p->cache == object->cache) { - if (fscache_object_is_dying(p)) - ret = -ENOBUFS; - goto cant_attach_object; - } - } - - /* pin the parent object */ - spin_lock_nested(&cookie->parent->lock, 1); - hlist_for_each_entry(p, &cookie->parent->backing_objects, - cookie_link) { - if (p->cache == object->cache) { - if (fscache_object_is_dying(p)) { - ret = -ENOBUFS; - spin_unlock(&cookie->parent->lock); - goto cant_attach_object; - } - object->parent = p; - spin_lock(&p->lock); - p->n_children++; - spin_unlock(&p->lock); - break; - } - } - spin_unlock(&cookie->parent->lock); - - /* attach to the cache's object list */ - if (list_empty(&object->cache_link)) { - spin_lock(&cache->object_list_lock); - list_add(&object->cache_link, &cache->object_list); - spin_unlock(&cache->object_list_lock); - } - - /* Attach to the cookie. The object already has a ref on it. */ - hlist_add_head(&object->cookie_link, &cookie->backing_objects); - ret = 0; - -cant_attach_object: - spin_unlock(&cookie->lock); - _leave(" = %d", ret); - return ret; -} - -/* - * Invalidate an object. Callable with spinlocks held. - */ -void __fscache_invalidate(struct fscache_cookie *cookie) -{ - struct fscache_object *object; - - _enter("{%s}", cookie->def->name); - - fscache_stat(&fscache_n_invalidates); - - /* Only permit invalidation of data files. Invalidating an index will - * require the caller to release all its attachments to the tree rooted - * there, and if it's doing that, it may as well just retire the - * cookie. - */ - ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); - - /* If there's an object, we tell the object state machine to handle the - * invalidation on our behalf, otherwise there's nothing to do. - */ - if (!hlist_empty(&cookie->backing_objects)) { - spin_lock(&cookie->lock); - - if (fscache_cookie_enabled(cookie) && - !hlist_empty(&cookie->backing_objects) && - !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING, - &cookie->flags)) { - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, - cookie_link); - if (fscache_object_is_live(object)) - fscache_raise_event( - object, FSCACHE_OBJECT_EV_INVALIDATE); - } - - spin_unlock(&cookie->lock); - } - - _leave(""); -} -EXPORT_SYMBOL(__fscache_invalidate); - -/* - * Wait for object invalidation to complete. - */ -void __fscache_wait_on_invalidate(struct fscache_cookie *cookie) -{ - _enter("%x", cookie->debug_id); - - wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING, - TASK_UNINTERRUPTIBLE); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_wait_on_invalidate); - -/* - * update the index entries backing a cookie - */ -void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data) -{ - struct fscache_object *object; - - fscache_stat(&fscache_n_updates); - - if (!cookie) { - fscache_stat(&fscache_n_updates_null); - _leave(" [no cookie]"); - return; - } - - _enter("{%s}", cookie->def->name); - - spin_lock(&cookie->lock); - - fscache_update_aux(cookie, aux_data); - - if (fscache_cookie_enabled(cookie)) { - /* update the index entry on disk in each cache backing this - * cookie. - */ - hlist_for_each_entry(object, - &cookie->backing_objects, cookie_link) { - fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE); - } - } - - spin_unlock(&cookie->lock); - _leave(""); -} -EXPORT_SYMBOL(__fscache_update_cookie); - -/* - * Disable a cookie to stop it from accepting new requests from the netfs. - */ -void __fscache_disable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool invalidate) -{ - struct fscache_object *object; - bool awaken = false; - - _enter("%x,%u", cookie->debug_id, invalidate); - - trace_fscache_disable(cookie); - - ASSERTCMP(atomic_read(&cookie->n_active), >, 0); - - if (atomic_read(&cookie->n_children) != 0) { - pr_err("Cookie '%s' still has children\n", - cookie->def->name); - BUG(); - } - - wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, - TASK_UNINTERRUPTIBLE); - - fscache_update_aux(cookie, aux_data); - - if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) - goto out_unlock_enable; - - /* If the cookie is being invalidated, wait for that to complete first - * so that we can reuse the flag. - */ - __fscache_wait_on_invalidate(cookie); - - /* Dispose of the backing objects */ - set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags); - - spin_lock(&cookie->lock); - if (!hlist_empty(&cookie->backing_objects)) { - hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) { - if (invalidate) - set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL); - } - } else { - if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - awaken = true; - } - spin_unlock(&cookie->lock); - if (awaken) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - - /* Wait for cessation of activity requiring access to the netfs (when - * n_active reaches 0). This makes sure outstanding reads and writes - * have completed. - */ - if (!atomic_dec_and_test(&cookie->n_active)) { - wait_var_event(&cookie->n_active, - !atomic_read(&cookie->n_active)); - } - - /* Make sure any pending writes are cancelled. */ - if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) - fscache_invalidate_writes(cookie); - - /* Reset the cookie state if it wasn't relinquished */ - if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) { - atomic_inc(&cookie->n_active); - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - } - -out_unlock_enable: - clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); - _leave(""); -} -EXPORT_SYMBOL(__fscache_disable_cookie); - -/* - * release a cookie back to the cache - * - the object will be marked as recyclable on disk if retire is true - * - all dependents of this cookie must have already been unregistered - * (indices/files/pages) - */ -void __fscache_relinquish_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool retire) -{ - fscache_stat(&fscache_n_relinquishes); - if (retire) - fscache_stat(&fscache_n_relinquishes_retire); - - if (!cookie) { - fscache_stat(&fscache_n_relinquishes_null); - _leave(" [no cookie]"); - return; - } - - _enter("%x{%s,%d},%d", - cookie->debug_id, cookie->def->name, - atomic_read(&cookie->n_active), retire); - - trace_fscache_relinquish(cookie, retire); - - /* No further netfs-accessing operations on this cookie permitted */ - if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) - BUG(); - - __fscache_disable_cookie(cookie, aux_data, retire); - - /* Clear pointers back to the netfs */ - cookie->netfs_data = NULL; - cookie->def = NULL; - BUG_ON(!radix_tree_empty(&cookie->stores)); - - if (cookie->parent) { - ASSERTCMP(refcount_read(&cookie->parent->ref), >, 0); - ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0); - atomic_dec(&cookie->parent->n_children); - } - - /* Dispose of the netfs's link to the cookie */ - fscache_cookie_put(cookie, fscache_cookie_put_relinquish); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_relinquish_cookie); - -/* - * Remove a cookie from the hash table. - */ -static void fscache_unhash_cookie(struct fscache_cookie *cookie) -{ - struct hlist_bl_head *h; - unsigned int bucket; - - bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); - h = &fscache_cookie_hash[bucket]; - - hlist_bl_lock(h); - hlist_bl_del(&cookie->hash_link); - hlist_bl_unlock(h); -} - -/* - * Drop a reference to a cookie. - */ -void fscache_cookie_put(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - struct fscache_cookie *parent; - int ref; - - _enter("%x", cookie->debug_id); - - do { - unsigned int cookie_debug_id = cookie->debug_id; - bool zero = __refcount_dec_and_test(&cookie->ref, &ref); - - trace_fscache_cookie(cookie_debug_id, ref - 1, where); - if (!zero) - return; - - parent = cookie->parent; - fscache_unhash_cookie(cookie); - fscache_free_cookie(cookie); - - cookie = parent; - where = fscache_cookie_put_parent; - } while (cookie); - - _leave(""); -} - -/* - * Get a reference to a cookie. - */ -struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - int ref; - - __refcount_inc(&cookie->ref, &ref); - trace_fscache_cookie(cookie->debug_id, ref + 1, where); - return cookie; -} - -/* - * check the consistency between the netfs inode and the backing cache - * - * NOTE: it only serves no-index type - */ -int __fscache_check_consistency(struct fscache_cookie *cookie, - const void *aux_data) -{ - struct fscache_operation *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,", cookie); - - ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - if (hlist_empty(&cookie->backing_objects)) - return 0; - - op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY); - if (!op) - return -ENOMEM; - - fscache_operation_init(cookie, op, NULL, NULL, NULL); - op->flags = FSCACHE_OP_MYTHREAD | - (1 << FSCACHE_OP_WAITING) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency); - - spin_lock(&cookie->lock); - - fscache_update_aux(cookie, aux_data); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto inconsistent; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) - goto inconsistent; - - op->debug_id = atomic_inc_return(&fscache_op_debug_id); - - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, op) < 0) - goto submit_failed; - - /* the work queue now carries its own ref on the object */ - spin_unlock(&cookie->lock); - - ret = fscache_wait_for_operation_activation(object, op, NULL, NULL); - if (ret == 0) { - /* ask the cache to honour the operation */ - ret = object->cache->ops->check_consistency(op); - fscache_op_complete(op, false); - } else if (ret == -ENOBUFS) { - ret = 0; - } - - fscache_put_operation(op); - _leave(" = %d", ret); - return ret; - -submit_failed: - wake_cookie = __fscache_unuse_cookie(cookie); -inconsistent: - spin_unlock(&cookie->lock); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - kfree(op); - _leave(" = -ESTALE"); - return -ESTALE; -} -EXPORT_SYMBOL(__fscache_check_consistency); - -/* - * Generate a list of extant cookies in /proc/fs/fscache/cookies - */ -static int fscache_cookies_seq_show(struct seq_file *m, void *v) -{ - struct fscache_cookie *cookie; - unsigned int keylen = 0, auxlen = 0; - char _type[3], *type; - u8 *p; - - if (v == &fscache_cookies) { - seq_puts(m, - "COOKIE PARENT USAGE CHILD ACT TY FL DEF NETFS_DATA\n" - "======== ======== ===== ===== === == === ================ ==========\n" - ); - return 0; - } - - cookie = list_entry(v, struct fscache_cookie, proc_link); - - switch (cookie->type) { - case 0: - type = "IX"; - break; - case 1: - type = "DT"; - break; - default: - snprintf(_type, sizeof(_type), "%02u", - cookie->type); - type = _type; - break; - } - - seq_printf(m, - "%08x %08x %5u %5u %3u %s %03lx %-16s %px", - cookie->debug_id, - cookie->parent ? cookie->parent->debug_id : 0, - refcount_read(&cookie->ref), - atomic_read(&cookie->n_children), - atomic_read(&cookie->n_active), - type, - cookie->flags, - cookie->def->name, - cookie->netfs_data); - - keylen = cookie->key_len; - auxlen = cookie->aux_len; - - if (keylen > 0 || auxlen > 0) { - seq_puts(m, " "); - p = keylen <= sizeof(cookie->inline_key) ? - cookie->inline_key : cookie->key; - for (; keylen > 0; keylen--) - seq_printf(m, "%02x", *p++); - if (auxlen > 0) { - seq_puts(m, ", "); - p = auxlen <= sizeof(cookie->inline_aux) ? - cookie->inline_aux : cookie->aux; - for (; auxlen > 0; auxlen--) - seq_printf(m, "%02x", *p++); - } - } - - seq_puts(m, "\n"); - return 0; -} - -static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) - __acquires(fscache_cookies_lock) -{ - read_lock(&fscache_cookies_lock); - return seq_list_start_head(&fscache_cookies, *_pos); -} - -static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) -{ - return seq_list_next(v, &fscache_cookies, _pos); -} - -static void fscache_cookies_seq_stop(struct seq_file *m, void *v) - __releases(rcu) -{ - read_unlock(&fscache_cookies_lock); -} - - -const struct seq_operations fscache_cookies_seq_ops = { - .start = fscache_cookies_seq_start, - .next = fscache_cookies_seq_next, - .stop = fscache_cookies_seq_stop, - .show = fscache_cookies_seq_show, -}; diff --git a/fs/fscache/fsdef.c b/fs/fscache/fsdef.c deleted file mode 100644 index 0402673c680e..000000000000 --- a/fs/fscache/fsdef.c +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Filesystem index definition - * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include "internal.h" - -static -enum fscache_checkaux fscache_fsdef_netfs_check_aux(void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size); - -/* - * The root index is owned by FS-Cache itself. - * - * When a netfs requests caching facilities, FS-Cache will, if one doesn't - * already exist, create an entry in the root index with the key being the name - * of the netfs ("AFS" for example), and the auxiliary data holding the index - * structure version supplied by the netfs: - * - * FSDEF - * | - * +-----------+ - * | | - * NFS AFS - * [v=1] [v=1] - * - * If an entry with the appropriate name does already exist, the version is - * compared. If the version is different, the entire subtree from that entry - * will be discarded and a new entry created. - * - * The new entry will be an index, and a cookie referring to it will be passed - * to the netfs. This is then the root handle by which the netfs accesses the - * cache. It can create whatever objects it likes in that index, including - * further indices. - */ -static struct fscache_cookie_def fscache_fsdef_index_def = { - .name = ".FS-Cache", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -struct fscache_cookie fscache_fsdef_index = { - .debug_id = 1, - .ref = REFCOUNT_INIT(1), - .n_active = ATOMIC_INIT(1), - .lock = __SPIN_LOCK_UNLOCKED(fscache_fsdef_index.lock), - .backing_objects = HLIST_HEAD_INIT, - .def = &fscache_fsdef_index_def, - .flags = 1 << FSCACHE_COOKIE_ENABLED, - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; -EXPORT_SYMBOL(fscache_fsdef_index); - -/* - * Definition of an entry in the root index. Each entry is an index, keyed to - * a specific netfs and only applicable to a particular version of the index - * structure used by that netfs. - */ -struct fscache_cookie_def fscache_fsdef_netfs_def = { - .name = "FSDEF.netfs", - .type = FSCACHE_COOKIE_TYPE_INDEX, - .check_aux = fscache_fsdef_netfs_check_aux, -}; - -/* - * check that the index structure version number stored in the auxiliary data - * matches the one the netfs gave us - */ -static enum fscache_checkaux fscache_fsdef_netfs_check_aux( - void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size) -{ - struct fscache_netfs *netfs = cookie_netfs_data; - uint32_t version; - - _enter("{%s},,%hu", netfs->name, datalen); - - if (datalen != sizeof(version)) { - _leave(" = OBSOLETE [dl=%d v=%zu]", datalen, sizeof(version)); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - memcpy(&version, data, sizeof(version)); - if (version != netfs->version) { - _leave(" = OBSOLETE [ver=%x net=%x]", version, netfs->version); - return FSCACHE_CHECKAUX_OBSOLETE; - } - - _leave(" = OKAY"); - return FSCACHE_CHECKAUX_OKAY; -} diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h deleted file mode 100644 index c3e4804b8fcb..000000000000 --- a/fs/fscache/internal.h +++ /dev/null @@ -1,461 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Internal definitions for FS-Cache - * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -/* - * Lock order, in the order in which multiple locks should be obtained: - * - fscache_addremove_sem - * - cookie->lock - * - cookie->parent->lock - * - cache->object_list_lock - * - object->lock - * - object->parent->lock - * - cookie->stores_lock - * - fscache_thread_lock - * - */ - -#ifdef pr_fmt -#undef pr_fmt -#endif - -#define pr_fmt(fmt) "FS-Cache: " fmt - -#include -#include -#include -#include - -#define FSCACHE_MIN_THREADS 4 -#define FSCACHE_MAX_THREADS 32 - -/* - * cache.c - */ -extern struct list_head fscache_cache_list; -extern struct rw_semaphore fscache_addremove_sem; - -extern struct fscache_cache *fscache_select_cache_for_object( - struct fscache_cookie *); - -/* - * cookie.c - */ -extern struct kmem_cache *fscache_cookie_jar; -extern const struct seq_operations fscache_cookies_seq_ops; - -extern void fscache_free_cookie(struct fscache_cookie *); -extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *, - const struct fscache_cookie_def *, - const void *, size_t, - const void *, size_t, - void *, loff_t); -extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *); -extern struct fscache_cookie *fscache_cookie_get(struct fscache_cookie *, - enum fscache_cookie_trace); -extern void fscache_cookie_put(struct fscache_cookie *, - enum fscache_cookie_trace); - -static inline void fscache_cookie_see(struct fscache_cookie *cookie, - enum fscache_cookie_trace where) -{ - trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), - where); -} - -/* - * fsdef.c - */ -extern struct fscache_cookie fscache_fsdef_index; -extern struct fscache_cookie_def fscache_fsdef_netfs_def; - -/* - * main.c - */ -extern unsigned fscache_defer_lookup; -extern unsigned fscache_defer_create; -extern unsigned fscache_debug; -extern struct kobject *fscache_root; -extern struct workqueue_struct *fscache_object_wq; -extern struct workqueue_struct *fscache_op_wq; -DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); - -extern unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n); - -static inline bool fscache_object_congested(void) -{ - return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq); -} - -/* - * object.c - */ -extern void fscache_enqueue_object(struct fscache_object *); - -/* - * operation.c - */ -extern int fscache_submit_exclusive_op(struct fscache_object *, - struct fscache_operation *); -extern int fscache_submit_op(struct fscache_object *, - struct fscache_operation *); -extern int fscache_cancel_op(struct fscache_operation *, bool); -extern void fscache_cancel_all_ops(struct fscache_object *); -extern void fscache_abort_object(struct fscache_object *); -extern void fscache_start_operations(struct fscache_object *); -extern void fscache_operation_gc(struct work_struct *); - -/* - * page.c - */ -extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *); -extern int fscache_wait_for_operation_activation(struct fscache_object *, - struct fscache_operation *, - atomic_t *, - atomic_t *); -extern void fscache_invalidate_writes(struct fscache_cookie *); -struct fscache_retrieval *fscache_alloc_retrieval(struct fscache_cookie *cookie, - struct address_space *mapping, - fscache_rw_complete_t end_io_func, - void *context); - -/* - * proc.c - */ -#ifdef CONFIG_PROC_FS -extern int __init fscache_proc_init(void); -extern void fscache_proc_cleanup(void); -#else -#define fscache_proc_init() (0) -#define fscache_proc_cleanup() do {} while (0) -#endif - -/* - * stats.c - */ -#ifdef CONFIG_FSCACHE_STATS -extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS]; -extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS]; - -extern atomic_t fscache_n_op_pend; -extern atomic_t fscache_n_op_run; -extern atomic_t fscache_n_op_enqueue; -extern atomic_t fscache_n_op_deferred_release; -extern atomic_t fscache_n_op_initialised; -extern atomic_t fscache_n_op_release; -extern atomic_t fscache_n_op_gc; -extern atomic_t fscache_n_op_cancelled; -extern atomic_t fscache_n_op_rejected; - -extern atomic_t fscache_n_attr_changed; -extern atomic_t fscache_n_attr_changed_ok; -extern atomic_t fscache_n_attr_changed_nobufs; -extern atomic_t fscache_n_attr_changed_nomem; -extern atomic_t fscache_n_attr_changed_calls; - -extern atomic_t fscache_n_allocs; -extern atomic_t fscache_n_allocs_ok; -extern atomic_t fscache_n_allocs_wait; -extern atomic_t fscache_n_allocs_nobufs; -extern atomic_t fscache_n_allocs_intr; -extern atomic_t fscache_n_allocs_object_dead; -extern atomic_t fscache_n_alloc_ops; -extern atomic_t fscache_n_alloc_op_waits; - -extern atomic_t fscache_n_retrievals; -extern atomic_t fscache_n_retrievals_ok; -extern atomic_t fscache_n_retrievals_wait; -extern atomic_t fscache_n_retrievals_nodata; -extern atomic_t fscache_n_retrievals_nobufs; -extern atomic_t fscache_n_retrievals_intr; -extern atomic_t fscache_n_retrievals_nomem; -extern atomic_t fscache_n_retrievals_object_dead; -extern atomic_t fscache_n_retrieval_ops; -extern atomic_t fscache_n_retrieval_op_waits; - -extern atomic_t fscache_n_stores; -extern atomic_t fscache_n_stores_ok; -extern atomic_t fscache_n_stores_again; -extern atomic_t fscache_n_stores_nobufs; -extern atomic_t fscache_n_stores_oom; -extern atomic_t fscache_n_store_ops; -extern atomic_t fscache_n_store_calls; -extern atomic_t fscache_n_store_pages; -extern atomic_t fscache_n_store_radix_deletes; -extern atomic_t fscache_n_store_pages_over_limit; - -extern atomic_t fscache_n_store_vmscan_not_storing; -extern atomic_t fscache_n_store_vmscan_gone; -extern atomic_t fscache_n_store_vmscan_busy; -extern atomic_t fscache_n_store_vmscan_cancelled; -extern atomic_t fscache_n_store_vmscan_wait; - -extern atomic_t fscache_n_marks; -extern atomic_t fscache_n_uncaches; - -extern atomic_t fscache_n_acquires; -extern atomic_t fscache_n_acquires_null; -extern atomic_t fscache_n_acquires_no_cache; -extern atomic_t fscache_n_acquires_ok; -extern atomic_t fscache_n_acquires_nobufs; -extern atomic_t fscache_n_acquires_oom; - -extern atomic_t fscache_n_invalidates; -extern atomic_t fscache_n_invalidates_run; - -extern atomic_t fscache_n_updates; -extern atomic_t fscache_n_updates_null; -extern atomic_t fscache_n_updates_run; - -extern atomic_t fscache_n_relinquishes; -extern atomic_t fscache_n_relinquishes_null; -extern atomic_t fscache_n_relinquishes_waitcrt; -extern atomic_t fscache_n_relinquishes_retire; - -extern atomic_t fscache_n_cookie_index; -extern atomic_t fscache_n_cookie_data; -extern atomic_t fscache_n_cookie_special; - -extern atomic_t fscache_n_object_alloc; -extern atomic_t fscache_n_object_no_alloc; -extern atomic_t fscache_n_object_lookups; -extern atomic_t fscache_n_object_lookups_negative; -extern atomic_t fscache_n_object_lookups_positive; -extern atomic_t fscache_n_object_lookups_timed_out; -extern atomic_t fscache_n_object_created; -extern atomic_t fscache_n_object_avail; -extern atomic_t fscache_n_object_dead; - -extern atomic_t fscache_n_checkaux_none; -extern atomic_t fscache_n_checkaux_okay; -extern atomic_t fscache_n_checkaux_update; -extern atomic_t fscache_n_checkaux_obsolete; - -extern atomic_t fscache_n_cop_alloc_object; -extern atomic_t fscache_n_cop_lookup_object; -extern atomic_t fscache_n_cop_lookup_complete; -extern atomic_t fscache_n_cop_grab_object; -extern atomic_t fscache_n_cop_invalidate_object; -extern atomic_t fscache_n_cop_update_object; -extern atomic_t fscache_n_cop_drop_object; -extern atomic_t fscache_n_cop_put_object; -extern atomic_t fscache_n_cop_sync_cache; -extern atomic_t fscache_n_cop_attr_changed; -extern atomic_t fscache_n_cop_read_or_alloc_page; -extern atomic_t fscache_n_cop_read_or_alloc_pages; -extern atomic_t fscache_n_cop_allocate_page; -extern atomic_t fscache_n_cop_allocate_pages; -extern atomic_t fscache_n_cop_write_page; -extern atomic_t fscache_n_cop_uncache_page; -extern atomic_t fscache_n_cop_dissociate_pages; - -extern atomic_t fscache_n_cache_no_space_reject; -extern atomic_t fscache_n_cache_stale_objects; -extern atomic_t fscache_n_cache_retired_objects; -extern atomic_t fscache_n_cache_culled_objects; - -static inline void fscache_stat(atomic_t *stat) -{ - atomic_inc(stat); -} - -static inline void fscache_stat_d(atomic_t *stat) -{ - atomic_dec(stat); -} - -#define __fscache_stat(stat) (stat) - -int fscache_stats_show(struct seq_file *m, void *v); -#else - -#define __fscache_stat(stat) (NULL) -#define fscache_stat(stat) do {} while (0) -#define fscache_stat_d(stat) do {} while (0) -#endif - -/* - * raise an event on an object - * - if the event is not masked for that object, then the object is - * queued for attention by the thread pool. - */ -static inline void fscache_raise_event(struct fscache_object *object, - unsigned event) -{ - BUG_ON(event >= NR_FSCACHE_OBJECT_EVENTS); -#if 0 - printk("*** fscache_raise_event(OBJ%d{%lx},%x)\n", - object->debug_id, object->event_mask, (1 << event)); -#endif - if (!test_and_set_bit(event, &object->events) && - test_bit(event, &object->event_mask)) - fscache_enqueue_object(object); -} - -/* - * get an extra reference to a netfs retrieval context - */ -static inline -void *fscache_get_context(struct fscache_cookie *cookie, void *context) -{ - if (cookie->def->get_context) - cookie->def->get_context(cookie->netfs_data, context); - return context; -} - -/* - * release a reference to a netfs retrieval context - */ -static inline -void fscache_put_context(struct fscache_cookie *cookie, void *context) -{ - if (cookie->def->put_context) - cookie->def->put_context(cookie->netfs_data, context); -} - -/* - * Update the auxiliary data on a cookie. - */ -static inline -void fscache_update_aux(struct fscache_cookie *cookie, const void *aux_data) -{ - void *p; - - if (!aux_data) - return; - if (cookie->aux_len <= sizeof(cookie->inline_aux)) - p = cookie->inline_aux; - else - p = cookie->aux; - - if (memcmp(p, aux_data, cookie->aux_len) != 0) { - memcpy(p, aux_data, cookie->aux_len); - set_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags); - } -} - -/*****************************************************************************/ -/* - * debug tracing - */ -#define dbgprintk(FMT, ...) \ - printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) - -#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) -#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) -#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) - -#define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) - -#ifdef __KDEBUG -#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) -#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) -#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) - -#elif defined(CONFIG_FSCACHE_DEBUG) -#define _enter(FMT, ...) \ -do { \ - if (__do_kdebug(ENTER)) \ - kenter(FMT, ##__VA_ARGS__); \ -} while (0) - -#define _leave(FMT, ...) \ -do { \ - if (__do_kdebug(LEAVE)) \ - kleave(FMT, ##__VA_ARGS__); \ -} while (0) - -#define _debug(FMT, ...) \ -do { \ - if (__do_kdebug(DEBUG)) \ - kdebug(FMT, ##__VA_ARGS__); \ -} while (0) - -#else -#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) -#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) -#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) -#endif - -/* - * determine whether a particular optional debugging point should be logged - * - we need to go through three steps to persuade cpp to correctly join the - * shorthand in FSCACHE_DEBUG_LEVEL with its prefix - */ -#define ____do_kdebug(LEVEL, POINT) \ - unlikely((fscache_debug & \ - (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) -#define ___do_kdebug(LEVEL, POINT) \ - ____do_kdebug(LEVEL, POINT) -#define __do_kdebug(POINT) \ - ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) - -#define FSCACHE_DEBUG_CACHE 0 -#define FSCACHE_DEBUG_COOKIE 1 -#define FSCACHE_DEBUG_PAGE 2 -#define FSCACHE_DEBUG_OPERATION 3 - -#define FSCACHE_POINT_ENTER 1 -#define FSCACHE_POINT_LEAVE 2 -#define FSCACHE_POINT_DEBUG 4 - -#ifndef FSCACHE_DEBUG_LEVEL -#define FSCACHE_DEBUG_LEVEL CACHE -#endif - -/* - * assertions - */ -#if 1 /* defined(__KDEBUGALL) */ - -#define ASSERT(X) \ -do { \ - if (unlikely(!(X))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTCMP(X, OP, Y) \ -do { \ - if (unlikely(!((X) OP (Y)))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - pr_err("%lx " #OP " %lx is false\n", \ - (unsigned long)(X), (unsigned long)(Y)); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTIF(C, X) \ -do { \ - if (unlikely((C) && !(X))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - BUG(); \ - } \ -} while (0) - -#define ASSERTIFCMP(C, X, OP, Y) \ -do { \ - if (unlikely((C) && !((X) OP (Y)))) { \ - pr_err("\n"); \ - pr_err("Assertion failed\n"); \ - pr_err("%lx " #OP " %lx is false\n", \ - (unsigned long)(X), (unsigned long)(Y)); \ - BUG(); \ - } \ -} while (0) - -#else - -#define ASSERT(X) do {} while (0) -#define ASSERTCMP(X, OP, Y) do {} while (0) -#define ASSERTIF(C, X) do {} while (0) -#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) - -#endif /* assert or not */ diff --git a/fs/fscache/io.c b/fs/fscache/io.c deleted file mode 100644 index 8ecc1141802f..000000000000 --- a/fs/fscache/io.c +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Cache data I/O routines - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL PAGE -#include -#define FSCACHE_USE_NEW_IO_API -#include -#include -#include -#include "internal.h" - -/* - * Start a cache read operation. - * - we return: - * -ENOMEM - out of memory, some pages may be being read - * -ERESTARTSYS - interrupted, some pages may be being read - * -ENOBUFS - no backing object or space available in which to cache any - * pages not being read - * -ENODATA - no data available in the backing object for some or all of - * the pages - * 0 - dispatched a read on all pages - */ -int __fscache_begin_read_operation(struct netfs_read_request *rreq, - struct fscache_cookie *cookie) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("rr=%08x", rreq->debug_id); - - fscache_stat(&fscache_n_retrievals); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, NULL, NULL, NULL); - if (!op) - return -ENOMEM; - trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); - - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_retrieval_ops); - - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - ret = object->cache->ops->begin_read_operation(rreq, op); - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_begin_read_operation); diff --git a/fs/fscache/main.c b/fs/fscache/main.c deleted file mode 100644 index 4207f98e405f..000000000000 --- a/fs/fscache/main.c +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* General filesystem local caching manager - * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL CACHE -#include -#include -#include -#include -#include -#include -#define CREATE_TRACE_POINTS -#include "internal.h" - -MODULE_DESCRIPTION("FS Cache Manager"); -MODULE_AUTHOR("Red Hat, Inc."); -MODULE_LICENSE("GPL"); - -unsigned fscache_defer_lookup = 1; -module_param_named(defer_lookup, fscache_defer_lookup, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_defer_lookup, - "Defer cookie lookup to background thread"); - -unsigned fscache_defer_create = 1; -module_param_named(defer_create, fscache_defer_create, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_defer_create, - "Defer cookie creation to background thread"); - -unsigned fscache_debug; -module_param_named(debug, fscache_debug, uint, - S_IWUSR | S_IRUGO); -MODULE_PARM_DESC(fscache_debug, - "FS-Cache debugging mask"); - -struct kobject *fscache_root; -struct workqueue_struct *fscache_object_wq; -struct workqueue_struct *fscache_op_wq; - -DEFINE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); - -/* these values serve as lower bounds, will be adjusted in fscache_init() */ -static unsigned fscache_object_max_active = 4; -static unsigned fscache_op_max_active = 2; - -#ifdef CONFIG_SYSCTL -static struct ctl_table_header *fscache_sysctl_header; - -static int fscache_max_active_sysctl(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - struct workqueue_struct **wqp = table->extra1; - unsigned int *datap = table->data; - int ret; - - ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret == 0) - workqueue_set_max_active(*wqp, *datap); - return ret; -} - -static struct ctl_table fscache_sysctls[] = { - { - .procname = "object_max_active", - .data = &fscache_object_max_active, - .maxlen = sizeof(unsigned), - .mode = 0644, - .proc_handler = fscache_max_active_sysctl, - .extra1 = &fscache_object_wq, - }, - { - .procname = "operation_max_active", - .data = &fscache_op_max_active, - .maxlen = sizeof(unsigned), - .mode = 0644, - .proc_handler = fscache_max_active_sysctl, - .extra1 = &fscache_op_wq, - }, - {} -}; - -static struct ctl_table fscache_sysctls_root[] = { - { - .procname = "fscache", - .mode = 0555, - .child = fscache_sysctls, - }, - {} -}; -#endif - -/* - * Mixing scores (in bits) for (7,20): - * Input delta: 1-bit 2-bit - * 1 round: 330.3 9201.6 - * 2 rounds: 1246.4 25475.4 - * 3 rounds: 1907.1 31295.1 - * 4 rounds: 2042.3 31718.6 - * Perfect: 2048 31744 - * (32*64) (32*31/2 * 64) - */ -#define HASH_MIX(x, y, a) \ - ( x ^= (a), \ - y ^= x, x = rol32(x, 7),\ - x += y, y = rol32(y,20),\ - y *= 9 ) - -static inline unsigned int fold_hash(unsigned long x, unsigned long y) -{ - /* Use arch-optimized multiply if one exists */ - return __hash_32(y ^ __hash_32(x)); -} - -/* - * Generate a hash. This is derived from full_name_hash(), but we want to be - * sure it is arch independent and that it doesn't change as bits of the - * computed hash value might appear on disk. The caller also guarantees that - * the hashed data will be a series of aligned 32-bit words. - */ -unsigned int fscache_hash(unsigned int salt, unsigned int *data, unsigned int n) -{ - unsigned int a, x = 0, y = salt; - - for (; n; n--) { - a = *data++; - HASH_MIX(x, y, a); - } - return fold_hash(x, y); -} - -/* - * initialise the fs caching module - */ -static int __init fscache_init(void) -{ - unsigned int nr_cpus = num_possible_cpus(); - unsigned int cpu; - int ret; - - fscache_object_max_active = - clamp_val(nr_cpus, - fscache_object_max_active, WQ_UNBOUND_MAX_ACTIVE); - - ret = -ENOMEM; - fscache_object_wq = alloc_workqueue("fscache_object", WQ_UNBOUND, - fscache_object_max_active); - if (!fscache_object_wq) - goto error_object_wq; - - fscache_op_max_active = - clamp_val(fscache_object_max_active / 2, - fscache_op_max_active, WQ_UNBOUND_MAX_ACTIVE); - - ret = -ENOMEM; - fscache_op_wq = alloc_workqueue("fscache_operation", WQ_UNBOUND, - fscache_op_max_active); - if (!fscache_op_wq) - goto error_op_wq; - - for_each_possible_cpu(cpu) - init_waitqueue_head(&per_cpu(fscache_object_cong_wait, cpu)); - - ret = fscache_proc_init(); - if (ret < 0) - goto error_proc; - -#ifdef CONFIG_SYSCTL - ret = -ENOMEM; - fscache_sysctl_header = register_sysctl_table(fscache_sysctls_root); - if (!fscache_sysctl_header) - goto error_sysctl; -#endif - - fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", - sizeof(struct fscache_cookie), - 0, 0, NULL); - if (!fscache_cookie_jar) { - pr_notice("Failed to allocate a cookie jar\n"); - ret = -ENOMEM; - goto error_cookie_jar; - } - - fscache_root = kobject_create_and_add("fscache", kernel_kobj); - if (!fscache_root) - goto error_kobj; - - pr_notice("Loaded\n"); - return 0; - -error_kobj: - kmem_cache_destroy(fscache_cookie_jar); -error_cookie_jar: -#ifdef CONFIG_SYSCTL - unregister_sysctl_table(fscache_sysctl_header); -error_sysctl: -#endif - fscache_proc_cleanup(); -error_proc: - destroy_workqueue(fscache_op_wq); -error_op_wq: - destroy_workqueue(fscache_object_wq); -error_object_wq: - return ret; -} - -fs_initcall(fscache_init); - -/* - * clean up on module removal - */ -static void __exit fscache_exit(void) -{ - _enter(""); - - kobject_put(fscache_root); - kmem_cache_destroy(fscache_cookie_jar); -#ifdef CONFIG_SYSCTL - unregister_sysctl_table(fscache_sysctl_header); -#endif - fscache_proc_cleanup(); - destroy_workqueue(fscache_op_wq); - destroy_workqueue(fscache_object_wq); - pr_notice("Unloaded\n"); -} - -module_exit(fscache_exit); diff --git a/fs/fscache/netfs.c b/fs/fscache/netfs.c deleted file mode 100644 index d6bdb7b5e723..000000000000 --- a/fs/fscache/netfs.c +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache netfs (client) registration - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include -#include -#include "internal.h" - -/* - * register a network filesystem for caching - */ -int __fscache_register_netfs(struct fscache_netfs *netfs) -{ - struct fscache_cookie *candidate, *cookie; - - _enter("{%s}", netfs->name); - - /* allocate a cookie for the primary index */ - candidate = fscache_alloc_cookie(&fscache_fsdef_index, - &fscache_fsdef_netfs_def, - netfs->name, strlen(netfs->name), - &netfs->version, sizeof(netfs->version), - netfs, 0); - if (!candidate) { - _leave(" = -ENOMEM"); - return -ENOMEM; - } - - candidate->flags = 1 << FSCACHE_COOKIE_ENABLED; - - /* check the netfs type is not already present */ - cookie = fscache_hash_cookie(candidate); - if (!cookie) - goto already_registered; - if (cookie != candidate) { - trace_fscache_cookie(candidate->debug_id, 1, fscache_cookie_discard); - fscache_free_cookie(candidate); - } - - fscache_cookie_get(cookie->parent, fscache_cookie_get_register_netfs); - atomic_inc(&cookie->parent->n_children); - - netfs->primary_index = cookie; - - pr_notice("Netfs '%s' registered for caching\n", netfs->name); - trace_fscache_netfs(netfs); - _leave(" = 0"); - return 0; - -already_registered: - fscache_cookie_put(candidate, fscache_cookie_put_dup_netfs); - _leave(" = -EEXIST"); - return -EEXIST; -} -EXPORT_SYMBOL(__fscache_register_netfs); - -/* - * unregister a network filesystem from the cache - * - all cookies must have been released first - */ -void __fscache_unregister_netfs(struct fscache_netfs *netfs) -{ - _enter("{%s.%u}", netfs->name, netfs->version); - - fscache_relinquish_cookie(netfs->primary_index, NULL, false); - pr_notice("Netfs '%s' unregistered from caching\n", netfs->name); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_unregister_netfs); diff --git a/fs/fscache/object.c b/fs/fscache/object.c deleted file mode 100644 index 6a675652129b..000000000000 --- a/fs/fscache/object.c +++ /dev/null @@ -1,1125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache object state machine handler - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/object.rst for a description of the - * object state machine and the in-kernel representations. - */ - -#define FSCACHE_DEBUG_LEVEL COOKIE -#include -#include -#include -#include "internal.h" - -static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *, int); -static const struct fscache_state *fscache_kill_dependents(struct fscache_object *, int); -static const struct fscache_state *fscache_drop_object(struct fscache_object *, int); -static const struct fscache_state *fscache_initialise_object(struct fscache_object *, int); -static const struct fscache_state *fscache_invalidate_object(struct fscache_object *, int); -static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *, int); -static const struct fscache_state *fscache_kill_object(struct fscache_object *, int); -static const struct fscache_state *fscache_lookup_failure(struct fscache_object *, int); -static const struct fscache_state *fscache_look_up_object(struct fscache_object *, int); -static const struct fscache_state *fscache_object_available(struct fscache_object *, int); -static const struct fscache_state *fscache_parent_ready(struct fscache_object *, int); -static const struct fscache_state *fscache_update_object(struct fscache_object *, int); -static const struct fscache_state *fscache_object_dead(struct fscache_object *, int); - -#define __STATE_NAME(n) fscache_osm_##n -#define STATE(n) (&__STATE_NAME(n)) - -/* - * Define a work state. Work states are execution states. No event processing - * is performed by them. The function attached to a work state returns a - * pointer indicating the next state to which the state machine should - * transition. Returning NO_TRANSIT repeats the current state, but goes back - * to the scheduler first. - */ -#define WORK_STATE(n, sn, f) \ - const struct fscache_state __STATE_NAME(n) = { \ - .name = #n, \ - .short_name = sn, \ - .work = f \ - } - -/* - * Returns from work states. - */ -#define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); }) - -#define NO_TRANSIT ((struct fscache_state *)NULL) - -/* - * Define a wait state. Wait states are event processing states. No execution - * is performed by them. Wait states are just tables of "if event X occurs, - * clear it and transition to state Y". The dispatcher returns to the - * scheduler if none of the events in which the wait state has an interest are - * currently pending. - */ -#define WAIT_STATE(n, sn, ...) \ - const struct fscache_state __STATE_NAME(n) = { \ - .name = #n, \ - .short_name = sn, \ - .work = NULL, \ - .transitions = { __VA_ARGS__, { 0, NULL } } \ - } - -#define TRANSIT_TO(state, emask) \ - { .events = (emask), .transit_to = STATE(state) } - -/* - * The object state machine. - */ -static WORK_STATE(INIT_OBJECT, "INIT", fscache_initialise_object); -static WORK_STATE(PARENT_READY, "PRDY", fscache_parent_ready); -static WORK_STATE(ABORT_INIT, "ABRT", fscache_abort_initialisation); -static WORK_STATE(LOOK_UP_OBJECT, "LOOK", fscache_look_up_object); -static WORK_STATE(OBJECT_AVAILABLE, "AVBL", fscache_object_available); -static WORK_STATE(JUMPSTART_DEPS, "JUMP", fscache_jumpstart_dependents); - -static WORK_STATE(INVALIDATE_OBJECT, "INVL", fscache_invalidate_object); -static WORK_STATE(UPDATE_OBJECT, "UPDT", fscache_update_object); - -static WORK_STATE(LOOKUP_FAILURE, "LCFL", fscache_lookup_failure); -static WORK_STATE(KILL_OBJECT, "KILL", fscache_kill_object); -static WORK_STATE(KILL_DEPENDENTS, "KDEP", fscache_kill_dependents); -static WORK_STATE(DROP_OBJECT, "DROP", fscache_drop_object); -static WORK_STATE(OBJECT_DEAD, "DEAD", fscache_object_dead); - -static WAIT_STATE(WAIT_FOR_INIT, "?INI", - TRANSIT_TO(INIT_OBJECT, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); - -static WAIT_STATE(WAIT_FOR_PARENT, "?PRN", - TRANSIT_TO(PARENT_READY, 1 << FSCACHE_OBJECT_EV_PARENT_READY)); - -static WAIT_STATE(WAIT_FOR_CMD, "?CMD", - TRANSIT_TO(INVALIDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_INVALIDATE), - TRANSIT_TO(UPDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_UPDATE), - TRANSIT_TO(JUMPSTART_DEPS, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); - -static WAIT_STATE(WAIT_FOR_CLEARANCE, "?CLR", - TRANSIT_TO(KILL_OBJECT, 1 << FSCACHE_OBJECT_EV_CLEARED)); - -/* - * Out-of-band event transition tables. These are for handling unexpected - * events, such as an I/O error. If an OOB event occurs, the state machine - * clears and disables the event and forces a transition to the nominated work - * state (acurrently executing work states will complete first). - * - * In such a situation, object->state remembers the state the machine should - * have been in/gone to and returning NO_TRANSIT returns to that. - */ -static const struct fscache_transition fscache_osm_init_oob[] = { - TRANSIT_TO(ABORT_INIT, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static const struct fscache_transition fscache_osm_lookup_oob[] = { - TRANSIT_TO(LOOKUP_FAILURE, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static const struct fscache_transition fscache_osm_run_oob[] = { - TRANSIT_TO(KILL_OBJECT, - (1 << FSCACHE_OBJECT_EV_ERROR) | - (1 << FSCACHE_OBJECT_EV_KILL)), - { 0, NULL } -}; - -static int fscache_get_object(struct fscache_object *, - enum fscache_obj_ref_trace); -static void fscache_put_object(struct fscache_object *, - enum fscache_obj_ref_trace); -static bool fscache_enqueue_dependents(struct fscache_object *, int); -static void fscache_dequeue_object(struct fscache_object *); -static void fscache_update_aux_data(struct fscache_object *); - -/* - * we need to notify the parent when an op completes that we had outstanding - * upon it - */ -static inline void fscache_done_parent_op(struct fscache_object *object) -{ - struct fscache_object *parent = object->parent; - - _enter("OBJ%x {OBJ%x,%x}", - object->debug_id, parent->debug_id, parent->n_ops); - - spin_lock_nested(&parent->lock, 1); - parent->n_obj_ops--; - parent->n_ops--; - if (parent->n_ops == 0) - fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); - spin_unlock(&parent->lock); -} - -/* - * Object state machine dispatcher. - */ -static void fscache_object_sm_dispatcher(struct fscache_object *object) -{ - const struct fscache_transition *t; - const struct fscache_state *state, *new_state; - unsigned long events, event_mask; - bool oob; - int event = -1; - - ASSERT(object != NULL); - - _enter("{OBJ%x,%s,%lx}", - object->debug_id, object->state->name, object->events); - - event_mask = object->event_mask; -restart: - object->event_mask = 0; /* Mask normal event handling */ - state = object->state; -restart_masked: - events = object->events; - - /* Handle any out-of-band events (typically an error) */ - if (events & object->oob_event_mask) { - _debug("{OBJ%x} oob %lx", - object->debug_id, events & object->oob_event_mask); - oob = true; - for (t = object->oob_table; t->events; t++) { - if (events & t->events) { - state = t->transit_to; - ASSERT(state->work != NULL); - event = fls(events & t->events) - 1; - __clear_bit(event, &object->oob_event_mask); - clear_bit(event, &object->events); - goto execute_work_state; - } - } - } - oob = false; - - /* Wait states are just transition tables */ - if (!state->work) { - if (events & event_mask) { - for (t = state->transitions; t->events; t++) { - if (events & t->events) { - new_state = t->transit_to; - event = fls(events & t->events) - 1; - trace_fscache_osm(object, state, - true, false, event); - clear_bit(event, &object->events); - _debug("{OBJ%x} ev %d: %s -> %s", - object->debug_id, event, - state->name, new_state->name); - object->state = state = new_state; - goto execute_work_state; - } - } - - /* The event mask didn't include all the tabled bits */ - BUG(); - } - /* Randomly woke up */ - goto unmask_events; - } - -execute_work_state: - _debug("{OBJ%x} exec %s", object->debug_id, state->name); - - trace_fscache_osm(object, state, false, oob, event); - new_state = state->work(object, event); - event = -1; - if (new_state == NO_TRANSIT) { - _debug("{OBJ%x} %s notrans", object->debug_id, state->name); - if (unlikely(state == STATE(OBJECT_DEAD))) { - _leave(" [dead]"); - return; - } - fscache_enqueue_object(object); - event_mask = object->oob_event_mask; - goto unmask_events; - } - - _debug("{OBJ%x} %s -> %s", - object->debug_id, state->name, new_state->name); - object->state = state = new_state; - - if (state->work) { - if (unlikely(state == STATE(OBJECT_DEAD))) { - _leave(" [dead]"); - return; - } - goto restart_masked; - } - - /* Transited to wait state */ - event_mask = object->oob_event_mask; - for (t = state->transitions; t->events; t++) - event_mask |= t->events; - -unmask_events: - object->event_mask = event_mask; - smp_mb(); - events = object->events; - if (events & event_mask) - goto restart; - _leave(" [msk %lx]", event_mask); -} - -/* - * execute an object - */ -static void fscache_object_work_func(struct work_struct *work) -{ - struct fscache_object *object = - container_of(work, struct fscache_object, work); - - _enter("{OBJ%x}", object->debug_id); - - fscache_object_sm_dispatcher(object); - fscache_put_object(object, fscache_obj_put_work); -} - -/** - * fscache_object_init - Initialise a cache object description - * @object: Object description - * @cookie: Cookie object will be attached to - * @cache: Cache in which backing object will be found - * - * Initialise a cache object description to its basic values. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -void fscache_object_init(struct fscache_object *object, - struct fscache_cookie *cookie, - struct fscache_cache *cache) -{ - const struct fscache_transition *t; - - atomic_inc(&cache->object_count); - - object->state = STATE(WAIT_FOR_INIT); - object->oob_table = fscache_osm_init_oob; - object->flags = 1 << FSCACHE_OBJECT_IS_LIVE; - spin_lock_init(&object->lock); - INIT_LIST_HEAD(&object->cache_link); - INIT_HLIST_NODE(&object->cookie_link); - INIT_WORK(&object->work, fscache_object_work_func); - INIT_LIST_HEAD(&object->dependents); - INIT_LIST_HEAD(&object->dep_link); - INIT_LIST_HEAD(&object->pending_ops); - object->n_children = 0; - object->n_ops = object->n_in_progress = object->n_exclusive = 0; - object->events = 0; - object->store_limit = 0; - object->store_limit_l = 0; - object->cache = cache; - object->cookie = cookie; - fscache_cookie_get(cookie, fscache_cookie_get_attach_object); - object->parent = NULL; -#ifdef CONFIG_FSCACHE_OBJECT_LIST - RB_CLEAR_NODE(&object->objlist_link); -#endif - - object->oob_event_mask = 0; - for (t = object->oob_table; t->events; t++) - object->oob_event_mask |= t->events; - object->event_mask = object->oob_event_mask; - for (t = object->state->transitions; t->events; t++) - object->event_mask |= t->events; -} -EXPORT_SYMBOL(fscache_object_init); - -/* - * Mark the object as no longer being live, making sure that we synchronise - * against op submission. - */ -static inline void fscache_mark_object_dead(struct fscache_object *object) -{ - spin_lock(&object->lock); - clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags); - spin_unlock(&object->lock); -} - -/* - * Abort object initialisation before we start it. - */ -static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_event_mask = 0; - fscache_dequeue_object(object); - return transit_to(KILL_OBJECT); -} - -/* - * initialise an object - * - check the specified object's parent to see if we can make use of it - * immediately to do a creation - * - we may need to start the process of creating a parent and we need to wait - * for the parent's lookup and creation to complete if it's not there yet - */ -static const struct fscache_state *fscache_initialise_object(struct fscache_object *object, - int event) -{ - struct fscache_object *parent; - bool success; - - _enter("{OBJ%x},%d", object->debug_id, event); - - ASSERT(list_empty(&object->dep_link)); - - parent = object->parent; - if (!parent) { - _leave(" [no parent]"); - return transit_to(DROP_OBJECT); - } - - _debug("parent: %s of:%lx", parent->state->name, parent->flags); - - if (fscache_object_is_dying(parent)) { - _leave(" [bad parent]"); - return transit_to(DROP_OBJECT); - } - - if (fscache_object_is_available(parent)) { - _leave(" [ready]"); - return transit_to(PARENT_READY); - } - - _debug("wait"); - - spin_lock(&parent->lock); - fscache_stat(&fscache_n_cop_grab_object); - success = false; - if (fscache_object_is_live(parent) && - object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) { - list_add(&object->dep_link, &parent->dependents); - success = true; - } - fscache_stat_d(&fscache_n_cop_grab_object); - spin_unlock(&parent->lock); - if (!success) { - _leave(" [grab failed]"); - return transit_to(DROP_OBJECT); - } - - /* fscache_acquire_non_index_cookie() uses this - * to wake the chain up */ - fscache_raise_event(parent, FSCACHE_OBJECT_EV_NEW_CHILD); - _leave(" [wait]"); - return transit_to(WAIT_FOR_PARENT); -} - -/* - * Once the parent object is ready, we should kick off our lookup op. - */ -static const struct fscache_state *fscache_parent_ready(struct fscache_object *object, - int event) -{ - struct fscache_object *parent = object->parent; - - _enter("{OBJ%x},%d", object->debug_id, event); - - ASSERT(parent != NULL); - - spin_lock(&parent->lock); - parent->n_ops++; - parent->n_obj_ops++; - spin_unlock(&parent->lock); - - _leave(""); - return transit_to(LOOK_UP_OBJECT); -} - -/* - * look an object up in the cache from which it was allocated - * - we hold an "access lock" on the parent object, so the parent object cannot - * be withdrawn by either party till we've finished - */ -static const struct fscache_state *fscache_look_up_object(struct fscache_object *object, - int event) -{ - struct fscache_cookie *cookie = object->cookie; - struct fscache_object *parent = object->parent; - int ret; - - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_table = fscache_osm_lookup_oob; - - ASSERT(parent != NULL); - ASSERTCMP(parent->n_ops, >, 0); - ASSERTCMP(parent->n_obj_ops, >, 0); - - /* make sure the parent is still available */ - ASSERT(fscache_object_is_available(parent)); - - if (fscache_object_is_dying(parent) || - test_bit(FSCACHE_IOERROR, &object->cache->flags) || - !fscache_use_cookie(object)) { - _leave(" [unavailable]"); - return transit_to(LOOKUP_FAILURE); - } - - _debug("LOOKUP \"%s\" in \"%s\"", - cookie->def->name, object->cache->tag->name); - - fscache_stat(&fscache_n_object_lookups); - fscache_stat(&fscache_n_cop_lookup_object); - ret = object->cache->ops->lookup_object(object); - fscache_stat_d(&fscache_n_cop_lookup_object); - - fscache_unuse_cookie(object); - - if (ret == -ETIMEDOUT) { - /* probably stuck behind another object, so move this one to - * the back of the queue */ - fscache_stat(&fscache_n_object_lookups_timed_out); - _leave(" [timeout]"); - return NO_TRANSIT; - } - - if (ret < 0) { - _leave(" [error]"); - return transit_to(LOOKUP_FAILURE); - } - - _leave(" [ok]"); - return transit_to(OBJECT_AVAILABLE); -} - -/** - * fscache_object_lookup_negative - Note negative cookie lookup - * @object: Object pointing to cookie to mark - * - * Note negative lookup, permitting those waiting to read data from an already - * existing backing object to continue as there's no data for them to read. - */ -void fscache_object_lookup_negative(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x,%s}", object->debug_id, object->state->name); - - if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { - fscache_stat(&fscache_n_object_lookups_negative); - - /* Allow write requests to begin stacking up and read requests to begin - * returning ENODATA. - */ - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - } - _leave(""); -} -EXPORT_SYMBOL(fscache_object_lookup_negative); - -/** - * fscache_obtained_object - Note successful object lookup or creation - * @object: Object pointing to cookie to mark - * - * Note successful lookup and/or creation, permitting those waiting to write - * data to a backing object to continue. - * - * Note that after calling this, an object's cookie may be relinquished by the - * netfs, and so must be accessed with object lock held. - */ -void fscache_obtained_object(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x,%s}", object->debug_id, object->state->name); - - /* if we were still looking up, then we must have a positive lookup - * result, in which case there may be data available */ - if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { - fscache_stat(&fscache_n_object_lookups_positive); - - /* We do (presumably) have data */ - clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - - /* Allow write requests to begin stacking up and read requests - * to begin shovelling data. - */ - clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - } else { - fscache_stat(&fscache_n_object_created); - } - - set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags); - _leave(""); -} -EXPORT_SYMBOL(fscache_obtained_object); - -/* - * handle an object that has just become available - */ -static const struct fscache_state *fscache_object_available(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_table = fscache_osm_run_oob; - - spin_lock(&object->lock); - - fscache_done_parent_op(object); - if (object->n_in_progress == 0) { - if (object->n_ops > 0) { - ASSERTCMP(object->n_ops, >=, object->n_obj_ops); - fscache_start_operations(object); - } else { - ASSERT(list_empty(&object->pending_ops)); - } - } - spin_unlock(&object->lock); - - fscache_stat(&fscache_n_cop_lookup_complete); - object->cache->ops->lookup_complete(object); - fscache_stat_d(&fscache_n_cop_lookup_complete); - - fscache_stat(&fscache_n_object_avail); - - _leave(""); - return transit_to(JUMPSTART_DEPS); -} - -/* - * Wake up this object's dependent objects now that we've become available. - */ -static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_PARENT_READY)) - return NO_TRANSIT; /* Not finished; requeue */ - return transit_to(WAIT_FOR_CMD); -} - -/* - * Handle lookup or creation failute. - */ -static const struct fscache_state *fscache_lookup_failure(struct fscache_object *object, - int event) -{ - struct fscache_cookie *cookie; - - _enter("{OBJ%x},%d", object->debug_id, event); - - object->oob_event_mask = 0; - - fscache_stat(&fscache_n_cop_lookup_complete); - object->cache->ops->lookup_complete(object); - fscache_stat_d(&fscache_n_cop_lookup_complete); - - set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags); - - cookie = object->cookie; - set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); - if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - - fscache_done_parent_op(object); - return transit_to(KILL_OBJECT); -} - -/* - * Wait for completion of all active operations on this object and the death of - * all child objects of this object. - */ -static const struct fscache_state *fscache_kill_object(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x,%d,%d},%d", - object->debug_id, object->n_ops, object->n_children, event); - - fscache_mark_object_dead(object); - object->oob_event_mask = 0; - - if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) { - /* Reject any new read/write ops and abort any that are pending. */ - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_cancel_all_ops(object); - } - - if (list_empty(&object->dependents) && - object->n_ops == 0 && - object->n_children == 0) - return transit_to(DROP_OBJECT); - - if (object->n_in_progress == 0) { - spin_lock(&object->lock); - if (object->n_ops > 0 && object->n_in_progress == 0) - fscache_start_operations(object); - spin_unlock(&object->lock); - } - - if (!list_empty(&object->dependents)) - return transit_to(KILL_DEPENDENTS); - - return transit_to(WAIT_FOR_CLEARANCE); -} - -/* - * Kill dependent objects. - */ -static const struct fscache_state *fscache_kill_dependents(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_KILL)) - return NO_TRANSIT; /* Not finished */ - return transit_to(WAIT_FOR_CLEARANCE); -} - -/* - * Drop an object's attachments - */ -static const struct fscache_state *fscache_drop_object(struct fscache_object *object, - int event) -{ - struct fscache_object *parent = object->parent; - struct fscache_cookie *cookie = object->cookie; - struct fscache_cache *cache = object->cache; - bool awaken = false; - - _enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event); - - ASSERT(cookie != NULL); - ASSERT(!hlist_unhashed(&object->cookie_link)); - - if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) { - _debug("final update"); - fscache_update_aux_data(object); - } - - /* Make sure the cookie no longer points here and that the netfs isn't - * waiting for us. - */ - spin_lock(&cookie->lock); - hlist_del_init(&object->cookie_link); - if (hlist_empty(&cookie->backing_objects) && - test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - awaken = true; - spin_unlock(&cookie->lock); - - if (awaken) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); - - - /* Prevent a race with our last child, which has to signal EV_CLEARED - * before dropping our spinlock. - */ - spin_lock(&object->lock); - spin_unlock(&object->lock); - - /* Discard from the cache's collection of objects */ - spin_lock(&cache->object_list_lock); - list_del_init(&object->cache_link); - spin_unlock(&cache->object_list_lock); - - fscache_stat(&fscache_n_cop_drop_object); - cache->ops->drop_object(object); - fscache_stat_d(&fscache_n_cop_drop_object); - - /* The parent object wants to know when all it dependents have gone */ - if (parent) { - _debug("release parent OBJ%x {%d}", - parent->debug_id, parent->n_children); - - spin_lock(&parent->lock); - parent->n_children--; - if (parent->n_children == 0) - fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); - spin_unlock(&parent->lock); - object->parent = NULL; - } - - /* this just shifts the object release to the work processor */ - fscache_put_object(object, fscache_obj_put_drop_obj); - fscache_stat(&fscache_n_object_dead); - - _leave(""); - return transit_to(OBJECT_DEAD); -} - -/* - * get a ref on an object - */ -static int fscache_get_object(struct fscache_object *object, - enum fscache_obj_ref_trace why) -{ - int ret; - - fscache_stat(&fscache_n_cop_grab_object); - ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN; - fscache_stat_d(&fscache_n_cop_grab_object); - return ret; -} - -/* - * Discard a ref on an object - */ -static void fscache_put_object(struct fscache_object *object, - enum fscache_obj_ref_trace why) -{ - fscache_stat(&fscache_n_cop_put_object); - object->cache->ops->put_object(object, why); - fscache_stat_d(&fscache_n_cop_put_object); -} - -/** - * fscache_object_destroy - Note that a cache object is about to be destroyed - * @object: The object to be destroyed - * - * Note the imminent destruction and deallocation of a cache object record. - */ -void fscache_object_destroy(struct fscache_object *object) -{ - /* We can get rid of the cookie now */ - fscache_cookie_put(object->cookie, fscache_cookie_put_object); - object->cookie = NULL; -} -EXPORT_SYMBOL(fscache_object_destroy); - -/* - * enqueue an object for metadata-type processing - */ -void fscache_enqueue_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - if (fscache_get_object(object, fscache_obj_get_queue) >= 0) { - wait_queue_head_t *cong_wq = - &get_cpu_var(fscache_object_cong_wait); - - if (queue_work(fscache_object_wq, &object->work)) { - if (fscache_object_congested()) - wake_up(cong_wq); - } else - fscache_put_object(object, fscache_obj_put_queue); - - put_cpu_var(fscache_object_cong_wait); - } -} - -/** - * fscache_object_sleep_till_congested - Sleep until object wq is congested - * @timeoutp: Scheduler sleep timeout - * - * Allow an object handler to sleep until the object workqueue is congested. - * - * The caller must set up a wake up event before calling this and must have set - * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own - * condition before calling this function as no test is made here. - * - * %true is returned if the object wq is congested, %false otherwise. - */ -bool fscache_object_sleep_till_congested(signed long *timeoutp) -{ - wait_queue_head_t *cong_wq = this_cpu_ptr(&fscache_object_cong_wait); - DEFINE_WAIT(wait); - - if (fscache_object_congested()) - return true; - - add_wait_queue_exclusive(cong_wq, &wait); - if (!fscache_object_congested()) - *timeoutp = schedule_timeout(*timeoutp); - finish_wait(cong_wq, &wait); - - return fscache_object_congested(); -} -EXPORT_SYMBOL_GPL(fscache_object_sleep_till_congested); - -/* - * Enqueue the dependents of an object for metadata-type processing. - * - * If we don't manage to finish the list before the scheduler wants to run - * again then return false immediately. We return true if the list was - * cleared. - */ -static bool fscache_enqueue_dependents(struct fscache_object *object, int event) -{ - struct fscache_object *dep; - bool ret = true; - - _enter("{OBJ%x}", object->debug_id); - - if (list_empty(&object->dependents)) - return true; - - spin_lock(&object->lock); - - while (!list_empty(&object->dependents)) { - dep = list_entry(object->dependents.next, - struct fscache_object, dep_link); - list_del_init(&dep->dep_link); - - fscache_raise_event(dep, event); - fscache_put_object(dep, fscache_obj_put_enq_dep); - - if (!list_empty(&object->dependents) && need_resched()) { - ret = false; - break; - } - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * remove an object from whatever queue it's waiting on - */ -static void fscache_dequeue_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - if (!list_empty(&object->dep_link)) { - spin_lock(&object->parent->lock); - list_del_init(&object->dep_link); - spin_unlock(&object->parent->lock); - } - - _leave(""); -} - -/** - * fscache_check_aux - Ask the netfs whether an object on disk is still valid - * @object: The object to ask about - * @data: The auxiliary data for the object - * @datalen: The size of the auxiliary data - * @object_size: The size of the object according to the server. - * - * This function consults the netfs about the coherency state of an object. - * The caller must be holding a ref on cookie->n_active (held by - * fscache_look_up_object() on behalf of the cache backend during object lookup - * and creation). - */ -enum fscache_checkaux fscache_check_aux(struct fscache_object *object, - const void *data, uint16_t datalen, - loff_t object_size) -{ - enum fscache_checkaux result; - - if (!object->cookie->def->check_aux) { - fscache_stat(&fscache_n_checkaux_none); - return FSCACHE_CHECKAUX_OKAY; - } - - result = object->cookie->def->check_aux(object->cookie->netfs_data, - data, datalen, object_size); - switch (result) { - /* entry okay as is */ - case FSCACHE_CHECKAUX_OKAY: - fscache_stat(&fscache_n_checkaux_okay); - break; - - /* entry requires update */ - case FSCACHE_CHECKAUX_NEEDS_UPDATE: - fscache_stat(&fscache_n_checkaux_update); - break; - - /* entry requires deletion */ - case FSCACHE_CHECKAUX_OBSOLETE: - fscache_stat(&fscache_n_checkaux_obsolete); - break; - - default: - BUG(); - } - - return result; -} -EXPORT_SYMBOL(fscache_check_aux); - -/* - * Asynchronously invalidate an object. - */ -static const struct fscache_state *_fscache_invalidate_object(struct fscache_object *object, - int event) -{ - struct fscache_operation *op; - struct fscache_cookie *cookie = object->cookie; - - _enter("{OBJ%x},%d", object->debug_id, event); - - /* We're going to need the cookie. If the cookie is not available then - * retire the object instead. - */ - if (!fscache_use_cookie(object)) { - ASSERT(radix_tree_empty(&object->cookie->stores)); - set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); - _leave(" [no cookie]"); - return transit_to(KILL_OBJECT); - } - - /* Reject any new read/write ops and abort any that are pending. */ - fscache_invalidate_writes(cookie); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - fscache_cancel_all_ops(object); - - /* Now we have to wait for in-progress reads and writes */ - op = kzalloc(sizeof(*op), GFP_KERNEL); - if (!op) - goto nomem; - - fscache_operation_init(cookie, op, object->cache->ops->invalidate_object, - NULL, NULL); - op->flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_EXCLUSIVE) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_invalidate); - - spin_lock(&cookie->lock); - if (fscache_submit_exclusive_op(object, op) < 0) - goto submit_op_failed; - spin_unlock(&cookie->lock); - fscache_put_operation(op); - - /* Once we've completed the invalidation, we know there will be no data - * stored in the cache and thus we can reinstate the data-check-skip - * optimisation. - */ - set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - - /* We can allow read and write requests to come in once again. They'll - * queue up behind our exclusive invalidation operation. - */ - if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) - wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); - _leave(" [ok]"); - return transit_to(UPDATE_OBJECT); - -nomem: - fscache_mark_object_dead(object); - fscache_unuse_cookie(object); - _leave(" [ENOMEM]"); - return transit_to(KILL_OBJECT); - -submit_op_failed: - fscache_mark_object_dead(object); - spin_unlock(&cookie->lock); - fscache_unuse_cookie(object); - kfree(op); - _leave(" [EIO]"); - return transit_to(KILL_OBJECT); -} - -static const struct fscache_state *fscache_invalidate_object(struct fscache_object *object, - int event) -{ - const struct fscache_state *s; - - fscache_stat(&fscache_n_invalidates_run); - fscache_stat(&fscache_n_cop_invalidate_object); - s = _fscache_invalidate_object(object, event); - fscache_stat_d(&fscache_n_cop_invalidate_object); - return s; -} - -/* - * Update auxiliary data. - */ -static void fscache_update_aux_data(struct fscache_object *object) -{ - fscache_stat(&fscache_n_updates_run); - fscache_stat(&fscache_n_cop_update_object); - object->cache->ops->update_object(object); - fscache_stat_d(&fscache_n_cop_update_object); -} - -/* - * Asynchronously update an object. - */ -static const struct fscache_state *fscache_update_object(struct fscache_object *object, - int event) -{ - _enter("{OBJ%x},%d", object->debug_id, event); - - fscache_update_aux_data(object); - - _leave(""); - return transit_to(WAIT_FOR_CMD); -} - -/** - * fscache_object_retrying_stale - Note retrying stale object - * @object: The object that will be retried - * - * Note that an object lookup found an on-disk object that was adjudged to be - * stale and has been deleted. The lookup will be retried. - */ -void fscache_object_retrying_stale(struct fscache_object *object) -{ - fscache_stat(&fscache_n_cache_no_space_reject); -} -EXPORT_SYMBOL(fscache_object_retrying_stale); - -/** - * fscache_object_mark_killed - Note that an object was killed - * @object: The object that was culled - * @why: The reason the object was killed. - * - * Note that an object was killed. Returns true if the object was - * already marked killed, false if it wasn't. - */ -void fscache_object_mark_killed(struct fscache_object *object, - enum fscache_why_object_killed why) -{ - if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) { - pr_err("Error: Object already killed by cache [%s]\n", - object->cache->identifier); - return; - } - - switch (why) { - case FSCACHE_OBJECT_NO_SPACE: - fscache_stat(&fscache_n_cache_no_space_reject); - break; - case FSCACHE_OBJECT_IS_STALE: - fscache_stat(&fscache_n_cache_stale_objects); - break; - case FSCACHE_OBJECT_WAS_RETIRED: - fscache_stat(&fscache_n_cache_retired_objects); - break; - case FSCACHE_OBJECT_WAS_CULLED: - fscache_stat(&fscache_n_cache_culled_objects); - break; - } -} -EXPORT_SYMBOL(fscache_object_mark_killed); - -/* - * The object is dead. We can get here if an object gets queued by an event - * that would lead to its death (such as EV_KILL) when the dispatcher is - * already running (and so can be requeued) but hasn't yet cleared the event - * mask. - */ -static const struct fscache_state *fscache_object_dead(struct fscache_object *object, - int event) -{ - if (!test_and_set_bit(FSCACHE_OBJECT_RUN_AFTER_DEAD, - &object->flags)) - return NO_TRANSIT; - - WARN(true, "FS-Cache object redispatched after death"); - return NO_TRANSIT; -} diff --git a/fs/fscache/operation.c b/fs/fscache/operation.c deleted file mode 100644 index e002cdfaf3cc..000000000000 --- a/fs/fscache/operation.c +++ /dev/null @@ -1,633 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache worker operation management routines - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - * - * See Documentation/filesystems/caching/operations.rst - */ - -#define FSCACHE_DEBUG_LEVEL OPERATION -#include -#include -#include -#include "internal.h" - -atomic_t fscache_op_debug_id; -EXPORT_SYMBOL(fscache_op_debug_id); - -static void fscache_operation_dummy_cancel(struct fscache_operation *op) -{ -} - -/** - * fscache_operation_init - Do basic initialisation of an operation - * @cookie: The cookie to operate on - * @op: The operation to initialise - * @processor: The function to perform the operation - * @cancel: A function to handle operation cancellation - * @release: The release function to assign - * - * Do basic initialisation of an operation. The caller must still set flags, - * object and processor if needed. - */ -void fscache_operation_init(struct fscache_cookie *cookie, - struct fscache_operation *op, - fscache_operation_processor_t processor, - fscache_operation_cancel_t cancel, - fscache_operation_release_t release) -{ - INIT_WORK(&op->work, fscache_op_work_func); - atomic_set(&op->usage, 1); - op->state = FSCACHE_OP_ST_INITIALISED; - op->debug_id = atomic_inc_return(&fscache_op_debug_id); - op->processor = processor; - op->cancel = cancel ?: fscache_operation_dummy_cancel; - op->release = release; - INIT_LIST_HEAD(&op->pend_link); - fscache_stat(&fscache_n_op_initialised); - trace_fscache_op(cookie, op, fscache_op_init); -} -EXPORT_SYMBOL(fscache_operation_init); - -/** - * fscache_enqueue_operation - Enqueue an operation for processing - * @op: The operation to enqueue - * - * Enqueue an operation for processing by the FS-Cache thread pool. - * - * This will get its own ref on the object. - */ -void fscache_enqueue_operation(struct fscache_operation *op) -{ - struct fscache_cookie *cookie = op->object->cookie; - - _enter("{OBJ%x OP%x,%u}", - op->object->debug_id, op->debug_id, atomic_read(&op->usage)); - - ASSERT(list_empty(&op->pend_link)); - ASSERT(op->processor != NULL); - ASSERT(fscache_object_is_available(op->object)); - ASSERTCMP(atomic_read(&op->usage), >, 0); - ASSERTIFCMP(op->state != FSCACHE_OP_ST_IN_PROGRESS, - op->state, ==, FSCACHE_OP_ST_CANCELLED); - - fscache_stat(&fscache_n_op_enqueue); - switch (op->flags & FSCACHE_OP_TYPE) { - case FSCACHE_OP_ASYNC: - trace_fscache_op(cookie, op, fscache_op_enqueue_async); - _debug("queue async"); - atomic_inc(&op->usage); - if (!queue_work(fscache_op_wq, &op->work)) - fscache_put_operation(op); - break; - case FSCACHE_OP_MYTHREAD: - trace_fscache_op(cookie, op, fscache_op_enqueue_mythread); - _debug("queue for caller's attention"); - break; - default: - pr_err("Unexpected op type %lx", op->flags); - BUG(); - break; - } -} -EXPORT_SYMBOL(fscache_enqueue_operation); - -/* - * start an op running - */ -static void fscache_run_op(struct fscache_object *object, - struct fscache_operation *op) -{ - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING); - - op->state = FSCACHE_OP_ST_IN_PROGRESS; - object->n_in_progress++; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - if (op->processor) - fscache_enqueue_operation(op); - else - trace_fscache_op(object->cookie, op, fscache_op_run); - fscache_stat(&fscache_n_op_run); -} - -/* - * report an unexpected submission - */ -static void fscache_report_unexpected_submission(struct fscache_object *object, - struct fscache_operation *op, - const struct fscache_state *ostate) -{ - static bool once_only; - struct fscache_operation *p; - unsigned n; - - if (once_only) - return; - once_only = true; - - kdebug("unexpected submission OP%x [OBJ%x %s]", - op->debug_id, object->debug_id, object->state->name); - kdebug("objstate=%s [%s]", object->state->name, ostate->name); - kdebug("objflags=%lx", object->flags); - kdebug("objevent=%lx [%lx]", object->events, object->event_mask); - kdebug("ops=%u inp=%u exc=%u", - object->n_ops, object->n_in_progress, object->n_exclusive); - - if (!list_empty(&object->pending_ops)) { - n = 0; - list_for_each_entry(p, &object->pending_ops, pend_link) { - ASSERTCMP(p->object, ==, object); - kdebug("%p %p", op->processor, op->release); - n++; - } - - kdebug("n=%u", n); - } - - dump_stack(); -} - -/* - * submit an exclusive operation for an object - * - other ops are excluded from running simultaneously with this one - * - this gets any extra refs it needs on an op - */ -int fscache_submit_exclusive_op(struct fscache_object *object, - struct fscache_operation *op) -{ - const struct fscache_state *ostate; - unsigned long flags; - int ret; - - _enter("{OBJ%x OP%x},", object->debug_id, op->debug_id); - - trace_fscache_op(object->cookie, op, fscache_op_submit_ex); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - ASSERTCMP(object->n_ops, >=, object->n_in_progress); - ASSERTCMP(object->n_ops, >=, object->n_exclusive); - ASSERT(list_empty(&op->pend_link)); - - ostate = object->state; - smp_rmb(); - - op->state = FSCACHE_OP_ST_PENDING; - flags = READ_ONCE(object->flags); - if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) { - fscache_stat(&fscache_n_op_rejected); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else if (unlikely(fscache_cache_is_broken(object))) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -EIO; - } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) { - op->object = object; - object->n_ops++; - object->n_exclusive++; /* reads and writes must wait */ - - if (object->n_in_progress > 0) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - } else if (!list_empty(&object->pending_ops)) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - fscache_start_operations(object); - } else { - ASSERTCMP(object->n_in_progress, ==, 0); - fscache_run_op(object, op); - } - - /* need to issue a new write op after this */ - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) { - op->object = object; - object->n_ops++; - object->n_exclusive++; /* reads and writes must wait */ - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else { - fscache_report_unexpected_submission(object, op, ostate); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * submit an operation for an object - * - objects may be submitted only in the following states: - * - during object creation (write ops may be submitted) - * - whilst the object is active - * - after an I/O error incurred in one of the two above states (op rejected) - * - this gets any extra refs it needs on an op - */ -int fscache_submit_op(struct fscache_object *object, - struct fscache_operation *op) -{ - const struct fscache_state *ostate; - unsigned long flags; - int ret; - - _enter("{OBJ%x OP%x},{%u}", - object->debug_id, op->debug_id, atomic_read(&op->usage)); - - trace_fscache_op(object->cookie, op, fscache_op_submit); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - ASSERTCMP(object->n_ops, >=, object->n_in_progress); - ASSERTCMP(object->n_ops, >=, object->n_exclusive); - ASSERT(list_empty(&op->pend_link)); - - ostate = object->state; - smp_rmb(); - - op->state = FSCACHE_OP_ST_PENDING; - flags = READ_ONCE(object->flags); - if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) { - fscache_stat(&fscache_n_op_rejected); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else if (unlikely(fscache_cache_is_broken(object))) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -EIO; - } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) { - op->object = object; - object->n_ops++; - - if (object->n_exclusive > 0) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - } else if (!list_empty(&object->pending_ops)) { - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - fscache_start_operations(object); - } else { - ASSERTCMP(object->n_exclusive, ==, 0); - fscache_run_op(object, op); - } - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) { - op->object = object; - object->n_ops++; - atomic_inc(&op->usage); - list_add_tail(&op->pend_link, &object->pending_ops); - fscache_stat(&fscache_n_op_pend); - ret = 0; - } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) { - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } else { - fscache_report_unexpected_submission(object, op, ostate); - ASSERT(!fscache_object_is_active(object)); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - ret = -ENOBUFS; - } - - spin_unlock(&object->lock); - return ret; -} - -/* - * queue an object for withdrawal on error, aborting all following asynchronous - * operations - */ -void fscache_abort_object(struct fscache_object *object) -{ - _enter("{OBJ%x}", object->debug_id); - - fscache_raise_event(object, FSCACHE_OBJECT_EV_ERROR); -} - -/* - * Jump start the operation processing on an object. The caller must hold - * object->lock. - */ -void fscache_start_operations(struct fscache_object *object) -{ - struct fscache_operation *op; - bool stop = false; - - while (!list_empty(&object->pending_ops) && !stop) { - op = list_entry(object->pending_ops.next, - struct fscache_operation, pend_link); - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) { - if (object->n_in_progress > 0) - break; - stop = true; - } - list_del_init(&op->pend_link); - fscache_run_op(object, op); - - /* the pending queue was holding a ref on the object */ - fscache_put_operation(op); - } - - ASSERTCMP(object->n_in_progress, <=, object->n_ops); - - _debug("woke %d ops on OBJ%x", - object->n_in_progress, object->debug_id); -} - -/* - * cancel an operation that's pending on an object - */ -int fscache_cancel_op(struct fscache_operation *op, - bool cancel_in_progress_op) -{ - struct fscache_object *object = op->object; - bool put = false; - int ret; - - _enter("OBJ%x OP%x}", op->object->debug_id, op->debug_id); - - trace_fscache_op(object->cookie, op, fscache_op_cancel); - - ASSERTCMP(op->state, >=, FSCACHE_OP_ST_PENDING); - ASSERTCMP(op->state, !=, FSCACHE_OP_ST_CANCELLED); - ASSERTCMP(atomic_read(&op->usage), >, 0); - - spin_lock(&object->lock); - - ret = -EBUSY; - if (op->state == FSCACHE_OP_ST_PENDING) { - ASSERT(!list_empty(&op->pend_link)); - list_del_init(&op->pend_link); - put = true; - - fscache_stat(&fscache_n_op_cancelled); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - ret = 0; - } else if (op->state == FSCACHE_OP_ST_IN_PROGRESS && cancel_in_progress_op) { - ASSERTCMP(object->n_in_progress, >, 0); - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - object->n_in_progress--; - if (object->n_in_progress == 0) - fscache_start_operations(object); - - fscache_stat(&fscache_n_op_cancelled); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - ret = 0; - } - - if (put) - fscache_put_operation(op); - spin_unlock(&object->lock); - _leave(" = %d", ret); - return ret; -} - -/* - * Cancel all pending operations on an object - */ -void fscache_cancel_all_ops(struct fscache_object *object) -{ - struct fscache_operation *op; - - _enter("OBJ%x", object->debug_id); - - spin_lock(&object->lock); - - while (!list_empty(&object->pending_ops)) { - op = list_entry(object->pending_ops.next, - struct fscache_operation, pend_link); - fscache_stat(&fscache_n_op_cancelled); - list_del_init(&op->pend_link); - - trace_fscache_op(object->cookie, op, fscache_op_cancel_all); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING); - op->cancel(op); - op->state = FSCACHE_OP_ST_CANCELLED; - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags)) - wake_up_bit(&op->flags, FSCACHE_OP_WAITING); - fscache_put_operation(op); - cond_resched_lock(&object->lock); - } - - spin_unlock(&object->lock); - _leave(""); -} - -/* - * Record the completion or cancellation of an in-progress operation. - */ -void fscache_op_complete(struct fscache_operation *op, bool cancelled) -{ - struct fscache_object *object = op->object; - - _enter("OBJ%x", object->debug_id); - - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_IN_PROGRESS); - ASSERTCMP(object->n_in_progress, >, 0); - ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags), - object->n_exclusive, >, 0); - ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags), - object->n_in_progress, ==, 1); - - spin_lock(&object->lock); - - if (!cancelled) { - trace_fscache_op(object->cookie, op, fscache_op_completed); - op->state = FSCACHE_OP_ST_COMPLETE; - } else { - op->cancel(op); - trace_fscache_op(object->cookie, op, fscache_op_cancelled); - op->state = FSCACHE_OP_ST_CANCELLED; - } - - if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) - object->n_exclusive--; - object->n_in_progress--; - if (object->n_in_progress == 0) - fscache_start_operations(object); - - spin_unlock(&object->lock); - _leave(""); -} -EXPORT_SYMBOL(fscache_op_complete); - -/* - * release an operation - * - queues pending ops if this is the last in-progress op - */ -void fscache_put_operation(struct fscache_operation *op) -{ - struct fscache_object *object; - struct fscache_cache *cache; - - _enter("{OBJ%x OP%x,%d}", - op->object ? op->object->debug_id : 0, - op->debug_id, atomic_read(&op->usage)); - - ASSERTCMP(atomic_read(&op->usage), >, 0); - - if (!atomic_dec_and_test(&op->usage)) - return; - - trace_fscache_op(op->object ? op->object->cookie : NULL, op, fscache_op_put); - - _debug("PUT OP"); - ASSERTIFCMP(op->state != FSCACHE_OP_ST_INITIALISED && - op->state != FSCACHE_OP_ST_COMPLETE, - op->state, ==, FSCACHE_OP_ST_CANCELLED); - - fscache_stat(&fscache_n_op_release); - - if (op->release) { - op->release(op); - op->release = NULL; - } - op->state = FSCACHE_OP_ST_DEAD; - - object = op->object; - if (likely(object)) { - if (test_bit(FSCACHE_OP_DEC_READ_CNT, &op->flags)) - atomic_dec(&object->n_reads); - if (test_bit(FSCACHE_OP_UNUSE_COOKIE, &op->flags)) - fscache_unuse_cookie(object); - - /* now... we may get called with the object spinlock held, so we - * complete the cleanup here only if we can immediately acquire the - * lock, and defer it otherwise */ - if (!spin_trylock(&object->lock)) { - _debug("defer put"); - fscache_stat(&fscache_n_op_deferred_release); - - cache = object->cache; - spin_lock(&cache->op_gc_list_lock); - list_add_tail(&op->pend_link, &cache->op_gc_list); - spin_unlock(&cache->op_gc_list_lock); - schedule_work(&cache->op_gc); - _leave(" [defer]"); - return; - } - - ASSERTCMP(object->n_ops, >, 0); - object->n_ops--; - if (object->n_ops == 0) - fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED); - - spin_unlock(&object->lock); - } - - kfree(op); - _leave(" [done]"); -} -EXPORT_SYMBOL(fscache_put_operation); - -/* - * garbage collect operations that have had their release deferred - */ -void fscache_operation_gc(struct work_struct *work) -{ - struct fscache_operation *op; - struct fscache_object *object; - struct fscache_cache *cache = - container_of(work, struct fscache_cache, op_gc); - int count = 0; - - _enter(""); - - do { - spin_lock(&cache->op_gc_list_lock); - if (list_empty(&cache->op_gc_list)) { - spin_unlock(&cache->op_gc_list_lock); - break; - } - - op = list_entry(cache->op_gc_list.next, - struct fscache_operation, pend_link); - list_del(&op->pend_link); - spin_unlock(&cache->op_gc_list_lock); - - object = op->object; - trace_fscache_op(object->cookie, op, fscache_op_gc); - - spin_lock(&object->lock); - - _debug("GC DEFERRED REL OBJ%x OP%x", - object->debug_id, op->debug_id); - fscache_stat(&fscache_n_op_gc); - - ASSERTCMP(atomic_read(&op->usage), ==, 0); - ASSERTCMP(op->state, ==, FSCACHE_OP_ST_DEAD); - - ASSERTCMP(object->n_ops, >, 0); - object->n_ops--; - if (object->n_ops == 0) - fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED); - - spin_unlock(&object->lock); - kfree(op); - - } while (count++ < 20); - - if (!list_empty(&cache->op_gc_list)) - schedule_work(&cache->op_gc); - - _leave(""); -} - -/* - * execute an operation using fs_op_wq to provide processing context - - * the caller holds a ref to this object, so we don't need to hold one - */ -void fscache_op_work_func(struct work_struct *work) -{ - struct fscache_operation *op = - container_of(work, struct fscache_operation, work); - - _enter("{OBJ%x OP%x,%d}", - op->object->debug_id, op->debug_id, atomic_read(&op->usage)); - - trace_fscache_op(op->object->cookie, op, fscache_op_work); - - ASSERT(op->processor != NULL); - op->processor(op); - fscache_put_operation(op); - - _leave(""); -} diff --git a/fs/fscache/page.c b/fs/fscache/page.c deleted file mode 100644 index 27df94ef0e0b..000000000000 --- a/fs/fscache/page.c +++ /dev/null @@ -1,1242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Cache page management and data I/O routines - * - * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL PAGE -#include -#include -#include -#include -#include -#include "internal.h" - -/* - * check to see if a page is being written to the cache - */ -bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page) -{ - void *val; - - rcu_read_lock(); - val = radix_tree_lookup(&cookie->stores, page->index); - rcu_read_unlock(); - trace_fscache_check_page(cookie, page, val, 0); - - return val != NULL; -} -EXPORT_SYMBOL(__fscache_check_page_write); - -/* - * wait for a page to finish being written to the cache - */ -void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page) -{ - wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); - - trace_fscache_page(cookie, page, fscache_page_write_wait); - - wait_event(*wq, !__fscache_check_page_write(cookie, page)); -} -EXPORT_SYMBOL(__fscache_wait_on_page_write); - -/* - * wait for a page to finish being written to the cache. Put a timeout here - * since we might be called recursively via parent fs. - */ -static -bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page) -{ - wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); - - return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page), - HZ); -} - -/* - * decide whether a page can be released, possibly by cancelling a store to it - * - we're allowed to sleep if __GFP_DIRECT_RECLAIM is flagged - */ -bool __fscache_maybe_release_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - struct page *xpage; - void *val; - - _enter("%p,%p,%x", cookie, page, gfp); - - trace_fscache_page(cookie, page, fscache_page_maybe_release); - -try_again: - rcu_read_lock(); - val = radix_tree_lookup(&cookie->stores, page->index); - if (!val) { - rcu_read_unlock(); - fscache_stat(&fscache_n_store_vmscan_not_storing); - __fscache_uncache_page(cookie, page); - return true; - } - - /* see if the page is actually undergoing storage - if so we can't get - * rid of it till the cache has finished with it */ - if (radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG)) { - rcu_read_unlock(); - goto page_busy; - } - - /* the page is pending storage, so we attempt to cancel the store and - * discard the store request so that the page can be reclaimed */ - spin_lock(&cookie->stores_lock); - rcu_read_unlock(); - - if (radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG)) { - /* the page started to undergo storage whilst we were looking, - * so now we can only wait or return */ - spin_unlock(&cookie->stores_lock); - goto page_busy; - } - - xpage = radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - spin_unlock(&cookie->stores_lock); - - if (xpage) { - fscache_stat(&fscache_n_store_vmscan_cancelled); - fscache_stat(&fscache_n_store_radix_deletes); - ASSERTCMP(xpage, ==, page); - } else { - fscache_stat(&fscache_n_store_vmscan_gone); - } - - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - if (xpage) - put_page(xpage); - __fscache_uncache_page(cookie, page); - return true; - -page_busy: - /* We will wait here if we're allowed to, but that could deadlock the - * allocator as the work threads writing to the cache may all end up - * sleeping on memory allocation, so we may need to impose a timeout - * too. */ - if (!(gfp & __GFP_DIRECT_RECLAIM) || !(gfp & __GFP_FS)) { - fscache_stat(&fscache_n_store_vmscan_busy); - return false; - } - - fscache_stat(&fscache_n_store_vmscan_wait); - if (!release_page_wait_timeout(cookie, page)) - _debug("fscache writeout timeout page: %p{%lx}", - page, page->index); - - gfp &= ~__GFP_DIRECT_RECLAIM; - goto try_again; -} -EXPORT_SYMBOL(__fscache_maybe_release_page); - -/* - * note that a page has finished being written to the cache - */ -static void fscache_end_page_write(struct fscache_object *object, - struct page *page) -{ - struct fscache_cookie *cookie; - struct page *xpage = NULL, *val; - - spin_lock(&object->lock); - cookie = object->cookie; - if (cookie) { - /* delete the page from the tree if it is now no longer - * pending */ - spin_lock(&cookie->stores_lock); - radix_tree_tag_clear(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_clear_store); - if (!radix_tree_tag_get(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG)) { - fscache_stat(&fscache_n_store_radix_deletes); - xpage = radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - trace_fscache_page(cookie, page, fscache_page_write_end); - - val = radix_tree_lookup(&cookie->stores, page->index); - trace_fscache_check_page(cookie, page, val, 1); - } else { - trace_fscache_page(cookie, page, fscache_page_write_end_pend); - } - spin_unlock(&cookie->stores_lock); - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - } else { - trace_fscache_page(cookie, page, fscache_page_write_end_noc); - } - spin_unlock(&object->lock); - if (xpage) - put_page(xpage); -} - -/* - * actually apply the changed attributes to a cache object - */ -static void fscache_attr_changed_op(struct fscache_operation *op) -{ - struct fscache_object *object = op->object; - int ret; - - _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id); - - fscache_stat(&fscache_n_attr_changed_calls); - - if (fscache_object_is_active(object)) { - fscache_stat(&fscache_n_cop_attr_changed); - ret = object->cache->ops->attr_changed(object); - fscache_stat_d(&fscache_n_cop_attr_changed); - if (ret < 0) - fscache_abort_object(object); - fscache_op_complete(op, ret < 0); - } else { - fscache_op_complete(op, true); - } - - _leave(""); -} - -/* - * notification that the attributes on an object have changed - */ -int __fscache_attr_changed(struct fscache_cookie *cookie) -{ - struct fscache_operation *op; - struct fscache_object *object; - bool wake_cookie = false; - - _enter("%p", cookie); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - - fscache_stat(&fscache_n_attr_changed); - - op = kzalloc(sizeof(*op), GFP_KERNEL); - if (!op) { - fscache_stat(&fscache_n_attr_changed_nomem); - _leave(" = -ENOMEM"); - return -ENOMEM; - } - - fscache_operation_init(cookie, op, fscache_attr_changed_op, NULL, NULL); - trace_fscache_page_op(cookie, NULL, op, fscache_page_op_attr_changed); - op->flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_EXCLUSIVE) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - if (fscache_submit_exclusive_op(object, op) < 0) - goto nobufs_dec; - spin_unlock(&cookie->lock); - fscache_stat(&fscache_n_attr_changed_ok); - fscache_put_operation(op); - _leave(" = 0"); - return 0; - -nobufs_dec: - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs: - spin_unlock(&cookie->lock); - fscache_put_operation(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_stat(&fscache_n_attr_changed_nobufs); - _leave(" = %d", -ENOBUFS); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_attr_changed); - -/* - * Handle cancellation of a pending retrieval op - */ -static void fscache_do_cancel_retrieval(struct fscache_operation *_op) -{ - struct fscache_retrieval *op = - container_of(_op, struct fscache_retrieval, op); - - atomic_set(&op->n_pages, 0); -} - -/* - * release a retrieval op reference - */ -static void fscache_release_retrieval_op(struct fscache_operation *_op) -{ - struct fscache_retrieval *op = - container_of(_op, struct fscache_retrieval, op); - - _enter("{OP%x}", op->op.debug_id); - - ASSERTIFCMP(op->op.state != FSCACHE_OP_ST_INITIALISED, - atomic_read(&op->n_pages), ==, 0); - - if (op->context) - fscache_put_context(op->cookie, op->context); - - _leave(""); -} - -/* - * allocate a retrieval op - */ -struct fscache_retrieval *fscache_alloc_retrieval( - struct fscache_cookie *cookie, - struct address_space *mapping, - fscache_rw_complete_t end_io_func, - void *context) -{ - struct fscache_retrieval *op; - - /* allocate a retrieval operation and attempt to submit it */ - op = kzalloc(sizeof(*op), GFP_NOIO); - if (!op) { - fscache_stat(&fscache_n_retrievals_nomem); - return NULL; - } - - fscache_operation_init(cookie, &op->op, NULL, - fscache_do_cancel_retrieval, - fscache_release_retrieval_op); - op->op.flags = FSCACHE_OP_MYTHREAD | - (1UL << FSCACHE_OP_WAITING) | - (1UL << FSCACHE_OP_UNUSE_COOKIE); - op->cookie = cookie; - op->mapping = mapping; - op->end_io_func = end_io_func; - op->context = context; - INIT_LIST_HEAD(&op->to_do); - - /* Pin the netfs read context in case we need to do the actual netfs - * read because we've encountered a cache read failure. - */ - if (context) - fscache_get_context(op->cookie, context); - return op; -} - -/* - * wait for a deferred lookup to complete - */ -int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie) -{ - _enter(""); - - if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) { - _leave(" = 0 [imm]"); - return 0; - } - - fscache_stat(&fscache_n_retrievals_wait); - - if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, - TASK_INTERRUPTIBLE) != 0) { - fscache_stat(&fscache_n_retrievals_intr); - _leave(" = -ERESTARTSYS"); - return -ERESTARTSYS; - } - - ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)); - - smp_rmb(); - _leave(" = 0 [dly]"); - return 0; -} - -/* - * wait for an object to become active (or dead) - */ -int fscache_wait_for_operation_activation(struct fscache_object *object, - struct fscache_operation *op, - atomic_t *stat_op_waits, - atomic_t *stat_object_dead) -{ - int ret; - - if (!test_bit(FSCACHE_OP_WAITING, &op->flags)) - goto check_if_dead; - - _debug(">>> WT"); - if (stat_op_waits) - fscache_stat(stat_op_waits); - if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING, - TASK_INTERRUPTIBLE) != 0) { - trace_fscache_op(object->cookie, op, fscache_op_signal); - ret = fscache_cancel_op(op, false); - if (ret == 0) - return -ERESTARTSYS; - - /* it's been removed from the pending queue by another party, - * so we should get to run shortly */ - wait_on_bit(&op->flags, FSCACHE_OP_WAITING, - TASK_UNINTERRUPTIBLE); - } - _debug("<<< GO"); - -check_if_dead: - if (op->state == FSCACHE_OP_ST_CANCELLED) { - if (stat_object_dead) - fscache_stat(stat_object_dead); - _leave(" = -ENOBUFS [cancelled]"); - return -ENOBUFS; - } - if (unlikely(fscache_object_is_dying(object) || - fscache_cache_is_broken(object))) { - enum fscache_operation_state state = op->state; - trace_fscache_op(object->cookie, op, fscache_op_signal); - fscache_cancel_op(op, true); - if (stat_object_dead) - fscache_stat(stat_object_dead); - _leave(" = -ENOBUFS [obj dead %d]", state); - return -ENOBUFS; - } - return 0; -} - -/* - * read a page from the cache or allocate a block in which to store it - * - we return: - * -ENOMEM - out of memory, nothing done - * -ERESTARTSYS - interrupted - * -ENOBUFS - no backing object available in which to cache the block - * -ENODATA - no data available in the backing object for this block - * 0 - dispatched a read - it'll call end_io_func() when finished - */ -int __fscache_read_or_alloc_page(struct fscache_cookie *cookie, - struct page *page, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%p,,,", cookie, page); - - fscache_stat(&fscache_n_retrievals); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, page->mapping, - end_io_func, context); - if (!op) { - _leave(" = -ENOMEM"); - return -ENOMEM; - } - atomic_set(&op->n_pages, 1); - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_retr_one); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)); - - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); - - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_retrieval_ops); - - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { - fscache_stat(&fscache_n_cop_allocate_page); - ret = object->cache->ops->allocate_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_allocate_page); - if (ret == 0) - ret = -ENODATA; - } else { - fscache_stat(&fscache_n_cop_read_or_alloc_page); - ret = object->cache->ops->read_or_alloc_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_read_or_alloc_page); - } - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_put_retrieval(op); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_read_or_alloc_page); - -/* - * read a list of page from the cache or allocate a block in which to store - * them - * - we return: - * -ENOMEM - out of memory, some pages may be being read - * -ERESTARTSYS - interrupted, some pages may be being read - * -ENOBUFS - no backing object or space available in which to cache any - * pages not being read - * -ENODATA - no data available in the backing object for some or all of - * the pages - * 0 - dispatched a read on all pages - * - * end_io_func() will be called for each page read from the cache as it is - * finishes being read - * - * any pages for which a read is dispatched will be removed from pages and - * nr_pages - */ -int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,,%d,,,", cookie, *nr_pages); - - fscache_stat(&fscache_n_retrievals); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(*nr_pages, >, 0); - ASSERT(!list_empty(pages)); - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context); - if (!op) - return -ENOMEM; - atomic_set(&op->n_pages, *nr_pages); - trace_fscache_page_op(cookie, NULL, &op->op, fscache_page_op_retr_multi); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - atomic_inc(&object->n_reads); - __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); - - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_retrieval_ops); - - /* we wait for the operation to become active, and then process it - * *here*, in this thread, and not in the thread pool */ - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_retrieval_op_waits), - __fscache_stat(&fscache_n_retrievals_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { - fscache_stat(&fscache_n_cop_allocate_pages); - ret = object->cache->ops->allocate_pages( - op, pages, nr_pages, gfp); - fscache_stat_d(&fscache_n_cop_allocate_pages); - } else { - fscache_stat(&fscache_n_cop_read_or_alloc_pages); - ret = object->cache->ops->read_or_alloc_pages( - op, pages, nr_pages, gfp); - fscache_stat_d(&fscache_n_cop_read_or_alloc_pages); - } - -error: - if (ret == -ENOMEM) - fscache_stat(&fscache_n_retrievals_nomem); - else if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_retrievals_intr); - else if (ret == -ENODATA) - fscache_stat(&fscache_n_retrievals_nodata); - else if (ret < 0) - fscache_stat(&fscache_n_retrievals_nobufs); - else - fscache_stat(&fscache_n_retrievals_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - atomic_dec(&object->n_reads); - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_retrievals_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_read_or_alloc_pages); - -/* - * allocate a block in the cache on which to store a page - * - we return: - * -ENOMEM - out of memory, nothing done - * -ERESTARTSYS - interrupted - * -ENOBUFS - no backing object available in which to cache the block - * 0 - block allocated - */ -int __fscache_alloc_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - struct fscache_retrieval *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%p,,,", cookie, page); - - fscache_stat(&fscache_n_allocs); - - if (hlist_empty(&cookie->backing_objects)) - goto nobufs; - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - if (fscache_wait_for_deferred_lookup(cookie) < 0) - return -ERESTARTSYS; - - op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL); - if (!op) - return -ENOMEM; - atomic_set(&op->n_pages, 1); - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_alloc_one); - - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs_unlock; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, &op->op) < 0) - goto nobufs_unlock_dec; - spin_unlock(&cookie->lock); - - fscache_stat(&fscache_n_alloc_ops); - - ret = fscache_wait_for_operation_activation( - object, &op->op, - __fscache_stat(&fscache_n_alloc_op_waits), - __fscache_stat(&fscache_n_allocs_object_dead)); - if (ret < 0) - goto error; - - /* ask the cache to honour the operation */ - fscache_stat(&fscache_n_cop_allocate_page); - ret = object->cache->ops->allocate_page(op, page, gfp); - fscache_stat_d(&fscache_n_cop_allocate_page); - -error: - if (ret == -ERESTARTSYS) - fscache_stat(&fscache_n_allocs_intr); - else if (ret < 0) - fscache_stat(&fscache_n_allocs_nobufs); - else - fscache_stat(&fscache_n_allocs_ok); - - fscache_put_retrieval(op); - _leave(" = %d", ret); - return ret; - -nobufs_unlock_dec: - wake_cookie = __fscache_unuse_cookie(cookie); -nobufs_unlock: - spin_unlock(&cookie->lock); - fscache_put_retrieval(op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); -nobufs: - fscache_stat(&fscache_n_allocs_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; -} -EXPORT_SYMBOL(__fscache_alloc_page); - -/* - * Unmark pages allocate in the readahead code path (via: - * fscache_readpages_or_alloc) after delegating to the base filesystem - */ -void __fscache_readpages_cancel(struct fscache_cookie *cookie, - struct list_head *pages) -{ - struct page *page; - - list_for_each_entry(page, pages, lru) { - if (PageFsCache(page)) - __fscache_uncache_page(cookie, page); - } -} -EXPORT_SYMBOL(__fscache_readpages_cancel); - -/* - * release a write op reference - */ -static void fscache_release_write_op(struct fscache_operation *_op) -{ - _enter("{OP%x}", _op->debug_id); -} - -/* - * perform the background storage of a page into the cache - */ -static void fscache_write_op(struct fscache_operation *_op) -{ - struct fscache_storage *op = - container_of(_op, struct fscache_storage, op); - struct fscache_object *object = op->op.object; - struct fscache_cookie *cookie; - struct page *page; - unsigned n; - void *results[1]; - int ret; - - _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage)); - -again: - spin_lock(&object->lock); - cookie = object->cookie; - - if (!fscache_object_is_active(object)) { - /* If we get here, then the on-disk cache object likely no - * longer exists, so we should just cancel this write - * operation. - */ - spin_unlock(&object->lock); - fscache_op_complete(&op->op, true); - _leave(" [inactive]"); - return; - } - - if (!cookie) { - /* If we get here, then the cookie belonging to the object was - * detached, probably by the cookie being withdrawn due to - * memory pressure, which means that the pages we might write - * to the cache from no longer exist - therefore, we can just - * cancel this write operation. - */ - spin_unlock(&object->lock); - fscache_op_complete(&op->op, true); - _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}", - _op->flags, _op->state, object->state->short_name, - object->flags); - return; - } - - spin_lock(&cookie->stores_lock); - - fscache_stat(&fscache_n_store_calls); - - /* find a page to store */ - results[0] = NULL; - page = NULL; - n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_gang_lookup(cookie, &op->op, results, n, op->store_limit); - if (n != 1) - goto superseded; - page = results[0]; - _debug("gang %d [%lx]", n, page->index); - - radix_tree_tag_set(&cookie->stores, page->index, - FSCACHE_COOKIE_STORING_TAG); - radix_tree_tag_clear(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_pend2store); - - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - - if (page->index >= op->store_limit) - goto discard_page; - - fscache_stat(&fscache_n_store_pages); - fscache_stat(&fscache_n_cop_write_page); - ret = object->cache->ops->write_page(op, page); - fscache_stat_d(&fscache_n_cop_write_page); - trace_fscache_wrote_page(cookie, page, &op->op, ret); - fscache_end_page_write(object, page); - if (ret < 0) { - fscache_abort_object(object); - fscache_op_complete(&op->op, true); - } else { - fscache_enqueue_operation(&op->op); - } - - _leave(""); - return; - -discard_page: - fscache_stat(&fscache_n_store_pages_over_limit); - trace_fscache_wrote_page(cookie, page, &op->op, -ENOBUFS); - fscache_end_page_write(object, page); - goto again; - -superseded: - /* this writer is going away and there aren't any more things to - * write */ - _debug("cease"); - spin_unlock(&cookie->stores_lock); - clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); - spin_unlock(&object->lock); - fscache_op_complete(&op->op, false); - _leave(""); -} - -/* - * Clear the pages pending writing for invalidation - */ -void fscache_invalidate_writes(struct fscache_cookie *cookie) -{ - struct page *page; - void *results[16]; - int n, i; - - _enter(""); - - for (;;) { - spin_lock(&cookie->stores_lock); - n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, - ARRAY_SIZE(results), - FSCACHE_COOKIE_PENDING_TAG); - if (n == 0) { - spin_unlock(&cookie->stores_lock); - break; - } - - for (i = n - 1; i >= 0; i--) { - page = results[i]; - radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - trace_fscache_page(cookie, page, fscache_page_inval); - } - - spin_unlock(&cookie->stores_lock); - - for (i = n - 1; i >= 0; i--) - put_page(results[i]); - } - - wake_up_bit(&cookie->flags, 0); - trace_fscache_wake_cookie(cookie); - - _leave(""); -} - -/* - * request a page be stored in the cache - * - returns: - * -ENOMEM - out of memory, nothing done - * -ENOBUFS - no backing object available in which to cache the page - * 0 - dispatched a write - it'll call end_io_func() when finished - * - * if the cookie still has a backing object at this point, that object can be - * in one of a few states with respect to storage processing: - * - * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is - * set) - * - * (a) no writes yet - * - * (b) writes deferred till post-creation (mark page for writing and - * return immediately) - * - * (2) negative lookup, object created, initial fill being made from netfs - * - * (a) fill point not yet reached this page (mark page for writing and - * return) - * - * (b) fill point passed this page (queue op to store this page) - * - * (3) object extant (queue op to store this page) - * - * any other state is invalid - */ -int __fscache_write_page(struct fscache_cookie *cookie, - struct page *page, - loff_t object_size, - gfp_t gfp) -{ - struct fscache_storage *op; - struct fscache_object *object; - bool wake_cookie = false; - int ret; - - _enter("%p,%x,", cookie, (u32) page->flags); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERT(PageFsCache(page)); - - fscache_stat(&fscache_n_stores); - - if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) { - _leave(" = -ENOBUFS [invalidating]"); - return -ENOBUFS; - } - - op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY); - if (!op) - goto nomem; - - fscache_operation_init(cookie, &op->op, fscache_write_op, NULL, - fscache_release_write_op); - op->op.flags = FSCACHE_OP_ASYNC | - (1 << FSCACHE_OP_WAITING) | - (1 << FSCACHE_OP_UNUSE_COOKIE); - - ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM); - if (ret < 0) - goto nomem_free; - - trace_fscache_page_op(cookie, page, &op->op, fscache_page_op_write_one); - - ret = -ENOBUFS; - spin_lock(&cookie->lock); - - if (!fscache_cookie_enabled(cookie) || - hlist_empty(&cookie->backing_objects)) - goto nobufs; - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) - goto nobufs; - - trace_fscache_page(cookie, page, fscache_page_write); - - /* add the page to the pending-storage radix tree on the backing - * object */ - spin_lock(&object->lock); - - if (object->store_limit_l != object_size) - fscache_set_store_limit(object, object_size); - - spin_lock(&cookie->stores_lock); - - _debug("store limit %llx", (unsigned long long) object->store_limit); - - ret = radix_tree_insert(&cookie->stores, page->index, page); - if (ret < 0) { - if (ret == -EEXIST) - goto already_queued; - _debug("insert failed %d", ret); - goto nobufs_unlock_obj; - } - - trace_fscache_page(cookie, page, fscache_page_radix_insert); - radix_tree_tag_set(&cookie->stores, page->index, - FSCACHE_COOKIE_PENDING_TAG); - trace_fscache_page(cookie, page, fscache_page_radix_set_pend); - get_page(page); - - /* we only want one writer at a time, but we do need to queue new - * writers after exclusive ops */ - if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags)) - goto already_pending; - - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - - op->op.debug_id = atomic_inc_return(&fscache_op_debug_id); - op->store_limit = object->store_limit; - - __fscache_use_cookie(cookie); - if (fscache_submit_op(object, &op->op) < 0) - goto submit_failed; - - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_stat(&fscache_n_store_ops); - fscache_stat(&fscache_n_stores_ok); - - /* the work queue now carries its own ref on the object */ - fscache_put_operation(&op->op); - _leave(" = 0"); - return 0; - -already_queued: - fscache_stat(&fscache_n_stores_again); -already_pending: - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_put_operation(&op->op); - fscache_stat(&fscache_n_stores_ok); - _leave(" = 0"); - return 0; - -submit_failed: - spin_lock(&cookie->stores_lock); - radix_tree_delete(&cookie->stores, page->index); - trace_fscache_page(cookie, page, fscache_page_radix_delete); - spin_unlock(&cookie->stores_lock); - wake_cookie = __fscache_unuse_cookie(cookie); - put_page(page); - ret = -ENOBUFS; - goto nobufs; - -nobufs_unlock_obj: - spin_unlock(&cookie->stores_lock); - spin_unlock(&object->lock); -nobufs: - spin_unlock(&cookie->lock); - radix_tree_preload_end(); - fscache_put_operation(&op->op); - if (wake_cookie) - __fscache_wake_unused_cookie(cookie); - fscache_stat(&fscache_n_stores_nobufs); - _leave(" = -ENOBUFS"); - return -ENOBUFS; - -nomem_free: - fscache_put_operation(&op->op); -nomem: - fscache_stat(&fscache_n_stores_oom); - _leave(" = -ENOMEM"); - return -ENOMEM; -} -EXPORT_SYMBOL(__fscache_write_page); - -/* - * remove a page from the cache - */ -void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page) -{ - struct fscache_object *object; - - _enter(",%p", page); - - ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); - ASSERTCMP(page, !=, NULL); - - fscache_stat(&fscache_n_uncaches); - - /* cache withdrawal may beat us to it */ - if (!PageFsCache(page)) - goto done; - - trace_fscache_page(cookie, page, fscache_page_uncache); - - /* get the object */ - spin_lock(&cookie->lock); - - if (hlist_empty(&cookie->backing_objects)) { - ClearPageFsCache(page); - goto done_unlock; - } - - object = hlist_entry(cookie->backing_objects.first, - struct fscache_object, cookie_link); - - /* there might now be stuff on disk we could read */ - clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); - - /* only invoke the cache backend if we managed to mark the page - * uncached here; this deals with synchronisation vs withdrawal */ - if (TestClearPageFsCache(page) && - object->cache->ops->uncache_page) { - /* the cache backend releases the cookie lock */ - fscache_stat(&fscache_n_cop_uncache_page); - object->cache->ops->uncache_page(object, page); - fscache_stat_d(&fscache_n_cop_uncache_page); - goto done; - } - -done_unlock: - spin_unlock(&cookie->lock); -done: - _leave(""); -} -EXPORT_SYMBOL(__fscache_uncache_page); - -/** - * fscache_mark_page_cached - Mark a page as being cached - * @op: The retrieval op pages are being marked for - * @page: The page to be marked - * - * Mark a netfs page as being cached. After this is called, the netfs - * must call fscache_uncache_page() to remove the mark. - */ -void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page) -{ - struct fscache_cookie *cookie = op->op.object->cookie; - -#ifdef CONFIG_FSCACHE_STATS - atomic_inc(&fscache_n_marks); -#endif - - trace_fscache_page(cookie, page, fscache_page_cached); - - _debug("- mark %p{%lx}", page, page->index); - if (TestSetPageFsCache(page)) { - static bool once_only; - if (!once_only) { - once_only = true; - pr_warn("Cookie type %s marked page %lx multiple times\n", - cookie->def->name, page->index); - } - } - - if (cookie->def->mark_page_cached) - cookie->def->mark_page_cached(cookie->netfs_data, - op->mapping, page); -} -EXPORT_SYMBOL(fscache_mark_page_cached); - -/** - * fscache_mark_pages_cached - Mark pages as being cached - * @op: The retrieval op pages are being marked for - * @pagevec: The pages to be marked - * - * Mark a bunch of netfs pages as being cached. After this is called, - * the netfs must call fscache_uncache_page() to remove the mark. - */ -void fscache_mark_pages_cached(struct fscache_retrieval *op, - struct pagevec *pagevec) -{ - unsigned long loop; - - for (loop = 0; loop < pagevec->nr; loop++) - fscache_mark_page_cached(op, pagevec->pages[loop]); - - pagevec_reinit(pagevec); -} -EXPORT_SYMBOL(fscache_mark_pages_cached); - -/* - * Uncache all the pages in an inode that are marked PG_fscache, assuming them - * to be associated with the given cookie. - */ -void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie, - struct inode *inode) -{ - struct address_space *mapping = inode->i_mapping; - struct pagevec pvec; - pgoff_t next; - int i; - - _enter("%p,%p", cookie, inode); - - if (!mapping || mapping->nrpages == 0) { - _leave(" [no pages]"); - return; - } - - pagevec_init(&pvec); - next = 0; - do { - if (!pagevec_lookup(&pvec, mapping, &next)) - break; - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; - if (PageFsCache(page)) { - __fscache_wait_on_page_write(cookie, page); - __fscache_uncache_page(cookie, page); - } - } - pagevec_release(&pvec); - cond_resched(); - } while (next); - - _leave(""); -} -EXPORT_SYMBOL(__fscache_uncache_all_inode_pages); diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c deleted file mode 100644 index 061df8f61ffc..000000000000 --- a/fs/fscache/proc.c +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache statistics viewing interface - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL OPERATION -#include -#include -#include -#include "internal.h" - -/* - * initialise the /proc/fs/fscache/ directory - */ -int __init fscache_proc_init(void) -{ - _enter(""); - - if (!proc_mkdir("fs/fscache", NULL)) - goto error_dir; - - if (!proc_create_seq("fs/fscache/cookies", S_IFREG | 0444, NULL, - &fscache_cookies_seq_ops)) - goto error_cookies; - -#ifdef CONFIG_FSCACHE_STATS - if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, - fscache_stats_show)) - goto error_stats; -#endif - -#ifdef CONFIG_FSCACHE_OBJECT_LIST - if (!proc_create("fs/fscache/objects", S_IFREG | 0444, NULL, - &fscache_objlist_proc_ops)) - goto error_objects; -#endif - - _leave(" = 0"); - return 0; - -#ifdef CONFIG_FSCACHE_OBJECT_LIST -error_objects: -#endif -#ifdef CONFIG_FSCACHE_STATS - remove_proc_entry("fs/fscache/stats", NULL); -error_stats: -#endif - remove_proc_entry("fs/fscache/cookies", NULL); -error_cookies: - remove_proc_entry("fs/fscache", NULL); -error_dir: - _leave(" = -ENOMEM"); - return -ENOMEM; -} - -/* - * clean up the /proc/fs/fscache/ directory - */ -void fscache_proc_cleanup(void) -{ -#ifdef CONFIG_FSCACHE_OBJECT_LIST - remove_proc_entry("fs/fscache/objects", NULL); -#endif -#ifdef CONFIG_FSCACHE_STATS - remove_proc_entry("fs/fscache/stats", NULL); -#endif - remove_proc_entry("fs/fscache/cookies", NULL); - remove_proc_entry("fs/fscache", NULL); -} diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c deleted file mode 100644 index a7c3ed89a3e0..000000000000 --- a/fs/fscache/stats.c +++ /dev/null @@ -1,283 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* FS-Cache statistics - * - * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#define FSCACHE_DEBUG_LEVEL THREAD -#include -#include -#include -#include "internal.h" - -/* - * operation counters - */ -atomic_t fscache_n_op_pend; -atomic_t fscache_n_op_run; -atomic_t fscache_n_op_enqueue; -atomic_t fscache_n_op_deferred_release; -atomic_t fscache_n_op_initialised; -atomic_t fscache_n_op_release; -atomic_t fscache_n_op_gc; -atomic_t fscache_n_op_cancelled; -atomic_t fscache_n_op_rejected; - -atomic_t fscache_n_attr_changed; -atomic_t fscache_n_attr_changed_ok; -atomic_t fscache_n_attr_changed_nobufs; -atomic_t fscache_n_attr_changed_nomem; -atomic_t fscache_n_attr_changed_calls; - -atomic_t fscache_n_allocs; -atomic_t fscache_n_allocs_ok; -atomic_t fscache_n_allocs_wait; -atomic_t fscache_n_allocs_nobufs; -atomic_t fscache_n_allocs_intr; -atomic_t fscache_n_allocs_object_dead; -atomic_t fscache_n_alloc_ops; -atomic_t fscache_n_alloc_op_waits; - -atomic_t fscache_n_retrievals; -atomic_t fscache_n_retrievals_ok; -atomic_t fscache_n_retrievals_wait; -atomic_t fscache_n_retrievals_nodata; -atomic_t fscache_n_retrievals_nobufs; -atomic_t fscache_n_retrievals_intr; -atomic_t fscache_n_retrievals_nomem; -atomic_t fscache_n_retrievals_object_dead; -atomic_t fscache_n_retrieval_ops; -atomic_t fscache_n_retrieval_op_waits; - -atomic_t fscache_n_stores; -atomic_t fscache_n_stores_ok; -atomic_t fscache_n_stores_again; -atomic_t fscache_n_stores_nobufs; -atomic_t fscache_n_stores_oom; -atomic_t fscache_n_store_ops; -atomic_t fscache_n_store_calls; -atomic_t fscache_n_store_pages; -atomic_t fscache_n_store_radix_deletes; -atomic_t fscache_n_store_pages_over_limit; - -atomic_t fscache_n_store_vmscan_not_storing; -atomic_t fscache_n_store_vmscan_gone; -atomic_t fscache_n_store_vmscan_busy; -atomic_t fscache_n_store_vmscan_cancelled; -atomic_t fscache_n_store_vmscan_wait; - -atomic_t fscache_n_marks; -atomic_t fscache_n_uncaches; - -atomic_t fscache_n_acquires; -atomic_t fscache_n_acquires_null; -atomic_t fscache_n_acquires_no_cache; -atomic_t fscache_n_acquires_ok; -atomic_t fscache_n_acquires_nobufs; -atomic_t fscache_n_acquires_oom; - -atomic_t fscache_n_invalidates; -atomic_t fscache_n_invalidates_run; - -atomic_t fscache_n_updates; -atomic_t fscache_n_updates_null; -atomic_t fscache_n_updates_run; - -atomic_t fscache_n_relinquishes; -atomic_t fscache_n_relinquishes_null; -atomic_t fscache_n_relinquishes_waitcrt; -atomic_t fscache_n_relinquishes_retire; - -atomic_t fscache_n_cookie_index; -atomic_t fscache_n_cookie_data; -atomic_t fscache_n_cookie_special; - -atomic_t fscache_n_object_alloc; -atomic_t fscache_n_object_no_alloc; -atomic_t fscache_n_object_lookups; -atomic_t fscache_n_object_lookups_negative; -atomic_t fscache_n_object_lookups_positive; -atomic_t fscache_n_object_lookups_timed_out; -atomic_t fscache_n_object_created; -atomic_t fscache_n_object_avail; -atomic_t fscache_n_object_dead; - -atomic_t fscache_n_checkaux_none; -atomic_t fscache_n_checkaux_okay; -atomic_t fscache_n_checkaux_update; -atomic_t fscache_n_checkaux_obsolete; - -atomic_t fscache_n_cop_alloc_object; -atomic_t fscache_n_cop_lookup_object; -atomic_t fscache_n_cop_lookup_complete; -atomic_t fscache_n_cop_grab_object; -atomic_t fscache_n_cop_invalidate_object; -atomic_t fscache_n_cop_update_object; -atomic_t fscache_n_cop_drop_object; -atomic_t fscache_n_cop_put_object; -atomic_t fscache_n_cop_sync_cache; -atomic_t fscache_n_cop_attr_changed; -atomic_t fscache_n_cop_read_or_alloc_page; -atomic_t fscache_n_cop_read_or_alloc_pages; -atomic_t fscache_n_cop_allocate_page; -atomic_t fscache_n_cop_allocate_pages; -atomic_t fscache_n_cop_write_page; -atomic_t fscache_n_cop_uncache_page; -atomic_t fscache_n_cop_dissociate_pages; - -atomic_t fscache_n_cache_no_space_reject; -atomic_t fscache_n_cache_stale_objects; -atomic_t fscache_n_cache_retired_objects; -atomic_t fscache_n_cache_culled_objects; - -/* - * display the general statistics - */ -int fscache_stats_show(struct seq_file *m, void *v) -{ - seq_puts(m, "FS-Cache statistics\n"); - - seq_printf(m, "Cookies: idx=%u dat=%u spc=%u\n", - atomic_read(&fscache_n_cookie_index), - atomic_read(&fscache_n_cookie_data), - atomic_read(&fscache_n_cookie_special)); - - seq_printf(m, "Objects: alc=%u nal=%u avl=%u ded=%u\n", - atomic_read(&fscache_n_object_alloc), - atomic_read(&fscache_n_object_no_alloc), - atomic_read(&fscache_n_object_avail), - atomic_read(&fscache_n_object_dead)); - seq_printf(m, "ChkAux : non=%u ok=%u upd=%u obs=%u\n", - atomic_read(&fscache_n_checkaux_none), - atomic_read(&fscache_n_checkaux_okay), - atomic_read(&fscache_n_checkaux_update), - atomic_read(&fscache_n_checkaux_obsolete)); - - seq_printf(m, "Pages : mrk=%u unc=%u\n", - atomic_read(&fscache_n_marks), - atomic_read(&fscache_n_uncaches)); - - seq_printf(m, "Acquire: n=%u nul=%u noc=%u ok=%u nbf=%u" - " oom=%u\n", - atomic_read(&fscache_n_acquires), - atomic_read(&fscache_n_acquires_null), - atomic_read(&fscache_n_acquires_no_cache), - atomic_read(&fscache_n_acquires_ok), - atomic_read(&fscache_n_acquires_nobufs), - atomic_read(&fscache_n_acquires_oom)); - - seq_printf(m, "Lookups: n=%u neg=%u pos=%u crt=%u tmo=%u\n", - atomic_read(&fscache_n_object_lookups), - atomic_read(&fscache_n_object_lookups_negative), - atomic_read(&fscache_n_object_lookups_positive), - atomic_read(&fscache_n_object_created), - atomic_read(&fscache_n_object_lookups_timed_out)); - - seq_printf(m, "Invals : n=%u run=%u\n", - atomic_read(&fscache_n_invalidates), - atomic_read(&fscache_n_invalidates_run)); - - seq_printf(m, "Updates: n=%u nul=%u run=%u\n", - atomic_read(&fscache_n_updates), - atomic_read(&fscache_n_updates_null), - atomic_read(&fscache_n_updates_run)); - - seq_printf(m, "Relinqs: n=%u nul=%u wcr=%u rtr=%u\n", - atomic_read(&fscache_n_relinquishes), - atomic_read(&fscache_n_relinquishes_null), - atomic_read(&fscache_n_relinquishes_waitcrt), - atomic_read(&fscache_n_relinquishes_retire)); - - seq_printf(m, "AttrChg: n=%u ok=%u nbf=%u oom=%u run=%u\n", - atomic_read(&fscache_n_attr_changed), - atomic_read(&fscache_n_attr_changed_ok), - atomic_read(&fscache_n_attr_changed_nobufs), - atomic_read(&fscache_n_attr_changed_nomem), - atomic_read(&fscache_n_attr_changed_calls)); - - seq_printf(m, "Allocs : n=%u ok=%u wt=%u nbf=%u int=%u\n", - atomic_read(&fscache_n_allocs), - atomic_read(&fscache_n_allocs_ok), - atomic_read(&fscache_n_allocs_wait), - atomic_read(&fscache_n_allocs_nobufs), - atomic_read(&fscache_n_allocs_intr)); - seq_printf(m, "Allocs : ops=%u owt=%u abt=%u\n", - atomic_read(&fscache_n_alloc_ops), - atomic_read(&fscache_n_alloc_op_waits), - atomic_read(&fscache_n_allocs_object_dead)); - - seq_printf(m, "Retrvls: n=%u ok=%u wt=%u nod=%u nbf=%u" - " int=%u oom=%u\n", - atomic_read(&fscache_n_retrievals), - atomic_read(&fscache_n_retrievals_ok), - atomic_read(&fscache_n_retrievals_wait), - atomic_read(&fscache_n_retrievals_nodata), - atomic_read(&fscache_n_retrievals_nobufs), - atomic_read(&fscache_n_retrievals_intr), - atomic_read(&fscache_n_retrievals_nomem)); - seq_printf(m, "Retrvls: ops=%u owt=%u abt=%u\n", - atomic_read(&fscache_n_retrieval_ops), - atomic_read(&fscache_n_retrieval_op_waits), - atomic_read(&fscache_n_retrievals_object_dead)); - - seq_printf(m, "Stores : n=%u ok=%u agn=%u nbf=%u oom=%u\n", - atomic_read(&fscache_n_stores), - atomic_read(&fscache_n_stores_ok), - atomic_read(&fscache_n_stores_again), - atomic_read(&fscache_n_stores_nobufs), - atomic_read(&fscache_n_stores_oom)); - seq_printf(m, "Stores : ops=%u run=%u pgs=%u rxd=%u olm=%u\n", - atomic_read(&fscache_n_store_ops), - atomic_read(&fscache_n_store_calls), - atomic_read(&fscache_n_store_pages), - atomic_read(&fscache_n_store_radix_deletes), - atomic_read(&fscache_n_store_pages_over_limit)); - - seq_printf(m, "VmScan : nos=%u gon=%u bsy=%u can=%u wt=%u\n", - atomic_read(&fscache_n_store_vmscan_not_storing), - atomic_read(&fscache_n_store_vmscan_gone), - atomic_read(&fscache_n_store_vmscan_busy), - atomic_read(&fscache_n_store_vmscan_cancelled), - atomic_read(&fscache_n_store_vmscan_wait)); - - seq_printf(m, "Ops : pend=%u run=%u enq=%u can=%u rej=%u\n", - atomic_read(&fscache_n_op_pend), - atomic_read(&fscache_n_op_run), - atomic_read(&fscache_n_op_enqueue), - atomic_read(&fscache_n_op_cancelled), - atomic_read(&fscache_n_op_rejected)); - seq_printf(m, "Ops : ini=%u dfr=%u rel=%u gc=%u\n", - atomic_read(&fscache_n_op_initialised), - atomic_read(&fscache_n_op_deferred_release), - atomic_read(&fscache_n_op_release), - atomic_read(&fscache_n_op_gc)); - - seq_printf(m, "CacheOp: alo=%d luo=%d luc=%d gro=%d\n", - atomic_read(&fscache_n_cop_alloc_object), - atomic_read(&fscache_n_cop_lookup_object), - atomic_read(&fscache_n_cop_lookup_complete), - atomic_read(&fscache_n_cop_grab_object)); - seq_printf(m, "CacheOp: inv=%d upo=%d dro=%d pto=%d atc=%d syn=%d\n", - atomic_read(&fscache_n_cop_invalidate_object), - atomic_read(&fscache_n_cop_update_object), - atomic_read(&fscache_n_cop_drop_object), - atomic_read(&fscache_n_cop_put_object), - atomic_read(&fscache_n_cop_attr_changed), - atomic_read(&fscache_n_cop_sync_cache)); - seq_printf(m, "CacheOp: rap=%d ras=%d alp=%d als=%d wrp=%d ucp=%d dsp=%d\n", - atomic_read(&fscache_n_cop_read_or_alloc_page), - atomic_read(&fscache_n_cop_read_or_alloc_pages), - atomic_read(&fscache_n_cop_allocate_page), - atomic_read(&fscache_n_cop_allocate_pages), - atomic_read(&fscache_n_cop_write_page), - atomic_read(&fscache_n_cop_uncache_page), - atomic_read(&fscache_n_cop_dissociate_pages)); - seq_printf(m, "CacheEv: nsp=%d stl=%d rtr=%d cul=%d\n", - atomic_read(&fscache_n_cache_no_space_reject), - atomic_read(&fscache_n_cache_stale_objects), - atomic_read(&fscache_n_cache_retired_objects), - atomic_read(&fscache_n_cache_culled_objects)); - netfs_stats_show(m); - return 0; -} diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 8d39491c5f9f..47f21a53ac4b 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* General filesystem caching backing cache interface * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * * NOTE!!! See: @@ -15,551 +15,5 @@ #define _LINUX_FSCACHE_CACHE_H #include -#include -#include - -#define NR_MAXCACHES BITS_PER_LONG - -struct fscache_cache; -struct fscache_cache_ops; -struct fscache_object; -struct fscache_operation; - -enum fscache_obj_ref_trace { - fscache_obj_get_add_to_deps, - fscache_obj_get_queue, - fscache_obj_put_alloc_fail, - fscache_obj_put_attach_fail, - fscache_obj_put_drop_obj, - fscache_obj_put_enq_dep, - fscache_obj_put_queue, - fscache_obj_put_work, - fscache_obj_ref__nr_traces -}; - -/* - * cache tag definition - */ -struct fscache_cache_tag { - struct list_head link; - struct fscache_cache *cache; /* cache referred to by this tag */ - unsigned long flags; -#define FSCACHE_TAG_RESERVED 0 /* T if tag is reserved for a cache */ - atomic_t usage; - char name[]; /* tag name */ -}; - -/* - * cache definition - */ -struct fscache_cache { - const struct fscache_cache_ops *ops; - struct fscache_cache_tag *tag; /* tag representing this cache */ - struct kobject *kobj; /* system representation of this cache */ - struct list_head link; /* link in list of caches */ - size_t max_index_size; /* maximum size of index data */ - char identifier[36]; /* cache label */ - - /* node management */ - struct work_struct op_gc; /* operation garbage collector */ - struct list_head object_list; /* list of data/index objects */ - struct list_head op_gc_list; /* list of ops to be deleted */ - spinlock_t object_list_lock; - spinlock_t op_gc_list_lock; - atomic_t object_count; /* no. of live objects in this cache */ - struct fscache_object *fsdef; /* object for the fsdef index */ - unsigned long flags; -#define FSCACHE_IOERROR 0 /* cache stopped on I/O error */ -#define FSCACHE_CACHE_WITHDRAWN 1 /* cache has been withdrawn */ -}; - -extern wait_queue_head_t fscache_cache_cleared_wq; - -/* - * operation to be applied to a cache object - * - retrieval initiation operations are done in the context of the process - * that issued them, and not in an async thread pool - */ -typedef void (*fscache_operation_release_t)(struct fscache_operation *op); -typedef void (*fscache_operation_processor_t)(struct fscache_operation *op); -typedef void (*fscache_operation_cancel_t)(struct fscache_operation *op); - -enum fscache_operation_state { - FSCACHE_OP_ST_BLANK, /* Op is not yet submitted */ - FSCACHE_OP_ST_INITIALISED, /* Op is initialised */ - FSCACHE_OP_ST_PENDING, /* Op is blocked from running */ - FSCACHE_OP_ST_IN_PROGRESS, /* Op is in progress */ - FSCACHE_OP_ST_COMPLETE, /* Op is complete */ - FSCACHE_OP_ST_CANCELLED, /* Op has been cancelled */ - FSCACHE_OP_ST_DEAD /* Op is now dead */ -}; - -struct fscache_operation { - struct work_struct work; /* record for async ops */ - struct list_head pend_link; /* link in object->pending_ops */ - struct fscache_object *object; /* object to be operated upon */ - - unsigned long flags; -#define FSCACHE_OP_TYPE 0x000f /* operation type */ -#define FSCACHE_OP_ASYNC 0x0001 /* - async op, processor may sleep for disk */ -#define FSCACHE_OP_MYTHREAD 0x0002 /* - processing is done be issuing thread, not pool */ -#define FSCACHE_OP_WAITING 4 /* cleared when op is woken */ -#define FSCACHE_OP_EXCLUSIVE 5 /* exclusive op, other ops must wait */ -#define FSCACHE_OP_DEC_READ_CNT 6 /* decrement object->n_reads on destruction */ -#define FSCACHE_OP_UNUSE_COOKIE 7 /* call fscache_unuse_cookie() on completion */ -#define FSCACHE_OP_KEEP_FLAGS 0x00f0 /* flags to keep when repurposing an op */ - - enum fscache_operation_state state; - atomic_t usage; - unsigned debug_id; /* debugging ID */ - - /* operation processor callback - * - can be NULL if FSCACHE_OP_WAITING is going to be used to perform - * the op in a non-pool thread */ - fscache_operation_processor_t processor; - - /* Operation cancellation cleanup (optional) */ - fscache_operation_cancel_t cancel; - - /* operation releaser */ - fscache_operation_release_t release; -}; - -extern atomic_t fscache_op_debug_id; -extern void fscache_op_work_func(struct work_struct *work); - -extern void fscache_enqueue_operation(struct fscache_operation *); -extern void fscache_op_complete(struct fscache_operation *, bool); -extern void fscache_put_operation(struct fscache_operation *); -extern void fscache_operation_init(struct fscache_cookie *, - struct fscache_operation *, - fscache_operation_processor_t, - fscache_operation_cancel_t, - fscache_operation_release_t); - -/* - * data read operation - */ -struct fscache_retrieval { - struct fscache_operation op; - struct fscache_cookie *cookie; /* The netfs cookie */ - struct address_space *mapping; /* netfs pages */ - fscache_rw_complete_t end_io_func; /* function to call on I/O completion */ - void *context; /* netfs read context (pinned) */ - struct list_head to_do; /* list of things to be done by the backend */ - atomic_t n_pages; /* number of pages to be retrieved */ -}; - -typedef int (*fscache_page_retrieval_func_t)(struct fscache_retrieval *op, - struct page *page, - gfp_t gfp); - -typedef int (*fscache_pages_retrieval_func_t)(struct fscache_retrieval *op, - struct list_head *pages, - unsigned *nr_pages, - gfp_t gfp); - -/** - * fscache_get_retrieval - Get an extra reference on a retrieval operation - * @op: The retrieval operation to get a reference on - * - * Get an extra reference on a retrieval operation. - */ -static inline -struct fscache_retrieval *fscache_get_retrieval(struct fscache_retrieval *op) -{ - atomic_inc(&op->op.usage); - return op; -} - -/** - * fscache_enqueue_retrieval - Enqueue a retrieval operation for processing - * @op: The retrieval operation affected - * - * Enqueue a retrieval operation for processing by the FS-Cache thread pool. - */ -static inline void fscache_enqueue_retrieval(struct fscache_retrieval *op) -{ - fscache_enqueue_operation(&op->op); -} - -/** - * fscache_retrieval_complete - Record (partial) completion of a retrieval - * @op: The retrieval operation affected - * @n_pages: The number of pages to account for - */ -static inline void fscache_retrieval_complete(struct fscache_retrieval *op, - int n_pages) -{ - if (atomic_sub_return_relaxed(n_pages, &op->n_pages) <= 0) - fscache_op_complete(&op->op, false); -} - -/** - * fscache_put_retrieval - Drop a reference to a retrieval operation - * @op: The retrieval operation affected - * - * Drop a reference to a retrieval operation. - */ -static inline void fscache_put_retrieval(struct fscache_retrieval *op) -{ - fscache_put_operation(&op->op); -} - -/* - * cached page storage work item - * - used to do three things: - * - batch writes to the cache - * - do cache writes asynchronously - * - defer writes until cache object lookup completion - */ -struct fscache_storage { - struct fscache_operation op; - pgoff_t store_limit; /* don't write more than this */ -}; - -/* - * cache operations - */ -struct fscache_cache_ops { - /* name of cache provider */ - const char *name; - - /* allocate an object record for a cookie */ - struct fscache_object *(*alloc_object)(struct fscache_cache *cache, - struct fscache_cookie *cookie); - - /* look up the object for a cookie - * - return -ETIMEDOUT to be requeued - */ - int (*lookup_object)(struct fscache_object *object); - - /* finished looking up */ - void (*lookup_complete)(struct fscache_object *object); - - /* increment the usage count on this object (may fail if unmounting) */ - struct fscache_object *(*grab_object)(struct fscache_object *object, - enum fscache_obj_ref_trace why); - - /* pin an object in the cache */ - int (*pin_object)(struct fscache_object *object); - - /* unpin an object in the cache */ - void (*unpin_object)(struct fscache_object *object); - - /* check the consistency between the backing cache and the FS-Cache - * cookie */ - int (*check_consistency)(struct fscache_operation *op); - - /* store the updated auxiliary data on an object */ - void (*update_object)(struct fscache_object *object); - - /* Invalidate an object */ - void (*invalidate_object)(struct fscache_operation *op); - - /* discard the resources pinned by an object and effect retirement if - * necessary */ - void (*drop_object)(struct fscache_object *object); - - /* dispose of a reference to an object */ - void (*put_object)(struct fscache_object *object, - enum fscache_obj_ref_trace why); - - /* sync a cache */ - void (*sync_cache)(struct fscache_cache *cache); - - /* notification that the attributes of a non-index object (such as - * i_size) have changed */ - int (*attr_changed)(struct fscache_object *object); - - /* reserve space for an object's data and associated metadata */ - int (*reserve_space)(struct fscache_object *object, loff_t i_size); - - /* request a backing block for a page be read or allocated in the - * cache */ - fscache_page_retrieval_func_t read_or_alloc_page; - - /* request backing blocks for a list of pages be read or allocated in - * the cache */ - fscache_pages_retrieval_func_t read_or_alloc_pages; - - /* request a backing block for a page be allocated in the cache so that - * it can be written directly */ - fscache_page_retrieval_func_t allocate_page; - - /* request backing blocks for pages be allocated in the cache so that - * they can be written directly */ - fscache_pages_retrieval_func_t allocate_pages; - - /* write a page to its backing block in the cache */ - int (*write_page)(struct fscache_storage *op, struct page *page); - - /* detach backing block from a page (optional) - * - must release the cookie lock before returning - * - may sleep - */ - void (*uncache_page)(struct fscache_object *object, - struct page *page); - - /* dissociate a cache from all the pages it was backing */ - void (*dissociate_pages)(struct fscache_cache *cache); - - /* Begin a read operation for the netfs lib */ - int (*begin_read_operation)(struct netfs_read_request *rreq, - struct fscache_retrieval *op); -}; - -extern struct fscache_cookie fscache_fsdef_index; - -/* - * Event list for fscache_object::{event_mask,events} - */ -enum { - FSCACHE_OBJECT_EV_NEW_CHILD, /* T if object has a new child */ - FSCACHE_OBJECT_EV_PARENT_READY, /* T if object's parent is ready */ - FSCACHE_OBJECT_EV_UPDATE, /* T if object should be updated */ - FSCACHE_OBJECT_EV_INVALIDATE, /* T if cache requested object invalidation */ - FSCACHE_OBJECT_EV_CLEARED, /* T if accessors all gone */ - FSCACHE_OBJECT_EV_ERROR, /* T if fatal error occurred during processing */ - FSCACHE_OBJECT_EV_KILL, /* T if netfs relinquished or cache withdrew object */ - NR_FSCACHE_OBJECT_EVENTS -}; - -#define FSCACHE_OBJECT_EVENTS_MASK ((1UL << NR_FSCACHE_OBJECT_EVENTS) - 1) - -/* - * States for object state machine. - */ -struct fscache_transition { - unsigned long events; - const struct fscache_state *transit_to; -}; - -struct fscache_state { - char name[24]; - char short_name[8]; - const struct fscache_state *(*work)(struct fscache_object *object, - int event); - const struct fscache_transition transitions[]; -}; - -/* - * on-disk cache file or index handle - */ -struct fscache_object { - const struct fscache_state *state; /* Object state machine state */ - const struct fscache_transition *oob_table; /* OOB state transition table */ - int debug_id; /* debugging ID */ - int n_children; /* number of child objects */ - int n_ops; /* number of extant ops on object */ - int n_obj_ops; /* number of object ops outstanding on object */ - int n_in_progress; /* number of ops in progress */ - int n_exclusive; /* number of exclusive ops queued or in progress */ - atomic_t n_reads; /* number of read ops in progress */ - spinlock_t lock; /* state and operations lock */ - - unsigned long lookup_jif; /* time at which lookup started */ - unsigned long oob_event_mask; /* OOB events this object is interested in */ - unsigned long event_mask; /* events this object is interested in */ - unsigned long events; /* events to be processed by this object - * (order is important - using fls) */ - - unsigned long flags; -#define FSCACHE_OBJECT_LOCK 0 /* T if object is busy being processed */ -#define FSCACHE_OBJECT_PENDING_WRITE 1 /* T if object has pending write */ -#define FSCACHE_OBJECT_WAITING 2 /* T if object is waiting on its parent */ -#define FSCACHE_OBJECT_IS_LIVE 3 /* T if object is not withdrawn or relinquished */ -#define FSCACHE_OBJECT_IS_LOOKED_UP 4 /* T if object has been looked up */ -#define FSCACHE_OBJECT_IS_AVAILABLE 5 /* T if object has become active */ -#define FSCACHE_OBJECT_RETIRED 6 /* T if object was retired on relinquishment */ -#define FSCACHE_OBJECT_KILLED_BY_CACHE 7 /* T if object was killed by the cache */ -#define FSCACHE_OBJECT_RUN_AFTER_DEAD 8 /* T if object has been dispatched after death */ - - struct list_head cache_link; /* link in cache->object_list */ - struct hlist_node cookie_link; /* link in cookie->backing_objects */ - struct fscache_cache *cache; /* cache that supplied this object */ - struct fscache_cookie *cookie; /* netfs's file/index object */ - struct fscache_object *parent; /* parent object */ - struct work_struct work; /* attention scheduling record */ - struct list_head dependents; /* FIFO of dependent objects */ - struct list_head dep_link; /* link in parent's dependents list */ - struct list_head pending_ops; /* unstarted operations on this object */ - pgoff_t store_limit; /* current storage limit */ - loff_t store_limit_l; /* current storage limit */ -}; - -extern void fscache_object_init(struct fscache_object *, struct fscache_cookie *, - struct fscache_cache *); -extern void fscache_object_destroy(struct fscache_object *); - -extern void fscache_object_lookup_negative(struct fscache_object *object); -extern void fscache_obtained_object(struct fscache_object *object); - -static inline bool fscache_object_is_live(struct fscache_object *object) -{ - return test_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags); -} - -static inline bool fscache_object_is_dying(struct fscache_object *object) -{ - return !fscache_object_is_live(object); -} - -static inline bool fscache_object_is_available(struct fscache_object *object) -{ - return test_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags); -} - -static inline bool fscache_cache_is_broken(struct fscache_object *object) -{ - return test_bit(FSCACHE_IOERROR, &object->cache->flags); -} - -static inline bool fscache_object_is_active(struct fscache_object *object) -{ - return fscache_object_is_available(object) && - fscache_object_is_live(object) && - !fscache_cache_is_broken(object); -} - -/** - * fscache_object_destroyed - Note destruction of an object in a cache - * @cache: The cache from which the object came - * - * Note the destruction and deallocation of an object record in a cache. - */ -static inline void fscache_object_destroyed(struct fscache_cache *cache) -{ - if (atomic_dec_and_test(&cache->object_count)) - wake_up_all(&fscache_cache_cleared_wq); -} - -/** - * fscache_object_lookup_error - Note an object encountered an error - * @object: The object on which the error was encountered - * - * Note that an object encountered a fatal error (usually an I/O error) and - * that it should be withdrawn as soon as possible. - */ -static inline void fscache_object_lookup_error(struct fscache_object *object) -{ - set_bit(FSCACHE_OBJECT_EV_ERROR, &object->events); -} - -/** - * fscache_set_store_limit - Set the maximum size to be stored in an object - * @object: The object to set the maximum on - * @i_size: The limit to set in bytes - * - * Set the maximum size an object is permitted to reach, implying the highest - * byte that may be written. Intended to be called by the attr_changed() op. - * - * See Documentation/filesystems/caching/backend-api.rst for a complete - * description. - */ -static inline -void fscache_set_store_limit(struct fscache_object *object, loff_t i_size) -{ - object->store_limit_l = i_size; - object->store_limit = i_size >> PAGE_SHIFT; - if (i_size & ~PAGE_MASK) - object->store_limit++; -} - -/** - * fscache_end_io - End a retrieval operation on a page - * @op: The FS-Cache operation covering the retrieval - * @page: The page that was to be fetched - * @error: The error code (0 if successful) - * - * Note the end of an operation to retrieve a page, as covered by a particular - * operation record. - */ -static inline void fscache_end_io(struct fscache_retrieval *op, - struct page *page, int error) -{ - op->end_io_func(page, op->context, error); -} - -static inline void __fscache_use_cookie(struct fscache_cookie *cookie) -{ - atomic_inc(&cookie->n_active); -} - -/** - * fscache_use_cookie - Request usage of cookie attached to an object - * @object: Object description - * - * Request usage of the cookie attached to an object. NULL is returned if the - * relinquishment had reduced the cookie usage count to 0. - */ -static inline bool fscache_use_cookie(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - return atomic_inc_not_zero(&cookie->n_active) != 0; -} - -static inline bool __fscache_unuse_cookie(struct fscache_cookie *cookie) -{ - return atomic_dec_and_test(&cookie->n_active); -} - -static inline void __fscache_wake_unused_cookie(struct fscache_cookie *cookie) -{ - wake_up_var(&cookie->n_active); -} - -/** - * fscache_unuse_cookie - Cease usage of cookie attached to an object - * @object: Object description - * - * Cease usage of the cookie attached to an object. When the users count - * reaches zero then the cookie relinquishment will be permitted to proceed. - */ -static inline void fscache_unuse_cookie(struct fscache_object *object) -{ - struct fscache_cookie *cookie = object->cookie; - if (__fscache_unuse_cookie(cookie)) - __fscache_wake_unused_cookie(cookie); -} - -/* - * out-of-line cache backend functions - */ -extern __printf(3, 4) -void fscache_init_cache(struct fscache_cache *cache, - const struct fscache_cache_ops *ops, - const char *idfmt, ...); - -extern int fscache_add_cache(struct fscache_cache *cache, - struct fscache_object *fsdef, - const char *tagname); -extern void fscache_withdraw_cache(struct fscache_cache *cache); - -extern void fscache_io_error(struct fscache_cache *cache); - -extern void fscache_mark_page_cached(struct fscache_retrieval *op, - struct page *page); - -extern void fscache_mark_pages_cached(struct fscache_retrieval *op, - struct pagevec *pagevec); - -extern bool fscache_object_sleep_till_congested(signed long *timeoutp); - -extern enum fscache_checkaux fscache_check_aux(struct fscache_object *object, - const void *data, - uint16_t datalen, - loff_t object_size); - -extern void fscache_object_retrying_stale(struct fscache_object *object); - -enum fscache_why_object_killed { - FSCACHE_OBJECT_IS_STALE, - FSCACHE_OBJECT_NO_SPACE, - FSCACHE_OBJECT_WAS_RETIRED, - FSCACHE_OBJECT_WAS_CULLED, -}; -extern void fscache_object_mark_killed(struct fscache_object *object, - enum fscache_why_object_killed why); #endif /* _LINUX_FSCACHE_CACHE_H */ diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 3b2282c157f7..0364a4ca16f6 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -15,861 +15,14 @@ #define _LINUX_FSCACHE_H #include -#include -#include -#include -#include #include #if defined(CONFIG_FSCACHE) || defined(CONFIG_FSCACHE_MODULE) -#define fscache_available() (1) #define fscache_cookie_valid(cookie) (cookie) +#define fscache_cookie_enabled(cookie) (cookie) #else -#define fscache_available() (0) #define fscache_cookie_valid(cookie) (0) +#define fscache_cookie_enabled(cookie) (0) #endif - -/* pattern used to fill dead space in an index entry */ -#define FSCACHE_INDEX_DEADFILL_PATTERN 0x79 - -struct pagevec; -struct fscache_cache_tag; -struct fscache_cookie; -struct fscache_netfs; -struct netfs_read_request; - -typedef void (*fscache_rw_complete_t)(struct page *page, - void *context, - int error); - -/* result of index entry consultation */ -enum fscache_checkaux { - FSCACHE_CHECKAUX_OKAY, /* entry okay as is */ - FSCACHE_CHECKAUX_NEEDS_UPDATE, /* entry requires update */ - FSCACHE_CHECKAUX_OBSOLETE, /* entry requires deletion */ -}; - -/* - * fscache cookie definition - */ -struct fscache_cookie_def { - /* name of cookie type */ - char name[16]; - - /* cookie type */ - uint8_t type; -#define FSCACHE_COOKIE_TYPE_INDEX 0 -#define FSCACHE_COOKIE_TYPE_DATAFILE 1 - - /* select the cache into which to insert an entry in this index - * - optional - * - should return a cache identifier or NULL to cause the cache to be - * inherited from the parent if possible or the first cache picked - * for a non-index file if not - */ - struct fscache_cache_tag *(*select_cache)( - const void *parent_netfs_data, - const void *cookie_netfs_data); - - /* consult the netfs about the state of an object - * - this function can be absent if the index carries no state data - * - the netfs data from the cookie being used as the target is - * presented, as is the auxiliary data and the object size - */ - enum fscache_checkaux (*check_aux)(void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size); - - /* get an extra reference on a read context - * - this function can be absent if the completion function doesn't - * require a context - */ - void (*get_context)(void *cookie_netfs_data, void *context); - - /* release an extra reference on a read context - * - this function can be absent if the completion function doesn't - * require a context - */ - void (*put_context)(void *cookie_netfs_data, void *context); - - /* indicate page that now have cache metadata retained - * - this function should mark the specified page as now being cached - * - the page will have been marked with PG_fscache before this is - * called, so this is optional - */ - void (*mark_page_cached)(void *cookie_netfs_data, - struct address_space *mapping, - struct page *page); -}; - -/* - * fscache cached network filesystem type - * - name, version and ops must be filled in before registration - * - all other fields will be set during registration - */ -struct fscache_netfs { - uint32_t version; /* indexing version */ - const char *name; /* filesystem name */ - struct fscache_cookie *primary_index; -}; - -/* - * data file or index object cookie - * - a file will only appear in one cache - * - a request to cache a file may or may not be honoured, subject to - * constraints such as disk space - * - indices are created on disk just-in-time - */ -struct fscache_cookie { - refcount_t ref; /* number of users of this cookie */ - atomic_t n_children; /* number of children of this cookie */ - atomic_t n_active; /* number of active users of netfs ptrs */ - unsigned int debug_id; - spinlock_t lock; - spinlock_t stores_lock; /* lock on page store tree */ - struct hlist_head backing_objects; /* object(s) backing this file/index */ - const struct fscache_cookie_def *def; /* definition */ - struct fscache_cookie *parent; /* parent of this entry */ - struct hlist_bl_node hash_link; /* Link in hash table */ - struct list_head proc_link; /* Link in proc list */ - void *netfs_data; /* back pointer to netfs */ - struct radix_tree_root stores; /* pages to be stored on this cookie */ -#define FSCACHE_COOKIE_PENDING_TAG 0 /* pages tag: pending write to cache */ -#define FSCACHE_COOKIE_STORING_TAG 1 /* pages tag: writing to cache */ - - unsigned long flags; -#define FSCACHE_COOKIE_LOOKING_UP 0 /* T if non-index cookie being looked up still */ -#define FSCACHE_COOKIE_NO_DATA_YET 1 /* T if new object with no cached data yet */ -#define FSCACHE_COOKIE_UNAVAILABLE 2 /* T if cookie is unavailable (error, etc) */ -#define FSCACHE_COOKIE_INVALIDATING 3 /* T if cookie is being invalidated */ -#define FSCACHE_COOKIE_RELINQUISHED 4 /* T if cookie has been relinquished */ -#define FSCACHE_COOKIE_ENABLED 5 /* T if cookie is enabled */ -#define FSCACHE_COOKIE_ENABLEMENT_LOCK 6 /* T if cookie is being en/disabled */ -#define FSCACHE_COOKIE_AUX_UPDATED 8 /* T if the auxiliary data was updated */ -#define FSCACHE_COOKIE_ACQUIRED 9 /* T if cookie is in use */ -#define FSCACHE_COOKIE_RELINQUISHING 10 /* T if cookie is being relinquished */ - - u8 type; /* Type of object */ - u8 key_len; /* Length of index key */ - u8 aux_len; /* Length of auxiliary data */ - u32 key_hash; /* Hash of parent, type, key, len */ - union { - void *key; /* Index key */ - u8 inline_key[16]; /* - If the key is short enough */ - }; - union { - void *aux; /* Auxiliary data */ - u8 inline_aux[8]; /* - If the aux data is short enough */ - }; -}; - -static inline bool fscache_cookie_enabled(struct fscache_cookie *cookie) -{ - return fscache_cookie_valid(cookie) && test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); -} - -/* - * slow-path functions for when there is actually caching available, and the - * netfs does actually have a valid token - * - these are not to be called directly - * - these are undefined symbols when FS-Cache is not configured and the - * optimiser takes care of not using them - */ -extern int __fscache_register_netfs(struct fscache_netfs *); -extern void __fscache_unregister_netfs(struct fscache_netfs *); -extern struct fscache_cache_tag *__fscache_lookup_cache_tag(const char *); -extern void __fscache_release_cache_tag(struct fscache_cache_tag *); - -extern struct fscache_cookie *__fscache_acquire_cookie( - struct fscache_cookie *, - const struct fscache_cookie_def *, - const void *, size_t, - const void *, size_t, - void *, loff_t, bool); -extern void __fscache_relinquish_cookie(struct fscache_cookie *, const void *, bool); -extern int __fscache_check_consistency(struct fscache_cookie *, const void *); -extern void __fscache_update_cookie(struct fscache_cookie *, const void *); -extern int __fscache_attr_changed(struct fscache_cookie *); -extern void __fscache_invalidate(struct fscache_cookie *); -extern void __fscache_wait_on_invalidate(struct fscache_cookie *); - -#ifdef FSCACHE_USE_NEW_IO_API -extern int __fscache_begin_read_operation(struct netfs_read_request *, struct fscache_cookie *); -#else -extern int __fscache_read_or_alloc_page(struct fscache_cookie *, - struct page *, - fscache_rw_complete_t, - void *, - gfp_t); -extern int __fscache_read_or_alloc_pages(struct fscache_cookie *, - struct address_space *, - struct list_head *, - unsigned *, - fscache_rw_complete_t, - void *, - gfp_t); -extern int __fscache_alloc_page(struct fscache_cookie *, struct page *, gfp_t); -extern int __fscache_write_page(struct fscache_cookie *, struct page *, loff_t, gfp_t); -extern void __fscache_uncache_page(struct fscache_cookie *, struct page *); -extern bool __fscache_check_page_write(struct fscache_cookie *, struct page *); -extern void __fscache_wait_on_page_write(struct fscache_cookie *, struct page *); -extern bool __fscache_maybe_release_page(struct fscache_cookie *, struct page *, - gfp_t); -extern void __fscache_uncache_all_inode_pages(struct fscache_cookie *, - struct inode *); -extern void __fscache_readpages_cancel(struct fscache_cookie *cookie, - struct list_head *pages); -#endif /* FSCACHE_USE_NEW_IO_API */ - -extern void __fscache_disable_cookie(struct fscache_cookie *, const void *, bool); -extern void __fscache_enable_cookie(struct fscache_cookie *, const void *, loff_t, - bool (*)(void *), void *); - -/** - * fscache_register_netfs - Register a filesystem as desiring caching services - * @netfs: The description of the filesystem - * - * Register a filesystem as desiring caching services if they're available. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_register_netfs(struct fscache_netfs *netfs) -{ - if (fscache_available()) - return __fscache_register_netfs(netfs); - else - return 0; -} - -/** - * fscache_unregister_netfs - Indicate that a filesystem no longer desires - * caching services - * @netfs: The description of the filesystem - * - * Indicate that a filesystem no longer desires caching services for the - * moment. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_unregister_netfs(struct fscache_netfs *netfs) -{ - if (fscache_available()) - __fscache_unregister_netfs(netfs); -} - -/** - * fscache_lookup_cache_tag - Look up a cache tag - * @name: The name of the tag to search for - * - * Acquire a specific cache referral tag that can be used to select a specific - * cache in which to cache an index. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -struct fscache_cache_tag *fscache_lookup_cache_tag(const char *name) -{ - if (fscache_available()) - return __fscache_lookup_cache_tag(name); - else - return NULL; -} - -/** - * fscache_release_cache_tag - Release a cache tag - * @tag: The tag to release - * - * Release a reference to a cache referral tag previously looked up. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_release_cache_tag(struct fscache_cache_tag *tag) -{ - if (fscache_available()) - __fscache_release_cache_tag(tag); -} - -/** - * fscache_acquire_cookie - Acquire a cookie to represent a cache object - * @parent: The cookie that's to be the parent of this one - * @def: A description of the cache object, including callback operations - * @index_key: The index key for this cookie - * @index_key_len: Size of the index key - * @aux_data: The auxiliary data for the cookie (may be NULL) - * @aux_data_len: Size of the auxiliary data buffer - * @netfs_data: An arbitrary piece of data to be kept in the cookie to - * represent the cache object to the netfs - * @object_size: The initial size of object - * @enable: Whether or not to enable a data cookie immediately - * - * This function is used to inform FS-Cache about part of an index hierarchy - * that can be used to locate files. This is done by requesting a cookie for - * each index in the path to the file. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -struct fscache_cookie *fscache_acquire_cookie( - struct fscache_cookie *parent, - const struct fscache_cookie_def *def, - const void *index_key, - size_t index_key_len, - const void *aux_data, - size_t aux_data_len, - void *netfs_data, - loff_t object_size, - bool enable) -{ - if (fscache_cookie_valid(parent) && fscache_cookie_enabled(parent)) - return __fscache_acquire_cookie(parent, def, - index_key, index_key_len, - aux_data, aux_data_len, - netfs_data, object_size, enable); - else - return NULL; -} - -/** - * fscache_relinquish_cookie - Return the cookie to the cache, maybe discarding - * it - * @cookie: The cookie being returned - * @aux_data: The updated auxiliary data for the cookie (may be NULL) - * @retire: True if the cache object the cookie represents is to be discarded - * - * This function returns a cookie to the cache, forcibly discarding the - * associated cache object if retire is set to true. The opportunity is - * provided to update the auxiliary data in the cache before the object is - * disconnected. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_relinquish_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool retire) -{ - if (fscache_cookie_valid(cookie)) - __fscache_relinquish_cookie(cookie, aux_data, retire); -} - -/** - * fscache_check_consistency - Request validation of a cache's auxiliary data - * @cookie: The cookie representing the cache object - * @aux_data: The updated auxiliary data for the cookie (may be NULL) - * - * Request an consistency check from fscache, which passes the request to the - * backing cache. The auxiliary data on the cookie will be updated first if - * @aux_data is set. - * - * Returns 0 if consistent and -ESTALE if inconsistent. May also - * return -ENOMEM and -ERESTARTSYS. - */ -static inline -int fscache_check_consistency(struct fscache_cookie *cookie, - const void *aux_data) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_check_consistency(cookie, aux_data); - else - return 0; -} - -/** - * fscache_update_cookie - Request that a cache object be updated - * @cookie: The cookie representing the cache object - * @aux_data: The updated auxiliary data for the cookie (may be NULL) - * - * Request an update of the index data for the cache object associated with the - * cookie. The auxiliary data on the cookie will be updated first if @aux_data - * is set. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - __fscache_update_cookie(cookie, aux_data); -} - -/** - * fscache_pin_cookie - Pin a data-storage cache object in its cache - * @cookie: The cookie representing the cache object - * - * Permit data-storage cache objects to be pinned in the cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_pin_cookie(struct fscache_cookie *cookie) -{ - return -ENOBUFS; -} - -/** - * fscache_pin_cookie - Unpin a data-storage cache object in its cache - * @cookie: The cookie representing the cache object - * - * Permit data-storage cache objects to be unpinned from the cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_unpin_cookie(struct fscache_cookie *cookie) -{ -} - -/** - * fscache_attr_changed - Notify cache that an object's attributes changed - * @cookie: The cookie representing the cache object - * - * Send a notification to the cache indicating that an object's attributes have - * changed. This includes the data size. These attributes will be obtained - * through the get_attr() cookie definition op. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_attr_changed(struct fscache_cookie *cookie) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_attr_changed(cookie); - else - return -ENOBUFS; -} - -/** - * fscache_invalidate - Notify cache that an object needs invalidation - * @cookie: The cookie representing the cache object - * - * Notify the cache that an object is needs to be invalidated and that it - * should abort any retrievals or stores it is doing on the cache. The object - * is then marked non-caching until such time as the invalidation is complete. - * - * This can be called with spinlocks held. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_invalidate(struct fscache_cookie *cookie) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - __fscache_invalidate(cookie); -} - -/** - * fscache_wait_on_invalidate - Wait for invalidation to complete - * @cookie: The cookie representing the cache object - * - * Wait for the invalidation of an object to complete. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_wait_on_invalidate(struct fscache_cookie *cookie) -{ - if (fscache_cookie_valid(cookie)) - __fscache_wait_on_invalidate(cookie); -} - -/** - * fscache_reserve_space - Reserve data space for a cached object - * @cookie: The cookie representing the cache object - * @i_size: The amount of space to be reserved - * - * Reserve an amount of space in the cache for the cache object attached to a - * cookie so that a write to that object within the space can always be - * honoured. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_reserve_space(struct fscache_cookie *cookie, loff_t size) -{ - return -ENOBUFS; -} - -#ifdef FSCACHE_USE_NEW_IO_API - -/** - * fscache_begin_read_operation - Begin a read operation for the netfs lib - * @rreq: The read request being undertaken - * @cookie: The cookie representing the cache object - * - * Begin a read operation on behalf of the netfs helper library. @rreq - * indicates the read request to which the operation state should be attached; - * @cookie indicates the cache object that will be accessed. - * - * This is intended to be called from the ->begin_cache_operation() netfs lib - * operation as implemented by the network filesystem. - * - * Returns: - * * 0 - Success - * * -ENOBUFS - No caching available - * * Other error code from the cache, such as -ENOMEM. - */ -static inline -int fscache_begin_read_operation(struct netfs_read_request *rreq, - struct fscache_cookie *cookie) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_begin_read_operation(rreq, cookie); - return -ENOBUFS; -} - -#else /* FSCACHE_USE_NEW_IO_API */ - -/** - * fscache_read_or_alloc_page - Read a page from the cache or allocate a block - * in which to store it - * @cookie: The cookie representing the cache object - * @page: The netfs page to fill if possible - * @end_io_func: The callback to invoke when and if the page is filled - * @context: An arbitrary piece of data to pass on to end_io_func() - * @gfp: The conditions under which memory allocation should be made - * - * Read a page from the cache, or if that's not possible make a potential - * one-block reservation in the cache into which the page may be stored once - * fetched from the server. - * - * If the page is not backed by the cache object, or if it there's some reason - * it can't be, -ENOBUFS will be returned and nothing more will be done for - * that page. - * - * Else, if that page is backed by the cache, a read will be initiated directly - * to the netfs's page and 0 will be returned by this function. The - * end_io_func() callback will be invoked when the operation terminates on a - * completion or failure. Note that the callback may be invoked before the - * return. - * - * Else, if the page is unbacked, -ENODATA is returned and a block may have - * been allocated in the cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_read_or_alloc_page(struct fscache_cookie *cookie, - struct page *page, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_read_or_alloc_page(cookie, page, end_io_func, - context, gfp); - else - return -ENOBUFS; -} - -/** - * fscache_read_or_alloc_pages - Read pages from the cache and/or allocate - * blocks in which to store them - * @cookie: The cookie representing the cache object - * @mapping: The netfs inode mapping to which the pages will be attached - * @pages: A list of potential netfs pages to be filled - * @nr_pages: Number of pages to be read and/or allocated - * @end_io_func: The callback to invoke when and if each page is filled - * @context: An arbitrary piece of data to pass on to end_io_func() - * @gfp: The conditions under which memory allocation should be made - * - * Read a set of pages from the cache, or if that's not possible, attempt to - * make a potential one-block reservation for each page in the cache into which - * that page may be stored once fetched from the server. - * - * If some pages are not backed by the cache object, or if it there's some - * reason they can't be, -ENOBUFS will be returned and nothing more will be - * done for that pages. - * - * Else, if some of the pages are backed by the cache, a read will be initiated - * directly to the netfs's page and 0 will be returned by this function. The - * end_io_func() callback will be invoked when the operation terminates on a - * completion or failure. Note that the callback may be invoked before the - * return. - * - * Else, if a page is unbacked, -ENODATA is returned and a block may have - * been allocated in the cache. - * - * Because the function may want to return all of -ENOBUFS, -ENODATA and 0 in - * regard to different pages, the return values are prioritised in that order. - * Any pages submitted for reading are removed from the pages list. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_read_or_alloc_pages(struct fscache_cookie *cookie, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages, - fscache_rw_complete_t end_io_func, - void *context, - gfp_t gfp) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_read_or_alloc_pages(cookie, mapping, pages, - nr_pages, end_io_func, - context, gfp); - else - return -ENOBUFS; -} - -/** - * fscache_alloc_page - Allocate a block in which to store a page - * @cookie: The cookie representing the cache object - * @page: The netfs page to allocate a page for - * @gfp: The conditions under which memory allocation should be made - * - * Request Allocation a block in the cache in which to store a netfs page - * without retrieving any contents from the cache. - * - * If the page is not backed by a file then -ENOBUFS will be returned and - * nothing more will be done, and no reservation will be made. - * - * Else, a block will be allocated if one wasn't already, and 0 will be - * returned - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_alloc_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_alloc_page(cookie, page, gfp); - else - return -ENOBUFS; -} - -/** - * fscache_readpages_cancel - Cancel read/alloc on pages - * @cookie: The cookie representing the inode's cache object. - * @pages: The netfs pages that we canceled write on in readpages() - * - * Uncache/unreserve the pages reserved earlier in readpages() via - * fscache_readpages_or_alloc() and similar. In most successful caches in - * readpages() this doesn't do anything. In cases when the underlying netfs's - * readahead failed we need to clean up the pagelist (unmark and uncache). - * - * This function may sleep as it may have to clean up disk state. - */ -static inline -void fscache_readpages_cancel(struct fscache_cookie *cookie, - struct list_head *pages) -{ - if (fscache_cookie_valid(cookie)) - __fscache_readpages_cancel(cookie, pages); -} - -/** - * fscache_write_page - Request storage of a page in the cache - * @cookie: The cookie representing the cache object - * @page: The netfs page to store - * @object_size: Updated size of object - * @gfp: The conditions under which memory allocation should be made - * - * Request the contents of the netfs page be written into the cache. This - * request may be ignored if no cache block is currently allocated, in which - * case it will return -ENOBUFS. - * - * If a cache block was already allocated, a write will be initiated and 0 will - * be returned. The PG_fscache_write page bit is set immediately and will then - * be cleared at the completion of the write to indicate the success or failure - * of the operation. Note that the completion may happen before the return. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -int fscache_write_page(struct fscache_cookie *cookie, - struct page *page, - loff_t object_size, - gfp_t gfp) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - return __fscache_write_page(cookie, page, object_size, gfp); - else - return -ENOBUFS; -} - -/** - * fscache_uncache_page - Indicate that caching is no longer required on a page - * @cookie: The cookie representing the cache object - * @page: The netfs page that was being cached. - * - * Tell the cache that we no longer want a page to be cached and that it should - * remove any knowledge of the netfs page it may have. - * - * Note that this cannot cancel any outstanding I/O operations between this - * page and the cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_uncache_page(struct fscache_cookie *cookie, - struct page *page) -{ - if (fscache_cookie_valid(cookie)) - __fscache_uncache_page(cookie, page); -} - -/** - * fscache_check_page_write - Ask if a page is being writing to the cache - * @cookie: The cookie representing the cache object - * @page: The netfs page that is being cached. - * - * Ask the cache if a page is being written to the cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -bool fscache_check_page_write(struct fscache_cookie *cookie, - struct page *page) -{ - if (fscache_cookie_valid(cookie)) - return __fscache_check_page_write(cookie, page); - return false; -} - -/** - * fscache_wait_on_page_write - Wait for a page to complete writing to the cache - * @cookie: The cookie representing the cache object - * @page: The netfs page that is being cached. - * - * Ask the cache to wake us up when a page is no longer being written to the - * cache. - * - * See Documentation/filesystems/caching/netfs-api.rst for a complete - * description. - */ -static inline -void fscache_wait_on_page_write(struct fscache_cookie *cookie, - struct page *page) -{ - if (fscache_cookie_valid(cookie)) - __fscache_wait_on_page_write(cookie, page); -} - -/** - * fscache_maybe_release_page - Consider releasing a page, cancelling a store - * @cookie: The cookie representing the cache object - * @page: The netfs page that is being cached. - * @gfp: The gfp flags passed to releasepage() - * - * Consider releasing a page for the vmscan algorithm, on behalf of the netfs's - * releasepage() call. A storage request on the page may cancelled if it is - * not currently being processed. - * - * The function returns true if the page no longer has a storage request on it, - * and false if a storage request is left in place. If true is returned, the - * page will have been passed to fscache_uncache_page(). If false is returned - * the page cannot be freed yet. - */ -static inline -bool fscache_maybe_release_page(struct fscache_cookie *cookie, - struct page *page, - gfp_t gfp) -{ - if (fscache_cookie_valid(cookie) && PageFsCache(page)) - return __fscache_maybe_release_page(cookie, page, gfp); - return true; -} - -/** - * fscache_uncache_all_inode_pages - Uncache all an inode's pages - * @cookie: The cookie representing the inode's cache object. - * @inode: The inode to uncache pages from. - * - * Uncache all the pages in an inode that are marked PG_fscache, assuming them - * to be associated with the given cookie. - * - * This function may sleep. It will wait for pages that are being written out - * and will wait whilst the PG_fscache mark is removed by the cache. - */ -static inline -void fscache_uncache_all_inode_pages(struct fscache_cookie *cookie, - struct inode *inode) -{ - if (fscache_cookie_valid(cookie)) - __fscache_uncache_all_inode_pages(cookie, inode); -} - -#endif /* FSCACHE_USE_NEW_IO_API */ - -/** - * fscache_disable_cookie - Disable a cookie - * @cookie: The cookie representing the cache object - * @aux_data: The updated auxiliary data for the cookie (may be NULL) - * @invalidate: Invalidate the backing object - * - * Disable a cookie from accepting further alloc, read, write, invalidate, - * update or acquire operations. Outstanding operations can still be waited - * upon and pages can still be uncached and the cookie relinquished. - * - * This will not return until all outstanding operations have completed. - * - * If @invalidate is set, then the backing object will be invalidated and - * detached, otherwise it will just be detached. - * - * If @aux_data is set, then auxiliary data will be updated from that. - */ -static inline -void fscache_disable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - bool invalidate) -{ - if (fscache_cookie_valid(cookie) && fscache_cookie_enabled(cookie)) - __fscache_disable_cookie(cookie, aux_data, invalidate); -} - -/** - * fscache_enable_cookie - Reenable a cookie - * @cookie: The cookie representing the cache object - * @aux_data: The updated auxiliary data for the cookie (may be NULL) - * @object_size: Current size of object - * @can_enable: A function to permit enablement once lock is held - * @data: Data for can_enable() - * - * Reenable a previously disabled cookie, allowing it to accept further alloc, - * read, write, invalidate, update or acquire operations. An attempt will be - * made to immediately reattach the cookie to a backing object. If @aux_data - * is set, the auxiliary data attached to the cookie will be updated. - * - * The can_enable() function is called (if not NULL) once the enablement lock - * is held to rule on whether enablement is still permitted to go ahead. - */ -static inline -void fscache_enable_cookie(struct fscache_cookie *cookie, - const void *aux_data, - loff_t object_size, - bool (*can_enable)(void *data), - void *data) -{ - if (fscache_cookie_valid(cookie) && !fscache_cookie_enabled(cookie)) - __fscache_enable_cookie(cookie, aux_data, object_size, - can_enable, data); -} - #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h deleted file mode 100644 index 446392f5ba83..000000000000 --- a/include/trace/events/fscache.h +++ /dev/null @@ -1,523 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* FS-Cache tracepoints - * - * Copyright (C) 2016 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ -#undef TRACE_SYSTEM -#define TRACE_SYSTEM fscache - -#if !defined(_TRACE_FSCACHE_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_FSCACHE_H - -#include -#include - -/* - * Define enums for tracing information. - */ -#ifndef __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY -#define __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY - -enum fscache_cookie_trace { - fscache_cookie_collision, - fscache_cookie_discard, - fscache_cookie_get_acquire_parent, - fscache_cookie_get_attach_object, - fscache_cookie_get_reacquire, - fscache_cookie_get_register_netfs, - fscache_cookie_put_acquire_nobufs, - fscache_cookie_put_dup_netfs, - fscache_cookie_put_relinquish, - fscache_cookie_put_object, - fscache_cookie_put_parent, -}; - -enum fscache_page_trace { - fscache_page_cached, - fscache_page_inval, - fscache_page_maybe_release, - fscache_page_radix_clear_store, - fscache_page_radix_delete, - fscache_page_radix_insert, - fscache_page_radix_pend2store, - fscache_page_radix_set_pend, - fscache_page_uncache, - fscache_page_write, - fscache_page_write_end, - fscache_page_write_end_pend, - fscache_page_write_end_noc, - fscache_page_write_wait, - fscache_page_trace__nr -}; - -enum fscache_op_trace { - fscache_op_cancel, - fscache_op_cancel_all, - fscache_op_cancelled, - fscache_op_completed, - fscache_op_enqueue_async, - fscache_op_enqueue_mythread, - fscache_op_gc, - fscache_op_init, - fscache_op_put, - fscache_op_run, - fscache_op_signal, - fscache_op_submit, - fscache_op_submit_ex, - fscache_op_work, - fscache_op_trace__nr -}; - -enum fscache_page_op_trace { - fscache_page_op_alloc_one, - fscache_page_op_attr_changed, - fscache_page_op_check_consistency, - fscache_page_op_invalidate, - fscache_page_op_retr_multi, - fscache_page_op_retr_one, - fscache_page_op_write_one, - fscache_page_op_trace__nr -}; - -#endif - -/* - * Declare tracing information enums and their string mappings for display. - */ -#define fscache_cookie_traces \ - EM(fscache_cookie_collision, "*COLLISION*") \ - EM(fscache_cookie_discard, "DISCARD") \ - EM(fscache_cookie_get_acquire_parent, "GET prn") \ - EM(fscache_cookie_get_attach_object, "GET obj") \ - EM(fscache_cookie_get_reacquire, "GET raq") \ - EM(fscache_cookie_get_register_netfs, "GET net") \ - EM(fscache_cookie_put_acquire_nobufs, "PUT nbf") \ - EM(fscache_cookie_put_dup_netfs, "PUT dnt") \ - EM(fscache_cookie_put_relinquish, "PUT rlq") \ - EM(fscache_cookie_put_object, "PUT obj") \ - E_(fscache_cookie_put_parent, "PUT prn") - -#define fscache_page_traces \ - EM(fscache_page_cached, "Cached ") \ - EM(fscache_page_inval, "InvalPg") \ - EM(fscache_page_maybe_release, "MayRels") \ - EM(fscache_page_uncache, "Uncache") \ - EM(fscache_page_radix_clear_store, "RxCStr ") \ - EM(fscache_page_radix_delete, "RxDel ") \ - EM(fscache_page_radix_insert, "RxIns ") \ - EM(fscache_page_radix_pend2store, "RxP2S ") \ - EM(fscache_page_radix_set_pend, "RxSPend ") \ - EM(fscache_page_write, "WritePg") \ - EM(fscache_page_write_end, "EndPgWr") \ - EM(fscache_page_write_end_pend, "EndPgWP") \ - EM(fscache_page_write_end_noc, "EndPgNC") \ - E_(fscache_page_write_wait, "WtOnWrt") - -#define fscache_op_traces \ - EM(fscache_op_cancel, "Cancel1") \ - EM(fscache_op_cancel_all, "CancelA") \ - EM(fscache_op_cancelled, "Canclld") \ - EM(fscache_op_completed, "Complet") \ - EM(fscache_op_enqueue_async, "EnqAsyn") \ - EM(fscache_op_enqueue_mythread, "EnqMyTh") \ - EM(fscache_op_gc, "GC ") \ - EM(fscache_op_init, "Init ") \ - EM(fscache_op_put, "Put ") \ - EM(fscache_op_run, "Run ") \ - EM(fscache_op_signal, "Signal ") \ - EM(fscache_op_submit, "Submit ") \ - EM(fscache_op_submit_ex, "SubmitX") \ - E_(fscache_op_work, "Work ") - -#define fscache_page_op_traces \ - EM(fscache_page_op_alloc_one, "Alloc1 ") \ - EM(fscache_page_op_attr_changed, "AttrChg") \ - EM(fscache_page_op_check_consistency, "CheckCn") \ - EM(fscache_page_op_invalidate, "Inval ") \ - EM(fscache_page_op_retr_multi, "RetrMul") \ - EM(fscache_page_op_retr_one, "Retr1 ") \ - E_(fscache_page_op_write_one, "Write1 ") - -/* - * Export enum symbols via userspace. - */ -#undef EM -#undef E_ -#define EM(a, b) TRACE_DEFINE_ENUM(a); -#define E_(a, b) TRACE_DEFINE_ENUM(a); - -fscache_cookie_traces; - -/* - * Now redefine the EM() and E_() macros to map the enums to the strings that - * will be printed in the output. - */ -#undef EM -#undef E_ -#define EM(a, b) { a, b }, -#define E_(a, b) { a, b } - - -TRACE_EVENT(fscache_cookie, - TP_PROTO(unsigned int cookie_debug_id, - int ref, - enum fscache_cookie_trace where), - - TP_ARGS(cookie_debug_id, ref, where), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(enum fscache_cookie_trace, where ) - __field(int, ref ) - ), - - TP_fast_assign( - __entry->cookie = cookie_debug_id; - __entry->where = where; - __entry->ref = ref; - ), - - TP_printk("%s c=%08x r=%d", - __print_symbolic(__entry->where, fscache_cookie_traces), - __entry->cookie, __entry->ref) - ); - -TRACE_EVENT(fscache_netfs, - TP_PROTO(struct fscache_netfs *netfs), - - TP_ARGS(netfs), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __array(char, name, 8 ) - ), - - TP_fast_assign( - __entry->cookie = netfs->primary_index->debug_id; - strncpy(__entry->name, netfs->name, 8); - __entry->name[7] = 0; - ), - - TP_printk("c=%08x n=%s", - __entry->cookie, __entry->name) - ); - -TRACE_EVENT(fscache_acquire, - TP_PROTO(struct fscache_cookie *cookie), - - TP_ARGS(cookie), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, parent ) - __array(char, name, 8 ) - __field(int, p_ref ) - __field(int, p_n_children ) - __field(u8, p_flags ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->parent = cookie->parent->debug_id; - __entry->p_ref = refcount_read(&cookie->parent->ref); - __entry->p_n_children = atomic_read(&cookie->parent->n_children); - __entry->p_flags = cookie->parent->flags; - memcpy(__entry->name, cookie->def->name, 8); - __entry->name[7] = 0; - ), - - TP_printk("c=%08x p=%08x pr=%d pc=%d pf=%02x n=%s", - __entry->cookie, __entry->parent, __entry->p_ref, - __entry->p_n_children, __entry->p_flags, __entry->name) - ); - -TRACE_EVENT(fscache_relinquish, - TP_PROTO(struct fscache_cookie *cookie, bool retire), - - TP_ARGS(cookie, retire), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, parent ) - __field(int, ref ) - __field(int, n_children ) - __field(int, n_active ) - __field(u8, flags ) - __field(bool, retire ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->parent = cookie->parent->debug_id; - __entry->ref = refcount_read(&cookie->ref); - __entry->n_children = atomic_read(&cookie->n_children); - __entry->n_active = atomic_read(&cookie->n_active); - __entry->flags = cookie->flags; - __entry->retire = retire; - ), - - TP_printk("c=%08x r=%d p=%08x Nc=%d Na=%d f=%02x r=%u", - __entry->cookie, __entry->ref, - __entry->parent, __entry->n_children, __entry->n_active, - __entry->flags, __entry->retire) - ); - -TRACE_EVENT(fscache_enable, - TP_PROTO(struct fscache_cookie *cookie), - - TP_ARGS(cookie), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(int, ref ) - __field(int, n_children ) - __field(int, n_active ) - __field(u8, flags ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->ref = refcount_read(&cookie->ref); - __entry->n_children = atomic_read(&cookie->n_children); - __entry->n_active = atomic_read(&cookie->n_active); - __entry->flags = cookie->flags; - ), - - TP_printk("c=%08x r=%d Nc=%d Na=%d f=%02x", - __entry->cookie, __entry->ref, - __entry->n_children, __entry->n_active, __entry->flags) - ); - -TRACE_EVENT(fscache_disable, - TP_PROTO(struct fscache_cookie *cookie), - - TP_ARGS(cookie), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(int, ref ) - __field(int, n_children ) - __field(int, n_active ) - __field(u8, flags ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->ref = refcount_read(&cookie->ref); - __entry->n_children = atomic_read(&cookie->n_children); - __entry->n_active = atomic_read(&cookie->n_active); - __entry->flags = cookie->flags; - ), - - TP_printk("c=%08x r=%d Nc=%d Na=%d f=%02x", - __entry->cookie, __entry->ref, - __entry->n_children, __entry->n_active, __entry->flags) - ); - -TRACE_EVENT(fscache_osm, - TP_PROTO(struct fscache_object *object, - const struct fscache_state *state, - bool wait, bool oob, s8 event_num), - - TP_ARGS(object, state, wait, oob, event_num), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, object ) - __array(char, state, 8 ) - __field(bool, wait ) - __field(bool, oob ) - __field(s8, event_num ) - ), - - TP_fast_assign( - __entry->cookie = object->cookie->debug_id; - __entry->object = object->debug_id; - __entry->wait = wait; - __entry->oob = oob; - __entry->event_num = event_num; - memcpy(__entry->state, state->short_name, 8); - ), - - TP_printk("c=%08x o=%08d %s %s%sev=%d", - __entry->cookie, - __entry->object, - __entry->state, - __print_symbolic(__entry->wait, - { true, "WAIT" }, - { false, "WORK" }), - __print_symbolic(__entry->oob, - { true, " OOB " }, - { false, " " }), - __entry->event_num) - ); - -TRACE_EVENT(fscache_page, - TP_PROTO(struct fscache_cookie *cookie, struct page *page, - enum fscache_page_trace why), - - TP_ARGS(cookie, page, why), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(pgoff_t, page ) - __field(enum fscache_page_trace, why ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->page = page->index; - __entry->why = why; - ), - - TP_printk("c=%08x %s pg=%lx", - __entry->cookie, - __print_symbolic(__entry->why, fscache_page_traces), - __entry->page) - ); - -TRACE_EVENT(fscache_check_page, - TP_PROTO(struct fscache_cookie *cookie, struct page *page, - void *val, int n), - - TP_ARGS(cookie, page, val, n), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(void *, page ) - __field(void *, val ) - __field(int, n ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->page = page; - __entry->val = val; - __entry->n = n; - ), - - TP_printk("c=%08x pg=%p val=%p n=%d", - __entry->cookie, __entry->page, __entry->val, __entry->n) - ); - -TRACE_EVENT(fscache_wake_cookie, - TP_PROTO(struct fscache_cookie *cookie), - - TP_ARGS(cookie), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - ), - - TP_printk("c=%08x", __entry->cookie) - ); - -TRACE_EVENT(fscache_op, - TP_PROTO(struct fscache_cookie *cookie, struct fscache_operation *op, - enum fscache_op_trace why), - - TP_ARGS(cookie, op, why), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, op ) - __field(enum fscache_op_trace, why ) - ), - - TP_fast_assign( - __entry->cookie = cookie ? cookie->debug_id : 0; - __entry->op = op->debug_id; - __entry->why = why; - ), - - TP_printk("c=%08x op=%08x %s", - __entry->cookie, __entry->op, - __print_symbolic(__entry->why, fscache_op_traces)) - ); - -TRACE_EVENT(fscache_page_op, - TP_PROTO(struct fscache_cookie *cookie, struct page *page, - struct fscache_operation *op, enum fscache_page_op_trace what), - - TP_ARGS(cookie, page, op, what), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, op ) - __field(pgoff_t, page ) - __field(enum fscache_page_op_trace, what ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->page = page ? page->index : 0; - __entry->op = op->debug_id; - __entry->what = what; - ), - - TP_printk("c=%08x %s pg=%lx op=%08x", - __entry->cookie, - __print_symbolic(__entry->what, fscache_page_op_traces), - __entry->page, __entry->op) - ); - -TRACE_EVENT(fscache_wrote_page, - TP_PROTO(struct fscache_cookie *cookie, struct page *page, - struct fscache_operation *op, int ret), - - TP_ARGS(cookie, page, op, ret), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, op ) - __field(pgoff_t, page ) - __field(int, ret ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->page = page->index; - __entry->op = op->debug_id; - __entry->ret = ret; - ), - - TP_printk("c=%08x pg=%lx op=%08x ret=%d", - __entry->cookie, __entry->page, __entry->op, __entry->ret) - ); - -TRACE_EVENT(fscache_gang_lookup, - TP_PROTO(struct fscache_cookie *cookie, struct fscache_operation *op, - void **results, int n, pgoff_t store_limit), - - TP_ARGS(cookie, op, results, n, store_limit), - - TP_STRUCT__entry( - __field(unsigned int, cookie ) - __field(unsigned int, op ) - __field(pgoff_t, results0 ) - __field(int, n ) - __field(pgoff_t, store_limit ) - ), - - TP_fast_assign( - __entry->cookie = cookie->debug_id; - __entry->op = op->debug_id; - __entry->results0 = results[0] ? ((struct page *)results[0])->index : (pgoff_t)-1; - __entry->n = n; - __entry->store_limit = store_limit; - ), - - TP_printk("c=%08x op=%08x r0=%lx n=%d sl=%lx", - __entry->cookie, __entry->op, __entry->results0, __entry->n, - __entry->store_limit) - ); - -#endif /* _TRACE_FSCACHE_H */ - -/* This part must be outside protection */ -#include -- cgit v1.2.3-59-g8ed1b From a39c41b853ee51f4dcd19f5556f860ae8e2f23d3 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:30:37 +0100 Subject: netfs: Pass a flag to ->prepare_write() to say if there's no alloc'd space Pass a flag to ->prepare_write() to indicate if there's definitely no space allocated in the cache yet (for instance if we've already checked as we were asked to do a read). Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819583123.215744.12783808230464471417.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906886835.143852.6689886781122679769.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967079100.1823006.12889542712309574359.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021489334.640689.3131206613015409076.stgit@warthog.procyon.org.uk/ # v4 --- fs/netfs/read_helper.c | 2 +- include/linux/netfs.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/netfs/read_helper.c b/fs/netfs/read_helper.c index 75c76cbb27cc..9dd76b8914f2 100644 --- a/fs/netfs/read_helper.c +++ b/fs/netfs/read_helper.c @@ -323,7 +323,7 @@ static void netfs_rreq_do_write_to_cache(struct netfs_read_request *rreq) } ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len, - rreq->i_size); + rreq->i_size, true); if (ret < 0) { trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write); trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index ca0683b9e3d1..1ea22fc48818 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -232,7 +232,8 @@ struct netfs_cache_ops { * actually do. */ int (*prepare_write)(struct netfs_cache_resources *cres, - loff_t *_start, size_t *_len, loff_t i_size); + loff_t *_start, size_t *_len, loff_t i_size, + bool no_space_allocated_yet); }; struct readahead_control; -- cgit v1.2.3-59-g8ed1b From 1e1236b841166f1d2daf36fdf6bb3e656bc5f5ca Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:34:41 +0100 Subject: fscache: Introduce new driver Introduce basic skeleton of the new, rewritten fscache driver. Changes ======= ver #3: - Use remove_proc_subtree(), not remove_proc_entry() to remove a populated dir. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819584034.215744.4290533472390439030.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906887770.143852.3577888294989185666.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967080039.1823006.5702921801104057922.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021491014.640689.4292699878317589512.stgit@warthog.procyon.org.uk/ # v4 --- fs/Makefile | 1 + fs/fscache/Kconfig | 39 +++++++++ fs/fscache/Makefile | 12 +++ fs/fscache/internal.h | 183 +++++++++++++++++++++++++++++++++++++++++ fs/fscache/main.c | 65 +++++++++++++++ fs/fscache/proc.c | 42 ++++++++++ fs/fscache/stats.c | 22 +++++ include/linux/fscache-cache.h | 2 + include/linux/fscache.h | 6 +- include/trace/events/fscache.h | 49 +++++++++++ 10 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 fs/fscache/Makefile create mode 100644 fs/fscache/internal.h create mode 100644 fs/fscache/main.c create mode 100644 fs/fscache/proc.c create mode 100644 fs/fscache/stats.c create mode 100644 include/trace/events/fscache.h (limited to 'include/linux') diff --git a/fs/Makefile b/fs/Makefile index 23ddd0803d14..290815f3fd31 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -67,6 +67,7 @@ obj-$(CONFIG_DLM) += dlm/ # Do not add any filesystems before this line obj-$(CONFIG_NETFS_SUPPORT) += netfs/ +obj-$(CONFIG_FSCACHE) += fscache/ obj-$(CONFIG_REISERFS_FS) += reiserfs/ obj-$(CONFIG_EXT4_FS) += ext4/ # We place ext4 before ext2 so that clean ext3 root fs's do NOT mount using the diff --git a/fs/fscache/Kconfig b/fs/fscache/Kconfig index 6440484d9461..76316c4a3fb7 100644 --- a/fs/fscache/Kconfig +++ b/fs/fscache/Kconfig @@ -1,4 +1,43 @@ # SPDX-License-Identifier: GPL-2.0-only +config FSCACHE + tristate "General filesystem local caching manager" + select NETFS_SUPPORT + help + This option enables a generic filesystem caching manager that can be + used by various network and other filesystems to cache data locally. + Different sorts of caches can be plugged in, depending on the + resources available. + + See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_STATS + bool "Gather statistical information on local caching" + depends on FSCACHE && PROC_FS + select NETFS_STATS + help + This option causes statistical information to be gathered on local + caching and exported through file: + + /proc/fs/fscache/stats + + The gathering of statistics adds a certain amount of overhead to + execution as there are a quite a few stats gathered, and on a + multi-CPU system these may be on cachelines that keep bouncing + between CPUs. On the other hand, the stats are very useful for + debugging purposes. Saying 'Y' here is recommended. + + See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_DEBUG + bool "Debug FS-Cache" + depends on FSCACHE + help + This permits debugging to be dynamically enabled in the local caching + management module. If this is set, the debugging output may be + enabled by setting bits in /sys/modules/fscache/parameter/debug. + + See Documentation/filesystems/caching/fscache.rst for more information. + config FSCACHE_OLD_API bool diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile new file mode 100644 index 000000000000..f9722de32247 --- /dev/null +++ b/fs/fscache/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for general filesystem caching code +# + +fscache-y := \ + main.o + +fscache-$(CONFIG_PROC_FS) += proc.o +fscache-$(CONFIG_FSCACHE_STATS) += stats.o + +obj-$(CONFIG_FSCACHE) := fscache.o diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h new file mode 100644 index 000000000000..ea52f8594a77 --- /dev/null +++ b/fs/fscache/internal.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Internal definitions for FS-Cache + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "FS-Cache: " fmt + +#include +#include +#include +#include +#include + +/* + * main.c + */ +extern unsigned fscache_debug; + +/* + * proc.c + */ +#ifdef CONFIG_PROC_FS +extern int __init fscache_proc_init(void); +extern void fscache_proc_cleanup(void); +#else +#define fscache_proc_init() (0) +#define fscache_proc_cleanup() do {} while (0) +#endif + +/* + * stats.c + */ +#ifdef CONFIG_FSCACHE_STATS + +static inline void fscache_stat(atomic_t *stat) +{ + atomic_inc(stat); +} + +static inline void fscache_stat_d(atomic_t *stat) +{ + atomic_dec(stat); +} + +#define __fscache_stat(stat) (stat) + +int fscache_stats_show(struct seq_file *m, void *v); +#else + +#define __fscache_stat(stat) (NULL) +#define fscache_stat(stat) do {} while (0) +#define fscache_stat_d(stat) do {} while (0) +#endif + + +/*****************************************************************************/ +/* + * debug tracing + */ +#define dbgprintk(FMT, ...) \ + printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) + +#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) + +#define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) + +#ifdef __KDEBUG +#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) +#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) +#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) + +#elif defined(CONFIG_FSCACHE_DEBUG) +#define _enter(FMT, ...) \ +do { \ + if (__do_kdebug(ENTER)) \ + kenter(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _leave(FMT, ...) \ +do { \ + if (__do_kdebug(LEAVE)) \ + kleave(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _debug(FMT, ...) \ +do { \ + if (__do_kdebug(DEBUG)) \ + kdebug(FMT, ##__VA_ARGS__); \ +} while (0) + +#else +#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) +#endif + +/* + * determine whether a particular optional debugging point should be logged + * - we need to go through three steps to persuade cpp to correctly join the + * shorthand in FSCACHE_DEBUG_LEVEL with its prefix + */ +#define ____do_kdebug(LEVEL, POINT) \ + unlikely((fscache_debug & \ + (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) +#define ___do_kdebug(LEVEL, POINT) \ + ____do_kdebug(LEVEL, POINT) +#define __do_kdebug(POINT) \ + ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) + +#define FSCACHE_DEBUG_CACHE 0 +#define FSCACHE_DEBUG_COOKIE 1 +#define FSCACHE_DEBUG_OBJECT 2 +#define FSCACHE_DEBUG_OPERATION 3 + +#define FSCACHE_POINT_ENTER 1 +#define FSCACHE_POINT_LEAVE 2 +#define FSCACHE_POINT_DEBUG 4 + +#ifndef FSCACHE_DEBUG_LEVEL +#define FSCACHE_DEBUG_LEVEL CACHE +#endif + +/* + * assertions + */ +#if 1 /* defined(__KDEBUGALL) */ + +#define ASSERT(X) \ +do { \ + if (unlikely(!(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ + if (unlikely(!((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIF(C, X) \ +do { \ + if (unlikely((C) && !(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ + if (unlikely((C) && !((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#else + +#define ASSERT(X) do {} while (0) +#define ASSERTCMP(X, OP, Y) do {} while (0) +#define ASSERTIF(C, X) do {} while (0) +#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) + +#endif /* assert or not */ diff --git a/fs/fscache/main.c b/fs/fscache/main.c new file mode 100644 index 000000000000..819de2ee1276 --- /dev/null +++ b/fs/fscache/main.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* General filesystem local caching manager + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#define CREATE_TRACE_POINTS +#include "internal.h" + +MODULE_DESCRIPTION("FS Cache Manager"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +unsigned fscache_debug; +module_param_named(debug, fscache_debug, uint, + S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(fscache_debug, + "FS-Cache debugging mask"); + +struct workqueue_struct *fscache_wq; +EXPORT_SYMBOL(fscache_wq); + +/* + * initialise the fs caching module + */ +static int __init fscache_init(void) +{ + int ret = -ENOMEM; + + fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!fscache_wq) + goto error_wq; + + ret = fscache_proc_init(); + if (ret < 0) + goto error_proc; + + pr_notice("Loaded\n"); + return 0; + +error_proc: + destroy_workqueue(fscache_wq); +error_wq: + return ret; +} + +fs_initcall(fscache_init); + +/* + * clean up on module removal + */ +static void __exit fscache_exit(void) +{ + _enter(""); + + fscache_proc_cleanup(); + destroy_workqueue(fscache_wq); + pr_notice("Unloaded\n"); +} + +module_exit(fscache_exit); diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c new file mode 100644 index 000000000000..4d866ac41776 --- /dev/null +++ b/fs/fscache/proc.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics viewing interface + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include +#include "internal.h" + +/* + * initialise the /proc/fs/fscache/ directory + */ +int __init fscache_proc_init(void) +{ + if (!proc_mkdir("fs/fscache", NULL)) + goto error_dir; + +#ifdef CONFIG_FSCACHE_STATS + if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, + fscache_stats_show)) + goto error; +#endif + + return 0; + +error: + remove_proc_entry("fs/fscache", NULL); +error_dir: + return -ENOMEM; +} + +/* + * clean up the /proc/fs/fscache/ directory + */ +void fscache_proc_cleanup(void) +{ + remove_proc_subtree("fs/fscache", NULL); +} diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c new file mode 100644 index 000000000000..bd92f93e1680 --- /dev/null +++ b/fs/fscache/stats.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +/* + * display the general statistics + */ +int fscache_stats_show(struct seq_file *m, void *v) +{ + seq_puts(m, "FS-Cache statistics\n"); + + netfs_stats_show(m); + return 0; +} diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 47f21a53ac4b..d6910a913918 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -16,4 +16,6 @@ #include +extern struct workqueue_struct *fscache_wq; + #endif /* _LINUX_FSCACHE_CACHE_H */ diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 0364a4ca16f6..1cf90c252aac 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* General filesystem caching interface * - * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) * * NOTE!!! See: @@ -18,9 +18,13 @@ #include #if defined(CONFIG_FSCACHE) || defined(CONFIG_FSCACHE_MODULE) +#define __fscache_available (1) +#define fscache_available() (1) #define fscache_cookie_valid(cookie) (cookie) #define fscache_cookie_enabled(cookie) (cookie) #else +#define __fscache_available (0) +#define fscache_available() (0) #define fscache_cookie_valid(cookie) (0) #define fscache_cookie_enabled(cookie) (0) #endif diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h new file mode 100644 index 000000000000..fe214c5cc87f --- /dev/null +++ b/include/trace/events/fscache.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* FS-Cache tracepoints + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM fscache + +#if !defined(_TRACE_FSCACHE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_FSCACHE_H + +#include +#include + +/* + * Define enums for tracing information. + */ +#ifndef __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY +#define __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY + +#endif + +/* + * Declare tracing information enums and their string mappings for display. + */ + +/* + * Export enum symbols via userspace. + */ +#undef EM +#undef E_ +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define E_(a, b) TRACE_DEFINE_ENUM(a); + +/* + * Now redefine the EM() and E_() macros to map the enums to the strings that + * will be printed in the output. + */ +#undef EM +#undef E_ +#define EM(a, b) { a, b }, +#define E_(a, b) { a, b } + + +#endif /* _TRACE_FSCACHE_H */ + +/* This part must be outside protection */ +#include -- cgit v1.2.3-59-g8ed1b From 9549332df4ed4e761a1d41c83f2c25d28bb22431 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:00:26 +0100 Subject: fscache: Implement cache registration Implement a register of caches and provide functions to manage it. Two functions are provided for the cache backend to use: (1) Acquire a cache cookie: struct fscache_cache *fscache_acquire_cache(const char *name) This gets the cache cookie for a cache of the specified name and moves it to the preparation state. If a nameless cache cookie exists, that will be given this name and used. (2) Relinquish a cache cookie: void fscache_relinquish_cache(struct fscache_cache *cache); This relinquishes a cache cookie, cleans it and makes it available if it's still referenced by a network filesystem. Note that network filesystems don't deal with cache cookies directly, but rather go straight to the volume registration. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819587157.215744.13523139317322503286.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906889665.143852.10378009165231294456.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967085081.1823006.2218944206363626210.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021494847.640689.10109692261640524343.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/Makefile | 1 + fs/fscache/cache.c | 274 +++++++++++++++++++++++++++++++++++++++++ fs/fscache/internal.h | 33 +++++ fs/fscache/proc.c | 4 + include/linux/fscache-cache.h | 34 +++++ include/trace/events/fscache.h | 43 +++++++ 6 files changed, 389 insertions(+) create mode 100644 fs/fscache/cache.c (limited to 'include/linux') diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index f9722de32247..d9fc22c18090 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -4,6 +4,7 @@ # fscache-y := \ + cache.o \ main.o fscache-$(CONFIG_PROC_FS) += proc.o diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c new file mode 100644 index 000000000000..8db77bb9f8e2 --- /dev/null +++ b/fs/fscache/cache.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache cache handling + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +static LIST_HEAD(fscache_caches); +DECLARE_RWSEM(fscache_addremove_sem); +EXPORT_SYMBOL(fscache_addremove_sem); + +static atomic_t fscache_cache_debug_id; + +/* + * Allocate a cache cookie. + */ +static struct fscache_cache *fscache_alloc_cache(const char *name) +{ + struct fscache_cache *cache; + + cache = kzalloc(sizeof(*cache), GFP_KERNEL); + if (cache) { + if (name) { + cache->name = kstrdup(name, GFP_KERNEL); + if (!cache->name) { + kfree(cache); + return NULL; + } + } + refcount_set(&cache->ref, 1); + INIT_LIST_HEAD(&cache->cache_link); + cache->debug_id = atomic_inc_return(&fscache_cache_debug_id); + } + return cache; +} + +static bool fscache_get_cache_maybe(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + bool success; + int ref; + + success = __refcount_inc_not_zero(&cache->ref, &ref); + if (success) + trace_fscache_cache(cache->debug_id, ref + 1, where); + return success; +} + +/* + * Look up a cache cookie. + */ +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache) +{ + struct fscache_cache *candidate, *cache, *unnamed = NULL; + + /* firstly check for the existence of the cache under read lock */ + down_read(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + if (!cache->name && !name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + } + + up_read(&fscache_addremove_sem); + + /* the cache does not exist - create a candidate */ + candidate = fscache_alloc_cache(name); + if (!candidate) + return ERR_PTR(-ENOMEM); + + /* write lock, search again and add if still not present */ + down_write(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + if (!cache->name) { + unnamed = cache; + if (!name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + if (unnamed && is_cache && + fscache_get_cache_maybe(unnamed, fscache_cache_get_acquire)) + goto use_unnamed_cache; + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + list_add_tail(&candidate->cache_link, &fscache_caches); + trace_fscache_cache(candidate->debug_id, + refcount_read(&candidate->ref), + fscache_cache_new_acquire); + up_write(&fscache_addremove_sem); + return candidate; + +got_cache_r: + up_read(&fscache_addremove_sem); + return cache; +use_unnamed_cache: + cache = unnamed; + cache->name = candidate->name; + candidate->name = NULL; +got_cache_w: + up_write(&fscache_addremove_sem); + kfree(candidate->name); + kfree(candidate); + return cache; +} + +/** + * fscache_acquire_cache - Acquire a cache-level cookie. + * @name: The name of the cache. + * + * Get a cookie to represent an actual cache. If a name is given and there is + * a nameless cache record available, this will acquire that and set its name, + * directing all the volumes using it to this cache. + * + * The cache will be switched over to the preparing state if not currently in + * use, otherwise -EBUSY will be returned. + */ +struct fscache_cache *fscache_acquire_cache(const char *name) +{ + struct fscache_cache *cache; + + ASSERT(name); + cache = fscache_lookup_cache(name, true); + if (IS_ERR(cache)) + return cache; + + if (!fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_NOT_PRESENT, + FSCACHE_CACHE_IS_PREPARING)) { + pr_warn("Cache tag %s in use\n", name); + fscache_put_cache(cache, fscache_cache_put_cache); + return ERR_PTR(-EBUSY); + } + + return cache; +} +EXPORT_SYMBOL(fscache_acquire_cache); + +/** + * fscache_put_cache - Release a cache-level cookie. + * @cache: The cache cookie to be released + * @where: An indication of where the release happened + * + * Release the caller's reference on a cache-level cookie. The @where + * indication should give information about the circumstances in which the call + * occurs and will be logged through a tracepoint. + */ +void fscache_put_cache(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + unsigned int debug_id = cache->debug_id; + bool zero; + int ref; + + if (IS_ERR_OR_NULL(cache)) + return; + + zero = __refcount_dec_and_test(&cache->ref, &ref); + trace_fscache_cache(debug_id, ref - 1, where); + + if (zero) { + down_write(&fscache_addremove_sem); + list_del_init(&cache->cache_link); + up_write(&fscache_addremove_sem); + kfree(cache->name); + kfree(cache); + } +} + +/** + * fscache_relinquish_cache - Reset cache state and release cookie + * @cache: The cache cookie to be released + * + * Reset the state of a cache and release the caller's reference on a cache + * cookie. + */ +void fscache_relinquish_cache(struct fscache_cache *cache) +{ + enum fscache_cache_trace where = + (cache->state == FSCACHE_CACHE_IS_PREPARING) ? + fscache_cache_put_prep_failed : + fscache_cache_put_relinquish; + + cache->cache_priv = NULL; + smp_store_release(&cache->state, FSCACHE_CACHE_IS_NOT_PRESENT); + fscache_put_cache(cache, where); +} +EXPORT_SYMBOL(fscache_relinquish_cache); + +#ifdef CONFIG_PROC_FS +static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] = "-PAEW"; + +/* + * Generate a list of caches in /proc/fs/fscache/caches + */ +static int fscache_caches_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cache *cache; + + if (v == &fscache_caches) { + seq_puts(m, + "CACHE REF VOLS OBJS ACCES S NAME\n" + "======== ===== ===== ===== ===== = ===============\n" + ); + return 0; + } + + cache = list_entry(v, struct fscache_cache, cache_link); + seq_printf(m, + "%08x %5d %5d %5d %5d %c %s\n", + cache->debug_id, + refcount_read(&cache->ref), + atomic_read(&cache->n_volumes), + atomic_read(&cache->object_count), + atomic_read(&cache->n_accesses), + fscache_cache_states[cache->state], + cache->name ?: "-"); + return 0; +} + +static void *fscache_caches_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_caches, *_pos); +} + +static void *fscache_caches_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_caches, _pos); +} + +static void fscache_caches_seq_stop(struct seq_file *m, void *v) + __releases(fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_caches_seq_ops = { + .start = fscache_caches_seq_start, + .next = fscache_caches_seq_next, + .stop = fscache_caches_seq_stop, + .show = fscache_caches_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index f345bdb018ba..8fd39e7735fc 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -17,6 +17,39 @@ #include #include +/* + * cache.c + */ +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_caches_seq_ops; +#endif +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); +void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); + +static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) +{ + return smp_load_acquire(&cache->state); +} + +static inline bool fscache_cache_is_live(const struct fscache_cache *cache) +{ + return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; +} + +static inline void fscache_set_cache_state(struct fscache_cache *cache, + enum fscache_cache_state new_state) +{ + smp_store_release(&cache->state, new_state); + +} + +static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, + enum fscache_cache_state old_state, + enum fscache_cache_state new_state) +{ + return try_cmpxchg_release(&cache->state, &old_state, new_state); +} + /* * main.c */ diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c index 4d866ac41776..93b925709e09 100644 --- a/fs/fscache/proc.c +++ b/fs/fscache/proc.c @@ -19,6 +19,10 @@ int __init fscache_proc_init(void) if (!proc_mkdir("fs/fscache", NULL)) goto error_dir; + if (!proc_create_seq("fs/fscache/caches", S_IFREG | 0444, NULL, + &fscache_caches_seq_ops)) + goto error; + #ifdef CONFIG_FSCACHE_STATS if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, fscache_stats_show)) diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index d6910a913918..18cd5c9877bb 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -16,6 +16,40 @@ #include +enum fscache_cache_trace; +enum fscache_access_trace; + +enum fscache_cache_state { + FSCACHE_CACHE_IS_NOT_PRESENT, /* No cache is present for this name */ + FSCACHE_CACHE_IS_PREPARING, /* A cache is preparing to come live */ + FSCACHE_CACHE_IS_ACTIVE, /* Attached cache is active and can be used */ + FSCACHE_CACHE_GOT_IOERROR, /* Attached cache stopped on I/O error */ + FSCACHE_CACHE_IS_WITHDRAWN, /* Attached cache is being withdrawn */ +#define NR__FSCACHE_CACHE_STATE (FSCACHE_CACHE_IS_WITHDRAWN + 1) +}; + +/* + * Cache cookie. + */ +struct fscache_cache { + struct list_head cache_link; /* Link in cache list */ + void *cache_priv; /* Private cache data (or NULL) */ + refcount_t ref; + atomic_t n_volumes; /* Number of active volumes; */ + atomic_t n_accesses; /* Number of in-progress accesses on the cache */ + atomic_t object_count; /* no. of live objects in this cache */ + unsigned int debug_id; + enum fscache_cache_state state; + char *name; +}; + extern struct workqueue_struct *fscache_wq; +/* + * out-of-line cache backend functions + */ +extern struct rw_semaphore fscache_addremove_sem; +extern struct fscache_cache *fscache_acquire_cache(const char *name); +extern void fscache_relinquish_cache(struct fscache_cache *cache); + #endif /* _LINUX_FSCACHE_CACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index fe214c5cc87f..3b8e0597b2c1 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -19,11 +19,27 @@ #ifndef __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY #define __FSCACHE_DECLARE_TRACE_ENUMS_ONCE_ONLY +enum fscache_cache_trace { + fscache_cache_collision, + fscache_cache_get_acquire, + fscache_cache_new_acquire, + fscache_cache_put_cache, + fscache_cache_put_prep_failed, + fscache_cache_put_relinquish, +}; + #endif /* * Declare tracing information enums and their string mappings for display. */ +#define fscache_cache_traces \ + EM(fscache_cache_collision, "*COLLIDE*") \ + EM(fscache_cache_get_acquire, "GET acq ") \ + EM(fscache_cache_new_acquire, "NEW acq ") \ + EM(fscache_cache_put_cache, "PUT cache") \ + EM(fscache_cache_put_prep_failed, "PUT pfail") \ + E_(fscache_cache_put_relinquish, "PUT relnq") /* * Export enum symbols via userspace. @@ -33,6 +49,8 @@ #define EM(a, b) TRACE_DEFINE_ENUM(a); #define E_(a, b) TRACE_DEFINE_ENUM(a); +fscache_cache_traces; + /* * Now redefine the EM() and E_() macros to map the enums to the strings that * will be printed in the output. @@ -43,6 +61,31 @@ #define E_(a, b) { a, b } +TRACE_EVENT(fscache_cache, + TP_PROTO(unsigned int cache_debug_id, + int usage, + enum fscache_cache_trace where), + + TP_ARGS(cache_debug_id, usage, where), + + TP_STRUCT__entry( + __field(unsigned int, cache ) + __field(int, usage ) + __field(enum fscache_cache_trace, where ) + ), + + TP_fast_assign( + __entry->cache = cache_debug_id; + __entry->usage = usage; + __entry->where = where; + ), + + TP_printk("C=%08x %s r=%d", + __entry->cache, + __print_symbolic(__entry->where, fscache_cache_traces), + __entry->usage) + ); + #endif /* _TRACE_FSCACHE_H */ /* This part must be outside protection */ -- cgit v1.2.3-59-g8ed1b From 62ab63352350e881ae693a8236b35d7d0516c78b Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:26:17 +0100 Subject: fscache: Implement volume registration Add functions to the fscache API to allow volumes to be acquired and relinquished by the network filesystem. A volume is an index of data storage cache objects. A volume is represented by a volume cookie in the API. A filesystem would typically create a volume for a superblock and then create per-inode cookies within it. To request a volume, the filesystem calls: struct fscache_volume * fscache_acquire_volume(const char *volume_key, const char *cache_name, const void *coherency_data, size_t coherency_len) The volume_key is a printable string used to match the volume in the cache. It should not contain any '/' characters. For AFS, for example, this would be "afs,,", e.g. "afs,example.com,523001". The cache_name can be NULL, but if not it should be a string indicating the name of the cache to use if there's more than one available. The coherency data, if given, is an arbitrarily-sized blob that's attached to the volume and is compared when the volume is looked up. If it doesn't match, the old volume is judged to be out of date and it and everything within it is discarded. Acquiring a volume twice concurrently is disallowed, though the function will wait if an old volume cookie is being relinquishing. When a network filesystem has finished with a volume, it should return the volume cookie by calling: void fscache_relinquish_volume(struct fscache_volume *volume, const void *coherency_data, bool invalidate) If invalidate is true, the entire volume will be discarded; if false, the volume will be synced and the coherency data will be updated. Changes ======= ver #4: - Removed an extraneous param from kdoc on fscache_relinquish_volume()[3]. ver #3: - fscache_hash()'s size parameter is now in bytes. Use __le32 as the unit to round up to. - When comparing cookies, simply see if the attributes are the same rather than subtracting them to produce a strcmp-style return[2]. - Make the coherency data an arbitrary blob rather than a u64, but don't store it for the moment. ver #2: - Fix error check[1]. - Make a fscache_acquire_volume() return errors, including EBUSY if a conflicting volume cookie already exists. No error is printed now - that's left to the netfs. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/20211203095608.GC2480@kili/ [1] Link: https://lore.kernel.org/r/CAHk-=whtkzB446+hX0zdLsdcUJsJ=8_-0S1mE_R+YurThfUbLA@mail.gmail.com/ [2] Link: https://lore.kernel.org/r/20211220224646.30e8205c@canb.auug.org.au/ [3] Link: https://lore.kernel.org/r/163819588944.215744.1629085755564865996.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906890630.143852.13972180614535611154.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967086836.1823006.8191672796841981763.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021495816.640689.4403156093668590217.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/Makefile | 3 +- fs/fscache/internal.h | 14 ++ fs/fscache/proc.c | 4 + fs/fscache/stats.c | 12 ++ fs/fscache/volume.c | 340 +++++++++++++++++++++++++++++++++++++++++ include/linux/fscache.h | 84 ++++++++++ include/trace/events/fscache.h | 61 +++++++- 7 files changed, 516 insertions(+), 2 deletions(-) create mode 100644 fs/fscache/volume.c (limited to 'include/linux') diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index d9fc22c18090..bb5282ae682f 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -5,7 +5,8 @@ fscache-y := \ cache.o \ - main.o + main.o \ + volume.o fscache-$(CONFIG_PROC_FS) += proc.o fscache-$(CONFIG_FSCACHE_STATS) += stats.o diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 8fd39e7735fc..07dc9cbc2280 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -72,6 +72,9 @@ extern void fscache_proc_cleanup(void); * stats.c */ #ifdef CONFIG_FSCACHE_STATS +extern atomic_t fscache_n_volumes; +extern atomic_t fscache_n_volumes_collision; +extern atomic_t fscache_n_volumes_nomem; static inline void fscache_stat(atomic_t *stat) { @@ -93,6 +96,17 @@ int fscache_stats_show(struct seq_file *m, void *v); #define fscache_stat_d(stat) do {} while (0) #endif +/* + * volume.c + */ +extern const struct seq_operations fscache_volumes_seq_ops; + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +void fscache_create_volume(struct fscache_volume *volume, bool wait); + /*****************************************************************************/ /* diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c index 93b925709e09..bc6ecbdd065d 100644 --- a/fs/fscache/proc.c +++ b/fs/fscache/proc.c @@ -23,6 +23,10 @@ int __init fscache_proc_init(void) &fscache_caches_seq_ops)) goto error; + if (!proc_create_seq("fs/fscache/volumes", S_IFREG | 0444, NULL, + &fscache_volumes_seq_ops)) + goto error; + #ifdef CONFIG_FSCACHE_STATS if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, fscache_stats_show)) diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index bd92f93e1680..b811a4d03585 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -10,12 +10,24 @@ #include #include "internal.h" +/* + * operation counters + */ +atomic_t fscache_n_volumes; +atomic_t fscache_n_volumes_collision; +atomic_t fscache_n_volumes_nomem; + /* * display the general statistics */ int fscache_stats_show(struct seq_file *m, void *v) { seq_puts(m, "FS-Cache statistics\n"); + seq_printf(m, "Cookies: v=%d vcol=%u voom=%u\n", + atomic_read(&fscache_n_volumes), + atomic_read(&fscache_n_volumes_collision), + atomic_read(&fscache_n_volumes_nomem) + ); netfs_stats_show(m); return 0; diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c new file mode 100644 index 000000000000..630894fefd02 --- /dev/null +++ b/fs/fscache/volume.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume-level cache cookie handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +#define fscache_volume_hash_shift 10 +static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; +static atomic_t fscache_volume_debug_id; +static LIST_HEAD(fscache_volumes); + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref; + + __refcount_inc(&volume->ref, &ref); + trace_fscache_volume(volume->debug_id, ref + 1, where); + return volume; +} + +static void fscache_see_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref = refcount_read(&volume->ref); + + trace_fscache_volume(volume->debug_id, ref, where); +} + +static bool fscache_volume_same(const struct fscache_volume *a, + const struct fscache_volume *b) +{ + size_t klen; + + if (a->key_hash != b->key_hash || + a->cache != b->cache || + a->key[0] != b->key[0]) + return false; + + klen = round_up(a->key[0] + 1, sizeof(__le32)); + return memcmp(a->key, b->key, klen) == 0; +} + +static bool fscache_is_acquire_pending(struct fscache_volume *volume) +{ + return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); +} + +static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, + unsigned int collidee_debug_id) +{ + wait_var_event_timeout(&candidate->flags, + fscache_is_acquire_pending(candidate), 20 * HZ); + if (!fscache_is_acquire_pending(candidate)) { + pr_notice("Potential volume collision new=%08x old=%08x", + candidate->debug_id, collidee_debug_id); + fscache_stat(&fscache_n_volumes_collision); + wait_var_event(&candidate->flags, fscache_is_acquire_pending(candidate)); + } +} + +/* + * Attempt to insert the new volume into the hash. If there's a collision, we + * wait for the old volume to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_volume(struct fscache_volume *candidate) +{ + struct fscache_volume *cursor; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket, collidee_debug_id = 0; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(candidate, cursor)) { + if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) + goto collision; + fscache_see_volume(cursor, fscache_volume_get_hash_collision); + set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); + set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); + collidee_debug_id = cursor->debug_id; + break; + } + } + + hlist_bl_add_head(&candidate->hash_link, h); + hlist_bl_unlock(h); + + if (test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags)) + fscache_wait_on_volume_collision(candidate, collidee_debug_id); + return true; + +collision: + fscache_see_volume(cursor, fscache_volume_collision); + hlist_bl_unlock(h); + return false; +} + +/* + * Allocate and initialise a volume representation cookie. + */ +static struct fscache_volume *fscache_alloc_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + struct fscache_cache *cache; + size_t klen, hlen; + char *key; + + cache = fscache_lookup_cache(cache_name, false); + if (IS_ERR(cache)) + return NULL; + + volume = kzalloc(sizeof(*volume), GFP_KERNEL); + if (!volume) + goto err_cache; + + volume->cache = cache; + INIT_LIST_HEAD(&volume->proc_link); + INIT_WORK(&volume->work, NULL /* PLACEHOLDER */); + refcount_set(&volume->ref, 1); + spin_lock_init(&volume->lock); + + /* Stick the length on the front of the key and pad it out to make + * hashing easier. + */ + klen = strlen(volume_key); + hlen = round_up(1 + klen + 1, sizeof(__le32)); + key = kzalloc(hlen, GFP_KERNEL); + if (!key) + goto err_vol; + key[0] = klen; + memcpy(key + 1, volume_key, klen); + + volume->key = key; + volume->key_hash = fscache_hash(0, key, hlen); + + volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); + down_write(&fscache_addremove_sem); + atomic_inc(&cache->n_volumes); + list_add_tail(&volume->proc_link, &fscache_volumes); + fscache_see_volume(volume, fscache_volume_new_acquire); + fscache_stat(&fscache_n_volumes); + up_write(&fscache_addremove_sem); + _leave(" = v=%x", volume->debug_id); + return volume; + +err_vol: + kfree(volume); +err_cache: + fscache_put_cache(cache, fscache_cache_put_alloc_volume); + fscache_stat(&fscache_n_volumes_nomem); + return NULL; +} + +/* + * Acquire a volume representation cookie and link it to a (proposed) cache. + */ +struct fscache_volume *__fscache_acquire_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + + volume = fscache_alloc_volume(volume_key, cache_name, + coherency_data, coherency_len); + if (!volume) + return ERR_PTR(-ENOMEM); + + if (!fscache_hash_volume(volume)) { + fscache_put_volume(volume, fscache_volume_put_hash_collision); + return ERR_PTR(-EBUSY); + } + + // PLACEHOLDER: Create the volume if we have a cache available + return volume; +} +EXPORT_SYMBOL(__fscache_acquire_volume); + +static void fscache_wake_pending_volume(struct fscache_volume *volume, + struct hlist_bl_head *h) +{ + struct fscache_volume *cursor; + struct hlist_bl_node *p; + + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(cursor, volume)) { + fscache_see_volume(cursor, fscache_volume_see_hash_wake); + clear_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &cursor->flags); + wake_up_bit(&cursor->flags, FSCACHE_VOLUME_ACQUIRE_PENDING); + return; + } + } +} + +/* + * Remove a volume cookie from the hash table. + */ +static void fscache_unhash_volume(struct fscache_volume *volume) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&volume->hash_link); + if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) + fscache_wake_pending_volume(volume, h); + hlist_bl_unlock(h); +} + +/* + * Drop a cache's volume attachments. + */ +static void fscache_free_volume(struct fscache_volume *volume) +{ + struct fscache_cache *cache = volume->cache; + + if (volume->cache_priv) { + // PLACEHOLDER: Detach any attached cache + } + + down_write(&fscache_addremove_sem); + list_del_init(&volume->proc_link); + atomic_dec(&volume->cache->n_volumes); + up_write(&fscache_addremove_sem); + + if (!hlist_bl_unhashed(&volume->hash_link)) + fscache_unhash_volume(volume); + + trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); + kfree(volume->key); + kfree(volume); + fscache_stat_d(&fscache_n_volumes); + fscache_put_cache(cache, fscache_cache_put_volume); +} + +/* + * Drop a reference to a volume cookie. + */ +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + if (volume) { + unsigned int debug_id = volume->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&volume->ref, &ref); + trace_fscache_volume(debug_id, ref - 1, where); + if (zero) + fscache_free_volume(volume); + } +} + +/* + * Relinquish a volume representation cookie. + */ +void __fscache_relinquish_volume(struct fscache_volume *volume, + const void *coherency_data, + bool invalidate) +{ + if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) + return; + + if (invalidate) + set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); + + fscache_put_volume(volume, fscache_volume_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_volume); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of volumes in /proc/fs/fscache/volumes + */ +static int fscache_volumes_seq_show(struct seq_file *m, void *v) +{ + struct fscache_volume *volume; + + if (v == &fscache_volumes) { + seq_puts(m, + "VOLUME REF nCOOK ACC FL CACHE KEY\n" + "======== ===== ===== === == =============== ================\n"); + return 0; + } + + volume = list_entry(v, struct fscache_volume, proc_link); + seq_printf(m, + "%08x %5d %5d %3d %02lx %-15.15s %s\n", + volume->debug_id, + refcount_read(&volume->ref), + atomic_read(&volume->n_cookies), + atomic_read(&volume->n_accesses), + volume->flags, + volume->cache->name ?: "-", + volume->key + 1); + return 0; +} + +static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(&fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_volumes, *_pos); +} + +static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_volumes, _pos); +} + +static void fscache_volumes_seq_stop(struct seq_file *m, void *v) + __releases(&fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_volumes_seq_ops = { + .start = fscache_volumes_seq_start, + .next = fscache_volumes_seq_next, + .stop = fscache_volumes_seq_stop, + .show = fscache_volumes_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 1cf90c252aac..131a741a6652 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -20,13 +20,97 @@ #if defined(CONFIG_FSCACHE) || defined(CONFIG_FSCACHE_MODULE) #define __fscache_available (1) #define fscache_available() (1) +#define fscache_volume_valid(volume) (volume) #define fscache_cookie_valid(cookie) (cookie) #define fscache_cookie_enabled(cookie) (cookie) #else #define __fscache_available (0) #define fscache_available() (0) +#define fscache_volume_valid(volume) (0) #define fscache_cookie_valid(cookie) (0) #define fscache_cookie_enabled(cookie) (0) #endif +/* + * Volume representation cookie. + */ +struct fscache_volume { + refcount_t ref; + atomic_t n_cookies; /* Number of data cookies in volume */ + atomic_t n_accesses; /* Number of cache accesses in progress */ + unsigned int debug_id; + unsigned int key_hash; /* Hash of key string */ + char *key; /* Volume ID, eg. "afs@example.com@1234" */ + struct list_head proc_link; /* Link in /proc/fs/fscache/volumes */ + struct hlist_bl_node hash_link; /* Link in hash table */ + struct work_struct work; + struct fscache_cache *cache; /* The cache in which this resides */ + void *cache_priv; /* Cache private data */ + spinlock_t lock; + unsigned long flags; +#define FSCACHE_VOLUME_RELINQUISHED 0 /* Volume is being cleaned up */ +#define FSCACHE_VOLUME_INVALIDATE 1 /* Volume was invalidated */ +#define FSCACHE_VOLUME_COLLIDED_WITH 2 /* Volume was collided with */ +#define FSCACHE_VOLUME_ACQUIRE_PENDING 3 /* Volume is waiting to complete acquisition */ +#define FSCACHE_VOLUME_CREATING 4 /* Volume is being created on disk */ +}; + +/* + * slow-path functions for when there is actually caching available, and the + * netfs does actually have a valid token + * - these are not to be called directly + * - these are undefined symbols when FS-Cache is not configured and the + * optimiser takes care of not using them + */ +extern struct fscache_volume *__fscache_acquire_volume(const char *, const char *, + const void *, size_t); +extern void __fscache_relinquish_volume(struct fscache_volume *, const void *, bool); + +/** + * fscache_acquire_volume - Register a volume as desiring caching services + * @volume_key: An identification string for the volume + * @cache_name: The name of the cache to use (or NULL for the default) + * @coherency_data: Piece of arbitrary coherency data to check (or NULL) + * @coherency_len: The size of the coherency data + * + * Register a volume as desiring caching services if they're available. The + * caller must provide an identifier for the volume and may also indicate which + * cache it should be in. If a preexisting volume entry is found in the cache, + * the coherency data must match otherwise the entry will be invalidated. + * + * Returns a cookie pointer on success, -ENOMEM if out of memory or -EBUSY if a + * cache volume of that name is already acquired. Note that "NULL" is a valid + * cookie pointer and can be returned if caching is refused. + */ +static inline +struct fscache_volume *fscache_acquire_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + if (!fscache_available()) + return NULL; + return __fscache_acquire_volume(volume_key, cache_name, + coherency_data, coherency_len); +} + +/** + * fscache_relinquish_volume - Cease caching a volume + * @volume: The volume cookie + * @coherency_data: Piece of arbitrary coherency data to set (or NULL) + * @invalidate: True if the volume should be invalidated + * + * Indicate that a filesystem no longer desires caching services for a volume. + * The caller must have relinquished all file cookies prior to calling this. + * The stored coherency data is updated. + */ +static inline +void fscache_relinquish_volume(struct fscache_volume *volume, + const void *coherency_data, + bool invalidate) +{ + if (fscache_volume_valid(volume)) + __fscache_relinquish_volume(volume, coherency_data, invalidate); +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 3b8e0597b2c1..eeb3e7d88e20 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -23,9 +23,26 @@ enum fscache_cache_trace { fscache_cache_collision, fscache_cache_get_acquire, fscache_cache_new_acquire, + fscache_cache_put_alloc_volume, fscache_cache_put_cache, fscache_cache_put_prep_failed, fscache_cache_put_relinquish, + fscache_cache_put_volume, +}; + +enum fscache_volume_trace { + fscache_volume_collision, + fscache_volume_get_cookie, + fscache_volume_get_create_work, + fscache_volume_get_hash_collision, + fscache_volume_free, + fscache_volume_new_acquire, + fscache_volume_put_cookie, + fscache_volume_put_create_work, + fscache_volume_put_hash_collision, + fscache_volume_put_relinquish, + fscache_volume_see_create_work, + fscache_volume_see_hash_wake, }; #endif @@ -37,9 +54,25 @@ enum fscache_cache_trace { EM(fscache_cache_collision, "*COLLIDE*") \ EM(fscache_cache_get_acquire, "GET acq ") \ EM(fscache_cache_new_acquire, "NEW acq ") \ + EM(fscache_cache_put_alloc_volume, "PUT alvol") \ EM(fscache_cache_put_cache, "PUT cache") \ EM(fscache_cache_put_prep_failed, "PUT pfail") \ - E_(fscache_cache_put_relinquish, "PUT relnq") + EM(fscache_cache_put_relinquish, "PUT relnq") \ + E_(fscache_cache_put_volume, "PUT vol ") + +#define fscache_volume_traces \ + EM(fscache_volume_collision, "*COLLIDE*") \ + EM(fscache_volume_get_cookie, "GET cook ") \ + EM(fscache_volume_get_create_work, "GET creat") \ + EM(fscache_volume_get_hash_collision, "GET hcoll") \ + EM(fscache_volume_free, "FREE ") \ + EM(fscache_volume_new_acquire, "NEW acq ") \ + EM(fscache_volume_put_cookie, "PUT cook ") \ + EM(fscache_volume_put_create_work, "PUT creat") \ + EM(fscache_volume_put_hash_collision, "PUT hcoll") \ + EM(fscache_volume_put_relinquish, "PUT relnq") \ + EM(fscache_volume_see_create_work, "SEE creat") \ + E_(fscache_volume_see_hash_wake, "SEE hwake") /* * Export enum symbols via userspace. @@ -50,6 +83,7 @@ enum fscache_cache_trace { #define E_(a, b) TRACE_DEFINE_ENUM(a); fscache_cache_traces; +fscache_volume_traces; /* * Now redefine the EM() and E_() macros to map the enums to the strings that @@ -86,6 +120,31 @@ TRACE_EVENT(fscache_cache, __entry->usage) ); +TRACE_EVENT(fscache_volume, + TP_PROTO(unsigned int volume_debug_id, + int usage, + enum fscache_volume_trace where), + + TP_ARGS(volume_debug_id, usage, where), + + TP_STRUCT__entry( + __field(unsigned int, volume ) + __field(int, usage ) + __field(enum fscache_volume_trace, where ) + ), + + TP_fast_assign( + __entry->volume = volume_debug_id; + __entry->usage = usage; + __entry->where = where; + ), + + TP_printk("V=%08x %s u=%d", + __entry->volume, + __print_symbolic(__entry->where, fscache_volume_traces), + __entry->usage) + ); + #endif /* _TRACE_FSCACHE_H */ /* This part must be outside protection */ -- cgit v1.2.3-59-g8ed1b From 7f3283aba39a0f395700c3b5defa4ec49d9914b3 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:53:34 +0100 Subject: fscache: Implement cookie registration Add functions to the fscache API to allow data file cookies to be acquired and relinquished by the network filesystem. It is intended that the filesystem will create such cookies per-inode under a volume. To request a cookie, the filesystem should call: struct fscache_cookie * fscache_acquire_cookie(struct fscache_volume *volume, u8 advice, const void *index_key, size_t index_key_len, const void *aux_data, size_t aux_data_len, loff_t object_size) The filesystem must first have created a volume cookie, which is passed in here. If it passes in NULL then the function will just return a NULL cookie. A binary key should be passed in index_key and is of size index_key_len. This is saved in the cookie and is used to locate the associated data in the cache. A coherency data buffer of size aux_data_len will be allocated and initialised from the buffer pointed to by aux_data. This is used to validate cache objects when they're opened and is stored on disk with them when they're committed. The data is stored in the cookie and will be updateable by various functions in later patches. The object_size must also be given. This is also used to perform a coherency check and to size the backing storage appropriately. This function disallows a cookie from being acquired twice in parallel, though it will cause the second user to wait if the first is busy relinquishing its cookie. When a network filesystem has finished with a cookie, it should call: void fscache_relinquish_cookie(struct fscache_volume *volume, bool retire) If retire is true, any backing data will be discarded immediately. Changes ======= ver #3: - fscache_hash()'s size parameter is now in bytes. Use __le32 as the unit to round up to. - When comparing cookies, simply see if the attributes are the same rather than subtracting them to produce a strcmp-style return[1]. - Add a check to see if the cookie is still hashed at the point of freeing. ver #2: - Don't hold n_accesses elevated whilst cache is bound to a cookie, but rather add a flag that prevents the state machine from being queued when n_accesses reaches 0. - Remove the unused cookie pointer field from the fscache_acquire tracepoint. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/CAHk-=whtkzB446+hX0zdLsdcUJsJ=8_-0S1mE_R+YurThfUbLA@mail.gmail.com/ [1] Link: https://lore.kernel.org/r/163819590658.215744.14934902514281054323.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906891983.143852.6219772337558577395.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967088507.1823006.12659006350221417165.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021498432.640689.12743483856927722772.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/Makefile | 1 + fs/fscache/cookie.c | 497 +++++++++++++++++++++++++++++++++++++++++ fs/fscache/internal.h | 23 ++ fs/fscache/main.c | 12 + fs/fscache/proc.c | 4 + fs/fscache/stats.c | 28 ++- include/linux/fscache-cache.h | 22 ++ include/linux/fscache.h | 134 +++++++++++ include/trace/events/fscache.h | 111 +++++++++ 9 files changed, 831 insertions(+), 1 deletion(-) create mode 100644 fs/fscache/cookie.c (limited to 'include/linux') diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index bb5282ae682f..bcc79615f93a 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -5,6 +5,7 @@ fscache-y := \ cache.o \ + cookie.o \ main.o \ volume.o diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c new file mode 100644 index 000000000000..438b0098aa73 --- /dev/null +++ b/fs/fscache/cookie.c @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* netfs cookie management + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * See Documentation/filesystems/caching/netfs-api.rst for more information on + * the netfs API. + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +struct kmem_cache *fscache_cookie_jar; + +static void fscache_drop_cookie(struct fscache_cookie *cookie); + +#define fscache_cookie_hash_shift 15 +static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; +static LIST_HEAD(fscache_cookies); +static DEFINE_RWLOCK(fscache_cookies_lock); +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAFWRD"; + +void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) +{ + const u8 *k; + + pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", + prefix, + cookie->debug_id, + cookie->flags, + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state]); + pr_err("%c-cookie V=%08x [%s]\n", + prefix, + cookie->volume->debug_id, + cookie->volume->key); + + k = (cookie->key_len <= sizeof(cookie->inline_key)) ? + cookie->inline_key : cookie->key; + pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); +} + +static void fscache_free_cookie(struct fscache_cookie *cookie) +{ + if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { + fscache_print_cookie(cookie, 'F'); + return; + } + + write_lock(&fscache_cookies_lock); + list_del(&cookie->proc_link); + write_unlock(&fscache_cookies_lock); + if (cookie->aux_len > sizeof(cookie->inline_aux)) + kfree(cookie->aux); + if (cookie->key_len > sizeof(cookie->inline_key)) + kfree(cookie->key); + fscache_stat_d(&fscache_n_cookies); + kmem_cache_free(fscache_cookie_jar, cookie); +} + +static inline void wake_up_cookie_state(struct fscache_cookie *cookie) +{ + /* Use a barrier to ensure that waiters see the state variable + * change, as spin_unlock doesn't guarantee a barrier. + * + * See comments over wake_up_bit() and waitqueue_active(). + */ + smp_mb(); + wake_up_var(&cookie->state); +} + +static void __fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + cookie->state = state; +} + +/* + * Change the state a cookie is at and wake up anyone waiting for that - but + * only if the cookie isn't already marked as being in a cleanup state. + */ +void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + bool changed = false; + + spin_lock(&cookie->lock); + switch (cookie->state) { + case FSCACHE_COOKIE_STATE_RELINQUISHING: + break; + default: + __fscache_set_cookie_state(cookie, state); + changed = true; + break; + } + spin_unlock(&cookie->lock); + if (changed) + wake_up_cookie_state(cookie); +} +EXPORT_SYMBOL(fscache_set_cookie_state); + +/* + * Set the index key in a cookie. The cookie struct has space for a 16-byte + * key plus length and hash, but if that's not big enough, it's instead a + * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then + * the key data. + */ +static int fscache_set_key(struct fscache_cookie *cookie, + const void *index_key, size_t index_key_len) +{ + void *buf; + size_t buf_size; + + buf_size = round_up(index_key_len, sizeof(__le32)); + + if (index_key_len > sizeof(cookie->inline_key)) { + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + cookie->key = buf; + } else { + buf = cookie->inline_key; + } + + memcpy(buf, index_key, index_key_len); + cookie->key_hash = fscache_hash(cookie->volume->key_hash, + buf, buf_size); + return 0; +} + +static bool fscache_cookie_same(const struct fscache_cookie *a, + const struct fscache_cookie *b) +{ + const void *ka, *kb; + + if (a->key_hash != b->key_hash || + a->volume != b->volume || + a->key_len != b->key_len) + return false; + + if (a->key_len <= sizeof(a->inline_key)) { + ka = &a->inline_key; + kb = &b->inline_key; + } else { + ka = a->key; + kb = b->key; + } + return memcmp(ka, kb, a->key_len) == 0; +} + +static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); + +/* + * Allocate a cookie. + */ +static struct fscache_cookie *fscache_alloc_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + /* allocate and initialise a cookie */ + cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); + if (!cookie) + return NULL; + fscache_stat(&fscache_n_cookies); + + cookie->volume = volume; + cookie->advice = advice; + cookie->key_len = index_key_len; + cookie->aux_len = aux_data_len; + cookie->object_size = object_size; + if (object_size == 0) + __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + + if (fscache_set_key(cookie, index_key, index_key_len) < 0) + goto nomem; + + if (cookie->aux_len <= sizeof(cookie->inline_aux)) { + memcpy(cookie->inline_aux, aux_data, cookie->aux_len); + } else { + cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); + if (!cookie->aux) + goto nomem; + } + + refcount_set(&cookie->ref, 1); + cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); + cookie->state = FSCACHE_COOKIE_STATE_QUIESCENT; + spin_lock_init(&cookie->lock); + INIT_LIST_HEAD(&cookie->commit_link); + INIT_WORK(&cookie->work, NULL /* PLACEHOLDER */); + + write_lock(&fscache_cookies_lock); + list_add_tail(&cookie->proc_link, &fscache_cookies); + write_unlock(&fscache_cookies_lock); + fscache_see_cookie(cookie, fscache_cookie_new_acquire); + return cookie; + +nomem: + fscache_free_cookie(cookie); + return NULL; +} + +static void fscache_wait_on_collision(struct fscache_cookie *candidate, + struct fscache_cookie *wait_for) +{ + enum fscache_cookie_state *statep = &wait_for->state; + + wait_var_event_timeout(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED, + 20 * HZ); + if (READ_ONCE(*statep) != FSCACHE_COOKIE_STATE_DROPPED) { + pr_notice("Potential collision c=%08x old: c=%08x", + candidate->debug_id, wait_for->debug_id); + wait_var_event(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED); + } +} + +/* + * Attempt to insert the new cookie into the hash. If there's a collision, we + * wait for the old cookie to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_cookie(struct fscache_cookie *candidate) +{ + struct fscache_cookie *cursor, *wait_for = NULL; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_cookie_same(candidate, cursor)) { + if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) + goto collision; + wait_for = fscache_get_cookie(cursor, + fscache_cookie_get_hash_collision); + break; + } + } + + fscache_get_volume(candidate->volume, fscache_volume_get_cookie); + atomic_inc(&candidate->volume->n_cookies); + hlist_bl_add_head(&candidate->hash_link, h); + set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); + hlist_bl_unlock(h); + + if (wait_for) { + fscache_wait_on_collision(candidate, wait_for); + fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); + } + return true; + +collision: + trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), + fscache_cookie_collision); + pr_err("Duplicate cookie detected\n"); + fscache_print_cookie(cursor, 'O'); + fscache_print_cookie(candidate, 'N'); + hlist_bl_unlock(h); + return false; +} + +/* + * Request a cookie to represent a data storage object within a volume. + * + * We never let on to the netfs about errors. We may set a negative cookie + * pointer, but that's okay + */ +struct fscache_cookie *__fscache_acquire_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + _enter("V=%x", volume->debug_id); + + if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) + return NULL; + if (!aux_data || !aux_data_len) { + aux_data = NULL; + aux_data_len = 0; + } + + fscache_stat(&fscache_n_acquires); + + cookie = fscache_alloc_cookie(volume, advice, + index_key, index_key_len, + aux_data, aux_data_len, + object_size); + if (!cookie) { + fscache_stat(&fscache_n_acquires_oom); + return NULL; + } + + if (!fscache_hash_cookie(cookie)) { + fscache_see_cookie(cookie, fscache_cookie_discard); + fscache_free_cookie(cookie); + return NULL; + } + + trace_fscache_acquire(cookie); + fscache_stat(&fscache_n_acquires_ok); + _leave(" = c=%08x", cookie->debug_id); + return cookie; +} +EXPORT_SYMBOL(__fscache_acquire_cookie); + +/* + * Remove a cookie from the hash table. + */ +static void fscache_unhash_cookie(struct fscache_cookie *cookie) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&cookie->hash_link); + clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); + hlist_bl_unlock(h); +} + +/* + * Finalise a cookie after all its resources have been disposed of. + */ +static void fscache_drop_cookie(struct fscache_cookie *cookie) +{ + spin_lock(&cookie->lock); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + + fscache_unhash_cookie(cookie); + fscache_stat(&fscache_n_relinquishes_dropped); +} + +/* + * Allow the netfs to release a cookie back to the cache. + * - the object will be marked as recyclable on disk if retire is true + */ +void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) +{ + fscache_stat(&fscache_n_relinquishes); + if (retire) + fscache_stat(&fscache_n_relinquishes_retire); + + _enter("c=%08x{%d},%d", + cookie->debug_id, atomic_read(&cookie->n_active), retire); + + if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Cookie c=%x already relinquished\n", cookie->debug_id)) + return; + + if (retire) + set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); + trace_fscache_relinquish(cookie, retire); + + ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); + ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); + atomic_dec(&cookie->volume->n_cookies); + + set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); + + if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) + ; // PLACEHOLDER: Do something here if the cookie was cached + else + fscache_drop_cookie(cookie); + fscache_put_cookie(cookie, fscache_cookie_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_cookie); + +/* + * Drop a reference to a cookie. + */ +void fscache_put_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + struct fscache_volume *volume = cookie->volume; + unsigned int cookie_debug_id = cookie->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&cookie->ref, &ref); + trace_fscache_cookie(cookie_debug_id, ref - 1, where); + if (zero) { + fscache_free_cookie(cookie); + fscache_put_volume(volume, fscache_volume_put_cookie); + } +} +EXPORT_SYMBOL(fscache_put_cookie); + +/* + * Get a reference to a cookie. + */ +struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + int ref; + + __refcount_inc(&cookie->ref, &ref); + trace_fscache_cookie(cookie->debug_id, ref + 1, where); + return cookie; +} +EXPORT_SYMBOL(fscache_get_cookie); + +/* + * Generate a list of extant cookies in /proc/fs/fscache/cookies + */ +static int fscache_cookies_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cookie *cookie; + unsigned int keylen = 0, auxlen = 0; + u8 *p; + + if (v == &fscache_cookies) { + seq_puts(m, + "COOKIE VOLUME REF ACT ACC S FL DEF \n" + "======== ======== === === === = == ================\n" + ); + return 0; + } + + cookie = list_entry(v, struct fscache_cookie, proc_link); + + seq_printf(m, + "%08x %08x %3d %3d %3d %c %02lx", + cookie->debug_id, + cookie->volume->debug_id, + refcount_read(&cookie->ref), + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state], + cookie->flags); + + keylen = cookie->key_len; + auxlen = cookie->aux_len; + + if (keylen > 0 || auxlen > 0) { + seq_puts(m, " "); + p = keylen <= sizeof(cookie->inline_key) ? + cookie->inline_key : cookie->key; + for (; keylen > 0; keylen--) + seq_printf(m, "%02x", *p++); + if (auxlen > 0) { + seq_puts(m, ", "); + p = auxlen <= sizeof(cookie->inline_aux) ? + cookie->inline_aux : cookie->aux; + for (; auxlen > 0; auxlen--) + seq_printf(m, "%02x", *p++); + } + } + + seq_puts(m, "\n"); + return 0; +} + +static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_cookies_lock) +{ + read_lock(&fscache_cookies_lock); + return seq_list_start_head(&fscache_cookies, *_pos); +} + +static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_cookies, _pos); +} + +static void fscache_cookies_seq_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + read_unlock(&fscache_cookies_lock); +} + + +const struct seq_operations fscache_cookies_seq_ops = { + .start = fscache_cookies_seq_start, + .next = fscache_cookies_seq_next, + .stop = fscache_cookies_seq_stop, + .show = fscache_cookies_seq_show, +}; diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 07dc9cbc2280..71c897757d44 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -50,6 +50,20 @@ static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, return try_cmpxchg_release(&cache->state, &old_state, new_state); } +/* + * cookie.c + */ +extern struct kmem_cache *fscache_cookie_jar; +extern const struct seq_operations fscache_cookies_seq_ops; + +extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); +static inline void fscache_see_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), + where); +} + /* * main.c */ @@ -75,6 +89,15 @@ extern void fscache_proc_cleanup(void); extern atomic_t fscache_n_volumes; extern atomic_t fscache_n_volumes_collision; extern atomic_t fscache_n_volumes_nomem; +extern atomic_t fscache_n_cookies; + +extern atomic_t fscache_n_acquires; +extern atomic_t fscache_n_acquires_ok; +extern atomic_t fscache_n_acquires_oom; + +extern atomic_t fscache_n_relinquishes; +extern atomic_t fscache_n_relinquishes_retire; +extern atomic_t fscache_n_relinquishes_dropped; static inline void fscache_stat(atomic_t *stat) { diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 687b34903d5b..ae493e9ca1c9 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -79,9 +79,20 @@ static int __init fscache_init(void) if (ret < 0) goto error_proc; + fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", + sizeof(struct fscache_cookie), + 0, 0, NULL); + if (!fscache_cookie_jar) { + pr_notice("Failed to allocate a cookie jar\n"); + ret = -ENOMEM; + goto error_cookie_jar; + } + pr_notice("Loaded\n"); return 0; +error_cookie_jar: + fscache_proc_cleanup(); error_proc: destroy_workqueue(fscache_wq); error_wq: @@ -97,6 +108,7 @@ static void __exit fscache_exit(void) { _enter(""); + kmem_cache_destroy(fscache_cookie_jar); fscache_proc_cleanup(); destroy_workqueue(fscache_wq); pr_notice("Unloaded\n"); diff --git a/fs/fscache/proc.c b/fs/fscache/proc.c index bc6ecbdd065d..dc3b0e9c8cce 100644 --- a/fs/fscache/proc.c +++ b/fs/fscache/proc.c @@ -27,6 +27,10 @@ int __init fscache_proc_init(void) &fscache_volumes_seq_ops)) goto error; + if (!proc_create_seq("fs/fscache/cookies", S_IFREG | 0444, NULL, + &fscache_cookies_seq_ops)) + goto error; + #ifdef CONFIG_FSCACHE_STATS if (!proc_create_single("fs/fscache/stats", S_IFREG | 0444, NULL, fscache_stats_show)) diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index b811a4d03585..252e883ae148 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -16,6 +16,18 @@ atomic_t fscache_n_volumes; atomic_t fscache_n_volumes_collision; atomic_t fscache_n_volumes_nomem; +atomic_t fscache_n_cookies; + +atomic_t fscache_n_acquires; +atomic_t fscache_n_acquires_ok; +atomic_t fscache_n_acquires_oom; + +atomic_t fscache_n_updates; +EXPORT_SYMBOL(fscache_n_updates); + +atomic_t fscache_n_relinquishes; +atomic_t fscache_n_relinquishes_retire; +atomic_t fscache_n_relinquishes_dropped; /* * display the general statistics @@ -23,12 +35,26 @@ atomic_t fscache_n_volumes_nomem; int fscache_stats_show(struct seq_file *m, void *v) { seq_puts(m, "FS-Cache statistics\n"); - seq_printf(m, "Cookies: v=%d vcol=%u voom=%u\n", + seq_printf(m, "Cookies: n=%d v=%d vcol=%u voom=%u\n", + atomic_read(&fscache_n_cookies), atomic_read(&fscache_n_volumes), atomic_read(&fscache_n_volumes_collision), atomic_read(&fscache_n_volumes_nomem) ); + seq_printf(m, "Acquire: n=%u ok=%u oom=%u\n", + atomic_read(&fscache_n_acquires), + atomic_read(&fscache_n_acquires_ok), + atomic_read(&fscache_n_acquires_oom)); + + seq_printf(m, "Updates: n=%u\n", + atomic_read(&fscache_n_updates)); + + seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", + atomic_read(&fscache_n_relinquishes), + atomic_read(&fscache_n_relinquishes_retire), + atomic_read(&fscache_n_relinquishes_dropped)); + netfs_stats_show(m); return 0; } diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 18cd5c9877bb..c4355b888c91 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -17,6 +17,7 @@ #include enum fscache_cache_trace; +enum fscache_cookie_trace; enum fscache_access_trace; enum fscache_cache_state { @@ -52,4 +53,25 @@ extern struct rw_semaphore fscache_addremove_sem; extern struct fscache_cache *fscache_acquire_cache(const char *name); extern void fscache_relinquish_cache(struct fscache_cache *cache); +extern struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where); +extern void fscache_put_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where); +extern void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state); + +/** + * fscache_get_key - Get a pointer to the cookie key + * @cookie: The cookie to query + * + * Return a pointer to the where a cookie's key is stored. + */ +static inline void *fscache_get_key(struct fscache_cookie *cookie) +{ + if (cookie->key_len <= sizeof(cookie->inline_key)) + return cookie->inline_key; + else + return cookie->key; +} + #endif /* _LINUX_FSCACHE_CACHE_H */ diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 131a741a6652..4450d17c11e8 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -31,6 +31,27 @@ #define fscache_cookie_enabled(cookie) (0) #endif +struct fscache_cookie; + +#define FSCACHE_ADV_SINGLE_CHUNK 0x01 /* The object is a single chunk of data */ +#define FSCACHE_ADV_WRITE_CACHE 0x00 /* Do cache if written to locally */ +#define FSCACHE_ADV_WRITE_NOCACHE 0x02 /* Don't cache if written to locally */ + +/* + * Data object state. + */ +enum fscache_cookie_state { + FSCACHE_COOKIE_STATE_QUIESCENT, /* The cookie is uncached */ + FSCACHE_COOKIE_STATE_LOOKING_UP, /* The cache object is being looked up */ + FSCACHE_COOKIE_STATE_CREATING, /* The cache object is being created */ + FSCACHE_COOKIE_STATE_ACTIVE, /* The cache is active, readable and writable */ + FSCACHE_COOKIE_STATE_FAILED, /* The cache failed, withdraw to clear */ + FSCACHE_COOKIE_STATE_WITHDRAWING, /* The cookie is being withdrawn */ + FSCACHE_COOKIE_STATE_RELINQUISHING, /* The cookie is being relinquished */ + FSCACHE_COOKIE_STATE_DROPPED, /* The cookie has been dropped */ +#define FSCACHE_COOKIE_STATE__NR (FSCACHE_COOKIE_STATE_DROPPED + 1) +} __attribute__((mode(byte))); + /* * Volume representation cookie. */ @@ -55,6 +76,60 @@ struct fscache_volume { #define FSCACHE_VOLUME_CREATING 4 /* Volume is being created on disk */ }; +/* + * Data file representation cookie. + * - a file will only appear in one cache + * - a request to cache a file may or may not be honoured, subject to + * constraints such as disk space + * - indices are created on disk just-in-time + */ +struct fscache_cookie { + refcount_t ref; + atomic_t n_active; /* number of active users of cookie */ + atomic_t n_accesses; /* Number of cache accesses in progress */ + unsigned int debug_id; + unsigned int inval_counter; /* Number of invalidations made */ + spinlock_t lock; + struct fscache_volume *volume; /* Parent volume of this file. */ + void *cache_priv; /* Cache-side representation */ + struct hlist_bl_node hash_link; /* Link in hash table */ + struct list_head proc_link; /* Link in proc list */ + struct list_head commit_link; /* Link in commit queue */ + struct work_struct work; /* Commit/relinq/withdraw work */ + loff_t object_size; /* Size of the netfs object */ + unsigned long unused_at; /* Time at which unused (jiffies) */ + unsigned long flags; +#define FSCACHE_COOKIE_RELINQUISHED 0 /* T if cookie has been relinquished */ +#define FSCACHE_COOKIE_RETIRED 1 /* T if this cookie has retired on relinq */ +#define FSCACHE_COOKIE_IS_CACHING 2 /* T if this cookie is cached */ +#define FSCACHE_COOKIE_NO_DATA_TO_READ 3 /* T if this cookie has nothing to read */ +#define FSCACHE_COOKIE_NEEDS_UPDATE 4 /* T if attrs have been updated */ +#define FSCACHE_COOKIE_HAS_BEEN_CACHED 5 /* T if cookie needs withdraw-on-relinq */ +#define FSCACHE_COOKIE_DISABLED 6 /* T if cookie has been disabled */ +#define FSCACHE_COOKIE_LOCAL_WRITE 7 /* T if cookie has been modified locally */ +#define FSCACHE_COOKIE_NO_ACCESS_WAKE 8 /* T if no wake when n_accesses goes 0 */ +#define FSCACHE_COOKIE_DO_RELINQUISH 9 /* T if this cookie needs relinquishment */ +#define FSCACHE_COOKIE_DO_WITHDRAW 10 /* T if this cookie needs withdrawing */ +#define FSCACHE_COOKIE_DO_LRU_DISCARD 11 /* T if this cookie needs LRU discard */ +#define FSCACHE_COOKIE_DO_PREP_TO_WRITE 12 /* T if cookie needs write preparation */ +#define FSCACHE_COOKIE_HAVE_DATA 13 /* T if this cookie has data stored */ +#define FSCACHE_COOKIE_IS_HASHED 14 /* T if this cookie is hashed */ + + enum fscache_cookie_state state; + u8 advice; /* FSCACHE_ADV_* */ + u8 key_len; /* Length of index key */ + u8 aux_len; /* Length of auxiliary data */ + u32 key_hash; /* Hash of volume, key, len */ + union { + void *key; /* Index key */ + u8 inline_key[16]; /* - If the key is short enough */ + }; + union { + void *aux; /* Auxiliary data */ + u8 inline_aux[8]; /* - If the aux data is short enough */ + }; +}; + /* * slow-path functions for when there is actually caching available, and the * netfs does actually have a valid token @@ -66,6 +141,14 @@ extern struct fscache_volume *__fscache_acquire_volume(const char *, const char const void *, size_t); extern void __fscache_relinquish_volume(struct fscache_volume *, const void *, bool); +extern struct fscache_cookie *__fscache_acquire_cookie( + struct fscache_volume *, + u8, + const void *, size_t, + const void *, size_t, + loff_t); +extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); + /** * fscache_acquire_volume - Register a volume as desiring caching services * @volume_key: An identification string for the volume @@ -113,4 +196,55 @@ void fscache_relinquish_volume(struct fscache_volume *volume, __fscache_relinquish_volume(volume, coherency_data, invalidate); } +/** + * fscache_acquire_cookie - Acquire a cookie to represent a cache object + * @volume: The volume in which to locate/create this cookie + * @advice: Advice flags (FSCACHE_COOKIE_ADV_*) + * @index_key: The index key for this cookie + * @index_key_len: Size of the index key + * @aux_data: The auxiliary data for the cookie (may be NULL) + * @aux_data_len: Size of the auxiliary data buffer + * @object_size: The initial size of object + * + * Acquire a cookie to represent a data file within the given cache volume. + * + * See Documentation/filesystems/caching/netfs-api.rst for a complete + * description. + */ +static inline +struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume, + u8 advice, + const void *index_key, + size_t index_key_len, + const void *aux_data, + size_t aux_data_len, + loff_t object_size) +{ + if (!fscache_volume_valid(volume)) + return NULL; + return __fscache_acquire_cookie(volume, advice, + index_key, index_key_len, + aux_data, aux_data_len, + object_size); +} + +/** + * fscache_relinquish_cookie - Return the cookie to the cache, maybe discarding + * it + * @cookie: The cookie being returned + * @retire: True if the cache object the cookie represents is to be discarded + * + * This function returns a cookie to the cache, forcibly discarding the + * associated cache object if retire is set to true. + * + * See Documentation/filesystems/caching/netfs-api.rst for a complete + * description. + */ +static inline +void fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) +{ + if (fscache_cookie_valid(cookie)) + __fscache_relinquish_cookie(cookie, retire); +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index eeb3e7d88e20..9286e1c4b2ac 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -45,6 +45,23 @@ enum fscache_volume_trace { fscache_volume_see_hash_wake, }; +enum fscache_cookie_trace { + fscache_cookie_collision, + fscache_cookie_discard, + fscache_cookie_get_end_access, + fscache_cookie_get_hash_collision, + fscache_cookie_new_acquire, + fscache_cookie_put_hash_collision, + fscache_cookie_put_over_queued, + fscache_cookie_put_relinquish, + fscache_cookie_put_withdrawn, + fscache_cookie_put_work, + fscache_cookie_see_active, + fscache_cookie_see_relinquish, + fscache_cookie_see_withdraw, + fscache_cookie_see_work, +}; + #endif /* @@ -74,6 +91,22 @@ enum fscache_volume_trace { EM(fscache_volume_see_create_work, "SEE creat") \ E_(fscache_volume_see_hash_wake, "SEE hwake") +#define fscache_cookie_traces \ + EM(fscache_cookie_collision, "*COLLIDE*") \ + EM(fscache_cookie_discard, "DISCARD ") \ + EM(fscache_cookie_get_hash_collision, "GET hcoll") \ + EM(fscache_cookie_get_end_access, "GQ endac") \ + EM(fscache_cookie_new_acquire, "NEW acq ") \ + EM(fscache_cookie_put_hash_collision, "PUT hcoll") \ + EM(fscache_cookie_put_over_queued, "PQ overq") \ + EM(fscache_cookie_put_relinquish, "PUT relnq") \ + EM(fscache_cookie_put_withdrawn, "PUT wthdn") \ + EM(fscache_cookie_put_work, "PQ work ") \ + EM(fscache_cookie_see_active, "- activ") \ + EM(fscache_cookie_see_relinquish, "- x-rlq") \ + EM(fscache_cookie_see_withdraw, "- x-wth") \ + E_(fscache_cookie_see_work, "- work ") + /* * Export enum symbols via userspace. */ @@ -84,6 +117,7 @@ enum fscache_volume_trace { fscache_cache_traces; fscache_volume_traces; +fscache_cookie_traces; /* * Now redefine the EM() and E_() macros to map the enums to the strings that @@ -145,6 +179,83 @@ TRACE_EVENT(fscache_volume, __entry->usage) ); +TRACE_EVENT(fscache_cookie, + TP_PROTO(unsigned int cookie_debug_id, + int ref, + enum fscache_cookie_trace where), + + TP_ARGS(cookie_debug_id, ref, where), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(int, ref ) + __field(enum fscache_cookie_trace, where ) + ), + + TP_fast_assign( + __entry->cookie = cookie_debug_id; + __entry->ref = ref; + __entry->where = where; + ), + + TP_printk("c=%08x %s r=%d", + __entry->cookie, + __print_symbolic(__entry->where, fscache_cookie_traces), + __entry->ref) + ); + +TRACE_EVENT(fscache_acquire, + TP_PROTO(struct fscache_cookie *cookie), + + TP_ARGS(cookie), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(unsigned int, volume ) + __field(int, v_ref ) + __field(int, v_n_cookies ) + ), + + TP_fast_assign( + __entry->cookie = cookie->debug_id; + __entry->volume = cookie->volume->debug_id; + __entry->v_ref = refcount_read(&cookie->volume->ref); + __entry->v_n_cookies = atomic_read(&cookie->volume->n_cookies); + ), + + TP_printk("c=%08x V=%08x vr=%d vc=%d", + __entry->cookie, + __entry->volume, __entry->v_ref, __entry->v_n_cookies) + ); + +TRACE_EVENT(fscache_relinquish, + TP_PROTO(struct fscache_cookie *cookie, bool retire), + + TP_ARGS(cookie, retire), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(unsigned int, volume ) + __field(int, ref ) + __field(int, n_active ) + __field(u8, flags ) + __field(bool, retire ) + ), + + TP_fast_assign( + __entry->cookie = cookie->debug_id; + __entry->volume = cookie->volume->debug_id; + __entry->ref = refcount_read(&cookie->ref); + __entry->n_active = atomic_read(&cookie->n_active); + __entry->flags = cookie->flags; + __entry->retire = retire; + ), + + TP_printk("c=%08x V=%08x r=%d U=%d f=%02x rt=%u", + __entry->cookie, __entry->volume, __entry->ref, + __entry->n_active, __entry->flags, __entry->retire) + ); + #endif /* _TRACE_FSCACHE_H */ /* This part must be outside protection */ -- cgit v1.2.3-59-g8ed1b From e6acd3299badbfb5fb0231d42481d4f5dedf5599 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:26:17 +0100 Subject: fscache: Implement volume-level access helpers Add a pair of helper functions to manage access to a volume, pinning the volume in place for the duration to prevent cache withdrawal from removing it: bool fscache_begin_volume_access(struct fscache_volume *volume, enum fscache_access_trace why); void fscache_end_volume_access(struct fscache_volume *volume, enum fscache_access_trace why); The way the access gate on the volume works/will work is: (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), then we return false to indicate access was not permitted. (2) If the cache tests as live, then we increment the volume's n_accesses count and then recheck the cache liveness, ending the access if it ceased to be live. (3) When we end the access, we decrement the volume's n_accesses and wake up the any waiters if it reaches 0. (4) Whilst the cache is caching, the volume's n_accesses is kept artificially incremented to prevent wakeups from happening. (5) When the cache is taken offline, the state is changed to prevent new accesses, the volume's n_accesses is decremented and we wait for it to become 0. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819594158.215744.8285859817391683254.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906894315.143852.5454793807544710479.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967095028.1823006.9173132503876627466.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021501546.640689.9631510472149608443.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/internal.h | 3 ++ fs/fscache/main.c | 1 + fs/fscache/volume.c | 84 ++++++++++++++++++++++++++++++++++++++++++ include/linux/fscache-cache.h | 4 ++ include/trace/events/fscache.h | 34 +++++++++++++++++ 5 files changed, 126 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index be29816b37ef..91a4ea08ec0b 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -130,6 +130,9 @@ struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, enum fscache_volume_trace where); void fscache_put_volume(struct fscache_volume *volume, enum fscache_volume_trace where); +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why); void fscache_create_volume(struct fscache_volume *volume, bool wait); diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 876f4bee5840..6cab5d99ba4c 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -22,6 +22,7 @@ MODULE_PARM_DESC(fscache_debug, "FS-Cache debugging mask"); EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); +EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); struct workqueue_struct *fscache_wq; EXPORT_SYMBOL(fscache_wq); diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c index 630894fefd02..20497f0f10bb 100644 --- a/fs/fscache/volume.c +++ b/fs/fscache/volume.c @@ -33,6 +33,90 @@ static void fscache_see_volume(struct fscache_volume *volume, trace_fscache_volume(volume->debug_id, ref, where); } +/* + * Pin the cache behind a volume so that we can access it. + */ +static void __fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&volume->n_accesses); + smp_mb__after_atomic(); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); +} + +/** + * fscache_begin_volume_access - Pin a cache so a volume can be accessed + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing a volume and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the volume's n_accesses + * count and then recheck the cache liveness, ending the access if it + * ceased to be live. + * + * (3) When we end the access, we decrement the volume's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cache is caching, the volume's n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, the volume's n_accesses is decremented and we wait for it to + * become 0. + * + * The datafile @cookie and the @why indicator are merely provided for tracing + * purposes. + */ +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!fscache_cache_is_live(volume->cache)) + return false; + __fscache_begin_volume_access(volume, cookie, why); + if (!fscache_cache_is_live(volume->cache)) { + fscache_end_volume_access(volume, cookie, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_volume_access - Unpin a cache at the end of an access. + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache volume after we've accessed it. The datafile @cookie and the + * @why indicator are merely provided for tracing purposes. + */ +void fscache_end_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&volume->n_accesses); +} +EXPORT_SYMBOL(fscache_end_volume_access); + static bool fscache_volume_same(const struct fscache_volume *a, const struct fscache_volume *b) { diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index c4355b888c91..fbbd8a2afe12 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -53,6 +53,10 @@ extern struct rw_semaphore fscache_addremove_sem; extern struct fscache_cache *fscache_acquire_cache(const char *name); extern void fscache_relinquish_cache(struct fscache_cache *cache); +extern void fscache_end_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why); + extern struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where); extern void fscache_put_cookie(struct fscache_cookie *cookie, diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 734966bc49e1..4f40cfa52469 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -43,6 +43,7 @@ enum fscache_volume_trace { fscache_volume_put_relinquish, fscache_volume_see_create_work, fscache_volume_see_hash_wake, + fscache_volume_wait_create_work, }; enum fscache_cookie_trace { @@ -245,6 +246,39 @@ TRACE_EVENT(fscache_access_cache, __entry->n_accesses) ); +TRACE_EVENT(fscache_access_volume, + TP_PROTO(unsigned int volume_debug_id, + unsigned int cookie_debug_id, + int ref, + int n_accesses, + enum fscache_access_trace why), + + TP_ARGS(volume_debug_id, cookie_debug_id, ref, n_accesses, why), + + TP_STRUCT__entry( + __field(unsigned int, volume ) + __field(unsigned int, cookie ) + __field(int, ref ) + __field(int, n_accesses ) + __field(enum fscache_access_trace, why ) + ), + + TP_fast_assign( + __entry->volume = volume_debug_id; + __entry->cookie = cookie_debug_id; + __entry->ref = ref; + __entry->n_accesses = n_accesses; + __entry->why = why; + ), + + TP_printk("V=%08x c=%08x %s r=%d a=%d", + __entry->volume, + __entry->cookie, + __print_symbolic(__entry->why, fscache_access_traces), + __entry->ref, + __entry->n_accesses) + ); + TRACE_EVENT(fscache_acquire, TP_PROTO(struct fscache_cookie *cookie), -- cgit v1.2.3-59-g8ed1b From a7733fb632722a2f085f9324f14783effe268ed3 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:53:34 +0100 Subject: fscache: Implement cookie-level access helpers Add a number of helper functions to manage access to a cookie, pinning the cache object in place for the duration to prevent cache withdrawal from removing it: (1) void fscache_init_access_gate(struct fscache_cookie *cookie); This function initialises the access count when a cache binds to a cookie. An extra ref is taken on the access count to prevent wakeups while the cache is active. We're only interested in the wakeup when a cookie is being withdrawn and we're waiting for it to quiesce - at which point the counter will be decremented before the wait. The FSCACHE_COOKIE_NACC_ELEVATED flag is set on the cookie to keep track of the extra ref in order to handle a race between relinquishment and withdrawal both trying to drop the extra ref. (2) bool fscache_begin_cookie_access(struct fscache_cookie *cookie, enum fscache_access_trace why); This function attempts to begin access upon a cookie, pinning it in place if it's cached. If successful, it returns true and leaves a the access count incremented. (3) void fscache_end_cookie_access(struct fscache_cookie *cookie, enum fscache_access_trace why); This function drops the access count obtained by (2), permitting object withdrawal to take place when it reaches zero. A tracepoint is provided to track changes to the access counter on a cookie. Changes ======= ver #2: - Don't hold n_accesses elevated whilst cache is bound to a cookie, but rather add a flag that prevents the state machine from being queued when n_accesses reaches 0. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819595085.215744.1706073049250505427.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906895313.143852.10141619544149102193.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967095980.1823006.1133648159424418877.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021503063.640689.8870918985269528670.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cookie.c | 98 ++++++++++++++++++++++++++++++++++++++++++ fs/fscache/internal.h | 3 ++ fs/fscache/main.c | 1 + include/linux/fscache-cache.h | 2 + include/trace/events/fscache.h | 29 +++++++++++++ 5 files changed, 133 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 438b0098aa73..04d2127bd354 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -62,6 +62,104 @@ static void fscache_free_cookie(struct fscache_cookie *cookie) kmem_cache_free(fscache_cookie_jar, cookie); } +/* + * Initialise the access gate on a cookie by setting a flag to prevent the + * state machine from being queued when the access counter transitions to 0. + * We're only interested in this when we withdraw caching services from the + * cookie. + */ +static void fscache_init_access_gate(struct fscache_cookie *cookie) +{ + int n_accesses; + + n_accesses = atomic_read(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_pin); + set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); +} + +/** + * fscache_end_cookie_access - Unpin a cache at the end of an access. + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache cookie after we've accessed it and bring a deferred + * relinquishment or withdrawal state into effect. + * + * The @why indicator is provided for tracing purposes. + */ +void fscache_end_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); + if (n_accesses == 0 && + !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) { + // PLACEHOLDER: Need to poke the state machine + } +} +EXPORT_SYMBOL(fscache_end_cookie_access); + +/* + * Pin the cache behind a cookie so that we can access it. + */ +static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&cookie->n_accesses); + smp_mb__after_atomic(); /* (Future) read state after is-caching. + * Reread n_accesses after is-caching + */ + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); +} + +/** + * fscache_begin_cookie_access - Pin a cache so data can be accessed + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing data and returns true if successful. This works as follows: + * + * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not + * set), we return false to indicate access was not permitted. + * + * (2) If the cookie is being cached, we increment its n_accesses count and + * then recheck the IS_CACHING flag, ending the access if it got cleared. + * + * (3) When we end the access, we decrement the cookie's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cookie is actively being cached, its n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline or if the cookie is culled, the flag is + * cleared to prevent new accesses, the cookie's n_accesses is decremented + * and we wait for it to become 0. + * + * The @why indicator are merely provided for tracing purposes. + */ +bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return false; + __fscache_begin_cookie_access(cookie, why); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || + !fscache_cache_is_live(cookie->volume->cache)) { + fscache_end_cookie_access(cookie, fscache_access_unlive); + return false; + } + return true; +} + static inline void wake_up_cookie_state(struct fscache_cookie *cookie) { /* Use a barrier to ensure that waiters see the state variable diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 91a4ea08ec0b..e0d8ef212e82 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -59,6 +59,9 @@ extern struct kmem_cache *fscache_cookie_jar; extern const struct seq_operations fscache_cookies_seq_ops; extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); +extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why); + static inline void fscache_see_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where) { diff --git a/fs/fscache/main.c b/fs/fscache/main.c index 6cab5d99ba4c..dad85fd84f6f 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c @@ -23,6 +23,7 @@ MODULE_PARM_DESC(fscache_debug, EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); +EXPORT_TRACEPOINT_SYMBOL(fscache_access); struct workqueue_struct *fscache_wq; EXPORT_SYMBOL(fscache_wq); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index fbbd8a2afe12..66624407ba84 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -61,6 +61,8 @@ extern struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where); extern void fscache_put_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where); +extern void fscache_end_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why); extern void fscache_set_cookie_state(struct fscache_cookie *cookie, enum fscache_cookie_state state); diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 4f40cfa52469..b1a962adfd16 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -279,6 +279,35 @@ TRACE_EVENT(fscache_access_volume, __entry->n_accesses) ); +TRACE_EVENT(fscache_access, + TP_PROTO(unsigned int cookie_debug_id, + int ref, + int n_accesses, + enum fscache_access_trace why), + + TP_ARGS(cookie_debug_id, ref, n_accesses, why), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(int, ref ) + __field(int, n_accesses ) + __field(enum fscache_access_trace, why ) + ), + + TP_fast_assign( + __entry->cookie = cookie_debug_id; + __entry->ref = ref; + __entry->n_accesses = n_accesses; + __entry->why = why; + ), + + TP_printk("c=%08x %s r=%d a=%d", + __entry->cookie, + __print_symbolic(__entry->why, fscache_access_traces), + __entry->ref, + __entry->n_accesses) + ); + TRACE_EVENT(fscache_acquire, TP_PROTO(struct fscache_cookie *cookie), -- cgit v1.2.3-59-g8ed1b From 2e0c76aee25f33c482abda6224bd87732359354d Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:00:26 +0100 Subject: fscache: Implement functions add/remove a cache Implement functions to allow the cache backend to add or remove a cache: (1) Declare a cache to be live: int fscache_add_cache(struct fscache_cache *cache, const struct fscache_cache_ops *ops, void *cache_priv); Take a previously acquired cache cookie, set the operations table and private data and mark the cache open for access. (2) Withdraw a cache from service: void fscache_withdraw_cache(struct fscache_cache *cache); This marks the cache as withdrawn and thus prevents further cache-level and volume-level accesses. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819596022.215744.8799712491432238827.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906896599.143852.17049208999019262884.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967097870.1823006.3470041000971522030.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021505541.640689.1819714759326331054.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cache.c | 70 +++++++++++++++++++++++++++++++++++++++++++ include/linux/fscache-cache.h | 13 ++++++++ 2 files changed, 83 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index e867cff53a70..bbd102be91c4 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -210,12 +210,55 @@ void fscache_relinquish_cache(struct fscache_cache *cache) fscache_cache_put_prep_failed : fscache_cache_put_relinquish; + cache->ops = NULL; cache->cache_priv = NULL; smp_store_release(&cache->state, FSCACHE_CACHE_IS_NOT_PRESENT); fscache_put_cache(cache, where); } EXPORT_SYMBOL(fscache_relinquish_cache); +/** + * fscache_add_cache - Declare a cache as being open for business + * @cache: The cache-level cookie representing the cache + * @ops: Table of cache operations to use + * @cache_priv: Private data for the cache record + * + * Add a cache to the system, making it available for netfs's to use. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +int fscache_add_cache(struct fscache_cache *cache, + const struct fscache_cache_ops *ops, + void *cache_priv) +{ + int n_accesses; + + _enter("{%s,%s}", ops->name, cache->name); + + BUG_ON(fscache_cache_state(cache) != FSCACHE_CACHE_IS_PREPARING); + + /* Get a ref on the cache cookie and keep its n_accesses counter raised + * by 1 to prevent wakeups from transitioning it to 0 until we're + * withdrawing caching services from it. + */ + n_accesses = atomic_inc_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_pin); + + down_write(&fscache_addremove_sem); + + cache->ops = ops; + cache->cache_priv = cache_priv; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_ACTIVE); + + up_write(&fscache_addremove_sem); + pr_notice("Cache \"%s\" added (type %s)\n", cache->name, ops->name); + _leave(" = 0 [%s]", cache->name); + return 0; +} +EXPORT_SYMBOL(fscache_add_cache); + /** * fscache_begin_cache_access - Pin a cache so it can be accessed * @cache: The cache-level cookie @@ -278,6 +321,33 @@ void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_t wake_up_var(&cache->n_accesses); } +/** + * fscache_withdraw_cache - Withdraw a cache from the active service + * @cache: The cache cookie + * + * Begin the process of withdrawing a cache from service. This stops new + * cache-level and volume-level accesses from taking place and waits for + * currently ongoing cache-level accesses to end. + */ +void fscache_withdraw_cache(struct fscache_cache *cache) +{ + int n_accesses; + + pr_notice("Withdrawing cache \"%s\" (%u objs)\n", + cache->name, atomic_read(&cache->object_count)); + + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_WITHDRAWN); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&cache->n_accesses, + atomic_read(&cache->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_cache); + #ifdef CONFIG_PROC_FS static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] = "-PAEW"; diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 66624407ba84..f78add6e7823 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -33,6 +33,7 @@ enum fscache_cache_state { * Cache cookie. */ struct fscache_cache { + const struct fscache_cache_ops *ops; struct list_head cache_link; /* Link in cache list */ void *cache_priv; /* Private cache data (or NULL) */ refcount_t ref; @@ -44,6 +45,14 @@ struct fscache_cache { char *name; }; +/* + * cache operations + */ +struct fscache_cache_ops { + /* name of cache provider */ + const char *name; +}; + extern struct workqueue_struct *fscache_wq; /* @@ -52,6 +61,10 @@ extern struct workqueue_struct *fscache_wq; extern struct rw_semaphore fscache_addremove_sem; extern struct fscache_cache *fscache_acquire_cache(const char *name); extern void fscache_relinquish_cache(struct fscache_cache *cache); +extern int fscache_add_cache(struct fscache_cache *cache, + const struct fscache_cache_ops *ops, + void *cache_priv); +extern void fscache_withdraw_cache(struct fscache_cache *cache); extern void fscache_end_volume_access(struct fscache_volume *volume, struct fscache_cookie *cookie, -- cgit v1.2.3-59-g8ed1b From bfa22da3ed652aa15acd4246fa13a0de6dbe4a59 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:26:17 +0100 Subject: fscache: Provide and use cache methods to lookup/create/free a volume Add cache methods to lookup, create and remove a volume. Looking up or creating the volume requires the cache pinning for access; freeing the volume requires the volume pinning for access. The ->acquire_volume() method is used to ask the cache backend to lookup and, if necessary, create a volume; the ->free_volume() method is used to free the resources for a volume. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819597821.215744.5225318658134989949.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906898645.143852.8537799955945956818.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967099771.1823006.1455197910571061835.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021507345.640689.4073511598838843040.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/volume.c | 89 ++++++++++++++++++++++++++++++++++++++++-- include/linux/fscache-cache.h | 7 ++++ include/trace/events/fscache.h | 11 +++++- 3 files changed, 103 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c index 20497f0f10bb..e1a8e92a6adb 100644 --- a/fs/fscache/volume.c +++ b/fs/fscache/volume.c @@ -15,6 +15,8 @@ static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; static atomic_t fscache_volume_debug_id; static LIST_HEAD(fscache_volumes); +static void fscache_create_volume_work(struct work_struct *work); + struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, enum fscache_volume_trace where) { @@ -213,7 +215,7 @@ static struct fscache_volume *fscache_alloc_volume(const char *volume_key, volume->cache = cache; INIT_LIST_HEAD(&volume->proc_link); - INIT_WORK(&volume->work, NULL /* PLACEHOLDER */); + INIT_WORK(&volume->work, fscache_create_volume_work); refcount_set(&volume->ref, 1); spin_lock_init(&volume->lock); @@ -249,6 +251,58 @@ err_cache: return NULL; } +/* + * Create a volume's representation on disk. Have a volume ref and a cache + * access we have to release. + */ +static void fscache_create_volume_work(struct work_struct *work) +{ + const struct fscache_cache_ops *ops; + struct fscache_volume *volume = + container_of(work, struct fscache_volume, work); + + fscache_see_volume(volume, fscache_volume_see_create_work); + + ops = volume->cache->ops; + if (ops->acquire_volume) + ops->acquire_volume(volume); + fscache_end_cache_access(volume->cache, + fscache_access_acquire_volume_end); + + clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); + wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); + fscache_put_volume(volume, fscache_volume_put_create_work); +} + +/* + * Dispatch a worker thread to create a volume's representation on disk. + */ +void fscache_create_volume(struct fscache_volume *volume, bool wait) +{ + if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) + goto maybe_wait; + if (volume->cache_priv) + goto no_wait; /* We raced */ + if (!fscache_begin_cache_access(volume->cache, + fscache_access_acquire_volume)) + goto no_wait; + + fscache_get_volume(volume, fscache_volume_get_create_work); + if (!schedule_work(&volume->work)) + fscache_put_volume(volume, fscache_volume_put_create_work); + +maybe_wait: + if (wait) { + fscache_see_volume(volume, fscache_volume_wait_create_work); + wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, + TASK_UNINTERRUPTIBLE); + } + return; +no_wait: + clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); + wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); +} + /* * Acquire a volume representation cookie and link it to a (proposed) cache. */ @@ -269,7 +323,7 @@ struct fscache_volume *__fscache_acquire_volume(const char *volume_key, return ERR_PTR(-EBUSY); } - // PLACEHOLDER: Create the volume if we have a cache available + fscache_create_volume(volume, false); return volume; } EXPORT_SYMBOL(__fscache_acquire_volume); @@ -316,7 +370,12 @@ static void fscache_free_volume(struct fscache_volume *volume) struct fscache_cache *cache = volume->cache; if (volume->cache_priv) { - // PLACEHOLDER: Detach any attached cache + __fscache_begin_volume_access(volume, NULL, + fscache_access_relinquish_volume); + if (volume->cache_priv) + cache->ops->free_volume(volume); + fscache_end_volume_access(volume, NULL, + fscache_access_relinquish_volume_end); } down_write(&fscache_addremove_sem); @@ -369,6 +428,30 @@ void __fscache_relinquish_volume(struct fscache_volume *volume, } EXPORT_SYMBOL(__fscache_relinquish_volume); +/** + * fscache_withdraw_volume - Withdraw a volume from being cached + * @volume: Volume cookie + * + * Withdraw a cache volume from service, waiting for all accesses to complete + * before returning. + */ +void fscache_withdraw_volume(struct fscache_volume *volume) +{ + int n_accesses; + + _debug("withdraw V=%x", volume->debug_id); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, 0, + refcount_read(&volume->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&volume->n_accesses, + atomic_read(&volume->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_volume); + #ifdef CONFIG_PROC_FS /* * Generate a list of volumes in /proc/fs/fscache/volumes diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index f78add6e7823..a10b66ca3544 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -51,6 +51,12 @@ struct fscache_cache { struct fscache_cache_ops { /* name of cache provider */ const char *name; + + /* Acquire a volume */ + void (*acquire_volume)(struct fscache_volume *volume); + + /* Free the cache's data attached to a volume */ + void (*free_volume)(struct fscache_volume *volume); }; extern struct workqueue_struct *fscache_wq; @@ -65,6 +71,7 @@ extern int fscache_add_cache(struct fscache_cache *cache, const struct fscache_cache_ops *ops, void *cache_priv); extern void fscache_withdraw_cache(struct fscache_cache *cache); +extern void fscache_withdraw_volume(struct fscache_volume *volume); extern void fscache_end_volume_access(struct fscache_volume *volume, struct fscache_cookie *cookie, diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index b1a962adfd16..1d576bd8112e 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -64,8 +64,12 @@ enum fscache_cookie_trace { }; enum fscache_access_trace { + fscache_access_acquire_volume, + fscache_access_acquire_volume_end, fscache_access_cache_pin, fscache_access_cache_unpin, + fscache_access_relinquish_volume, + fscache_access_relinquish_volume_end, fscache_access_unlive, }; @@ -96,7 +100,8 @@ enum fscache_access_trace { EM(fscache_volume_put_hash_collision, "PUT hcoll") \ EM(fscache_volume_put_relinquish, "PUT relnq") \ EM(fscache_volume_see_create_work, "SEE creat") \ - E_(fscache_volume_see_hash_wake, "SEE hwake") + EM(fscache_volume_see_hash_wake, "SEE hwake") \ + E_(fscache_volume_wait_create_work, "WAIT crea") #define fscache_cookie_traces \ EM(fscache_cookie_collision, "*COLLIDE*") \ @@ -115,8 +120,12 @@ enum fscache_access_trace { E_(fscache_cookie_see_work, "- work ") #define fscache_access_traces \ + EM(fscache_access_acquire_volume, "BEGIN acq_vol") \ + EM(fscache_access_acquire_volume_end, "END acq_vol") \ EM(fscache_access_cache_pin, "PIN cache ") \ EM(fscache_access_cache_unpin, "UNPIN cache ") \ + EM(fscache_access_relinquish_volume, "BEGIN rlq_vol") \ + EM(fscache_access_relinquish_volume_end,"END rlq_vol") \ E_(fscache_access_unlive, "END unlive ") /* -- cgit v1.2.3-59-g8ed1b From 29f18e79fe7c5f8011befeda9be6b220a350f947 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:00:26 +0100 Subject: fscache: Add a function for a cache backend to note an I/O error Add a function to the backend API to note an I/O error in a cache. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819598741.215744.891281275151382095.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906901316.143852.15225412215771586528.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967100721.1823006.16435671567428949398.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021508840.640689.11902836226570620424.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cache.c | 20 ++++++++++++++++++++ include/linux/fscache-cache.h | 2 ++ 2 files changed, 22 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index bbd102be91c4..25eac61f1c29 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -321,6 +321,26 @@ void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_t wake_up_var(&cache->n_accesses); } +/** + * fscache_io_error - Note a cache I/O error + * @cache: The record describing the cache + * + * Note that an I/O error occurred in a cache and that it should no longer be + * used for anything. This also reports the error into the kernel log. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +void fscache_io_error(struct fscache_cache *cache) +{ + if (fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_ACTIVE, + FSCACHE_CACHE_GOT_IOERROR)) + pr_err("Cache '%s' stopped due to I/O error\n", + cache->name); +} +EXPORT_SYMBOL(fscache_io_error); + /** * fscache_withdraw_cache - Withdraw a cache from the active service * @cache: The cache cookie diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index a10b66ca3544..936ef731bbc7 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -73,6 +73,8 @@ extern int fscache_add_cache(struct fscache_cache *cache, extern void fscache_withdraw_cache(struct fscache_cache *cache); extern void fscache_withdraw_volume(struct fscache_volume *volume); +extern void fscache_io_error(struct fscache_cache *cache); + extern void fscache_end_volume_access(struct fscache_volume *volume, struct fscache_cookie *cookie, enum fscache_access_trace why); -- cgit v1.2.3-59-g8ed1b From 5d00e426f95e7ea036fec2a0aceb3f71d6dbdf92 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:53:34 +0100 Subject: fscache: Implement simple cookie state machine Implement a very simple cookie state machine to handle lookup, invalidation, withdrawal, relinquishment and, to be added later, commit on LRU discard. Three cache methods are provided: ->lookup_cookie() to look up and, if necessary, create a data storage object; ->withdraw_cookie() to free the resources associated with that object and potentially delete it; and ->prepare_to_write(), to do prepare for changes to the cached data to be modified locally. Changes ======= ver #3: - Fix a race between LRU discard and relinquishment whereby the former would override the latter and thus the latter would never happen[1]. ver #2: - Don't hold n_accesses elevated whilst cache is bound to a cookie, but rather add a flag that prevents the state machine from being queued when n_accesses reaches 0. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/599331.1639410068@warthog.procyon.org.uk/ [1] Link: https://lore.kernel.org/r/163819599657.215744.15799615296912341745.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906903925.143852.1805855338154353867.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967105456.1823006.14730395299835841776.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021510706.640689.7961423370243272583.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cookie.c | 313 +++++++++++++++++++++++++++++++++++------ include/linux/fscache-cache.h | 27 +++- include/trace/events/fscache.h | 4 + 3 files changed, 300 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 04d2127bd354..336046de08ee 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -15,7 +15,8 @@ struct kmem_cache *fscache_cookie_jar; -static void fscache_drop_cookie(struct fscache_cookie *cookie); +static void fscache_cookie_worker(struct work_struct *work); +static void fscache_unhash_cookie(struct fscache_cookie *cookie); #define fscache_cookie_hash_shift 15 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; @@ -62,6 +63,19 @@ static void fscache_free_cookie(struct fscache_cookie *cookie) kmem_cache_free(fscache_cookie_jar, cookie); } +static void __fscache_queue_cookie(struct fscache_cookie *cookie) +{ + if (!queue_work(fscache_wq, &cookie->work)) + fscache_put_cookie(cookie, fscache_cookie_put_over_queued); +} + +static void fscache_queue_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + fscache_get_cookie(cookie, where); + __fscache_queue_cookie(cookie); +} + /* * Initialise the access gate on a cookie by setting a flag to prevent the * state machine from being queued when the access counter transitions to 0. @@ -98,9 +112,8 @@ void fscache_end_cookie_access(struct fscache_cookie *cookie, trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), n_accesses, why); if (n_accesses == 0 && - !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) { - // PLACEHOLDER: Need to poke the state machine - } + !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); } EXPORT_SYMBOL(fscache_end_cookie_access); @@ -171,35 +184,58 @@ static inline void wake_up_cookie_state(struct fscache_cookie *cookie) wake_up_var(&cookie->state); } +/* + * Change the state a cookie is at and wake up anyone waiting for that. Impose + * an ordering between the stuff stored in the cookie and the state member. + * Paired with fscache_cookie_state(). + */ static void __fscache_set_cookie_state(struct fscache_cookie *cookie, enum fscache_cookie_state state) { - cookie->state = state; + smp_store_release(&cookie->state, state); } -/* - * Change the state a cookie is at and wake up anyone waiting for that - but - * only if the cookie isn't already marked as being in a cleanup state. - */ -void fscache_set_cookie_state(struct fscache_cookie *cookie, - enum fscache_cookie_state state) +static void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) { - bool changed = false; - spin_lock(&cookie->lock); - switch (cookie->state) { - case FSCACHE_COOKIE_STATE_RELINQUISHING: - break; - default: - __fscache_set_cookie_state(cookie, state); - changed = true; - break; - } + __fscache_set_cookie_state(cookie, state); spin_unlock(&cookie->lock); - if (changed) - wake_up_cookie_state(cookie); + wake_up_cookie_state(cookie); +} + +/** + * fscache_cookie_lookup_negative - Note negative lookup + * @cookie: The cookie that was being looked up + * + * Note that some part of the metadata path in the cache doesn't exist and so + * we can release any waiting readers in the certain knowledge that there's + * nothing for them to actually read. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); +} +EXPORT_SYMBOL(fscache_cookie_lookup_negative); + +/** + * fscache_caching_failed - Report that a failure stopped caching on a cookie + * @cookie: The cookie that was affected + * + * Tell fscache that caching on a cookie needs to be stopped due to some sort + * of failure. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_caching_failed(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); } -EXPORT_SYMBOL(fscache_set_cookie_state); +EXPORT_SYMBOL(fscache_caching_failed); /* * Set the index key in a cookie. The cookie struct has space for a 16-byte @@ -291,10 +327,10 @@ static struct fscache_cookie *fscache_alloc_cookie( refcount_set(&cookie->ref, 1); cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); - cookie->state = FSCACHE_COOKIE_STATE_QUIESCENT; spin_lock_init(&cookie->lock); INIT_LIST_HEAD(&cookie->commit_link); - INIT_WORK(&cookie->work, NULL /* PLACEHOLDER */); + INIT_WORK(&cookie->work, fscache_cookie_worker); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); write_lock(&fscache_cookies_lock); list_add_tail(&cookie->proc_link, &fscache_cookies); @@ -417,6 +453,192 @@ struct fscache_cookie *__fscache_acquire_cookie( } EXPORT_SYMBOL(__fscache_acquire_cookie); +/* + * Prepare a cache object to be written to. + */ +static void fscache_prepare_to_write(struct fscache_cookie *cookie) +{ + cookie->volume->cache->ops->prepare_to_write(cookie); +} + +/* + * Look up a cookie in the cache. + */ +static void fscache_perform_lookup(struct fscache_cookie *cookie) +{ + enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; + bool need_withdraw = false; + + _enter(""); + + if (!cookie->volume->cache_priv) { + fscache_create_volume(cookie->volume, true); + if (!cookie->volume->cache_priv) { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + goto out; + } + } + + if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { + if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + need_withdraw = true; + _leave(" [fail]"); + goto out; + } + + fscache_see_cookie(cookie, fscache_cookie_see_active); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); + trace = fscache_access_lookup_cookie_end; + +out: + fscache_end_cookie_access(cookie, trace); + if (need_withdraw) + fscache_withdraw_cookie(cookie); + fscache_end_volume_access(cookie->volume, cookie, trace); +} + +/* + * Perform work upon the cookie, such as committing its cache state, + * relinquishing it or withdrawing the backing cache. We're protected from the + * cache going away under us as object withdrawal must come through this + * non-reentrant work item. + */ +static void fscache_cookie_state_machine(struct fscache_cookie *cookie) +{ + enum fscache_cookie_state state; + bool wake = false; + + _enter("c=%x", cookie->debug_id); + +again: + spin_lock(&cookie->lock); +again_locked: + state = cookie->state; + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + /* The QUIESCENT state is jumped to the LOOKING_UP state by + * fscache_use_cookie(). + */ + + if (atomic_read(&cookie->n_accesses) == 0 && + test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + spin_unlock(&cookie->lock); + fscache_init_access_gate(cookie); + fscache_perform_lookup(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { + spin_unlock(&cookie->lock); + fscache_prepare_to_write(cookie); + spin_lock(&cookie->lock); + } + fallthrough; + + case FSCACHE_COOKIE_STATE_FAILED: + if (atomic_read(&cookie->n_accesses) != 0) + break; + if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_WITHDRAWING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_RELINQUISHING: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + if (cookie->cache_priv) { + spin_unlock(&cookie->lock); + cookie->volume->cache->ops->withdraw_cookie(cookie); + spin_lock(&cookie->lock); + } + + switch (state) { + case FSCACHE_COOKIE_STATE_RELINQUISHING: + fscache_see_cookie(cookie, fscache_cookie_see_relinquish); + fscache_unhash_cookie(cookie); + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_DROPPED); + wake = true; + goto out; + case FSCACHE_COOKIE_STATE_WITHDRAWING: + fscache_see_cookie(cookie, fscache_cookie_see_withdraw); + break; + default: + BUG(); + } + + clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + wake = true; + goto again_locked; + + case FSCACHE_COOKIE_STATE_DROPPED: + break; + + default: + WARN_ONCE(1, "Cookie %x in unexpected state %u\n", + cookie->debug_id, state); + break; + } + +out: + spin_unlock(&cookie->lock); + if (wake) + wake_up_cookie_state(cookie); + _leave(""); +} + +static void fscache_cookie_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); + + fscache_see_cookie(cookie, fscache_cookie_see_work); + fscache_cookie_state_machine(cookie); + fscache_put_cookie(cookie, fscache_cookie_put_work); +} + +/* + * Wait for the object to become inactive. The cookie's work item will be + * scheduled when someone transitions n_accesses to 0 - but if someone's + * already done that, schedule it anyway. + */ +static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + int n_accesses; + bool unpinned; + + unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); + + /* Need to read the access count after unpinning */ + n_accesses = atomic_read(&cookie->n_accesses); + if (unpinned) + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_unpin); + if (n_accesses == 0) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} + /* * Remove a cookie from the hash table. */ @@ -432,21 +654,27 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie) hlist_bl_del(&cookie->hash_link); clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); hlist_bl_unlock(h); + fscache_stat(&fscache_n_relinquishes_dropped); } -/* - * Finalise a cookie after all its resources have been disposed of. - */ -static void fscache_drop_cookie(struct fscache_cookie *cookie) +static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) { - spin_lock(&cookie->lock); - __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); - spin_unlock(&cookie->lock); - wake_up_cookie_state(cookie); + __fscache_withdraw_cookie(cookie); +} - fscache_unhash_cookie(cookie); - fscache_stat(&fscache_n_relinquishes_dropped); +/** + * fscache_withdraw_cookie - Mark a cookie for withdrawal + * @cookie: The cookie to be withdrawn. + * + * Allow the cache backend to withdraw the backing for a cookie for its own + * reasons, even if that cookie is in active use. + */ +void fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); } +EXPORT_SYMBOL(fscache_withdraw_cookie); /* * Allow the netfs to release a cookie back to the cache. @@ -473,12 +701,13 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); atomic_dec(&cookie->volume->n_cookies); - set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); - - if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) - ; // PLACEHOLDER: Do something here if the cookie was cached - else - fscache_drop_cookie(cookie); + if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); + } else { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); + fscache_unhash_cookie(cookie); + } fscache_put_cookie(cookie, fscache_cookie_put_relinquish); } EXPORT_SYMBOL(__fscache_relinquish_cookie); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 936ef731bbc7..ae6a75976450 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -57,6 +57,15 @@ struct fscache_cache_ops { /* Free the cache's data attached to a volume */ void (*free_volume)(struct fscache_volume *volume); + + /* Look up a cookie in the cache */ + bool (*lookup_cookie)(struct fscache_cookie *cookie); + + /* Withdraw an object without any cookie access counts held */ + void (*withdraw_cookie)(struct fscache_cookie *cookie); + + /* Prepare to write to a live cache object */ + void (*prepare_to_write)(struct fscache_cookie *cookie); }; extern struct workqueue_struct *fscache_wq; @@ -72,6 +81,7 @@ extern int fscache_add_cache(struct fscache_cache *cache, void *cache_priv); extern void fscache_withdraw_cache(struct fscache_cache *cache); extern void fscache_withdraw_volume(struct fscache_volume *volume); +extern void fscache_withdraw_cookie(struct fscache_cookie *cookie); extern void fscache_io_error(struct fscache_cache *cache); @@ -85,8 +95,21 @@ extern void fscache_put_cookie(struct fscache_cookie *cookie, enum fscache_cookie_trace where); extern void fscache_end_cookie_access(struct fscache_cookie *cookie, enum fscache_access_trace why); -extern void fscache_set_cookie_state(struct fscache_cookie *cookie, - enum fscache_cookie_state state); +extern void fscache_cookie_lookup_negative(struct fscache_cookie *cookie); +extern void fscache_caching_failed(struct fscache_cookie *cookie); + +/** + * fscache_cookie_state - Read the state of a cookie + * @cookie: The cookie to query + * + * Get the state of a cookie, imposing an ordering between the cookie contents + * and the state value. Paired with fscache_set_cookie_state(). + */ +static inline +enum fscache_cookie_state fscache_cookie_state(struct fscache_cookie *cookie) +{ + return smp_load_acquire(&cookie->state); +} /** * fscache_get_key - Get a pointer to the cookie key diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 1d576bd8112e..030c97bb9c8b 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -68,6 +68,8 @@ enum fscache_access_trace { fscache_access_acquire_volume_end, fscache_access_cache_pin, fscache_access_cache_unpin, + fscache_access_lookup_cookie_end, + fscache_access_lookup_cookie_end_failed, fscache_access_relinquish_volume, fscache_access_relinquish_volume_end, fscache_access_unlive, @@ -124,6 +126,8 @@ enum fscache_access_trace { EM(fscache_access_acquire_volume_end, "END acq_vol") \ EM(fscache_access_cache_pin, "PIN cache ") \ EM(fscache_access_cache_unpin, "UNPIN cache ") \ + EM(fscache_access_lookup_cookie_end, "END lookup ") \ + EM(fscache_access_lookup_cookie_end_failed,"END lookupf") \ EM(fscache_access_relinquish_volume, "BEGIN rlq_vol") \ EM(fscache_access_relinquish_volume_end,"END rlq_vol") \ E_(fscache_access_unlive, "END unlive ") -- cgit v1.2.3-59-g8ed1b From 12bb21a29c19aae50cfad4e2bb5c943108f34a7d Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:53:34 +0100 Subject: fscache: Implement cookie user counting and resource pinning Provide a pair of functions to count the number of users of a cookie (open files, writeback, invalidation, resizing, reads, writes), to obtain and pin resources for the cookie and to prevent culling for the whilst there are users. The first function marks a cookie as being in use: void fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify); The caller should indicate the cookie to use and whether or not the caller is in a context that may modify the cookie (e.g. a file open O_RDWR). If the cookie is not already resourced, fscache will ask the cache backend in the background to do whatever it needs to look up, create or otherwise obtain the resources necessary to access data. This is pinned to the cookie and may not be culled, though it may be withdrawn if the cache as a whole is withdrawn. The second function removes the in-use mark from a cookie and, optionally, updates the coherency data: void fscache_unuse_cookie(struct fscache_cookie *cookie, const void *aux_data, const loff_t *object_size); If non-NULL, the aux_data buffer and/or the object_size will be saved into the cookie and will be set on the backing store when the object is committed. If this removes the last usage on a cookie, the cookie is placed onto an LRU list from which it will be removed and closed after a couple of seconds if it doesn't get reused. This prevents resource overload in the cache - in particular it prevents it from holding too many files open. Changes ======= ver #2: - Fix fscache_unuse_cookie() to use atomic_dec_and_lock() to avoid a potential race if the cookie gets reused before it completes the unusement. - Added missing transition to LRU_DISCARDING state. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819600612.215744.13678350304176542741.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906907567.143852.16979631199380722019.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967106467.1823006.6790864931048582667.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021511674.640689.10084988363699111860.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cookie.c | 218 ++++++++++++++++++++++++++++++++++++++++- fs/fscache/internal.h | 5 + fs/fscache/stats.c | 12 +++ include/linux/fscache.h | 82 +++++++++++++++- include/trace/events/fscache.h | 12 +++ 5 files changed, 327 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 336046de08ee..2f5ee717f2bb 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -15,6 +15,8 @@ struct kmem_cache *fscache_cookie_jar; +static void fscache_cookie_lru_timed_out(struct timer_list *timer); +static void fscache_cookie_lru_worker(struct work_struct *work); static void fscache_cookie_worker(struct work_struct *work); static void fscache_unhash_cookie(struct fscache_cookie *cookie); @@ -22,7 +24,12 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie); static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; static LIST_HEAD(fscache_cookies); static DEFINE_RWLOCK(fscache_cookies_lock); -static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAFWRD"; +static LIST_HEAD(fscache_cookie_lru); +static DEFINE_SPINLOCK(fscache_cookie_lru_lock); +DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); +static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAFUWRD"; +unsigned int fscache_lru_cookie_timeout = 10 * HZ; void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) { @@ -47,6 +54,14 @@ void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) static void fscache_free_cookie(struct fscache_cookie *cookie) { + if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { + spin_lock(&fscache_cookie_lru_lock); + list_del_init(&cookie->commit_link); + spin_unlock(&fscache_cookie_lru_lock); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_removed); + } + if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { fscache_print_cookie(cookie, 'F'); return; @@ -498,6 +513,126 @@ out: fscache_end_volume_access(cookie->volume, cookie, trace); } +/* + * Begin the process of looking up a cookie. We offload the actual process to + * a worker thread. + */ +static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) +{ + if (will_modify) { + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + } + if (!fscache_begin_volume_access(cookie->volume, cookie, + fscache_access_lookup_cookie)) + return false; + + __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); + set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); + return true; +} + +/* + * Start using the cookie for I/O. This prevents the backing object from being + * reaped by VM pressure. + */ +void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) +{ + enum fscache_cookie_state state; + bool queue = false; + + _enter("c=%08x", cookie->debug_id); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to use relinquished cookie\n")) + return; + + spin_lock(&cookie->lock); + + atomic_inc(&cookie->n_active); + +again: + state = fscache_cookie_state(cookie); + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + queue = fscache_begin_lookup(cookie, will_modify); + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + if (will_modify) + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + break; + case FSCACHE_COOKIE_STATE_ACTIVE: + if (will_modify && + !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + queue = true; + } + break; + + case FSCACHE_COOKIE_STATE_FAILED: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + break; + + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + spin_unlock(&cookie->lock); + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + spin_lock(&cookie->lock); + goto again; + + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", state); + break; + } + + spin_unlock(&cookie->lock); + if (queue) + fscache_queue_cookie(cookie, fscache_cookie_get_use_work); + _leave(""); +} +EXPORT_SYMBOL(__fscache_use_cookie); + +static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return; + + cookie->unused_at = jiffies; + spin_lock(&fscache_cookie_lru_lock); + if (list_empty(&cookie->commit_link)) { + fscache_get_cookie(cookie, fscache_cookie_get_lru); + fscache_stat(&fscache_n_cookies_lru); + } + list_move_tail(&cookie->commit_link, &fscache_cookie_lru); + + spin_unlock(&fscache_cookie_lru_lock); + timer_reduce(&fscache_cookie_lru_timer, + jiffies + fscache_lru_cookie_timeout); +} + +/* + * Stop using the cookie for I/O. + */ +void __fscache_unuse_cookie(struct fscache_cookie *cookie, + const void *aux_data, const loff_t *object_size) +{ + if (aux_data || object_size) + __fscache_update_cookie(cookie, aux_data, object_size); + + if (atomic_dec_and_lock(&cookie->n_active, &cookie->lock)) { + fscache_unuse_cookie_locked(cookie); + spin_unlock(&cookie->lock); + } +} +EXPORT_SYMBOL(__fscache_unuse_cookie); + /* * Perform work upon the cookie, such as committing its cache state, * relinquishing it or withdrawing the backing cache. We're protected from the @@ -542,6 +677,12 @@ again_locked: fscache_prepare_to_write(cookie); spin_lock(&cookie->lock); } + if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + wake = true; + goto again_locked; + } fallthrough; case FSCACHE_COOKIE_STATE_FAILED: @@ -561,6 +702,7 @@ again_locked: } break; + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: case FSCACHE_COOKIE_STATE_RELINQUISHING: case FSCACHE_COOKIE_STATE_WITHDRAWING: if (cookie->cache_priv) { @@ -577,6 +719,9 @@ again_locked: FSCACHE_COOKIE_STATE_DROPPED); wake = true; goto out; + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); + break; case FSCACHE_COOKIE_STATE_WITHDRAWING: fscache_see_cookie(cookie, fscache_cookie_see_withdraw); break; @@ -639,6 +784,76 @@ static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) fscache_queue_cookie(cookie, fscache_cookie_get_end_access); } +static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) +{ + fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); + + spin_lock(&cookie->lock); + if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || + time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || + atomic_read(&cookie->n_active) > 0) { + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_removed); + } else { + set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_expired); + _debug("lru c=%x", cookie->debug_id); + __fscache_withdraw_cookie(cookie); + } + + fscache_put_cookie(cookie, fscache_cookie_put_lru); +} + +static void fscache_cookie_lru_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie; + unsigned long unused_at; + + spin_lock(&fscache_cookie_lru_lock); + + while (!list_empty(&fscache_cookie_lru)) { + cookie = list_first_entry(&fscache_cookie_lru, + struct fscache_cookie, commit_link); + unused_at = cookie->unused_at + fscache_lru_cookie_timeout; + if (time_before(jiffies, unused_at)) { + timer_reduce(&fscache_cookie_lru_timer, unused_at); + break; + } + + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + spin_unlock(&fscache_cookie_lru_lock); + fscache_cookie_lru_do_one(cookie); + spin_lock(&fscache_cookie_lru_lock); + } + + spin_unlock(&fscache_cookie_lru_lock); +} + +static void fscache_cookie_lru_timed_out(struct timer_list *timer) +{ + queue_work(fscache_wq, &fscache_cookie_lru_work); +} + +static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) +{ + bool need_put = false; + + if (!list_empty(&cookie->commit_link)) { + spin_lock(&fscache_cookie_lru_lock); + if (!list_empty(&cookie->commit_link)) { + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_dropped); + need_put = true; + } + spin_unlock(&fscache_cookie_lru_lock); + if (need_put) + fscache_put_cookie(cookie, fscache_cookie_put_lru); + } +} + /* * Remove a cookie from the hash table. */ @@ -659,6 +874,7 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie) static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) { + fscache_cookie_drop_from_lru(cookie); __fscache_withdraw_cookie(cookie); } diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index e0d8ef212e82..ca938e00eaa0 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -57,6 +57,7 @@ static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, */ extern struct kmem_cache *fscache_cookie_jar; extern const struct seq_operations fscache_cookies_seq_ops; +extern struct timer_list fscache_cookie_lru_timer; extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, @@ -95,6 +96,10 @@ extern atomic_t fscache_n_volumes; extern atomic_t fscache_n_volumes_collision; extern atomic_t fscache_n_volumes_nomem; extern atomic_t fscache_n_cookies; +extern atomic_t fscache_n_cookies_lru; +extern atomic_t fscache_n_cookies_lru_expired; +extern atomic_t fscache_n_cookies_lru_removed; +extern atomic_t fscache_n_cookies_lru_dropped; extern atomic_t fscache_n_acquires; extern atomic_t fscache_n_acquires_ok; diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index 252e883ae148..5aa4bd9fe207 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -17,6 +17,10 @@ atomic_t fscache_n_volumes; atomic_t fscache_n_volumes_collision; atomic_t fscache_n_volumes_nomem; atomic_t fscache_n_cookies; +atomic_t fscache_n_cookies_lru; +atomic_t fscache_n_cookies_lru_expired; +atomic_t fscache_n_cookies_lru_removed; +atomic_t fscache_n_cookies_lru_dropped; atomic_t fscache_n_acquires; atomic_t fscache_n_acquires_ok; @@ -47,6 +51,14 @@ int fscache_stats_show(struct seq_file *m, void *v) atomic_read(&fscache_n_acquires_ok), atomic_read(&fscache_n_acquires_oom)); + seq_printf(m, "LRU : n=%u exp=%u rmv=%u drp=%u at=%ld\n", + atomic_read(&fscache_n_cookies_lru), + atomic_read(&fscache_n_cookies_lru_expired), + atomic_read(&fscache_n_cookies_lru_removed), + atomic_read(&fscache_n_cookies_lru_dropped), + timer_pending(&fscache_cookie_lru_timer) ? + fscache_cookie_lru_timer.expires - jiffies : 0); + seq_printf(m, "Updates: n=%u\n", atomic_read(&fscache_n_updates)); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 4450d17c11e8..e6c321e5bf73 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -22,12 +22,14 @@ #define fscache_available() (1) #define fscache_volume_valid(volume) (volume) #define fscache_cookie_valid(cookie) (cookie) -#define fscache_cookie_enabled(cookie) (cookie) +#define fscache_resources_valid(cres) ((cres)->cache_priv) +#define fscache_cookie_enabled(cookie) (cookie && !test_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) #else #define __fscache_available (0) #define fscache_available() (0) #define fscache_volume_valid(volume) (0) #define fscache_cookie_valid(cookie) (0) +#define fscache_resources_valid(cres) (false) #define fscache_cookie_enabled(cookie) (0) #endif @@ -46,6 +48,7 @@ enum fscache_cookie_state { FSCACHE_COOKIE_STATE_CREATING, /* The cache object is being created */ FSCACHE_COOKIE_STATE_ACTIVE, /* The cache is active, readable and writable */ FSCACHE_COOKIE_STATE_FAILED, /* The cache failed, withdraw to clear */ + FSCACHE_COOKIE_STATE_LRU_DISCARDING, /* The cookie is being discarded by the LRU */ FSCACHE_COOKIE_STATE_WITHDRAWING, /* The cookie is being withdrawn */ FSCACHE_COOKIE_STATE_RELINQUISHING, /* The cookie is being relinquished */ FSCACHE_COOKIE_STATE_DROPPED, /* The cookie has been dropped */ @@ -147,6 +150,8 @@ extern struct fscache_cookie *__fscache_acquire_cookie( const void *, size_t, const void *, size_t, loff_t); +extern void __fscache_use_cookie(struct fscache_cookie *, bool); +extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); /** @@ -228,6 +233,39 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume, object_size); } +/** + * fscache_use_cookie - Request usage of cookie attached to an object + * @object: Object description + * @will_modify: If cache is expected to be modified locally + * + * Request usage of the cookie attached to an object. The caller should tell + * the cache if the object's contents are about to be modified locally and then + * the cache can apply the policy that has been set to handle this case. + */ +static inline void fscache_use_cookie(struct fscache_cookie *cookie, + bool will_modify) +{ + if (fscache_cookie_valid(cookie)) + __fscache_use_cookie(cookie, will_modify); +} + +/** + * fscache_unuse_cookie - Cease usage of cookie attached to an object + * @object: Object description + * @aux_data: Updated auxiliary data (or NULL) + * @object_size: Revised size of the object (or NULL) + * + * Cease usage of the cookie attached to an object. When the users count + * reaches zero then the cookie relinquishment will be permitted to proceed. + */ +static inline void fscache_unuse_cookie(struct fscache_cookie *cookie, + const void *aux_data, + const loff_t *object_size) +{ + if (fscache_cookie_valid(cookie)) + __fscache_unuse_cookie(cookie, aux_data, object_size); +} + /** * fscache_relinquish_cookie - Return the cookie to the cache, maybe discarding * it @@ -247,4 +285,46 @@ void fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) __fscache_relinquish_cookie(cookie, retire); } +/* + * Find the auxiliary data on a cookie. + */ +static inline void *fscache_get_aux(struct fscache_cookie *cookie) +{ + if (cookie->aux_len <= sizeof(cookie->inline_aux)) + return cookie->inline_aux; + else + return cookie->aux; +} + +/* + * Update the auxiliary data on a cookie. + */ +static inline +void fscache_update_aux(struct fscache_cookie *cookie, + const void *aux_data, const loff_t *object_size) +{ + void *p = fscache_get_aux(cookie); + + if (aux_data && p) + memcpy(p, aux_data, cookie->aux_len); + if (object_size) + cookie->object_size = *object_size; +} + +#ifdef CONFIG_FSCACHE_STATS +extern atomic_t fscache_n_updates; +#endif + +static inline +void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, + const loff_t *object_size) +{ +#ifdef CONFIG_FSCACHE_STATS + atomic_inc(&fscache_n_updates); +#endif + fscache_update_aux(cookie, aux_data, object_size); + smp_wmb(); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 030c97bb9c8b..b0409b1fad23 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -51,13 +51,18 @@ enum fscache_cookie_trace { fscache_cookie_discard, fscache_cookie_get_end_access, fscache_cookie_get_hash_collision, + fscache_cookie_get_lru, + fscache_cookie_get_use_work, fscache_cookie_new_acquire, fscache_cookie_put_hash_collision, + fscache_cookie_put_lru, fscache_cookie_put_over_queued, fscache_cookie_put_relinquish, fscache_cookie_put_withdrawn, fscache_cookie_put_work, fscache_cookie_see_active, + fscache_cookie_see_lru_discard, + fscache_cookie_see_lru_do_one, fscache_cookie_see_relinquish, fscache_cookie_see_withdraw, fscache_cookie_see_work, @@ -68,6 +73,7 @@ enum fscache_access_trace { fscache_access_acquire_volume_end, fscache_access_cache_pin, fscache_access_cache_unpin, + fscache_access_lookup_cookie, fscache_access_lookup_cookie_end, fscache_access_lookup_cookie_end_failed, fscache_access_relinquish_volume, @@ -110,13 +116,18 @@ enum fscache_access_trace { EM(fscache_cookie_discard, "DISCARD ") \ EM(fscache_cookie_get_hash_collision, "GET hcoll") \ EM(fscache_cookie_get_end_access, "GQ endac") \ + EM(fscache_cookie_get_lru, "GET lru ") \ + EM(fscache_cookie_get_use_work, "GQ use ") \ EM(fscache_cookie_new_acquire, "NEW acq ") \ EM(fscache_cookie_put_hash_collision, "PUT hcoll") \ + EM(fscache_cookie_put_lru, "PUT lru ") \ EM(fscache_cookie_put_over_queued, "PQ overq") \ EM(fscache_cookie_put_relinquish, "PUT relnq") \ EM(fscache_cookie_put_withdrawn, "PUT wthdn") \ EM(fscache_cookie_put_work, "PQ work ") \ EM(fscache_cookie_see_active, "- activ") \ + EM(fscache_cookie_see_lru_discard, "- x-lru") \ + EM(fscache_cookie_see_lru_do_one, "- lrudo") \ EM(fscache_cookie_see_relinquish, "- x-rlq") \ EM(fscache_cookie_see_withdraw, "- x-wth") \ E_(fscache_cookie_see_work, "- work ") @@ -126,6 +137,7 @@ enum fscache_access_trace { EM(fscache_access_acquire_volume_end, "END acq_vol") \ EM(fscache_access_cache_pin, "PIN cache ") \ EM(fscache_access_cache_unpin, "UNPIN cache ") \ + EM(fscache_access_lookup_cookie, "BEGIN lookup ") \ EM(fscache_access_lookup_cookie_end, "END lookup ") \ EM(fscache_access_lookup_cookie_end_failed,"END lookupf") \ EM(fscache_access_relinquish_volume, "BEGIN rlq_vol") \ -- cgit v1.2.3-59-g8ed1b From d24af13e2e2358a602740c7817ea90da43d3e740 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 15:53:34 +0100 Subject: fscache: Implement cookie invalidation Add a function to invalidate the cache behind a cookie: void fscache_invalidate(struct fscache_cookie *cookie, const void *aux_data, loff_t size, unsigned int flags) This causes any cached data for the specified cookie to be discarded. If the cookie is marked as being in use, a new cache object will be created if possible and future I/O will use that instead. In-flight I/O should be abandoned (writes) or reconsidered (reads). Each time it is called cookie->inval_counter is incremented and this can be used to detect invalidation at the end of an I/O operation. The coherency data attached to the cookie can be updated and the cookie size should be reset. One flag is available, FSCACHE_INVAL_DIO_WRITE, which should be used to indicate invalidation due to a DIO write on a file. This will temporarily disable caching for this cookie. Changes ======= ver #2: - Should only change to inval state if can get access to cache. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819602231.215744.11206598147269491575.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906909707.143852.18056070560477964891.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967107447.1823006.5945029409592119962.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021512640.640689.11418616313147754172.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cookie.c | 88 +++++++++++++++++++++++++++++++++++++++++- fs/fscache/internal.h | 2 + fs/fscache/stats.c | 5 +++ include/linux/fscache-cache.h | 4 ++ include/linux/fscache.h | 31 +++++++++++++++ include/linux/netfs.h | 1 + include/trace/events/fscache.h | 25 ++++++++++++ 7 files changed, 155 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index 2f5ee717f2bb..a7ea7d1db032 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -19,6 +19,7 @@ static void fscache_cookie_lru_timed_out(struct timer_list *timer); static void fscache_cookie_lru_worker(struct work_struct *work); static void fscache_cookie_worker(struct work_struct *work); static void fscache_unhash_cookie(struct fscache_cookie *cookie); +static void fscache_perform_invalidation(struct fscache_cookie *cookie); #define fscache_cookie_hash_shift 15 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; @@ -28,7 +29,7 @@ static LIST_HEAD(fscache_cookie_lru); static DEFINE_SPINLOCK(fscache_cookie_lru_lock); DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); -static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAFUWRD"; +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; unsigned int fscache_lru_cookie_timeout = 10 * HZ; void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) @@ -236,6 +237,19 @@ void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) } EXPORT_SYMBOL(fscache_cookie_lookup_negative); +/** + * fscache_resume_after_invalidation - Allow I/O to resume after invalidation + * @cookie: The cookie that was invalidated + * + * Tell fscache that invalidation is sufficiently complete that I/O can be + * allowed again. + */ +void fscache_resume_after_invalidation(struct fscache_cookie *cookie) +{ + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); +} +EXPORT_SYMBOL(fscache_resume_after_invalidation); + /** * fscache_caching_failed - Report that a failure stopped caching on a cookie * @cookie: The cookie that was affected @@ -566,6 +580,7 @@ again: set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); break; case FSCACHE_COOKIE_STATE_ACTIVE: + case FSCACHE_COOKIE_STATE_INVALIDATING: if (will_modify && !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); @@ -671,6 +686,11 @@ again_locked: fscache_perform_lookup(cookie); goto again; + case FSCACHE_COOKIE_STATE_INVALIDATING: + spin_unlock(&cookie->lock); + fscache_perform_invalidation(cookie); + goto again; + case FSCACHE_COOKIE_STATE_ACTIVE: if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { spin_unlock(&cookie->lock); @@ -962,6 +982,72 @@ struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, } EXPORT_SYMBOL(fscache_get_cookie); +/* + * Ask the cache to effect invalidation of a cookie. + */ +static void fscache_perform_invalidation(struct fscache_cookie *cookie) +{ + if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) + fscache_caching_failed(cookie); + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); +} + +/* + * Invalidate an object. + */ +void __fscache_invalidate(struct fscache_cookie *cookie, + const void *aux_data, loff_t new_size, + unsigned int flags) +{ + bool is_caching; + + _enter("c=%x", cookie->debug_id); + + fscache_stat(&fscache_n_invalidates); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to invalidate relinquished cookie\n")) + return; + + if ((flags & FSCACHE_INVAL_DIO_WRITE) && + test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) + return; + + spin_lock(&cookie->lock); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_update_aux(cookie, aux_data, &new_size); + cookie->inval_counter++; + trace_fscache_invalidate(cookie, new_size); + + switch (cookie->state) { + case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ + default: + spin_unlock(&cookie->lock); + _leave(" [no %u]", cookie->state); + return; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + spin_unlock(&cookie->lock); + _leave(" [look %x]", cookie->inval_counter); + return; + + case FSCACHE_COOKIE_STATE_ACTIVE: + is_caching = fscache_begin_cookie_access( + cookie, fscache_access_invalidate_cookie); + if (is_caching) + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + + if (is_caching) + fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); + _leave(" [inv]"); + return; + } +} +EXPORT_SYMBOL(__fscache_invalidate); + /* * Generate a list of extant cookies in /proc/fs/fscache/cookies */ diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index ca938e00eaa0..7fb83d216360 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -105,6 +105,8 @@ extern atomic_t fscache_n_acquires; extern atomic_t fscache_n_acquires_ok; extern atomic_t fscache_n_acquires_oom; +extern atomic_t fscache_n_invalidates; + extern atomic_t fscache_n_relinquishes; extern atomic_t fscache_n_relinquishes_retire; extern atomic_t fscache_n_relinquishes_dropped; diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index 5aa4bd9fe207..cdbb672a274f 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -26,6 +26,8 @@ atomic_t fscache_n_acquires; atomic_t fscache_n_acquires_ok; atomic_t fscache_n_acquires_oom; +atomic_t fscache_n_invalidates; + atomic_t fscache_n_updates; EXPORT_SYMBOL(fscache_n_updates); @@ -59,6 +61,9 @@ int fscache_stats_show(struct seq_file *m, void *v) timer_pending(&fscache_cookie_lru_timer) ? fscache_cookie_lru_timer.expires - jiffies : 0); + seq_printf(m, "Invals : n=%u\n", + atomic_read(&fscache_n_invalidates)); + seq_printf(m, "Updates: n=%u\n", atomic_read(&fscache_n_updates)); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index ae6a75976450..1ad56bfd9d72 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -64,6 +64,9 @@ struct fscache_cache_ops { /* Withdraw an object without any cookie access counts held */ void (*withdraw_cookie)(struct fscache_cookie *cookie); + /* Invalidate an object */ + bool (*invalidate_cookie)(struct fscache_cookie *cookie); + /* Prepare to write to a live cache object */ void (*prepare_to_write)(struct fscache_cookie *cookie); }; @@ -96,6 +99,7 @@ extern void fscache_put_cookie(struct fscache_cookie *cookie, extern void fscache_end_cookie_access(struct fscache_cookie *cookie, enum fscache_access_trace why); extern void fscache_cookie_lookup_negative(struct fscache_cookie *cookie); +extern void fscache_resume_after_invalidation(struct fscache_cookie *cookie); extern void fscache_caching_failed(struct fscache_cookie *cookie); /** diff --git a/include/linux/fscache.h b/include/linux/fscache.h index e6c321e5bf73..0f36d1fac237 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -39,6 +39,8 @@ struct fscache_cookie; #define FSCACHE_ADV_WRITE_CACHE 0x00 /* Do cache if written to locally */ #define FSCACHE_ADV_WRITE_NOCACHE 0x02 /* Don't cache if written to locally */ +#define FSCACHE_INVAL_DIO_WRITE 0x01 /* Invalidate due to DIO write */ + /* * Data object state. */ @@ -47,6 +49,7 @@ enum fscache_cookie_state { FSCACHE_COOKIE_STATE_LOOKING_UP, /* The cache object is being looked up */ FSCACHE_COOKIE_STATE_CREATING, /* The cache object is being created */ FSCACHE_COOKIE_STATE_ACTIVE, /* The cache is active, readable and writable */ + FSCACHE_COOKIE_STATE_INVALIDATING, /* The cache is being invalidated */ FSCACHE_COOKIE_STATE_FAILED, /* The cache failed, withdraw to clear */ FSCACHE_COOKIE_STATE_LRU_DISCARDING, /* The cookie is being discarded by the LRU */ FSCACHE_COOKIE_STATE_WITHDRAWING, /* The cookie is being withdrawn */ @@ -153,6 +156,7 @@ extern struct fscache_cookie *__fscache_acquire_cookie( extern void __fscache_use_cookie(struct fscache_cookie *, bool); extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); +extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); /** * fscache_acquire_volume - Register a volume as desiring caching services @@ -327,4 +331,31 @@ void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); } +/** + * fscache_invalidate - Notify cache that an object needs invalidation + * @cookie: The cookie representing the cache object + * @aux_data: The updated auxiliary data for the cookie (may be NULL) + * @size: The revised size of the object. + * @flags: Invalidation flags (FSCACHE_INVAL_*) + * + * Notify the cache that an object is needs to be invalidated and that it + * should abort any retrievals or stores it is doing on the cache. This + * increments inval_counter on the cookie which can be used by the caller to + * reconsider I/O requests as they complete. + * + * If @flags has FSCACHE_INVAL_DIO_WRITE set, this indicates that this is due + * to a direct I/O write and will cause caching to be disabled on this cookie + * until it is completely unused. + * + * See Documentation/filesystems/caching/netfs-api.rst for a complete + * description. + */ +static inline +void fscache_invalidate(struct fscache_cookie *cookie, + const void *aux_data, loff_t size, unsigned int flags) +{ + if (fscache_cookie_enabled(cookie)) + __fscache_invalidate(cookie, aux_data, size, flags); +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 1ea22fc48818..5a46fde65759 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -124,6 +124,7 @@ struct netfs_cache_resources { void *cache_priv; void *cache_priv2; unsigned int debug_id; /* Cookie debug ID */ + unsigned int inval_counter; /* object->inval_counter at begin_op */ }; /* diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index b0409b1fad23..294792881434 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -51,6 +51,7 @@ enum fscache_cookie_trace { fscache_cookie_discard, fscache_cookie_get_end_access, fscache_cookie_get_hash_collision, + fscache_cookie_get_inval_work, fscache_cookie_get_lru, fscache_cookie_get_use_work, fscache_cookie_new_acquire, @@ -73,6 +74,8 @@ enum fscache_access_trace { fscache_access_acquire_volume_end, fscache_access_cache_pin, fscache_access_cache_unpin, + fscache_access_invalidate_cookie, + fscache_access_invalidate_cookie_end, fscache_access_lookup_cookie, fscache_access_lookup_cookie_end, fscache_access_lookup_cookie_end_failed, @@ -116,6 +119,7 @@ enum fscache_access_trace { EM(fscache_cookie_discard, "DISCARD ") \ EM(fscache_cookie_get_hash_collision, "GET hcoll") \ EM(fscache_cookie_get_end_access, "GQ endac") \ + EM(fscache_cookie_get_inval_work, "GQ inval") \ EM(fscache_cookie_get_lru, "GET lru ") \ EM(fscache_cookie_get_use_work, "GQ use ") \ EM(fscache_cookie_new_acquire, "NEW acq ") \ @@ -137,6 +141,8 @@ enum fscache_access_trace { EM(fscache_access_acquire_volume_end, "END acq_vol") \ EM(fscache_access_cache_pin, "PIN cache ") \ EM(fscache_access_cache_unpin, "UNPIN cache ") \ + EM(fscache_access_invalidate_cookie, "BEGIN inval ") \ + EM(fscache_access_invalidate_cookie_end,"END inval ") \ EM(fscache_access_lookup_cookie, "BEGIN lookup ") \ EM(fscache_access_lookup_cookie_end, "END lookup ") \ EM(fscache_access_lookup_cookie_end_failed,"END lookupf") \ @@ -385,6 +391,25 @@ TRACE_EVENT(fscache_relinquish, __entry->n_active, __entry->flags, __entry->retire) ); +TRACE_EVENT(fscache_invalidate, + TP_PROTO(struct fscache_cookie *cookie, loff_t new_size), + + TP_ARGS(cookie, new_size), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(loff_t, new_size ) + ), + + TP_fast_assign( + __entry->cookie = cookie->debug_id; + __entry->new_size = new_size; + ), + + TP_printk("c=%08x sz=%llx", + __entry->cookie, __entry->new_size) + ); + #endif /* _TRACE_FSCACHE_H */ /* This part must be outside protection */ -- cgit v1.2.3-59-g8ed1b From d64f4554dd177c5891c02424a8d9e80590b55b35 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:06:34 +0100 Subject: fscache: Provide a means to begin an operation Provide a function to begin a read operation: int fscache_begin_read_operation( struct netfs_cache_resources *cres, struct fscache_cookie *cookie) This is primarily intended to be called by network filesystems on behalf of netfslib, but may also be called to use the I/O access functions directly. It attaches the resources required by the cache to cres struct from the supplied cookie. This holds access to the cache behind the cookie for the duration of the operation and forces cache withdrawal and cookie invalidation to perform synchronisation on the operation. cres->inval_counter is set from the cookie at this point so that it can be compared at the end of the operation. Note that this does not guarantee that the cache state is fully set up and able to perform I/O immediately; looking up and creation may be left in progress in the background. The operations intended to be called by the network filesystem, such as reading and writing, are expected to wait for the cookie to move to the correct state. This will, however, potentially sleep, waiting for a certain minimum state to be set or for operations such as invalidate to advance far enough that I/O can resume. Also provide a function for the cache to call to wait for the cache object to get to a state where it can be used for certain things: bool fscache_wait_for_operation(struct netfs_cache_resources *cres, enum fscache_want_stage stage); This looks at the cache resources provided by the begin function and waits for them to get to an appropriate stage. There's a choice of wanting just some parameters (FSCACHE_WANT_PARAM) or the ability to do I/O (FSCACHE_WANT_READ or FSCACHE_WANT_WRITE). Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819603692.215744.146724961588817028.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906910672.143852.13856103384424986357.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967110245.1823006.2239170567540431836.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021513617.640689.16627329360866150606.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/Makefile | 1 + fs/fscache/internal.h | 11 +++ fs/fscache/io.c | 151 +++++++++++++++++++++++++++++++++++++++++ include/linux/fscache-cache.h | 11 +++ include/linux/fscache.h | 49 +++++++++++++ include/trace/events/fscache.h | 6 ++ 6 files changed, 229 insertions(+) create mode 100644 fs/fscache/io.c (limited to 'include/linux') diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index bcc79615f93a..afb090ea16c4 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -6,6 +6,7 @@ fscache-y := \ cache.o \ cookie.o \ + io.o \ main.o \ volume.o diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 7fb83d216360..017bf3d346a4 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -70,6 +70,17 @@ static inline void fscache_see_cookie(struct fscache_cookie *cookie, where); } +/* + * io.c + */ +static inline void fscache_end_operation(struct netfs_cache_resources *cres) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + + if (ops) + ops->end_operation(cres); +} + /* * main.c */ diff --git a/fs/fscache/io.c b/fs/fscache/io.c new file mode 100644 index 000000000000..460a43473019 --- /dev/null +++ b/fs/fscache/io.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Cache data I/O routines + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ +#define FSCACHE_DEBUG_LEVEL OPERATION +#include +#include +#include +#include +#include +#include "internal.h" + +/** + * fscache_wait_for_operation - Wait for an object become accessible + * @cres: The cache resources for the operation being performed + * @want_state: The minimum state the object must be at + * + * See if the target cache object is at the specified minimum state of + * accessibility yet, and if not, wait for it. + */ +bool fscache_wait_for_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) +{ + struct fscache_cookie *cookie = fscache_cres_cookie(cres); + enum fscache_cookie_state state; + +again: + if (!fscache_cache_is_live(cookie->volume->cache)) { + _leave(" [broken]"); + return false; + } + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_CREATING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + fallthrough; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != state); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + default: + _leave(" [not live]"); + return false; + } + +ready: + if (!cres->cache_priv2) + return cookie->volume->cache->ops->begin_operation(cres, want_state); + return true; +} +EXPORT_SYMBOL(fscache_wait_for_operation); + +/* + * Begin an I/O operation on the cache, waiting till we reach the right state. + * + * Attaches the resources required to the operation resources record. + */ +static int fscache_begin_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie, + enum fscache_want_state want_state, + enum fscache_access_trace why) +{ + enum fscache_cookie_state state; + long timeo; + bool once_only = false; + + cres->ops = NULL; + cres->cache_priv = cookie; + cres->cache_priv2 = NULL; + cres->debug_id = cookie->debug_id; + cres->inval_counter = cookie->inval_counter; + + if (!fscache_begin_cookie_access(cookie, why)) + return -ENOBUFS; + +again: + spin_lock(&cookie->lock); + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_CREATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", cookie->state); + goto not_live; + default: + goto not_live; + } + +ready: + spin_unlock(&cookie->lock); + if (!cookie->volume->cache->ops->begin_operation(cres, want_state)) + goto failed; + return 0; + +wait_for_file_wrangling: + spin_unlock(&cookie->lock); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + atomic_read(&cookie->n_accesses), + fscache_access_io_wait); + timeo = wait_var_event_timeout(&cookie->state, + fscache_cookie_state(cookie) != state, 20 * HZ); + if (timeo <= 1 && !once_only) { + pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u", + __func__, fscache_cookie_state(cookie), state); + fscache_print_cookie(cookie, 'O'); + once_only = true; + } + goto again; + +not_live: + spin_unlock(&cookie->lock); +failed: + cres->cache_priv = NULL; + cres->ops = NULL; + fscache_end_cookie_access(cookie, fscache_access_io_not_live); + _leave(" = -ENOBUFS"); + return -ENOBUFS; +} + +int __fscache_begin_read_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_read); +} +EXPORT_SYMBOL(__fscache_begin_read_operation); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 1ad56bfd9d72..566497cf5f13 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -67,6 +67,10 @@ struct fscache_cache_ops { /* Invalidate an object */ bool (*invalidate_cookie)(struct fscache_cookie *cookie); + /* Begin an operation for the netfs lib */ + bool (*begin_operation)(struct netfs_cache_resources *cres, + enum fscache_want_state want_state); + /* Prepare to write to a live cache object */ void (*prepare_to_write)(struct fscache_cookie *cookie); }; @@ -101,6 +105,8 @@ extern void fscache_end_cookie_access(struct fscache_cookie *cookie, extern void fscache_cookie_lookup_negative(struct fscache_cookie *cookie); extern void fscache_resume_after_invalidation(struct fscache_cookie *cookie); extern void fscache_caching_failed(struct fscache_cookie *cookie); +extern bool fscache_wait_for_operation(struct netfs_cache_resources *cred, + enum fscache_want_state state); /** * fscache_cookie_state - Read the state of a cookie @@ -129,4 +135,9 @@ static inline void *fscache_get_key(struct fscache_cookie *cookie) return cookie->key; } +static inline struct fscache_cookie *fscache_cres_cookie(struct netfs_cache_resources *cres) +{ + return cres->cache_priv; +} + #endif /* _LINUX_FSCACHE_CACHE_H */ diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 0f36d1fac237..7cdc63c4fe35 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -41,6 +41,12 @@ struct fscache_cookie; #define FSCACHE_INVAL_DIO_WRITE 0x01 /* Invalidate due to DIO write */ +enum fscache_want_state { + FSCACHE_WANT_PARAMS, + FSCACHE_WANT_WRITE, + FSCACHE_WANT_READ, +}; + /* * Data object state. */ @@ -157,6 +163,7 @@ extern void __fscache_use_cookie(struct fscache_cookie *, bool); extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); +extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); /** * fscache_acquire_volume - Register a volume as desiring caching services @@ -358,4 +365,46 @@ void fscache_invalidate(struct fscache_cookie *cookie, __fscache_invalidate(cookie, aux_data, size, flags); } +/** + * fscache_operation_valid - Return true if operations resources are usable + * @cres: The resources to check. + * + * Returns a pointer to the operations table if usable or NULL if not. + */ +static inline +const struct netfs_cache_ops *fscache_operation_valid(const struct netfs_cache_resources *cres) +{ + return fscache_resources_valid(cres) ? cres->ops : NULL; +} + +/** + * fscache_begin_read_operation - Begin a read operation for the netfs lib + * @cres: The cache resources for the read being performed + * @cookie: The cookie representing the cache object + * + * Begin a read operation on behalf of the netfs helper library. @cres + * indicates the cache resources to which the operation state should be + * attached; @cookie indicates the cache object that will be accessed. + * + * This is intended to be called from the ->begin_cache_operation() netfs lib + * operation as implemented by the network filesystem. + * + * @cres->inval_counter is set from @cookie->inval_counter for comparison at + * the end of the operation. This allows invalidation during the operation to + * be detected by the caller. + * + * Returns: + * * 0 - Success + * * -ENOBUFS - No caching available + * * Other error code from the cache, such as -ENOMEM. + */ +static inline +int fscache_begin_read_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + if (fscache_cookie_enabled(cookie)) + return __fscache_begin_read_operation(cres, cookie); + return -ENOBUFS; +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 294792881434..9f78c903b00a 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -76,6 +76,9 @@ enum fscache_access_trace { fscache_access_cache_unpin, fscache_access_invalidate_cookie, fscache_access_invalidate_cookie_end, + fscache_access_io_not_live, + fscache_access_io_read, + fscache_access_io_wait, fscache_access_lookup_cookie, fscache_access_lookup_cookie_end, fscache_access_lookup_cookie_end_failed, @@ -143,6 +146,9 @@ enum fscache_access_trace { EM(fscache_access_cache_unpin, "UNPIN cache ") \ EM(fscache_access_invalidate_cookie, "BEGIN inval ") \ EM(fscache_access_invalidate_cookie_end,"END inval ") \ + EM(fscache_access_io_not_live, "END io_notl") \ + EM(fscache_access_io_read, "BEGIN io_read") \ + EM(fscache_access_io_wait, "WAIT io ") \ EM(fscache_access_lookup_cookie, "BEGIN lookup ") \ EM(fscache_access_lookup_cookie_end, "END lookup ") \ EM(fscache_access_lookup_cookie_end_failed,"END lookupf") \ -- cgit v1.2.3-59-g8ed1b From cdf262f29488e6c3432911ec487ea41918fcbcd7 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 11 Nov 2021 23:14:29 +0000 Subject: fscache: Count data storage objects in a cache Count the data storage objects that are currently allocated in a cache. This is used to pin certain cache structures until cache withdrawal is complete. Three helpers are provided to manage and make use of the count: (1) void fscache_count_object(struct fscache_cache *cache); This should be called by the cache backend to note that an object has been allocated and attached to the cache. (2) void fscache_uncount_object(struct fscache_cache *cache); This should be called by the backend to note that an object has been destroyed. This sends a wakeup event that allows cache withdrawal to proceed if it was waiting for that object. (3) void fscache_wait_for_objects(struct fscache_cache *cache); This can be used by the backend to wait for all outstanding cache object to be destroyed. Each cache's counter is displayed as part of /proc/fs/fscache/caches. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819608594.215744.1812706538117388252.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906911646.143852.168184059935530127.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967111846.1823006.9868154941573671255.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021516219.640689.4934796654308958158.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/cache.c | 2 ++ include/linux/fscache-cache.h | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index 25eac61f1c29..2749933852a9 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -13,6 +13,8 @@ static LIST_HEAD(fscache_caches); DECLARE_RWSEM(fscache_addremove_sem); EXPORT_SYMBOL(fscache_addremove_sem); +DECLARE_WAIT_QUEUE_HEAD(fscache_clearance_waiters); +EXPORT_SYMBOL(fscache_clearance_waiters); static atomic_t fscache_cache_debug_id; diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 566497cf5f13..337335d7a5e2 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -76,6 +76,7 @@ struct fscache_cache_ops { }; extern struct workqueue_struct *fscache_wq; +extern wait_queue_head_t fscache_clearance_waiters; /* * out-of-line cache backend functions @@ -140,4 +141,42 @@ static inline struct fscache_cookie *fscache_cres_cookie(struct netfs_cache_reso return cres->cache_priv; } +/** + * fscache_count_object - Tell fscache that an object has been added + * @cache: The cache to account to + * + * Tell fscache that an object has been added to the cache. This prevents the + * cache from tearing down the cache structure until the object is uncounted. + */ +static inline void fscache_count_object(struct fscache_cache *cache) +{ + atomic_inc(&cache->object_count); +} + +/** + * fscache_uncount_object - Tell fscache that an object has been removed + * @cache: The cache to account to + * + * Tell fscache that an object has been removed from the cache and will no + * longer be accessed. After this point, the cache cookie may be destroyed. + */ +static inline void fscache_uncount_object(struct fscache_cache *cache) +{ + if (atomic_dec_and_test(&cache->object_count)) + wake_up_all(&fscache_clearance_waiters); +} + +/** + * fscache_wait_for_objects - Wait for all objects to be withdrawn + * @cache: The cache to query + * + * Wait for all extant objects in a cache to finish being withdrawn + * and go away. + */ +static inline void fscache_wait_for_objects(struct fscache_cache *cache) +{ + wait_event(fscache_clearance_waiters, + atomic_read(&cache->object_count) == 0); +} + #endif /* _LINUX_FSCACHE_CACHE_H */ -- cgit v1.2.3-59-g8ed1b From 8e7a867bb7309fbf47e8c2a68798b919fc02523f Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 23:06:16 +0100 Subject: fscache: Provide read/write stat counters for the cache Provide read/write stat counters for the cache backend to use. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819609532.215744.10821082637727410554.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906912598.143852.12960327989649429069.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967113830.1823006.3222957649202368162.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021517502.640689.6077928311710357342.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/stats.c | 9 +++++++++ include/linux/fscache-cache.h | 10 ++++++++++ 2 files changed, 19 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index cdbb672a274f..db42beb1ba3f 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -35,6 +35,11 @@ atomic_t fscache_n_relinquishes; atomic_t fscache_n_relinquishes_retire; atomic_t fscache_n_relinquishes_dropped; +atomic_t fscache_n_read; +EXPORT_SYMBOL(fscache_n_read); +atomic_t fscache_n_write; +EXPORT_SYMBOL(fscache_n_write); + /* * display the general statistics */ @@ -72,6 +77,10 @@ int fscache_stats_show(struct seq_file *m, void *v) atomic_read(&fscache_n_relinquishes_retire), atomic_read(&fscache_n_relinquishes_dropped)); + seq_printf(m, "IO : rd=%u wr=%u\n", + atomic_read(&fscache_n_read), + atomic_read(&fscache_n_write)); + netfs_stats_show(m); return 0; } diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 337335d7a5e2..796c8b5c5305 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -179,4 +179,14 @@ static inline void fscache_wait_for_objects(struct fscache_cache *cache) atomic_read(&cache->object_count) == 0); } +#ifdef CONFIG_FSCACHE_STATS +extern atomic_t fscache_n_read; +extern atomic_t fscache_n_write; +#define fscache_count_read() atomic_inc(&fscache_n_read) +#define fscache_count_write() atomic_inc(&fscache_n_write) +#else +#define fscache_count_read() do {} while(0) +#define fscache_count_write() do {} while(0) +#endif + #endif /* _LINUX_FSCACHE_CACHE_H */ -- cgit v1.2.3-59-g8ed1b From ed1235eb78a7421cd0ac2ad09e931f8f07ccdc7c Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 23:10:46 +0100 Subject: fscache: Provide a function to let the netfs update its coherency data Provide a function to let the netfs update its coherency data: void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, const loff_t *object_size); This will update the auxiliary data and/or the size of the object attached to a cookie if either pointer is not-NULL and flag that the disk needs to be updated. Note that fscache_unuse_cookie() also allows this to be done. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819610438.215744.4223265964131424954.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906913530.143852.18150303220217653820.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967117795.1823006.7493373142653442595.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021518440.640689.6369952464473039268.stgit@warthog.procyon.org.uk/ # v4 --- include/linux/fscache.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 7cdc63c4fe35..fc77648c8af6 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -338,6 +338,28 @@ void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); } +/** + * fscache_update_cookie - Request that a cache object be updated + * @cookie: The cookie representing the cache object + * @aux_data: The updated auxiliary data for the cookie (may be NULL) + * @object_size: The current size of the object (may be NULL) + * + * Request an update of the index data for the cache object associated with the + * cookie. The auxiliary data on the cookie will be updated first if @aux_data + * is set and the object size will be updated and the object possibly trimmed + * if @object_size is set. + * + * See Documentation/filesystems/caching/netfs-api.rst for a complete + * description. + */ +static inline +void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, + const loff_t *object_size) +{ + if (fscache_cookie_enabled(cookie)) + __fscache_update_cookie(cookie, aux_data, object_size); +} + /** * fscache_invalidate - Notify cache that an object needs invalidation * @cookie: The cookie representing the cache object -- cgit v1.2.3-59-g8ed1b From 3a11b3a86366ccbf0818b088ffecadf8b2d61177 Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 14 Sep 2021 09:47:45 +0100 Subject: netfs: Pass more information on how to deal with a hole in the cache Pass more information to the cache on how to deal with a hole if it encounters one when trying to read from the cache. Three options are provided: (1) NETFS_READ_HOLE_IGNORE. Read the hole along with the data, assuming it to be a punched-out extent by the backing filesystem. (2) NETFS_READ_HOLE_CLEAR. If there's a hole, erase the requested region of the cache and clear the read buffer. (3) NETFS_READ_HOLE_FAIL. Fail the read if a hole is detected. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819612321.215744.9738308885948264476.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906914460.143852.6284247083607910189.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967119923.1823006.15637375885194297582.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021519762.640689.16994364383313159319.stgit@warthog.procyon.org.uk/ # v4 --- fs/netfs/read_helper.c | 8 ++++---- include/linux/netfs.h | 11 ++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/netfs/read_helper.c b/fs/netfs/read_helper.c index 9dd76b8914f2..6169659857b3 100644 --- a/fs/netfs/read_helper.c +++ b/fs/netfs/read_helper.c @@ -170,7 +170,7 @@ static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error */ static void netfs_read_from_cache(struct netfs_read_request *rreq, struct netfs_read_subrequest *subreq, - bool seek_data) + enum netfs_read_from_hole read_hole) { struct netfs_cache_resources *cres = &rreq->cache_resources; struct iov_iter iter; @@ -180,7 +180,7 @@ static void netfs_read_from_cache(struct netfs_read_request *rreq, subreq->start + subreq->transferred, subreq->len - subreq->transferred); - cres->ops->read(cres, subreq->start, &iter, seek_data, + cres->ops->read(cres, subreq->start, &iter, read_hole, netfs_cache_read_terminated, subreq); } @@ -461,7 +461,7 @@ static void netfs_rreq_short_read(struct netfs_read_request *rreq, netfs_get_read_subrequest(subreq); atomic_inc(&rreq->nr_rd_ops); if (subreq->source == NETFS_READ_FROM_CACHE) - netfs_read_from_cache(rreq, subreq, true); + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR); else netfs_read_from_server(rreq, subreq); } @@ -789,7 +789,7 @@ static bool netfs_rreq_submit_slice(struct netfs_read_request *rreq, netfs_read_from_server(rreq, subreq); break; case NETFS_READ_FROM_CACHE: - netfs_read_from_cache(rreq, subreq, false); + netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE); break; default: BUG(); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 5a46fde65759..b46c39d98bbd 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -196,6 +196,15 @@ struct netfs_read_request_ops { void (*cleanup)(struct address_space *mapping, void *netfs_priv); }; +/* + * How to handle reading from a hole. + */ +enum netfs_read_from_hole { + NETFS_READ_HOLE_IGNORE, + NETFS_READ_HOLE_CLEAR, + NETFS_READ_HOLE_FAIL, +}; + /* * Table of operations for access to a cache. This is obtained by * rreq->ops->begin_cache_operation(). @@ -208,7 +217,7 @@ struct netfs_cache_ops { int (*read)(struct netfs_cache_resources *cres, loff_t start_pos, struct iov_iter *iter, - bool seek_data, + enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, void *term_func_priv); -- cgit v1.2.3-59-g8ed1b From 9af1c6c3089b294ffa240e0fbba356666698b6d0 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:06:34 +0100 Subject: fscache: Implement raw I/O interface Provide a pair of functions to perform raw I/O on the cache. The first function allows an arbitrary asynchronous direct-IO read to be made against a cache object, though the read should be aligned and sized appropriately for the backing device: int fscache_read(struct netfs_cache_resources *cres, loff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, void *term_func_priv); The cache resources must have been previously initialised by fscache_begin_read_operation(). A read operation is sent to the backing filesystem, starting at start_pos within the file. The size of the read is specified by the iterator, as is the location of the output buffer. If there is a hole in the data it can be ignored and left to the backing filesystem to deal with (NETFS_READ_HOLE_IGNORE), a hole at the beginning can be skipped over and the buffer padded with zeros (NETFS_READ_HOLE_CLEAR) or -ENODATA can be given (NETFS_READ_HOLE_FAIL). If term_func is not NULL, the operation may be performed asynchronously. Upon completion, successful or otherwise, (*term_func)() will be called and passed term_func_priv, along with an error or the amount of data transferred. If the op is run asynchronously, fscache_read() will return -EIOCBQUEUED. The second function allows an arbitrary asynchronous direct-IO write to be made against a cache object, though the write should be aligned and sized appropriately for the backing device: int fscache_write(struct netfs_cache_resources *cres, loff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv); This works in very similar way to fscache_read(), except that there's no need to deal with holes (they're just overwritten). The caller is responsible for preventing concurrent overlapping writes. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819613224.215744.7877577215582621254.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906915386.143852.16936177636106480724.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967122632.1823006.7487049517698562172.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021521420.640689.12747258780542678309.stgit@warthog.procyon.org.uk/ # v4 --- include/linux/fscache.h | 74 ++++++++++++++++++++++++++++++++++++++++++ include/trace/events/fscache.h | 2 ++ 2 files changed, 76 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fscache.h b/include/linux/fscache.h index fc77648c8af6..ae753cae0fdd 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -429,4 +429,78 @@ int fscache_begin_read_operation(struct netfs_cache_resources *cres, return -ENOBUFS; } +/** + * fscache_read - Start a read from the cache. + * @cres: The cache resources to use + * @start_pos: The beginning file offset in the cache file + * @iter: The buffer to fill - and also the length + * @read_hole: How to handle a hole in the data. + * @term_func: The function to call upon completion + * @term_func_priv: The private data for @term_func + * + * Start a read from the cache. @cres indicates the cache object to read from + * and must be obtained by a call to fscache_begin_operation() beforehand. + * + * The data is read into the iterator, @iter, and that also indicates the size + * of the operation. @start_pos is the start position in the file, though if + * @seek_data is set appropriately, the cache can use SEEK_DATA to find the + * next piece of data, writing zeros for the hole into the iterator. + * + * Upon termination of the operation, @term_func will be called and supplied + * with @term_func_priv plus the amount of data written, if successful, or the + * error code otherwise. + * + * @read_hole indicates how a partially populated region in the cache should be + * handled. It can be one of a number of settings: + * + * NETFS_READ_HOLE_IGNORE - Just try to read (may return a short read). + * + * NETFS_READ_HOLE_CLEAR - Seek for data, clearing the part of the buffer + * skipped over, then do as for IGNORE. + * + * NETFS_READ_HOLE_FAIL - Give ENODATA if we encounter a hole. + */ +static inline +int fscache_read(struct netfs_cache_resources *cres, + loff_t start_pos, + struct iov_iter *iter, + enum netfs_read_from_hole read_hole, + netfs_io_terminated_t term_func, + void *term_func_priv) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + return ops->read(cres, start_pos, iter, read_hole, + term_func, term_func_priv); +} + +/** + * fscache_write - Start a write to the cache. + * @cres: The cache resources to use + * @start_pos: The beginning file offset in the cache file + * @iter: The data to write - and also the length + * @term_func: The function to call upon completion + * @term_func_priv: The private data for @term_func + * + * Start a write to the cache. @cres indicates the cache object to write to and + * must be obtained by a call to fscache_begin_operation() beforehand. + * + * The data to be written is obtained from the iterator, @iter, and that also + * indicates the size of the operation. @start_pos is the start position in + * the file. + * + * Upon termination of the operation, @term_func will be called and supplied + * with @term_func_priv plus the amount of data written, if successful, or the + * error code otherwise. + */ +static inline +int fscache_write(struct netfs_cache_resources *cres, + loff_t start_pos, + struct iov_iter *iter, + netfs_io_terminated_t term_func, + void *term_func_priv) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + return ops->write(cres, start_pos, iter, term_func, term_func_priv); +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 9f78c903b00a..2459d75659cf 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -79,6 +79,7 @@ enum fscache_access_trace { fscache_access_io_not_live, fscache_access_io_read, fscache_access_io_wait, + fscache_access_io_write, fscache_access_lookup_cookie, fscache_access_lookup_cookie_end, fscache_access_lookup_cookie_end_failed, @@ -149,6 +150,7 @@ enum fscache_access_trace { EM(fscache_access_io_not_live, "END io_notl") \ EM(fscache_access_io_read, "BEGIN io_read") \ EM(fscache_access_io_wait, "WAIT io ") \ + EM(fscache_access_io_write, "BEGIN io_writ") \ EM(fscache_access_lookup_cookie, "BEGIN lookup ") \ EM(fscache_access_lookup_cookie_end, "END lookup ") \ EM(fscache_access_lookup_cookie_end_failed,"END lookupf") \ -- cgit v1.2.3-59-g8ed1b From b6e16652d6c0e4f9e9b120f66966ec153f0623fc Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:06:34 +0100 Subject: fscache: Implement higher-level write I/O interface Provide a higher-level function than fscache_write() to perform a write from an inode's pagecache to the cache, whilst fending off concurrent writes by means of the PG_fscache mark on a page: void fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, loff_t start, size_t len, loff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool caching); If caching is false, this function does nothing except call (*term_func)() if given. It assumes that, in such a case, PG_fscache will not have been set on the pages. Otherwise, if caching is true, this function requires the source pages to have had PG_fscache set on them before calling. start and len define the region of the file to be modified and i_size indicates the new file size. The source pages are extracted from the mapping. term_func and term_func_priv work as for fscache_write(). The PG_fscache marks will be cleared at the end of the operation, before term_func is called or the function otherwise returns. There is an additonal helper function to clear the PG_fscache bits from a range of pages: void fscache_clear_page_bits(struct fscache_cookie *cookie, struct address_space *mapping, loff_t start, size_t len, bool caching); If caching is true, the pages to be managed are expected to be located on mapping in the range defined by start and len. If caching is false, it does nothing. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819614155.215744.5528123235123721230.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906916346.143852.15632773570362489926.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967123599.1823006.12946816026724657428.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021522672.640689.4381958316198807813.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/io.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/fscache.h | 63 +++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) (limited to 'include/linux') diff --git a/fs/fscache/io.c b/fs/fscache/io.c index 460a43473019..74cde7acf434 100644 --- a/fs/fscache/io.c +++ b/fs/fscache/io.c @@ -149,3 +149,107 @@ int __fscache_begin_read_operation(struct netfs_cache_resources *cres, fscache_access_io_read); } EXPORT_SYMBOL(__fscache_begin_read_operation); + +struct fscache_write_request { + struct netfs_cache_resources cache_resources; + struct address_space *mapping; + loff_t start; + size_t len; + bool set_bits; + netfs_io_terminated_t term_func; + void *term_func_priv; +}; + +void __fscache_clear_page_bits(struct address_space *mapping, + loff_t start, size_t len) +{ + pgoff_t first = start / PAGE_SIZE; + pgoff_t last = (start + len - 1) / PAGE_SIZE; + struct page *page; + + if (len) { + XA_STATE(xas, &mapping->i_pages, first); + + rcu_read_lock(); + xas_for_each(&xas, page, last) { + end_page_fscache(page); + } + rcu_read_unlock(); + } +} +EXPORT_SYMBOL(__fscache_clear_page_bits); + +/* + * Deal with the completion of writing the data to the cache. + */ +static void fscache_wreq_done(void *priv, ssize_t transferred_or_error, + bool was_async) +{ + struct fscache_write_request *wreq = priv; + + fscache_clear_page_bits(fscache_cres_cookie(&wreq->cache_resources), + wreq->mapping, wreq->start, wreq->len, + wreq->set_bits); + + if (wreq->term_func) + wreq->term_func(wreq->term_func_priv, transferred_or_error, + was_async); + fscache_end_operation(&wreq->cache_resources); + kfree(wreq); +} + +void __fscache_write_to_cache(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, loff_t i_size, + netfs_io_terminated_t term_func, + void *term_func_priv, + bool cond) +{ + struct fscache_write_request *wreq; + struct netfs_cache_resources *cres; + struct iov_iter iter; + int ret = -ENOBUFS; + + if (len == 0) + goto abandon; + + _enter("%llx,%zx", start, len); + + wreq = kzalloc(sizeof(struct fscache_write_request), GFP_NOFS); + if (!wreq) + goto abandon; + wreq->mapping = mapping; + wreq->start = start; + wreq->len = len; + wreq->set_bits = cond; + wreq->term_func = term_func; + wreq->term_func_priv = term_func_priv; + + cres = &wreq->cache_resources; + if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_write) < 0) + goto abandon_free; + + ret = cres->ops->prepare_write(cres, &start, &len, i_size, false); + if (ret < 0) + goto abandon_end; + + /* TODO: Consider clearing page bits now for space the write isn't + * covering. This is more complicated than it appears when THPs are + * taken into account. + */ + + iov_iter_xarray(&iter, WRITE, &mapping->i_pages, start, len); + fscache_write(cres, start, &iter, fscache_wreq_done, wreq); + return; + +abandon_end: + return fscache_wreq_done(wreq, ret, false); +abandon_free: + kfree(wreq); +abandon: + fscache_clear_page_bits(cookie, mapping, start, len, cond); + if (term_func) + term_func(term_func_priv, ret, false); +} +EXPORT_SYMBOL(__fscache_write_to_cache); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index ae753cae0fdd..9d469613e16c 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -165,6 +165,11 @@ extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); +extern void __fscache_write_to_cache(struct fscache_cookie *, struct address_space *, + loff_t, size_t, loff_t, netfs_io_terminated_t, void *, + bool); +extern void __fscache_clear_page_bits(struct address_space *, loff_t, size_t); + /** * fscache_acquire_volume - Register a volume as desiring caching services * @volume_key: An identification string for the volume @@ -503,4 +508,62 @@ int fscache_write(struct netfs_cache_resources *cres, return ops->write(cres, start_pos, iter, term_func, term_func_priv); } +/** + * fscache_clear_page_bits - Clear the PG_fscache bits from a set of pages + * @cookie: The cookie representing the cache object + * @mapping: The netfs inode to use as the source + * @start: The start position in @mapping + * @len: The amount of data to unlock + * @caching: If PG_fscache has been set + * + * Clear the PG_fscache flag from a sequence of pages and wake up anyone who's + * waiting. + */ +static inline void fscache_clear_page_bits(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, + bool caching) +{ + if (caching) + __fscache_clear_page_bits(mapping, start, len); +} + +/** + * fscache_write_to_cache - Save a write to the cache and clear PG_fscache + * @cookie: The cookie representing the cache object + * @mapping: The netfs inode to use as the source + * @start: The start position in @mapping + * @len: The amount of data to write back + * @i_size: The new size of the inode + * @term_func: The function to call upon completion + * @term_func_priv: The private data for @term_func + * @caching: If PG_fscache has been set + * + * Helper function for a netfs to write dirty data from an inode into the cache + * object that's backing it. + * + * @start and @len describe the range of the data. This does not need to be + * page-aligned, but to satisfy DIO requirements, the cache may expand it up to + * the page boundaries on either end. All the pages covering the range must be + * marked with PG_fscache. + * + * If given, @term_func will be called upon completion and supplied with + * @term_func_priv. Note that the PG_fscache flags will have been cleared by + * this point, so the netfs must retain its own pin on the mapping. + */ +static inline void fscache_write_to_cache(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, loff_t i_size, + netfs_io_terminated_t term_func, + void *term_func_priv, + bool caching) +{ + if (caching) + __fscache_write_to_cache(cookie, mapping, start, len, i_size, + term_func, term_func_priv, caching); + else if (term_func) + term_func(term_func_priv, -ENOBUFS, false); + +} + #endif /* _LINUX_FSCACHE_H */ -- cgit v1.2.3-59-g8ed1b From 08276bdae68b022a7726edf7416b6748e3df5395 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 23:50:01 +0100 Subject: vfs, fscache: Implement pinning of cache usage for writeback Cachefiles has a problem in that it needs to keep the backing file for a cookie open whilst there are local modifications pending that need to be written to it. However, we don't want to keep the file open indefinitely, as that causes EMFILE/ENFILE/ENOMEM problems. Reopening the cache file, however, is a problem if this is being done due to writeback triggered by exit(). Some filesystems will oops if we try to open a file in that context because they want to access current->fs or other resources that have already been dismantled. To get around this, I added the following: (1) An inode flag, I_PINNING_FSCACHE_WB, to be set on a network filesystem inode to indicate that we have a usage count on the cookie caching that inode. (2) A flag in struct writeback_control, unpinned_fscache_wb, that is set when __writeback_single_inode() clears the last dirty page from i_pages - at which point it clears I_PINNING_FSCACHE_WB and sets this flag. This has to be done here so that clearing I_PINNING_FSCACHE_WB can be done atomically with the check of PAGECACHE_TAG_DIRTY that clears I_DIRTY_PAGES. (3) A function, fscache_set_page_dirty(), which if it is not set, sets I_PINNING_FSCACHE_WB and calls fscache_use_cookie() to pin the cache resources. (4) A function, fscache_unpin_writeback(), to be called by ->write_inode() to unuse the cookie. (5) A function, fscache_clear_inode_writeback(), to be called when the inode is evicted, before clear_inode() is called. This cleans up any lingering I_PINNING_FSCACHE_WB. The network filesystem can then use these tools to make sure that fscache_write_to_cache() can write locally modified data to the cache as well as to the server. For the future, I'm working on write helpers for netfs lib that should allow this facility to be removed by keeping track of the dirty regions separately - but that's incomplete at the moment and is also going to be affected by folios, one way or another, since it deals with pages Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819615157.215744.17623791756928043114.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906917856.143852.8224898306177154573.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967124567.1823006.14188359004568060298.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021524705.640689.17824932021727663017.stgit@warthog.procyon.org.uk/ # v4 --- fs/fs-writeback.c | 8 ++++++++ fs/fscache/io.c | 38 ++++++++++++++++++++++++++++++++++++++ include/linux/fs.h | 3 +++ include/linux/fscache.h | 41 +++++++++++++++++++++++++++++++++++++++++ include/linux/writeback.h | 1 + 5 files changed, 91 insertions(+) (limited to 'include/linux') diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 67f0e88eed01..8294a60ce323 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1666,6 +1666,13 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) inode->i_state |= I_DIRTY_PAGES; + else if (unlikely(inode->i_state & I_PINNING_FSCACHE_WB)) { + if (!(inode->i_state & I_DIRTY_PAGES)) { + inode->i_state &= ~I_PINNING_FSCACHE_WB; + wbc->unpinned_fscache_wb = true; + dirty |= I_PINNING_FSCACHE_WB; /* Cause write_inode */ + } + } spin_unlock(&inode->i_lock); @@ -1675,6 +1682,7 @@ __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) if (ret == 0) ret = err; } + wbc->unpinned_fscache_wb = false; trace_writeback_single_inode(inode, wbc, nr_to_write); return ret; } diff --git a/fs/fscache/io.c b/fs/fscache/io.c index 74cde7acf434..e9e5d6758ea8 100644 --- a/fs/fscache/io.c +++ b/fs/fscache/io.c @@ -150,6 +150,44 @@ int __fscache_begin_read_operation(struct netfs_cache_resources *cres, } EXPORT_SYMBOL(__fscache_begin_read_operation); +/** + * fscache_set_page_dirty - Mark page dirty and pin a cache object for writeback + * @page: The page being dirtied + * @cookie: The cookie referring to the cache object + * + * Set the dirty flag on a page and pin an in-use cache object in memory when + * dirtying a page so that writeback can later write to it. This is intended + * to be called from the filesystem's ->set_page_dirty() method. + * + * Returns 1 if PG_dirty was set on the page, 0 otherwise. + */ +int fscache_set_page_dirty(struct page *page, struct fscache_cookie *cookie) +{ + struct inode *inode = page->mapping->host; + bool need_use = false; + + _enter(""); + + if (!__set_page_dirty_nobuffers(page)) + return 0; + if (!fscache_cookie_valid(cookie)) + return 1; + + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + spin_lock(&inode->i_lock); + if (!(inode->i_state & I_PINNING_FSCACHE_WB)) { + inode->i_state |= I_PINNING_FSCACHE_WB; + need_use = true; + } + spin_unlock(&inode->i_lock); + + if (need_use) + fscache_use_cookie(cookie, true); + } + return 1; +} +EXPORT_SYMBOL(fscache_set_page_dirty); + struct fscache_write_request { struct netfs_cache_resources cache_resources; struct address_space *mapping; diff --git a/include/linux/fs.h b/include/linux/fs.h index bbf812ce89a8..2c0b8e77d9ab 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2418,6 +2418,8 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, * Used to detect that mark_inode_dirty() should not move * inode between dirty lists. * + * I_PINNING_FSCACHE_WB Inode is pinning an fscache object for writeback. + * * Q: What is the difference between I_WILL_FREE and I_FREEING? */ #define I_DIRTY_SYNC (1 << 0) @@ -2440,6 +2442,7 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, #define I_CREATING (1 << 15) #define I_DONTCACHE (1 << 16) #define I_SYNC_QUEUED (1 << 17) +#define I_PINNING_FSCACHE_WB (1 << 18) #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC) #define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES) diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 9d469613e16c..18e725671594 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -16,6 +16,7 @@ #include #include +#include #if defined(CONFIG_FSCACHE) || defined(CONFIG_FSCACHE_MODULE) #define __fscache_available (1) @@ -566,4 +567,44 @@ static inline void fscache_write_to_cache(struct fscache_cookie *cookie, } +#if __fscache_available +extern int fscache_set_page_dirty(struct page *page, struct fscache_cookie *cookie); +#else +#define fscache_set_page_dirty(PAGE, COOKIE) (__set_page_dirty_nobuffers((PAGE))) +#endif + +/** + * fscache_unpin_writeback - Unpin writeback resources + * @wbc: The writeback control + * @cookie: The cookie referring to the cache object + * + * Unpin the writeback resources pinned by fscache_set_page_dirty(). This is + * intended to be called by the netfs's ->write_inode() method. + */ +static inline void fscache_unpin_writeback(struct writeback_control *wbc, + struct fscache_cookie *cookie) +{ + if (wbc->unpinned_fscache_wb) + fscache_unuse_cookie(cookie, NULL, NULL); +} + +/** + * fscache_clear_inode_writeback - Clear writeback resources pinned by an inode + * @cookie: The cookie referring to the cache object + * @inode: The inode to clean up + * @aux: Auxiliary data to apply to the inode + * + * Clear any writeback resources held by an inode when the inode is evicted. + * This must be called before clear_inode() is called. + */ +static inline void fscache_clear_inode_writeback(struct fscache_cookie *cookie, + struct inode *inode, + const void *aux) +{ + if (inode->i_state & I_PINNING_FSCACHE_WB) { + loff_t i_size = i_size_read(inode); + fscache_unuse_cookie(cookie, aux, &i_size); + } +} + #endif /* _LINUX_FSCACHE_H */ diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 3bfd487d1dd2..fec248ab1fec 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -68,6 +68,7 @@ struct writeback_control { unsigned for_reclaim:1; /* Invoked from the page allocator */ unsigned range_cyclic:1; /* range_start is cyclic */ unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ + unsigned unpinned_fscache_wb:1; /* Cleared I_PINNING_FSCACHE_WB */ /* * When writeback IOs are bounced through async layers, only the -- cgit v1.2.3-59-g8ed1b From 1f67e6d0b18853c641d861a671f46a4964a88510 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:06:34 +0100 Subject: fscache: Provide a function to note the release of a page Provide a function to be called from a network filesystem's releasepage method to indicate that a page has been released that might have been a reflection of data upon the server - and now that data must be reloaded from the server or the cache. This is used to end an optimisation for empty files, in particular files that have just been created locally, whereby we know there cannot yet be any data that we would need to read from the server or the cache. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819617128.215744.4725572296135656508.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906920354.143852.7511819614661372008.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967128061.1823006.611781655060034988.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021525963.640689.9264556596205140044.stgit@warthog.procyon.org.uk/ # v4 --- include/linux/fscache.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 18e725671594..28ce258c1f87 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -607,4 +607,20 @@ static inline void fscache_clear_inode_writeback(struct fscache_cookie *cookie, } } +/** + * fscache_note_page_release - Note that a netfs page got released + * @cookie: The cookie corresponding to the file + * + * Note that a page that has been copied to the cache has been released. This + * means that future reads will need to look in the cache to see if it's there. + */ +static inline +void fscache_note_page_release(struct fscache_cookie *cookie) +{ + if (cookie && + test_bit(FSCACHE_COOKIE_HAVE_DATA, &cookie->flags) && + test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) + clear_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); +} + #endif /* _LINUX_FSCACHE_H */ -- cgit v1.2.3-59-g8ed1b From 16a96bdf92d5af06f9fa6a01a4b08e2fdfed2e5b Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 20 Oct 2021 14:06:34 +0100 Subject: fscache: Provide a function to resize a cookie Provide a function to change the size of the storage attached to a cookie, to match the size of the file being cached when it's changed by truncate or fallocate: void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size); This acts synchronously and is expected to run under the inode lock of the caller. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819621839.215744.7895597119803515402.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906922387.143852.16394459879816147793.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967128998.1823006.10740669081985775576.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021527861.640689.3466382085497236267.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/internal.h | 3 +++ fs/fscache/io.c | 25 +++++++++++++++++++++++++ fs/fscache/stats.c | 9 +++++++-- include/linux/fscache-cache.h | 4 ++++ include/linux/fscache.h | 18 ++++++++++++++++++ include/trace/events/fscache.h | 25 +++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h index 017bf3d346a4..f121c21590dc 100644 --- a/fs/fscache/internal.h +++ b/fs/fscache/internal.h @@ -122,6 +122,9 @@ extern atomic_t fscache_n_relinquishes; extern atomic_t fscache_n_relinquishes_retire; extern atomic_t fscache_n_relinquishes_dropped; +extern atomic_t fscache_n_resizes; +extern atomic_t fscache_n_resizes_null; + static inline void fscache_stat(atomic_t *stat) { atomic_inc(stat); diff --git a/fs/fscache/io.c b/fs/fscache/io.c index e9e5d6758ea8..bed7628a5a9d 100644 --- a/fs/fscache/io.c +++ b/fs/fscache/io.c @@ -291,3 +291,28 @@ abandon: term_func(term_func_priv, ret, false); } EXPORT_SYMBOL(__fscache_write_to_cache); + +/* + * Change the size of a backing object. + */ +void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +{ + struct netfs_cache_resources cres; + + trace_fscache_resize(cookie, new_size); + if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_resize) == 0) { + fscache_stat(&fscache_n_resizes); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + + /* We cannot defer a resize as we need to do it inside the + * netfs's inode lock so that we're serialised with respect to + * writes. + */ + cookie->volume->cache->ops->resize_cookie(&cres, new_size); + fscache_end_operation(&cres); + } else { + fscache_stat(&fscache_n_resizes_null); + } +} +EXPORT_SYMBOL(__fscache_resize_cookie); diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index db42beb1ba3f..798ee68b3e9d 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -35,6 +35,9 @@ atomic_t fscache_n_relinquishes; atomic_t fscache_n_relinquishes_retire; atomic_t fscache_n_relinquishes_dropped; +atomic_t fscache_n_resizes; +atomic_t fscache_n_resizes_null; + atomic_t fscache_n_read; EXPORT_SYMBOL(fscache_n_read); atomic_t fscache_n_write; @@ -69,8 +72,10 @@ int fscache_stats_show(struct seq_file *m, void *v) seq_printf(m, "Invals : n=%u\n", atomic_read(&fscache_n_invalidates)); - seq_printf(m, "Updates: n=%u\n", - atomic_read(&fscache_n_updates)); + seq_printf(m, "Updates: n=%u rsz=%u rsn=%u\n", + atomic_read(&fscache_n_updates), + atomic_read(&fscache_n_resizes), + atomic_read(&fscache_n_resizes_null)); seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", atomic_read(&fscache_n_relinquishes), diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 796c8b5c5305..3fa4902dc87c 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -64,6 +64,10 @@ struct fscache_cache_ops { /* Withdraw an object without any cookie access counts held */ void (*withdraw_cookie)(struct fscache_cookie *cookie); + /* Change the size of a data object */ + void (*resize_cookie)(struct netfs_cache_resources *cres, + loff_t new_size); + /* Invalidate an object */ bool (*invalidate_cookie)(struct fscache_cookie *cookie); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 28ce258c1f87..86b1c0db1de5 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -163,6 +163,7 @@ extern struct fscache_cookie *__fscache_acquire_cookie( extern void __fscache_use_cookie(struct fscache_cookie *, bool); extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); +extern void __fscache_resize_cookie(struct fscache_cookie *, loff_t); extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); @@ -366,6 +367,23 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, __fscache_update_cookie(cookie, aux_data, object_size); } +/** + * fscache_resize_cookie - Request that a cache object be resized + * @cookie: The cookie representing the cache object + * @new_size: The new size of the object (may be NULL) + * + * Request that the size of an object be changed. + * + * See Documentation/filesystems/caching/netfs-api.txt for a complete + * description. + */ +static inline +void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +{ + if (fscache_cookie_enabled(cookie)) + __fscache_resize_cookie(cookie, new_size); +} + /** * fscache_invalidate - Notify cache that an object needs invalidation * @cookie: The cookie representing the cache object diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index 2459d75659cf..5fa37a8b4ec7 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -78,6 +78,7 @@ enum fscache_access_trace { fscache_access_invalidate_cookie_end, fscache_access_io_not_live, fscache_access_io_read, + fscache_access_io_resize, fscache_access_io_wait, fscache_access_io_write, fscache_access_lookup_cookie, @@ -149,6 +150,7 @@ enum fscache_access_trace { EM(fscache_access_invalidate_cookie_end,"END inval ") \ EM(fscache_access_io_not_live, "END io_notl") \ EM(fscache_access_io_read, "BEGIN io_read") \ + EM(fscache_access_io_resize, "BEGIN io_resz") \ EM(fscache_access_io_wait, "WAIT io ") \ EM(fscache_access_io_write, "BEGIN io_writ") \ EM(fscache_access_lookup_cookie, "BEGIN lookup ") \ @@ -418,6 +420,29 @@ TRACE_EVENT(fscache_invalidate, __entry->cookie, __entry->new_size) ); +TRACE_EVENT(fscache_resize, + TP_PROTO(struct fscache_cookie *cookie, loff_t new_size), + + TP_ARGS(cookie, new_size), + + TP_STRUCT__entry( + __field(unsigned int, cookie ) + __field(loff_t, old_size ) + __field(loff_t, new_size ) + ), + + TP_fast_assign( + __entry->cookie = cookie->debug_id; + __entry->old_size = cookie->object_size; + __entry->new_size = new_size; + ), + + TP_printk("c=%08x os=%08llx sz=%08llx", + __entry->cookie, + __entry->old_size, + __entry->new_size) + ); + #endif /* _TRACE_FSCACHE_H */ /* This part must be outside protection */ -- cgit v1.2.3-59-g8ed1b From 1bd9c4e4f0494915b2391f373d25096579f835ff Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 18 Nov 2021 08:58:08 +0000 Subject: vfs, cachefiles: Mark a backing file in use with an inode flag Use an inode flag, S_KERNEL_FILE, to mark that a backing file is in use by the kernel to prevent cachefiles or other kernel services from interfering with that file. Alter rmdir to reject attempts to remove a directory marked with this flag. This is used by cachefiles to prevent cachefilesd from removing them. Using S_SWAPFILE instead isn't really viable as that has other effects in the I/O paths. Changes ======= ver #3: - Check for the object pointer being NULL in the tracepoints rather than the caller. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819630256.215744.4815885535039369574.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906931596.143852.8642051223094013028.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967141000.1823006.12920680657559677789.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021541207.640689.564689725898537127.stgit@warthog.procyon.org.uk/ # v4 --- fs/cachefiles/Makefile | 1 + fs/cachefiles/namei.c | 43 +++++++++++++++++++++++++++++++++++++++ fs/namei.c | 3 ++- include/linux/fs.h | 1 + include/trace/events/cachefiles.h | 42 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 fs/cachefiles/namei.c (limited to 'include/linux') diff --git a/fs/cachefiles/Makefile b/fs/cachefiles/Makefile index 463e3d608b75..e0b092ca077f 100644 --- a/fs/cachefiles/Makefile +++ b/fs/cachefiles/Makefile @@ -7,6 +7,7 @@ cachefiles-y := \ cache.o \ daemon.o \ main.o \ + namei.o \ security.o cachefiles-$(CONFIG_CACHEFILES_ERROR_INJECTION) += error_inject.o diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c new file mode 100644 index 000000000000..913f83f1c900 --- /dev/null +++ b/fs/cachefiles/namei.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* CacheFiles path walking and related routines + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include "internal.h" + +/* + * Mark the backing file as being a cache file if it's not already in use. The + * mark tells the culling request command that it's not allowed to cull the + * file or directory. The caller must hold the inode lock. + */ +static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, + struct dentry *dentry) +{ + struct inode *inode = d_backing_inode(dentry); + bool can_use = false; + + if (!(inode->i_flags & S_KERNEL_FILE)) { + inode->i_flags |= S_KERNEL_FILE; + trace_cachefiles_mark_active(object, inode); + can_use = true; + } else { + pr_notice("cachefiles: Inode already in use: %pd\n", dentry); + } + + return can_use; +} + +/* + * Unmark a backing inode. The caller must hold the inode lock. + */ +static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct dentry *dentry) +{ + struct inode *inode = d_backing_inode(dentry); + + inode->i_flags &= ~S_KERNEL_FILE; + trace_cachefiles_mark_inactive(object, inode); +} diff --git a/fs/namei.c b/fs/namei.c index 1f9d2187c765..d81f04f8d818 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3958,7 +3958,8 @@ int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir, inode_lock(dentry->d_inode); error = -EBUSY; - if (is_local_mountpoint(dentry)) + if (is_local_mountpoint(dentry) || + (dentry->d_inode->i_flags & S_KERNEL_FILE)) goto out; error = security_inode_rmdir(dir, dentry); diff --git a/include/linux/fs.h b/include/linux/fs.h index 2c0b8e77d9ab..bcf1ca430139 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2249,6 +2249,7 @@ struct super_operations { #define S_ENCRYPTED (1 << 14) /* Encrypted file (using fs/crypto/) */ #define S_CASEFOLD (1 << 15) /* Casefolded file */ #define S_VERITY (1 << 16) /* Verity file (using fs/verity/) */ +#define S_KERNEL_FILE (1 << 17) /* File is in use by the kernel (eg. fs/cachefiles) */ /* * Note that nosuid etc flags are inode-specific: setting some file-system diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index 9bd5a8a60801..6331cd29880d 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -83,6 +83,48 @@ cachefiles_error_traces; #define E_(a, b) { a, b } +TRACE_EVENT(cachefiles_mark_active, + TP_PROTO(struct cachefiles_object *obj, + struct inode *inode), + + TP_ARGS(obj, inode), + + /* Note that obj may be NULL */ + TP_STRUCT__entry( + __field(unsigned int, obj ) + __field(ino_t, inode ) + ), + + TP_fast_assign( + __entry->obj = obj ? obj->debug_id : 0; + __entry->inode = inode->i_ino; + ), + + TP_printk("o=%08x i=%lx", + __entry->obj, __entry->inode) + ); + +TRACE_EVENT(cachefiles_mark_inactive, + TP_PROTO(struct cachefiles_object *obj, + struct inode *inode), + + TP_ARGS(obj, inode), + + /* Note that obj may be NULL */ + TP_STRUCT__entry( + __field(unsigned int, obj ) + __field(ino_t, inode ) + ), + + TP_fast_assign( + __entry->obj = obj ? obj->debug_id : 0; + __entry->inode = inode->i_ino; + ), + + TP_printk("o=%08x i=%lx", + __entry->obj, __entry->inode) + ); + TRACE_EVENT(cachefiles_vfs_error, TP_PROTO(struct cachefiles_object *obj, struct inode *backer, int error, enum cachefiles_error_trace where), -- cgit v1.2.3-59-g8ed1b From 32e150037dce368d129996ffe5f98217b1974d9e Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 14 Dec 2021 09:51:43 +0000 Subject: fscache, cachefiles: Store the volume coherency data Store the volume coherency data in an xattr and check it when we rebind the volume. If it doesn't match the cache volume is moved to the graveyard and rebuilt anew. Changes ======= ver #4: - Remove a couple of debugging prints. Signed-off-by: David Howells Reviewed-by: Jeff Layton Link: https://lore.kernel.org/r/163967164397.1823006.2950539849831291830.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021563138.640689.15851092065380543119.stgit@warthog.procyon.org.uk/ # v4 --- fs/cachefiles/internal.h | 2 + fs/cachefiles/volume.c | 25 ++++++++++++- fs/cachefiles/xattr.c | 78 +++++++++++++++++++++++++++++++++++++++ fs/fscache/volume.c | 14 ++++++- include/linux/fscache.h | 2 + include/trace/events/cachefiles.h | 42 ++++++++++++++++++++- 6 files changed, 157 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index d5868f5514d3..abdd1b66f6b9 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -270,6 +270,8 @@ extern int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, struct cachefiles_object *object, struct dentry *dentry); extern void cachefiles_prepare_to_write(struct fscache_cookie *cookie); +extern bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume); +extern int cachefiles_check_volume_xattr(struct cachefiles_volume *volume); /* * Error handling diff --git a/fs/cachefiles/volume.c b/fs/cachefiles/volume.c index 4a14f5e72764..89df0ba8ba5e 100644 --- a/fs/cachefiles/volume.c +++ b/fs/cachefiles/volume.c @@ -22,7 +22,8 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) struct dentry *vdentry, *fan; size_t len; char *name; - int n_accesses, i; + bool is_new = false; + int ret, n_accesses, i; _enter(""); @@ -43,11 +44,29 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) memcpy(name + 1, vcookie->key + 1, len); name[len + 1] = 0; - vdentry = cachefiles_get_directory(cache, cache->store, name, NULL); +retry: + vdentry = cachefiles_get_directory(cache, cache->store, name, &is_new); if (IS_ERR(vdentry)) goto error_name; volume->dentry = vdentry; + if (is_new) { + if (!cachefiles_set_volume_xattr(volume)) + goto error_dir; + } else { + ret = cachefiles_check_volume_xattr(volume); + if (ret < 0) { + if (ret != -ESTALE) + goto error_dir; + inode_lock_nested(d_inode(cache->store), I_MUTEX_PARENT); + cachefiles_bury_object(cache, NULL, cache->store, vdentry, + FSCACHE_VOLUME_IS_WEIRD); + cachefiles_put_directory(volume->dentry); + cond_resched(); + goto retry; + } + } + for (i = 0; i < 256; i++) { sprintf(name, "@%02x", i); fan = cachefiles_get_directory(cache, vdentry, name, NULL); @@ -74,6 +93,7 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) error_fan: for (i = 0; i < 256; i++) cachefiles_put_directory(volume->fanout[i]); +error_dir: cachefiles_put_directory(volume->dentry); error_name: kfree(name); @@ -114,5 +134,6 @@ void cachefiles_free_volume(struct fscache_volume *vcookie) void cachefiles_withdraw_volume(struct cachefiles_volume *volume) { fscache_withdraw_volume(volume->vcookie); + cachefiles_set_volume_xattr(volume); __cachefiles_free_volume(volume); } diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index 0601c46a22ef..83f41bd0c3a9 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -179,3 +179,81 @@ void cachefiles_prepare_to_write(struct fscache_cookie *cookie) cachefiles_end_secure(cache, saved_cred); } } + +/* + * Set the state xattr on a volume directory. + */ +bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume) +{ + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + struct dentry *dentry = volume->dentry; + int ret; + + _enter("%x,#%d", volume->vcookie->debug_id, len); + + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_setxattr(&init_user_ns, dentry, cachefiles_xattr_cache, + p, len, 0); + if (ret < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(dentry), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_fail); + if (ret != -ENOMEM) + cachefiles_io_error( + volume->cache, "Failed to set xattr with error %d", ret); + } else { + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_ok); + } + + _leave(" = %d", ret); + return ret == 0; +} + +/* + * Check the consistency between the backing cache and the volume cookie. + */ +int cachefiles_check_volume_xattr(struct cachefiles_volume *volume) +{ + struct cachefiles_xattr *buf; + struct dentry *dentry = volume->dentry; + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + enum cachefiles_coherency_trace why; + ssize_t xlen; + int ret = -ESTALE; + + _enter(""); + + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + xlen = cachefiles_inject_read_error(); + if (xlen == 0) + xlen = vfs_getxattr(&init_user_ns, dentry, cachefiles_xattr_cache, buf, len); + if (xlen != len) { + if (xlen < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(dentry), xlen, + cachefiles_trace_getxattr_error); + if (xlen == -EIO) + cachefiles_io_error( + volume->cache, + "Failed to read xattr with error %zd", xlen); + } + why = cachefiles_coherency_vol_check_xattr; + } else if (memcmp(buf->data, p, len) != 0) { + why = cachefiles_coherency_vol_check_cmp; + } else { + why = cachefiles_coherency_vol_check_ok; + ret = 0; + } + + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, why); + kfree(buf); + _leave(" = %d", ret); + return ret; +} diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c index e1a8e92a6adb..a57c6cbee858 100644 --- a/fs/fscache/volume.c +++ b/fs/fscache/volume.c @@ -205,15 +205,22 @@ static struct fscache_volume *fscache_alloc_volume(const char *volume_key, size_t klen, hlen; char *key; + if (!coherency_data) + coherency_len = 0; + cache = fscache_lookup_cache(cache_name, false); if (IS_ERR(cache)) return NULL; - volume = kzalloc(sizeof(*volume), GFP_KERNEL); + volume = kzalloc(struct_size(volume, coherency, coherency_len), + GFP_KERNEL); if (!volume) goto err_cache; volume->cache = cache; + volume->coherency_len = coherency_len; + if (coherency_data) + memcpy(volume->coherency, coherency_data, coherency_len); INIT_LIST_HEAD(&volume->proc_link); INIT_WORK(&volume->work, fscache_create_volume_work); refcount_set(&volume->ref, 1); @@ -421,8 +428,11 @@ void __fscache_relinquish_volume(struct fscache_volume *volume, if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) return; - if (invalidate) + if (invalidate) { set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); + } else if (coherency_data) { + memcpy(volume->coherency, coherency_data, volume->coherency_len); + } fscache_put_volume(volume, fscache_volume_put_relinquish); } diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 86b1c0db1de5..7bd35f60d19a 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -87,6 +87,8 @@ struct fscache_volume { #define FSCACHE_VOLUME_COLLIDED_WITH 2 /* Volume was collided with */ #define FSCACHE_VOLUME_ACQUIRE_PENDING 3 /* Volume is waiting to complete acquisition */ #define FSCACHE_VOLUME_CREATING 4 /* Volume is being created on disk */ + u8 coherency_len; /* Length of the coherency data */ + u8 coherency[]; /* Coherency data */ }; /* diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index ab1376ebc3ab..1172529b5b49 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -40,6 +40,7 @@ enum fscache_why_object_killed { FSCACHE_OBJECT_NO_SPACE, FSCACHE_OBJECT_WAS_RETIRED, FSCACHE_OBJECT_WAS_CULLED, + FSCACHE_VOLUME_IS_WEIRD, }; enum cachefiles_coherency_trace { @@ -53,6 +54,11 @@ enum cachefiles_coherency_trace { cachefiles_coherency_check_xattr, cachefiles_coherency_set_fail, cachefiles_coherency_set_ok, + cachefiles_coherency_vol_check_cmp, + cachefiles_coherency_vol_check_ok, + cachefiles_coherency_vol_check_xattr, + cachefiles_coherency_vol_set_fail, + cachefiles_coherency_vol_set_ok, }; enum cachefiles_trunc_trace { @@ -103,7 +109,8 @@ enum cachefiles_error_trace { EM(FSCACHE_OBJECT_INVALIDATED, "inval") \ EM(FSCACHE_OBJECT_NO_SPACE, "no_space") \ EM(FSCACHE_OBJECT_WAS_RETIRED, "was_retired") \ - E_(FSCACHE_OBJECT_WAS_CULLED, "was_culled") + EM(FSCACHE_OBJECT_WAS_CULLED, "was_culled") \ + E_(FSCACHE_VOLUME_IS_WEIRD, "volume_weird") #define cachefiles_obj_ref_traces \ EM(cachefiles_obj_get_ioreq, "GET ioreq") \ @@ -129,7 +136,12 @@ enum cachefiles_error_trace { EM(cachefiles_coherency_check_type, "BAD type") \ EM(cachefiles_coherency_check_xattr, "BAD xatt") \ EM(cachefiles_coherency_set_fail, "SET fail") \ - E_(cachefiles_coherency_set_ok, "SET ok ") + EM(cachefiles_coherency_set_ok, "SET ok ") \ + EM(cachefiles_coherency_vol_check_cmp, "VOL BAD cmp ") \ + EM(cachefiles_coherency_vol_check_ok, "VOL OK ") \ + EM(cachefiles_coherency_vol_check_xattr,"VOL BAD xatt") \ + EM(cachefiles_coherency_vol_set_fail, "VOL SET fail") \ + E_(cachefiles_coherency_vol_set_ok, "VOL SET ok ") #define cachefiles_trunc_traces \ EM(cachefiles_trunc_dio_adjust, "DIOADJ") \ @@ -365,6 +377,32 @@ TRACE_EVENT(cachefiles_coherency, __entry->content) ); +TRACE_EVENT(cachefiles_vol_coherency, + TP_PROTO(struct cachefiles_volume *volume, + ino_t ino, + enum cachefiles_coherency_trace why), + + TP_ARGS(volume, ino, why), + + /* Note that obj may be NULL */ + TP_STRUCT__entry( + __field(unsigned int, vol ) + __field(enum cachefiles_coherency_trace, why ) + __field(u64, ino ) + ), + + TP_fast_assign( + __entry->vol = volume->vcookie->debug_id; + __entry->why = why; + __entry->ino = ino; + ), + + TP_printk("V=%08x %s i=%llx", + __entry->vol, + __print_symbolic(__entry->why, cachefiles_coherency_traces), + __entry->ino) + ); + TRACE_EVENT(cachefiles_prep_read, TP_PROTO(struct netfs_read_subrequest *sreq, enum netfs_read_source source, -- cgit v1.2.3-59-g8ed1b From 3929eca769b5a231010b4978acc61c0735da198f Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 21 Oct 2021 21:58:29 +0100 Subject: fscache, cachefiles: Display stats of no-space events Add stat counters of no-space events that caused caching not to happen and display in /proc/fs/fscache/stats. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819653216.215744.17210522251617386509.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906958369.143852.7257100711818401748.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967166917.1823006.14842444049198947892.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021566184.640689.4417328329632709265.stgit@warthog.procyon.org.uk/ # v4 --- fs/cachefiles/cache.c | 18 +++++++++++++++--- fs/cachefiles/daemon.c | 2 +- fs/cachefiles/internal.h | 11 +++++++++-- fs/cachefiles/io.c | 7 +++++-- fs/cachefiles/namei.c | 6 ++++-- fs/fscache/stats.c | 8 ++++++++ include/linux/fscache-cache.h | 6 ++++++ 7 files changed, 48 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/fs/cachefiles/cache.c b/fs/cachefiles/cache.c index e2cbbc08bad9..809519286335 100644 --- a/fs/cachefiles/cache.c +++ b/fs/cachefiles/cache.c @@ -147,7 +147,7 @@ int cachefiles_add_cache(struct cachefiles_cache *cache) pr_info("File cache on %s registered\n", cache_cookie->name); /* check how much space the cache has */ - cachefiles_has_space(cache, 0, 0); + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); cachefiles_end_secure(cache, saved_cred); _leave(" = 0 [%px]", cache->cache); return 0; @@ -175,7 +175,8 @@ error_getsec: * cache */ int cachefiles_has_space(struct cachefiles_cache *cache, - unsigned fnr, unsigned bnr) + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason) { struct kstatfs stats; u64 b_avail, b_writing; @@ -233,7 +234,7 @@ int cachefiles_has_space(struct cachefiles_cache *cache, ret = -ENOBUFS; if (stats.f_ffree < cache->fstop || b_avail < cache->bstop) - goto begin_cull; + goto stop_and_begin_cull; ret = 0; if (stats.f_ffree < cache->fcull || @@ -252,6 +253,17 @@ int cachefiles_has_space(struct cachefiles_cache *cache, //_leave(" = 0"); return 0; +stop_and_begin_cull: + switch (reason) { + case cachefiles_has_space_for_write: + fscache_count_no_write_space(); + break; + case cachefiles_has_space_for_create: + fscache_count_no_create_space(); + break; + default: + break; + } begin_cull: if (!test_and_set_bit(CACHEFILES_CULLING, &cache->flags)) { _debug("### CULL CACHE ###"); diff --git a/fs/cachefiles/daemon.c b/fs/cachefiles/daemon.c index 45af558a696e..40a792421fc1 100644 --- a/fs/cachefiles/daemon.c +++ b/fs/cachefiles/daemon.c @@ -170,7 +170,7 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, return 0; /* check how much space the cache has */ - cachefiles_has_space(cache, 0, 0); + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); /* summarise */ f_released = atomic_xchg(&cache->f_released, 0); diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index abdd1b66f6b9..8dd54d9375b6 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -130,10 +130,17 @@ static inline void cachefiles_state_changed(struct cachefiles_cache *cache) * cache.c */ extern int cachefiles_add_cache(struct cachefiles_cache *cache); -extern int cachefiles_has_space(struct cachefiles_cache *cache, - unsigned fnr, unsigned bnr); extern void cachefiles_withdraw_cache(struct cachefiles_cache *cache); +enum cachefiles_has_space_for { + cachefiles_has_space_check, + cachefiles_has_space_for_write, + cachefiles_has_space_for_create, +}; +extern int cachefiles_has_space(struct cachefiles_cache *cache, + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason); + /* * daemon.c */ diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index 6f4dce0cfc36..60b1eac2ce78 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -468,7 +468,8 @@ static int __cachefiles_prepare_write(struct netfs_cache_resources *cres, * space, we need to see if it's fully allocated. If it's not, we may * want to cull it. */ - if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE) == 0) + if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_check) == 0) return 0; /* Enough space to simply overwrite the whole block */ pos = cachefiles_inject_read_error(); @@ -483,6 +484,7 @@ static int __cachefiles_prepare_write(struct netfs_cache_resources *cres, return 0; /* Fully allocated */ /* Partially allocated, but insufficient space: cull. */ + fscache_count_no_write_space(); ret = cachefiles_inject_remove_error(); if (ret == 0) ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, @@ -498,7 +500,8 @@ static int __cachefiles_prepare_write(struct netfs_cache_resources *cres, return ret; check_space: - return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE); + return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_for_write); } static int cachefiles_prepare_write(struct netfs_cache_resources *cres, diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index b549e9f79c01..ab3ca598acac 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -115,7 +115,8 @@ retry: /* we need to create the subdir if it doesn't exist yet */ if (d_is_negative(subdir)) { - ret = cachefiles_has_space(cache, 1, 0); + ret = cachefiles_has_space(cache, 1, 0, + cachefiles_has_space_for_create); if (ret < 0) goto mkdir_error; @@ -513,7 +514,8 @@ static bool cachefiles_create_file(struct cachefiles_object *object) struct file *file; int ret; - ret = cachefiles_has_space(object->volume->cache, 1, 0); + ret = cachefiles_has_space(object->volume->cache, 1, 0, + cachefiles_has_space_for_create); if (ret < 0) return false; diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index 798ee68b3e9d..db2f4e225dd9 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -42,6 +42,10 @@ atomic_t fscache_n_read; EXPORT_SYMBOL(fscache_n_read); atomic_t fscache_n_write; EXPORT_SYMBOL(fscache_n_write); +atomic_t fscache_n_no_write_space; +EXPORT_SYMBOL(fscache_n_no_write_space); +atomic_t fscache_n_no_create_space; +EXPORT_SYMBOL(fscache_n_no_create_space); /* * display the general statistics @@ -82,6 +86,10 @@ int fscache_stats_show(struct seq_file *m, void *v) atomic_read(&fscache_n_relinquishes_retire), atomic_read(&fscache_n_relinquishes_dropped)); + seq_printf(m, "NoSpace: nwr=%u ncr=%u\n", + atomic_read(&fscache_n_no_write_space), + atomic_read(&fscache_n_no_create_space)); + seq_printf(m, "IO : rd=%u wr=%u\n", atomic_read(&fscache_n_read), atomic_read(&fscache_n_write)); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 3fa4902dc87c..007e47f38610 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -186,11 +186,17 @@ static inline void fscache_wait_for_objects(struct fscache_cache *cache) #ifdef CONFIG_FSCACHE_STATS extern atomic_t fscache_n_read; extern atomic_t fscache_n_write; +extern atomic_t fscache_n_no_write_space; +extern atomic_t fscache_n_no_create_space; #define fscache_count_read() atomic_inc(&fscache_n_read) #define fscache_count_write() atomic_inc(&fscache_n_write) +#define fscache_count_no_write_space() atomic_inc(&fscache_n_no_write_space) +#define fscache_count_no_create_space() atomic_inc(&fscache_n_no_create_space) #else #define fscache_count_read() do {} while(0) #define fscache_count_write() do {} while(0) +#define fscache_count_no_write_space() do {} while(0) +#define fscache_count_no_create_space() do {} while(0) #endif #endif /* _LINUX_FSCACHE_CACHE_H */ -- cgit v1.2.3-59-g8ed1b From 9f08ebc3438baaaefcc79654b330209b83397f17 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 22 Oct 2021 09:17:58 +0100 Subject: fscache, cachefiles: Display stat of culling events Add a stat counter of culling events whereby the cache backend culls a file to make space (when asked by cachefilesd in this case) and display in /proc/fs/fscache/stats. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/163819654165.215744.3797804661644212436.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906961387.143852.9291157239960289090.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967168266.1823006.14436200166581605746.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021567619.640689.4339228906248763197.stgit@warthog.procyon.org.uk/ # v4 --- fs/cachefiles/namei.c | 1 + fs/fscache/stats.c | 7 +++++-- include/linux/fscache-cache.h | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index ab3ca598acac..9bd692870617 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -805,6 +805,7 @@ int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, if (ret < 0) goto error; + fscache_count_culled(); dput(victim); _leave(" = 0"); return 0; diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c index db2f4e225dd9..fc94e5e79f1c 100644 --- a/fs/fscache/stats.c +++ b/fs/fscache/stats.c @@ -46,6 +46,8 @@ atomic_t fscache_n_no_write_space; EXPORT_SYMBOL(fscache_n_no_write_space); atomic_t fscache_n_no_create_space; EXPORT_SYMBOL(fscache_n_no_create_space); +atomic_t fscache_n_culled; +EXPORT_SYMBOL(fscache_n_culled); /* * display the general statistics @@ -86,9 +88,10 @@ int fscache_stats_show(struct seq_file *m, void *v) atomic_read(&fscache_n_relinquishes_retire), atomic_read(&fscache_n_relinquishes_dropped)); - seq_printf(m, "NoSpace: nwr=%u ncr=%u\n", + seq_printf(m, "NoSpace: nwr=%u ncr=%u cull=%u\n", atomic_read(&fscache_n_no_write_space), - atomic_read(&fscache_n_no_create_space)); + atomic_read(&fscache_n_no_create_space), + atomic_read(&fscache_n_culled)); seq_printf(m, "IO : rd=%u wr=%u\n", atomic_read(&fscache_n_read), diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 007e47f38610..a174cedf4d90 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -188,15 +188,18 @@ extern atomic_t fscache_n_read; extern atomic_t fscache_n_write; extern atomic_t fscache_n_no_write_space; extern atomic_t fscache_n_no_create_space; +extern atomic_t fscache_n_culled; #define fscache_count_read() atomic_inc(&fscache_n_read) #define fscache_count_write() atomic_inc(&fscache_n_write) #define fscache_count_no_write_space() atomic_inc(&fscache_n_no_write_space) #define fscache_count_no_create_space() atomic_inc(&fscache_n_no_create_space) +#define fscache_count_culled() atomic_inc(&fscache_n_culled) #else #define fscache_count_read() do {} while(0) #define fscache_count_write() do {} while(0) #define fscache_count_no_write_space() do {} while(0) #define fscache_count_no_create_space() do {} while(0) +#define fscache_count_culled() do {} while(0) #endif #endif /* _LINUX_FSCACHE_CACHE_H */ -- cgit v1.2.3-59-g8ed1b From 44ea62813f0ab3d718de480504f4dfd0bdd01858 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 3 Sep 2021 18:31:40 -0700 Subject: spi: don't include ptp_clock_kernel.h in spi.h Commit b42faeee718c ("spi: Add a PTP system timestamp to the transfer structure") added an include of ptp_clock_kernel.h to spi.h for struct ptp_system_timestamp but a forward declaration is enough. Let's use that to limit the number of objects we have to rebuild every time we touch networking headers. Signed-off-by: Jakub Kicinski Tested-by: Vladimir Oltean Link: https://lore.kernel.org/r/20210904013140.2377609-1-kuba@kernel.org Signed-off-by: Mark Brown --- drivers/spi/spi.c | 1 + include/linux/spi/spi.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 7c790858547c..4599b121d744 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -33,6 +33,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index eb7ac8a1e03c..7ab3fed7b804 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -14,12 +14,12 @@ #include #include #include -#include #include struct dma_chan; struct software_node; +struct ptp_system_timestamp; struct spi_controller; struct spi_transfer; struct spi_controller_mem_ops; -- cgit v1.2.3-59-g8ed1b From 3506659e18a61ae525f3b9b4f5af23b4b149d4db Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 28 Nov 2021 14:53:35 -0500 Subject: mm: Add unmap_mapping_folio() Convert both callers of unmap_mapping_page() to call unmap_mapping_folio() instead. Also move zap_details from linux/mm.h to mm/memory.c Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- include/linux/mm.h | 24 ------------------------ mm/internal.h | 4 +++- mm/memory.c | 49 +++++++++++++++++++++++++++++++++++-------------- mm/truncate.c | 4 ++-- 4 files changed, 40 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 145f045b0ddc..c9cdb26802fb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1825,28 +1825,6 @@ static inline bool can_do_mlock(void) { return false; } extern int user_shm_lock(size_t, struct ucounts *); extern void user_shm_unlock(size_t, struct ucounts *); -/* - * Parameter block passed down to zap_pte_range in exceptional cases. - */ -struct zap_details { - struct address_space *zap_mapping; /* Check page->mapping if set */ - struct page *single_page; /* Locked page to be unmapped */ -}; - -/* - * We set details->zap_mappings when we want to unmap shared but keep private - * pages. Return true if skip zapping this page, false otherwise. - */ -static inline bool -zap_skip_check_mapping(struct zap_details *details, struct page *page) -{ - if (!details || !page) - return false; - - return details->zap_mapping && - (details->zap_mapping != page_rmapping(page)); -} - struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte); struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, @@ -1892,7 +1870,6 @@ extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, extern int fixup_user_fault(struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked); -void unmap_mapping_page(struct page *page); void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows); void unmap_mapping_range(struct address_space *mapping, @@ -1913,7 +1890,6 @@ static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, BUG(); return -EFAULT; } -static inline void unmap_mapping_page(struct page *page) { } static inline void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows) { } static inline void unmap_mapping_range(struct address_space *mapping, diff --git a/mm/internal.h b/mm/internal.h index 3b79a5c9427a..1ca93c6cb18c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -74,6 +74,7 @@ static inline bool can_madv_lru_vma(struct vm_area_struct *vma) return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP)); } +struct zap_details; void unmap_page_range(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long addr, unsigned long end, @@ -388,6 +389,7 @@ void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); #ifdef CONFIG_MMU +void unmap_mapping_folio(struct folio *folio); extern long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked); extern long faultin_vma_page_range(struct vm_area_struct *vma, @@ -491,8 +493,8 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, } return fpin; } - #else /* !CONFIG_MMU */ +static inline void unmap_mapping_folio(struct folio *folio) { } static inline void clear_page_mlock(struct page *page) { } static inline void mlock_vma_page(struct page *page) { } static inline void vunmap_range_noflush(unsigned long start, unsigned long end) diff --git a/mm/memory.c b/mm/memory.c index 8f1de811a1dc..23f2f1300d42 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1304,6 +1304,28 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) return ret; } +/* + * Parameter block passed down to zap_pte_range in exceptional cases. + */ +struct zap_details { + struct address_space *zap_mapping; /* Check page->mapping if set */ + struct folio *single_folio; /* Locked folio to be unmapped */ +}; + +/* + * We set details->zap_mapping when we want to unmap shared but keep private + * pages. Return true if skip zapping this page, false otherwise. + */ +static inline bool +zap_skip_check_mapping(struct zap_details *details, struct page *page) +{ + if (!details || !page) + return false; + + return details->zap_mapping && + (details->zap_mapping != page_rmapping(page)); +} + static unsigned long zap_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, @@ -1443,8 +1465,8 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, else if (zap_huge_pmd(tlb, vma, pmd, addr)) goto next; /* fall through */ - } else if (details && details->single_page && - PageTransCompound(details->single_page) && + } else if (details && details->single_folio && + folio_test_pmd_mappable(details->single_folio) && next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) { spinlock_t *ptl = pmd_lock(tlb->mm, pmd); /* @@ -3332,31 +3354,30 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, } /** - * unmap_mapping_page() - Unmap single page from processes. - * @page: The locked page to be unmapped. + * unmap_mapping_folio() - Unmap single folio from processes. + * @folio: The locked folio to be unmapped. * - * Unmap this page from any userspace process which still has it mmaped. + * Unmap this folio from any userspace process which still has it mmaped. * Typically, for efficiency, the range of nearby pages has already been * unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once - * truncation or invalidation holds the lock on a page, it may find that - * the page has been remapped again: and then uses unmap_mapping_page() + * truncation or invalidation holds the lock on a folio, it may find that + * the page has been remapped again: and then uses unmap_mapping_folio() * to unmap it finally. */ -void unmap_mapping_page(struct page *page) +void unmap_mapping_folio(struct folio *folio) { - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; struct zap_details details = { }; pgoff_t first_index; pgoff_t last_index; - VM_BUG_ON(!PageLocked(page)); - VM_BUG_ON(PageTail(page)); + VM_BUG_ON(!folio_test_locked(folio)); - first_index = page->index; - last_index = page->index + thp_nr_pages(page) - 1; + first_index = folio->index; + last_index = folio->index + folio_nr_pages(folio) - 1; details.zap_mapping = mapping; - details.single_page = page; + details.single_folio = folio; i_mmap_lock_write(mapping); if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))) diff --git a/mm/truncate.c b/mm/truncate.c index ab86b07c1e9c..c98feea75a10 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -180,7 +180,7 @@ void do_invalidatepage(struct page *page, unsigned int offset, static void truncate_cleanup_folio(struct folio *folio) { if (folio_mapped(folio)) - unmap_mapping_page(&folio->page); + unmap_mapping_folio(folio); if (folio_has_private(folio)) do_invalidatepage(&folio->page, 0, folio_size(folio)); @@ -670,7 +670,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping, wait_on_page_writeback(page); if (page_mapped(page)) - unmap_mapping_page(page); + unmap_mapping_folio(page_folio(page)); BUG_ON(page_mapped(page)); ret2 = do_launder_page(mapping, page); -- cgit v1.2.3-59-g8ed1b From 1e84a3d997b74c33491899e31d48774f252213ab Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 2 Dec 2021 16:01:55 -0500 Subject: truncate,shmem: Add truncate_inode_folio() Convert all callers of truncate_inode_page() to call truncate_inode_folio() instead, and move the declaration to mm/internal.h. Move the assertion that the caller is not passing in a tail page to generic_error_remove_page(). We can't entirely remove the struct page from the callers yet because the page pointer in the pvec might be a shadow/dax/swap entry instead of actually a page. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/mm.h | 1 - mm/internal.h | 1 + mm/shmem.c | 5 +++-- mm/truncate.c | 23 ++++++++++++----------- 4 files changed, 16 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index c9cdb26802fb..d8b7d7ed14dd 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1859,7 +1859,6 @@ extern void truncate_pagecache(struct inode *inode, loff_t new); extern void truncate_setsize(struct inode *inode, loff_t newsize); void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to); void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); -int truncate_inode_page(struct address_space *mapping, struct page *page); int generic_error_remove_page(struct address_space *mapping, struct page *page); int invalidate_inode_page(struct page *page); diff --git a/mm/internal.h b/mm/internal.h index 1ca93c6cb18c..f9967b0be8bf 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -92,6 +92,7 @@ static inline void force_page_cache_readahead(struct address_space *mapping, unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, pgoff_t end, struct pagevec *pvec, pgoff_t *indices); +int truncate_inode_folio(struct address_space *mapping, struct folio *folio); /** * folio_evictable - Test whether a folio is evictable. diff --git a/mm/shmem.c b/mm/shmem.c index 40da9075374b..dbef008fb6e5 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -950,7 +950,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, index += folio_nr_pages(folio) - 1; if (!unfalloc || !folio_test_uptodate(folio)) - truncate_inode_page(mapping, &folio->page); + truncate_inode_folio(mapping, folio); folio_unlock(folio); } pagevec_remove_exceptionals(&pvec); @@ -1027,7 +1027,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, } VM_BUG_ON_PAGE(PageWriteback(page), page); if (shmem_punch_compound(page, start, end)) - truncate_inode_page(mapping, page); + truncate_inode_folio(mapping, + page_folio(page)); else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { /* Wipe the page and don't get stuck */ clear_highpage(page); diff --git a/mm/truncate.c b/mm/truncate.c index c98feea75a10..0000424fc56b 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -218,12 +218,9 @@ invalidate_complete_page(struct address_space *mapping, struct page *page) return ret; } -int truncate_inode_page(struct address_space *mapping, struct page *page) +int truncate_inode_folio(struct address_space *mapping, struct folio *folio) { - struct folio *folio = page_folio(page); - VM_BUG_ON_PAGE(PageTail(page), page); - - if (page->mapping != mapping) + if (folio->mapping != mapping) return -EIO; truncate_cleanup_folio(folio); @@ -236,6 +233,8 @@ int truncate_inode_page(struct address_space *mapping, struct page *page) */ int generic_error_remove_page(struct address_space *mapping, struct page *page) { + VM_BUG_ON_PAGE(PageTail(page), page); + if (!mapping) return -EINVAL; /* @@ -244,7 +243,7 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page) */ if (!S_ISREG(mapping->host->i_mode)) return -EIO; - return truncate_inode_page(mapping, page); + return truncate_inode_folio(mapping, page_folio(page)); } EXPORT_SYMBOL(generic_error_remove_page); @@ -395,18 +394,20 @@ void truncate_inode_pages_range(struct address_space *mapping, for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; + struct folio *folio; /* We rely upon deletion not changing page->index */ index = indices[i]; if (xa_is_value(page)) continue; + folio = page_folio(page); - lock_page(page); - WARN_ON(page_to_index(page) != index); - wait_on_page_writeback(page); - truncate_inode_page(mapping, page); - unlock_page(page); + folio_lock(folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + folio_wait_writeback(folio); + truncate_inode_folio(mapping, folio); + folio_unlock(folio); } truncate_exceptional_pvec_entries(mapping, &pvec, indices); pagevec_release(&pvec); -- cgit v1.2.3-59-g8ed1b From 0e499ed3d7a216706e02eeded562627d3e69dcfd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 1 Sep 2020 23:17:50 -0400 Subject: filemap: Return only folios from find_get_entries() The callers have all been converted to work on folios, so convert find_get_entries() to return a batch of folios instead of pages. We also now return multiple large folios in a single call. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jan Kara Reviewed-by: William Kucharski Reviewed-by: Christoph Hellwig --- include/linux/pagemap.h | 2 -- mm/filemap.c | 43 +++++++++++-------------------------------- mm/internal.h | 4 ++++ mm/shmem.c | 36 ++++++++++++++++++++---------------- mm/truncate.c | 43 ++++++++++++++++++++++++------------------- 5 files changed, 59 insertions(+), 69 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index eb6e58e106c8..d2259a1da51c 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -592,8 +592,6 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index) return head + (index & (thp_nr_pages(head) - 1)); } -unsigned find_get_entries(struct address_space *mapping, pgoff_t start, - pgoff_t end, struct pagevec *pvec, pgoff_t *indices); unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, pgoff_t end, unsigned int nr_pages, struct page **pages); diff --git a/mm/filemap.c b/mm/filemap.c index aefa6082b81b..021214fd5354 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2015,57 +2015,36 @@ reset: * @mapping: The address_space to search * @start: The starting page cache index * @end: The final page index (inclusive). - * @pvec: Where the resulting entries are placed. + * @fbatch: Where the resulting entries are placed. * @indices: The cache indices corresponding to the entries in @entries * * find_get_entries() will search for and return a batch of entries in - * the mapping. The entries are placed in @pvec. find_get_entries() - * takes a reference on any actual pages it returns. + * the mapping. The entries are placed in @fbatch. find_get_entries() + * takes a reference on any actual folios it returns. * - * The search returns a group of mapping-contiguous page cache entries - * with ascending indexes. There may be holes in the indices due to - * not-present pages. + * The entries have ascending indexes. The indices may not be consecutive + * due to not-present entries or large folios. * - * Any shadow entries of evicted pages, or swap entries from + * Any shadow entries of evicted folios, or swap entries from * shmem/tmpfs, are included in the returned array. * - * If it finds a Transparent Huge Page, head or tail, find_get_entries() - * stops at that page: the caller is likely to have a better way to handle - * the compound page as a whole, and then skip its extent, than repeatedly - * calling find_get_entries() to return all its tails. - * - * Return: the number of pages and shadow entries which were found. + * Return: The number of entries which were found. */ unsigned find_get_entries(struct address_space *mapping, pgoff_t start, - pgoff_t end, struct pagevec *pvec, pgoff_t *indices) + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) { XA_STATE(xas, &mapping->i_pages, start); struct folio *folio; - unsigned int ret = 0; - unsigned nr_entries = PAGEVEC_SIZE; rcu_read_lock(); while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { - struct page *page = &folio->page; - /* - * Terminate early on finding a THP, to allow the caller to - * handle it all at once; but continue if this is hugetlbfs. - */ - if (!xa_is_value(folio) && folio_test_large(folio) && - !folio_test_hugetlb(folio)) { - page = folio_file_page(folio, xas.xa_index); - nr_entries = ret + 1; - } - - indices[ret] = xas.xa_index; - pvec->pages[ret] = page; - if (++ret == nr_entries) + indices[fbatch->nr] = xas.xa_index; + if (!folio_batch_add(fbatch, folio)) break; } rcu_read_unlock(); - pvec->nr = ret; - return ret; + return folio_batch_count(fbatch); } /** diff --git a/mm/internal.h b/mm/internal.h index e5f3ff3ae24e..07124e95e790 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -12,6 +12,8 @@ #include #include +struct folio_batch; + /* * The set of flags that only affect watermark checking and reclaim * behaviour. This is used by the MM to obey the caller constraints @@ -92,6 +94,8 @@ static inline void force_page_cache_readahead(struct address_space *mapping, unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, pgoff_t end, struct pagevec *pvec, pgoff_t *indices); +unsigned find_get_entries(struct address_space *mapping, pgoff_t start, + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); void filemap_free_folio(struct address_space *mapping, struct folio *folio); int truncate_inode_folio(struct address_space *mapping, struct folio *folio); diff --git a/mm/shmem.c b/mm/shmem.c index dbef008fb6e5..e909c163fb38 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -920,6 +920,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, unsigned int partial_start = lstart & (PAGE_SIZE - 1); unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); struct pagevec pvec; + struct folio_batch fbatch; pgoff_t indices[PAGEVEC_SIZE]; long nr_swaps_freed = 0; pgoff_t index; @@ -987,11 +988,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (start >= end) return; + folio_batch_init(&fbatch); index = start; while (index < end) { cond_resched(); - if (!find_get_entries(mapping, index, end - 1, &pvec, + if (!find_get_entries(mapping, index, end - 1, &fbatch, indices)) { /* If all gone or hole-punch or unfalloc, we're done */ if (index == start || end != -1) @@ -1000,14 +1002,14 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, index = start; continue; } - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i]; index = indices[i]; - if (xa_is_value(page)) { + if (xa_is_value(folio)) { if (unfalloc) continue; - if (shmem_free_swap(mapping, index, page)) { + if (shmem_free_swap(mapping, index, folio)) { /* Swap was replaced by page: retry */ index--; break; @@ -1016,33 +1018,35 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, continue; } - lock_page(page); + folio_lock(folio); - if (!unfalloc || !PageUptodate(page)) { - if (page_mapping(page) != mapping) { + if (!unfalloc || !folio_test_uptodate(folio)) { + struct page *page = folio_file_page(folio, + index); + if (folio_mapping(folio) != mapping) { /* Page was replaced by swap: retry */ - unlock_page(page); + folio_unlock(folio); index--; break; } - VM_BUG_ON_PAGE(PageWriteback(page), page); + VM_BUG_ON_FOLIO(folio_test_writeback(folio), + folio); if (shmem_punch_compound(page, start, end)) - truncate_inode_folio(mapping, - page_folio(page)); + truncate_inode_folio(mapping, folio); else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { /* Wipe the page and don't get stuck */ clear_highpage(page); flush_dcache_page(page); - set_page_dirty(page); + folio_mark_dirty(folio); if (index < round_up(start, HPAGE_PMD_NR)) start = index + 1; } } - unlock_page(page); + folio_unlock(folio); } - pagevec_remove_exceptionals(&pvec); - pagevec_release(&pvec); + folio_batch_remove_exceptionals(&fbatch); + folio_batch_release(&fbatch); index++; } diff --git a/mm/truncate.c b/mm/truncate.c index 5370094641d6..357af144df63 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -108,6 +108,13 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, pvec->nr = j; } +static void truncate_folio_batch_exceptionals(struct address_space *mapping, + struct folio_batch *fbatch, pgoff_t *indices) +{ + truncate_exceptional_pvec_entries(mapping, (struct pagevec *)fbatch, + indices); +} + /* * Invalidate exceptional entry if easily possible. This handles exceptional * entries for invalidate_inode_pages(). @@ -297,6 +304,7 @@ void truncate_inode_pages_range(struct address_space *mapping, unsigned int partial_start; /* inclusive */ unsigned int partial_end; /* exclusive */ struct pagevec pvec; + struct folio_batch fbatch; pgoff_t indices[PAGEVEC_SIZE]; pgoff_t index; int i; @@ -379,10 +387,11 @@ void truncate_inode_pages_range(struct address_space *mapping, if (start >= end) goto out; + folio_batch_init(&fbatch); index = start; for ( ; ; ) { cond_resched(); - if (!find_get_entries(mapping, index, end - 1, &pvec, + if (!find_get_entries(mapping, index, end - 1, &fbatch, indices)) { /* If all gone from start onwards, we're done */ if (index == start) @@ -392,16 +401,14 @@ void truncate_inode_pages_range(struct address_space *mapping, continue; } - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; - struct folio *folio; + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i]; /* We rely upon deletion not changing page->index */ index = indices[i]; - if (xa_is_value(page)) + if (xa_is_value(folio)) continue; - folio = page_folio(page); folio_lock(folio); VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); @@ -410,8 +417,8 @@ void truncate_inode_pages_range(struct address_space *mapping, folio_unlock(folio); index = folio_index(folio) + folio_nr_pages(folio) - 1; } - truncate_exceptional_pvec_entries(mapping, &pvec, indices); - pagevec_release(&pvec); + truncate_folio_batch_exceptionals(mapping, &fbatch, indices); + folio_batch_release(&fbatch); index++; } @@ -625,7 +632,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end) { pgoff_t indices[PAGEVEC_SIZE]; - struct pagevec pvec; + struct folio_batch fbatch; pgoff_t index; int i; int ret = 0; @@ -635,23 +642,21 @@ int invalidate_inode_pages2_range(struct address_space *mapping, if (mapping_empty(mapping)) goto out; - pagevec_init(&pvec); + folio_batch_init(&fbatch); index = start; - while (find_get_entries(mapping, index, end, &pvec, indices)) { - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; - struct folio *folio; + while (find_get_entries(mapping, index, end, &fbatch, indices)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i]; /* We rely upon deletion not changing folio->index */ index = indices[i]; - if (xa_is_value(page)) { + if (xa_is_value(folio)) { if (!invalidate_exceptional_entry2(mapping, - index, page)) + index, folio)) ret = -EBUSY; continue; } - folio = page_folio(page); if (!did_range_unmap && folio_mapped(folio)) { /* @@ -684,8 +689,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping, ret = ret2; folio_unlock(folio); } - pagevec_remove_exceptionals(&pvec); - pagevec_release(&pvec); + folio_batch_remove_exceptionals(&fbatch); + folio_batch_release(&fbatch); cond_resched(); index++; } -- cgit v1.2.3-59-g8ed1b From 51dcbdac28d4dde915f78adf08bb3fac87f516e9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 7 Dec 2021 14:15:07 -0500 Subject: mm: Convert find_lock_entries() to use a folio_batch find_lock_entries() already only returned the head page of folios, so convert it to return a folio_batch instead of a pagevec. That cascades through converting truncate_inode_pages_range() to delete_from_page_cache_batch() and page_cache_delete_batch(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- fs/f2fs/f2fs.h | 2 ++ include/linux/pagemap.h | 4 +-- mm/filemap.c | 60 +++++++++++++++++++++---------------------- mm/internal.h | 2 +- mm/shmem.c | 14 +++++------ mm/truncate.c | 67 +++++++++++++++++++++---------------------------- 6 files changed, 69 insertions(+), 80 deletions(-) (limited to 'include/linux') diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index ce9fc9f13000..d0d603187171 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -28,6 +28,8 @@ #include #include +struct pagevec; + #ifdef CONFIG_F2FS_CHECK_FS #define f2fs_bug_on(sbi, condition) BUG_ON(condition) #else diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index d2259a1da51c..6e038811f4c8 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -16,7 +16,7 @@ #include /* for in_interrupt() */ #include -struct pagevec; +struct folio_batch; static inline bool mapping_empty(struct address_space *mapping) { @@ -936,7 +936,7 @@ static inline void __delete_from_page_cache(struct page *page, void *shadow) } void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, - struct pagevec *pvec); + struct folio_batch *fbatch); int try_to_release_page(struct page *page, gfp_t gfp); bool filemap_release_folio(struct folio *folio, gfp_t gfp); loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end, diff --git a/mm/filemap.c b/mm/filemap.c index 021214fd5354..9d3bae3e36c3 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -270,30 +270,29 @@ void filemap_remove_folio(struct folio *folio) } /* - * page_cache_delete_batch - delete several pages from page cache - * @mapping: the mapping to which pages belong - * @pvec: pagevec with pages to delete + * page_cache_delete_batch - delete several folios from page cache + * @mapping: the mapping to which folios belong + * @fbatch: batch of folios to delete * - * The function walks over mapping->i_pages and removes pages passed in @pvec - * from the mapping. The function expects @pvec to be sorted by page index - * and is optimised for it to be dense. - * It tolerates holes in @pvec (mapping entries at those indices are not - * modified). The function expects only THP head pages to be present in the - * @pvec. + * The function walks over mapping->i_pages and removes folios passed in + * @fbatch from the mapping. The function expects @fbatch to be sorted + * by page index and is optimised for it to be dense. + * It tolerates holes in @fbatch (mapping entries at those indices are not + * modified). * * The function expects the i_pages lock to be held. */ static void page_cache_delete_batch(struct address_space *mapping, - struct pagevec *pvec) + struct folio_batch *fbatch) { - XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); + XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); int total_pages = 0; int i = 0; struct folio *folio; mapping_set_update(&xas, mapping); xas_for_each(&xas, folio, ULONG_MAX) { - if (i >= pagevec_count(pvec)) + if (i >= folio_batch_count(fbatch)) break; /* A swap/dax/shadow entry got inserted? Skip it. */ @@ -306,9 +305,9 @@ static void page_cache_delete_batch(struct address_space *mapping, * means our page has been removed, which shouldn't be * possible because we're holding the PageLock. */ - if (&folio->page != pvec->pages[i]) { + if (folio != fbatch->folios[i]) { VM_BUG_ON_FOLIO(folio->index > - pvec->pages[i]->index, folio); + fbatch->folios[i]->index, folio); continue; } @@ -316,12 +315,11 @@ static void page_cache_delete_batch(struct address_space *mapping, if (folio->index == xas.xa_index) folio->mapping = NULL; - /* Leave page->index set: truncation lookup relies on it */ + /* Leave folio->index set: truncation lookup relies on it */ /* - * Move to the next page in the vector if this is a regular - * page or the index is of the last sub-page of this compound - * page. + * Move to the next folio in the batch if this is a regular + * folio or the index is of the last sub-page of this folio. */ if (folio->index + folio_nr_pages(folio) - 1 == xas.xa_index) i++; @@ -332,29 +330,29 @@ static void page_cache_delete_batch(struct address_space *mapping, } void delete_from_page_cache_batch(struct address_space *mapping, - struct pagevec *pvec) + struct folio_batch *fbatch) { int i; - if (!pagevec_count(pvec)) + if (!folio_batch_count(fbatch)) return; spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); - for (i = 0; i < pagevec_count(pvec); i++) { - struct folio *folio = page_folio(pvec->pages[i]); + for (i = 0; i < folio_batch_count(fbatch); i++) { + struct folio *folio = fbatch->folios[i]; trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); } - page_cache_delete_batch(mapping, pvec); + page_cache_delete_batch(mapping, fbatch); xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_add_lru(mapping->host); spin_unlock(&mapping->host->i_lock); - for (i = 0; i < pagevec_count(pvec); i++) - filemap_free_folio(mapping, page_folio(pvec->pages[i])); + for (i = 0; i < folio_batch_count(fbatch); i++) + filemap_free_folio(mapping, fbatch->folios[i]); } int filemap_check_errors(struct address_space *mapping) @@ -2052,8 +2050,8 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start, * @mapping: The address_space to search. * @start: The starting page cache index. * @end: The final page index (inclusive). - * @pvec: Where the resulting entries are placed. - * @indices: The cache indices of the entries in @pvec. + * @fbatch: Where the resulting entries are placed. + * @indices: The cache indices of the entries in @fbatch. * * find_lock_entries() will return a batch of entries from @mapping. * Swap, shadow and DAX entries are included. Folios are returned @@ -2068,7 +2066,7 @@ unsigned find_get_entries(struct address_space *mapping, pgoff_t start, * Return: The number of entries which were found. */ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, - pgoff_t end, struct pagevec *pvec, pgoff_t *indices) + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) { XA_STATE(xas, &mapping->i_pages, start); struct folio *folio; @@ -2088,8 +2086,8 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), folio); } - indices[pvec->nr] = xas.xa_index; - if (!pagevec_add(pvec, &folio->page)) + indices[fbatch->nr] = xas.xa_index; + if (!folio_batch_add(fbatch, folio)) break; goto next; unlock: @@ -2106,7 +2104,7 @@ next: } rcu_read_unlock(); - return pagevec_count(pvec); + return folio_batch_count(fbatch); } /** diff --git a/mm/internal.h b/mm/internal.h index 07124e95e790..c52c05dc6b1f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -93,7 +93,7 @@ static inline void force_page_cache_readahead(struct address_space *mapping, } unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, - pgoff_t end, struct pagevec *pvec, pgoff_t *indices); + pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); unsigned find_get_entries(struct address_space *mapping, pgoff_t start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); void filemap_free_folio(struct address_space *mapping, struct folio *folio); diff --git a/mm/shmem.c b/mm/shmem.c index e909c163fb38..bbfa2d05e787 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -919,7 +919,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, pgoff_t end = (lend + 1) >> PAGE_SHIFT; unsigned int partial_start = lstart & (PAGE_SIZE - 1); unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); - struct pagevec pvec; struct folio_batch fbatch; pgoff_t indices[PAGEVEC_SIZE]; long nr_swaps_freed = 0; @@ -932,12 +931,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (info->fallocend > start && info->fallocend <= end && !unfalloc) info->fallocend = start; - pagevec_init(&pvec); + folio_batch_init(&fbatch); index = start; while (index < end && find_lock_entries(mapping, index, end - 1, - &pvec, indices)) { - for (i = 0; i < pagevec_count(&pvec); i++) { - struct folio *folio = (struct folio *)pvec.pages[i]; + &fbatch, indices)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i]; index = indices[i]; @@ -954,8 +953,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, truncate_inode_folio(mapping, folio); folio_unlock(folio); } - pagevec_remove_exceptionals(&pvec); - pagevec_release(&pvec); + folio_batch_remove_exceptionals(&fbatch); + folio_batch_release(&fbatch); cond_resched(); index++; } @@ -988,7 +987,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (start >= end) return; - folio_batch_init(&fbatch); index = start; while (index < end) { cond_resched(); diff --git a/mm/truncate.c b/mm/truncate.c index 357af144df63..e7f5762c43d3 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -56,11 +56,11 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, /* * Unconditionally remove exceptional entries. Usually called from truncate - * path. Note that the pagevec may be altered by this function by removing + * path. Note that the folio_batch may be altered by this function by removing * exceptional entries similar to what pagevec_remove_exceptionals does. */ -static void truncate_exceptional_pvec_entries(struct address_space *mapping, - struct pagevec *pvec, pgoff_t *indices) +static void truncate_folio_batch_exceptionals(struct address_space *mapping, + struct folio_batch *fbatch, pgoff_t *indices) { int i, j; bool dax; @@ -69,11 +69,11 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, if (shmem_mapping(mapping)) return; - for (j = 0; j < pagevec_count(pvec); j++) - if (xa_is_value(pvec->pages[j])) + for (j = 0; j < folio_batch_count(fbatch); j++) + if (xa_is_value(fbatch->folios[j])) break; - if (j == pagevec_count(pvec)) + if (j == folio_batch_count(fbatch)) return; dax = dax_mapping(mapping); @@ -82,12 +82,12 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, xa_lock_irq(&mapping->i_pages); } - for (i = j; i < pagevec_count(pvec); i++) { - struct page *page = pvec->pages[i]; + for (i = j; i < folio_batch_count(fbatch); i++) { + struct folio *folio = fbatch->folios[i]; pgoff_t index = indices[i]; - if (!xa_is_value(page)) { - pvec->pages[j++] = page; + if (!xa_is_value(folio)) { + fbatch->folios[j++] = folio; continue; } @@ -96,7 +96,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, continue; } - __clear_shadow_entry(mapping, index, page); + __clear_shadow_entry(mapping, index, folio); } if (!dax) { @@ -105,14 +105,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, inode_add_lru(mapping->host); spin_unlock(&mapping->host->i_lock); } - pvec->nr = j; -} - -static void truncate_folio_batch_exceptionals(struct address_space *mapping, - struct folio_batch *fbatch, pgoff_t *indices) -{ - truncate_exceptional_pvec_entries(mapping, (struct pagevec *)fbatch, - indices); + fbatch->nr = j; } /* @@ -303,7 +296,6 @@ void truncate_inode_pages_range(struct address_space *mapping, pgoff_t end; /* exclusive */ unsigned int partial_start; /* inclusive */ unsigned int partial_end; /* exclusive */ - struct pagevec pvec; struct folio_batch fbatch; pgoff_t indices[PAGEVEC_SIZE]; pgoff_t index; @@ -333,18 +325,18 @@ void truncate_inode_pages_range(struct address_space *mapping, else end = (lend + 1) >> PAGE_SHIFT; - pagevec_init(&pvec); + folio_batch_init(&fbatch); index = start; while (index < end && find_lock_entries(mapping, index, end - 1, - &pvec, indices)) { - index = indices[pagevec_count(&pvec) - 1] + 1; - truncate_exceptional_pvec_entries(mapping, &pvec, indices); - for (i = 0; i < pagevec_count(&pvec); i++) - truncate_cleanup_folio(page_folio(pvec.pages[i])); - delete_from_page_cache_batch(mapping, &pvec); - for (i = 0; i < pagevec_count(&pvec); i++) - unlock_page(pvec.pages[i]); - pagevec_release(&pvec); + &fbatch, indices)) { + index = indices[folio_batch_count(&fbatch) - 1] + 1; + truncate_folio_batch_exceptionals(mapping, &fbatch, indices); + for (i = 0; i < folio_batch_count(&fbatch); i++) + truncate_cleanup_folio(fbatch.folios[i]); + delete_from_page_cache_batch(mapping, &fbatch); + for (i = 0; i < folio_batch_count(&fbatch); i++) + folio_unlock(fbatch.folios[i]); + folio_batch_release(&fbatch); cond_resched(); } @@ -387,7 +379,6 @@ void truncate_inode_pages_range(struct address_space *mapping, if (start >= end) goto out; - folio_batch_init(&fbatch); index = start; for ( ; ; ) { cond_resched(); @@ -489,16 +480,16 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t end, unsigned long *nr_pagevec) { pgoff_t indices[PAGEVEC_SIZE]; - struct pagevec pvec; + struct folio_batch fbatch; pgoff_t index = start; unsigned long ret; unsigned long count = 0; int i; - pagevec_init(&pvec); - while (find_lock_entries(mapping, index, end, &pvec, indices)) { - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; + folio_batch_init(&fbatch); + while (find_lock_entries(mapping, index, end, &fbatch, indices)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct page *page = &fbatch.folios[i]->page; /* We rely upon deletion not changing page->index */ index = indices[i]; @@ -525,8 +516,8 @@ static unsigned long __invalidate_mapping_pages(struct address_space *mapping, } count += ret; } - pagevec_remove_exceptionals(&pvec); - pagevec_release(&pvec); + folio_batch_remove_exceptionals(&fbatch); + folio_batch_release(&fbatch); cond_resched(); index++; } -- cgit v1.2.3-59-g8ed1b From 1613fac9aaf840af76faa747ea428a714af98dbd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 7 Dec 2021 14:28:49 -0500 Subject: mm: Remove pagevec_remove_exceptionals() All of its callers now call folio_batch_remove_exceptionals(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/pagevec.h | 6 +----- mm/swap.c | 26 +++++++++++++------------- mm/truncate.c | 2 +- 3 files changed, 15 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index c3fa616d7ae7..dda8d5868c81 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -27,7 +27,6 @@ struct pagevec { void __pagevec_release(struct pagevec *pvec); void __pagevec_lru_add(struct pagevec *pvec); -void pagevec_remove_exceptionals(struct pagevec *pvec); unsigned pagevec_lookup_range(struct pagevec *pvec, struct address_space *mapping, pgoff_t *start, pgoff_t end); @@ -146,8 +145,5 @@ static inline void folio_batch_release(struct folio_batch *fbatch) pagevec_release((struct pagevec *)fbatch); } -static inline void folio_batch_remove_exceptionals(struct folio_batch *fbatch) -{ - pagevec_remove_exceptionals((struct pagevec *)fbatch); -} +void folio_batch_remove_exceptionals(struct folio_batch *fbatch); #endif /* _LINUX_PAGEVEC_H */ diff --git a/mm/swap.c b/mm/swap.c index e8c9dc6d0377..74f6b311d7ee 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -1077,24 +1077,24 @@ void __pagevec_lru_add(struct pagevec *pvec) } /** - * pagevec_remove_exceptionals - pagevec exceptionals pruning - * @pvec: The pagevec to prune + * folio_batch_remove_exceptionals() - Prune non-folios from a batch. + * @fbatch: The batch to prune * - * find_get_entries() fills both pages and XArray value entries (aka - * exceptional entries) into the pagevec. This function prunes all - * exceptionals from @pvec without leaving holes, so that it can be - * passed on to page-only pagevec operations. + * find_get_entries() fills a batch with both folios and shadow/swap/DAX + * entries. This function prunes all the non-folio entries from @fbatch + * without leaving holes, so that it can be passed on to folio-only batch + * operations. */ -void pagevec_remove_exceptionals(struct pagevec *pvec) +void folio_batch_remove_exceptionals(struct folio_batch *fbatch) { - int i, j; + unsigned int i, j; - for (i = 0, j = 0; i < pagevec_count(pvec); i++) { - struct page *page = pvec->pages[i]; - if (!xa_is_value(page)) - pvec->pages[j++] = page; + for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { + struct folio *folio = fbatch->folios[i]; + if (!xa_is_value(folio)) + fbatch->folios[j++] = folio; } - pvec->nr = j; + fbatch->nr = j; } /** diff --git a/mm/truncate.c b/mm/truncate.c index e7f5762c43d3..a1113b0abb30 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -57,7 +57,7 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, /* * Unconditionally remove exceptional entries. Usually called from truncate * path. Note that the folio_batch may be altered by this function by removing - * exceptional entries similar to what pagevec_remove_exceptionals does. + * exceptional entries similar to what folio_batch_remove_exceptionals() does. */ static void truncate_folio_batch_exceptionals(struct address_space *mapping, struct folio_batch *fbatch, pgoff_t *indices) -- cgit v1.2.3-59-g8ed1b From 25a8de7f8d970ffa7263bd9d32a08138cd949f17 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 27 Aug 2021 07:21:49 -0400 Subject: XArray: Add xas_advance() Add a new helper function to help iterate over multi-index entries. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski --- include/linux/xarray.h | 18 ++++++++++++++++++ lib/xarray.c | 6 +++--- 2 files changed, 21 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index a91e3d90df8a..d6d5da6ed735 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -1580,6 +1580,24 @@ static inline void xas_set(struct xa_state *xas, unsigned long index) xas->xa_node = XAS_RESTART; } +/** + * xas_advance() - Skip over sibling entries. + * @xas: XArray operation state. + * @index: Index of last sibling entry. + * + * Move the operation state to refer to the last sibling entry. + * This is useful for loops that normally want to see sibling + * entries but sometimes want to skip them. Use xas_set() if you + * want to move to an index which is not part of this entry. + */ +static inline void xas_advance(struct xa_state *xas, unsigned long index) +{ + unsigned char shift = xas_is_node(xas) ? xas->xa_node->shift : 0; + + xas->xa_index = index; + xas->xa_offset = (index >> shift) & XA_CHUNK_MASK; +} + /** * xas_set_order() - Set up XArray operation state for a multislot entry. * @xas: XArray operation state. diff --git a/lib/xarray.c b/lib/xarray.c index f5d8f54907b4..6f47f6375808 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -157,7 +157,7 @@ static void xas_move_index(struct xa_state *xas, unsigned long offset) xas->xa_index += offset << shift; } -static void xas_advance(struct xa_state *xas) +static void xas_next_offset(struct xa_state *xas) { xas->xa_offset++; xas_move_index(xas, xas->xa_offset); @@ -1250,7 +1250,7 @@ void *xas_find(struct xa_state *xas, unsigned long max) xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1; } - xas_advance(xas); + xas_next_offset(xas); while (xas->xa_node && (xas->xa_index <= max)) { if (unlikely(xas->xa_offset == XA_CHUNK_SIZE)) { @@ -1268,7 +1268,7 @@ void *xas_find(struct xa_state *xas, unsigned long max) if (entry && !xa_is_sibling(entry)) return entry; - xas_advance(xas); + xas_next_offset(xas); } if (!xas->xa_node) -- cgit v1.2.3-59-g8ed1b From 6b24ca4a1a8d4ee3221d6d44ddbb99f542e4bda3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 27 Jun 2020 22:19:08 -0400 Subject: mm: Use multi-index entries in the page cache We currently store large folios as 2^N consecutive entries. While this consumes rather more memory than necessary, it also turns out to be buggy. A writeback operation which starts within a tail page of a dirty folio will not write back the folio as the xarray's dirty bit is only set on the head index. With multi-index entries, the dirty bit will be found no matter where in the folio the operation starts. This does end up simplifying the page cache slightly, although not as much as I had hoped. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- include/linux/pagemap.h | 10 -------- mm/filemap.c | 61 +++++++++++++++++++++++++++++++------------------ mm/huge_memory.c | 18 +++++++++++---- mm/khugepaged.c | 12 +++++++++- mm/migrate.c | 8 ------- mm/shmem.c | 19 +++++++-------- 6 files changed, 72 insertions(+), 56 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 6e038811f4c8..704cb1b4b15d 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1125,16 +1125,6 @@ static inline unsigned int __readahead_batch(struct readahead_control *rac, VM_BUG_ON_PAGE(PageTail(page), page); array[i++] = page; rac->_batch_count += thp_nr_pages(page); - - /* - * The page cache isn't using multi-index entries yet, - * so the xas cursor needs to be manually moved to the - * next index. This can be removed once the page cache - * is converted. - */ - if (PageHead(page)) - xas_set(&xas, rac->_index + rac->_batch_count); - if (i == array_sz) break; } diff --git a/mm/filemap.c b/mm/filemap.c index 9d3bae3e36c3..33077c264d79 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -135,7 +135,6 @@ static void page_cache_delete(struct address_space *mapping, } VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); - VM_BUG_ON_FOLIO(nr != 1 && shadow, folio); xas_store(&xas, shadow); xas_init_marks(&xas); @@ -286,7 +285,7 @@ static void page_cache_delete_batch(struct address_space *mapping, struct folio_batch *fbatch) { XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); - int total_pages = 0; + long total_pages = 0; int i = 0; struct folio *folio; @@ -313,18 +312,12 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); - if (folio->index == xas.xa_index) - folio->mapping = NULL; + folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ - /* - * Move to the next folio in the batch if this is a regular - * folio or the index is of the last sub-page of this folio. - */ - if (folio->index + folio_nr_pages(folio) - 1 == xas.xa_index) - i++; + i++; xas_store(&xas, NULL); - total_pages++; + total_pages += folio_nr_pages(folio); } mapping->nrpages -= total_pages; } @@ -2089,24 +2082,27 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, indices[fbatch->nr] = xas.xa_index; if (!folio_batch_add(fbatch, folio)) break; - goto next; + continue; unlock: folio_unlock(folio); put: folio_put(folio); -next: - if (!xa_is_value(folio) && folio_test_large(folio)) { - xas_set(&xas, folio->index + folio_nr_pages(folio)); - /* Did we wrap on 32-bit? */ - if (!xas.xa_index) - break; - } } rcu_read_unlock(); return folio_batch_count(fbatch); } +static inline +bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max) +{ + if (!folio_test_large(folio) || folio_test_hugetlb(folio)) + return false; + if (index >= max) + return false; + return index < folio->index + folio_nr_pages(folio) - 1; +} + /** * find_get_pages_range - gang pagecache lookup * @mapping: The address_space to search @@ -2145,11 +2141,17 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, if (xa_is_value(folio)) continue; +again: pages[ret] = folio_file_page(folio, xas.xa_index); if (++ret == nr_pages) { *start = xas.xa_index + 1; goto out; } + if (folio_more_pages(folio, xas.xa_index, end)) { + xas.xa_index++; + folio_ref_inc(folio); + goto again; + } } /* @@ -2207,9 +2209,15 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, if (unlikely(folio != xas_reload(&xas))) goto put_page; - pages[ret] = &folio->page; +again: + pages[ret] = folio_file_page(folio, xas.xa_index); if (++ret == nr_pages) break; + if (folio_more_pages(folio, xas.xa_index, ULONG_MAX)) { + xas.xa_index++; + folio_ref_inc(folio); + goto again; + } continue; put_page: folio_put(folio); @@ -2334,8 +2342,7 @@ static void filemap_get_read_batch(struct address_space *mapping, break; if (folio_test_readahead(folio)) break; - xas.xa_index = folio->index + folio_nr_pages(folio) - 1; - xas.xa_offset = (xas.xa_index >> xas.xa_shift) & XA_CHUNK_MASK; + xas_advance(&xas, folio->index + folio_nr_pages(folio) - 1); continue; put_folio: folio_put(folio); @@ -3284,6 +3291,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); do { +again: page = folio_file_page(folio, xas.xa_index); if (PageHWPoison(page)) goto unlock; @@ -3305,9 +3313,18 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, do_set_pte(vmf, page, addr); /* no need to invalidate: a not-present page won't be cached */ update_mmu_cache(vma, addr, vmf->pte); + if (folio_more_pages(folio, xas.xa_index, end_pgoff)) { + xas.xa_index++; + folio_ref_inc(folio); + goto again; + } folio_unlock(folio); continue; unlock: + if (folio_more_pages(folio, xas.xa_index, end_pgoff)) { + xas.xa_index++; + goto again; + } folio_unlock(folio); folio_put(folio); } while ((folio = next_map_page(mapping, &xas, end_pgoff)) != NULL); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index e5483347291c..f58524394dc1 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2614,6 +2614,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) { struct page *head = compound_head(page); struct deferred_split *ds_queue = get_deferred_split_queue(head); + XA_STATE(xas, &head->mapping->i_pages, head->index); struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; int extra_pins, ret; @@ -2652,6 +2653,13 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) goto out; } + xas_split_alloc(&xas, head, compound_order(head), + mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); + if (xas_error(&xas)) { + ret = xas_error(&xas); + goto out; + } + anon_vma = NULL; i_mmap_lock_read(mapping); @@ -2681,13 +2689,12 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) /* block interrupt reentry in xa_lock and spinlock */ local_irq_disable(); if (mapping) { - XA_STATE(xas, &mapping->i_pages, page_index(head)); - /* * Check if the head page is present in page cache. * We assume all tail are present too, if head is there. */ - xa_lock(&mapping->i_pages); + xas_lock(&xas); + xas_reset(&xas); if (xas_load(&xas) != head) goto fail; } @@ -2703,6 +2710,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) if (mapping) { int nr = thp_nr_pages(head); + xas_split(&xas, head, thp_order(head)); if (PageSwapBacked(head)) { __mod_lruvec_page_state(head, NR_SHMEM_THPS, -nr); @@ -2719,7 +2727,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) spin_unlock(&ds_queue->split_queue_lock); fail: if (mapping) - xa_unlock(&mapping->i_pages); + xas_unlock(&xas); local_irq_enable(); remap_page(head, thp_nr_pages(head)); ret = -EBUSY; @@ -2733,6 +2741,8 @@ out_unlock: if (mapping) i_mmap_unlock_read(mapping); out: + /* Free any memory we didn't use */ + xas_nomem(&xas, 0); count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); return ret; } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index e99101162f1a..2e1911cc3466 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1667,7 +1667,10 @@ static void collapse_file(struct mm_struct *mm, } count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); - /* This will be less messy when we use multi-index entries */ + /* + * Ensure we have slots for all the pages in the range. This is + * almost certainly a no-op because most of the pages must be present + */ do { xas_lock_irq(&xas); xas_create_range(&xas); @@ -1892,6 +1895,9 @@ out_unlock: __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); } + /* Join all the small entries into a single multi-index entry */ + xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_store(&xas, new_page); xa_locked: xas_unlock_irq(&xas); xa_unlocked: @@ -2013,6 +2019,10 @@ static void khugepaged_scan_file(struct mm_struct *mm, continue; } + /* + * XXX: khugepaged should compact smaller compound pages + * into a PMD sized page + */ if (PageTransCompound(page)) { result = SCAN_PAGE_COMPOUND; break; diff --git a/mm/migrate.c b/mm/migrate.c index 311638177536..7079e6b7dbe7 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -433,14 +433,6 @@ int folio_migrate_mapping(struct address_space *mapping, } xas_store(&xas, newfolio); - if (nr > 1) { - int i; - - for (i = 1; i < nr; i++) { - xas_next(&xas); - xas_store(&xas, newfolio); - } - } /* * Drop cache reference from old page by unfreezing diff --git a/mm/shmem.c b/mm/shmem.c index e4c9e5c7081f..28d627444a24 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -694,7 +694,6 @@ static int shmem_add_to_page_cache(struct page *page, struct mm_struct *charge_mm) { XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page)); - unsigned long i = 0; unsigned long nr = compound_nr(page); int error; @@ -721,20 +720,18 @@ static int shmem_add_to_page_cache(struct page *page, cgroup_throttle_swaprate(page, gfp); do { - void *entry; xas_lock_irq(&xas); - entry = xas_find_conflict(&xas); - if (entry != expected) + if (expected != xas_find_conflict(&xas)) { + xas_set_err(&xas, -EEXIST); + goto unlock; + } + if (expected && xas_find_conflict(&xas)) { xas_set_err(&xas, -EEXIST); - xas_create_range(&xas); - if (xas_error(&xas)) goto unlock; -next: - xas_store(&xas, page); - if (++i < nr) { - xas_next(&xas); - goto next; } + xas_store(&xas, page); + if (xas_error(&xas)) + goto unlock; if (PageTransHuge(page)) { count_vm_event(THP_FILE_ALLOC); __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr); -- cgit v1.2.3-59-g8ed1b From e32cf5dfbe227b355776948b2c9b5691b84d1cbd Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Wed, 22 Dec 2021 22:10:09 -0600 Subject: kthread: Generalize pf_io_worker so it can point to struct kthread The point of using set_child_tid to hold the kthread pointer was that it already did what is necessary. There are now restrictions on when set_child_tid can be initialized and when set_child_tid can be used in schedule_tail. Which indicates that continuing to use set_child_tid to hold the kthread pointer is a bad idea. Instead of continuing to use the set_child_tid field of task_struct generalize the pf_io_worker field of task_struct and use it to hold the kthread pointer. Rename pf_io_worker (which is a void * pointer) to worker_private so it can be used to store kthreads struct kthread pointer. Update the kthread code to store the kthread pointer in the worker_private field. Remove the places where set_child_tid had to be dealt with carefully because kthreads also used it. Link: https://lkml.kernel.org/r/CAHk-=wgtFAA9SbVYg0gR1tqPMC17-NYcs0GQkaYg1bGhh1uJQQ@mail.gmail.com Link: https://lkml.kernel.org/r/87a6grvqy8.fsf_-_@email.froward.int.ebiederm.org Suggested-by: Linus Torvalds Signed-off-by: "Eric W. Biederman" --- fs/io-wq.c | 6 +++--- fs/io-wq.h | 2 +- include/linux/sched.h | 4 ++-- kernel/fork.c | 8 +------- kernel/kthread.c | 14 +++++--------- kernel/sched/core.c | 2 +- 6 files changed, 13 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/fs/io-wq.c b/fs/io-wq.c index 88202de519f6..e4fc7384b40c 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -657,7 +657,7 @@ loop: */ void io_wq_worker_running(struct task_struct *tsk) { - struct io_worker *worker = tsk->pf_io_worker; + struct io_worker *worker = tsk->worker_private; if (!worker) return; @@ -675,7 +675,7 @@ void io_wq_worker_running(struct task_struct *tsk) */ void io_wq_worker_sleeping(struct task_struct *tsk) { - struct io_worker *worker = tsk->pf_io_worker; + struct io_worker *worker = tsk->worker_private; if (!worker) return; @@ -694,7 +694,7 @@ void io_wq_worker_sleeping(struct task_struct *tsk) static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker, struct task_struct *tsk) { - tsk->pf_io_worker = worker; + tsk->worker_private = worker; worker->task = tsk; set_cpus_allowed_ptr(tsk, wqe->cpu_mask); tsk->flags |= PF_NO_SETAFFINITY; diff --git a/fs/io-wq.h b/fs/io-wq.h index 41bf37674a49..c7c23947cbcd 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -200,6 +200,6 @@ static inline void io_wq_worker_running(struct task_struct *tsk) static inline bool io_wq_current_is_worker(void) { return in_task() && (current->flags & PF_IO_WORKER) && - current->pf_io_worker; + current->worker_private; } #endif diff --git a/include/linux/sched.h b/include/linux/sched.h index 78c351e35fec..52f2fdffa3ab 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -987,8 +987,8 @@ struct task_struct { /* CLONE_CHILD_CLEARTID: */ int __user *clear_child_tid; - /* PF_IO_WORKER */ - void *pf_io_worker; + /* PF_KTHREAD | PF_IO_WORKER */ + void *worker_private; u64 utime; u64 stime; diff --git a/kernel/fork.c b/kernel/fork.c index 0816be1bb044..6f0293cb29c9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -950,7 +950,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->splice_pipe = NULL; tsk->task_frag.page = NULL; tsk->wake_q.next = NULL; - tsk->pf_io_worker = NULL; + tsk->worker_private = NULL; account_kernel_stack(tsk, 1); @@ -2032,12 +2032,6 @@ static __latent_entropy struct task_struct *copy_process( siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); } - /* - * This _must_ happen before we call free_task(), i.e. before we jump - * to any of the bad_fork_* labels. This is to avoid freeing - * p->set_child_tid which is (ab)used as a kthread's data pointer for - * kernel threads (PF_KTHREAD). - */ p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; /* * Clear TID on mm_release()? diff --git a/kernel/kthread.c b/kernel/kthread.c index c14707d15341..261a3c3b9c6c 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -72,7 +72,7 @@ enum KTHREAD_BITS { static inline struct kthread *to_kthread(struct task_struct *k) { WARN_ON(!(k->flags & PF_KTHREAD)); - return (__force void *)k->set_child_tid; + return k->worker_private; } /* @@ -80,7 +80,7 @@ static inline struct kthread *to_kthread(struct task_struct *k) * * Per construction; when: * - * (p->flags & PF_KTHREAD) && p->set_child_tid + * (p->flags & PF_KTHREAD) && p->worker_private * * the task is both a kthread and struct kthread is persistent. However * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and @@ -88,7 +88,7 @@ static inline struct kthread *to_kthread(struct task_struct *k) */ static inline struct kthread *__to_kthread(struct task_struct *p) { - void *kthread = (__force void *)p->set_child_tid; + void *kthread = p->worker_private; if (kthread && !(p->flags & PF_KTHREAD)) kthread = NULL; return kthread; @@ -109,11 +109,7 @@ bool set_kthread_struct(struct task_struct *p) init_completion(&kthread->parked); p->vfork_done = &kthread->exited; - /* - * We abuse ->set_child_tid to avoid the new member and because it - * can't be wrongly copied by copy_process(). - */ - p->set_child_tid = (__force void __user *)kthread; + p->worker_private = kthread; return true; } @@ -128,7 +124,7 @@ void free_kthread_struct(struct task_struct *k) #ifdef CONFIG_BLK_CGROUP WARN_ON_ONCE(kthread && kthread->blkcg_css); #endif - k->set_child_tid = (__force void __user *)NULL; + k->worker_private = NULL; kfree(kthread); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d8adbea77be1..ee222b89c692 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4908,7 +4908,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) finish_task_switch(prev); preempt_enable(); - if (!(current->flags & PF_KTHREAD) && current->set_child_tid) + if (current->set_child_tid) put_user(task_pid_vnr(current), current->set_child_tid); calculate_sigpending(); -- cgit v1.2.3-59-g8ed1b From 3367d1bd738c01b2737eaab7d922bfe5f1a41f38 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Sat, 8 Jan 2022 16:31:58 +0100 Subject: power: supply: Provide stubs for charge_behaviour helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When CONFIG_SYSFS is not enabled provide stubs for the helper functions to not break their callers. Fixes: 539b9c94ac83 ("power: supply: add helpers for charge_behaviour sysfs") Reported-by: kernel test robot Signed-off-by: Thomas Weißschuh Link: https://lore.kernel.org/r/20220108153158.189489-1-linux@weissschuh.net Signed-off-by: Hans de Goede --- include/linux/power_supply.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 71f0379c2af8..f6b9ed4630fa 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -553,6 +553,21 @@ ssize_t power_supply_charge_behaviour_show(struct device *dev, char *buf); int power_supply_charge_behaviour_parse(unsigned int available_behaviours, const char *buf); +#else +static inline +ssize_t power_supply_charge_behaviour_show(struct device *dev, + unsigned int available_behaviours, + enum power_supply_charge_behaviour behaviour, + char *buf) +{ + return -EOPNOTSUPP; +} + +static inline int power_supply_charge_behaviour_parse(unsigned int available_behaviours, + const char *buf) +{ + return -EOPNOTSUPP; +} #endif #endif /* __LINUX_POWER_SUPPLY_H__ */ -- cgit v1.2.3-59-g8ed1b From 2f824d4d197e02275562359a2ae5274177ce500c Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Sat, 8 Jan 2022 09:48:31 -0600 Subject: signal: Remove SIGNAL_GROUP_COREDUMP After the previous cleanups "signal->core_state" is set whenever SIGNAL_GROUP_COREDUMP is set and "signal->core_state" is tested whenver the code wants to know if a coredump is in progress. The remaining tests of SIGNAL_GROUP_COREDUMP also test to see if SIGNAL_GROUP_EXIT is set. Similarly the only place that sets SIGNAL_GROUP_COREDUMP also sets SIGNAL_GROUP_EXIT. Which makes SIGNAL_GROUP_COREDUMP unecessary and redundant. So stop setting SIGNAL_GROUP_COREDUMP, stop testing SIGNAL_GROUP_COREDUMP, and remove it's definition. With the setting of SIGNAL_GROUP_COREDUMP gone, coredump_finish no longer needs to clear SIGNAL_GROUP_COREDUMP out of signal->flags by setting SIGNAL_GROUP_EXIT. Link: https://lkml.kernel.org/r/20211213225350.27481-5-ebiederm@xmission.com Signed-off-by: "Eric W. Biederman" --- fs/coredump.c | 3 +-- include/linux/sched/signal.h | 3 +-- kernel/signal.c | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/coredump.c b/fs/coredump.c index 0864941a879b..fee1c57aee89 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -353,7 +353,7 @@ static int zap_process(struct task_struct *start, int exit_code) int nr = 0; /* ignore all signals except SIGKILL, see prepare_signal() */ - start->signal->flags = SIGNAL_GROUP_EXIT | SIGNAL_GROUP_COREDUMP; + start->signal->flags = SIGNAL_GROUP_EXIT; start->signal->group_exit_code = exit_code; start->signal->group_stop_count = 0; @@ -427,7 +427,6 @@ static void coredump_finish(bool core_dumped) if (core_dumped && !__fatal_signal_pending(current)) current->signal->group_exit_code |= 0x80; current->signal->group_exit_task = NULL; - current->signal->flags = SIGNAL_GROUP_EXIT; next = current->signal->core_state->dumper.next; current->signal->core_state = NULL; spin_unlock_irq(¤t->sighand->siglock); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index fa26d2a58413..ecc10e148799 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -256,7 +256,6 @@ struct signal_struct { #define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */ #define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */ #define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */ -#define SIGNAL_GROUP_COREDUMP 0x00000008 /* coredump in progress */ /* * Pending notifications to parent. */ @@ -272,7 +271,7 @@ struct signal_struct { static inline void signal_set_stop_flags(struct signal_struct *sig, unsigned int flags) { - WARN_ON(sig->flags & (SIGNAL_GROUP_EXIT|SIGNAL_GROUP_COREDUMP)); + WARN_ON(sig->flags & SIGNAL_GROUP_EXIT); sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags; } diff --git a/kernel/signal.c b/kernel/signal.c index 0706c1345a71..bae231bc2f4a 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -906,7 +906,7 @@ static bool prepare_signal(int sig, struct task_struct *p, bool force) struct task_struct *t; sigset_t flush; - if ((signal->flags & SIGNAL_GROUP_EXIT) || signal->core_state) { + if (signal->flags & SIGNAL_GROUP_EXIT) { if (signal->core_state) return sig == SIGKILL; /* -- cgit v1.2.3-59-g8ed1b From 60700e38fb68e800607ca7a027060d5419fc5798 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Sun, 6 Jun 2021 13:47:53 -0500 Subject: signal: Rename group_exit_task group_exec_task The only remaining user of group_exit_task is exec. Rename the field so that it is clear which part of the code uses it. Update the comment above the definition of group_exec_task to document how it is currently used. Link: https://lkml.kernel.org/r/20211213225350.27481-7-ebiederm@xmission.com Signed-off-by: "Eric W. Biederman" --- fs/exec.c | 8 ++++---- include/linux/sched/signal.h | 12 ++++-------- kernel/exit.c | 4 ++-- 3 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/exec.c b/fs/exec.c index 59cac7c18178..9d2925811011 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1054,7 +1054,7 @@ static int de_thread(struct task_struct *tsk) return -EAGAIN; } - sig->group_exit_task = tsk; + sig->group_exec_task = tsk; sig->notify_count = zap_other_threads(tsk); if (!thread_group_leader(tsk)) sig->notify_count--; @@ -1082,7 +1082,7 @@ static int de_thread(struct task_struct *tsk) write_lock_irq(&tasklist_lock); /* * Do this under tasklist_lock to ensure that - * exit_notify() can't miss ->group_exit_task + * exit_notify() can't miss ->group_exec_task */ sig->notify_count = -1; if (likely(leader->exit_state)) @@ -1149,7 +1149,7 @@ static int de_thread(struct task_struct *tsk) release_task(leader); } - sig->group_exit_task = NULL; + sig->group_exec_task = NULL; sig->notify_count = 0; no_thread_group: @@ -1162,7 +1162,7 @@ no_thread_group: killed: /* protects against exit_notify() and __exit_signal() */ read_lock(&tasklist_lock); - sig->group_exit_task = NULL; + sig->group_exec_task = NULL; sig->notify_count = 0; read_unlock(&tasklist_lock); return -EAGAIN; diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index ecc10e148799..d3248aba5183 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -109,13 +109,9 @@ struct signal_struct { /* thread group exit support */ int group_exit_code; - /* overloaded: - * - notify group_exit_task when ->count is equal to notify_count - * - everyone except group_exit_task is stopped during signal delivery - * of fatal signals, group_exit_task processes the signal. - */ + /* notify group_exec_task when notify_count is less or equal to 0 */ int notify_count; - struct task_struct *group_exit_task; + struct task_struct *group_exec_task; /* thread group stop support, overloads group_exit_code too */ int group_stop_count; @@ -275,11 +271,11 @@ static inline void signal_set_stop_flags(struct signal_struct *sig, sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags; } -/* If true, all threads except ->group_exit_task have pending SIGKILL */ +/* If true, all threads except ->group_exec_task have pending SIGKILL */ static inline int signal_group_exit(const struct signal_struct *sig) { return (sig->flags & SIGNAL_GROUP_EXIT) || - (sig->group_exit_task != NULL); + (sig->group_exec_task != NULL); } extern void flush_signals(struct task_struct *); diff --git a/kernel/exit.c b/kernel/exit.c index fc0726cb22db..b05578abbf26 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -116,7 +116,7 @@ static void __exit_signal(struct task_struct *tsk) * then notify it: */ if (sig->notify_count > 0 && !--sig->notify_count) - wake_up_process(sig->group_exit_task); + wake_up_process(sig->group_exec_task); if (tsk == sig->curr_target) sig->curr_target = next_thread(tsk); @@ -697,7 +697,7 @@ static void exit_notify(struct task_struct *tsk, int group_dead) /* mt-exec, de_thread() is waiting for group leader */ if (unlikely(tsk->signal->notify_count < 0)) - wake_up_process(tsk->signal->group_exit_task); + wake_up_process(tsk->signal->group_exec_task); write_unlock_irq(&tasklist_lock); list_for_each_entry_safe(p, n, &dead, ptrace_entry) { -- cgit v1.2.3-59-g8ed1b From 49697335e0b441b0553598c1b48ee9ebb053d2f1 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Thu, 24 Jun 2021 02:14:30 -0500 Subject: signal: Remove the helper signal_group_exit This helper is misleading. It tests for an ongoing exec as well as the process having received a fatal signal. Sometimes it is appropriate to treat an on-going exec differently than a process that is shutting down due to a fatal signal. In particular taking the fast path out of exit_signals instead of retargeting signals is not appropriate during exec, and not changing the the exit code in do_group_exit during exec. Removing the helper makes it more obvious what is going on as both cases must be coded for explicitly. While removing the helper fix the two cases where I have observed using signal_group_exit resulted in the wrong result. In exit_signals only test for SIGNAL_GROUP_EXIT so that signals are retargetted during an exec. In do_group_exit use 0 as the exit code during an exec as de_thread does not set group_exit_code. As best as I can determine group_exit_code has been is set to 0 most of the time during de_thread. During a thread group stop group_exit_code is set to the stop signal and when the thread group receives SIGCONT group_exit_code is reset to 0. Link: https://lkml.kernel.org/r/20211213225350.27481-8-ebiederm@xmission.com Signed-off-by: "Eric W. Biederman" --- fs/coredump.c | 5 +++-- fs/exec.c | 2 +- include/linux/sched/signal.h | 7 ------- kernel/exit.c | 8 ++++++-- kernel/signal.c | 8 +++++--- 5 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/coredump.c b/fs/coredump.c index c92ffc0bf2c2..7dece20b162b 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -372,11 +372,12 @@ static int zap_process(struct task_struct *start, int exit_code) static int zap_threads(struct task_struct *tsk, struct core_state *core_state, int exit_code) { + struct signal_struct *signal = tsk->signal; int nr = -EAGAIN; spin_lock_irq(&tsk->sighand->siglock); - if (!signal_group_exit(tsk->signal)) { - tsk->signal->core_state = core_state; + if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task) { + signal->core_state = core_state; nr = zap_process(tsk, exit_code); clear_tsk_thread_flag(tsk, TIF_SIGPENDING); tsk->flags |= PF_DUMPCORE; diff --git a/fs/exec.c b/fs/exec.c index 9d2925811011..82db656ca709 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1045,7 +1045,7 @@ static int de_thread(struct task_struct *tsk) * Kill all other threads in the thread group. */ spin_lock_irq(lock); - if (signal_group_exit(sig)) { + if ((sig->flags & SIGNAL_GROUP_EXIT) || sig->group_exec_task) { /* * Another group action in progress, just * return so that the signal is processed. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index d3248aba5183..b6ecb9fc4cd2 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -271,13 +271,6 @@ static inline void signal_set_stop_flags(struct signal_struct *sig, sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags; } -/* If true, all threads except ->group_exec_task have pending SIGKILL */ -static inline int signal_group_exit(const struct signal_struct *sig) -{ - return (sig->flags & SIGNAL_GROUP_EXIT) || - (sig->group_exec_task != NULL); -} - extern void flush_signals(struct task_struct *); extern void ignore_signals(struct task_struct *); extern void flush_signal_handlers(struct task_struct *, int force_default); diff --git a/kernel/exit.c b/kernel/exit.c index b05578abbf26..861cfb1e2f77 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -914,15 +914,19 @@ do_group_exit(int exit_code) BUG_ON(exit_code & 0x80); /* core dumps don't get here */ - if (signal_group_exit(sig)) + if (sig->flags & SIGNAL_GROUP_EXIT) exit_code = sig->group_exit_code; + else if (sig->group_exec_task) + exit_code = 0; else if (!thread_group_empty(current)) { struct sighand_struct *const sighand = current->sighand; spin_lock_irq(&sighand->siglock); - if (signal_group_exit(sig)) + if (sig->flags & SIGNAL_GROUP_EXIT) /* Another thread got here before we took the lock. */ exit_code = sig->group_exit_code; + else if (sig->group_exec_task) + exit_code = 0; else { sig->group_exit_code = exit_code; sig->flags = SIGNAL_GROUP_EXIT; diff --git a/kernel/signal.c b/kernel/signal.c index bae231bc2f4a..167b8e196a79 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -2386,7 +2386,8 @@ static bool do_signal_stop(int signr) WARN_ON_ONCE(signr & ~JOBCTL_STOP_SIGMASK); if (!likely(current->jobctl & JOBCTL_STOP_DEQUEUED) || - unlikely(signal_group_exit(sig))) + unlikely(sig->flags & SIGNAL_GROUP_EXIT) || + unlikely(sig->group_exec_task)) return false; /* * There is no group stop already in progress. We must @@ -2693,7 +2694,8 @@ relock: enum pid_type type; /* Has this task already been marked for death? */ - if (signal_group_exit(signal)) { + if ((signal->flags & SIGNAL_GROUP_EXIT) || + signal->group_exec_task) { ksig->info.si_signo = signr = SIGKILL; sigdelset(¤t->pending.signal, SIGKILL); trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO, @@ -2949,7 +2951,7 @@ void exit_signals(struct task_struct *tsk) */ cgroup_threadgroup_change_begin(tsk); - if (thread_group_empty(tsk) || signal_group_exit(tsk->signal)) { + if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) { tsk->flags |= PF_EXITING; cgroup_threadgroup_change_end(tsk); return; -- cgit v1.2.3-59-g8ed1b From 2d4bcf886e42f0f4846a3d9bdc3a90d278903a2e Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Sat, 8 Jan 2022 11:23:02 -0600 Subject: exit: Remove profile_task_exit & profile_munmap When I say remove I mean remove. All profile_task_exit and profile_munmap do is call a blocking notifier chain. The helpers profile_task_register and profile_task_unregister are not called anywhere in the tree. Which means this is all dead code. So remove the dead code and make it easier to read do_exit. Reviewed-by: Christoph Hellwig Link: https://lkml.kernel.org/r/20220103213312.9144-1-ebiederm@xmission.com Signed-off-by: "Eric W. Biederman" --- include/linux/profile.h | 26 ------------------------- kernel/exit.c | 1 - kernel/profile.c | 50 ------------------------------------------------- mm/mmap.c | 1 - 4 files changed, 78 deletions(-) (limited to 'include/linux') diff --git a/include/linux/profile.h b/include/linux/profile.h index fd18ca96f557..f7eb2b57d890 100644 --- a/include/linux/profile.h +++ b/include/linux/profile.h @@ -31,11 +31,6 @@ static inline int create_proc_profile(void) } #endif -enum profile_type { - PROFILE_TASK_EXIT, - PROFILE_MUNMAP -}; - #ifdef CONFIG_PROFILING extern int prof_on __read_mostly; @@ -66,23 +61,14 @@ static inline void profile_hit(int type, void *ip) struct task_struct; struct mm_struct; -/* task is in do_exit() */ -void profile_task_exit(struct task_struct * task); - /* task is dead, free task struct ? Returns 1 if * the task was taken, 0 if the task should be freed. */ int profile_handoff_task(struct task_struct * task); -/* sys_munmap */ -void profile_munmap(unsigned long addr); - int task_handoff_register(struct notifier_block * n); int task_handoff_unregister(struct notifier_block * n); -int profile_event_register(enum profile_type, struct notifier_block * n); -int profile_event_unregister(enum profile_type, struct notifier_block * n); - #else #define prof_on 0 @@ -117,19 +103,7 @@ static inline int task_handoff_unregister(struct notifier_block * n) return -ENOSYS; } -static inline int profile_event_register(enum profile_type t, struct notifier_block * n) -{ - return -ENOSYS; -} - -static inline int profile_event_unregister(enum profile_type t, struct notifier_block * n) -{ - return -ENOSYS; -} - -#define profile_task_exit(a) do { } while (0) #define profile_handoff_task(a) (0) -#define profile_munmap(a) do { } while (0) #endif /* CONFIG_PROFILING */ diff --git a/kernel/exit.c b/kernel/exit.c index 861cfb1e2f77..64e907bc87d5 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -751,7 +751,6 @@ void __noreturn do_exit(long code) */ force_uaccess_begin(); - profile_task_exit(tsk); kcov_task_exit(tsk); coredump_task_exit(tsk); diff --git a/kernel/profile.c b/kernel/profile.c index eb9c7f0f5ac5..9355cc934a96 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -135,14 +135,7 @@ int __ref profile_init(void) /* Profile event notifications */ -static BLOCKING_NOTIFIER_HEAD(task_exit_notifier); static ATOMIC_NOTIFIER_HEAD(task_free_notifier); -static BLOCKING_NOTIFIER_HEAD(munmap_notifier); - -void profile_task_exit(struct task_struct *task) -{ - blocking_notifier_call_chain(&task_exit_notifier, 0, task); -} int profile_handoff_task(struct task_struct *task) { @@ -151,11 +144,6 @@ int profile_handoff_task(struct task_struct *task) return (ret == NOTIFY_OK) ? 1 : 0; } -void profile_munmap(unsigned long addr) -{ - blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr); -} - int task_handoff_register(struct notifier_block *n) { return atomic_notifier_chain_register(&task_free_notifier, n); @@ -168,44 +156,6 @@ int task_handoff_unregister(struct notifier_block *n) } EXPORT_SYMBOL_GPL(task_handoff_unregister); -int profile_event_register(enum profile_type type, struct notifier_block *n) -{ - int err = -EINVAL; - - switch (type) { - case PROFILE_TASK_EXIT: - err = blocking_notifier_chain_register( - &task_exit_notifier, n); - break; - case PROFILE_MUNMAP: - err = blocking_notifier_chain_register( - &munmap_notifier, n); - break; - } - - return err; -} -EXPORT_SYMBOL_GPL(profile_event_register); - -int profile_event_unregister(enum profile_type type, struct notifier_block *n) -{ - int err = -EINVAL; - - switch (type) { - case PROFILE_TASK_EXIT: - err = blocking_notifier_chain_unregister( - &task_exit_notifier, n); - break; - case PROFILE_MUNMAP: - err = blocking_notifier_chain_unregister( - &munmap_notifier, n); - break; - } - - return err; -} -EXPORT_SYMBOL_GPL(profile_event_unregister); - #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS) /* * Each cpu has a pair of open-addressed hashtables for pending diff --git a/mm/mmap.c b/mm/mmap.c index bfb0ea164a90..70318c2a47c3 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2928,7 +2928,6 @@ EXPORT_SYMBOL(vm_munmap); SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) { addr = untagged_addr(addr); - profile_munmap(addr); return __vm_munmap(addr, len, true); } -- cgit v1.2.3-59-g8ed1b From 2873cd31a20c25b5e763b35e5fb886f0938c6dd5 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Sat, 8 Jan 2022 10:03:24 -0600 Subject: exit: Remove profile_handoff_task All profile_handoff_task does is notify the task_free_notifier chain. The helpers task_handoff_register and task_handoff_unregister are used to add and delete entries from that chain and are never called. So remove the dead code and make it much easier to read and reason about __put_task_struct. Suggested-by: Al Viro Link: https://lkml.kernel.org/r/87fspyw6m0.fsf@email.froward.int.ebiederm.org Signed-off-by: "Eric W. Biederman" --- include/linux/profile.h | 19 ------------------- kernel/fork.c | 4 +--- kernel/profile.c | 23 ----------------------- 3 files changed, 1 insertion(+), 45 deletions(-) (limited to 'include/linux') diff --git a/include/linux/profile.h b/include/linux/profile.h index f7eb2b57d890..11db1ec516e2 100644 --- a/include/linux/profile.h +++ b/include/linux/profile.h @@ -61,14 +61,6 @@ static inline void profile_hit(int type, void *ip) struct task_struct; struct mm_struct; -/* task is dead, free task struct ? Returns 1 if - * the task was taken, 0 if the task should be freed. - */ -int profile_handoff_task(struct task_struct * task); - -int task_handoff_register(struct notifier_block * n); -int task_handoff_unregister(struct notifier_block * n); - #else #define prof_on 0 @@ -93,17 +85,6 @@ static inline void profile_hit(int type, void *ip) return; } -static inline int task_handoff_register(struct notifier_block * n) -{ - return -ENOSYS; -} - -static inline int task_handoff_unregister(struct notifier_block * n) -{ - return -ENOSYS; -} - -#define profile_handoff_task(a) (0) #endif /* CONFIG_PROFILING */ diff --git a/kernel/fork.c b/kernel/fork.c index 6f0293cb29c9..494539ecb6d3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -754,9 +754,7 @@ void __put_task_struct(struct task_struct *tsk) delayacct_tsk_free(tsk); put_signal_struct(tsk->signal); sched_core_free(tsk); - - if (!profile_handoff_task(tsk)) - free_task(tsk); + free_task(tsk); } EXPORT_SYMBOL_GPL(__put_task_struct); diff --git a/kernel/profile.c b/kernel/profile.c index 9355cc934a96..37640a0bd8a3 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -133,29 +133,6 @@ int __ref profile_init(void) return -ENOMEM; } -/* Profile event notifications */ - -static ATOMIC_NOTIFIER_HEAD(task_free_notifier); - -int profile_handoff_task(struct task_struct *task) -{ - int ret; - ret = atomic_notifier_call_chain(&task_free_notifier, 0, task); - return (ret == NOTIFY_OK) ? 1 : 0; -} - -int task_handoff_register(struct notifier_block *n) -{ - return atomic_notifier_chain_register(&task_free_notifier, n); -} -EXPORT_SYMBOL_GPL(task_handoff_register); - -int task_handoff_unregister(struct notifier_block *n) -{ - return atomic_notifier_chain_unregister(&task_free_notifier, n); -} -EXPORT_SYMBOL_GPL(task_handoff_unregister); - #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS) /* * Each cpu has a pair of open-addressed hashtables for pending -- cgit v1.2.3-59-g8ed1b From 4264178416cd52a55a3eccbefb3973866e060280 Mon Sep 17 00:00:00 2001 From: "Eric W. Biederman" Date: Mon, 20 Dec 2021 16:28:53 -0600 Subject: ptrace: Remove unused regs argument from ptrace_report_syscall Link: https://lkml.kernel.org/r/20220103213312.9144-7-ebiederm@xmission.com Signed-off-by: "Eric W. Biederman" --- include/linux/tracehook.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tracehook.h b/include/linux/tracehook.h index 2564b7434b4d..88c007ab5ebc 100644 --- a/include/linux/tracehook.h +++ b/include/linux/tracehook.h @@ -54,8 +54,7 @@ struct linux_binprm; /* * ptrace report for syscall entry and exit looks identical. */ -static inline int ptrace_report_syscall(struct pt_regs *regs, - unsigned long message) +static inline int ptrace_report_syscall(unsigned long message) { int ptrace = current->ptrace; @@ -102,7 +101,7 @@ static inline int ptrace_report_syscall(struct pt_regs *regs, static inline __must_check int tracehook_report_syscall_entry( struct pt_regs *regs) { - return ptrace_report_syscall(regs, PTRACE_EVENTMSG_SYSCALL_ENTRY); + return ptrace_report_syscall(PTRACE_EVENTMSG_SYSCALL_ENTRY); } /** @@ -127,7 +126,7 @@ static inline void tracehook_report_syscall_exit(struct pt_regs *regs, int step) if (step) user_single_step_report(regs); else - ptrace_report_syscall(regs, PTRACE_EVENTMSG_SYSCALL_EXIT); + ptrace_report_syscall(PTRACE_EVENTMSG_SYSCALL_EXIT); } /** -- cgit v1.2.3-59-g8ed1b From 40595cdc93edf4110c0f0c0b06f8d82008f23929 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Thu, 16 Dec 2021 12:20:13 -0500 Subject: nfs: block notification on fs with its own ->lock NFSv4.1 supports an optional lock notification feature which notifies the client when a lock comes available. (Normally NFSv4 clients just poll for locks if necessary.) To make that work, we need to request a blocking lock from the filesystem. We turned that off for NFS in commit f657f8eef3ff ("nfs: don't atempt blocking locks on nfs reexports") [sic] because it actually blocks the nfsd thread while waiting for the lock. Thanks to Vasily Averin for pointing out that NFS isn't the only filesystem with that problem. Any filesystem that leaves ->lock NULL will use posix_lock_file(), which does the right thing. Simplest is just to assume that any filesystem that defines its own ->lock is not safe to request a blocking lock from. So, this patch mostly reverts commit f657f8eef3ff ("nfs: don't atempt blocking locks on nfs reexports") [sic] and commit b840be2f00c0 ("lockd: don't attempt blocking locks on nfs reexports"), and instead uses a check of ->lock (Vasily's suggestion) to decide whether to support blocking lock notifications on a given filesystem. Also add a little documentation. Perhaps someday we could add back an export flag later to allow filesystems with "good" ->lock methods to support blocking lock notifications. Reported-by: Vasily Averin Signed-off-by: J. Bruce Fields [ cel: Description rewritten to address checkpatch nits ] [ cel: Fixed warning when SUNRPC debugging is disabled ] [ cel: Fixed NULL check ] Signed-off-by: Chuck Lever Reviewed-by: Vasily Averin --- fs/lockd/svclock.c | 6 ++++-- fs/nfs/export.c | 2 +- fs/nfsd/nfs4state.c | 18 ++++++++++++------ include/linux/exportfs.h | 2 -- include/linux/lockd/lockd.h | 9 +++++++-- 5 files changed, 24 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index e9b85d8fd5fe..cb3658ab9b7a 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -470,8 +470,10 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, struct nlm_host *host, struct nlm_lock *lock, int wait, struct nlm_cookie *cookie, int reclaim) { - struct nlm_block *block = NULL; +#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) struct inode *inode = nlmsvc_file_inode(file); +#endif + struct nlm_block *block = NULL; int error; int mode; int async_block = 0; @@ -484,7 +486,7 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, (long long)lock->fl.fl_end, wait); - if (inode->i_sb->s_export_op->flags & EXPORT_OP_SYNC_LOCKS) { + if (nlmsvc_file_file(file)->f_op->lock) { async_block = wait; wait = 0; } diff --git a/fs/nfs/export.c b/fs/nfs/export.c index 171c424cb6d5..01596f2d0a1e 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -158,5 +158,5 @@ const struct export_operations nfs_export_ops = { .fetch_iversion = nfs_fetch_iversion, .flags = EXPORT_OP_NOWCC|EXPORT_OP_NOSUBTREECHK| EXPORT_OP_CLOSE_BEFORE_UNLINK|EXPORT_OP_REMOTE_FS| - EXPORT_OP_NOATOMIC_ATTR|EXPORT_OP_SYNC_LOCKS, + EXPORT_OP_NOATOMIC_ATTR, }; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 72e3833c3034..d8faccc55479 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6842,7 +6842,6 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_blocked_lock *nbl = NULL; struct file_lock *file_lock = NULL; struct file_lock *conflock = NULL; - struct super_block *sb; __be32 status = 0; int lkflg; int err; @@ -6864,7 +6863,6 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, dprintk("NFSD: nfsd4_lock: permission denied!\n"); return status; } - sb = cstate->current_fh.fh_dentry->d_sb; if (lock->lk_is_new) { if (nfsd4_has_session(cstate)) @@ -6916,8 +6914,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, fp = lock_stp->st_stid.sc_file; switch (lock->lk_type) { case NFS4_READW_LT: - if (nfsd4_has_session(cstate) && - !(sb->s_export_op->flags & EXPORT_OP_SYNC_LOCKS)) + if (nfsd4_has_session(cstate)) fl_flags |= FL_SLEEP; fallthrough; case NFS4_READ_LT: @@ -6929,8 +6926,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, fl_type = F_RDLCK; break; case NFS4_WRITEW_LT: - if (nfsd4_has_session(cstate) && - !(sb->s_export_op->flags & EXPORT_OP_SYNC_LOCKS)) + if (nfsd4_has_session(cstate)) fl_flags |= FL_SLEEP; fallthrough; case NFS4_WRITE_LT: @@ -6951,6 +6947,16 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out; } + /* + * Most filesystems with their own ->lock operations will block + * the nfsd thread waiting to acquire the lock. That leads to + * deadlocks (we don't want every nfsd thread tied up waiting + * for file locks), so don't attempt blocking lock notifications + * on those filesystems: + */ + if (nf->nf_file->f_op->lock) + fl_flags &= ~FL_SLEEP; + nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn); if (!nbl) { dprintk("NFSD: %s: unable to allocate block!\n", __func__); diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 3260fe714846..fe848901fcc3 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -221,8 +221,6 @@ struct export_operations { #define EXPORT_OP_NOATOMIC_ATTR (0x10) /* Filesystem cannot supply atomic attribute updates */ -#define EXPORT_OP_SYNC_LOCKS (0x20) /* Filesystem can't do - asychronous blocking locks */ unsigned long flags; }; diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index c4ae6506b8b3..fcef192e5e45 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -303,10 +303,15 @@ void nlmsvc_invalidate_all(void); int nlmsvc_unlock_all_by_sb(struct super_block *sb); int nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr); +static inline struct file *nlmsvc_file_file(struct nlm_file *file) +{ + return file->f_file[O_RDONLY] ? + file->f_file[O_RDONLY] : file->f_file[O_WRONLY]; +} + static inline struct inode *nlmsvc_file_inode(struct nlm_file *file) { - return locks_inode(file->f_file[O_RDONLY] ? - file->f_file[O_RDONLY] : file->f_file[O_WRONLY]); + return locks_inode(nlmsvc_file_file(file)); } static inline int __nlm_privileged_request4(const struct sockaddr *sap) -- cgit v1.2.3-59-g8ed1b From 0aa698787aa2a9e8840987e54ba2982559de6404 Mon Sep 17 00:00:00 2001 From: axelj Date: Mon, 13 Dec 2021 08:09:25 +0100 Subject: tpm: Add Upgrade/Reduced mode support for TPM2 modules If something went wrong during the TPM firmware upgrade, like power failure or the firmware image file get corrupted, the TPM might end up in Upgrade or Failure mode upon the next start. The state is persistent between the TPM power cycle/restart. According to TPM specification: * If the TPM is in Upgrade mode, it will answer with TPM2_RC_UPGRADE to all commands except TPM2_FieldUpgradeData(). It may also accept other commands if it is able to complete them using the previously installed firmware. * If the TPM is in Failure mode, it will allow performing TPM initialization but will not provide any crypto operations. Will happily respond to Field Upgrade calls. Change the behavior of the tpm2_auto_startup(), so it detects the active running mode of the TPM by adding the following checks. If tpm2_do_selftest() call returns TPM2_RC_UPGRADE, the TPM is in Upgrade mode. If the TPM is in Failure mode, it will successfully respond to both tpm2_do_selftest() and tpm2_startup() calls. Although, will fail to answer to tpm2_get_cc_attrs_tbl(). Use this fact to conclude that TPM is in Failure mode. If detected that the TPM is in the Upgrade or Failure mode, the function sets TPM_CHIP_FLAG_FIRMWARE_UPGRADE_MODE flag. The TPM_CHIP_FLAG_FIRMWARE_UPGRADE_MODE flag is used later during driver initialization/deinitialization to disable functionality which makes no sense or will fail in the current TPM state. Following functionality is affected: * Do not register TPM as a hwrng * Do not register sysfs entries which provide information impossible to obtain in limited mode * Do not register resource managed character device Signed-off-by: axelj Reviewed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen --- drivers/char/tpm/tpm-chip.c | 19 ++++++++++++------- drivers/char/tpm/tpm-sysfs.c | 3 +++ drivers/char/tpm/tpm2-cmd.c | 6 ++++++ include/linux/tpm.h | 10 ++++++++++ 4 files changed, 31 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index ddaeceb7e109..b4ed3ae67a4b 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -444,7 +444,7 @@ static int tpm_add_char_device(struct tpm_chip *chip) return rc; } - if (chip->flags & TPM_CHIP_FLAG_TPM2) { + if (chip->flags & TPM_CHIP_FLAG_TPM2 && !tpm_is_firmware_upgrade(chip)) { rc = cdev_device_add(&chip->cdevs, &chip->devs); if (rc) { dev_err(&chip->devs, @@ -488,7 +488,8 @@ static void tpm_del_legacy_sysfs(struct tpm_chip *chip) { struct attribute **i; - if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL)) + if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL) || + tpm_is_firmware_upgrade(chip)) return; sysfs_remove_link(&chip->dev.parent->kobj, "ppi"); @@ -506,7 +507,8 @@ static int tpm_add_legacy_sysfs(struct tpm_chip *chip) struct attribute **i; int rc; - if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL)) + if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL) || + tpm_is_firmware_upgrade(chip)) return 0; rc = compat_only_sysfs_link_entry_to_kobj( @@ -536,7 +538,7 @@ static int tpm_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait) static int tpm_add_hwrng(struct tpm_chip *chip) { - if (!IS_ENABLED(CONFIG_HW_RANDOM_TPM)) + if (!IS_ENABLED(CONFIG_HW_RANDOM_TPM) || tpm_is_firmware_upgrade(chip)) return 0; snprintf(chip->hwrng_name, sizeof(chip->hwrng_name), @@ -550,6 +552,9 @@ static int tpm_get_pcr_allocation(struct tpm_chip *chip) { int rc; + if (tpm_is_firmware_upgrade(chip)) + return 0; + rc = (chip->flags & TPM_CHIP_FLAG_TPM2) ? tpm2_get_pcr_allocation(chip) : tpm1_get_pcr_allocation(chip); @@ -612,7 +617,7 @@ int tpm_chip_register(struct tpm_chip *chip) return 0; out_hwrng: - if (IS_ENABLED(CONFIG_HW_RANDOM_TPM)) + if (IS_ENABLED(CONFIG_HW_RANDOM_TPM) && !tpm_is_firmware_upgrade(chip)) hwrng_unregister(&chip->hwrng); out_ppi: tpm_bios_log_teardown(chip); @@ -637,10 +642,10 @@ EXPORT_SYMBOL_GPL(tpm_chip_register); void tpm_chip_unregister(struct tpm_chip *chip) { tpm_del_legacy_sysfs(chip); - if (IS_ENABLED(CONFIG_HW_RANDOM_TPM)) + if (IS_ENABLED(CONFIG_HW_RANDOM_TPM) && !tpm_is_firmware_upgrade(chip)) hwrng_unregister(&chip->hwrng); tpm_bios_log_teardown(chip); - if (chip->flags & TPM_CHIP_FLAG_TPM2) + if (chip->flags & TPM_CHIP_FLAG_TPM2 && !tpm_is_firmware_upgrade(chip)) cdev_device_del(&chip->cdevs, &chip->devs); tpm_del_char_device(chip); } diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c index 63f03cfb8e6a..54c71473aa29 100644 --- a/drivers/char/tpm/tpm-sysfs.c +++ b/drivers/char/tpm/tpm-sysfs.c @@ -480,6 +480,9 @@ void tpm_sysfs_add_device(struct tpm_chip *chip) WARN_ON(chip->groups_cnt != 0); + if (tpm_is_firmware_upgrade(chip)) + return; + if (chip->flags & TPM_CHIP_FLAG_TPM2) chip->groups[chip->groups_cnt++] = &tpm2_dev_group; else diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index a25815a6f625..4704fa553098 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -745,6 +745,12 @@ int tpm2_auto_startup(struct tpm_chip *chip) rc = tpm2_get_cc_attrs_tbl(chip); out: + if (rc == TPM2_RC_UPGRADE) { + dev_info(&chip->dev, "TPM in field upgrade mode, requires firmware upgrade\n"); + chip->flags |= TPM_CHIP_FLAG_FIRMWARE_UPGRADE; + rc = 0; + } + if (rc > 0) rc = -ENODEV; return rc; diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 12d827734686..dfeb25a0362d 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -207,6 +207,7 @@ enum tpm2_return_codes { TPM2_RC_INITIALIZE = 0x0100, /* RC_VER1 */ TPM2_RC_FAILURE = 0x0101, TPM2_RC_DISABLED = 0x0120, + TPM2_RC_UPGRADE = 0x012D, TPM2_RC_COMMAND_CODE = 0x0143, TPM2_RC_TESTING = 0x090A, /* RC_WARN */ TPM2_RC_REFERENCE_H0 = 0x0910, @@ -278,6 +279,7 @@ enum tpm_chip_flags { TPM_CHIP_FLAG_HAVE_TIMEOUTS = BIT(4), TPM_CHIP_FLAG_ALWAYS_POWERED = BIT(5), TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED = BIT(6), + TPM_CHIP_FLAG_FIRMWARE_UPGRADE = BIT(7), }; #define to_tpm_chip(d) container_of(d, struct tpm_chip, dev) @@ -399,6 +401,14 @@ static inline void tpm_buf_append_u32(struct tpm_buf *buf, const u32 value) tpm_buf_append(buf, (u8 *) &value2, 4); } +/* + * Check if TPM device is in the firmware upgrade mode. + */ +static inline bool tpm_is_firmware_upgrade(struct tpm_chip *chip) +{ + return chip->flags & TPM_CHIP_FLAG_FIRMWARE_UPGRADE; +} + static inline u32 tpm2_rc_value(u32 rc) { return (rc & BIT(7)) ? rc & 0xff : rc; -- cgit v1.2.3-59-g8ed1b From 292c33c95defd0b814fec1fc8cd60d16556cf7b8 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 7 Jan 2022 08:52:28 +0800 Subject: block: fix old-style declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the 'inline' keyword to the front of 'void'. Remove a warning found by clang(make W=1 LLVM=1) ./include/linux/blk-mq.h:259:1: warning: ‘inline’ is not at beginning of declaration Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/20220107005228.103927-1-yang.lee@linux.alibaba.com Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index f40a05ecca4a..d319ffa59354 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -256,7 +256,7 @@ static inline unsigned short req_get_ioprio(struct request *req) * @rq: The request to move * @prev: The request preceding @rq in @src (NULL if @rq is the head) */ -static void inline rq_list_move(struct request **src, struct request **dst, +static inline void rq_list_move(struct request **src, struct request **dst, struct request *rq, struct request *prev) { if (prev) -- cgit v1.2.3-59-g8ed1b From 0ea9fc15b1d7d6636d429e74ffe3f86bf2f2f7d6 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 23 Nov 2021 17:05:07 +0100 Subject: fs/locks: fix fcntl_getlk64/fcntl_setlk64 stub prototypes My patch to rework oabi fcntl64() introduced a harmless sparse warning when file locking is disabled: arch/arm/kernel/sys_oabi-compat.c:251:51: sparse: sparse: incorrect type in argument 3 (different address spaces) @@ expected struct flock64 [noderef] __user *user @@ got struct flock64 * @@ arch/arm/kernel/sys_oabi-compat.c:251:51: sparse: expected struct flock64 [noderef] __user *user arch/arm/kernel/sys_oabi-compat.c:251:51: sparse: got struct flock64 * arch/arm/kernel/sys_oabi-compat.c:265:55: sparse: sparse: incorrect type in argument 4 (different address spaces) @@ expected struct flock64 [noderef] __user *user @@ got struct flock64 * @@ arch/arm/kernel/sys_oabi-compat.c:265:55: sparse: expected struct flock64 [noderef] __user *user arch/arm/kernel/sys_oabi-compat.c:265:55: sparse: got struct flock64 * When file locking is enabled, everything works correctly and the right data gets passed, but the stub declarations in linux/fs.h did not get modified when the calling conventions changed in an earlier patch. Reported-by: kernel test robot Fixes: 7e2d8c29ecdd ("ARM: 9111/1: oabi-compat: rework fcntl64() emulation") Fixes: a75d30c77207 ("fs/locks: pass kernel struct flock to fcntl_getlk/setlk") Cc: Christoph Hellwig Reviewed-by: Christoph Hellwig Acked-by: Christian Brauner Signed-off-by: Arnd Bergmann Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/fs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index bbf812ce89a8..5122d13775c2 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1220,13 +1220,13 @@ static inline int fcntl_setlk(unsigned int fd, struct file *file, #if BITS_PER_LONG == 32 static inline int fcntl_getlk64(struct file *file, unsigned int cmd, - struct flock64 __user *user) + struct flock64 *user) { return -EINVAL; } static inline int fcntl_setlk64(unsigned int fd, struct file *file, - unsigned int cmd, struct flock64 __user *user) + unsigned int cmd, struct flock64 *user) { return -EACCES; } -- cgit v1.2.3-59-g8ed1b From 719774377622bc4025d2a74f551b5dc2158c6c30 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 7 Jan 2022 05:03:22 +0100 Subject: netfilter: conntrack: convert to refcount_t api Convert nf_conn reference counting from atomic_t to refcount_t based api. refcount_t api provides more runtime sanity checks and will warn on certain constructs, e.g. refcount_inc() on a zero reference count, which usually indicates use-after-free. For this reason template allocation is changed to init the refcount to 1, the subsequenct add operations are removed. Likewise, init_conntrack() is changed to set the initial refcount to 1 instead refcount_inc(). This is safe because the new entry is not (yet) visible to other cpus. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter/nf_conntrack_common.h | 8 ++++---- net/netfilter/nf_conntrack_core.c | 26 +++++++++++++------------- net/netfilter/nf_conntrack_expect.c | 4 ++-- net/netfilter/nf_conntrack_netlink.c | 6 +++--- net/netfilter/nf_conntrack_standalone.c | 4 ++-- net/netfilter/nf_flow_table_core.c | 2 +- net/netfilter/nf_synproxy_core.c | 1 - net/netfilter/nft_ct.c | 4 +--- net/netfilter/xt_CT.c | 3 +-- net/openvswitch/conntrack.c | 1 - net/sched/act_ct.c | 1 - 11 files changed, 27 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter/nf_conntrack_common.h b/include/linux/netfilter/nf_conntrack_common.h index 700ea077ce2d..a03f7a80b9ab 100644 --- a/include/linux/netfilter/nf_conntrack_common.h +++ b/include/linux/netfilter/nf_conntrack_common.h @@ -2,7 +2,7 @@ #ifndef _NF_CONNTRACK_COMMON_H #define _NF_CONNTRACK_COMMON_H -#include +#include #include struct ip_conntrack_stat { @@ -25,19 +25,19 @@ struct ip_conntrack_stat { #define NFCT_PTRMASK ~(NFCT_INFOMASK) struct nf_conntrack { - atomic_t use; + refcount_t use; }; void nf_conntrack_destroy(struct nf_conntrack *nfct); static inline void nf_conntrack_put(struct nf_conntrack *nfct) { - if (nfct && atomic_dec_and_test(&nfct->use)) + if (nfct && refcount_dec_and_test(&nfct->use)) nf_conntrack_destroy(nfct); } static inline void nf_conntrack_get(struct nf_conntrack *nfct) { if (nfct) - atomic_inc(&nfct->use); + refcount_inc(&nfct->use); } #endif /* _NF_CONNTRACK_COMMON_H */ diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index bed0017cadb0..328d359fcabe 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -585,7 +585,7 @@ struct nf_conn *nf_ct_tmpl_alloc(struct net *net, tmpl->status = IPS_TEMPLATE; write_pnet(&tmpl->ct_net, net); nf_ct_zone_add(tmpl, zone); - atomic_set(&tmpl->ct_general.use, 0); + refcount_set(&tmpl->ct_general.use, 1); return tmpl; } @@ -618,7 +618,7 @@ destroy_conntrack(struct nf_conntrack *nfct) struct nf_conn *ct = (struct nf_conn *)nfct; pr_debug("destroy_conntrack(%p)\n", ct); - WARN_ON(atomic_read(&nfct->use) != 0); + WARN_ON(refcount_read(&nfct->use) != 0); if (unlikely(nf_ct_is_template(ct))) { nf_ct_tmpl_free(ct); @@ -742,7 +742,7 @@ nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2) /* caller must hold rcu readlock and none of the nf_conntrack_locks */ static void nf_ct_gc_expired(struct nf_conn *ct) { - if (!atomic_inc_not_zero(&ct->ct_general.use)) + if (!refcount_inc_not_zero(&ct->ct_general.use)) return; if (nf_ct_should_gc(ct)) @@ -810,7 +810,7 @@ __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone, * in, try to obtain a reference and re-check tuple */ ct = nf_ct_tuplehash_to_ctrack(h); - if (likely(atomic_inc_not_zero(&ct->ct_general.use))) { + if (likely(refcount_inc_not_zero(&ct->ct_general.use))) { if (likely(nf_ct_key_equal(h, tuple, zone, net))) goto found; @@ -907,7 +907,7 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct) smp_wmb(); /* The caller holds a reference to this object */ - atomic_set(&ct->ct_general.use, 2); + refcount_set(&ct->ct_general.use, 2); __nf_conntrack_hash_insert(ct, hash, reply_hash); nf_conntrack_double_unlock(hash, reply_hash); NF_CT_STAT_INC(net, insert); @@ -958,7 +958,7 @@ static void __nf_conntrack_insert_prepare(struct nf_conn *ct) { struct nf_conn_tstamp *tstamp; - atomic_inc(&ct->ct_general.use); + refcount_inc(&ct->ct_general.use); ct->status |= IPS_CONFIRMED; /* set conntrack timestamp, if enabled. */ @@ -1351,7 +1351,7 @@ static unsigned int early_drop_list(struct net *net, nf_ct_is_dying(tmp)) continue; - if (!atomic_inc_not_zero(&tmp->ct_general.use)) + if (!refcount_inc_not_zero(&tmp->ct_general.use)) continue; /* kill only if still in same netns -- might have moved due to @@ -1469,7 +1469,7 @@ static void gc_worker(struct work_struct *work) continue; /* need to take reference to avoid possible races */ - if (!atomic_inc_not_zero(&tmp->ct_general.use)) + if (!refcount_inc_not_zero(&tmp->ct_general.use)) continue; if (gc_worker_skip_ct(tmp)) { @@ -1569,7 +1569,7 @@ __nf_conntrack_alloc(struct net *net, /* Because we use RCU lookups, we set ct_general.use to zero before * this is inserted in any list. */ - atomic_set(&ct->ct_general.use, 0); + refcount_set(&ct->ct_general.use, 0); return ct; out: atomic_dec(&cnet->count); @@ -1594,7 +1594,7 @@ void nf_conntrack_free(struct nf_conn *ct) /* A freed object has refcnt == 0, that's * the golden rule for SLAB_TYPESAFE_BY_RCU */ - WARN_ON(atomic_read(&ct->ct_general.use) != 0); + WARN_ON(refcount_read(&ct->ct_general.use) != 0); nf_ct_ext_destroy(ct); kmem_cache_free(nf_conntrack_cachep, ct); @@ -1686,8 +1686,8 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, if (!exp) __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC); - /* Now it is inserted into the unconfirmed list, bump refcount */ - nf_conntrack_get(&ct->ct_general); + /* Now it is inserted into the unconfirmed list, set refcount to 1. */ + refcount_set(&ct->ct_general.use, 1); nf_ct_add_to_unconfirmed_list(ct); local_bh_enable(); @@ -2300,7 +2300,7 @@ get_next_corpse(int (*iter)(struct nf_conn *i, void *data), return NULL; found: - atomic_inc(&ct->ct_general.use); + refcount_inc(&ct->ct_general.use); spin_unlock(lockp); local_bh_enable(); return ct; diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 1e89b595ecd0..96948e98ec53 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -203,12 +203,12 @@ nf_ct_find_expectation(struct net *net, * about to invoke ->destroy(), or nf_ct_delete() via timeout * or early_drop(). * - * The atomic_inc_not_zero() check tells: If that fails, we + * The refcount_inc_not_zero() check tells: If that fails, we * know that the ct is being destroyed. If it succeeds, we * can be sure the ct cannot disappear underneath. */ if (unlikely(nf_ct_is_dying(exp->master) || - !atomic_inc_not_zero(&exp->master->ct_general.use))) + !refcount_inc_not_zero(&exp->master->ct_general.use))) return NULL; if (exp->flags & NF_CT_EXPECT_PERMANENT) { diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 4f53d9480ed5..13e2e0390c77 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -508,7 +508,7 @@ nla_put_failure: static int ctnetlink_dump_use(struct sk_buff *skb, const struct nf_conn *ct) { - if (nla_put_be32(skb, CTA_USE, htonl(atomic_read(&ct->ct_general.use)))) + if (nla_put_be32(skb, CTA_USE, htonl(refcount_read(&ct->ct_general.use)))) goto nla_put_failure; return 0; @@ -1200,7 +1200,7 @@ restart: ct = nf_ct_tuplehash_to_ctrack(h); if (nf_ct_is_expired(ct)) { if (i < ARRAY_SIZE(nf_ct_evict) && - atomic_inc_not_zero(&ct->ct_general.use)) + refcount_inc_not_zero(&ct->ct_general.use)) nf_ct_evict[i++] = ct; continue; } @@ -1748,7 +1748,7 @@ restart: NFNL_MSG_TYPE(cb->nlh->nlmsg_type), ct, dying, 0); if (res < 0) { - if (!atomic_inc_not_zero(&ct->ct_general.use)) + if (!refcount_inc_not_zero(&ct->ct_general.use)) continue; cb->args[0] = cpu; cb->args[1] = (unsigned long)ct; diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c index 80f675d884b2..3e1afd10a9b6 100644 --- a/net/netfilter/nf_conntrack_standalone.c +++ b/net/netfilter/nf_conntrack_standalone.c @@ -303,7 +303,7 @@ static int ct_seq_show(struct seq_file *s, void *v) int ret = 0; WARN_ON(!ct); - if (unlikely(!atomic_inc_not_zero(&ct->ct_general.use))) + if (unlikely(!refcount_inc_not_zero(&ct->ct_general.use))) return 0; if (nf_ct_should_gc(ct)) { @@ -370,7 +370,7 @@ static int ct_seq_show(struct seq_file *s, void *v) ct_show_zone(s, ct, NF_CT_DEFAULT_ZONE_DIR); ct_show_delta_time(s, ct); - seq_printf(s, "use=%u\n", atomic_read(&ct->ct_general.use)); + seq_printf(s, "use=%u\n", refcount_read(&ct->ct_general.use)); if (seq_has_overflowed(s)) goto release; diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index ed37bb9b4e58..b90eca7a2f22 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -48,7 +48,7 @@ struct flow_offload *flow_offload_alloc(struct nf_conn *ct) struct flow_offload *flow; if (unlikely(nf_ct_is_dying(ct) || - !atomic_inc_not_zero(&ct->ct_general.use))) + !refcount_inc_not_zero(&ct->ct_general.use))) return NULL; flow = kzalloc(sizeof(*flow), GFP_ATOMIC); diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c index 3d6d49420db8..2dfc5dae0656 100644 --- a/net/netfilter/nf_synproxy_core.c +++ b/net/netfilter/nf_synproxy_core.c @@ -349,7 +349,6 @@ static int __net_init synproxy_net_init(struct net *net) goto err2; __set_bit(IPS_CONFIRMED_BIT, &ct->status); - nf_conntrack_get(&ct->ct_general); snet->tmpl = ct; snet->stats = alloc_percpu(struct synproxy_stats); diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 99b1de14ff7e..518d96c8c247 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -259,7 +259,7 @@ static void nft_ct_set_zone_eval(const struct nft_expr *expr, ct = this_cpu_read(nft_ct_pcpu_template); - if (likely(atomic_read(&ct->ct_general.use) == 1)) { + if (likely(refcount_read(&ct->ct_general.use) == 1)) { nf_ct_zone_add(ct, &zone); } else { /* previous skb got queued to userspace */ @@ -270,7 +270,6 @@ static void nft_ct_set_zone_eval(const struct nft_expr *expr, } } - atomic_inc(&ct->ct_general.use); nf_ct_set(skb, ct, IP_CT_NEW); } #endif @@ -375,7 +374,6 @@ static bool nft_ct_tmpl_alloc_pcpu(void) return false; } - atomic_set(&tmp->ct_general.use, 1); per_cpu(nft_ct_pcpu_template, cpu) = tmp; } diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index 0a913ce07425..267757b0392a 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -24,7 +24,7 @@ static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct) return XT_CONTINUE; if (ct) { - atomic_inc(&ct->ct_general.use); + refcount_inc(&ct->ct_general.use); nf_ct_set(skb, ct, IP_CT_NEW); } else { nf_ct_set(skb, ct, IP_CT_UNTRACKED); @@ -201,7 +201,6 @@ static int xt_ct_tg_check(const struct xt_tgchk_param *par, goto err4; } __set_bit(IPS_CONFIRMED_BIT, &ct->status); - nf_conntrack_get(&ct->ct_general); out: info->ct = ct; return 0; diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 1b5eae57bc90..121664e52271 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -1716,7 +1716,6 @@ int ovs_ct_copy_action(struct net *net, const struct nlattr *attr, goto err_free_ct; __set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status); - nf_conntrack_get(&ct_info.ct->ct_general); return 0; err_free_ct: __ovs_ct_free_action(&ct_info); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index ab1810f2e660..3fa904cf8f27 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1228,7 +1228,6 @@ static int tcf_ct_fill_params(struct net *net, return -ENOMEM; } __set_bit(IPS_CONFIRMED_BIT, &tmpl->status); - nf_conntrack_get(&tmpl->ct_general); p->tmpl = tmpl; return 0; -- cgit v1.2.3-59-g8ed1b From 3fce16493dc1aa2c9af3d7e7bd360dfe203a3e6a Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 7 Jan 2022 05:03:23 +0100 Subject: netfilter: core: move ip_ct_attach indirection to struct nf_ct_hook ip_ct_attach predates struct nf_ct_hook, we can place it there and remove the exported symbol. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter.h | 2 +- net/netfilter/core.c | 19 ++++++++----------- net/netfilter/nf_conntrack_core.c | 4 +--- 3 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 3fda1a508733..e0e3f3355ab1 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -440,7 +440,6 @@ nf_nat_decode_session(struct sk_buff *skb, struct flowi *fl, u_int8_t family) #if IS_ENABLED(CONFIG_NF_CONNTRACK) #include -extern void (*ip_ct_attach)(struct sk_buff *, const struct sk_buff *) __rcu; void nf_ct_attach(struct sk_buff *, const struct sk_buff *); struct nf_conntrack_tuple; bool nf_ct_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple, @@ -463,6 +462,7 @@ struct nf_ct_hook { void (*destroy)(struct nf_conntrack *); bool (*get_tuple_skb)(struct nf_conntrack_tuple *, const struct sk_buff *); + void (*attach)(struct sk_buff *nskb, const struct sk_buff *skb); }; extern struct nf_ct_hook __rcu *nf_ct_hook; diff --git a/net/netfilter/core.c b/net/netfilter/core.c index 6dec9cd395f1..dc806dc9fe28 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -673,25 +673,22 @@ struct nf_ct_hook __rcu *nf_ct_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_hook); #if IS_ENABLED(CONFIG_NF_CONNTRACK) -/* This does not belong here, but locally generated errors need it if connection - tracking in use: without this, connection may not be in hash table, and hence - manufactured ICMP or RST packets will not be associated with it. */ -void (*ip_ct_attach)(struct sk_buff *, const struct sk_buff *) - __rcu __read_mostly; -EXPORT_SYMBOL(ip_ct_attach); - struct nf_nat_hook __rcu *nf_nat_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_nat_hook); +/* This does not belong here, but locally generated errors need it if connection + * tracking in use: without this, connection may not be in hash table, and hence + * manufactured ICMP or RST packets will not be associated with it. + */ void nf_ct_attach(struct sk_buff *new, const struct sk_buff *skb) { - void (*attach)(struct sk_buff *, const struct sk_buff *); + const struct nf_ct_hook *ct_hook; if (skb->_nfct) { rcu_read_lock(); - attach = rcu_dereference(ip_ct_attach); - if (attach) - attach(new, skb); + ct_hook = rcu_dereference(nf_ct_hook); + if (ct_hook) + ct_hook->attach(new, skb); rcu_read_unlock(); } } diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 328d359fcabe..89f1e5f0090b 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -2455,7 +2455,6 @@ static int kill_all(struct nf_conn *i, void *data) void nf_conntrack_cleanup_start(void) { conntrack_gc_work.exiting = true; - RCU_INIT_POINTER(ip_ct_attach, NULL); } void nf_conntrack_cleanup_end(void) @@ -2774,12 +2773,11 @@ static struct nf_ct_hook nf_conntrack_hook = { .update = nf_conntrack_update, .destroy = destroy_conntrack, .get_tuple_skb = nf_conntrack_get_tuple_skb, + .attach = nf_conntrack_attach, }; void nf_conntrack_init_end(void) { - /* For use by REJECT target */ - RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach); RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook); } -- cgit v1.2.3-59-g8ed1b From 285c8a7a58158cb1805c97ff03875df2ba2ea1fe Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 7 Jan 2022 05:03:24 +0100 Subject: netfilter: make function op structures const No functional changes, these structures should be const. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter.h | 8 ++++---- net/netfilter/core.c | 10 +++++----- net/netfilter/nf_conntrack_core.c | 4 ++-- net/netfilter/nf_conntrack_netlink.c | 4 ++-- net/netfilter/nf_nat_core.c | 2 +- net/netfilter/nfnetlink_queue.c | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index e0e3f3355ab1..15e71bfff726 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -381,13 +381,13 @@ struct nf_nat_hook { enum ip_conntrack_dir dir); }; -extern struct nf_nat_hook __rcu *nf_nat_hook; +extern const struct nf_nat_hook __rcu *nf_nat_hook; static inline void nf_nat_decode_session(struct sk_buff *skb, struct flowi *fl, u_int8_t family) { #if IS_ENABLED(CONFIG_NF_NAT) - struct nf_nat_hook *nat_hook; + const struct nf_nat_hook *nat_hook; rcu_read_lock(); nat_hook = rcu_dereference(nf_nat_hook); @@ -464,7 +464,7 @@ struct nf_ct_hook { const struct sk_buff *); void (*attach)(struct sk_buff *nskb, const struct sk_buff *skb); }; -extern struct nf_ct_hook __rcu *nf_ct_hook; +extern const struct nf_ct_hook __rcu *nf_ct_hook; struct nlattr; @@ -479,7 +479,7 @@ struct nfnl_ct_hook { void (*seq_adjust)(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, s32 off); }; -extern struct nfnl_ct_hook __rcu *nfnl_ct_hook; +extern const struct nfnl_ct_hook __rcu *nfnl_ct_hook; /** * nf_skb_duplicated - TEE target has sent a packet diff --git a/net/netfilter/core.c b/net/netfilter/core.c index dc806dc9fe28..354cb472f386 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -666,14 +666,14 @@ EXPORT_SYMBOL(nf_hook_slow_list); /* This needs to be compiled in any case to avoid dependencies between the * nfnetlink_queue code and nf_conntrack. */ -struct nfnl_ct_hook __rcu *nfnl_ct_hook __read_mostly; +const struct nfnl_ct_hook __rcu *nfnl_ct_hook __read_mostly; EXPORT_SYMBOL_GPL(nfnl_ct_hook); -struct nf_ct_hook __rcu *nf_ct_hook __read_mostly; +const struct nf_ct_hook __rcu *nf_ct_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_hook); #if IS_ENABLED(CONFIG_NF_CONNTRACK) -struct nf_nat_hook __rcu *nf_nat_hook __read_mostly; +const struct nf_nat_hook __rcu *nf_nat_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_nat_hook); /* This does not belong here, but locally generated errors need it if connection @@ -696,7 +696,7 @@ EXPORT_SYMBOL(nf_ct_attach); void nf_conntrack_destroy(struct nf_conntrack *nfct) { - struct nf_ct_hook *ct_hook; + const struct nf_ct_hook *ct_hook; rcu_read_lock(); ct_hook = rcu_dereference(nf_ct_hook); @@ -709,7 +709,7 @@ EXPORT_SYMBOL(nf_conntrack_destroy); bool nf_ct_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple, const struct sk_buff *skb) { - struct nf_ct_hook *ct_hook; + const struct nf_ct_hook *ct_hook; bool ret = false; rcu_read_lock(); diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 89f1e5f0090b..cd3d07e418b5 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -2085,9 +2085,9 @@ static int __nf_conntrack_update(struct net *net, struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo) { + const struct nf_nat_hook *nat_hook; struct nf_conntrack_tuple_hash *h; struct nf_conntrack_tuple tuple; - struct nf_nat_hook *nat_hook; unsigned int status; int dataoff; u16 l3num; @@ -2769,7 +2769,7 @@ err_cachep: return ret; } -static struct nf_ct_hook nf_conntrack_hook = { +static const struct nf_ct_hook nf_conntrack_hook = { .update = nf_conntrack_update, .destroy = destroy_conntrack, .get_tuple_skb = nf_conntrack_get_tuple_skb, diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 13e2e0390c77..01cc69ab0b4e 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1819,7 +1819,7 @@ ctnetlink_parse_nat_setup(struct nf_conn *ct, const struct nlattr *attr) __must_hold(RCU) { - struct nf_nat_hook *nat_hook; + const struct nf_nat_hook *nat_hook; int err; nat_hook = rcu_dereference(nf_nat_hook); @@ -2921,7 +2921,7 @@ static void ctnetlink_glue_seqadj(struct sk_buff *skb, struct nf_conn *ct, nf_ct_tcp_seqadj_set(skb, ct, ctinfo, diff); } -static struct nfnl_ct_hook ctnetlink_glue_hook = { +static const struct nfnl_ct_hook ctnetlink_glue_hook = { .build_size = ctnetlink_glue_build_size, .build = ctnetlink_glue_build, .parse = ctnetlink_glue_parse, diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 3dd130487b5b..2d06a66899b2 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -1167,7 +1167,7 @@ static struct pernet_operations nat_net_ops = { .size = sizeof(struct nat_net), }; -static struct nf_nat_hook nat_hook = { +static const struct nf_nat_hook nat_hook = { .parse_nat_setup = nfnetlink_parse_nat_setup, #ifdef CONFIG_XFRM .decode_session = __nf_nat_decode_session, diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 08771f47d469..862d8a3a0911 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -225,7 +225,7 @@ find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { - struct nf_ct_hook *ct_hook; + const struct nf_ct_hook *ct_hook; int err; if (verdict == NF_ACCEPT || @@ -388,7 +388,7 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, struct net_device *outdev; struct nf_conn *ct = NULL; enum ip_conntrack_info ctinfo = 0; - struct nfnl_ct_hook *nfnl_ct; + const struct nfnl_ct_hook *nfnl_ct; bool csum_verify; char *secdata = NULL; u32 seclen = 0; @@ -1103,7 +1103,7 @@ static int nfqnl_recv_verdict_batch(struct sk_buff *skb, return 0; } -static struct nf_conn *nfqnl_ct_parse(struct nfnl_ct_hook *nfnl_ct, +static struct nf_conn *nfqnl_ct_parse(const struct nfnl_ct_hook *nfnl_ct, const struct nlmsghdr *nlh, const struct nlattr * const nfqa[], struct nf_queue_entry *entry, @@ -1170,11 +1170,11 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info, { struct nfnl_queue_net *q = nfnl_queue_pernet(info->net); u_int16_t queue_num = ntohs(info->nfmsg->res_id); + const struct nfnl_ct_hook *nfnl_ct; struct nfqnl_msg_verdict_hdr *vhdr; enum ip_conntrack_info ctinfo; struct nfqnl_instance *queue; struct nf_queue_entry *entry; - struct nfnl_ct_hook *nfnl_ct; struct nf_conn *ct = NULL; unsigned int verdict; int err; -- cgit v1.2.3-59-g8ed1b From 6ae7989c9af0d98ab64196f4f4c6f6499454bd23 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 7 Jan 2022 05:03:25 +0100 Subject: netfilter: conntrack: avoid useless indirection during conntrack destruction nf_ct_put() results in a usesless indirection: nf_ct_put -> nf_conntrack_put -> nf_conntrack_destroy -> rcu readlock + indirect call of ct_hooks->destroy(). There are two _put helpers: nf_ct_put and nf_conntrack_put. The latter is what should be used in code that MUST NOT cause a linker dependency on the conntrack module (e.g. calls from core network stack). Everyone else should call nf_ct_put() instead. A followup patch will convert a few nf_conntrack_put() calls to nf_ct_put(), in particular from modules that already have a conntrack dependency such as act_ct or even nf_conntrack itself. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter/nf_conntrack_common.h | 2 ++ include/net/netfilter/nf_conntrack.h | 8 ++++++-- net/netfilter/nf_conntrack_core.c | 12 ++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter/nf_conntrack_common.h b/include/linux/netfilter/nf_conntrack_common.h index a03f7a80b9ab..2770db2fa080 100644 --- a/include/linux/netfilter/nf_conntrack_common.h +++ b/include/linux/netfilter/nf_conntrack_common.h @@ -29,6 +29,8 @@ struct nf_conntrack { }; void nf_conntrack_destroy(struct nf_conntrack *nfct); + +/* like nf_ct_put, but without module dependency on nf_conntrack */ static inline void nf_conntrack_put(struct nf_conntrack *nfct) { if (nfct && refcount_dec_and_test(&nfct->use)) diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index a4a14f3a5e38..8731d5bcb47d 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -76,6 +76,8 @@ struct nf_conn { * Hint, SKB address this struct and refcnt via skb->_nfct and * helpers nf_conntrack_get() and nf_conntrack_put(). * Helper nf_ct_put() equals nf_conntrack_put() by dec refcnt, + * except that the latter uses internal indirection and does not + * result in a conntrack module dependency. * beware nf_ct_get() is different and don't inc refcnt. */ struct nf_conntrack ct_general; @@ -170,11 +172,13 @@ nf_ct_get(const struct sk_buff *skb, enum ip_conntrack_info *ctinfo) return (struct nf_conn *)(nfct & NFCT_PTRMASK); } +void nf_ct_destroy(struct nf_conntrack *nfct); + /* decrement reference count on a conntrack */ static inline void nf_ct_put(struct nf_conn *ct) { - WARN_ON(!ct); - nf_conntrack_put(&ct->ct_general); + if (ct && refcount_dec_and_test(&ct->ct_general.use)) + nf_ct_destroy(&ct->ct_general); } /* Protocol module loading */ diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index cd3d07e418b5..7a2063abae04 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -558,7 +558,7 @@ static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct) #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK) -/* Released via destroy_conntrack() */ +/* Released via nf_ct_destroy() */ struct nf_conn *nf_ct_tmpl_alloc(struct net *net, const struct nf_conntrack_zone *zone, gfp_t flags) @@ -612,12 +612,11 @@ static void destroy_gre_conntrack(struct nf_conn *ct) #endif } -static void -destroy_conntrack(struct nf_conntrack *nfct) +void nf_ct_destroy(struct nf_conntrack *nfct) { struct nf_conn *ct = (struct nf_conn *)nfct; - pr_debug("destroy_conntrack(%p)\n", ct); + pr_debug("%s(%p)\n", __func__, ct); WARN_ON(refcount_read(&nfct->use) != 0); if (unlikely(nf_ct_is_template(ct))) { @@ -643,9 +642,10 @@ destroy_conntrack(struct nf_conntrack *nfct) if (ct->master) nf_ct_put(ct->master); - pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); + pr_debug("%s: returning ct=%p to slab\n", __func__, ct); nf_conntrack_free(ct); } +EXPORT_SYMBOL(nf_ct_destroy); static void nf_ct_delete_from_lists(struct nf_conn *ct) { @@ -2771,7 +2771,7 @@ err_cachep: static const struct nf_ct_hook nf_conntrack_hook = { .update = nf_conntrack_update, - .destroy = destroy_conntrack, + .destroy = nf_ct_destroy, .get_tuple_skb = nf_conntrack_get_tuple_skb, .attach = nf_conntrack_attach, }; -- cgit v1.2.3-59-g8ed1b From 6316136ec6e3dd1c302f7e7289a9ee46ecc610ae Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 7 Jan 2022 15:46:16 +0100 Subject: netfilter: egress: avoid a lockdep splat include/linux/netfilter_netdev.h:97 suspicious rcu_dereference_check() usage! 2 locks held by sd-resolve/1100: 0: ..(rcu_read_lock_bh){1:3}, at: ip_finish_output2 1: ..(rcu_read_lock_bh){1:3}, at: __dev_queue_xmit __dev_queue_xmit+0 .. The helper has two callers, one uses rcu_read_lock, the other rcu_read_lock_bh(). Annotate the dereference to reflect this. Fixes: 42df6e1d221dd ("netfilter: Introduce egress hook") Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter_netdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/netfilter_netdev.h b/include/linux/netfilter_netdev.h index b71b57a83bb4..b4dd96e4dc8d 100644 --- a/include/linux/netfilter_netdev.h +++ b/include/linux/netfilter_netdev.h @@ -94,7 +94,7 @@ static inline struct sk_buff *nf_hook_egress(struct sk_buff *skb, int *rc, return skb; #endif - e = rcu_dereference(dev->nf_hooks_egress); + e = rcu_dereference_check(dev->nf_hooks_egress, rcu_read_lock_bh_held()); if (!e) return skb; -- cgit v1.2.3-59-g8ed1b From 6f022c2ddbcefaee79502ce5386dfe351d457070 Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Thu, 6 Jan 2022 17:38:04 +0200 Subject: net: openvswitch: Fix ct_state nat flags for conns arriving from tc Netfilter conntrack maintains NAT flags per connection indicating whether NAT was configured for the connection. Openvswitch maintains NAT flags on the per packet flow key ct_state field, indicating whether NAT was actually executed on the packet. When a packet misses from tc to ovs the conntrack NAT flags are set. However, NAT was not necessarily executed on the packet because the connection's state might still be in NEW state. As such, openvswitch wrongly assumes that NAT was executed and sets an incorrect flow key NAT flags. Fix this, by flagging to openvswitch which NAT was actually done in act_ct via tc_skb_ext and tc_skb_cb to the openvswitch module, so the packet flow key NAT flags will be correctly set. Fixes: b57dc7c13ea9 ("net/sched: Introduce action ct") Signed-off-by: Paul Blakey Acked-by: Jamal Hadi Salim Link: https://lore.kernel.org/r/20220106153804.26451-1-paulb@nvidia.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 4 +++- include/net/pkt_sched.h | 4 +++- net/openvswitch/flow.c | 16 +++++++++++++--- net/sched/act_ct.c | 6 ++++++ net/sched/cls_api.c | 2 ++ 5 files changed, 27 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 4507d77d6941..60ab0c2fe567 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -287,7 +287,9 @@ struct tc_skb_ext { __u32 chain; __u16 mru; __u16 zone; - bool post_ct; + u8 post_ct:1; + u8 post_ct_snat:1; + u8 post_ct_dnat:1; }; #endif diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 9e71691c491b..9e7b21c0b3a6 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -197,7 +197,9 @@ struct tc_skb_cb { struct qdisc_skb_cb qdisc_cb; u16 mru; - bool post_ct; + u8 post_ct:1; + u8 post_ct_snat:1; + u8 post_ct_dnat:1; u16 zone; /* Only valid if post_ct = true */ }; diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 6d262d9aa10e..02096f2ec678 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -859,7 +859,7 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info, #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) struct tc_skb_ext *tc_ext; #endif - bool post_ct = false; + bool post_ct = false, post_ct_snat = false, post_ct_dnat = false; int res, err; u16 zone = 0; @@ -900,6 +900,8 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info, key->recirc_id = tc_ext ? tc_ext->chain : 0; OVS_CB(skb)->mru = tc_ext ? tc_ext->mru : 0; post_ct = tc_ext ? tc_ext->post_ct : false; + post_ct_snat = post_ct ? tc_ext->post_ct_snat : false; + post_ct_dnat = post_ct ? tc_ext->post_ct_dnat : false; zone = post_ct ? tc_ext->zone : 0; } else { key->recirc_id = 0; @@ -911,8 +913,16 @@ int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info, err = key_extract(skb, key); if (!err) { ovs_ct_fill_key(skb, key, post_ct); /* Must be after key_extract(). */ - if (post_ct && !skb_get_nfct(skb)) - key->ct_zone = zone; + if (post_ct) { + if (!skb_get_nfct(skb)) { + key->ct_zone = zone; + } else { + if (!post_ct_dnat) + key->ct_state &= ~OVS_CS_F_DST_NAT; + if (!post_ct_snat) + key->ct_state &= ~OVS_CS_F_SRC_NAT; + } + } } return err; } diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index ab3591408419..2a17eb77c904 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -839,6 +839,12 @@ static int ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct, } err = nf_nat_packet(ct, ctinfo, hooknum, skb); + if (err == NF_ACCEPT) { + if (maniptype == NF_NAT_MANIP_SRC) + tc_skb_cb(skb)->post_ct_snat = 1; + if (maniptype == NF_NAT_MANIP_DST) + tc_skb_cb(skb)->post_ct_dnat = 1; + } out: return err; } diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 35c74bdde848..cc9409aa755e 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -1625,6 +1625,8 @@ int tcf_classify(struct sk_buff *skb, ext->chain = last_executed_chain; ext->mru = cb->mru; ext->post_ct = cb->post_ct; + ext->post_ct_snat = cb->post_ct_snat; + ext->post_ct_dnat = cb->post_ct_dnat; ext->zone = cb->zone; } -- cgit v1.2.3-59-g8ed1b From c504e5c2f9648a1e5c2be01e8c3f59d394192bd3 Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Sun, 9 Jan 2022 14:36:26 +0800 Subject: net: skb: introduce kfree_skb_reason() Introduce the interface kfree_skb_reason(), which is able to pass the reason why the skb is dropped to 'kfree_skb' tracepoint. Add the 'reason' field to 'trace_kfree_skb', therefor user can get more detail information about abnormal skb with 'drop_monitor' or eBPF. All drop reasons are defined in the enum 'skb_drop_reason', and they will be print as string in 'kfree_skb' tracepoint in format of 'reason: XXX'. ( Maybe the reasons should be defined in a uapi header file, so that user space can use them? ) Signed-off-by: Menglong Dong Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 23 ++++++++++++++++++++++- include/trace/events/skb.h | 36 +++++++++++++++++++++++++++++------- net/core/dev.c | 3 ++- net/core/drop_monitor.c | 10 +++++++--- net/core/skbuff.c | 12 +++++++----- 5 files changed, 67 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 642acb0d1646..ef0870abc791 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -305,6 +305,17 @@ struct sk_buff_head { struct sk_buff; +/* The reason of skb drop, which is used in kfree_skb_reason(). + * en...maybe they should be splited by group? + * + * Each item here should also be in 'TRACE_SKB_DROP_REASON', which is + * used to translate the reason to string. + */ +enum skb_drop_reason { + SKB_DROP_REASON_NOT_SPECIFIED, + SKB_DROP_REASON_MAX, +}; + /* To allow 64K frame to be packed as single skb without frag_list we * require 64K/PAGE_SIZE pages plus 1 additional page to allow for * buffers which do not start on a page boundary. @@ -1085,8 +1096,18 @@ static inline bool skb_unref(struct sk_buff *skb) return true; } +void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason); + +/** + * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason + * @skb: buffer to free + */ +static inline void kfree_skb(struct sk_buff *skb) +{ + kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED); +} + void skb_release_head_state(struct sk_buff *skb); -void kfree_skb(struct sk_buff *skb); void kfree_skb_list(struct sk_buff *segs); void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt); void skb_tx_error(struct sk_buff *skb); diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h index 9e92f22eb086..294c61bbe44b 100644 --- a/include/trace/events/skb.h +++ b/include/trace/events/skb.h @@ -9,29 +9,51 @@ #include #include +#define TRACE_SKB_DROP_REASON \ + EM(SKB_DROP_REASON_NOT_SPECIFIED, NOT_SPECIFIED) \ + EMe(SKB_DROP_REASON_MAX, MAX) + +#undef EM +#undef EMe + +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define EMe(a, b) TRACE_DEFINE_ENUM(a); + +TRACE_SKB_DROP_REASON + +#undef EM +#undef EMe +#define EM(a, b) { a, #b }, +#define EMe(a, b) { a, #b } + /* * Tracepoint for free an sk_buff: */ TRACE_EVENT(kfree_skb, - TP_PROTO(struct sk_buff *skb, void *location), + TP_PROTO(struct sk_buff *skb, void *location, + enum skb_drop_reason reason), - TP_ARGS(skb, location), + TP_ARGS(skb, location, reason), TP_STRUCT__entry( - __field( void *, skbaddr ) - __field( void *, location ) - __field( unsigned short, protocol ) + __field(void *, skbaddr) + __field(void *, location) + __field(unsigned short, protocol) + __field(enum skb_drop_reason, reason) ), TP_fast_assign( __entry->skbaddr = skb; __entry->location = location; __entry->protocol = ntohs(skb->protocol); + __entry->reason = reason; ), - TP_printk("skbaddr=%p protocol=%u location=%p", - __entry->skbaddr, __entry->protocol, __entry->location) + TP_printk("skbaddr=%p protocol=%u location=%p reason: %s", + __entry->skbaddr, __entry->protocol, __entry->location, + __print_symbolic(__entry->reason, + TRACE_SKB_DROP_REASON)) ); TRACE_EVENT(consume_skb, diff --git a/net/core/dev.c b/net/core/dev.c index 83a4089990a0..84a0d9542fe9 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4899,7 +4899,8 @@ static __latent_entropy void net_tx_action(struct softirq_action *h) if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) trace_consume_skb(skb); else - trace_kfree_skb(skb, net_tx_action); + trace_kfree_skb(skb, net_tx_action, + SKB_DROP_REASON_NOT_SPECIFIED); if (skb->fclone != SKB_FCLONE_UNAVAILABLE) __kfree_skb(skb); diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 3d0ab2eec916..7b288a121a41 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -110,7 +110,8 @@ static u32 net_dm_queue_len = 1000; struct net_dm_alert_ops { void (*kfree_skb_probe)(void *ignore, struct sk_buff *skb, - void *location); + void *location, + enum skb_drop_reason reason); void (*napi_poll_probe)(void *ignore, struct napi_struct *napi, int work, int budget); void (*work_item_func)(struct work_struct *work); @@ -262,7 +263,9 @@ out: spin_unlock_irqrestore(&data->lock, flags); } -static void trace_kfree_skb_hit(void *ignore, struct sk_buff *skb, void *location) +static void trace_kfree_skb_hit(void *ignore, struct sk_buff *skb, + void *location, + enum skb_drop_reason reason) { trace_drop_common(skb, location); } @@ -490,7 +493,8 @@ static const struct net_dm_alert_ops net_dm_alert_summary_ops = { static void net_dm_packet_trace_kfree_skb_hit(void *ignore, struct sk_buff *skb, - void *location) + void *location, + enum skb_drop_reason reason) { ktime_t tstamp = ktime_get_real(); struct per_cpu_dm_data *data; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index e514a36bcffc..0118f0afaa4f 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -759,21 +759,23 @@ void __kfree_skb(struct sk_buff *skb) EXPORT_SYMBOL(__kfree_skb); /** - * kfree_skb - free an sk_buff + * kfree_skb_reason - free an sk_buff with special reason * @skb: buffer to free + * @reason: reason why this skb is dropped * * Drop a reference to the buffer and free it if the usage count has - * hit zero. + * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' + * tracepoint. */ -void kfree_skb(struct sk_buff *skb) +void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) { if (!skb_unref(skb)) return; - trace_kfree_skb(skb, __builtin_return_address(0)); + trace_kfree_skb(skb, __builtin_return_address(0), reason); __kfree_skb(skb); } -EXPORT_SYMBOL(kfree_skb); +EXPORT_SYMBOL(kfree_skb_reason); void kfree_skb_list(struct sk_buff *segs) { -- cgit v1.2.3-59-g8ed1b From 85125597419aec3aa7b8f3b8713e415f997796f2 Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Sun, 9 Jan 2022 14:36:27 +0800 Subject: net: skb: use kfree_skb_reason() in tcp_v4_rcv() Replace kfree_skb() with kfree_skb_reason() in tcp_v4_rcv(). Following drop reasons are added: SKB_DROP_REASON_NO_SOCKET SKB_DROP_REASON_PKT_TOO_SMALL SKB_DROP_REASON_TCP_CSUM SKB_DROP_REASON_TCP_FILTER After this patch, 'kfree_skb' event will print message like this: $ TASK-PID CPU# ||||| TIMESTAMP FUNCTION $ | | | ||||| | | -0 [000] ..s1. 36.113438: kfree_skb: skbaddr=(____ptrval____) protocol=2048 location=(____ptrval____) reason: NO_SOCKET The reason of skb drop is printed too. Signed-off-by: Menglong Dong Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 4 ++++ include/trace/events/skb.h | 4 ++++ net/ipv4/tcp_ipv4.c | 14 +++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index ef0870abc791..c9c97b0d0fe9 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -313,6 +313,10 @@ struct sk_buff; */ enum skb_drop_reason { SKB_DROP_REASON_NOT_SPECIFIED, + SKB_DROP_REASON_NO_SOCKET, + SKB_DROP_REASON_PKT_TOO_SMALL, + SKB_DROP_REASON_TCP_CSUM, + SKB_DROP_REASON_TCP_FILTER, SKB_DROP_REASON_MAX, }; diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h index 294c61bbe44b..faa7d068a7bc 100644 --- a/include/trace/events/skb.h +++ b/include/trace/events/skb.h @@ -11,6 +11,10 @@ #define TRACE_SKB_DROP_REASON \ EM(SKB_DROP_REASON_NOT_SPECIFIED, NOT_SPECIFIED) \ + EM(SKB_DROP_REASON_NO_SOCKET, NO_SOCKET) \ + EM(SKB_DROP_REASON_PKT_TOO_SMALL, PKT_TOO_SMALL) \ + EM(SKB_DROP_REASON_TCP_CSUM, TCP_CSUM) \ + EM(SKB_DROP_REASON_TCP_FILTER, TCP_FILTER) \ EMe(SKB_DROP_REASON_MAX, MAX) #undef EM diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 9861786b8336..b3f34e366b27 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1971,8 +1971,10 @@ int tcp_v4_rcv(struct sk_buff *skb) const struct tcphdr *th; bool refcounted; struct sock *sk; + int drop_reason; int ret; + drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; if (skb->pkt_type != PACKET_HOST) goto discard_it; @@ -1984,8 +1986,10 @@ int tcp_v4_rcv(struct sk_buff *skb) th = (const struct tcphdr *)skb->data; - if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) + if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { + drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; goto bad_packet; + } if (!pskb_may_pull(skb, th->doff * 4)) goto discard_it; @@ -2090,8 +2094,10 @@ process: nf_reset_ct(skb); - if (tcp_filter(sk, skb)) + if (tcp_filter(sk, skb)) { + drop_reason = SKB_DROP_REASON_TCP_FILTER; goto discard_and_relse; + } th = (const struct tcphdr *)skb->data; iph = ip_hdr(skb); tcp_v4_fill_cb(skb, iph, th); @@ -2124,6 +2130,7 @@ put_and_return: return ret; no_tcp_socket: + drop_reason = SKB_DROP_REASON_NO_SOCKET; if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) goto discard_it; @@ -2131,6 +2138,7 @@ no_tcp_socket: if (tcp_checksum_complete(skb)) { csum_error: + drop_reason = SKB_DROP_REASON_TCP_CSUM; trace_tcp_bad_csum(skb); __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); bad_packet: @@ -2141,7 +2149,7 @@ bad_packet: discard_it: /* Discard frame. */ - kfree_skb(skb); + kfree_skb_reason(skb, drop_reason); return 0; discard_and_relse: -- cgit v1.2.3-59-g8ed1b From 1c7fab70df085d866a3765955f397ca2b4025b15 Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Sun, 9 Jan 2022 14:36:28 +0800 Subject: net: skb: use kfree_skb_reason() in __udp4_lib_rcv() Replace kfree_skb() with kfree_skb_reason() in __udp4_lib_rcv. New drop reason 'SKB_DROP_REASON_UDP_CSUM' is added for udp csum error. Signed-off-by: Menglong Dong Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 1 + include/trace/events/skb.h | 1 + net/ipv4/udp.c | 10 ++++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c9c97b0d0fe9..af64c7de9b53 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -317,6 +317,7 @@ enum skb_drop_reason { SKB_DROP_REASON_PKT_TOO_SMALL, SKB_DROP_REASON_TCP_CSUM, SKB_DROP_REASON_TCP_FILTER, + SKB_DROP_REASON_UDP_CSUM, SKB_DROP_REASON_MAX, }; diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h index faa7d068a7bc..3e042ca2cedb 100644 --- a/include/trace/events/skb.h +++ b/include/trace/events/skb.h @@ -15,6 +15,7 @@ EM(SKB_DROP_REASON_PKT_TOO_SMALL, PKT_TOO_SMALL) \ EM(SKB_DROP_REASON_TCP_CSUM, TCP_CSUM) \ EM(SKB_DROP_REASON_TCP_FILTER, TCP_FILTER) \ + EM(SKB_DROP_REASON_UDP_CSUM, UDP_CSUM) \ EMe(SKB_DROP_REASON_MAX, MAX) #undef EM diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index c2a4411d2b04..464590ea922e 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -2411,6 +2411,9 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, __be32 saddr, daddr; struct net *net = dev_net(skb->dev); bool refcounted; + int drop_reason; + + drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; /* * Validate the packet. @@ -2466,6 +2469,7 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, if (udp_lib_checksum_complete(skb)) goto csum_error; + drop_reason = SKB_DROP_REASON_NO_SOCKET; __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE); icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); @@ -2473,10 +2477,11 @@ int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, * Hmm. We got an UDP packet to a port to which we * don't wanna listen. Ignore it. */ - kfree_skb(skb); + kfree_skb_reason(skb, drop_reason); return 0; short_packet: + drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), @@ -2489,6 +2494,7 @@ csum_error: * RFC1122: OK. Discards the bad packet silently (as far as * the network is concerned, anyway) as per 4.1.3.4 (MUST). */ + drop_reason = SKB_DROP_REASON_UDP_CSUM; net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), @@ -2496,7 +2502,7 @@ csum_error: __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE); drop: __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE); - kfree_skb(skb); + kfree_skb_reason(skb, drop_reason); return 0; } -- cgit v1.2.3-59-g8ed1b From a6b5a28eb56c3f4988f7ff5290b954ba296e309a Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Sat, 14 Nov 2020 13:43:54 -0500 Subject: nfs: Convert to new fscache volume/cookie API Change the nfs filesystem to support fscache's indexing rewrite and reenable caching in nfs. The following changes have been made: (1) The fscache_netfs struct is no more, and there's no need to register the filesystem as a whole. (2) The session cookie is now an fscache_volume cookie, allocated with fscache_acquire_volume(). That takes three parameters: a string representing the "volume" in the index, a string naming the cache to use (or NULL) and a u64 that conveys coherency metadata for the volume. For nfs, I've made it render the volume name string as: "nfs,,,
,,,*<,param>[,]" (3) The fscache_cookie_def is no more and needed information is passed directly to fscache_acquire_cookie(). The cache no longer calls back into the filesystem, but rather metadata changes are indicated at other times. fscache_acquire_cookie() is passed the same keying and coherency information as before. (4) fscache_enable/disable_cookie() have been removed. Call fscache_use_cookie() and fscache_unuse_cookie() when a file is opened or closed to prevent a cache file from being culled and to keep resources to hand that are needed to do I/O. If a file is opened for writing, we invalidate it with FSCACHE_INVAL_DIO_WRITE in lieu of doing writeback to the cache, thereby making it cease caching until all currently open files are closed. This should give the same behaviour as the uptream code. Making the cache store local modifications isn't straightforward for NFS, so that's left for future patches. (5) fscache_invalidate() now needs to be given uptodate auxiliary data and a file size. It also takes a flag to indicate if this was due to a DIO write. (6) Call nfs_fscache_invalidate() with FSCACHE_INVAL_DIO_WRITE on a file to which a DIO write is made. (7) Call fscache_note_page_release() from nfs_release_page(). (8) Use a killable wait in nfs_vm_page_mkwrite() when waiting for PG_fscache to be cleared. (9) The functions to read and write data to/from the cache are stubbed out pending a conversion to use netfslib. Changes ======= ver #3: - Added missing =n fallback for nfs_fscache_release_file()[1][2]. ver #2: - Use gfpflags_allow_blocking() rather than using flag directly. - fscache_acquire_volume() now returns errors. - Remove NFS_INO_FSCACHE as it's no longer used. - Need to unuse a cookie on file-release, not inode-clear. Signed-off-by: Dave Wysochanski Co-developed-by: David Howells Signed-off-by: David Howells Tested-by: Dave Wysochanski Acked-by: Jeff Layton cc: Trond Myklebust cc: Anna Schumaker cc: linux-nfs@vger.kernel.org cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/202112100804.nksO8K4u-lkp@intel.com/ [1] Link: https://lore.kernel.org/r/202112100957.2oEDT20W-lkp@intel.com/ [2] Link: https://lore.kernel.org/r/163819668938.215744.14448852181937731615.stgit@warthog.procyon.org.uk/ # v1 Link: https://lore.kernel.org/r/163906979003.143852.2601189243864854724.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967182112.1823006.7791504655391213379.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021575950.640689.12069642327533368467.stgit@warthog.procyon.org.uk/ # v4 --- fs/nfs/Kconfig | 2 +- fs/nfs/Makefile | 2 +- fs/nfs/client.c | 4 - fs/nfs/direct.c | 2 + fs/nfs/file.c | 13 +- fs/nfs/fscache-index.c | 140 --------------- fs/nfs/fscache.c | 434 +++++++++++----------------------------------- fs/nfs/fscache.h | 127 ++++---------- fs/nfs/inode.c | 11 +- fs/nfs/nfstrace.h | 1 - fs/nfs/super.c | 28 +-- fs/nfs/write.c | 1 + include/linux/nfs_fs.h | 1 - include/linux/nfs_fs_sb.h | 9 +- 14 files changed, 172 insertions(+), 603 deletions(-) delete mode 100644 fs/nfs/fscache-index.c (limited to 'include/linux') diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index bdc11b89eac5..14a72224b657 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -170,7 +170,7 @@ config ROOT_NFS config NFS_FSCACHE bool "Provide NFS client caching support" - depends on NFS_FS=m && FSCACHE_OLD_API || NFS_FS=y && FSCACHE_OLD_API=y + depends on NFS_FS=m && FSCACHE || NFS_FS=y && FSCACHE=y help Say Y here if you want NFS data to be cached locally on disc through the general filesystem cache manager diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile index 22d11fdc6deb..5f6db37f461e 100644 --- a/fs/nfs/Makefile +++ b/fs/nfs/Makefile @@ -12,7 +12,7 @@ nfs-y := client.o dir.o file.o getroot.o inode.o super.o \ export.o sysfs.o fs_context.o nfs-$(CONFIG_ROOT_NFS) += nfsroot.o nfs-$(CONFIG_SYSCTL) += sysctl.o -nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o +nfs-$(CONFIG_NFS_FSCACHE) += fscache.o obj-$(CONFIG_NFS_V2) += nfsv2.o nfsv2-y := nfs2super.o proc.o nfs2xdr.o diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 1e4dc1ab9312..8d8b85b5a641 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -183,8 +183,6 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_net = get_net(cl_init->net); clp->cl_principal = "*"; - nfs_fscache_get_client_cookie(clp); - return clp; error_cleanup: @@ -238,8 +236,6 @@ static void pnfs_init_server(struct nfs_server *server) */ void nfs_free_client(struct nfs_client *clp) { - nfs_fscache_release_client_cookie(clp); - /* -EIO all pending I/O */ if (!IS_ERR(clp->cl_rpcclient)) rpc_shutdown_client(clp->cl_rpcclient); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 9cff8709c80a..eabfdab543c8 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -59,6 +59,7 @@ #include "internal.h" #include "iostat.h" #include "pnfs.h" +#include "fscache.h" #define NFSDBG_FACILITY NFSDBG_VFS @@ -959,6 +960,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter) } else { result = requested; } + nfs_fscache_invalidate(inode, FSCACHE_INVAL_DIO_WRITE); out_release: nfs_direct_req_release(dreq); out: diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 24e7dccce355..76d76acbc594 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -84,6 +84,7 @@ nfs_file_release(struct inode *inode, struct file *filp) nfs_inc_stats(inode, NFSIOS_VFSRELEASE); nfs_file_clear_open_context(filp); + nfs_fscache_release_file(inode, filp); return 0; } EXPORT_SYMBOL_GPL(nfs_file_release); @@ -415,8 +416,7 @@ static void nfs_invalidate_page(struct page *page, unsigned int offset, return; /* Cancel any unstarted writes on this page */ nfs_wb_page_cancel(page_file_mapping(page)->host, page); - - nfs_fscache_invalidate_page(page, page->mapping->host); + wait_on_page_fscache(page); } /* @@ -475,12 +475,11 @@ static void nfs_check_dirty_writeback(struct page *page, static int nfs_launder_page(struct page *page) { struct inode *inode = page_file_mapping(page)->host; - struct nfs_inode *nfsi = NFS_I(inode); dfprintk(PAGECACHE, "NFS: launder_page(%ld, %llu)\n", inode->i_ino, (long long)page_offset(page)); - nfs_fscache_wait_on_page_write(nfsi, page); + wait_on_page_fscache(page); return nfs_wb_page(inode, page); } @@ -555,7 +554,11 @@ static vm_fault_t nfs_vm_page_mkwrite(struct vm_fault *vmf) sb_start_pagefault(inode->i_sb); /* make sure the cache has finished storing the page */ - nfs_fscache_wait_on_page_write(NFS_I(inode), page); + if (PageFsCache(page) && + wait_on_page_fscache_killable(vmf->page) < 0) { + ret = VM_FAULT_RETRY; + goto out; + } wait_on_bit_action(&NFS_I(inode)->flags, NFS_INO_INVALIDATING, nfs_wait_bit_killable, TASK_KILLABLE); diff --git a/fs/nfs/fscache-index.c b/fs/nfs/fscache-index.c deleted file mode 100644 index 573b1da9342c..000000000000 --- a/fs/nfs/fscache-index.c +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* NFS FS-Cache index structure definition - * - * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "internal.h" -#include "fscache.h" - -#define NFSDBG_FACILITY NFSDBG_FSCACHE - -/* - * Define the NFS filesystem for FS-Cache. Upon registration FS-Cache sticks - * the cookie for the top-level index object for NFS into here. The top-level - * index can than have other cache objects inserted into it. - */ -struct fscache_netfs nfs_fscache_netfs = { - .name = "nfs", - .version = 0, -}; - -/* - * Register NFS for caching - */ -int nfs_fscache_register(void) -{ - return fscache_register_netfs(&nfs_fscache_netfs); -} - -/* - * Unregister NFS for caching - */ -void nfs_fscache_unregister(void) -{ - fscache_unregister_netfs(&nfs_fscache_netfs); -} - -/* - * Define the server object for FS-Cache. This is used to describe a server - * object to fscache_acquire_cookie(). It is keyed by the NFS protocol and - * server address parameters. - */ -const struct fscache_cookie_def nfs_fscache_server_index_def = { - .name = "NFS.server", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -/* - * Define the superblock object for FS-Cache. This is used to describe a - * superblock object to fscache_acquire_cookie(). It is keyed by all the NFS - * parameters that might cause a separate superblock. - */ -const struct fscache_cookie_def nfs_fscache_super_index_def = { - .name = "NFS.super", - .type = FSCACHE_COOKIE_TYPE_INDEX, -}; - -/* - * Consult the netfs about the state of an object - * - This function can be absent if the index carries no state data - * - The netfs data from the cookie being used as the target is - * presented, as is the auxiliary data - */ -static -enum fscache_checkaux nfs_fscache_inode_check_aux(void *cookie_netfs_data, - const void *data, - uint16_t datalen, - loff_t object_size) -{ - struct nfs_fscache_inode_auxdata auxdata; - struct nfs_inode *nfsi = cookie_netfs_data; - - if (datalen != sizeof(auxdata)) - return FSCACHE_CHECKAUX_OBSOLETE; - - memset(&auxdata, 0, sizeof(auxdata)); - auxdata.mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; - auxdata.mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; - auxdata.ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; - auxdata.ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; - - if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) - auxdata.change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); - - if (memcmp(data, &auxdata, datalen) != 0) - return FSCACHE_CHECKAUX_OBSOLETE; - - return FSCACHE_CHECKAUX_OKAY; -} - -/* - * Get an extra reference on a read context. - * - This function can be absent if the completion function doesn't require a - * context. - * - The read context is passed back to NFS in the event that a data read on the - * cache fails with EIO - in which case the server must be contacted to - * retrieve the data, which requires the read context for security. - */ -static void nfs_fh_get_context(void *cookie_netfs_data, void *context) -{ - get_nfs_open_context(context); -} - -/* - * Release an extra reference on a read context. - * - This function can be absent if the completion function doesn't require a - * context. - */ -static void nfs_fh_put_context(void *cookie_netfs_data, void *context) -{ - if (context) - put_nfs_open_context(context); -} - -/* - * Define the inode object for FS-Cache. This is used to describe an inode - * object to fscache_acquire_cookie(). It is keyed by the NFS file handle for - * an inode. - * - * Coherency is managed by comparing the copies of i_size, i_mtime and i_ctime - * held in the cache auxiliary data for the data storage object with those in - * the inode struct in memory. - */ -const struct fscache_cookie_def nfs_fscache_inode_object_def = { - .name = "NFS.fh", - .type = FSCACHE_COOKIE_TYPE_DATAFILE, - .check_aux = nfs_fscache_inode_check_aux, - .get_context = nfs_fh_get_context, - .put_context = nfs_fh_put_context, -}; diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index d743629e05e1..fac6438477a0 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -22,24 +22,18 @@ #define NFSDBG_FACILITY NFSDBG_FSCACHE -static struct rb_root nfs_fscache_keys = RB_ROOT; -static DEFINE_SPINLOCK(nfs_fscache_keys_lock); +#define NFS_MAX_KEY_LEN 1000 -/* - * Layout of the key for an NFS server cache object. - */ -struct nfs_server_key { - struct { - uint16_t nfsversion; /* NFS protocol version */ - uint32_t minorversion; /* NFSv4 minor version */ - uint16_t family; /* address family */ - __be16 port; /* IP port */ - } hdr; - union { - struct in_addr ipv4_addr; /* IPv4 address */ - struct in6_addr ipv6_addr; /* IPv6 address */ - }; -} __packed; +static bool nfs_append_int(char *key, int *_len, unsigned long long x) +{ + if (*_len > NFS_MAX_KEY_LEN) + return false; + if (x == 0) + key[(*_len)++] = ','; + else + *_len += sprintf(key + *_len, ",%llx", x); + return true; +} /* * Get the per-client index cookie for an NFS client if the appropriate mount @@ -47,160 +41,108 @@ struct nfs_server_key { * - We always try and get an index cookie for the client, but get filehandle * cookies on a per-superblock basis, depending on the mount flags */ -void nfs_fscache_get_client_cookie(struct nfs_client *clp) +static bool nfs_fscache_get_client_key(struct nfs_client *clp, + char *key, int *_len) { const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &clp->cl_addr; const struct sockaddr_in *sin = (struct sockaddr_in *) &clp->cl_addr; - struct nfs_server_key key; - uint16_t len = sizeof(key.hdr); - memset(&key, 0, sizeof(key)); - key.hdr.nfsversion = clp->rpc_ops->version; - key.hdr.minorversion = clp->cl_minorversion; - key.hdr.family = clp->cl_addr.ss_family; + *_len += snprintf(key + *_len, NFS_MAX_KEY_LEN - *_len, + ",%u.%u,%x", + clp->rpc_ops->version, + clp->cl_minorversion, + clp->cl_addr.ss_family); switch (clp->cl_addr.ss_family) { case AF_INET: - key.hdr.port = sin->sin_port; - key.ipv4_addr = sin->sin_addr; - len += sizeof(key.ipv4_addr); - break; + if (!nfs_append_int(key, _len, sin->sin_port) || + !nfs_append_int(key, _len, sin->sin_addr.s_addr)) + return false; + return true; case AF_INET6: - key.hdr.port = sin6->sin6_port; - key.ipv6_addr = sin6->sin6_addr; - len += sizeof(key.ipv6_addr); - break; + if (!nfs_append_int(key, _len, sin6->sin6_port) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[0]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[1]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[2]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[3])) + return false; + return true; default: printk(KERN_WARNING "NFS: Unknown network family '%d'\n", clp->cl_addr.ss_family); - clp->fscache = NULL; - return; + return false; } - - /* create a cache index for looking up filehandles */ - clp->fscache = fscache_acquire_cookie(nfs_fscache_netfs.primary_index, - &nfs_fscache_server_index_def, - &key, len, - NULL, 0, - clp, 0, true); - dfprintk(FSCACHE, "NFS: get client cookie (0x%p/0x%p)\n", - clp, clp->fscache); -} - -/* - * Dispose of a per-client cookie - */ -void nfs_fscache_release_client_cookie(struct nfs_client *clp) -{ - dfprintk(FSCACHE, "NFS: releasing client cookie (0x%p/0x%p)\n", - clp, clp->fscache); - - fscache_relinquish_cookie(clp->fscache, NULL, false); - clp->fscache = NULL; } /* - * Get the cache cookie for an NFS superblock. We have to handle - * uniquification here because the cache doesn't do it for us. + * Get the cache cookie for an NFS superblock. * * The default uniquifier is just an empty string, but it may be overridden * either by the 'fsc=xxx' option to mount, or by inheriting it from the parent * superblock across an automount point of some nature. */ -void nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int ulen) +int nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int ulen) { - struct nfs_fscache_key *key, *xkey; + struct fscache_volume *vcookie; struct nfs_server *nfss = NFS_SB(sb); - struct rb_node **p, *parent; - int diff; + unsigned int len = 3; + char *key; - nfss->fscache_key = NULL; - nfss->fscache = NULL; - if (!uniq) { - uniq = ""; - ulen = 1; + if (uniq) { + nfss->fscache_uniq = kmemdup_nul(uniq, ulen, GFP_KERNEL); + if (!nfss->fscache_uniq) + return -ENOMEM; } - key = kzalloc(sizeof(*key) + ulen, GFP_KERNEL); + key = kmalloc(NFS_MAX_KEY_LEN + 24, GFP_KERNEL); if (!key) - return; - - key->nfs_client = nfss->nfs_client; - key->key.super.s_flags = sb->s_flags & NFS_SB_MASK; - key->key.nfs_server.flags = nfss->flags; - key->key.nfs_server.rsize = nfss->rsize; - key->key.nfs_server.wsize = nfss->wsize; - key->key.nfs_server.acregmin = nfss->acregmin; - key->key.nfs_server.acregmax = nfss->acregmax; - key->key.nfs_server.acdirmin = nfss->acdirmin; - key->key.nfs_server.acdirmax = nfss->acdirmax; - key->key.nfs_server.fsid = nfss->fsid; - key->key.rpc_auth.au_flavor = nfss->client->cl_auth->au_flavor; - - key->key.uniq_len = ulen; - memcpy(key->key.uniquifier, uniq, ulen); - - spin_lock(&nfs_fscache_keys_lock); - p = &nfs_fscache_keys.rb_node; - parent = NULL; - while (*p) { - parent = *p; - xkey = rb_entry(parent, struct nfs_fscache_key, node); - - if (key->nfs_client < xkey->nfs_client) - goto go_left; - if (key->nfs_client > xkey->nfs_client) - goto go_right; - - diff = memcmp(&key->key, &xkey->key, sizeof(key->key)); - if (diff < 0) - goto go_left; - if (diff > 0) - goto go_right; - - if (key->key.uniq_len == 0) - goto non_unique; - diff = memcmp(key->key.uniquifier, - xkey->key.uniquifier, - key->key.uniq_len); - if (diff < 0) - goto go_left; - if (diff > 0) - goto go_right; - goto non_unique; - - go_left: - p = &(*p)->rb_left; - continue; - go_right: - p = &(*p)->rb_right; + return -ENOMEM; + + memcpy(key, "nfs", 3); + if (!nfs_fscache_get_client_key(nfss->nfs_client, key, &len) || + !nfs_append_int(key, &len, nfss->fsid.major) || + !nfs_append_int(key, &len, nfss->fsid.minor) || + !nfs_append_int(key, &len, sb->s_flags & NFS_SB_MASK) || + !nfs_append_int(key, &len, nfss->flags) || + !nfs_append_int(key, &len, nfss->rsize) || + !nfs_append_int(key, &len, nfss->wsize) || + !nfs_append_int(key, &len, nfss->acregmin) || + !nfs_append_int(key, &len, nfss->acregmax) || + !nfs_append_int(key, &len, nfss->acdirmin) || + !nfs_append_int(key, &len, nfss->acdirmax) || + !nfs_append_int(key, &len, nfss->client->cl_auth->au_flavor)) + goto out; + + if (ulen > 0) { + if (ulen > NFS_MAX_KEY_LEN - len) + goto out; + key[len++] = ','; + memcpy(key + len, uniq, ulen); + len += ulen; } - - rb_link_node(&key->node, parent, p); - rb_insert_color(&key->node, &nfs_fscache_keys); - spin_unlock(&nfs_fscache_keys_lock); - nfss->fscache_key = key; + key[len] = 0; /* create a cache index for looking up filehandles */ - nfss->fscache = fscache_acquire_cookie(nfss->nfs_client->fscache, - &nfs_fscache_super_index_def, - &key->key, - sizeof(key->key) + ulen, - NULL, 0, - nfss, 0, true); + vcookie = fscache_acquire_volume(key, + NULL, /* preferred_cache */ + NULL, 0 /* coherency_data */); dfprintk(FSCACHE, "NFS: get superblock cookie (0x%p/0x%p)\n", - nfss, nfss->fscache); - return; + nfss, vcookie); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(key); + return PTR_ERR(vcookie); + } + pr_err("NFS: Cache volume key already in use (%s)\n", key); + vcookie = NULL; + } + nfss->fscache = vcookie; -non_unique: - spin_unlock(&nfs_fscache_keys_lock); +out: kfree(key); - nfss->fscache_key = NULL; - nfss->fscache = NULL; - printk(KERN_WARNING "NFS:" - " Cache request denied due to non-unique superblock keys\n"); + return 0; } /* @@ -213,29 +155,9 @@ void nfs_fscache_release_super_cookie(struct super_block *sb) dfprintk(FSCACHE, "NFS: releasing superblock cookie (0x%p/0x%p)\n", nfss, nfss->fscache); - fscache_relinquish_cookie(nfss->fscache, NULL, false); + fscache_relinquish_volume(nfss->fscache, NULL, false); nfss->fscache = NULL; - - if (nfss->fscache_key) { - spin_lock(&nfs_fscache_keys_lock); - rb_erase(&nfss->fscache_key->node, &nfs_fscache_keys); - spin_unlock(&nfs_fscache_keys_lock); - kfree(nfss->fscache_key); - nfss->fscache_key = NULL; - } -} - -static void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, - struct nfs_inode *nfsi) -{ - memset(auxdata, 0, sizeof(*auxdata)); - auxdata->mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; - auxdata->mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; - auxdata->ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; - auxdata->ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; - - if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) - auxdata->change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); + kfree(nfss->fscache_uniq); } /* @@ -254,10 +176,12 @@ void nfs_fscache_init_inode(struct inode *inode) nfs_fscache_update_auxdata(&auxdata, nfsi); nfsi->fscache = fscache_acquire_cookie(NFS_SB(inode->i_sb)->fscache, - &nfs_fscache_inode_object_def, - nfsi->fh.data, nfsi->fh.size, - &auxdata, sizeof(auxdata), - nfsi, nfsi->vfs_inode.i_size, false); + 0, + nfsi->fh.data, /* index_key */ + nfsi->fh.size, + &auxdata, /* aux_data */ + sizeof(auxdata), + i_size_read(&nfsi->vfs_inode)); } /* @@ -265,24 +189,15 @@ void nfs_fscache_init_inode(struct inode *inode) */ void nfs_fscache_clear_inode(struct inode *inode) { - struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); dfprintk(FSCACHE, "NFS: clear cookie (0x%p/0x%p)\n", nfsi, cookie); - nfs_fscache_update_auxdata(&auxdata, nfsi); - fscache_relinquish_cookie(cookie, &auxdata, false); + fscache_relinquish_cookie(cookie, false); nfsi->fscache = NULL; } -static bool nfs_fscache_can_enable(void *data) -{ - struct inode *inode = data; - - return !inode_is_open_for_write(inode); -} - /* * Enable or disable caching for a file that is being opened as appropriate. * The cookie is allocated when the inode is initialised, but is not enabled at @@ -307,93 +222,31 @@ void nfs_fscache_open_file(struct inode *inode, struct file *filp) struct nfs_fscache_inode_auxdata auxdata; struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); + bool open_for_write = inode_is_open_for_write(inode); if (!fscache_cookie_valid(cookie)) return; - nfs_fscache_update_auxdata(&auxdata, nfsi); - - if (inode_is_open_for_write(inode)) { + fscache_use_cookie(cookie, open_for_write); + if (open_for_write) { dfprintk(FSCACHE, "NFS: nfsi 0x%p disabling cache\n", nfsi); - clear_bit(NFS_INO_FSCACHE, &nfsi->flags); - fscache_disable_cookie(cookie, &auxdata, true); - fscache_uncache_all_inode_pages(cookie, inode); - } else { - dfprintk(FSCACHE, "NFS: nfsi 0x%p enabling cache\n", nfsi); - fscache_enable_cookie(cookie, &auxdata, nfsi->vfs_inode.i_size, - nfs_fscache_can_enable, inode); - if (fscache_cookie_enabled(cookie)) - set_bit(NFS_INO_FSCACHE, &NFS_I(inode)->flags); + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_invalidate(cookie, &auxdata, i_size_read(inode), + FSCACHE_INVAL_DIO_WRITE); } } EXPORT_SYMBOL_GPL(nfs_fscache_open_file); -/* - * Release the caching state associated with a page, if the page isn't busy - * interacting with the cache. - * - Returns true (can release page) or false (page busy). - */ -int nfs_fscache_release_page(struct page *page, gfp_t gfp) -{ - if (PageFsCache(page)) { - struct fscache_cookie *cookie = nfs_i_fscache(page->mapping->host); - - BUG_ON(!cookie); - dfprintk(FSCACHE, "NFS: fscache releasepage (0x%p/0x%p/0x%p)\n", - cookie, page, NFS_I(page->mapping->host)); - - if (!fscache_maybe_release_page(cookie, page, gfp)) - return 0; - - nfs_inc_fscache_stats(page->mapping->host, - NFSIOS_FSCACHE_PAGES_UNCACHED); - } - - return 1; -} - -/* - * Release the caching state associated with a page if undergoing complete page - * invalidation. - */ -void __nfs_fscache_invalidate_page(struct page *page, struct inode *inode) +void nfs_fscache_release_file(struct inode *inode, struct file *filp) { + struct nfs_fscache_inode_auxdata auxdata; + struct nfs_inode *nfsi = NFS_I(inode); struct fscache_cookie *cookie = nfs_i_fscache(inode); - BUG_ON(!cookie); - - dfprintk(FSCACHE, "NFS: fscache invalidatepage (0x%p/0x%p/0x%p)\n", - cookie, page, NFS_I(inode)); - - fscache_wait_on_page_write(cookie, page); - - BUG_ON(!PageLocked(page)); - fscache_uncache_page(cookie, page); - nfs_inc_fscache_stats(page->mapping->host, - NFSIOS_FSCACHE_PAGES_UNCACHED); -} - -/* - * Handle completion of a page being read from the cache. - * - Called in process (keventd) context. - */ -static void nfs_readpage_from_fscache_complete(struct page *page, - void *context, - int error) -{ - dfprintk(FSCACHE, - "NFS: readpage_from_fscache_complete (0x%p/0x%p/%d)\n", - page, context, error); - - /* - * If the read completes with an error, mark the page with PG_checked, - * unlock the page, and let the VM reissue the readpage. - */ - if (!error) - SetPageUptodate(page); - else - SetPageChecked(page); - unlock_page(page); + if (fscache_cookie_valid(cookie)) { + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_unuse_cookie(cookie, &auxdata, NULL); + } } /* @@ -402,8 +255,6 @@ static void nfs_readpage_from_fscache_complete(struct page *page, int __nfs_readpage_from_fscache(struct nfs_open_context *ctx, struct inode *inode, struct page *page) { - int ret; - dfprintk(FSCACHE, "NFS: readpage_from_fscache(fsc:%p/p:%p(i:%lx f:%lx)/0x%p)\n", nfs_i_fscache(inode), page, page->index, page->flags, inode); @@ -413,31 +264,7 @@ int __nfs_readpage_from_fscache(struct nfs_open_context *ctx, return 1; } - ret = fscache_read_or_alloc_page(nfs_i_fscache(inode), - page, - nfs_readpage_from_fscache_complete, - ctx, - GFP_KERNEL); - - switch (ret) { - case 0: /* read BIO submitted (page in fscache) */ - dfprintk(FSCACHE, - "NFS: readpage_from_fscache: BIO submitted\n"); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK); - return ret; - - case -ENOBUFS: /* inode not in cache */ - case -ENODATA: /* page not in cache */ - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); - dfprintk(FSCACHE, - "NFS: readpage_from_fscache %d\n", ret); - return 1; - - default: - dfprintk(FSCACHE, "NFS: readpage_from_fscache %d\n", ret); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); - } - return ret; + return -ENOBUFS; // TODO: Use netfslib } /* @@ -449,45 +276,10 @@ int __nfs_readpages_from_fscache(struct nfs_open_context *ctx, struct list_head *pages, unsigned *nr_pages) { - unsigned npages = *nr_pages; - int ret; - dfprintk(FSCACHE, "NFS: nfs_getpages_from_fscache (0x%p/%u/0x%p)\n", - nfs_i_fscache(inode), npages, inode); - - ret = fscache_read_or_alloc_pages(nfs_i_fscache(inode), - mapping, pages, nr_pages, - nfs_readpage_from_fscache_complete, - ctx, - mapping_gfp_mask(mapping)); - if (*nr_pages < npages) - nfs_add_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK, - npages); - if (*nr_pages > 0) - nfs_add_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL, - *nr_pages); - - switch (ret) { - case 0: /* read submitted to the cache for all pages */ - BUG_ON(!list_empty(pages)); - BUG_ON(*nr_pages != 0); - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: submitted\n"); - - return ret; - - case -ENOBUFS: /* some pages aren't cached and can't be */ - case -ENODATA: /* some pages aren't cached */ - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: no page: %d\n", ret); - return 1; + nfs_i_fscache(inode), *nr_pages, inode); - default: - dfprintk(FSCACHE, - "NFS: nfs_getpages_from_fscache: ret %d\n", ret); - } - - return ret; + return -ENOBUFS; // TODO: Use netfslib } /* @@ -496,25 +288,9 @@ int __nfs_readpages_from_fscache(struct nfs_open_context *ctx, */ void __nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync) { - int ret; - dfprintk(FSCACHE, "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx)/%d)\n", nfs_i_fscache(inode), page, page->index, page->flags, sync); - ret = fscache_write_page(nfs_i_fscache(inode), page, - inode->i_size, GFP_KERNEL); - dfprintk(FSCACHE, - "NFS: readpage_to_fscache: p:%p(i:%lu f:%lx) ret %d\n", - page, page->index, page->flags, ret); - - if (ret != 0) { - fscache_uncache_page(nfs_i_fscache(inode), page); - nfs_inc_fscache_stats(inode, - NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_UNCACHED); - } else { - nfs_inc_fscache_stats(inode, - NFSIOS_FSCACHE_PAGES_WRITTEN_OK); - } + return; // TODO: Use netfslib } diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index 6754c8607230..0fa267243d26 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -12,46 +12,10 @@ #include #include #include +#include #ifdef CONFIG_NFS_FSCACHE -/* - * set of NFS FS-Cache objects that form a superblock key - */ -struct nfs_fscache_key { - struct rb_node node; - struct nfs_client *nfs_client; /* the server */ - - /* the elements of the unique key - as used by nfs_compare_super() and - * nfs_compare_mount_options() to distinguish superblocks */ - struct { - struct { - unsigned long s_flags; /* various flags - * (& NFS_MS_MASK) */ - } super; - - struct { - struct nfs_fsid fsid; - int flags; - unsigned int rsize; /* read size */ - unsigned int wsize; /* write size */ - unsigned int acregmin; /* attr cache timeouts */ - unsigned int acregmax; - unsigned int acdirmin; - unsigned int acdirmax; - } nfs_server; - - struct { - rpc_authflavor_t au_flavor; - } rpc_auth; - - /* uniquifier - can be used if nfs_server.flags includes - * NFS_MOUNT_UNSHARED */ - u8 uniq_len; - char uniquifier[0]; - } key; -}; - /* * Definition of the auxiliary data attached to NFS inode storage objects * within the cache. @@ -69,32 +33,18 @@ struct nfs_fscache_inode_auxdata { u64 change_attr; }; -/* - * fscache-index.c - */ -extern struct fscache_netfs nfs_fscache_netfs; -extern const struct fscache_cookie_def nfs_fscache_server_index_def; -extern const struct fscache_cookie_def nfs_fscache_super_index_def; -extern const struct fscache_cookie_def nfs_fscache_inode_object_def; - -extern int nfs_fscache_register(void); -extern void nfs_fscache_unregister(void); - /* * fscache.c */ -extern void nfs_fscache_get_client_cookie(struct nfs_client *); -extern void nfs_fscache_release_client_cookie(struct nfs_client *); - -extern void nfs_fscache_get_super_cookie(struct super_block *, const char *, int); +extern int nfs_fscache_get_super_cookie(struct super_block *, const char *, int); extern void nfs_fscache_release_super_cookie(struct super_block *); extern void nfs_fscache_init_inode(struct inode *); extern void nfs_fscache_clear_inode(struct inode *); extern void nfs_fscache_open_file(struct inode *, struct file *); +extern void nfs_fscache_release_file(struct inode *, struct file *); extern void __nfs_fscache_invalidate_page(struct page *, struct inode *); -extern int nfs_fscache_release_page(struct page *, gfp_t); extern int __nfs_readpage_from_fscache(struct nfs_open_context *, struct inode *, struct page *); @@ -103,25 +53,17 @@ extern int __nfs_readpages_from_fscache(struct nfs_open_context *, struct list_head *, unsigned *); extern void __nfs_readpage_to_fscache(struct inode *, struct page *, int); -/* - * wait for a page to complete writing to the cache - */ -static inline void nfs_fscache_wait_on_page_write(struct nfs_inode *nfsi, - struct page *page) -{ - if (PageFsCache(page)) - fscache_wait_on_page_write(nfsi->fscache, page); -} - -/* - * release the caching state associated with a page if undergoing complete page - * invalidation - */ -static inline void nfs_fscache_invalidate_page(struct page *page, - struct inode *inode) +static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { - if (PageFsCache(page)) - __nfs_fscache_invalidate_page(page, inode); + if (PageFsCache(page)) { + if (!gfpflags_allow_blocking(gfp) || !(gfp & __GFP_FS)) + return false; + wait_on_page_fscache(page); + fscache_note_page_release(nfs_i_fscache(page->mapping->host)); + nfs_inc_fscache_stats(page->mapping->host, + NFSIOS_FSCACHE_PAGES_UNCACHED); + } + return true; } /* @@ -163,20 +105,32 @@ static inline void nfs_readpage_to_fscache(struct inode *inode, __nfs_readpage_to_fscache(inode, page, sync); } -/* - * Invalidate the contents of fscache for this inode. This will not sleep. - */ -static inline void nfs_fscache_invalidate(struct inode *inode) +static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, + struct nfs_inode *nfsi) { - fscache_invalidate(NFS_I(inode)->fscache); + memset(auxdata, 0, sizeof(*auxdata)); + auxdata->mtime_sec = nfsi->vfs_inode.i_mtime.tv_sec; + auxdata->mtime_nsec = nfsi->vfs_inode.i_mtime.tv_nsec; + auxdata->ctime_sec = nfsi->vfs_inode.i_ctime.tv_sec; + auxdata->ctime_nsec = nfsi->vfs_inode.i_ctime.tv_nsec; + + if (NFS_SERVER(&nfsi->vfs_inode)->nfs_client->rpc_ops->version == 4) + auxdata->change_attr = inode_peek_iversion_raw(&nfsi->vfs_inode); } /* - * Wait for an object to finish being invalidated. + * Invalidate the contents of fscache for this inode. This will not sleep. */ -static inline void nfs_fscache_wait_on_invalidate(struct inode *inode) +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) { - fscache_wait_on_invalidate(NFS_I(inode)->fscache); + struct nfs_fscache_inode_auxdata auxdata; + struct nfs_inode *nfsi = NFS_I(inode); + + if (nfsi->fscache) { + nfs_fscache_update_auxdata(&auxdata, nfsi); + fscache_invalidate(nfsi->fscache, &auxdata, + i_size_read(&nfsi->vfs_inode), flags); + } } /* @@ -190,28 +144,18 @@ static inline const char *nfs_server_fscache_state(struct nfs_server *server) } #else /* CONFIG_NFS_FSCACHE */ -static inline int nfs_fscache_register(void) { return 0; } -static inline void nfs_fscache_unregister(void) {} - -static inline void nfs_fscache_get_client_cookie(struct nfs_client *clp) {} -static inline void nfs_fscache_release_client_cookie(struct nfs_client *clp) {} - static inline void nfs_fscache_release_super_cookie(struct super_block *sb) {} static inline void nfs_fscache_init_inode(struct inode *inode) {} static inline void nfs_fscache_clear_inode(struct inode *inode) {} static inline void nfs_fscache_open_file(struct inode *inode, struct file *filp) {} +static inline void nfs_fscache_release_file(struct inode *inode, struct file *file) {} static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { return 1; /* True: may release page */ } -static inline void nfs_fscache_invalidate_page(struct page *page, - struct inode *inode) {} -static inline void nfs_fscache_wait_on_page_write(struct nfs_inode *nfsi, - struct page *page) {} - static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx, struct inode *inode, struct page *page) @@ -230,8 +174,7 @@ static inline void nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync) {} -static inline void nfs_fscache_invalidate(struct inode *inode) {} -static inline void nfs_fscache_wait_on_invalidate(struct inode *inode) {} +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) {} static inline const char *nfs_server_fscache_state(struct nfs_server *server) { diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index fda530d5e764..a918c3a834b6 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -209,7 +209,7 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) if (!nfs_has_xattr_cache(nfsi)) flags &= ~NFS_INO_INVALID_XATTR; if (flags & NFS_INO_INVALID_DATA) - nfs_fscache_invalidate(inode); + nfs_fscache_invalidate(inode, 0); flags &= ~(NFS_INO_REVAL_PAGECACHE | NFS_INO_REVAL_FORCED); nfsi->cache_validity |= flags; @@ -1289,6 +1289,7 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map { int ret; + nfs_fscache_invalidate(inode, 0); if (mapping->nrpages != 0) { if (S_ISREG(inode->i_mode)) { ret = nfs_sync_mapping(mapping); @@ -1300,7 +1301,6 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map return ret; } nfs_inc_stats(inode, NFSIOS_DATAINVALIDATE); - nfs_fscache_wait_on_invalidate(inode); dfprintk(PAGECACHE, "NFS: (%s/%Lu) data cache invalidated\n", inode->i_sb->s_id, @@ -2374,10 +2374,6 @@ static int __init init_nfs_fs(void) if (err < 0) goto out9; - err = nfs_fscache_register(); - if (err < 0) - goto out8; - err = nfsiod_start(); if (err) goto out7; @@ -2429,8 +2425,6 @@ out5: out6: nfsiod_stop(); out7: - nfs_fscache_unregister(); -out8: unregister_pernet_subsys(&nfs_net_ops); out9: nfs_sysfs_exit(); @@ -2445,7 +2439,6 @@ static void __exit exit_nfs_fs(void) nfs_destroy_readpagecache(); nfs_destroy_inodecache(); nfs_destroy_nfspagecache(); - nfs_fscache_unregister(); unregister_pernet_subsys(&nfs_net_ops); rpc_proc_unregister(&init_net, "nfs"); unregister_nfs_fs(); diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index b3aee261801e..317ce27bdc4b 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -42,7 +42,6 @@ { BIT(NFS_INO_ACL_LRU_SET), "ACL_LRU_SET" }, \ { BIT(NFS_INO_INVALIDATING), "INVALIDATING" }, \ { BIT(NFS_INO_FSCACHE), "FSCACHE" }, \ - { BIT(NFS_INO_FSCACHE_LOCK), "FSCACHE_LOCK" }, \ { BIT(NFS_INO_LAYOUTCOMMIT), "NEED_LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTCOMMITTING), "LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTSTATS), "LAYOUTSTATS" }, \ diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 3aced401735c..6ab5eeb000dc 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -1204,42 +1204,42 @@ static int nfs_compare_super(struct super_block *sb, struct fs_context *fc) } #ifdef CONFIG_NFS_FSCACHE -static void nfs_get_cache_cookie(struct super_block *sb, - struct nfs_fs_context *ctx) +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) { struct nfs_server *nfss = NFS_SB(sb); char *uniq = NULL; int ulen = 0; - nfss->fscache_key = NULL; nfss->fscache = NULL; if (!ctx) - return; + return 0; if (ctx->clone_data.sb) { struct nfs_server *mnt_s = NFS_SB(ctx->clone_data.sb); if (!(mnt_s->options & NFS_OPTION_FSCACHE)) - return; - if (mnt_s->fscache_key) { - uniq = mnt_s->fscache_key->key.uniquifier; - ulen = mnt_s->fscache_key->key.uniq_len; + return 0; + if (mnt_s->fscache_uniq) { + uniq = mnt_s->fscache_uniq; + ulen = strlen(uniq); } } else { if (!(ctx->options & NFS_OPTION_FSCACHE)) - return; + return 0; if (ctx->fscache_uniq) { uniq = ctx->fscache_uniq; ulen = strlen(ctx->fscache_uniq); } } - nfs_fscache_get_super_cookie(sb, uniq, ulen); + return nfs_fscache_get_super_cookie(sb, uniq, ulen); } #else -static void nfs_get_cache_cookie(struct super_block *sb, - struct nfs_fs_context *ctx) +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) { + return 0; } #endif @@ -1299,7 +1299,9 @@ int nfs_get_tree_common(struct fs_context *fc) s->s_blocksize_bits = bsize; s->s_blocksize = 1U << bsize; } - nfs_get_cache_cookie(s, ctx); + error = nfs_get_cache_cookie(s, ctx); + if (error < 0) + goto error_splat_super; } error = nfs_get_root(s, fc); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 9b7619ce17a7..2b322170372a 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -294,6 +294,7 @@ static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int c nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); out: spin_unlock(&inode->i_lock); + nfs_fscache_invalidate(inode, 0); } /* A writeback failed: mark the page as bad, and invalidate the page cache */ diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 05f249f20f55..00835bacd236 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -275,7 +275,6 @@ struct nfs4_copy_state { #define NFS_INO_ACL_LRU_SET (2) /* Inode is on the LRU list */ #define NFS_INO_INVALIDATING (3) /* inode is being invalidated */ #define NFS_INO_FSCACHE (5) /* inode can be cached by FS-Cache */ -#define NFS_INO_FSCACHE_LOCK (6) /* FS-Cache cookie management lock */ #define NFS_INO_FORCE_READDIR (7) /* force readdirplus */ #define NFS_INO_LAYOUTCOMMIT (9) /* layoutcommit required */ #define NFS_INO_LAYOUTCOMMITTING (10) /* layoutcommit inflight */ diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 2a9acbfe00f0..77b2dba27bbb 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -120,11 +120,6 @@ struct nfs_client { * This is used to generate the mv0 callback address. */ char cl_ipaddr[48]; - -#ifdef CONFIG_NFS_FSCACHE - struct fscache_cookie *fscache; /* client index cache cookie */ -#endif - struct net *cl_net; struct list_head pending_cb_stateids; }; @@ -194,8 +189,8 @@ struct nfs_server { struct nfs_auth_info auth_info; /* parsed auth flavors */ #ifdef CONFIG_NFS_FSCACHE - struct nfs_fscache_key *fscache_key; /* unique key for superblock */ - struct fscache_cookie *fscache; /* superblock cookie */ + struct fscache_volume *fscache; /* superblock cookie */ + char *fscache_uniq; /* Uniquifier (or NULL) */ #endif u32 pnfs_blksize; /* layout_blksize attr */ -- cgit v1.2.3-59-g8ed1b From 16f2f4e679cfdaa9552574484f104014908a76c6 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 27 Aug 2021 15:19:34 +0100 Subject: nfs: Implement cache I/O by accessing the cache directly Move NFS to using fscache DIO API instead of the old upstream I/O API as that has been removed. This is a stopgap solution as the intention is that at sometime in the future, the cache will move to using larger blocks and won't be able to store individual pages in order to deal with the potential for data corruption due to the backing filesystem being able insert/remove bridging blocks of zeros into its extent list[1]. NFS then reads and writes cache pages synchronously and one page at a time. The preferred change would be to use the netfs lib, but the new I/O API can be used directly. It's just that as the cache now needs to track data for itself, caching blocks may exceed page size... This code is somewhat borrowed from my "fallback I/O" patchset[2]. Changes ======= ver #3: - Restore lost =n fallback for nfs_fscache_release_page()[2]. Signed-off-by: David Howells Tested-by: Dave Wysochanski Acked-by: Jeff Layton cc: Trond Myklebust cc: Anna Schumaker cc: linux-nfs@vger.kernel.org cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/YO17ZNOcq+9PajfQ@mit.edu [1] Link: https://lore.kernel.org/r/202112100957.2oEDT20W-lkp@intel.com/ [2] Link: https://lore.kernel.org/r/163189108292.2509237.12615909591150927232.stgit@warthog.procyon.org.uk/ [2] Link: https://lore.kernel.org/r/163906981318.143852.17220018647843475985.stgit@warthog.procyon.org.uk/ # v2 Link: https://lore.kernel.org/r/163967184451.1823006.6450645559828329590.stgit@warthog.procyon.org.uk/ # v3 Link: https://lore.kernel.org/r/164021577632.640689.11069627070150063812.stgit@warthog.procyon.org.uk/ # v4 --- fs/fscache/io.c | 8 +++ fs/nfs/fscache.c | 126 +++++++++++++++++++++++++++++++++++++++--------- fs/nfs/fscache.h | 52 +++++--------------- fs/nfs/read.c | 25 +++------- fs/nfs/write.c | 7 ++- include/linux/fscache.h | 28 +++++++++++ 6 files changed, 163 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/fs/fscache/io.c b/fs/fscache/io.c index bed7628a5a9d..7a769ea57720 100644 --- a/fs/fscache/io.c +++ b/fs/fscache/io.c @@ -150,6 +150,14 @@ int __fscache_begin_read_operation(struct netfs_cache_resources *cres, } EXPORT_SYMBOL(__fscache_begin_read_operation); +int __fscache_begin_write_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_write); +} +EXPORT_SYMBOL(__fscache_begin_write_operation); + /** * fscache_set_page_dirty - Mark page dirty and pin a cache object for writeback * @page: The page being dirtied diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index fac6438477a0..cfe901650ab0 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -249,48 +249,128 @@ void nfs_fscache_release_file(struct inode *inode, struct file *filp) } } +static inline void fscache_end_operation(struct netfs_cache_resources *cres) +{ + const struct netfs_cache_ops *ops = fscache_operation_valid(cres); + + if (ops) + ops->end_operation(cres); +} + +/* + * Fallback page reading interface. + */ +static int fscache_fallback_read_page(struct inode *inode, struct page *page) +{ + struct netfs_cache_resources cres; + struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct iov_iter iter; + struct bio_vec bvec[1]; + int ret; + + memset(&cres, 0, sizeof(cres)); + bvec[0].bv_page = page; + bvec[0].bv_offset = 0; + bvec[0].bv_len = PAGE_SIZE; + iov_iter_bvec(&iter, READ, bvec, ARRAY_SIZE(bvec), PAGE_SIZE); + + ret = fscache_begin_read_operation(&cres, cookie); + if (ret < 0) + return ret; + + ret = fscache_read(&cres, page_offset(page), &iter, NETFS_READ_HOLE_FAIL, + NULL, NULL); + fscache_end_operation(&cres); + return ret; +} + +/* + * Fallback page writing interface. + */ +static int fscache_fallback_write_page(struct inode *inode, struct page *page, + bool no_space_allocated_yet) +{ + struct netfs_cache_resources cres; + struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct iov_iter iter; + struct bio_vec bvec[1]; + loff_t start = page_offset(page); + size_t len = PAGE_SIZE; + int ret; + + memset(&cres, 0, sizeof(cres)); + bvec[0].bv_page = page; + bvec[0].bv_offset = 0; + bvec[0].bv_len = PAGE_SIZE; + iov_iter_bvec(&iter, WRITE, bvec, ARRAY_SIZE(bvec), PAGE_SIZE); + + ret = fscache_begin_write_operation(&cres, cookie); + if (ret < 0) + return ret; + + ret = cres.ops->prepare_write(&cres, &start, &len, i_size_read(inode), + no_space_allocated_yet); + if (ret == 0) + ret = fscache_write(&cres, page_offset(page), &iter, NULL, NULL); + fscache_end_operation(&cres); + return ret; +} + /* * Retrieve a page from fscache */ -int __nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, struct page *page) +int __nfs_readpage_from_fscache(struct inode *inode, struct page *page) { + int ret; + dfprintk(FSCACHE, "NFS: readpage_from_fscache(fsc:%p/p:%p(i:%lx f:%lx)/0x%p)\n", nfs_i_fscache(inode), page, page->index, page->flags, inode); if (PageChecked(page)) { + dfprintk(FSCACHE, "NFS: readpage_from_fscache: PageChecked\n"); ClearPageChecked(page); return 1; } - return -ENOBUFS; // TODO: Use netfslib -} - -/* - * Retrieve a set of pages from fscache - */ -int __nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - dfprintk(FSCACHE, "NFS: nfs_getpages_from_fscache (0x%p/%u/0x%p)\n", - nfs_i_fscache(inode), *nr_pages, inode); + ret = fscache_fallback_read_page(inode, page); + if (ret < 0) { + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); + dfprintk(FSCACHE, + "NFS: readpage_from_fscache failed %d\n", ret); + SetPageChecked(page); + return ret; + } - return -ENOBUFS; // TODO: Use netfslib + /* Read completed synchronously */ + dfprintk(FSCACHE, "NFS: readpage_from_fscache: read successful\n"); + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK); + SetPageUptodate(page); + return 0; } /* - * Store a newly fetched page in fscache - * - PG_fscache must be set on the page + * Store a newly fetched page in fscache. We can be certain there's no page + * stored in the cache as yet otherwise we would've read it from there. */ -void __nfs_readpage_to_fscache(struct inode *inode, struct page *page, int sync) +void __nfs_readpage_to_fscache(struct inode *inode, struct page *page) { + int ret; + dfprintk(FSCACHE, - "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx)/%d)\n", - nfs_i_fscache(inode), page, page->index, page->flags, sync); + "NFS: readpage_to_fscache(fsc:%p/p:%p(i:%lx f:%lx))\n", + nfs_i_fscache(inode), page, page->index, page->flags); - return; // TODO: Use netfslib + ret = fscache_fallback_write_page(inode, page, true); + + dfprintk(FSCACHE, + "NFS: readpage_to_fscache: p:%p(i:%lu f:%lx) ret %d\n", + page, page->index, page->flags, ret); + + if (ret != 0) { + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL); + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_UNCACHED); + } else { + nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_OK); + } } diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index 0fa267243d26..e0220fc40366 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -44,14 +44,10 @@ extern void nfs_fscache_clear_inode(struct inode *); extern void nfs_fscache_open_file(struct inode *, struct file *); extern void nfs_fscache_release_file(struct inode *, struct file *); -extern void __nfs_fscache_invalidate_page(struct page *, struct inode *); - -extern int __nfs_readpage_from_fscache(struct nfs_open_context *, - struct inode *, struct page *); -extern int __nfs_readpages_from_fscache(struct nfs_open_context *, - struct inode *, struct address_space *, - struct list_head *, unsigned *); -extern void __nfs_readpage_to_fscache(struct inode *, struct page *, int); +extern int __nfs_readpage_from_fscache(struct inode *, struct page *); +extern void __nfs_read_completion_to_fscache(struct nfs_pgio_header *hdr, + unsigned long bytes); +extern void __nfs_readpage_to_fscache(struct inode *, struct page *); static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { @@ -69,27 +65,11 @@ static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) /* * Retrieve a page from an inode data storage object. */ -static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, +static inline int nfs_readpage_from_fscache(struct inode *inode, struct page *page) { if (NFS_I(inode)->fscache) - return __nfs_readpage_from_fscache(ctx, inode, page); - return -ENOBUFS; -} - -/* - * Retrieve a set of pages from an inode data storage object. - */ -static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - if (NFS_I(inode)->fscache) - return __nfs_readpages_from_fscache(ctx, inode, mapping, pages, - nr_pages); + return __nfs_readpage_from_fscache(inode, page); return -ENOBUFS; } @@ -98,11 +78,10 @@ static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, * in the cache. */ static inline void nfs_readpage_to_fscache(struct inode *inode, - struct page *page, - int sync) + struct page *page) { - if (PageFsCache(page)) - __nfs_readpage_to_fscache(inode, page, sync); + if (NFS_I(inode)->fscache) + __nfs_readpage_to_fscache(inode, page); } static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, @@ -156,22 +135,13 @@ static inline int nfs_fscache_release_page(struct page *page, gfp_t gfp) { return 1; /* True: may release page */ } -static inline int nfs_readpage_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, +static inline int nfs_readpage_from_fscache(struct inode *inode, struct page *page) { return -ENOBUFS; } -static inline int nfs_readpages_from_fscache(struct nfs_open_context *ctx, - struct inode *inode, - struct address_space *mapping, - struct list_head *pages, - unsigned *nr_pages) -{ - return -ENOBUFS; -} static inline void nfs_readpage_to_fscache(struct inode *inode, - struct page *page, int sync) {} + struct page *page) {} static inline void nfs_fscache_invalidate(struct inode *inode, int flags) {} diff --git a/fs/nfs/read.c b/fs/nfs/read.c index d11af2a9299c..eb00229c1a50 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -123,7 +123,7 @@ static void nfs_readpage_release(struct nfs_page *req, int error) struct address_space *mapping = page_file_mapping(page); if (PageUptodate(page)) - nfs_readpage_to_fscache(inode, page, 0); + nfs_readpage_to_fscache(inode, page); else if (!PageError(page) && !PagePrivate(page)) generic_error_remove_page(mapping, page); unlock_page(page); @@ -305,6 +305,12 @@ readpage_async_filler(void *data, struct page *page) aligned_len = min_t(unsigned int, ALIGN(len, rsize), PAGE_SIZE); + if (!IS_SYNC(page->mapping->host)) { + error = nfs_readpage_from_fscache(page->mapping->host, page); + if (error == 0) + goto out_unlock; + } + new = nfs_create_request(desc->ctx, page, 0, aligned_len); if (IS_ERR(new)) goto out_error; @@ -320,6 +326,7 @@ readpage_async_filler(void *data, struct page *page) return 0; out_error: error = PTR_ERR(new); +out_unlock: unlock_page(page); out: return error; @@ -366,12 +373,6 @@ int nfs_readpage(struct file *file, struct page *page) desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); xchg(&desc.ctx->error, 0); - if (!IS_SYNC(inode)) { - ret = nfs_readpage_from_fscache(desc.ctx, inode, page); - if (ret == 0) - goto out_wait; - } - nfs_pageio_init_read(&desc.pgio, inode, false, &nfs_async_read_completion_ops); @@ -381,7 +382,6 @@ int nfs_readpage(struct file *file, struct page *page) nfs_pageio_complete_read(&desc.pgio); ret = desc.pgio.pg_error < 0 ? desc.pgio.pg_error : 0; -out_wait: if (!ret) { ret = wait_on_page_locked_killable(page); if (!PageUptodate(page) && !ret) @@ -419,14 +419,6 @@ int nfs_readpages(struct file *file, struct address_space *mapping, } else desc.ctx = get_nfs_open_context(nfs_file_open_context(file)); - /* attempt to read as many of the pages as possible from the cache - * - this returns -ENOBUFS immediately if the cookie is negative - */ - ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping, - pages, &nr_pages); - if (ret == 0) - goto read_complete; /* all pages were read */ - nfs_pageio_init_read(&desc.pgio, inode, false, &nfs_async_read_completion_ops); @@ -434,7 +426,6 @@ int nfs_readpages(struct file *file, struct address_space *mapping, nfs_pageio_complete_read(&desc.pgio); -read_complete: put_nfs_open_context(desc.ctx); out: trace_nfs_aop_readahead_done(inode, nr_pages, ret); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 2b322170372a..987a187bd39a 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -2126,8 +2126,11 @@ int nfs_migrate_page(struct address_space *mapping, struct page *newpage, if (PagePrivate(page)) return -EBUSY; - if (!nfs_fscache_release_page(page, GFP_KERNEL)) - return -EBUSY; + if (PageFsCache(page)) { + if (mode == MIGRATE_ASYNC) + return -EBUSY; + wait_on_page_fscache(page); + } return migrate_page(mapping, newpage, page, mode); } diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 7bd35f60d19a..ede50406bcb0 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -168,6 +168,7 @@ extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); extern void __fscache_resize_cookie(struct fscache_cookie *, loff_t); extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); +extern int __fscache_begin_write_operation(struct netfs_cache_resources *, struct fscache_cookie *); extern void __fscache_write_to_cache(struct fscache_cookie *, struct address_space *, loff_t, size_t, loff_t, netfs_io_terminated_t, void *, @@ -499,6 +500,33 @@ int fscache_read(struct netfs_cache_resources *cres, term_func, term_func_priv); } +/** + * fscache_begin_write_operation - Begin a write operation for the netfs lib + * @cres: The cache resources for the write being performed + * @cookie: The cookie representing the cache object + * + * Begin a write operation on behalf of the netfs helper library. @cres + * indicates the cache resources to which the operation state should be + * attached; @cookie indicates the cache object that will be accessed. + * + * @cres->inval_counter is set from @cookie->inval_counter for comparison at + * the end of the operation. This allows invalidation during the operation to + * be detected by the caller. + * + * Returns: + * * 0 - Success + * * -ENOBUFS - No caching available + * * Other error code from the cache, such as -ENOMEM. + */ +static inline +int fscache_begin_write_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + if (fscache_cookie_enabled(cookie)) + return __fscache_begin_write_operation(cres, cookie); + return -ENOBUFS; +} + /** * fscache_write - Start a write to the cache. * @cres: The cache resources to use -- cgit v1.2.3-59-g8ed1b From a59466ee91aaa9d43889a4c51e01de087d188448 Mon Sep 17 00:00:00 2001 From: Karolina Drobnik Date: Tue, 11 Jan 2022 10:28:47 +0000 Subject: memblock: Remove #ifdef __KERNEL__ from memblock.h memblock.h is not a uAPI header, so __KERNEL__ guard can be deleted Signed-off-by: Karolina Drobnik Signed-off-by: Mike Rapoport Link: https://lore.kernel.org/r/20220111102847.673746-1-karolinadrobnik@gmail.com --- include/linux/memblock.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 9dc7cb239d21..50ad19662a32 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ #ifndef _LINUX_MEMBLOCK_H #define _LINUX_MEMBLOCK_H -#ifdef __KERNEL__ /* * Logical memory blocks. @@ -605,6 +604,5 @@ static inline void early_memtest(phys_addr_t start, phys_addr_t end) } #endif -#endif /* __KERNEL__ */ #endif /* _LINUX_MEMBLOCK_H */ -- cgit v1.2.3-59-g8ed1b From 500b55b05d0a21c4adddf4c3b29ee6f32b502046 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 21 Dec 2021 10:45:07 -0600 Subject: PCI: Work around Intel I210 ROM BAR overlap defect Per PCIe r5, sec 7.5.1.2.4, a device must not claim accesses to its Expansion ROM unless both the Memory Space Enable and the Expansion ROM Enable bit are set. But apparently some Intel I210 NICs don't work correctly if the ROM BAR overlaps another BAR, even if the Expansion ROM is disabled. Michael reported that on a Kontron SMARC-sAL28 ARM64 system with U-Boot v2021.01-rc3, the ROM BAR overlaps BAR 3, and networking doesn't work at all: BAR 0: 0x40000000 (32-bit, non-prefetchable) [size=1M] BAR 3: 0x40200000 (32-bit, non-prefetchable) [size=16K] ROM: 0x40200000 (disabled) [size=1M] NETDEV WATCHDOG: enP2p1s0 (igb): transmit queue 0 timed out Hardware name: Kontron SMARC-sAL28 (Single PHY) on SMARC Eval 2.0 carrier (DT) igb 0002:01:00.0 enP2p1s0: Reset adapter Previously, pci_std_update_resource() wrote the assigned ROM address to the BAR only when the ROM was enabled. This meant that the I210 ROM BAR could be left with an address assigned by firmware, which might overlap with other BARs. Quirk these I210 devices so pci_std_update_resource() always writes the assigned address to the ROM BAR, whether or not the ROM is enabled. Link: https://lore.kernel.org/r/20211223163754.GA1267351@bhelgaas Link: https://lore.kernel.org/r/20201230185317.30915-1-michael@walle.cc Link: https://bugzilla.kernel.org/show_bug.cgi?id=211105 Reported-by: Michael Walle Tested-by: Michael Walle Signed-off-by: Bjorn Helgaas --- drivers/pci/quirks.c | 10 ++++++++++ drivers/pci/setup-res.c | 8 ++++++-- include/linux/pci.h | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 003950c738d2..46ff04091fa3 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5857,3 +5857,13 @@ static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) pdev->dev_flags |= PCI_DEV_FLAGS_HAS_MSI_MASKING; } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0ab8, nvidia_ion_ahci_fixup); + +static void rom_bar_overlap_defect(struct pci_dev *dev) +{ + pci_info(dev, "working around ROM BAR overlap defect\n"); + dev->rom_bar_overlap = 1; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1533, rom_bar_overlap_defect); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1536, rom_bar_overlap_defect); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1537, rom_bar_overlap_defect); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1538, rom_bar_overlap_defect); diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 7f1acb3918d0..439ac5f5907a 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -75,12 +75,16 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) * as zero when disabled, so don't update ROM BARs unless * they're enabled. See * https://lore.kernel.org/r/43147B3D.1030309@vc.cvut.cz/ + * But we must update ROM BAR for buggy devices where even a + * disabled ROM can conflict with other BARs. */ - if (!(res->flags & IORESOURCE_ROM_ENABLE)) + if (!(res->flags & IORESOURCE_ROM_ENABLE) && + !dev->rom_bar_overlap) return; reg = dev->rom_base_reg; - new |= PCI_ROM_ADDRESS_ENABLE; + if (res->flags & IORESOURCE_ROM_ENABLE) + new |= PCI_ROM_ADDRESS_ENABLE; } else return; diff --git a/include/linux/pci.h b/include/linux/pci.h index 18a75c8e615c..51c4a063f489 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -455,6 +455,7 @@ struct pci_dev { unsigned int link_active_reporting:1;/* Device capable of reporting link active */ unsigned int no_vf_scan:1; /* Don't scan for VFs after IOV enablement */ unsigned int no_command_memory:1; /* No PCI_COMMAND_MEMORY */ + unsigned int rom_bar_overlap:1; /* ROM BAR disable broken */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ -- cgit v1.2.3-59-g8ed1b From 1d3cfc2835c1754d19a743dc346a9e58cf0c07c0 Mon Sep 17 00:00:00 2001 From: Kelvin Cao Date: Thu, 23 Dec 2021 17:23:33 -0800 Subject: ntb_hw_switchtec: Remove code for disabling ID protection ID protection is a firmware setting for NT window access control. With it enabled, only the posted requests with requester IDs in the requester ID table will be allowed to access the NT windows. Otherwise all posted requests are allowed. Normally user will configure it statically via the Switchtec config file, and it will take effect when the firmware boots up. The driver can also toggle the ID protection setting dynamically, which will overwrite the static setting in the Switchtec config file as a side effect. Currently, the driver disables the ID protection. However, it's not necessary to disable the ID protection at the driver level as the driver has already configured the proper requester IDs in the requester ID table to allow the corresponding posted requests to hit the NT windows. Remove the code that disables the ID protection to make the static setting prevail. Note: ID protection is not applicable to non-posted requests. Signed-off-by: Kelvin Cao Signed-off-by: Jon Mason --- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 3 --- include/linux/switchtec.h | 2 -- 2 files changed, 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index ba6a9670c681..e79a355bafbf 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -967,9 +967,6 @@ static int config_req_id_table(struct switchtec_ntb *sndev, if (rc) return rc; - iowrite32(NTB_PART_CTRL_ID_PROT_DIS, - &mmio_ctrl->partition_ctrl); - for (i = 0; i < count; i++) { iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, &mmio_ctrl->req_id_table[i]); diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index be24056ac00f..48fabe36509e 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -337,8 +337,6 @@ enum { NTB_CTRL_REQ_ID_EN = 1 << 0, NTB_CTRL_LUT_EN = 1 << 0, - - NTB_PART_CTRL_ID_PROT_DIS = 1 << 0, }; struct ntb_ctrl_regs { -- cgit v1.2.3-59-g8ed1b From ca321ec74322e3c49552fc1ffc80b42d0dbf1a84 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sat, 8 Jan 2022 15:06:57 +0100 Subject: module.h: allow #define strings to work with MODULE_IMPORT_NS The MODULE_IMPORT_NS() macro does not allow defined strings to work properly with it, so add a layer of indirection to allow this to happen. Cc: Luis Chamberlain Cc: Jessica Yu Cc: Matthias Maennich Signed-off-by: Greg Kroah-Hartman Reviewed-by: Matthias Maennich Signed-off-by: Luis Chamberlain --- include/linux/module.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/module.h b/include/linux/module.h index c9f1200b2312..f4338235ed2c 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -290,7 +290,8 @@ extern typeof(name) __mod_##type##__##name##_device_table \ * files require multiple MODULE_FIRMWARE() specifiers */ #define MODULE_FIRMWARE(_firmware) MODULE_INFO(firmware, _firmware) -#define MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, #ns) +#define _MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, #ns) +#define MODULE_IMPORT_NS(ns) _MODULE_IMPORT_NS(ns) struct notifier_block; -- cgit v1.2.3-59-g8ed1b From 3f4b32511a77bc5a05cfbf26fec94c4e1b1cf46a Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 7 Jan 2022 18:17:18 +0000 Subject: PM: core: Remove DEFINE_UNIVERSAL_DEV_PM_OPS() macro The deprecated UNIVERSAL_DEV_PM_OPS() macro uses the provided callbacks for both runtime PM and system sleep, which is very likely to be a mistake, as a system sleep can be triggered while a given device is already PM-suspended, which would cause the suspend callback to be called twice. The amount of users of UNIVERSAL_DEV_PM_OPS() is also tiny (16 occurences) compared to the number of places where SET_SYSTEM_SLEEP_PM_OPS() is used with pm_runtime_force_suspend() and pm_runtime_force_resume(), which makes me think that none of these cases are actually valid. As the new macro DEFINE_UNIVERSAL_DEV_PM_OPS() which was introduced to replace UNIVERSAL_DEV_PM_OPS() is currently unused, remove it before someone starts to use it in yet another invalid case. Signed-off-by: Paul Cercueil Acked-by: Jonathan Cameron Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index e1e9402180b9..02f059d814bb 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -366,6 +366,12 @@ static const struct dev_pm_ops name = { \ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ } +/* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */ +#define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ +const struct dev_pm_ops __maybe_unused name = { \ + SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ +} + /* * Use this for defining a set of PM operations to be used in all situations * (system suspend, hibernation or runtime PM). @@ -378,20 +384,9 @@ static const struct dev_pm_ops name = { \ * suspend and "early" resume callback pointers, .suspend_late() and * .resume_early(), to the same routines as .runtime_suspend() and * .runtime_resume(), respectively (and analogously for hibernation). + * + * Deprecated. You most likely don't want this macro. */ -#define DEFINE_UNIVERSAL_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ -static const struct dev_pm_ops name = { \ - SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ - RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ -} - -/* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */ -#define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ -const struct dev_pm_ops __maybe_unused name = { \ - SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ -} - -/* Deprecated. Use DEFINE_UNIVERSAL_DEV_PM_OPS() instead. */ #define UNIVERSAL_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ const struct dev_pm_ops __maybe_unused name = { \ SET_SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ -- cgit v1.2.3-59-g8ed1b From 52cc1d7f9786d2be44a3ab9b5b48416a7618e713 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 7 Jan 2022 18:17:19 +0000 Subject: PM: core: Remove static qualifier in DEFINE_SIMPLE_DEV_PM_OPS macro Keep this macro in line with the other ones. This makes it possible to use them in the cases where the underlying dev_pm_ops structure is exported. Restore the "static" qualifier in the two drivers where the DEFINE_SIMPLE_DEV_PM_OPS macro was used. Signed-off-by: Paul Cercueil Acked-by: Jonathan Cameron Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/mmc/host/jz4740_mmc.c | 4 ++-- drivers/mmc/host/mxcmmc.c | 2 +- include/linux/pm.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c index bb612fce7ead..4ac87b0797bb 100644 --- a/drivers/mmc/host/jz4740_mmc.c +++ b/drivers/mmc/host/jz4740_mmc.c @@ -1113,8 +1113,8 @@ static int jz4740_mmc_resume(struct device *dev) return pinctrl_select_default_state(dev); } -DEFINE_SIMPLE_DEV_PM_OPS(jz4740_mmc_pm_ops, jz4740_mmc_suspend, - jz4740_mmc_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(jz4740_mmc_pm_ops, jz4740_mmc_suspend, + jz4740_mmc_resume); static struct platform_driver jz4740_mmc_driver = { .probe = jz4740_mmc_probe, diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c index 98c218bd6669..40b6878bea6c 100644 --- a/drivers/mmc/host/mxcmmc.c +++ b/drivers/mmc/host/mxcmmc.c @@ -1210,7 +1210,7 @@ static int mxcmci_resume(struct device *dev) return ret; } -DEFINE_SIMPLE_DEV_PM_OPS(mxcmci_pm_ops, mxcmci_suspend, mxcmci_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(mxcmci_pm_ops, mxcmci_suspend, mxcmci_resume); static struct platform_driver mxcmci_driver = { .probe = mxcmci_probe, diff --git a/include/linux/pm.h b/include/linux/pm.h index 02f059d814bb..8e13387e70ec 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -362,7 +362,7 @@ struct dev_pm_ops { * to RAM and hibernation. */ #define DEFINE_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ -static const struct dev_pm_ops name = { \ +const struct dev_pm_ops name = { \ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ } -- cgit v1.2.3-59-g8ed1b From 0ae101fdd3297b7165755340e05386f1e1379709 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 7 Jan 2022 18:17:20 +0000 Subject: PM: core: Add EXPORT[_GPL]_SIMPLE_DEV_PM_OPS macros These macros are defined conditionally, according to CONFIG_PM: - if CONFIG_PM is enabled, these macros resolve to DEFINE_SIMPLE_DEV_PM_OPS(), and the dev_pm_ops symbol will be exported. - if CONFIG_PM is disabled, these macros will result in a dummy static dev_pm_ops to be created with the __maybe_unused flag. The dev_pm_ops will then be discarded by the compiler, along with the provided callback functions if they are not used anywhere else. In the second case, the symbol is not exported, which should be perfectly fine - users of the symbol should all use the pm_ptr() or pm_sleep_ptr() macro, so the dev_pm_ops marked as "extern" in the client's code will never be accessed. Signed-off-by: Paul Cercueil Acked-by: Jonathan Cameron Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index 8e13387e70ec..8279af2c538a 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -8,6 +8,7 @@ #ifndef _LINUX_PM_H #define _LINUX_PM_H +#include #include #include #include @@ -357,14 +358,42 @@ struct dev_pm_ops { #define SET_RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) #endif +#define _DEFINE_DEV_PM_OPS(name, \ + suspend_fn, resume_fn, \ + runtime_suspend_fn, runtime_resume_fn, idle_fn) \ +const struct dev_pm_ops name = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + RUNTIME_PM_OPS(runtime_suspend_fn, runtime_resume_fn, idle_fn) \ +} + +#ifdef CONFIG_PM +#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ + runtime_resume_fn, idle_fn, sec) \ + _DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ + runtime_resume_fn, idle_fn); \ + _EXPORT_SYMBOL(name, sec) +#else +#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ + runtime_resume_fn, idle_fn, sec) \ +static __maybe_unused _DEFINE_DEV_PM_OPS(__static_##name, suspend_fn, \ + resume_fn, runtime_suspend_fn, \ + runtime_resume_fn, idle_fn) +#endif + /* * Use this if you want to use the same suspend and resume callbacks for suspend * to RAM and hibernation. + * + * If the underlying dev_pm_ops struct symbol has to be exported, use + * EXPORT_SIMPLE_DEV_PM_OPS() or EXPORT_GPL_SIMPLE_DEV_PM_OPS() instead. */ #define DEFINE_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ -const struct dev_pm_ops name = { \ - SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ -} + _DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL) + +#define EXPORT_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ + _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "") +#define EXPORT_GPL_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ + _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "_gpl") /* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */ #define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ -- cgit v1.2.3-59-g8ed1b From 9d8619190031af0a314bee865262d8975473e4dd Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 7 Jan 2022 18:17:21 +0000 Subject: PM: runtime: Add DEFINE_RUNTIME_DEV_PM_OPS() macro A lot of drivers create a dev_pm_ops struct with the system sleep suspend/resume callbacks set to pm_runtime_force_suspend() and pm_runtime_force_resume(). These drivers can now use the DEFINE_RUNTIME_DEV_PM_OPS() macro, which will use pm_runtime_force_{suspend,resume}() as the system sleep callbacks, while having the same dead code removal characteristic that is already provided by DEFINE_SIMPLE_DEV_PM_OPS(). Signed-off-by: Paul Cercueil Acked-by: Jonathan Cameron Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- include/linux/pm.h | 3 ++- include/linux/pm_runtime.h | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pm.h b/include/linux/pm.h index 8279af2c538a..f7d2be686359 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -414,7 +414,8 @@ const struct dev_pm_ops __maybe_unused name = { \ * .resume_early(), to the same routines as .runtime_suspend() and * .runtime_resume(), respectively (and analogously for hibernation). * - * Deprecated. You most likely don't want this macro. + * Deprecated. You most likely don't want this macro. Use + * DEFINE_RUNTIME_DEV_PM_OPS() instead. */ #define UNIVERSAL_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ const struct dev_pm_ops __maybe_unused name = { \ diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 016de5776b6d..4af454d29281 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -22,6 +22,20 @@ usage_count */ #define RPM_AUTO 0x08 /* Use autosuspend_delay */ +/* + * Use this for defining a set of PM operations to be used in all situations + * (system suspend, hibernation or runtime PM). + * + * Note that the behaviour differs from the deprecated UNIVERSAL_DEV_PM_OPS() + * macro, which uses the provided callbacks for both runtime PM and system + * sleep, while DEFINE_RUNTIME_DEV_PM_OPS() uses pm_runtime_force_suspend() + * and pm_runtime_force_resume() for its system sleep callbacks. + */ +#define DEFINE_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ + _DEFINE_DEV_PM_OPS(name, pm_runtime_force_suspend, \ + pm_runtime_force_resume, suspend_fn, \ + resume_fn, idle_fn) + #ifdef CONFIG_PM extern struct workqueue_struct *pm_wq; -- cgit v1.2.3-59-g8ed1b From d59ff7d9d84b03d22c5107f794e28fc8e1fce3a6 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 7 Jan 2022 18:17:22 +0000 Subject: PM: runtime: Add EXPORT[_GPL]_RUNTIME_DEV_PM_OPS macros Similar to EXPORT[_GPL]_SIMPLE_DEV_PM_OPS, but for users with runtime-PM suspend/resume callbacks. Signed-off-by: Paul Cercueil Acked-by: Jonathan Cameron Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- include/linux/pm_runtime.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 4af454d29281..9f09601c465a 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -30,12 +30,22 @@ * macro, which uses the provided callbacks for both runtime PM and system * sleep, while DEFINE_RUNTIME_DEV_PM_OPS() uses pm_runtime_force_suspend() * and pm_runtime_force_resume() for its system sleep callbacks. + * + * If the underlying dev_pm_ops struct symbol has to be exported, use + * EXPORT_RUNTIME_DEV_PM_OPS() or EXPORT_GPL_RUNTIME_DEV_PM_OPS() instead. */ #define DEFINE_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ _DEFINE_DEV_PM_OPS(name, pm_runtime_force_suspend, \ pm_runtime_force_resume, suspend_fn, \ resume_fn, idle_fn) +#define EXPORT_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ + _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ + suspend_fn, resume_fn, idle_fn, "") +#define EXPORT_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ + _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ + suspend_fn, resume_fn, idle_fn, "_gpl") + #ifdef CONFIG_PM extern struct workqueue_struct *pm_wq; -- cgit v1.2.3-59-g8ed1b From 8a59bb93b7e3cca389af44781a429ac12ac49be6 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Thu, 9 Dec 2021 14:53:30 -0500 Subject: NFSv4 store server support for fs_location attribute Define and store if server returns it supports fs_locations attribute as a capability. Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 2 ++ include/linux/nfs_fs_sb.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 44ab27f20f04..7d4c63282793 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -3875,6 +3875,8 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL) server->caps |= NFS_CAP_SECURITY_LABEL; #endif + if (res.attr_bitmask[0] & FATTR4_WORD0_FS_LOCATIONS) + server->caps |= NFS_CAP_FS_LOCATIONS; if (!(res.attr_bitmask[0] & FATTR4_WORD0_FILEID)) server->fattr_valid &= ~NFS_ATTR_FATTR_FILEID; if (!(res.attr_bitmask[1] & FATTR4_WORD1_MODE)) diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index f24fc67af42d..8c08b356c8ca 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -289,5 +289,5 @@ struct nfs_server { #define NFS_CAP_COPY_NOTIFY (1U << 27) #define NFS_CAP_XATTR (1U << 28) #define NFS_CAP_READ_PLUS (1U << 29) - +#define NFS_CAP_FS_LOCATIONS (1U << 30) #endif -- cgit v1.2.3-59-g8ed1b From 2d7c86a8f9cdce1408c4f3c69d94d007eff2f179 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 14 Jul 2021 15:35:50 +0530 Subject: libceph: generalize addr/ip parsing based on delimiter ... and remove hardcoded function name in ceph_parse_ips(). [ idryomov: delim parameter, drop CEPH_ADDR_PARSE_DEFAULT_DELIM ] Signed-off-by: Venky Shankar Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- drivers/block/rbd.c | 3 ++- fs/ceph/super.c | 2 +- include/linux/ceph/libceph.h | 2 +- include/linux/ceph/messenger.h | 2 +- net/ceph/ceph_common.c | 9 ++++----- net/ceph/messenger.c | 15 ++++++++------- 6 files changed, 17 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 953fa134cd3d..909dbe6111bf 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -6497,7 +6497,8 @@ static int rbd_add_parse_args(const char *buf, pctx.opts->exclusive = RBD_EXCLUSIVE_DEFAULT; pctx.opts->trim = RBD_TRIM_DEFAULT; - ret = ceph_parse_mon_ips(mon_addrs, mon_addrs_size, pctx.copts, NULL); + ret = ceph_parse_mon_ips(mon_addrs, mon_addrs_size, pctx.copts, NULL, + ','); if (ret) goto out_err; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index bab61232dc5a..c444371ebc38 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -272,7 +272,7 @@ static int ceph_parse_source(struct fs_parameter *param, struct fs_context *fc) dout("server path '%s'\n", fsopt->server_path); ret = ceph_parse_mon_ips(param->string, dev_name_end - dev_name, - pctx->copts, fc->log.log); + pctx->copts, fc->log.log, ','); if (ret) return ret; diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 409d8c29bc4f..c72285d8594e 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -301,7 +301,7 @@ struct fs_parameter; struct fc_log; struct ceph_options *ceph_alloc_options(void); int ceph_parse_mon_ips(const char *buf, size_t len, struct ceph_options *opt, - struct fc_log *l); + struct fc_log *l, char delim); int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, struct fc_log *l); int ceph_print_client_options(struct seq_file *m, struct ceph_client *client, diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 0e6e9ad3c3bf..ff99ce094cfa 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -532,7 +532,7 @@ extern const char *ceph_pr_addr(const struct ceph_entity_addr *addr); extern int ceph_parse_ips(const char *c, const char *end, struct ceph_entity_addr *addr, - int max_count, int *count); + int max_count, int *count, char delim); extern int ceph_msgr_init(void); extern void ceph_msgr_exit(void); diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 97d6ea763e32..851b0c4c5730 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -422,14 +422,14 @@ out: } int ceph_parse_mon_ips(const char *buf, size_t len, struct ceph_options *opt, - struct fc_log *l) + struct fc_log *l, char delim) { struct p_log log = {.prefix = "libceph", .log = l}; int ret; - /* ip1[:port1][,ip2[:port2]...] */ + /* ip1[:port1][ip2[:port2]...] */ ret = ceph_parse_ips(buf, buf + len, opt->mon_addr, CEPH_MAX_MON, - &opt->num_mon); + &opt->num_mon, delim); if (ret) { error_plog(&log, "Failed to parse monitor IPs: %d", ret); return ret; @@ -455,8 +455,7 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, case Opt_ip: err = ceph_parse_ips(param->string, param->string + param->size, - &opt->my_addr, - 1, NULL); + &opt->my_addr, 1, NULL, ','); if (err) { error_plog(&log, "Failed to parse ip: %d", err); return err; diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 57d043b382ed..929ed91f2ec3 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1267,30 +1267,31 @@ static int ceph_parse_server_name(const char *name, size_t namelen, */ int ceph_parse_ips(const char *c, const char *end, struct ceph_entity_addr *addr, - int max_count, int *count) + int max_count, int *count, char delim) { int i, ret = -EINVAL; const char *p = c; dout("parse_ips on '%.*s'\n", (int)(end-c), c); for (i = 0; i < max_count; i++) { + char cur_delim = delim; const char *ipend; int port; - char delim = ','; if (*p == '[') { - delim = ']'; + cur_delim = ']'; p++; } - ret = ceph_parse_server_name(p, end - p, &addr[i], delim, &ipend); + ret = ceph_parse_server_name(p, end - p, &addr[i], cur_delim, + &ipend); if (ret) goto bad; ret = -EINVAL; p = ipend; - if (delim == ']') { + if (cur_delim == ']') { if (*p != ']') { dout("missing matching ']'\n"); goto bad; @@ -1326,11 +1327,11 @@ int ceph_parse_ips(const char *c, const char *end, addr[i].type = CEPH_ENTITY_ADDR_TYPE_LEGACY; addr[i].nonce = 0; - dout("parse_ips got %s\n", ceph_pr_addr(&addr[i])); + dout("%s got %s\n", __func__, ceph_pr_addr(&addr[i])); if (p == end) break; - if (*p != ',') + if (*p != delim) goto bad; p++; } -- cgit v1.2.3-59-g8ed1b From 4153c7fc937a2afa077dbdb9fe3189b9981f423c Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 14 Jul 2021 15:35:51 +0530 Subject: libceph: rename parse_fsid() to ceph_parse_fsid() and export ... as it is too generic. also, use __func__ when logging rather than hardcoding the function name. Signed-off-by: Venky Shankar Reviewed-by: Jeff Layton Signed-off-by: Ilya Dryomov --- include/linux/ceph/libceph.h | 1 + net/ceph/ceph_common.c | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index c72285d8594e..644f224eccf7 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -296,6 +296,7 @@ extern bool libceph_compatible(void *data); extern const char *ceph_msg_type_name(int type); extern int ceph_check_fsid(struct ceph_client *client, struct ceph_fsid *fsid); extern void *ceph_kvmalloc(size_t size, gfp_t flags); +extern int ceph_parse_fsid(const char *str, struct ceph_fsid *fsid); struct fs_parameter; struct fc_log; diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 851b0c4c5730..decae43b4262 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -217,14 +217,14 @@ void *ceph_kvmalloc(size_t size, gfp_t flags) return p; } -static int parse_fsid(const char *str, struct ceph_fsid *fsid) +int ceph_parse_fsid(const char *str, struct ceph_fsid *fsid) { int i = 0; char tmp[3]; int err = -EINVAL; int d; - dout("parse_fsid '%s'\n", str); + dout("%s '%s'\n", __func__, str); tmp[2] = 0; while (*str && i < 16) { if (ispunct(*str)) { @@ -244,9 +244,10 @@ static int parse_fsid(const char *str, struct ceph_fsid *fsid) if (i == 16) err = 0; - dout("parse_fsid ret %d got fsid %pU\n", err, fsid); + dout("%s ret %d got fsid %pU\n", __func__, err, fsid); return err; } +EXPORT_SYMBOL(ceph_parse_fsid); /* * ceph options @@ -464,7 +465,7 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, break; case Opt_fsid: - err = parse_fsid(param->string, &opt->fsid); + err = ceph_parse_fsid(param->string, &opt->fsid); if (err) { error_plog(&log, "Failed to parse fsid: %d", err); return err; -- cgit v1.2.3-59-g8ed1b From 1976b2b31462151403c9fc110204fcc2a77bdfd1 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 12 Jan 2022 10:27:38 -0500 Subject: NFSv4.1 query for fs_location attr on a new file system Query the server for other possible trunkable locations for a given file system on a 4.1+ mount. v2: -- added missing static to nfs4_discover_trunking, reported by the kernel test robot Signed-off-by: Olga Kornievskaia Signed-off-by: Anna Schumaker --- fs/nfs/client.c | 7 +++++ fs/nfs/nfs4_fs.h | 9 +++--- fs/nfs/nfs4proc.c | 76 ++++++++++++++++++++++++++++++++++++++++++------- fs/nfs/nfs4state.c | 3 +- include/linux/nfs_xdr.h | 1 + 5 files changed, 81 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 1e4dc1ab9312..f7e39cc4472b 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -860,6 +860,13 @@ static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, str server->namelen = pathinfo.max_namelen; } + if (clp->rpc_ops->discover_trunking != NULL && + (server->caps & NFS_CAP_FS_LOCATIONS)) { + error = clp->rpc_ops->discover_trunking(server, mntfh); + if (error < 0) + return error; + } + return 0; } diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index 89076dd32334..166ea6112e61 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -260,8 +260,8 @@ struct nfs4_state_maintenance_ops { }; struct nfs4_mig_recovery_ops { - int (*get_locations)(struct inode *, struct nfs4_fs_locations *, - struct page *, const struct cred *); + int (*get_locations)(struct nfs_server *, struct nfs_fh *, + struct nfs4_fs_locations *, struct page *, const struct cred *); int (*fsid_present)(struct inode *, const struct cred *); }; @@ -302,8 +302,9 @@ extern int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait); extern int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle); extern int nfs4_proc_fs_locations(struct rpc_clnt *, struct inode *, const struct qstr *, struct nfs4_fs_locations *, struct page *); -extern int nfs4_proc_get_locations(struct inode *, struct nfs4_fs_locations *, - struct page *page, const struct cred *); +extern int nfs4_proc_get_locations(struct nfs_server *, struct nfs_fh *, + struct nfs4_fs_locations *, + struct page *page, const struct cred *); extern int nfs4_proc_fsid_present(struct inode *, const struct cred *); extern struct rpc_clnt *nfs4_proc_lookup_mountpoint(struct inode *, struct dentry *, diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 7d4c63282793..fc4629d2d2ab 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -3935,6 +3935,60 @@ int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) return err; } +static int _nfs4_discover_trunking(struct nfs_server *server, + struct nfs_fh *fhandle) +{ + struct nfs4_fs_locations *locations = NULL; + struct page *page; + const struct cred *cred; + struct nfs_client *clp = server->nfs_client; + const struct nfs4_state_maintenance_ops *ops = + clp->cl_mvops->state_renewal_ops; + int status = -ENOMEM; + + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) { + cred = nfs4_get_clid_cred(clp); + if (cred == NULL) + return -ENOKEY; + } + + page = alloc_page(GFP_KERNEL); + locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL); + if (page == NULL || locations == NULL) + goto out; + + status = nfs4_proc_get_locations(server, fhandle, locations, page, + cred); + if (status) + goto out; +out: + if (page) + __free_page(page); + kfree(locations); + return status; +} + +static int nfs4_discover_trunking(struct nfs_server *server, + struct nfs_fh *fhandle) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs_client *clp = server->nfs_client; + int err = 0; + + if (!nfs4_has_session(clp)) + goto out; + do { + err = nfs4_handle_exception(server, + _nfs4_discover_trunking(server, fhandle), + &exception); + } while (exception.retry); +out: + return err; +} + static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsinfo *info) { @@ -7823,18 +7877,18 @@ int nfs4_proc_fs_locations(struct rpc_clnt *client, struct inode *dir, * appended to this compound to identify the client ID which is * performing recovery. */ -static int _nfs40_proc_get_locations(struct inode *inode, +static int _nfs40_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs4_fs_locations *locations, struct page *page, const struct cred *cred) { - struct nfs_server *server = NFS_SERVER(inode); struct rpc_clnt *clnt = server->client; u32 bitmask[2] = { [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS, }; struct nfs4_fs_locations_arg args = { .clientid = server->nfs_client->cl_clientid, - .fh = NFS_FH(inode), + .fh = fhandle, .page = page, .bitmask = bitmask, .migration = 1, /* skip LOOKUP */ @@ -7880,17 +7934,17 @@ static int _nfs40_proc_get_locations(struct inode *inode, * When the client supports GETATTR(fs_locations_info), it can * be plumbed in here. */ -static int _nfs41_proc_get_locations(struct inode *inode, +static int _nfs41_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs4_fs_locations *locations, struct page *page, const struct cred *cred) { - struct nfs_server *server = NFS_SERVER(inode); struct rpc_clnt *clnt = server->client; u32 bitmask[2] = { [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS, }; struct nfs4_fs_locations_arg args = { - .fh = NFS_FH(inode), + .fh = fhandle, .page = page, .bitmask = bitmask, .migration = 1, /* skip LOOKUP */ @@ -7939,11 +7993,11 @@ static int _nfs41_proc_get_locations(struct inode *inode, * -NFS4ERR_LEASE_MOVED is returned if the server still has leases * from this client that require migration recovery. */ -int nfs4_proc_get_locations(struct inode *inode, +int nfs4_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs4_fs_locations *locations, struct page *page, const struct cred *cred) { - struct nfs_server *server = NFS_SERVER(inode); struct nfs_client *clp = server->nfs_client; const struct nfs4_mig_recovery_ops *ops = clp->cl_mvops->mig_recovery_ops; @@ -7956,10 +8010,11 @@ int nfs4_proc_get_locations(struct inode *inode, (unsigned long long)server->fsid.major, (unsigned long long)server->fsid.minor, clp->cl_hostname); - nfs_display_fhandle(NFS_FH(inode), __func__); + nfs_display_fhandle(fhandle, __func__); do { - status = ops->get_locations(inode, locations, page, cred); + status = ops->get_locations(server, fhandle, locations, page, + cred); if (status != -NFS4ERR_DELAY) break; nfs4_handle_exception(server, status, &exception); @@ -10428,6 +10483,7 @@ const struct nfs_rpc_ops nfs_v4_clientops = { .free_client = nfs4_free_client, .create_server = nfs4_create_server, .clone_server = nfs_clone_server, + .discover_trunking = nfs4_discover_trunking, }; static const struct xattr_handler nfs4_xattr_nfs4_acl_handler = { diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index f3265575c28d..499bef9fe118 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2098,7 +2098,8 @@ static int nfs4_try_migration(struct nfs_server *server, const struct cred *cred } inode = d_inode(server->super->s_root); - result = nfs4_proc_get_locations(inode, locations, page, cred); + result = nfs4_proc_get_locations(server, NFS_FH(inode), locations, + page, cred); if (result) { dprintk("<-- %s: failed to retrieve fs_locations: %d\n", __func__, result); diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index ddff92396a3f..728cb0c1f0b6 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1797,6 +1797,7 @@ struct nfs_rpc_ops { struct nfs_server *(*create_server)(struct fs_context *); struct nfs_server *(*clone_server)(struct nfs_server *, struct nfs_fh *, struct nfs_fattr *, rpc_authflavor_t); + int (*discover_trunking)(struct nfs_server *, struct nfs_fh *); }; /* -- cgit v1.2.3-59-g8ed1b From 180dccb0dba4f5e84a4a70c1be1d34cbb6528b32 Mon Sep 17 00:00:00 2001 From: Laibin Qiu Date: Thu, 13 Jan 2022 10:55:36 +0800 Subject: blk-mq: fix tag_get wait task can't be awakened In case of shared tags, there might be more than one hctx which allocates from the same tags, and each hctx is limited to allocate at most: hctx_max_depth = max((bt->sb.depth + users - 1) / users, 4U); tag idle detection is lazy, and may be delayed for 30sec, so there could be just one real active hctx(queue) but all others are actually idle and still accounted as active because of the lazy idle detection. Then if wake_batch is > hctx_max_depth, driver tag allocation may wait forever on this real active hctx. Fix this by recalculating wake_batch when inc or dec active_queues. Fixes: 0d2602ca30e41 ("blk-mq: improve support for shared tags maps") Suggested-by: Ming Lei Suggested-by: John Garry Signed-off-by: Laibin Qiu Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220113025536.1479653-1-qiulaibin@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 40 +++++++++++++++++++++++++++++++++------- include/linux/sbitmap.h | 11 +++++++++++ lib/sbitmap.c | 25 ++++++++++++++++++++++--- 3 files changed, 66 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index e55a6834c9a6..845f74e8dd7b 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -16,6 +16,21 @@ #include "blk-mq-sched.h" #include "blk-mq-tag.h" +/* + * Recalculate wakeup batch when tag is shared by hctx. + */ +static void blk_mq_update_wake_batch(struct blk_mq_tags *tags, + unsigned int users) +{ + if (!users) + return; + + sbitmap_queue_recalculate_wake_batch(&tags->bitmap_tags, + users); + sbitmap_queue_recalculate_wake_batch(&tags->breserved_tags, + users); +} + /* * If a previously inactive queue goes active, bump the active user count. * We need to do this before try to allocate driver tag, then even if fail @@ -24,18 +39,26 @@ */ bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) { + unsigned int users; + if (blk_mq_is_shared_tags(hctx->flags)) { struct request_queue *q = hctx->queue; - if (!test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) && - !test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) - atomic_inc(&hctx->tags->active_queues); + if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) || + test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags)) { + return true; + } } else { - if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) && - !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) - atomic_inc(&hctx->tags->active_queues); + if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) || + test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) { + return true; + } } + users = atomic_inc_return(&hctx->tags->active_queues); + + blk_mq_update_wake_batch(hctx->tags, users); + return true; } @@ -56,6 +79,7 @@ void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) { struct blk_mq_tags *tags = hctx->tags; + unsigned int users; if (blk_mq_is_shared_tags(hctx->flags)) { struct request_queue *q = hctx->queue; @@ -68,7 +92,9 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) return; } - atomic_dec(&tags->active_queues); + users = atomic_dec_return(&tags->active_queues); + + blk_mq_update_wake_batch(tags, users); blk_mq_tag_wakeup_all(tags, false); } diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index fc0357a6e19b..95df357ec009 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -415,6 +415,17 @@ static inline void sbitmap_queue_free(struct sbitmap_queue *sbq) sbitmap_free(&sbq->sb); } +/** + * sbitmap_queue_recalculate_wake_batch() - Recalculate wake batch + * @sbq: Bitmap queue to recalculate wake batch. + * @users: Number of shares. + * + * Like sbitmap_queue_update_wake_batch(), this will calculate wake batch + * by depth. This interface is for HCTX shared tags or queue shared tags. + */ +void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq, + unsigned int users); + /** * sbitmap_queue_resize() - Resize a &struct sbitmap_queue. * @sbq: Bitmap queue to resize. diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 2709ab825499..6220fa67fb7e 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -457,10 +457,9 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, } EXPORT_SYMBOL_GPL(sbitmap_queue_init_node); -static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, - unsigned int depth) +static inline void __sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, + unsigned int wake_batch) { - unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth); int i; if (sbq->wake_batch != wake_batch) { @@ -476,6 +475,26 @@ static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, } } +static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, + unsigned int depth) +{ + unsigned int wake_batch; + + wake_batch = sbq_calc_wake_batch(sbq, depth); + __sbitmap_queue_update_wake_batch(sbq, wake_batch); +} + +void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq, + unsigned int users) +{ + unsigned int wake_batch; + + wake_batch = clamp_val((sbq->sb.depth + users - 1) / + users, 4, SBQ_WAKE_BATCH); + __sbitmap_queue_update_wake_batch(sbq, wake_batch); +} +EXPORT_SYMBOL_GPL(sbitmap_queue_recalculate_wake_batch); + void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) { sbitmap_queue_update_wake_batch(sbq, depth); -- cgit v1.2.3-59-g8ed1b From 289e7b0f7eb47b87a0441e6c81336316f301eb39 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 13 Dec 2021 11:08:53 +0100 Subject: tracing: Account bottom half disabled sections. Disabling only bottom halves via local_bh_disable() disables also preemption but this remains invisible to tracing. On a CONFIG_PREEMPT kernel one might wonder why there is no scheduling happening despite the N flag in the trace. The reason might be the a rcu_read_lock_bh() section. Add a 'b' to the tracing output if in task context with disabled bottom halves. Link: https://lkml.kernel.org/r/YbcbtdtC/bjCKo57@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Steven Rostedt --- include/linux/trace_events.h | 1 + kernel/trace/trace.c | 6 ++++-- kernel/trace/trace_output.c | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 3900404aa063..70c069aef02c 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -172,6 +172,7 @@ enum trace_flag_type { TRACE_FLAG_SOFTIRQ = 0x10, TRACE_FLAG_PREEMPT_RESCHED = 0x20, TRACE_FLAG_NMI = 0x40, + TRACE_FLAG_BH_OFF = 0x80, }; #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 547d82628c2e..a73d78dcda2c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2603,6 +2603,8 @@ unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status) trace_flags |= TRACE_FLAG_HARDIRQ; if (in_serving_softirq()) trace_flags |= TRACE_FLAG_SOFTIRQ; + if (softirq_count() >> (SOFTIRQ_SHIFT + 1)) + trace_flags |= TRACE_FLAG_BH_OFF; if (tif_need_resched()) trace_flags |= TRACE_FLAG_NEED_RESCHED; @@ -4190,7 +4192,7 @@ unsigned long trace_total_entries(struct trace_array *tr) static void print_lat_help_header(struct seq_file *m) { seq_puts(m, "# _------=> CPU# \n" - "# / _-----=> irqs-off \n" + "# / _-----=> irqs-off/BH-disabled\n" "# | / _----=> need-resched \n" "# || / _---=> hardirq/softirq \n" "# ||| / _--=> preempt-depth \n" @@ -4231,7 +4233,7 @@ static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file print_event_info(buf, m); - seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space); + seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space); seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 3547e7176ff7..8aa493d25c73 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -445,14 +445,18 @@ int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry) char irqs_off; int hardirq; int softirq; + int bh_off; int nmi; nmi = entry->flags & TRACE_FLAG_NMI; hardirq = entry->flags & TRACE_FLAG_HARDIRQ; softirq = entry->flags & TRACE_FLAG_SOFTIRQ; + bh_off = entry->flags & TRACE_FLAG_BH_OFF; irqs_off = + (entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' : (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : + bh_off ? 'b' : (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.'; -- cgit v1.2.3-59-g8ed1b From 6840f9094f2bd788a316d8cb0a4e42538d3e47dd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 13 Jan 2022 16:44:19 -0500 Subject: pagevec: Initialise folio_batch->percpu_pvec_drained When UBSAN is enabled, it reports an invalid value in __pagevec_release() when accessing pvec->percpu_pvec_drained, which is simply whatever garbage was on the stack. Initialise it when initialising the rest of the folio_batch. Fixes: 10331795fb79 ("pagevec: Add folio_batch") Reported-by: Randy Dunlap Tested-by: Randy Dunlap Signed-off-by: Matthew Wilcox (Oracle) --- include/linux/pagevec.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index dda8d5868c81..67b1246f136b 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -111,6 +111,7 @@ static_assert(offsetof(struct pagevec, pages) == static inline void folio_batch_init(struct folio_batch *fbatch) { fbatch->nr = 0; + fbatch->percpu_pvec_drained = false; } static inline unsigned int folio_batch_count(struct folio_batch *fbatch) -- cgit v1.2.3-59-g8ed1b From e5b48ee30aec1fe6dff05e36b22e886c665b4736 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Tue, 4 Jan 2022 16:14:46 +0900 Subject: ata: sata_fsl: fix scsi host initialization When compiling with W=1, the sata_fsl driver compilation throws the warning: drivers/ata/sata_fsl.c:1385:22: error: initialized field overwritten [-Werror=override-init] 1385 | .can_queue = SATA_FSL_QUEUE_DEPTH, This is due to the driver scsi host template initialization overwriting the can_queue field that is already set using the ATA_NCQ_SHT() initializer macro, resulting in the same field being initialized twice in the host template declaration. To remove this warning, introduce the ATA_SUBBASE_SHT_QD() and ATA_NCQ_SHT_QD() initialization macros to allow specifying a queue depth different from the default ATA_DEF_QUEUE using an additional argument to the macro. Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke --- drivers/ata/sata_fsl.c | 3 +-- include/linux/libata.h | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c index 142e65d5efc7..101d4dd79f62 100644 --- a/drivers/ata/sata_fsl.c +++ b/drivers/ata/sata_fsl.c @@ -1380,8 +1380,7 @@ static void sata_fsl_host_stop(struct ata_host *host) * scsi mid-layer and libata interface structures */ static struct scsi_host_template sata_fsl_sht = { - ATA_NCQ_SHT("sata_fsl"), - .can_queue = SATA_FSL_QUEUE_DEPTH, + ATA_NCQ_SHT_QD("sata_fsl", SATA_FSL_QUEUE_DEPTH), .sg_tablesize = SATA_FSL_MAX_PRD_USABLE, .dma_boundary = ATA_DMA_BOUNDARY, }; diff --git a/include/linux/libata.h b/include/linux/libata.h index c258f69106f4..2e5e7c40c991 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -1385,6 +1385,12 @@ extern const struct attribute_group *ata_common_sdev_groups[]; .tag_alloc_policy = BLK_TAG_ALLOC_RR, \ .slave_configure = ata_scsi_slave_config +#define ATA_SUBBASE_SHT_QD(drv_name, drv_qd) \ + __ATA_BASE_SHT(drv_name), \ + .can_queue = drv_qd, \ + .tag_alloc_policy = BLK_TAG_ALLOC_RR, \ + .slave_configure = ata_scsi_slave_config + #define ATA_BASE_SHT(drv_name) \ ATA_SUBBASE_SHT(drv_name), \ .sdev_groups = ata_common_sdev_groups @@ -1396,6 +1402,11 @@ extern const struct attribute_group *ata_ncq_sdev_groups[]; ATA_SUBBASE_SHT(drv_name), \ .sdev_groups = ata_ncq_sdev_groups, \ .change_queue_depth = ata_scsi_change_queue_depth + +#define ATA_NCQ_SHT_QD(drv_name, drv_qd) \ + ATA_SUBBASE_SHT_QD(drv_name, drv_qd), \ + .sdev_groups = ata_ncq_sdev_groups, \ + .change_queue_depth = ata_scsi_change_queue_depth #endif /* -- cgit v1.2.3-59-g8ed1b From 0561e514c944da874ccdfbe2922f71b4c333c7e1 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Tue, 4 Jan 2022 17:54:18 +0900 Subject: ata: fix read_id() ata port operation interface Drivers that need to tweak a device IDENTIFY data implement the read_id() port operation. The IDENTIFY data buffer is passed as an argument to the read_id() operation for drivers to use. However, when this operation is called, the IDENTIFY data is not yet converted to CPU endian and contains le16 words. Change the interface of the read_id operation to pass a __le16 * pointer to the IDENTIFY data buffer to clarify the buffer endianness. Fix the pata_netcell, pata_it821x, ahci_xgene, ahci_ceva and ahci_brcm drivers implementation of this operation and modify the code to corretly deal with identify data words manipulation to avoid sparse warnings such as: drivers/ata/ahci_xgene.c:262:33: warning: invalid assignment: &= drivers/ata/ahci_xgene.c:262:33: left side has type unsigned short drivers/ata/ahci_xgene.c:262:33: right side has type restricted __le16 Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke --- drivers/ata/ahci_brcm.c | 2 +- drivers/ata/ahci_ceva.c | 5 ++--- drivers/ata/ahci_xgene.c | 2 +- drivers/ata/libata-core.c | 6 +++--- drivers/ata/pata_it821x.c | 23 +++++++++++------------ drivers/ata/pata_netcell.c | 5 +++-- include/linux/libata.h | 5 +++-- 7 files changed, 24 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c index 6e9c5ade4c2e..ba695338927a 100644 --- a/drivers/ata/ahci_brcm.c +++ b/drivers/ata/ahci_brcm.c @@ -246,7 +246,7 @@ static void brcm_sata_init(struct brcm_ahci_priv *priv) } static unsigned int brcm_ahci_read_id(struct ata_device *dev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { struct ata_port *ap = dev->link->ap; struct ata_host *host = ap->host; diff --git a/drivers/ata/ahci_ceva.c b/drivers/ata/ahci_ceva.c index e9c7c07fd84c..acf59f51b356 100644 --- a/drivers/ata/ahci_ceva.c +++ b/drivers/ata/ahci_ceva.c @@ -92,9 +92,8 @@ struct ceva_ahci_priv { }; static unsigned int ceva_ahci_read_id(struct ata_device *dev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { - __le16 *__id = (__le16 *)id; u32 err_mask; err_mask = ata_do_dev_read_id(dev, tf, id); @@ -104,7 +103,7 @@ static unsigned int ceva_ahci_read_id(struct ata_device *dev, * Since CEVA controller does not support device sleep feature, we * need to clear DEVSLP (bit 8) in word78 of the IDENTIFY DEVICE data. */ - __id[ATA_ID_FEATURE_SUPP] &= cpu_to_le16(~(1 << 8)); + id[ATA_ID_FEATURE_SUPP] &= cpu_to_le16(~(1 << 8)); return 0; } diff --git a/drivers/ata/ahci_xgene.c b/drivers/ata/ahci_xgene.c index 68ec7e9430b2..8e206379d699 100644 --- a/drivers/ata/ahci_xgene.c +++ b/drivers/ata/ahci_xgene.c @@ -237,7 +237,7 @@ static bool xgene_ahci_is_memram_inited(struct xgene_ahci_context *ctx) * does not support DEVSLP. */ static unsigned int xgene_ahci_read_id(struct ata_device *dev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { u32 err_mask; diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 9c2947905d1e..67f88027680a 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -1722,7 +1722,7 @@ static u32 ata_pio_mask_no_iordy(const struct ata_device *adev) * this function is wrapped or replaced by the driver */ unsigned int ata_do_dev_read_id(struct ata_device *dev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { return ata_exec_internal(dev, tf, NULL, DMA_FROM_DEVICE, id, sizeof(id[0]) * ATA_ID_WORDS, 0); @@ -1795,9 +1795,9 @@ retry: tf.flags |= ATA_TFLAG_POLLING; if (ap->ops->read_id) - err_mask = ap->ops->read_id(dev, &tf, id); + err_mask = ap->ops->read_id(dev, &tf, (__le16 *)id); else - err_mask = ata_do_dev_read_id(dev, &tf, id); + err_mask = ata_do_dev_read_id(dev, &tf, (__le16 *)id); if (err_mask) { if (err_mask & AC_ERR_NODEV_HINT) { diff --git a/drivers/ata/pata_it821x.c b/drivers/ata/pata_it821x.c index b77ef0046dbe..8a5b4e0079ab 100644 --- a/drivers/ata/pata_it821x.c +++ b/drivers/ata/pata_it821x.c @@ -537,7 +537,7 @@ static void it821x_dev_config(struct ata_device *adev) */ static unsigned int it821x_read_id(struct ata_device *adev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { unsigned int err_mask; unsigned char model_num[ATA_ID_PROD_LEN + 1]; @@ -545,21 +545,20 @@ static unsigned int it821x_read_id(struct ata_device *adev, err_mask = ata_do_dev_read_id(adev, tf, id); if (err_mask) return err_mask; - ata_id_c_string(id, model_num, ATA_ID_PROD, sizeof(model_num)); + ata_id_c_string((u16 *)id, model_num, ATA_ID_PROD, sizeof(model_num)); - id[83] &= ~(1 << 12); /* Cache flush is firmware handled */ - id[83] &= ~(1 << 13); /* Ditto for LBA48 flushes */ - id[84] &= ~(1 << 6); /* No FUA */ - id[85] &= ~(1 << 10); /* No HPA */ - id[76] = 0; /* No NCQ/AN etc */ + id[83] &= cpu_to_le16(~(1 << 12)); /* Cache flush is firmware handled */ + id[84] &= cpu_to_le16(~(1 << 6)); /* No FUA */ + id[85] &= cpu_to_le16(~(1 << 10)); /* No HPA */ + id[76] = 0; /* No NCQ/AN etc */ if (strstr(model_num, "Integrated Technology Express")) { /* Set feature bits the firmware neglects */ - id[49] |= 0x0300; /* LBA, DMA */ - id[83] &= 0x7FFF; - id[83] |= 0x4400; /* Word 83 is valid and LBA48 */ - id[86] |= 0x0400; /* LBA48 on */ - id[ATA_ID_MAJOR_VER] |= 0x1F; + id[49] |= cpu_to_le16(0x0300); /* LBA, DMA */ + id[83] &= cpu_to_le16(0x7FFF); + id[83] |= cpu_to_le16(0x4400); /* Word 83 is valid and LBA48 */ + id[86] |= cpu_to_le16(0x0400); /* LBA48 on */ + id[ATA_ID_MAJOR_VER] |= cpu_to_le16(0x1F); /* Clear the serial number because it's different each boot which breaks validation on resume */ memset(&id[ATA_ID_SERNO], 0x20, ATA_ID_SERNO_LEN); diff --git a/drivers/ata/pata_netcell.c b/drivers/ata/pata_netcell.c index a7ecc1a204b5..06929e77c491 100644 --- a/drivers/ata/pata_netcell.c +++ b/drivers/ata/pata_netcell.c @@ -21,12 +21,13 @@ /* No PIO or DMA methods needed for this device */ static unsigned int netcell_read_id(struct ata_device *adev, - struct ata_taskfile *tf, u16 *id) + struct ata_taskfile *tf, __le16 *id) { unsigned int err_mask = ata_do_dev_read_id(adev, tf, id); + /* Firmware forgets to mark words 85-87 valid */ if (err_mask == 0) - id[ATA_ID_CSF_DEFAULT] |= 0x4000; + id[ATA_ID_CSF_DEFAULT] |= cpu_to_le16(0x4000); return err_mask; } diff --git a/include/linux/libata.h b/include/linux/libata.h index 2e5e7c40c991..bf706cd45674 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -884,7 +884,8 @@ struct ata_port_operations { void (*set_piomode)(struct ata_port *ap, struct ata_device *dev); void (*set_dmamode)(struct ata_port *ap, struct ata_device *dev); int (*set_mode)(struct ata_link *link, struct ata_device **r_failed_dev); - unsigned int (*read_id)(struct ata_device *dev, struct ata_taskfile *tf, u16 *id); + unsigned int (*read_id)(struct ata_device *dev, struct ata_taskfile *tf, + __le16 *id); void (*dev_config)(struct ata_device *dev); @@ -1119,7 +1120,7 @@ extern void ata_id_string(const u16 *id, unsigned char *s, extern void ata_id_c_string(const u16 *id, unsigned char *s, unsigned int ofs, unsigned int len); extern unsigned int ata_do_dev_read_id(struct ata_device *dev, - struct ata_taskfile *tf, u16 *id); + struct ata_taskfile *tf, __le16 *id); extern void ata_qc_complete(struct ata_queued_cmd *qc); extern u64 ata_qc_get_active(struct ata_port *ap); extern void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd); -- cgit v1.2.3-59-g8ed1b From b9ba367c513dbc165dd6c01266a59db4be2a3564 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Wed, 5 Jan 2022 16:36:16 +0100 Subject: ata: libata: Rename link flag ATA_LFLAG_NO_DB_DELAY Rename the link flag ATA_LFLAG_NO_DB_DELAY to ATA_LFLAG_NO_DEBOUNCE_DELAY. The new name is longer, but clearer. Signed-off-by: Paul Menzel Signed-off-by: Damien Le Moal --- drivers/ata/ahci_brcm.c | 2 +- drivers/ata/libata-sata.c | 2 +- include/linux/libata.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c index ba695338927a..64dd8aa397d5 100644 --- a/drivers/ata/ahci_brcm.c +++ b/drivers/ata/ahci_brcm.c @@ -333,7 +333,7 @@ static struct ata_port_operations ahci_brcm_platform_ops = { static const struct ata_port_info ahci_brcm_port_info = { .flags = AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM, - .link_flags = ATA_LFLAG_NO_DB_DELAY, + .link_flags = ATA_LFLAG_NO_DEBOUNCE_DELAY, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, .port_ops = &ahci_brcm_platform_ops, diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index bfe9595d4f33..071158c0c44c 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -317,7 +317,7 @@ int sata_link_resume(struct ata_link *link, const unsigned long *params, * immediately after resuming. Delay 200ms before * debouncing. */ - if (!(link->flags & ATA_LFLAG_NO_DB_DELAY)) + if (!(link->flags & ATA_LFLAG_NO_DEBOUNCE_DELAY)) ata_msleep(link->ap, 200); /* is SControl restored correctly? */ diff --git a/include/linux/libata.h b/include/linux/libata.h index bf706cd45674..605756f645be 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -143,7 +143,7 @@ enum { ATA_LFLAG_NO_LPM = (1 << 8), /* disable LPM on this link */ ATA_LFLAG_RST_ONCE = (1 << 9), /* limit recovery to one reset */ ATA_LFLAG_CHANGED = (1 << 10), /* LPM state changed on this link */ - ATA_LFLAG_NO_DB_DELAY = (1 << 11), /* no debounce delay on link resume */ + ATA_LFLAG_NO_DEBOUNCE_DELAY = (1 << 11), /* no debounce delay on link resume */ /* struct ata_port flags */ ATA_FLAG_SLAVE_POSS = (1 << 0), /* host supports slave dev */ -- cgit v1.2.3-59-g8ed1b From d9679d0013a66849f23057978f92e76b255c50aa Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Wed, 13 Oct 2021 06:55:44 -0400 Subject: virtio: wrap config->reset calls This will enable cleanups down the road. The idea is to disable cbs, then add "flush_queued_cbs" callback as a parameter, this way drivers can flush any work queued after callbacks have been disabled. Signed-off-by: Michael S. Tsirkin Link: https://lore.kernel.org/r/20211013105226.20225-1-mst@redhat.com Signed-off-by: Michael S. Tsirkin --- arch/um/drivers/virt-pci.c | 2 +- drivers/block/virtio_blk.c | 4 ++-- drivers/bluetooth/virtio_bt.c | 2 +- drivers/char/hw_random/virtio-rng.c | 2 +- drivers/char/virtio_console.c | 4 ++-- drivers/crypto/virtio/virtio_crypto_core.c | 8 ++++---- drivers/firmware/arm_scmi/virtio.c | 2 +- drivers/gpio/gpio-virtio.c | 2 +- drivers/gpu/drm/virtio/virtgpu_kms.c | 2 +- drivers/i2c/busses/i2c-virtio.c | 2 +- drivers/iommu/virtio-iommu.c | 2 +- drivers/net/caif/caif_virtio.c | 2 +- drivers/net/virtio_net.c | 4 ++-- drivers/net/wireless/mac80211_hwsim.c | 2 +- drivers/nvdimm/virtio_pmem.c | 2 +- drivers/rpmsg/virtio_rpmsg_bus.c | 2 +- drivers/scsi/virtio_scsi.c | 2 +- drivers/virtio/virtio.c | 6 ++++++ drivers/virtio/virtio_balloon.c | 2 +- drivers/virtio/virtio_input.c | 2 +- drivers/virtio/virtio_mem.c | 2 +- fs/fuse/virtio_fs.c | 4 ++-- include/linux/virtio.h | 1 + net/9p/trans_virtio.c | 2 +- net/vmw_vsock/virtio_transport.c | 4 ++-- sound/virtio/virtio_card.c | 4 ++-- 26 files changed, 40 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c index c08066633023..22c4d87c9c15 100644 --- a/arch/um/drivers/virt-pci.c +++ b/arch/um/drivers/virt-pci.c @@ -616,7 +616,7 @@ static void um_pci_virtio_remove(struct virtio_device *vdev) int i; /* Stop all virtqueues */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); device_set_wakeup_enable(&vdev->dev, false); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 6ae38776e30e..644c6cddad6d 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -977,7 +977,7 @@ static void virtblk_remove(struct virtio_device *vdev) mutex_lock(&vblk->vdev_mutex); /* Stop all the virtqueues. */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */ vblk->vdev = NULL; @@ -996,7 +996,7 @@ static int virtblk_freeze(struct virtio_device *vdev) struct virtio_blk *vblk = vdev->priv; /* Ensure we don't receive any more interrupts */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); /* Make sure no work handler is accessing the device. */ flush_work(&vblk->config_work); diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 57908ce4fae8..24a9258962fa 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -364,7 +364,7 @@ static void virtbt_remove(struct virtio_device *vdev) struct hci_dev *hdev = vbt->hdev; hci_unregister_dev(hdev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); hci_free_dev(hdev); vbt->hdev = NULL; diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 0a7dde135db1..b2bf78b25630 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -179,7 +179,7 @@ static void remove_common(struct virtio_device *vdev) vi->data_avail = 0; vi->data_idx = 0; complete(&vi->have_data); - vdev->config->reset(vdev); + virtio_reset_device(vdev); if (vi->hwrng_register_done) hwrng_unregister(&vi->hwrng); vdev->config->del_vqs(vdev); diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index 660c5c388c29..2359889a35a0 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1958,7 +1958,7 @@ static void virtcons_remove(struct virtio_device *vdev) spin_unlock_irq(&pdrvdata_lock); /* Disable interrupts for vqs */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); /* Finish up work that's lined up */ if (use_multiport(portdev)) cancel_work_sync(&portdev->control_work); @@ -2148,7 +2148,7 @@ static int virtcons_freeze(struct virtio_device *vdev) portdev = vdev->priv; - vdev->config->reset(vdev); + virtio_reset_device(vdev); if (use_multiport(portdev)) virtqueue_disable_cb(portdev->c_ivq); diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index e2375d992308..8e977b7627cb 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -404,7 +404,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) free_engines: virtcrypto_clear_crypto_engines(vcrypto); free_vqs: - vcrypto->vdev->config->reset(vdev); + virtio_reset_device(vdev); virtcrypto_del_vqs(vcrypto); free_dev: virtcrypto_devmgr_rm_dev(vcrypto); @@ -436,7 +436,7 @@ static void virtcrypto_remove(struct virtio_device *vdev) if (virtcrypto_dev_started(vcrypto)) virtcrypto_dev_stop(vcrypto); - vdev->config->reset(vdev); + virtio_reset_device(vdev); virtcrypto_free_unused_reqs(vcrypto); virtcrypto_clear_crypto_engines(vcrypto); virtcrypto_del_vqs(vcrypto); @@ -456,7 +456,7 @@ static int virtcrypto_freeze(struct virtio_device *vdev) { struct virtio_crypto *vcrypto = vdev->priv; - vdev->config->reset(vdev); + virtio_reset_device(vdev); virtcrypto_free_unused_reqs(vcrypto); if (virtcrypto_dev_started(vcrypto)) virtcrypto_dev_stop(vcrypto); @@ -492,7 +492,7 @@ static int virtcrypto_restore(struct virtio_device *vdev) free_engines: virtcrypto_clear_crypto_engines(vcrypto); free_vqs: - vcrypto->vdev->config->reset(vdev); + virtio_reset_device(vdev); virtcrypto_del_vqs(vcrypto); return err; } diff --git a/drivers/firmware/arm_scmi/virtio.c b/drivers/firmware/arm_scmi/virtio.c index 87039c5c03fd..eefcc4146749 100644 --- a/drivers/firmware/arm_scmi/virtio.c +++ b/drivers/firmware/arm_scmi/virtio.c @@ -452,7 +452,7 @@ static void scmi_vio_remove(struct virtio_device *vdev) * outstanding message on any vqueue to be ignored by complete_cb: now * we can just stop processing buffers and destroy the vqueues. */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); /* Ensure scmi_vdev is visible as NULL */ smp_store_mb(scmi_vdev, NULL); diff --git a/drivers/gpio/gpio-virtio.c b/drivers/gpio/gpio-virtio.c index 9f4941bc5760..fcc5e8c08973 100644 --- a/drivers/gpio/gpio-virtio.c +++ b/drivers/gpio/gpio-virtio.c @@ -450,7 +450,7 @@ static void virtio_gpio_request_vq(struct virtqueue *vq) static void virtio_gpio_free_vqs(struct virtio_device *vdev) { - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); } diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c index 21f410901694..3313b92db531 100644 --- a/drivers/gpu/drm/virtio/virtgpu_kms.c +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c @@ -279,7 +279,7 @@ void virtio_gpu_deinit(struct drm_device *dev) flush_work(&vgdev->ctrlq.dequeue_work); flush_work(&vgdev->cursorq.dequeue_work); flush_work(&vgdev->config_changed_work); - vgdev->vdev->config->reset(vgdev->vdev); + virtio_reset_device(vgdev->vdev); vgdev->vdev->config->del_vqs(vgdev->vdev); } diff --git a/drivers/i2c/busses/i2c-virtio.c b/drivers/i2c/busses/i2c-virtio.c index 41eb0dcc3204..4b9536f50800 100644 --- a/drivers/i2c/busses/i2c-virtio.c +++ b/drivers/i2c/busses/i2c-virtio.c @@ -165,7 +165,7 @@ err_free: static void virtio_i2c_del_vqs(struct virtio_device *vdev) { - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); } diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 80930ce04a16..1d4e1e7cf175 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1115,7 +1115,7 @@ static void viommu_remove(struct virtio_device *vdev) iommu_device_unregister(&viommu->iommu); /* Stop all virtqueues */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); dev_info(&vdev->dev, "device removed\n"); diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c index 91230894692d..444ef6a342f6 100644 --- a/drivers/net/caif/caif_virtio.c +++ b/drivers/net/caif/caif_virtio.c @@ -754,7 +754,7 @@ static void cfv_remove(struct virtio_device *vdev) debugfs_remove_recursive(cfv->debugfs); vringh_kiov_cleanup(&cfv->ctx.riov); - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->vringh_config->del_vrhs(cfv->vdev); cfv->vr_rx = NULL; vdev->config->del_vqs(cfv->vdev); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index b107835242ad..eeed458c794e 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3310,7 +3310,7 @@ static int virtnet_probe(struct virtio_device *vdev) return 0; free_unregister_netdev: - vi->vdev->config->reset(vdev); + virtio_reset_device(vdev); unregister_netdev(dev); free_failover: @@ -3326,7 +3326,7 @@ free: static void remove_vq_common(struct virtnet_info *vi) { - vi->vdev->config->reset(vi->vdev); + virtio_reset_device(vi->vdev); /* Free unused buffers in both send and recv, if any. */ free_unused_bufs(vi); diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index 23219f3747f8..02d374360bcc 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -4498,7 +4498,7 @@ static void remove_vqs(struct virtio_device *vdev) { int i; - vdev->config->reset(vdev); + virtio_reset_device(vdev); for (i = 0; i < ARRAY_SIZE(hwsim_vqs); i++) { struct virtqueue *vq = hwsim_vqs[i]; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 726c7354d465..995b6cdc67ed 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -105,7 +105,7 @@ static void virtio_pmem_remove(struct virtio_device *vdev) nvdimm_bus_unregister(nvdimm_bus); vdev->config->del_vqs(vdev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); } static struct virtio_driver virtio_pmem_driver = { diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 9c112aa65040..4bda40568dc9 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -1024,7 +1024,7 @@ static void rpmsg_remove(struct virtio_device *vdev) size_t total_buf_space = vrp->num_bufs * vrp->buf_size; int ret; - vdev->config->reset(vdev); + virtio_reset_device(vdev); ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device); if (ret) diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 28e1d98ae102..c616c7171cc4 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -778,7 +778,7 @@ static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq, static void virtscsi_remove_vqs(struct virtio_device *vdev) { /* Stop all the virtqueues. */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); } diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 236081afe9a2..00ac9db792a4 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -204,6 +204,12 @@ int virtio_finalize_features(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(virtio_finalize_features); +void virtio_reset_device(struct virtio_device *dev) +{ + dev->config->reset(dev); +} +EXPORT_SYMBOL_GPL(virtio_reset_device); + static int virtio_dev_probe(struct device *_d) { int err, i; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index c22ff0117b46..f4c34a2a6b8e 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -1056,7 +1056,7 @@ static void remove_common(struct virtio_balloon *vb) return_free_pages_to_mm(vb, ULONG_MAX); /* Now we reset the device so we can clean up the queues. */ - vb->vdev->config->reset(vb->vdev); + virtio_reset_device(vb->vdev); vb->vdev->config->del_vqs(vb->vdev); } diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index ce51ae165943..3aa46703872d 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -347,7 +347,7 @@ static void virtinput_remove(struct virtio_device *vdev) spin_unlock_irqrestore(&vi->lock, flags); input_unregister_device(vi->idev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); while ((buf = virtqueue_detach_unused_buf(vi->sts)) != NULL) kfree(buf); vdev->config->del_vqs(vdev); diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 96e5a8782769..033fb93ed528 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -2850,7 +2850,7 @@ static void virtio_mem_remove(struct virtio_device *vdev) virtio_mem_deinit_hotplug(vm); /* reset the device and cleanup the queues */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); kfree(vm); diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 4cfa4bc1f579..ca2cac196d73 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -895,7 +895,7 @@ static int virtio_fs_probe(struct virtio_device *vdev) return 0; out_vqs: - vdev->config->reset(vdev); + virtio_reset_device(vdev); virtio_fs_cleanup_vqs(vdev, fs); kfree(fs->vqs); @@ -927,7 +927,7 @@ static void virtio_fs_remove(struct virtio_device *vdev) list_del_init(&fs->list); virtio_fs_stop_all_queues(fs); virtio_fs_drain_all_queues_locked(fs); - vdev->config->reset(vdev); + virtio_reset_device(vdev); virtio_fs_cleanup_vqs(vdev, fs); vdev->priv = NULL; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 41edbc01ffa4..72292a62cd90 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -138,6 +138,7 @@ int virtio_finalize_features(struct virtio_device *dev); int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); #endif +void virtio_reset_device(struct virtio_device *dev); size_t virtio_max_dma_size(struct virtio_device *vdev); diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index bd5a89c4960d..8ff6d7160677 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -721,7 +721,7 @@ static void p9_virtio_remove(struct virtio_device *vdev) mutex_unlock(&virtio_9p_lock); - vdev->config->reset(vdev); + virtio_reset_device(vdev); vdev->config->del_vqs(vdev); sysfs_remove_file(&(vdev->dev.kobj), &dev_attr_mount_tag.attr); diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 4f7c99dfd16c..fb3302fff627 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -665,7 +665,7 @@ static void virtio_vsock_remove(struct virtio_device *vdev) vsock_for_each_connected_socket(virtio_vsock_reset_sock); /* Stop all work handlers to make sure no one is accessing the device, - * so we can safely call vdev->config->reset(). + * so we can safely call virtio_reset_device(). */ mutex_lock(&vsock->rx_lock); vsock->rx_run = false; @@ -682,7 +682,7 @@ static void virtio_vsock_remove(struct virtio_device *vdev) /* Flush all device writes and interrupts, device will not use any * more buffers. */ - vdev->config->reset(vdev); + virtio_reset_device(vdev); mutex_lock(&vsock->rx_lock); while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) diff --git a/sound/virtio/virtio_card.c b/sound/virtio/virtio_card.c index 150ab3e37013..e2847c040f75 100644 --- a/sound/virtio/virtio_card.c +++ b/sound/virtio/virtio_card.c @@ -350,7 +350,7 @@ static void virtsnd_remove(struct virtio_device *vdev) snd_card_free(snd->card); vdev->config->del_vqs(vdev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); for (i = 0; snd->substreams && i < snd->nsubstreams; ++i) { struct virtio_pcm_substream *vss = &snd->substreams[i]; @@ -379,7 +379,7 @@ static int virtsnd_freeze(struct virtio_device *vdev) virtsnd_ctl_msg_cancel_all(snd); vdev->config->del_vqs(vdev); - vdev->config->reset(vdev); + virtio_reset_device(vdev); for (i = 0; i < snd->nsubstreams; ++i) cancel_work_sync(&snd->substreams[i].elapsed_period); -- cgit v1.2.3-59-g8ed1b From 539fec78edb4e084e7c532affc56cc42d4ceea4b Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Fri, 26 Nov 2021 17:47:53 +0100 Subject: vdpa: add driver_override support `driver_override` allows to control which of the vDPA bus drivers binds to a vDPA device. If `driver_override` is not set, the previous behaviour is followed: devices use the first vDPA bus driver loaded (unless auto binding is disabled). Tested on Fedora 34 with driverctl(8): $ modprobe virtio-vdpa $ modprobe vhost-vdpa $ modprobe vdpa-sim-net $ vdpa dev add mgmtdev vdpasim_net name dev1 # dev1 is attached to the first vDPA bus driver loaded $ driverctl -b vdpa list-devices dev1 virtio_vdpa $ driverctl -b vdpa set-override dev1 vhost_vdpa $ driverctl -b vdpa list-devices dev1 vhost_vdpa [*] Note: driverctl(8) integrates with udev so the binding is preserved. Suggested-by: Jason Wang Acked-by: Jason Wang Signed-off-by: Stefano Garzarella Link: https://lore.kernel.org/r/20211126164753.181829-3-sgarzare@redhat.com Signed-off-by: Michael S. Tsirkin --- Documentation/ABI/testing/sysfs-bus-vdpa | 20 +++++++++ drivers/vdpa/vdpa.c | 74 ++++++++++++++++++++++++++++++++ include/linux/vdpa.h | 2 + 3 files changed, 96 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-bus-vdpa b/Documentation/ABI/testing/sysfs-bus-vdpa index 4e55761a39df..28a6111202ba 100644 --- a/Documentation/ABI/testing/sysfs-bus-vdpa +++ b/Documentation/ABI/testing/sysfs-bus-vdpa @@ -35,3 +35,23 @@ Description: Writing a device name to this file will cause the driver to attempt to unbind from the device. This may be useful when overriding default bindings. + +What: /sys/bus/vdpa/devices/.../driver_override +Date: November 2021 +Contact: virtualization@lists.linux-foundation.org +Description: + This file allows the driver for a device to be specified. + When specified, only a driver with a name matching the value + written to driver_override will have an opportunity to bind to + the device. The override is specified by writing a string to the + driver_override file (echo vhost-vdpa > driver_override) and may + be cleared with an empty string (echo > driver_override). + This returns the device to standard matching rules binding. + Writing to driver_override does not automatically unbind the + device from its current driver or make any attempt to + automatically load the specified driver. If no driver with a + matching name is currently loaded in the kernel, the device will + not bind to any driver. This also allows devices to opt-out of + driver binding using a driver_override name such as "none". + Only a single driver may be specified in the override, there is + no support for parsing delimiters. diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 09bbe53c3ac4..59d0b8bbb79c 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -52,8 +52,81 @@ static void vdpa_dev_remove(struct device *d) drv->remove(vdev); } +static int vdpa_dev_match(struct device *dev, struct device_driver *drv) +{ + struct vdpa_device *vdev = dev_to_vdpa(dev); + + /* Check override first, and if set, only use the named driver */ + if (vdev->driver_override) + return strcmp(vdev->driver_override, drv->name) == 0; + + /* Currently devices must be supported by all vDPA bus drivers */ + return 1; +} + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct vdpa_device *vdev = dev_to_vdpa(dev); + const char *driver_override, *old; + char *cp; + + /* We need to keep extra room for a newline */ + if (count >= (PAGE_SIZE - 1)) + return -EINVAL; + + driver_override = kstrndup(buf, count, GFP_KERNEL); + if (!driver_override) + return -ENOMEM; + + cp = strchr(driver_override, '\n'); + if (cp) + *cp = '\0'; + + device_lock(dev); + old = vdev->driver_override; + if (strlen(driver_override)) { + vdev->driver_override = driver_override; + } else { + kfree(driver_override); + vdev->driver_override = NULL; + } + device_unlock(dev); + + kfree(old); + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct vdpa_device *vdev = dev_to_vdpa(dev); + ssize_t len; + + device_lock(dev); + len = snprintf(buf, PAGE_SIZE, "%s\n", vdev->driver_override); + device_unlock(dev); + + return len; +} +static DEVICE_ATTR_RW(driver_override); + +static struct attribute *vdpa_dev_attrs[] = { + &dev_attr_driver_override.attr, + NULL, +}; + +static const struct attribute_group vdpa_dev_group = { + .attrs = vdpa_dev_attrs, +}; +__ATTRIBUTE_GROUPS(vdpa_dev); + static struct bus_type vdpa_bus = { .name = "vdpa", + .dev_groups = vdpa_dev_groups, + .match = vdpa_dev_match, .probe = vdpa_dev_probe, .remove = vdpa_dev_remove, }; @@ -68,6 +141,7 @@ static void vdpa_release_dev(struct device *d) ida_simple_remove(&vdpa_index_ida, vdev->index); mutex_destroy(&vdev->cf_mutex); + kfree(vdev->driver_override); kfree(vdev); } diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index c3011ccda430..ae34015b37b7 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -64,6 +64,7 @@ struct vdpa_mgmt_dev; * struct vdpa_device - representation of a vDPA device * @dev: underlying device * @dma_dev: the actual device that is performing DMA + * @driver_override: driver name to force a match * @config: the configuration ops for this device. * @cf_mutex: Protects get and set access to configuration layout. * @index: device index @@ -76,6 +77,7 @@ struct vdpa_mgmt_dev; struct vdpa_device { struct device dev; struct device *dma_dev; + const char *driver_override; const struct vdpa_config_ops *config; struct mutex cf_mutex; /* Protects get/set config */ unsigned int index; -- cgit v1.2.3-59-g8ed1b From 28cc408be72cebb0f3fcc37bc74ab3196d4de726 Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Thu, 4 Nov 2021 20:52:48 +0100 Subject: vdpa: Mark vdpa_config_ops.get_vq_notification as optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since vhost_vdpa_mmap checks for its existence before calling it. Signed-off-by: Eugenio Pérez Link: https://lore.kernel.org/r/20211104195248.2088904-1-eperezma@redhat.com Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang Reviewed-by: Stefano Garzarella --- include/linux/vdpa.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index ae34015b37b7..2b7db96bb7d3 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -157,7 +157,7 @@ struct vdpa_map_file { * @vdev: vdpa device * @idx: virtqueue index * @state: pointer to returned state (last_avail_idx) - * @get_vq_notification: Get the notification area for a virtqueue + * @get_vq_notification: Get the notification area for a virtqueue (optional) * @vdev: vdpa device * @idx: virtqueue index * Returns the notifcation area -- cgit v1.2.3-59-g8ed1b From a64917bc2e9b1e0aa716b783c4ec879fdd280300 Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Wed, 5 Jan 2022 13:46:33 +0200 Subject: vdpa: Provide interface to read driver features Provide an interface to read the negotiated features. This is needed when building the netlink message in vdpa_dev_net_config_fill(). Also fix the implementation of vdpa_dev_net_config_fill() to use the negotiated features instead of the device features. To make APIs clearer, make the following name changes to struct vdpa_config_ops so they better describe their operations: get_features -> get_device_features set_features -> set_driver_features Finally, add get_driver_features to return the negotiated features and add implementation to all the upstream drivers. Acked-by: Jason Wang Signed-off-by: Eli Cohen Link: https://lore.kernel.org/r/20220105114646.577224-2-elic@nvidia.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/alibaba/eni_vdpa.c | 16 ++++++++++++---- drivers/vdpa/ifcvf/ifcvf_main.c | 16 ++++++++++++---- drivers/vdpa/mlx5/net/mlx5_vnet.c | 16 ++++++++++++---- drivers/vdpa/vdpa.c | 2 +- drivers/vdpa/vdpa_sim/vdpa_sim.c | 21 +++++++++++++++------ drivers/vdpa/vdpa_user/vduse_dev.c | 16 ++++++++++++---- drivers/vdpa/virtio_pci/vp_vdpa.c | 16 ++++++++++++---- drivers/vhost/vdpa.c | 2 +- drivers/virtio/virtio_vdpa.c | 2 +- include/linux/vdpa.h | 14 +++++++++----- 10 files changed, 87 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/drivers/vdpa/alibaba/eni_vdpa.c b/drivers/vdpa/alibaba/eni_vdpa.c index fe1b83b5f80d..f480d54f308c 100644 --- a/drivers/vdpa/alibaba/eni_vdpa.c +++ b/drivers/vdpa/alibaba/eni_vdpa.c @@ -58,7 +58,7 @@ static struct virtio_pci_legacy_device *vdpa_to_ldev(struct vdpa_device *vdpa) return &eni_vdpa->ldev; } -static u64 eni_vdpa_get_features(struct vdpa_device *vdpa) +static u64 eni_vdpa_get_device_features(struct vdpa_device *vdpa) { struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa); u64 features = vp_legacy_get_features(ldev); @@ -69,7 +69,7 @@ static u64 eni_vdpa_get_features(struct vdpa_device *vdpa) return features; } -static int eni_vdpa_set_features(struct vdpa_device *vdpa, u64 features) +static int eni_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) { struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa); @@ -84,6 +84,13 @@ static int eni_vdpa_set_features(struct vdpa_device *vdpa, u64 features) return 0; } +static u64 eni_vdpa_get_driver_features(struct vdpa_device *vdpa) +{ + struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa); + + return vp_legacy_get_driver_features(ldev); +} + static u8 eni_vdpa_get_status(struct vdpa_device *vdpa) { struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa); @@ -401,8 +408,9 @@ static void eni_vdpa_set_config_cb(struct vdpa_device *vdpa, } static const struct vdpa_config_ops eni_vdpa_ops = { - .get_features = eni_vdpa_get_features, - .set_features = eni_vdpa_set_features, + .get_device_features = eni_vdpa_get_device_features, + .set_driver_features = eni_vdpa_set_driver_features, + .get_driver_features = eni_vdpa_get_driver_features, .get_status = eni_vdpa_get_status, .set_status = eni_vdpa_set_status, .reset = eni_vdpa_reset, diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 92ba7126e5d6..d1a6b5ab543c 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -169,7 +169,7 @@ static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev) return &adapter->vf; } -static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev) +static u64 ifcvf_vdpa_get_device_features(struct vdpa_device *vdpa_dev) { struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev); struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); @@ -187,7 +187,7 @@ static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev) return features; } -static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features) +static int ifcvf_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features) { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); int ret; @@ -201,6 +201,13 @@ static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features) return 0; } +static u64 ifcvf_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) +{ + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); + + return vf->req_features; +} + static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) { struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); @@ -426,8 +433,9 @@ static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_devic * implemented set_map()/dma_map()/dma_unmap() */ static const struct vdpa_config_ops ifc_vdpa_ops = { - .get_features = ifcvf_vdpa_get_features, - .set_features = ifcvf_vdpa_set_features, + .get_device_features = ifcvf_vdpa_get_device_features, + .set_driver_features = ifcvf_vdpa_set_driver_features, + .get_driver_features = ifcvf_vdpa_get_driver_features, .get_status = ifcvf_vdpa_get_status, .set_status = ifcvf_vdpa_set_status, .reset = ifcvf_vdpa_reset, diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index c104d7699c16..3206e355230c 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1878,7 +1878,7 @@ static u64 mlx_to_vritio_features(u16 dev_features) return result; } -static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev) +static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); @@ -1971,7 +1971,7 @@ static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) } } -static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features) +static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); @@ -2338,6 +2338,13 @@ static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) return -EOPNOTSUPP; } +static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) +{ + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + + return mvdev->actual_features; +} + static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_vq_address = mlx5_vdpa_set_vq_address, .set_vq_num = mlx5_vdpa_set_vq_num, @@ -2350,8 +2357,9 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { .get_vq_notification = mlx5_get_vq_notification, .get_vq_irq = mlx5_get_vq_irq, .get_vq_align = mlx5_vdpa_get_vq_align, - .get_features = mlx5_vdpa_get_features, - .set_features = mlx5_vdpa_set_features, + .get_device_features = mlx5_vdpa_get_device_features, + .set_driver_features = mlx5_vdpa_set_driver_features, + .get_driver_features = mlx5_vdpa_get_driver_features, .set_config_cb = mlx5_vdpa_set_config_cb, .get_vq_num_max = mlx5_vdpa_get_vq_num_max, .get_device_id = mlx5_vdpa_get_device_id, diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 59d0b8bbb79c..42d71d60d5dc 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -808,7 +808,7 @@ static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *ms if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16)) return -EMSGSIZE; - features = vdev->config->get_features(vdev); + features = vdev->config->get_driver_features(vdev); return vdpa_dev_net_mq_config_fill(vdev, msg, features, &config); } diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 41b0cd17fcba..ddbe142af09a 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -399,14 +399,14 @@ static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) return VDPASIM_QUEUE_ALIGN; } -static u64 vdpasim_get_features(struct vdpa_device *vdpa) +static u64 vdpasim_get_device_features(struct vdpa_device *vdpa) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); return vdpasim->dev_attr.supported_features; } -static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features) +static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); @@ -419,6 +419,13 @@ static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features) return 0; } +static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + return vdpasim->features; +} + static void vdpasim_set_config_cb(struct vdpa_device *vdpa, struct vdpa_callback *cb) { @@ -613,8 +620,9 @@ static const struct vdpa_config_ops vdpasim_config_ops = { .set_vq_state = vdpasim_set_vq_state, .get_vq_state = vdpasim_get_vq_state, .get_vq_align = vdpasim_get_vq_align, - .get_features = vdpasim_get_features, - .set_features = vdpasim_set_features, + .get_device_features = vdpasim_get_device_features, + .set_driver_features = vdpasim_set_driver_features, + .get_driver_features = vdpasim_get_driver_features, .set_config_cb = vdpasim_set_config_cb, .get_vq_num_max = vdpasim_get_vq_num_max, .get_device_id = vdpasim_get_device_id, @@ -642,8 +650,9 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = { .set_vq_state = vdpasim_set_vq_state, .get_vq_state = vdpasim_get_vq_state, .get_vq_align = vdpasim_get_vq_align, - .get_features = vdpasim_get_features, - .set_features = vdpasim_set_features, + .get_device_features = vdpasim_get_device_features, + .set_driver_features = vdpasim_set_driver_features, + .get_driver_features = vdpasim_get_driver_features, .set_config_cb = vdpasim_set_config_cb, .get_vq_num_max = vdpasim_get_vq_num_max, .get_device_id = vdpasim_get_device_id, diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index f3cc7dde39af..f85d1a08ed87 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -573,14 +573,14 @@ static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) return dev->vq_align; } -static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa) +static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) { struct vduse_dev *dev = vdpa_to_vduse(vdpa); return dev->device_features; } -static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features) +static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) { struct vduse_dev *dev = vdpa_to_vduse(vdpa); @@ -588,6 +588,13 @@ static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features) return 0; } +static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) +{ + struct vduse_dev *dev = vdpa_to_vduse(vdpa); + + return dev->driver_features; +} + static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, struct vdpa_callback *cb) { @@ -721,8 +728,9 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = { .set_vq_state = vduse_vdpa_set_vq_state, .get_vq_state = vduse_vdpa_get_vq_state, .get_vq_align = vduse_vdpa_get_vq_align, - .get_features = vduse_vdpa_get_features, - .set_features = vduse_vdpa_set_features, + .get_device_features = vduse_vdpa_get_device_features, + .set_driver_features = vduse_vdpa_set_driver_features, + .get_driver_features = vduse_vdpa_get_driver_features, .set_config_cb = vduse_vdpa_set_config_cb, .get_vq_num_max = vduse_vdpa_get_vq_num_max, .get_device_id = vduse_vdpa_get_device_id, diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c index e3ff7875e123..a57e381e830b 100644 --- a/drivers/vdpa/virtio_pci/vp_vdpa.c +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c @@ -53,14 +53,14 @@ static struct virtio_pci_modern_device *vdpa_to_mdev(struct vdpa_device *vdpa) return &vp_vdpa->mdev; } -static u64 vp_vdpa_get_features(struct vdpa_device *vdpa) +static u64 vp_vdpa_get_device_features(struct vdpa_device *vdpa) { struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); return vp_modern_get_features(mdev); } -static int vp_vdpa_set_features(struct vdpa_device *vdpa, u64 features) +static int vp_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) { struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); @@ -69,6 +69,13 @@ static int vp_vdpa_set_features(struct vdpa_device *vdpa, u64 features) return 0; } +static u64 vp_vdpa_get_driver_features(struct vdpa_device *vdpa) +{ + struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); + + return vp_modern_get_driver_features(mdev); +} + static u8 vp_vdpa_get_status(struct vdpa_device *vdpa) { struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); @@ -415,8 +422,9 @@ vp_vdpa_get_vq_notification(struct vdpa_device *vdpa, u16 qid) } static const struct vdpa_config_ops vp_vdpa_ops = { - .get_features = vp_vdpa_get_features, - .set_features = vp_vdpa_set_features, + .get_device_features = vp_vdpa_get_device_features, + .set_driver_features = vp_vdpa_set_driver_features, + .get_driver_features = vp_vdpa_get_driver_features, .get_status = vp_vdpa_get_status, .set_status = vp_vdpa_set_status, .reset = vp_vdpa_reset, diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index ecfccd687ea0..a8fa7fc6db1e 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -262,7 +262,7 @@ static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) const struct vdpa_config_ops *ops = vdpa->config; u64 features; - features = ops->get_features(vdpa); + features = ops->get_device_features(vdpa); if (copy_to_user(featurep, &features, sizeof(features))) return -EFAULT; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index f85f860bc10b..a84b04ba3195 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -308,7 +308,7 @@ static u64 virtio_vdpa_get_features(struct virtio_device *vdev) struct vdpa_device *vdpa = vd_get_vdpa(vdev); const struct vdpa_config_ops *ops = vdpa->config; - return ops->get_features(vdpa); + return ops->get_device_features(vdpa); } static int virtio_vdpa_finalize_features(struct virtio_device *vdev) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 2b7db96bb7d3..9cc4291a79b3 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -171,14 +171,17 @@ struct vdpa_map_file { * for the device * @vdev: vdpa device * Returns virtqueue algin requirement - * @get_features: Get virtio features supported by the device + * @get_device_features: Get virtio features supported by the device * @vdev: vdpa device * Returns the virtio features support by the * device - * @set_features: Set virtio features supported by the driver + * @set_driver_features: Set virtio features supported by the driver * @vdev: vdpa device * @features: feature support by the driver * Returns integer: success (0) or error (< 0) + * @get_driver_features: Get the virtio driver features in action + * @vdev: vdpa device + * Returns the virtio features accepted * @set_config_cb: Set the config interrupt callback * @vdev: vdpa device * @cb: virtio-vdev interrupt callback structure @@ -278,8 +281,9 @@ struct vdpa_config_ops { /* Device ops */ u32 (*get_vq_align)(struct vdpa_device *vdev); - u64 (*get_features)(struct vdpa_device *vdev); - int (*set_features)(struct vdpa_device *vdev, u64 features); + u64 (*get_device_features)(struct vdpa_device *vdev); + int (*set_driver_features)(struct vdpa_device *vdev, u64 features); + u64 (*get_driver_features)(struct vdpa_device *vdev); void (*set_config_cb)(struct vdpa_device *vdev, struct vdpa_callback *cb); u16 (*get_vq_num_max)(struct vdpa_device *vdev); @@ -397,7 +401,7 @@ static inline int vdpa_set_features(struct vdpa_device *vdev, u64 features) const struct vdpa_config_ops *ops = vdev->config; vdev->features_valid = true; - return ops->set_features(vdev, features); + return ops->set_driver_features(vdev, features); } void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, -- cgit v1.2.3-59-g8ed1b From 73bc0dbb591baea322a7319c735e5f6c7dba9cfb Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Wed, 5 Jan 2022 13:46:35 +0200 Subject: vdpa: Sync calls set/get config/status with cf_mutex Add wrappers to get/set status and protect these operations with cf_mutex to serialize these operations with respect to get/set config operations. Signed-off-by: Eli Cohen Link: https://lore.kernel.org/r/20220105114646.577224-4-elic@nvidia.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 19 +++++++++++++++++++ drivers/vhost/vdpa.c | 7 +++---- drivers/virtio/virtio_vdpa.c | 3 +-- include/linux/vdpa.h | 3 +++ 4 files changed, 26 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 42d71d60d5dc..5134c83c4a22 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -21,6 +21,25 @@ static LIST_HEAD(mdev_head); static DEFINE_MUTEX(vdpa_dev_mutex); static DEFINE_IDA(vdpa_index_ida); +u8 vdpa_get_status(struct vdpa_device *vdev) +{ + u8 status; + + mutex_lock(&vdev->cf_mutex); + status = vdev->config->get_status(vdev); + mutex_unlock(&vdev->cf_mutex); + return status; +} +EXPORT_SYMBOL(vdpa_get_status); + +void vdpa_set_status(struct vdpa_device *vdev, u8 status) +{ + mutex_lock(&vdev->cf_mutex); + vdev->config->set_status(vdev, status); + mutex_unlock(&vdev->cf_mutex); +} +EXPORT_SYMBOL(vdpa_set_status); + static struct genl_family vdpa_nl_family; static int vdpa_dev_probe(struct device *d) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index a8fa7fc6db1e..293c51fdf9ab 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -142,10 +142,9 @@ static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) { struct vdpa_device *vdpa = v->vdpa; - const struct vdpa_config_ops *ops = vdpa->config; u8 status; - status = ops->get_status(vdpa); + status = vdpa_get_status(vdpa); if (copy_to_user(statusp, &status, sizeof(status))) return -EFAULT; @@ -164,7 +163,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) if (copy_from_user(&status, statusp, sizeof(status))) return -EFAULT; - status_old = ops->get_status(vdpa); + status_old = vdpa_get_status(vdpa); /* * Userspace shouldn't remove status bits unless reset the @@ -182,7 +181,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) if (ret) return ret; } else - ops->set_status(vdpa, status); + vdpa_set_status(vdpa, status); if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) for (i = 0; i < nvqs; i++) diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index a84b04ba3195..76504559bc25 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -91,9 +91,8 @@ static u8 virtio_vdpa_get_status(struct virtio_device *vdev) static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); - const struct vdpa_config_ops *ops = vdpa->config; - return ops->set_status(vdpa, status); + return vdpa_set_status(vdpa, status); } static void virtio_vdpa_reset(struct virtio_device *vdev) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 9cc4291a79b3..ae047fae2603 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -408,6 +408,9 @@ void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, unsigned int len); void vdpa_set_config(struct vdpa_device *dev, unsigned int offset, const void *buf, unsigned int length); +u8 vdpa_get_status(struct vdpa_device *vdev); +void vdpa_set_status(struct vdpa_device *vdev, u8 status); + /** * struct vdpa_mgmtdev_ops - vdpa device ops * @dev_add: Add a vdpa device using alloc and register -- cgit v1.2.3-59-g8ed1b From aba21aff772b8622e08f07219069be793429a48f Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Wed, 5 Jan 2022 13:46:37 +0200 Subject: vdpa: Allow to configure max data virtqueues Add netlink support to configure the max virtqueue pairs for a device. At least one pair is required. The maximum is dictated by the device. Example: $ vdpa dev add name vdpa-a mgmtdev auxiliary/mlx5_core.sf.1 max_vqp 4 Signed-off-by: Eli Cohen Link: https://lore.kernel.org/r/20220105114646.577224-6-elic@nvidia.com Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 15 +++++++++++++-- drivers/vhost/vdpa.c | 2 +- drivers/virtio/virtio_vdpa.c | 2 +- include/linux/vdpa.h | 19 ++++++++++++++++--- 4 files changed, 31 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 4494325cae91..96d31b80fdce 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -404,7 +404,7 @@ static void vdpa_get_config_unlocked(struct vdpa_device *vdev, * If it does happen we assume a legacy guest. */ if (!vdev->features_valid) - vdpa_set_features(vdev, 0); + vdpa_set_features(vdev, 0, true); ops->get_config(vdev, offset, buf, len); } @@ -581,7 +581,8 @@ out: } #define VDPA_DEV_NET_ATTRS_MASK ((1 << VDPA_ATTR_DEV_NET_CFG_MACADDR) | \ - (1 << VDPA_ATTR_DEV_NET_CFG_MTU)) + (1 << VDPA_ATTR_DEV_NET_CFG_MTU) | \ + (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info) { @@ -607,6 +608,16 @@ static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *i nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]); config.mask |= (1 << VDPA_ATTR_DEV_NET_CFG_MTU); } + if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]) { + config.net.max_vq_pairs = + nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]); + if (!config.net.max_vq_pairs) { + NL_SET_ERR_MSG_MOD(info->extack, + "At least one pair of VQs is required"); + return -EINVAL; + } + config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); + } /* Skip checking capability if user didn't prefer to configure any * device networking attributes. It is likely that user might have used diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 293c51fdf9ab..6e7edaf2472b 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -285,7 +285,7 @@ static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; - if (vdpa_set_features(vdpa, features)) + if (vdpa_set_features(vdpa, features, false)) return -EINVAL; return 0; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 76504559bc25..7767a7f0119b 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -317,7 +317,7 @@ static int virtio_vdpa_finalize_features(struct virtio_device *vdev) /* Give virtio_ring a chance to accept features. */ vring_transport_features(vdev); - return vdpa_set_features(vdpa, vdev->features); + return vdpa_set_features(vdpa, vdev->features, false); } static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index ae047fae2603..6d4d7e4fe208 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -101,6 +101,7 @@ struct vdpa_dev_set_config { struct { u8 mac[ETH_ALEN]; u16 mtu; + u16 max_vq_pairs; } net; u64 mask; }; @@ -391,17 +392,29 @@ static inline struct device *vdpa_get_dma_dev(struct vdpa_device *vdev) static inline int vdpa_reset(struct vdpa_device *vdev) { const struct vdpa_config_ops *ops = vdev->config; + int ret; + mutex_lock(&vdev->cf_mutex); vdev->features_valid = false; - return ops->reset(vdev); + ret = ops->reset(vdev); + mutex_unlock(&vdev->cf_mutex); + return ret; } -static inline int vdpa_set_features(struct vdpa_device *vdev, u64 features) +static inline int vdpa_set_features(struct vdpa_device *vdev, u64 features, bool locked) { const struct vdpa_config_ops *ops = vdev->config; + int ret; + + if (!locked) + mutex_lock(&vdev->cf_mutex); vdev->features_valid = true; - return ops->set_driver_features(vdev, features); + ret = ops->set_driver_features(vdev, features); + if (!locked) + mutex_unlock(&vdev->cf_mutex); + + return ret; } void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, -- cgit v1.2.3-59-g8ed1b From cd2629f6df1cab5b3df34705ae7f3bde6147fce3 Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Wed, 5 Jan 2022 13:46:42 +0200 Subject: vdpa: Support reporting max device capabilities Add max_supported_vqs and supported_features fields to struct vdpa_mgmt_dev. Upstream drivers need to feel these values according to the device capabilities. These values are reported back in a netlink message when showing management devices. Examples: $ auxiliary/mlx5_core.sf.1: supported_classes net max_supported_vqs 257 dev_features CSUM GUEST_CSUM MTU HOST_TSO4 HOST_TSO6 STATUS CTRL_VQ MQ \ CTRL_MAC_ADDR VERSION_1 ACCESS_PLATFORM $ vdpa -j mgmtdev show {"mgmtdev":{"auxiliary/mlx5_core.sf.1":{"supported_classes":["net"], \ "max_supported_vqs":257,"dev_features":["CSUM","GUEST_CSUM","MTU", \ "HOST_TSO4","HOST_TSO6","STATUS","CTRL_VQ","MQ","CTRL_MAC_ADDR", \ "VERSION_1","ACCESS_PLATFORM"]}}} $ vdpa -jp mgmtdev show { "mgmtdev": { "auxiliary/mlx5_core.sf.1": { "supported_classes": [ "net" ], "max_supported_vqs": 257, "dev_features": ["CSUM","GUEST_CSUM","MTU","HOST_TSO4", \ "HOST_TSO6","STATUS","CTRL_VQ","MQ", \ "CTRL_MAC_ADDR","VERSION_1","ACCESS_PLATFORM"] } } } Signed-off-by: Eli Cohen Link: https://lore.kernel.org/r/20220105114646.577224-11-elic@nvidia.com Signed-off-by: Michael S. Tsirkin Reviewed-by: Si-Wei Liu --- drivers/vdpa/vdpa.c | 10 ++++++++++ include/linux/vdpa.h | 2 ++ include/uapi/linux/vdpa.h | 2 ++ 3 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 60cf821175fa..34fa251db8cc 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -514,6 +514,16 @@ static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *m err = -EMSGSIZE; goto msg_err; } + if (nla_put_u32(msg, VDPA_ATTR_DEV_MGMTDEV_MAX_VQS, + mdev->max_supported_vqs)) { + err = -EMSGSIZE; + goto msg_err; + } + if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_SUPPORTED_FEATURES, + mdev->supported_features, VDPA_ATTR_PAD)) { + err = -EMSGSIZE; + goto msg_err; + } genlmsg_end(msg, hdr); return 0; diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 6d4d7e4fe208..a6047fd6cf12 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -460,6 +460,8 @@ struct vdpa_mgmt_dev { const struct virtio_device_id *id_table; u64 config_attr_mask; struct list_head list; + u64 supported_features; + u32 max_supported_vqs; }; int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev); diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index db3738ef3beb..1061d8d2d09d 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -44,6 +44,8 @@ enum vdpa_attr { VDPA_ATTR_DEV_NET_CFG_MTU, /* u16 */ VDPA_ATTR_DEV_NEGOTIATED_FEATURES, /* u64 */ + VDPA_ATTR_DEV_MGMTDEV_MAX_VQS, /* u32 */ + VDPA_ATTR_DEV_SUPPORTED_FEATURES, /* u64 */ /* new attributes must be added above here */ VDPA_ATTR_MAX, }; -- cgit v1.2.3-59-g8ed1b From f6d955d80830b6e6f6a170be68cc3628f36365dd Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Tue, 11 Jan 2022 20:33:57 +0200 Subject: vdpa: Avoid taking cf_mutex lock on get status Avoid the wrapper holding cf_mutex since it is not protecting anything. To avoid confusion and unnecessary overhead incurred by it, remove. Fixes: f489f27bc0ab ("vdpa: Sync calls set/get config/status with cf_mutex") Signed-off-by: Eli Cohen Link: https://lore.kernel.org/r/20220111183400.38418-2-elic@nvidia.com Signed-off-by: Michael S. Tsirkin Reviewed-by: Si-Wei Liu Acked-by: Jason Wang --- drivers/vdpa/vdpa.c | 11 ----------- drivers/vhost/vdpa.c | 5 +++-- include/linux/vdpa.h | 1 - 3 files changed, 3 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 4380367d00b5..9846c9de4bfa 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -21,17 +21,6 @@ static LIST_HEAD(mdev_head); static DEFINE_MUTEX(vdpa_dev_mutex); static DEFINE_IDA(vdpa_index_ida); -u8 vdpa_get_status(struct vdpa_device *vdev) -{ - u8 status; - - mutex_lock(&vdev->cf_mutex); - status = vdev->config->get_status(vdev); - mutex_unlock(&vdev->cf_mutex); - return status; -} -EXPORT_SYMBOL(vdpa_get_status); - void vdpa_set_status(struct vdpa_device *vdev, u8 status) { mutex_lock(&vdev->cf_mutex); diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 6e7edaf2472b..0ed6cbadb52d 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -142,9 +142,10 @@ static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) { struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; u8 status; - status = vdpa_get_status(vdpa); + status = ops->get_status(vdpa); if (copy_to_user(statusp, &status, sizeof(status))) return -EFAULT; @@ -163,7 +164,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) if (copy_from_user(&status, statusp, sizeof(status))) return -EFAULT; - status_old = vdpa_get_status(vdpa); + status_old = ops->get_status(vdpa); /* * Userspace shouldn't remove status bits unless reset the diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index a6047fd6cf12..2de442ececae 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -421,7 +421,6 @@ void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, unsigned int len); void vdpa_set_config(struct vdpa_device *dev, unsigned int offset, const void *buf, unsigned int length); -u8 vdpa_get_status(struct vdpa_device *vdev); void vdpa_set_status(struct vdpa_device *vdev, u8 status); /** -- cgit v1.2.3-59-g8ed1b From 800977f6f32e452cba6b04ef21d2f5383ca29209 Mon Sep 17 00:00:00 2001 From: Cai Huoqing Date: Fri, 14 Jan 2022 14:02:52 -0800 Subject: kthread: add the helper function kthread_run_on_cpu() Add a new helper function kthread_run_on_cpu(), which includes kthread_create_on_cpu/wake_up_process(). In some cases, use kthread_run_on_cpu() directly instead of kthread_create_on_node/kthread_bind/wake_up_process() or kthread_create_on_cpu/wake_up_process() or kthreadd_create/kthread_bind/wake_up_process() to simplify the code. [akpm@linux-foundation.org: export kthread_create_on_cpu to modules] Link: https://lkml.kernel.org/r/20211022025711.3673-2-caihuoqing@baidu.com Signed-off-by: Cai Huoqing Cc: Bernard Metzler Cc: Cai Huoqing Cc: Daniel Bristot de Oliveira Cc: Davidlohr Bueso Cc: Doug Ledford Cc: Ingo Molnar Cc: Jason Gunthorpe Cc: Joel Fernandes (Google) Cc: Josh Triplett Cc: Lai Jiangshan Cc: Mathieu Desnoyers Cc: "Paul E . McKenney" Cc: Steven Rostedt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kthread.h | 25 +++++++++++++++++++++++++ kernel/kthread.c | 1 + 2 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 346b0f269161..db47aae7c481 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -56,6 +56,31 @@ bool kthread_is_per_cpu(struct task_struct *k); __k; \ }) +/** + * kthread_run_on_cpu - create and wake a cpu bound thread. + * @threadfn: the function to run until signal_pending(current). + * @data: data ptr for @threadfn. + * @cpu: The cpu on which the thread should be bound, + * @namefmt: printf-style name for the thread. Format is restricted + * to "name.*%u". Code fills in cpu number. + * + * Description: Convenient wrapper for kthread_create_on_cpu() + * followed by wake_up_process(). Returns the kthread or + * ERR_PTR(-ENOMEM). + */ +static inline struct task_struct * +kthread_run_on_cpu(int (*threadfn)(void *data), void *data, + unsigned int cpu, const char *namefmt) +{ + struct task_struct *p; + + p = kthread_create_on_cpu(threadfn, data, cpu, namefmt); + if (!IS_ERR(p)) + wake_up_process(p); + + return p; +} + void free_kthread_struct(struct task_struct *k); void kthread_bind(struct task_struct *k, unsigned int cpu); void kthread_bind_mask(struct task_struct *k, const struct cpumask *mask); diff --git a/kernel/kthread.c b/kernel/kthread.c index 7113003fab63..4ed9e7bce9e8 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -523,6 +523,7 @@ struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), to_kthread(p)->cpu = cpu; return p; } +EXPORT_SYMBOL(kthread_create_on_cpu); void kthread_set_per_cpu(struct task_struct *k, int cpu) { -- cgit v1.2.3-59-g8ed1b From 60115fa54ad7b913b7cb5844e6b7ffeb842d55f2 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 14 Jan 2022 14:04:11 -0800 Subject: mm: defer kmemleak object creation of module_alloc() Yongqiang reports a kmemleak panic when module insmod/rmmod with KASAN enabled(without KASAN_VMALLOC) on x86[1]. When the module area allocates memory, it's kmemleak_object is created successfully, but the KASAN shadow memory of module allocation is not ready, so when kmemleak scan the module's pointer, it will panic due to no shadow memory with KASAN check. module_alloc __vmalloc_node_range kmemleak_vmalloc kmemleak_scan update_checksum kasan_module_alloc kmemleak_ignore Note, there is no problem if KASAN_VMALLOC enabled, the modules area entire shadow memory is preallocated. Thus, the bug only exits on ARCH which supports dynamic allocation of module area per module load, for now, only x86/arm64/s390 are involved. Add a VM_DEFER_KMEMLEAK flags, defer vmalloc'ed object register of kmemleak in module_alloc() to fix this issue. [1] https://lore.kernel.org/all/6d41e2b9-4692-5ec4-b1cd-cbe29ae89739@huawei.com/ [wangkefeng.wang@huawei.com: fix build] Link: https://lkml.kernel.org/r/20211125080307.27225-1-wangkefeng.wang@huawei.com [akpm@linux-foundation.org: simplify ifdefs, per Andrey] Link: https://lkml.kernel.org/r/CA+fCnZcnwJHUQq34VuRxpdoY6_XbJCDJ-jopksS5Eia4PijPzw@mail.gmail.com Link: https://lkml.kernel.org/r/20211124142034.192078-1-wangkefeng.wang@huawei.com Fixes: 793213a82de4 ("s390/kasan: dynamic shadow mem allocation for modules") Fixes: 39d114ddc682 ("arm64: add KASAN support") Fixes: bebf56a1b176 ("kasan: enable instrumentation of global variables") Signed-off-by: Kefeng Wang Reported-by: Yongqiang Liu Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Catalin Marinas Cc: Will Deacon Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Christian Borntraeger Cc: Alexander Gordeev Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: Alexander Potapenko Cc: Kefeng Wang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm64/kernel/module.c | 4 ++-- arch/s390/kernel/module.c | 5 +++-- arch/x86/kernel/module.c | 7 ++++--- include/linux/kasan.h | 4 ++-- include/linux/vmalloc.h | 7 +++++++ mm/kasan/shadow.c | 9 +++++++-- mm/vmalloc.c | 3 ++- 7 files changed, 27 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index b5ec010c481f..309a27553c87 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -36,7 +36,7 @@ void *module_alloc(unsigned long size) module_alloc_end = MODULES_END; p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, gfp_mask, PAGE_KERNEL, 0, + module_alloc_end, gfp_mask, PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE, __builtin_return_address(0)); if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && @@ -58,7 +58,7 @@ void *module_alloc(unsigned long size) PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); - if (p && (kasan_module_alloc(p, size) < 0)) { + if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) { vfree(p); return NULL; } diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index b01ba460b7ca..d52d85367bf7 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -37,14 +37,15 @@ void *module_alloc(unsigned long size) { + gfp_t gfp_mask = GFP_KERNEL; void *p; if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, + gfp_mask, PAGE_KERNEL_EXEC, VM_DEFER_KMEMLEAK, NUMA_NO_NODE, __builtin_return_address(0)); - if (p && (kasan_module_alloc(p, size) < 0)) { + if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) { vfree(p); return NULL; } diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 169fb6f4cd2e..95fa745e310a 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -67,6 +67,7 @@ static unsigned long int get_module_load_offset(void) void *module_alloc(unsigned long size) { + gfp_t gfp_mask = GFP_KERNEL; void *p; if (PAGE_ALIGN(size) > MODULES_LEN) @@ -74,10 +75,10 @@ void *module_alloc(unsigned long size) p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR + get_module_load_offset(), - MODULES_END, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, + MODULES_END, gfp_mask, + PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE, __builtin_return_address(0)); - if (p && (kasan_module_alloc(p, size) < 0)) { + if (p && (kasan_module_alloc(p, size, gfp_mask) < 0)) { vfree(p); return NULL; } diff --git a/include/linux/kasan.h b/include/linux/kasan.h index d8783b682669..89c99e5e67de 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -474,12 +474,12 @@ static inline void kasan_populate_early_vm_area_shadow(void *start, * allocations with real shadow memory. With KASAN vmalloc, the special * case is unnecessary, as the work is handled in the generic case. */ -int kasan_module_alloc(void *addr, size_t size); +int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask); void kasan_free_shadow(const struct vm_struct *vm); #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */ -static inline int kasan_module_alloc(void *addr, size_t size) { return 0; } +static inline int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) { return 0; } static inline void kasan_free_shadow(const struct vm_struct *vm) {} #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */ diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 6e022cc712e6..880227b9f044 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -28,6 +28,13 @@ struct notifier_block; /* in notifier.h */ #define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */ #define VM_NO_HUGE_VMAP 0x00000400 /* force PAGE_SIZE pte mapping */ +#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ + !defined(CONFIG_KASAN_VMALLOC) +#define VM_DEFER_KMEMLEAK 0x00000800 /* defer kmemleak object creation */ +#else +#define VM_DEFER_KMEMLEAK 0 +#endif + /* * VM_KASAN is used slightly differently depending on CONFIG_KASAN_VMALLOC. * diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 4a4929b29a23..94136f84b449 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -498,7 +498,7 @@ void kasan_release_vmalloc(unsigned long start, unsigned long end, #else /* CONFIG_KASAN_VMALLOC */ -int kasan_module_alloc(void *addr, size_t size) +int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) { void *ret; size_t scaled_size; @@ -520,9 +520,14 @@ int kasan_module_alloc(void *addr, size_t size) __builtin_return_address(0)); if (ret) { + struct vm_struct *vm = find_vm_area(addr); __memset(ret, KASAN_SHADOW_INIT, shadow_size); - find_vm_area(addr)->flags |= VM_KASAN; + vm->flags |= VM_KASAN; kmemleak_ignore(ret); + + if (vm->flags & VM_DEFER_KMEMLEAK) + kmemleak_vmalloc(vm, size, gfp_mask); + return 0; } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index d2a00ad4e1dd..bf3c2fe8f528 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3074,7 +3074,8 @@ again: clear_vm_uninitialized_flag(area); size = PAGE_ALIGN(size); - kmemleak_vmalloc(area, size, gfp_mask); + if (!(vm_flags & VM_DEFER_KMEMLEAK)) + kmemleak_vmalloc(area, size, gfp_mask); return addr; -- cgit v1.2.3-59-g8ed1b From c4386bd8ee3a921c3c799b7197dc898ade76a453 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Fri, 14 Jan 2022 14:04:22 -0800 Subject: mm/memremap: add ZONE_DEVICE support for compound pages Add a new @vmemmap_shift property for struct dev_pagemap which specifies that a devmap is composed of a set of compound pages of order @vmemmap_shift, instead of base pages. When a compound page devmap is requested, all but the first page are initialised as tail pages instead of order-0 pages. For certain ZONE_DEVICE users like device-dax which have a fixed page size, this creates an opportunity to optimize GUP and GUP-fast walkers, treating it the same way as THP or hugetlb pages. Additionally, commit 7118fc2906e2 ("hugetlb: address ref count racing in prep_compound_gigantic_page") removed set_page_count() because the setting of page ref count to zero was redundant. devmap pages don't come from page allocator though and only head page refcount is used for compound pages, hence initialize tail page count to zero. Link: https://lkml.kernel.org/r/20211202204422.26777-5-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Dan Williams Cc: Christoph Hellwig Cc: Dave Jiang Cc: Jane Chu Cc: Jason Gunthorpe Cc: Jason Gunthorpe Cc: John Hubbard Cc: Jonathan Corbet Cc: Matthew Wilcox (Oracle) Cc: Mike Kravetz Cc: Muchun Song Cc: Naoya Horiguchi Cc: Vishal Verma Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memremap.h | 11 +++++++++++ mm/memremap.c | 18 ++++++++++++------ mm/page_alloc.c | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memremap.h b/include/linux/memremap.h index c0e9d35889e8..61a6a0e27359 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -99,6 +99,11 @@ struct dev_pagemap_ops { * @done: completion for @internal_ref * @type: memory type: see MEMORY_* in memory_hotplug.h * @flags: PGMAP_* flags to specify defailed behavior + * @vmemmap_shift: structural definition of how the vmemmap page metadata + * is populated, specifically the metadata page order. + * A zero value (default) uses base pages as the vmemmap metadata + * representation. A bigger value will set up compound struct pages + * of the requested order value. * @ops: method table * @owner: an opaque pointer identifying the entity that manages this * instance. Used by various helpers to make sure that no @@ -114,6 +119,7 @@ struct dev_pagemap { struct completion done; enum memory_type type; unsigned int flags; + unsigned long vmemmap_shift; const struct dev_pagemap_ops *ops; void *owner; int nr_range; @@ -130,6 +136,11 @@ static inline struct vmem_altmap *pgmap_altmap(struct dev_pagemap *pgmap) return NULL; } +static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap) +{ + return 1 << pgmap->vmemmap_shift; +} + #ifdef CONFIG_ZONE_DEVICE void *memremap_pages(struct dev_pagemap *pgmap, int nid); void memunmap_pages(struct dev_pagemap *pgmap); diff --git a/mm/memremap.c b/mm/memremap.c index 5a66a71ab591..a2869d8519a2 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -102,15 +102,22 @@ static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id) return (range->start + range_len(range)) >> PAGE_SHIFT; } -static unsigned long pfn_next(unsigned long pfn) +static unsigned long pfn_next(struct dev_pagemap *pgmap, unsigned long pfn) { - if (pfn % 1024 == 0) + if (pfn % (1024 << pgmap->vmemmap_shift)) cond_resched(); - return pfn + 1; + return pfn + pgmap_vmemmap_nr(pgmap); +} + +static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id) +{ + return (pfn_end(pgmap, range_id) - + pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift; } #define for_each_device_pfn(pfn, map, i) \ - for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); pfn = pfn_next(pfn)) + for (pfn = pfn_first(map, i); pfn < pfn_end(map, i); \ + pfn = pfn_next(map, pfn)) static void dev_pagemap_kill(struct dev_pagemap *pgmap) { @@ -295,8 +302,7 @@ static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params, memmap_init_zone_device(&NODE_DATA(nid)->node_zones[ZONE_DEVICE], PHYS_PFN(range->start), PHYS_PFN(range_len(range)), pgmap); - percpu_ref_get_many(pgmap->ref, pfn_end(pgmap, range_id) - - pfn_first(pgmap, range_id)); + percpu_ref_get_many(pgmap->ref, pfn_len(pgmap, range_id)); return 0; err_add_memory: diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 23045a2a1339..d59023a676ed 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6612,6 +6612,35 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, } } +static void __ref memmap_init_compound(struct page *head, + unsigned long head_pfn, + unsigned long zone_idx, int nid, + struct dev_pagemap *pgmap, + unsigned long nr_pages) +{ + unsigned long pfn, end_pfn = head_pfn + nr_pages; + unsigned int order = pgmap->vmemmap_shift; + + __SetPageHead(head); + for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { + struct page *page = pfn_to_page(pfn); + + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(head, pfn - head_pfn); + set_page_count(page, 0); + + /* + * The first tail page stores compound_mapcount_ptr() and + * compound_order() and the second tail page stores + * compound_pincount_ptr(). Call prep_compound_head() after + * the first and second tail pages have been initialized to + * not have the data overwritten. + */ + if (pfn == head_pfn + 2) + prep_compound_head(head, order); + } +} + void __ref memmap_init_zone_device(struct zone *zone, unsigned long start_pfn, unsigned long nr_pages, @@ -6620,6 +6649,7 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long pfn, end_pfn = start_pfn + nr_pages; struct pglist_data *pgdat = zone->zone_pgdat; struct vmem_altmap *altmap = pgmap_altmap(pgmap); + unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); unsigned long zone_idx = zone_idx(zone); unsigned long start = jiffies; int nid = pgdat->node_id; @@ -6637,10 +6667,16 @@ void __ref memmap_init_zone_device(struct zone *zone, nr_pages = end_pfn - start_pfn; } - for (pfn = start_pfn; pfn < end_pfn; pfn++) { + for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { struct page *page = pfn_to_page(pfn); __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + + if (pfns_per_compound == 1) + continue; + + memmap_init_compound(page, pfn, zone_idx, nid, pgmap, + pfns_per_compound); } pr_info("%s initialised %lu pages in %ums\n", __func__, -- cgit v1.2.3-59-g8ed1b From 3e9d80a891df3b1a5d77db47fa7fdf33ba71e5cb Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 14 Jan 2022 14:05:04 -0800 Subject: mm,fs: split dump_mapping() out from dump_page() dump_mapping() is a big chunk of dump_page(), and it'd be handy to be able to call it when we don't have a struct page. Split it out and move it to fs/inode.c. Take the opportunity to simplify some of the debug messages a little. Link: https://lkml.kernel.org/r/20211121121056.2870061-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski Acked-by: Michal Hocko Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/inode.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/fs.h | 1 + mm/debug.c | 52 ++-------------------------------------------------- 3 files changed, 52 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index 6b80a51129d5..980e7b7a5460 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -526,6 +526,55 @@ void __remove_inode_hash(struct inode *inode) } EXPORT_SYMBOL(__remove_inode_hash); +void dump_mapping(const struct address_space *mapping) +{ + struct inode *host; + const struct address_space_operations *a_ops; + struct hlist_node *dentry_first; + struct dentry *dentry_ptr; + struct dentry dentry; + unsigned long ino; + + /* + * If mapping is an invalid pointer, we don't want to crash + * accessing it, so probe everything depending on it carefully. + */ + if (get_kernel_nofault(host, &mapping->host) || + get_kernel_nofault(a_ops, &mapping->a_ops)) { + pr_warn("invalid mapping:%px\n", mapping); + return; + } + + if (!host) { + pr_warn("aops:%ps\n", a_ops); + return; + } + + if (get_kernel_nofault(dentry_first, &host->i_dentry.first) || + get_kernel_nofault(ino, &host->i_ino)) { + pr_warn("aops:%ps invalid inode:%px\n", a_ops, host); + return; + } + + if (!dentry_first) { + pr_warn("aops:%ps ino:%lx\n", a_ops, ino); + return; + } + + dentry_ptr = container_of(dentry_first, struct dentry, d_u.d_alias); + if (get_kernel_nofault(dentry, dentry_ptr)) { + pr_warn("aops:%ps ino:%lx invalid dentry:%px\n", + a_ops, ino, dentry_ptr); + return; + } + + /* + * if dentry is corrupted, the %pd handler may still crash, + * but it's unlikely that we reach here with a corrupt mapping + */ + pr_warn("aops:%ps ino:%lx dentry name:\"%pd\"\n", a_ops, ino, &dentry); +} + void clear_inode(struct inode *inode) { /* diff --git a/include/linux/fs.h b/include/linux/fs.h index bbf812ce89a8..5315fa68f751 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3152,6 +3152,7 @@ extern void unlock_new_inode(struct inode *); extern void discard_new_inode(struct inode *); extern unsigned int get_next_ino(void); extern void evict_inodes(struct super_block *sb); +void dump_mapping(const struct address_space *); /* * Userspace may rely on the the inode number being non-zero. For example, glibc diff --git a/mm/debug.c b/mm/debug.c index a05a39ff8fe4..bc9ac87f0e08 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -112,56 +112,8 @@ static void __dump_page(struct page *page) type = "ksm "; else if (PageAnon(page)) type = "anon "; - else if (mapping) { - struct inode *host; - const struct address_space_operations *a_ops; - struct hlist_node *dentry_first; - struct dentry *dentry_ptr; - struct dentry dentry; - unsigned long ino; - - /* - * mapping can be invalid pointer and we don't want to crash - * accessing it, so probe everything depending on it carefully - */ - if (get_kernel_nofault(host, &mapping->host) || - get_kernel_nofault(a_ops, &mapping->a_ops)) { - pr_warn("failed to read mapping contents, not a valid kernel address?\n"); - goto out_mapping; - } - - if (!host) { - pr_warn("aops:%ps\n", a_ops); - goto out_mapping; - } - - if (get_kernel_nofault(dentry_first, &host->i_dentry.first) || - get_kernel_nofault(ino, &host->i_ino)) { - pr_warn("aops:%ps with invalid host inode %px\n", - a_ops, host); - goto out_mapping; - } - - if (!dentry_first) { - pr_warn("aops:%ps ino:%lx\n", a_ops, ino); - goto out_mapping; - } - - dentry_ptr = container_of(dentry_first, struct dentry, d_u.d_alias); - if (get_kernel_nofault(dentry, dentry_ptr)) { - pr_warn("aops:%ps ino:%lx with invalid dentry %px\n", - a_ops, ino, dentry_ptr); - } else { - /* - * if dentry is corrupted, the %pd handler may still - * crash, but it's unlikely that we reach here with a - * corrupted struct page - */ - pr_warn("aops:%ps ino:%lx dentry name:\"%pd\"\n", - a_ops, ino, &dentry); - } - } -out_mapping: + else if (mapping) + dump_mapping(mapping); BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS + 1); pr_warn("%sflags: %pGp%s\n", type, &head->flags, -- cgit v1.2.3-59-g8ed1b From b6bf9abb0aa44e53ffe9c1e6e1d32568f5b25e4a Mon Sep 17 00:00:00 2001 From: Dan Schatzberg Date: Fri, 14 Jan 2022 14:05:35 -0800 Subject: mm/memcg: add oom_group_kill memory event Our container agent wants to know when a container exits if it was OOM killed or not to report to the user. We use memory.oom.group = 1 to ensure that OOM kills within the container's cgroup kill everything. Existing memory.events are insufficient for knowing if this triggered: 1) Our current approach reads memory.events oom_kill and reports the container was killed if the value is non-zero. This is erroneous in some cases where containers create their children cgroups with memory.oom.group=1 as such OOM kills will get counted against the container cgroup's oom_kill counter despite not actually OOM killing the entire container. 2) Reading memory.events.local will fail to identify OOM kills in leaf cgroups (that don't set memory.oom.group) within the container cgroup. This patch adds a new oom_group_kill event when memory.oom.group triggers to allow userspace to cleanly identify when an entire cgroup is oom killed. [schatzberg.dan@gmail.com: changes from Johannes and Chris] Link: https://lkml.kernel.org/r/20211213162511.2492267-1-schatzberg.dan@gmail.com Link: https://lkml.kernel.org/r/20211203162426.3375036-1-schatzberg.dan@gmail.com Signed-off-by: Dan Schatzberg Reviewed-by: Roman Gushchin Acked-by: Johannes Weiner Acked-by: Chris Down Reviewed-by: Shakeel Butt Acked-by: Michal Hocko Cc: Tejun Heo Cc: Zefan Li Cc: Jonathan Corbet Cc: Vladimir Davydov Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Cc: Alex Shi Cc: Wei Yang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/cgroup-v2.rst | 3 +++ include/linux/memcontrol.h | 1 + mm/memcontrol.c | 2 ++ mm/oom_kill.c | 1 + 4 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 2aeb7ae8b393..8269bfa240f4 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1268,6 +1268,9 @@ PAGE_SIZE multiple when read back. The number of processes belonging to this cgroup killed by any kind of OOM killer. + oom_group_kill + The number of times a group OOM has occurred. + memory.events.local Similar to memory.events but the fields in the file are local to the cgroup i.e. not hierarchical. The file modified event diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 0c5c403f4be6..951f24f42147 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -42,6 +42,7 @@ enum memcg_memory_event { MEMCG_MAX, MEMCG_OOM, MEMCG_OOM_KILL, + MEMCG_OOM_GROUP_KILL, MEMCG_SWAP_HIGH, MEMCG_SWAP_MAX, MEMCG_SWAP_FAIL, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index bfe9bdec192b..2d39d58baccf 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6318,6 +6318,8 @@ static void __memory_events_show(struct seq_file *m, atomic_long_t *events) seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); seq_printf(m, "oom_kill %lu\n", atomic_long_read(&events[MEMCG_OOM_KILL])); + seq_printf(m, "oom_group_kill %lu\n", + atomic_long_read(&events[MEMCG_OOM_GROUP_KILL])); } static int memory_events_show(struct seq_file *m, void *v) diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 1ddabefcfb5a..e52ce0b1465d 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -994,6 +994,7 @@ static void oom_kill_process(struct oom_control *oc, const char *message) * If necessary, kill all tasks in the selected memory cgroup. */ if (oom_group) { + memcg_memory_event(oom_group, MEMCG_OOM_GROUP_KILL); mem_cgroup_print_oom_group(oom_group); mem_cgroup_scan_tasks(oom_group, oom_kill_memcg_member, (void *)message); -- cgit v1.2.3-59-g8ed1b From 4e5aa1f4c2b489bc6f3ab5ca54747b18a847289d Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 14 Jan 2022 14:05:45 -0800 Subject: memcg: add per-memcg vmalloc stat The kvmalloc* allocation functions can fallback to vmalloc allocations and more often on long running machines. In addition the kernel does have __GFP_ACCOUNT kvmalloc* calls. So, often on long running machines, the memory.stat does not tell the complete picture which type of memory is charged to the memcg. So add a per-memcg vmalloc stat. [shakeelb@google.com: page_memcg() within rcu lock, per Muchun] Link: https://lkml.kernel.org/r/20211222052457.1960701-1-shakeelb@google.com [akpm@linux-foundation.org: remove cast, per Muchun] [shakeelb@google.com: remove area->page[0] checks and move to page by page accounting per Michal] Link: https://lkml.kernel.org/r/20220104222341.3972772-1-shakeelb@google.com Link: https://lkml.kernel.org/r/20211221215336.1922823-1-shakeelb@google.com Signed-off-by: Shakeel Butt Acked-by: Roman Gushchin Reviewed-by: Muchun Song Acked-by: Michal Hocko Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/cgroup-v2.rst | 3 +++ include/linux/memcontrol.h | 21 +++++++++++++++++++++ mm/memcontrol.c | 1 + mm/vmalloc.c | 13 +++++++++++-- 4 files changed, 36 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 8269bfa240f4..4f400b03dddf 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1314,6 +1314,9 @@ PAGE_SIZE multiple when read back. sock (npn) Amount of memory used in network transmission buffers + vmalloc (npn) + Amount of memory used for vmap backed memory. + shmem Amount of cached filesystem data that is swap-backed, such as tmpfs, shm segments, shared anonymous mmap()s diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 951f24f42147..0131e5574c88 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -33,6 +33,7 @@ enum memcg_stat_item { MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS, MEMCG_SOCK, MEMCG_PERCPU_B, + MEMCG_VMALLOC, MEMCG_NR_STAT, }; @@ -992,6 +993,21 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, local_irq_restore(flags); } +static inline void mod_memcg_page_state(struct page *page, + int idx, int val) +{ + struct mem_cgroup *memcg; + + if (mem_cgroup_disabled()) + return; + + rcu_read_lock(); + memcg = page_memcg(page); + if (memcg) + mod_memcg_state(memcg, idx, val); + rcu_read_unlock(); +} + static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) { return READ_ONCE(memcg->vmstats.state[idx]); @@ -1447,6 +1463,11 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, { } +static inline void mod_memcg_page_state(struct page *page, + int idx, int val) +{ +} + static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) { return 0; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 88e1be912aa7..c9ddd02dc5de 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1375,6 +1375,7 @@ static const struct memory_stat memory_stats[] = { { "pagetables", NR_PAGETABLE }, { "percpu", MEMCG_PERCPU_B }, { "sock", MEMCG_SOCK }, + { "vmalloc", MEMCG_VMALLOC }, { "shmem", NR_SHMEM }, { "file_mapped", NR_FILE_MAPPED }, { "file_dirty", NR_FILE_DIRTY }, diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bf3c2fe8f528..80c6de4c425f 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -2623,12 +2624,13 @@ static void __vunmap(const void *addr, int deallocate_pages) if (deallocate_pages) { unsigned int page_order = vm_area_page_order(area); - int i; + int i, step = 1U << page_order; - for (i = 0; i < area->nr_pages; i += 1U << page_order) { + for (i = 0; i < area->nr_pages; i += step) { struct page *page = area->pages[i]; BUG_ON(!page); + mod_memcg_page_state(page, MEMCG_VMALLOC, -step); __free_pages(page, page_order); cond_resched(); } @@ -2955,6 +2957,13 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, page_order, nr_small_pages, area->pages); atomic_long_add(area->nr_pages, &nr_vmalloc_pages); + if (gfp_mask & __GFP_ACCOUNT) { + int i, step = 1U << page_order; + + for (i = 0; i < area->nr_pages; i += step) + mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, + step); + } /* * If not enough pages were obtained to accomplish an -- cgit v1.2.3-59-g8ed1b From 9a10064f5625d5572c3626c1516e0bebc6c9fe9b Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 14 Jan 2022 14:05:59 -0800 Subject: mm: add a field to store names for private anonymous memory In many userspace applications, and especially in VM based applications like Android uses heavily, there are multiple different allocators in use. At a minimum there is libc malloc and the stack, and in many cases there are libc malloc, the stack, direct syscalls to mmap anonymous memory, and multiple VM heaps (one for small objects, one for big objects, etc.). Each of these layers usually has its own tools to inspect its usage; malloc by compiling a debug version, the VM through heap inspection tools, and for direct syscalls there is usually no way to track them. On Android we heavily use a set of tools that use an extended version of the logic covered in Documentation/vm/pagemap.txt to walk all pages mapped in userspace and slice their usage by process, shared (COW) vs. unique mappings, backing, etc. This can account for real physical memory usage even in cases like fork without exec (which Android uses heavily to share as many private COW pages as possible between processes), Kernel SamePage Merging, and clean zero pages. It produces a measurement of the pages that only exist in that process (USS, for unique), and a measurement of the physical memory usage of that process with the cost of shared pages being evenly split between processes that share them (PSS). If all anonymous memory is indistinguishable then figuring out the real physical memory usage (PSS) of each heap requires either a pagemap walking tool that can understand the heap debugging of every layer, or for every layer's heap debugging tools to implement the pagemap walking logic, in which case it is hard to get a consistent view of memory across the whole system. Tracking the information in userspace leads to all sorts of problems. It either needs to be stored inside the process, which means every process has to have an API to export its current heap information upon request, or it has to be stored externally in a filesystem that somebody needs to clean up on crashes. It needs to be readable while the process is still running, so it has to have some sort of synchronization with every layer of userspace. Efficiently tracking the ranges requires reimplementing something like the kernel vma trees, and linking to it from every layer of userspace. It requires more memory, more syscalls, more runtime cost, and more complexity to separately track regions that the kernel is already tracking. This patch adds a field to /proc/pid/maps and /proc/pid/smaps to show a userspace-provided name for anonymous vmas. The names of named anonymous vmas are shown in /proc/pid/maps and /proc/pid/smaps as [anon:]. Userspace can set the name for a region of memory by calling prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, start, len, (unsigned long)name) Setting the name to NULL clears it. The name length limit is 80 bytes including NUL-terminator and is checked to contain only printable ascii characters (including space), except '[',']','\','$' and '`'. Ascii strings are being used to have a descriptive identifiers for vmas, which can be understood by the users reading /proc/pid/maps or /proc/pid/smaps. Names can be standardized for a given system and they can include some variable parts such as the name of the allocator or a library, tid of the thread using it, etc. The name is stored in a pointer in the shared union in vm_area_struct that points to a null terminated string. Anonymous vmas with the same name (equivalent strings) and are otherwise mergeable will be merged. The name pointers are not shared between vmas even if they contain the same name. The name pointer is stored in a union with fields that are only used on file-backed mappings, so it does not increase memory usage. CONFIG_ANON_VMA_NAME kernel configuration is introduced to enable this feature. It keeps the feature disabled by default to prevent any additional memory overhead and to avoid confusing procfs parsers on systems which are not ready to support named anonymous vmas. The patch is based on the original patch developed by Colin Cross, more specifically on its latest version [1] posted upstream by Sumit Semwal. It used a userspace pointer to store vma names. In that design, name pointers could be shared between vmas. However during the last upstreaming attempt, Kees Cook raised concerns [2] about this approach and suggested to copy the name into kernel memory space, perform validity checks [3] and store as a string referenced from vm_area_struct. One big concern is about fork() performance which would need to strdup anonymous vma names. Dave Hansen suggested experimenting with worst-case scenario of forking a process with 64k vmas having longest possible names [4]. I ran this experiment on an ARM64 Android device and recorded a worst-case regression of almost 40% when forking such a process. This regression is addressed in the followup patch which replaces the pointer to a name with a refcounted structure that allows sharing the name pointer between vmas of the same name. Instead of duplicating the string during fork() or when splitting a vma it increments the refcount. [1] https://lore.kernel.org/linux-mm/20200901161459.11772-4-sumit.semwal@linaro.org/ [2] https://lore.kernel.org/linux-mm/202009031031.D32EF57ED@keescook/ [3] https://lore.kernel.org/linux-mm/202009031022.3834F692@keescook/ [4] https://lore.kernel.org/linux-mm/5d0358ab-8c47-2f5f-8e43-23b89d6a8e95@intel.com/ Changes for prctl(2) manual page (in the options section): PR_SET_VMA Sets an attribute specified in arg2 for virtual memory areas starting from the address specified in arg3 and spanning the size specified in arg4. arg5 specifies the value of the attribute to be set. Note that assigning an attribute to a virtual memory area might prevent it from being merged with adjacent virtual memory areas due to the difference in that attribute's value. Currently, arg2 must be one of: PR_SET_VMA_ANON_NAME Set a name for anonymous virtual memory areas. arg5 should be a pointer to a null-terminated string containing the name. The name length including null byte cannot exceed 80 bytes. If arg5 is NULL, the name of the appropriate anonymous virtual memory areas will be reset. The name can contain only printable ascii characters (including space), except '[',']','\','$' and '`'. This feature is available only if the kernel is built with the CONFIG_ANON_VMA_NAME option enabled. [surenb@google.com: docs: proc.rst: /proc/PID/maps: fix malformed table] Link: https://lkml.kernel.org/r/20211123185928.2513763-1-surenb@google.com [surenb: rebased over v5.15-rc6, replaced userpointer with a kernel copy, added input sanitization and CONFIG_ANON_VMA_NAME config. The bulk of the work here was done by Colin Cross, therefore, with his permission, keeping him as the author] Link: https://lkml.kernel.org/r/20211019215511.3771969-2-surenb@google.com Signed-off-by: Colin Cross Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Cc: Stephen Rothwell Cc: Al Viro Cc: Cyrill Gorcunov Cc: Dave Hansen Cc: David Rientjes Cc: "Eric W. Biederman" Cc: Hugh Dickins Cc: Ingo Molnar Cc: Jan Glauber Cc: Johannes Weiner Cc: John Stultz Cc: Mel Gorman Cc: Minchan Kim Cc: Oleg Nesterov Cc: Pekka Enberg Cc: Peter Zijlstra Cc: Rob Landley Cc: "Serge E. Hallyn" Cc: Shaohua Li Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/filesystems/proc.rst | 6 +- fs/proc/task_mmu.c | 12 +++- fs/userfaultfd.c | 7 +- include/linux/mm.h | 13 +++- include/linux/mm_types.h | 64 ++++++++++++++++-- include/uapi/linux/prctl.h | 3 + kernel/fork.c | 2 + kernel/sys.c | 63 ++++++++++++++++++ mm/Kconfig | 14 ++++ mm/madvise.c | 129 +++++++++++++++++++++++++++++++++++-- mm/mempolicy.c | 3 +- mm/mlock.c | 2 +- mm/mmap.c | 38 ++++++----- mm/mprotect.c | 2 +- 14 files changed, 324 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 8d7f141c6fc7..061744c436d9 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -426,12 +426,14 @@ with the memory region, as the case would be with BSS (uninitialized data). The "pathname" shows the name associated file for this mapping. If the mapping is not associated with a file: - ======= ==================================== + ============= ==================================== [heap] the heap of the program [stack] the stack of the main process [vdso] the "virtual dynamic shared object", the kernel system call handler - ======= ==================================== + [anon:] an anonymous mapping that has been + named by userspace + ============= ==================================== or if empty, the mapping is anonymous. diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index ad667dbc96f5..e6998652fd67 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -308,6 +308,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) name = arch_vma_name(vma); if (!name) { + const char *anon_name; + if (!mm) { name = "[vdso]"; goto done; @@ -319,8 +321,16 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) goto done; } - if (is_stack(vma)) + if (is_stack(vma)) { name = "[stack]"; + goto done; + } + + anon_name = vma_anon_name(vma); + if (anon_name) { + seq_pad(m, ' '); + seq_printf(m, "[anon:%s]", anon_name); + } } done: diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 22bf14ab2d16..5b2af7b82776 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -877,7 +877,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - NULL_VM_UFFD_CTX); + NULL_VM_UFFD_CTX, vma_anon_name(vma)); if (prev) vma = prev; else @@ -1436,7 +1436,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, prev = vma_merge(mm, prev, start, vma_end, new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - ((struct vm_userfaultfd_ctx){ ctx })); + ((struct vm_userfaultfd_ctx){ ctx }), + vma_anon_name(vma)); if (prev) { vma = prev; goto next; @@ -1613,7 +1614,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, prev = vma_merge(mm, prev, start, vma_end, new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, vma_policy(vma), - NULL_VM_UFFD_CTX); + NULL_VM_UFFD_CTX, vma_anon_name(vma)); if (prev) { vma = prev; goto next; diff --git a/include/linux/mm.h b/include/linux/mm.h index a7e4a9e7d807..7000442984b9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2658,7 +2658,7 @@ static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start, extern struct vm_area_struct *vma_merge(struct mm_struct *, struct vm_area_struct *prev, unsigned long addr, unsigned long end, unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t, - struct mempolicy *, struct vm_userfaultfd_ctx); + struct mempolicy *, struct vm_userfaultfd_ctx, const char *); extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); extern int __split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); @@ -3391,5 +3391,16 @@ static inline int seal_check_future_write(int seals, struct vm_area_struct *vma) return 0; } +#ifdef CONFIG_ANON_VMA_NAME +int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, + unsigned long len_in, const char *name); +#else +static inline int +madvise_set_anon_name(struct mm_struct *mm, unsigned long start, + unsigned long len_in, const char *name) { + return 0; +} +#endif + #endif /* __KERNEL__ */ #endif /* _LINUX_MM_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c3a6e6209600..799e2ee626b2 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -426,11 +426,19 @@ struct vm_area_struct { /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. + * + * For private anonymous mappings, a pointer to a null terminated string + * containing the name given to the vma, or NULL if unnamed. */ - struct { - struct rb_node rb; - unsigned long rb_subtree_last; - } shared; + + union { + struct { + struct rb_node rb; + unsigned long rb_subtree_last; + } shared; + /* Serialized by mmap_sem. */ + char *anon_name; + }; /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma @@ -875,4 +883,52 @@ typedef struct { unsigned long val; } swp_entry_t; +#ifdef CONFIG_ANON_VMA_NAME +/* + * mmap_lock should be read-locked when calling vma_anon_name() and while using + * the returned pointer. + */ +extern const char *vma_anon_name(struct vm_area_struct *vma); + +/* + * mmap_lock should be read-locked for orig_vma->vm_mm. + * mmap_lock should be write-locked for new_vma->vm_mm or new_vma should be + * isolated. + */ +extern void dup_vma_anon_name(struct vm_area_struct *orig_vma, + struct vm_area_struct *new_vma); + +/* + * mmap_lock should be write-locked or vma should have been isolated under + * write-locked mmap_lock protection. + */ +extern void free_vma_anon_name(struct vm_area_struct *vma); + +/* mmap_lock should be read-locked */ +static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, + const char *name) +{ + const char *vma_name = vma_anon_name(vma); + + /* either both NULL, or pointers to same string */ + if (vma_name == name) + return true; + + return name && vma_name && !strcmp(name, vma_name); +} +#else /* CONFIG_ANON_VMA_NAME */ +static inline const char *vma_anon_name(struct vm_area_struct *vma) +{ + return NULL; +} +static inline void dup_vma_anon_name(struct vm_area_struct *orig_vma, + struct vm_area_struct *new_vma) {} +static inline void free_vma_anon_name(struct vm_area_struct *vma) {} +static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, + const char *name) +{ + return true; +} +#endif /* CONFIG_ANON_VMA_NAME */ + #endif /* _LINUX_MM_TYPES_H */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index bb73e9a0b24f..e998764f0262 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -272,4 +272,7 @@ struct prctl_mm_map { # define PR_SCHED_CORE_SCOPE_THREAD_GROUP 1 # define PR_SCHED_CORE_SCOPE_PROCESS_GROUP 2 +#define PR_SET_VMA 0x53564d41 +# define PR_SET_VMA_ANON_NAME 0 + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 3244cc56b697..4cf20b5f2da3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -365,12 +365,14 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) *new = data_race(*orig); INIT_LIST_HEAD(&new->anon_vma_chain); new->vm_next = new->vm_prev = NULL; + dup_vma_anon_name(orig, new); } return new; } void vm_area_free(struct vm_area_struct *vma) { + free_vma_anon_name(vma); kmem_cache_free(vm_area_cachep, vma); } diff --git a/kernel/sys.c b/kernel/sys.c index 8fdac0d90504..2450a9f33cb0 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2261,6 +2261,66 @@ int __weak arch_prctl_spec_ctrl_set(struct task_struct *t, unsigned long which, #define PR_IO_FLUSHER (PF_MEMALLOC_NOIO | PF_LOCAL_THROTTLE) +#ifdef CONFIG_ANON_VMA_NAME + +#define ANON_VMA_NAME_MAX_LEN 80 +#define ANON_VMA_NAME_INVALID_CHARS "\\`$[]" + +static inline bool is_valid_name_char(char ch) +{ + /* printable ascii characters, excluding ANON_VMA_NAME_INVALID_CHARS */ + return ch > 0x1f && ch < 0x7f && + !strchr(ANON_VMA_NAME_INVALID_CHARS, ch); +} + +static int prctl_set_vma(unsigned long opt, unsigned long addr, + unsigned long size, unsigned long arg) +{ + struct mm_struct *mm = current->mm; + const char __user *uname; + char *name, *pch; + int error; + + switch (opt) { + case PR_SET_VMA_ANON_NAME: + uname = (const char __user *)arg; + if (uname) { + name = strndup_user(uname, ANON_VMA_NAME_MAX_LEN); + + if (IS_ERR(name)) + return PTR_ERR(name); + + for (pch = name; *pch != '\0'; pch++) { + if (!is_valid_name_char(*pch)) { + kfree(name); + return -EINVAL; + } + } + } else { + /* Reset the name */ + name = NULL; + } + + mmap_write_lock(mm); + error = madvise_set_anon_name(mm, addr, size, name); + mmap_write_unlock(mm); + kfree(name); + break; + default: + error = -EINVAL; + } + + return error; +} + +#else /* CONFIG_ANON_VMA_NAME */ +static int prctl_set_vma(unsigned long opt, unsigned long start, + unsigned long size, unsigned long arg) +{ + return -EINVAL; +} +#endif /* CONFIG_ANON_VMA_NAME */ + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2530,6 +2590,9 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = sched_core_share_pid(arg2, arg3, arg4, arg5); break; #endif + case PR_SET_VMA: + error = prctl_set_vma(arg2, arg3, arg4, arg5); + break; default: error = -EINVAL; break; diff --git a/mm/Kconfig b/mm/Kconfig index 356f4f2c779e..53d7485fc38f 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -900,6 +900,20 @@ config IO_MAPPING config SECRETMEM def_bool ARCH_HAS_SET_DIRECT_MAP && !EMBEDDED +config ANON_VMA_NAME + bool "Anonymous VMA name support" + depends on PROC_FS && ADVISE_SYSCALLS && MMU + + help + Allow naming anonymous virtual memory areas. + + This feature allows assigning names to virtual memory areas. Assigned + names can be later retrieved from /proc/pid/maps and /proc/pid/smaps + and help identifying individual anonymous memory areas. + Assigning a name to anonymous virtual memory area might prevent that + area from being merged with adjacent virtual memory areas due to the + difference in their name. + source "mm/damon/Kconfig" endmenu diff --git a/mm/madvise.c b/mm/madvise.c index 4b9c5509990c..413bbc6e40a0 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -62,19 +63,84 @@ static int madvise_need_mmap_write(int behavior) } } +#ifdef CONFIG_ANON_VMA_NAME +static inline bool has_vma_anon_name(struct vm_area_struct *vma) +{ + return !vma->vm_file && vma->anon_name; +} + +const char *vma_anon_name(struct vm_area_struct *vma) +{ + if (!has_vma_anon_name(vma)) + return NULL; + + mmap_assert_locked(vma->vm_mm); + + return vma->anon_name; +} + +void dup_vma_anon_name(struct vm_area_struct *orig_vma, + struct vm_area_struct *new_vma) +{ + if (!has_vma_anon_name(orig_vma)) + return; + + new_vma->anon_name = kstrdup(orig_vma->anon_name, GFP_KERNEL); +} + +void free_vma_anon_name(struct vm_area_struct *vma) +{ + if (!has_vma_anon_name(vma)) + return; + + kfree(vma->anon_name); + vma->anon_name = NULL; +} + +/* mmap_lock should be write-locked */ +static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name) +{ + if (!name) { + free_vma_anon_name(vma); + return 0; + } + + if (vma->anon_name) { + /* Same name, nothing to do here */ + if (!strcmp(name, vma->anon_name)) + return 0; + + free_vma_anon_name(vma); + } + vma->anon_name = kstrdup(name, GFP_KERNEL); + if (!vma->anon_name) + return -ENOMEM; + + return 0; +} +#else /* CONFIG_ANON_VMA_NAME */ +static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name) +{ + if (name) + return -EINVAL; + + return 0; +} +#endif /* CONFIG_ANON_VMA_NAME */ /* * Update the vm_flags on region of a vma, splitting it or merging it as * necessary. Must be called with mmap_sem held for writing; */ static int madvise_update_vma(struct vm_area_struct *vma, struct vm_area_struct **prev, unsigned long start, - unsigned long end, unsigned long new_flags) + unsigned long end, unsigned long new_flags, + const char *name) { struct mm_struct *mm = vma->vm_mm; int error; pgoff_t pgoff; - if (new_flags == vma->vm_flags) { + if (new_flags == vma->vm_flags && is_same_vma_anon_name(vma, name)) { *prev = vma; return 0; } @@ -82,7 +148,7 @@ static int madvise_update_vma(struct vm_area_struct *vma, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, name); if (*prev) { vma = *prev; goto success; @@ -111,6 +177,11 @@ success: * vm_flags is protected by the mmap_lock held in write mode. */ vma->vm_flags = new_flags; + if (!vma->vm_file) { + error = replace_vma_anon_name(vma, name); + if (error) + return error; + } return 0; } @@ -938,7 +1009,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, break; } - error = madvise_update_vma(vma, prev, start, end, new_flags); + error = madvise_update_vma(vma, prev, start, end, new_flags, + vma_anon_name(vma)); out: /* @@ -1118,6 +1190,55 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, return unmapped_error; } +#ifdef CONFIG_ANON_VMA_NAME +static int madvise_vma_anon_name(struct vm_area_struct *vma, + struct vm_area_struct **prev, + unsigned long start, unsigned long end, + unsigned long name) +{ + int error; + + /* Only anonymous mappings can be named */ + if (vma->vm_file) + return -EBADF; + + error = madvise_update_vma(vma, prev, start, end, vma->vm_flags, + (const char *)name); + + /* + * madvise() returns EAGAIN if kernel resources, such as + * slab, are temporarily unavailable. + */ + if (error == -ENOMEM) + error = -EAGAIN; + return error; +} + +int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, + unsigned long len_in, const char *name) +{ + unsigned long end; + unsigned long len; + + if (start & ~PAGE_MASK) + return -EINVAL; + len = (len_in + ~PAGE_MASK) & PAGE_MASK; + + /* Check to see whether len was rounded up from small -ve to zero */ + if (len_in && !len) + return -EINVAL; + + end = start + len; + if (end < start) + return -EINVAL; + + if (end == start) + return 0; + + return madvise_walk_vmas(mm, start, end, (unsigned long)name, + madvise_vma_anon_name); +} +#endif /* CONFIG_ANON_VMA_NAME */ /* * The madvise(2) system call. * diff --git a/mm/mempolicy.c b/mm/mempolicy.c index f6248affaf38..679f47b3a079 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -810,7 +810,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, ((vmstart - vma->vm_start) >> PAGE_SHIFT); prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, vma->anon_vma, vma->vm_file, pgoff, - new_pol, vma->vm_userfaultfd_ctx); + new_pol, vma->vm_userfaultfd_ctx, + vma_anon_name(vma)); if (prev) { vma = prev; next = vma->vm_next; diff --git a/mm/mlock.c b/mm/mlock.c index e263d62ae2d0..8f584eddd305 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -512,7 +512,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_anon_name(vma)); if (*prev) { vma = *prev; goto success; diff --git a/mm/mmap.c b/mm/mmap.c index bfb0ea164a90..85edb0011453 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1029,7 +1029,8 @@ again: */ static inline int is_mergeable_vma(struct vm_area_struct *vma, struct file *file, unsigned long vm_flags, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char *anon_name) { /* * VM_SOFTDIRTY should not prevent from VMA merging, if we @@ -1047,6 +1048,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma, return 0; if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx)) return 0; + if (!is_same_vma_anon_name(vma, anon_name)) + return 0; return 1; } @@ -1079,9 +1082,10 @@ static int can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char *anon_name) { - if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && + if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { if (vma->vm_pgoff == vm_pgoff) return 1; @@ -1100,9 +1104,10 @@ static int can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char *anon_name) { - if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) && + if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) && is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) { pgoff_t vm_pglen; vm_pglen = vma_pages(vma); @@ -1113,9 +1118,9 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, } /* - * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out - * whether that can be merged with its predecessor or its successor. - * Or both (it neatly fills a hole). + * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name), + * figure out whether that can be merged with its predecessor or its + * successor. Or both (it neatly fills a hole). * * In most cases - when called for mmap, brk or mremap - [addr,end) is * certain not to be mapped by the time vma_merge is called; but when @@ -1160,7 +1165,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, unsigned long end, unsigned long vm_flags, struct anon_vma *anon_vma, struct file *file, pgoff_t pgoff, struct mempolicy *policy, - struct vm_userfaultfd_ctx vm_userfaultfd_ctx) + struct vm_userfaultfd_ctx vm_userfaultfd_ctx, + const char *anon_name) { pgoff_t pglen = (end - addr) >> PAGE_SHIFT; struct vm_area_struct *area, *next; @@ -1190,7 +1196,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, mpol_equal(vma_policy(prev), policy) && can_vma_merge_after(prev, vm_flags, anon_vma, file, pgoff, - vm_userfaultfd_ctx)) { + vm_userfaultfd_ctx, anon_name)) { /* * OK, it can. Can we now merge in the successor as well? */ @@ -1199,7 +1205,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, - vm_userfaultfd_ctx) && + vm_userfaultfd_ctx, anon_name) && is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) { /* cases 1, 6 */ @@ -1222,7 +1228,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, mpol_equal(policy, vma_policy(next)) && can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, - vm_userfaultfd_ctx)) { + vm_userfaultfd_ctx, anon_name)) { if (prev && addr < prev->vm_end) /* case 4 */ err = __vma_adjust(prev, prev->vm_start, addr, prev->vm_pgoff, NULL, next); @@ -1754,7 +1760,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, * Can we just expand an old mapping? */ vma = vma_merge(mm, prev, addr, addr + len, vm_flags, - NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX); + NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); if (vma) goto out; @@ -1803,7 +1809,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, */ if (unlikely(vm_flags != vma->vm_flags && prev)) { merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, - NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX); + NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); if (merge) { /* ->mmap() can change vma->vm_file and fput the original file. So * fput the vma->vm_file here or we would add an extra fput for file @@ -3056,7 +3062,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla /* Can we just expand an old private anonymous mapping? */ vma = vma_merge(mm, prev, addr, addr + len, flags, - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX); + NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); if (vma) goto out; @@ -3249,7 +3255,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_anon_name(vma)); if (new_vma) { /* * Source vma may have been merged into new_vma diff --git a/mm/mprotect.c b/mm/mprotect.c index e552f5e0ccbd..0138dfcdb1d8 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -464,7 +464,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *pprev = vma_merge(mm, *pprev, start, end, newflags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + vma->vm_userfaultfd_ctx, vma_anon_name(vma)); if (*pprev) { vma = *pprev; VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); -- cgit v1.2.3-59-g8ed1b From 78db3412833dc9c479cd17412035f216cfd01a29 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 14 Jan 2022 14:06:03 -0800 Subject: mm: add anonymous vma name refcounting While forking a process with high number (64K) of named anonymous vmas the overhead caused by strdup() is noticeable. Experiments with ARM64 Android device show up to 40% performance regression when forking a process with 64k unpopulated anonymous vmas using the max name lengths vs the same process with the same number of anonymous vmas having no name. Introduce anon_vma_name refcounted structure to avoid the overhead of copying vma names during fork() and when splitting named anonymous vmas. When a vma is duplicated, instead of copying the name we increment the refcount of this structure. Multiple vmas can point to the same anon_vma_name as long as they increment the refcount. The name member of anon_vma_name structure is assigned at structure allocation time and is never changed. If vma name changes then the refcount of the original structure is dropped, a new anon_vma_name structure is allocated to hold the new name and the vma pointer is updated to point to the new structure. With this approach the fork() performance regressions is reduced 3-4x times and with usecases using more reasonable number of VMAs (a few thousand) the regressions is not measurable. Link: https://lkml.kernel.org/r/20211019215511.3771969-3-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Kees Cook Cc: Al Viro Cc: Colin Cross Cc: Cyrill Gorcunov Cc: Dave Hansen Cc: David Rientjes Cc: "Eric W. Biederman" Cc: Hugh Dickins Cc: Ingo Molnar Cc: Jan Glauber Cc: Johannes Weiner Cc: John Stultz Cc: Mel Gorman Cc: Minchan Kim Cc: Oleg Nesterov Cc: Pekka Enberg Cc: Peter Zijlstra Cc: Rob Landley Cc: "Serge E. Hallyn" Cc: Shaohua Li Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_types.h | 9 ++++++++- mm/madvise.c | 42 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 799e2ee626b2..449b6eafc695 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -386,6 +387,12 @@ struct vm_userfaultfd_ctx { struct vm_userfaultfd_ctx {}; #endif /* CONFIG_USERFAULTFD */ +struct anon_vma_name { + struct kref kref; + /* The name needs to be at the end because it is dynamically sized. */ + char name[]; +}; + /* * This struct describes a virtual memory area. There is one of these * per VM-area/task. A VM area is any part of the process virtual memory @@ -437,7 +444,7 @@ struct vm_area_struct { unsigned long rb_subtree_last; } shared; /* Serialized by mmap_sem. */ - char *anon_name; + struct anon_vma_name *anon_name; }; /* diff --git a/mm/madvise.c b/mm/madvise.c index 413bbc6e40a0..c63aacbbfa78 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -64,6 +64,29 @@ static int madvise_need_mmap_write(int behavior) } #ifdef CONFIG_ANON_VMA_NAME +static struct anon_vma_name *anon_vma_name_alloc(const char *name) +{ + struct anon_vma_name *anon_name; + size_t count; + + /* Add 1 for NUL terminator at the end of the anon_name->name */ + count = strlen(name) + 1; + anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL); + if (anon_name) { + kref_init(&anon_name->kref); + memcpy(anon_name->name, name, count); + } + + return anon_name; +} + +static void vma_anon_name_free(struct kref *kref) +{ + struct anon_vma_name *anon_name = + container_of(kref, struct anon_vma_name, kref); + kfree(anon_name); +} + static inline bool has_vma_anon_name(struct vm_area_struct *vma) { return !vma->vm_file && vma->anon_name; @@ -76,7 +99,7 @@ const char *vma_anon_name(struct vm_area_struct *vma) mmap_assert_locked(vma->vm_mm); - return vma->anon_name; + return vma->anon_name->name; } void dup_vma_anon_name(struct vm_area_struct *orig_vma, @@ -85,34 +108,41 @@ void dup_vma_anon_name(struct vm_area_struct *orig_vma, if (!has_vma_anon_name(orig_vma)) return; - new_vma->anon_name = kstrdup(orig_vma->anon_name, GFP_KERNEL); + kref_get(&orig_vma->anon_name->kref); + new_vma->anon_name = orig_vma->anon_name; } void free_vma_anon_name(struct vm_area_struct *vma) { + struct anon_vma_name *anon_name; + if (!has_vma_anon_name(vma)) return; - kfree(vma->anon_name); + anon_name = vma->anon_name; vma->anon_name = NULL; + kref_put(&anon_name->kref, vma_anon_name_free); } /* mmap_lock should be write-locked */ static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name) { + const char *anon_name; + if (!name) { free_vma_anon_name(vma); return 0; } - if (vma->anon_name) { + anon_name = vma_anon_name(vma); + if (anon_name) { /* Same name, nothing to do here */ - if (!strcmp(name, vma->anon_name)) + if (!strcmp(name, anon_name)) return 0; free_vma_anon_name(vma); } - vma->anon_name = kstrdup(name, GFP_KERNEL); + vma->anon_name = anon_vma_name_alloc(name); if (!vma->anon_name) return -ENOMEM; -- cgit v1.2.3-59-g8ed1b From 17fca131cee21724ee953a17c185c14e9533af5b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 14 Jan 2022 14:06:07 -0800 Subject: mm: move anon_vma declarations to linux/mm_inline.h The patch to add anonymous vma names causes a build failure in some configurations: include/linux/mm_types.h: In function 'is_same_vma_anon_name': include/linux/mm_types.h:924:37: error: implicit declaration of function 'strcmp' [-Werror=implicit-function-declaration] 924 | return name && vma_name && !strcmp(name, vma_name); | ^~~~~~ include/linux/mm_types.h:22:1: note: 'strcmp' is defined in header ''; did you forget to '#include '? This should not really be part of linux/mm_types.h in the first place, as that header is meant to only contain structure defintions and need a minimum set of indirect includes itself. While the header clearly includes more than it should at this point, let's not make it worse by including string.h as well, which would pull in the expensive (compile-speed wise) fortify-string logic. Move the new functions into a separate header that only needs to be included in a couple of locations. Link: https://lkml.kernel.org/r/20211207125710.2503446-1-arnd@kernel.org Fixes: "mm: add a field to store names for private anonymous memory" Signed-off-by: Arnd Bergmann Cc: Al Viro Cc: Colin Cross Cc: Eric Biederman Cc: Kees Cook Cc: Matthew Wilcox (Oracle) Cc: Peter Xu Cc: Peter Zijlstra (Intel) Cc: Stephen Rothwell Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yu Zhao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 1 + fs/userfaultfd.c | 1 + include/linux/mm_inline.h | 50 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/mm_types.h | 48 --------------------------------------------- kernel/fork.c | 1 + mm/madvise.c | 1 + mm/mmap.c | 1 + 7 files changed, 55 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index e6998652fd67..18f8c3acbb85 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 5b2af7b82776..e26b10132d47 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index e2ec68b0515c..47d96d2647ca 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -4,6 +4,7 @@ #include #include +#include /** * folio_is_file_lru - Should the folio be on a file LRU or anon LRU? @@ -135,4 +136,53 @@ static __always_inline void del_page_from_lru_list(struct page *page, { lruvec_del_folio(lruvec, page_folio(page)); } + +#ifdef CONFIG_ANON_VMA_NAME +/* + * mmap_lock should be read-locked when calling vma_anon_name() and while using + * the returned pointer. + */ +extern const char *vma_anon_name(struct vm_area_struct *vma); + +/* + * mmap_lock should be read-locked for orig_vma->vm_mm. + * mmap_lock should be write-locked for new_vma->vm_mm or new_vma should be + * isolated. + */ +extern void dup_vma_anon_name(struct vm_area_struct *orig_vma, + struct vm_area_struct *new_vma); + +/* + * mmap_lock should be write-locked or vma should have been isolated under + * write-locked mmap_lock protection. + */ +extern void free_vma_anon_name(struct vm_area_struct *vma); + +/* mmap_lock should be read-locked */ +static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, + const char *name) +{ + const char *vma_name = vma_anon_name(vma); + + /* either both NULL, or pointers to same string */ + if (vma_name == name) + return true; + + return name && vma_name && !strcmp(name, vma_name); +} +#else /* CONFIG_ANON_VMA_NAME */ +static inline const char *vma_anon_name(struct vm_area_struct *vma) +{ + return NULL; +} +static inline void dup_vma_anon_name(struct vm_area_struct *orig_vma, + struct vm_area_struct *new_vma) {} +static inline void free_vma_anon_name(struct vm_area_struct *vma) {} +static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, + const char *name) +{ + return true; +} +#endif /* CONFIG_ANON_VMA_NAME */ + #endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 449b6eafc695..4d5fb84eed5e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -890,52 +890,4 @@ typedef struct { unsigned long val; } swp_entry_t; -#ifdef CONFIG_ANON_VMA_NAME -/* - * mmap_lock should be read-locked when calling vma_anon_name() and while using - * the returned pointer. - */ -extern const char *vma_anon_name(struct vm_area_struct *vma); - -/* - * mmap_lock should be read-locked for orig_vma->vm_mm. - * mmap_lock should be write-locked for new_vma->vm_mm or new_vma should be - * isolated. - */ -extern void dup_vma_anon_name(struct vm_area_struct *orig_vma, - struct vm_area_struct *new_vma); - -/* - * mmap_lock should be write-locked or vma should have been isolated under - * write-locked mmap_lock protection. - */ -extern void free_vma_anon_name(struct vm_area_struct *vma); - -/* mmap_lock should be read-locked */ -static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, - const char *name) -{ - const char *vma_name = vma_anon_name(vma); - - /* either both NULL, or pointers to same string */ - if (vma_name == name) - return true; - - return name && vma_name && !strcmp(name, vma_name); -} -#else /* CONFIG_ANON_VMA_NAME */ -static inline const char *vma_anon_name(struct vm_area_struct *vma) -{ - return NULL; -} -static inline void dup_vma_anon_name(struct vm_area_struct *orig_vma, - struct vm_area_struct *new_vma) {} -static inline void free_vma_anon_name(struct vm_area_struct *vma) {} -static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, - const char *name) -{ - return true; -} -#endif /* CONFIG_ANON_VMA_NAME */ - #endif /* _LINUX_MM_TYPES_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 4cf20b5f2da3..75737e566441 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/madvise.c b/mm/madvise.c index c63aacbbfa78..5604064df464 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/mmap.c b/mm/mmap.c index 85edb0011453..77733b113c40 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3-59-g8ed1b From 36090def7bad06a6346f86a7cfdbfda2d138cb64 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 14 Jan 2022 14:06:10 -0800 Subject: mm: move tlb_flush_pending inline helpers to mm_inline.h linux/mm_types.h should only define structure definitions, to make it cheap to include elsewhere. The atomic_t helper function definitions are particularly large, so it's better to move the helpers using those into the existing linux/mm_inline.h and only include that where needed. As a follow-up, we may want to go through all the indirect includes in mm_types.h and reduce them as much as possible. Link: https://lkml.kernel.org/r/20211207125710.2503446-2-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Al Viro Cc: Stephen Rothwell Cc: Suren Baghdasaryan Cc: Colin Cross Cc: Kees Cook Cc: Peter Xu Cc: Peter Zijlstra (Intel) Cc: Yu Zhao Cc: Vlastimil Babka Cc: Matthew Wilcox (Oracle) Cc: Eric Biederman Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/x86/include/asm/pgtable.h | 2 +- include/linux/mm.h | 45 -------------- include/linux/mm_inline.h | 86 +++++++++++++++++++++++++++ include/linux/mm_types.h | 129 ++++++++++++++--------------------------- mm/ksm.c | 1 + mm/mapping_dirty_helpers.c | 1 + mm/memory.c | 1 + mm/mmu_gather.c | 1 + mm/pgtable-generic.c | 1 + 9 files changed, 137 insertions(+), 130 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 448cd01eb3ec..5196958aa6ac 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -752,7 +752,7 @@ static inline bool pte_accessible(struct mm_struct *mm, pte_t a) return true; if ((pte_flags(a) & _PAGE_PROTNONE) && - mm_tlb_flush_pending(mm)) + atomic_read(&mm->tlb_flush_pending)) return true; return false; diff --git a/include/linux/mm.h b/include/linux/mm.h index 7000442984b9..c17e5cfc1e47 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -424,51 +424,6 @@ extern unsigned int kobjsize(const void *objp); */ extern pgprot_t protection_map[16]; -/** - * enum fault_flag - Fault flag definitions. - * @FAULT_FLAG_WRITE: Fault was a write fault. - * @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE. - * @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked. - * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_lock and wait when retrying. - * @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region. - * @FAULT_FLAG_TRIED: The fault has been tried once. - * @FAULT_FLAG_USER: The fault originated in userspace. - * @FAULT_FLAG_REMOTE: The fault is not for current task/mm. - * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch. - * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals. - * - * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify - * whether we would allow page faults to retry by specifying these two - * fault flags correctly. Currently there can be three legal combinations: - * - * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and - * this is the first try - * - * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and - * we've already tried at least once - * - * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry - * - * The unlisted combination (!ALLOW_RETRY && TRIED) is illegal and should never - * be used. Note that page faults can be allowed to retry for multiple times, - * in which case we'll have an initial fault with flags (a) then later on - * continuous faults with flags (b). We should always try to detect pending - * signals before a retry to make sure the continuous page faults can still be - * interrupted if necessary. - */ -enum fault_flag { - FAULT_FLAG_WRITE = 1 << 0, - FAULT_FLAG_MKWRITE = 1 << 1, - FAULT_FLAG_ALLOW_RETRY = 1 << 2, - FAULT_FLAG_RETRY_NOWAIT = 1 << 3, - FAULT_FLAG_KILLABLE = 1 << 4, - FAULT_FLAG_TRIED = 1 << 5, - FAULT_FLAG_USER = 1 << 6, - FAULT_FLAG_REMOTE = 1 << 7, - FAULT_FLAG_INSTRUCTION = 1 << 8, - FAULT_FLAG_INTERRUPTIBLE = 1 << 9, -}; - /* * The default fault flags that should be used by most of the * arch-specific page fault handlers. diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 47d96d2647ca..b725839dfe71 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -2,6 +2,7 @@ #ifndef LINUX_MM_INLINE_H #define LINUX_MM_INLINE_H +#include #include #include #include @@ -185,4 +186,89 @@ static inline bool is_same_vma_anon_name(struct vm_area_struct *vma, } #endif /* CONFIG_ANON_VMA_NAME */ +static inline void init_tlb_flush_pending(struct mm_struct *mm) +{ + atomic_set(&mm->tlb_flush_pending, 0); +} + +static inline void inc_tlb_flush_pending(struct mm_struct *mm) +{ + atomic_inc(&mm->tlb_flush_pending); + /* + * The only time this value is relevant is when there are indeed pages + * to flush. And we'll only flush pages after changing them, which + * requires the PTL. + * + * So the ordering here is: + * + * atomic_inc(&mm->tlb_flush_pending); + * spin_lock(&ptl); + * ... + * set_pte_at(); + * spin_unlock(&ptl); + * + * spin_lock(&ptl) + * mm_tlb_flush_pending(); + * .... + * spin_unlock(&ptl); + * + * flush_tlb_range(); + * atomic_dec(&mm->tlb_flush_pending); + * + * Where the increment if constrained by the PTL unlock, it thus + * ensures that the increment is visible if the PTE modification is + * visible. After all, if there is no PTE modification, nobody cares + * about TLB flushes either. + * + * This very much relies on users (mm_tlb_flush_pending() and + * mm_tlb_flush_nested()) only caring about _specific_ PTEs (and + * therefore specific PTLs), because with SPLIT_PTE_PTLOCKS and RCpc + * locks (PPC) the unlock of one doesn't order against the lock of + * another PTL. + * + * The decrement is ordered by the flush_tlb_range(), such that + * mm_tlb_flush_pending() will not return false unless all flushes have + * completed. + */ +} + +static inline void dec_tlb_flush_pending(struct mm_struct *mm) +{ + /* + * See inc_tlb_flush_pending(). + * + * This cannot be smp_mb__before_atomic() because smp_mb() simply does + * not order against TLB invalidate completion, which is what we need. + * + * Therefore we must rely on tlb_flush_*() to guarantee order. + */ + atomic_dec(&mm->tlb_flush_pending); +} + +static inline bool mm_tlb_flush_pending(struct mm_struct *mm) +{ + /* + * Must be called after having acquired the PTL; orders against that + * PTLs release and therefore ensures that if we observe the modified + * PTE we must also observe the increment from inc_tlb_flush_pending(). + * + * That is, it only guarantees to return true if there is a flush + * pending for _this_ PTL. + */ + return atomic_read(&mm->tlb_flush_pending); +} + +static inline bool mm_tlb_flush_nested(struct mm_struct *mm) +{ + /* + * Similar to mm_tlb_flush_pending(), we must have acquired the PTL + * for which there is a TLB flush pending in order to guarantee + * we've seen both that PTE modification and the increment. + * + * (no requirement on actually still holding the PTL, that is irrelevant) + */ + return atomic_read(&mm->tlb_flush_pending) > 1; +} + + #endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 4d5fb84eed5e..6a89f128c990 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -692,90 +692,6 @@ extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_finish_mmu(struct mmu_gather *tlb); -static inline void init_tlb_flush_pending(struct mm_struct *mm) -{ - atomic_set(&mm->tlb_flush_pending, 0); -} - -static inline void inc_tlb_flush_pending(struct mm_struct *mm) -{ - atomic_inc(&mm->tlb_flush_pending); - /* - * The only time this value is relevant is when there are indeed pages - * to flush. And we'll only flush pages after changing them, which - * requires the PTL. - * - * So the ordering here is: - * - * atomic_inc(&mm->tlb_flush_pending); - * spin_lock(&ptl); - * ... - * set_pte_at(); - * spin_unlock(&ptl); - * - * spin_lock(&ptl) - * mm_tlb_flush_pending(); - * .... - * spin_unlock(&ptl); - * - * flush_tlb_range(); - * atomic_dec(&mm->tlb_flush_pending); - * - * Where the increment if constrained by the PTL unlock, it thus - * ensures that the increment is visible if the PTE modification is - * visible. After all, if there is no PTE modification, nobody cares - * about TLB flushes either. - * - * This very much relies on users (mm_tlb_flush_pending() and - * mm_tlb_flush_nested()) only caring about _specific_ PTEs (and - * therefore specific PTLs), because with SPLIT_PTE_PTLOCKS and RCpc - * locks (PPC) the unlock of one doesn't order against the lock of - * another PTL. - * - * The decrement is ordered by the flush_tlb_range(), such that - * mm_tlb_flush_pending() will not return false unless all flushes have - * completed. - */ -} - -static inline void dec_tlb_flush_pending(struct mm_struct *mm) -{ - /* - * See inc_tlb_flush_pending(). - * - * This cannot be smp_mb__before_atomic() because smp_mb() simply does - * not order against TLB invalidate completion, which is what we need. - * - * Therefore we must rely on tlb_flush_*() to guarantee order. - */ - atomic_dec(&mm->tlb_flush_pending); -} - -static inline bool mm_tlb_flush_pending(struct mm_struct *mm) -{ - /* - * Must be called after having acquired the PTL; orders against that - * PTLs release and therefore ensures that if we observe the modified - * PTE we must also observe the increment from inc_tlb_flush_pending(). - * - * That is, it only guarantees to return true if there is a flush - * pending for _this_ PTL. - */ - return atomic_read(&mm->tlb_flush_pending); -} - -static inline bool mm_tlb_flush_nested(struct mm_struct *mm) -{ - /* - * Similar to mm_tlb_flush_pending(), we must have acquired the PTL - * for which there is a TLB flush pending in order to guarantee - * we've seen both that PTE modification and the increment. - * - * (no requirement on actually still holding the PTL, that is irrelevant) - */ - return atomic_read(&mm->tlb_flush_pending) > 1; -} - struct vm_fault; /** @@ -890,4 +806,49 @@ typedef struct { unsigned long val; } swp_entry_t; +/** + * enum fault_flag - Fault flag definitions. + * @FAULT_FLAG_WRITE: Fault was a write fault. + * @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE. + * @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked. + * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_lock and wait when retrying. + * @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region. + * @FAULT_FLAG_TRIED: The fault has been tried once. + * @FAULT_FLAG_USER: The fault originated in userspace. + * @FAULT_FLAG_REMOTE: The fault is not for current task/mm. + * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch. + * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals. + * + * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify + * whether we would allow page faults to retry by specifying these two + * fault flags correctly. Currently there can be three legal combinations: + * + * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and + * this is the first try + * + * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and + * we've already tried at least once + * + * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry + * + * The unlisted combination (!ALLOW_RETRY && TRIED) is illegal and should never + * be used. Note that page faults can be allowed to retry for multiple times, + * in which case we'll have an initial fault with flags (a) then later on + * continuous faults with flags (b). We should always try to detect pending + * signals before a retry to make sure the continuous page faults can still be + * interrupted if necessary. + */ +enum fault_flag { + FAULT_FLAG_WRITE = 1 << 0, + FAULT_FLAG_MKWRITE = 1 << 1, + FAULT_FLAG_ALLOW_RETRY = 1 << 2, + FAULT_FLAG_RETRY_NOWAIT = 1 << 3, + FAULT_FLAG_KILLABLE = 1 << 4, + FAULT_FLAG_TRIED = 1 << 5, + FAULT_FLAG_USER = 1 << 6, + FAULT_FLAG_REMOTE = 1 << 7, + FAULT_FLAG_INSTRUCTION = 1 << 8, + FAULT_FLAG_INTERRUPTIBLE = 1 << 9, +}; + #endif /* _LINUX_MM_TYPES_H */ diff --git a/mm/ksm.c b/mm/ksm.c index 0662093237e4..f34476ac0a41 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c index ea734f248fce..1b0ab8fcfd8b 100644 --- a/mm/mapping_dirty_helpers.c +++ b/mm/mapping_dirty_helpers.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/mm/memory.c b/mm/memory.c index 8f1de811a1dc..bc80d4effac9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -41,6 +41,7 @@ #include #include +#include #include #include #include diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 1b9837419bf9..afb7185ffdc4 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 4e640baf9794..6523fda274e5 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -10,6 +10,7 @@ #include #include #include +#include #include /* -- cgit v1.2.3-59-g8ed1b From cc6dcfee72509868271d42919a3c1081b6b0dc7e Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Fri, 14 Jan 2022 14:06:18 -0800 Subject: mm: document locking restrictions for vm_operations_struct::close Add comments for vm_operations_struct::close documenting locking requirements for this callback and its callers. Link: https://lkml.kernel.org/r/20211209191325.3069345-2-surenb@google.com Signed-off-by: Suren Baghdasaryan Acked-by: Michal Hocko Cc: Andrea Arcangeli Cc: Andy Lutomirski Cc: Christian Brauner Cc: Christian Brauner Cc: Christoph Hellwig Cc: David Hildenbrand Cc: David Rientjes Cc: Florian Weimer Cc: Jan Engelhardt Cc: Jann Horn Cc: Johannes Weiner Cc: Kirill A. Shutemov Cc: Matthew Wilcox Cc: Minchan Kim Cc: Oleg Nesterov Cc: Rik van Riel Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tim Murray Cc: Jason Gunthorpe Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index c17e5cfc1e47..4d7245e6802a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -532,6 +532,10 @@ enum page_entry_size { */ struct vm_operations_struct { void (*open)(struct vm_area_struct * area); + /** + * @close: Called when the VMA is being removed from the MM. + * Context: User context. May sleep. Caller holds mmap_lock. + */ void (*close)(struct vm_area_struct * area); /* Called any time before splitting to check if it's allowed */ int (*may_split)(struct vm_area_struct *area, unsigned long addr); -- cgit v1.2.3-59-g8ed1b From 08d5b29eac7dd5e6c79b66d390ecbb9219e05931 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Fri, 14 Jan 2022 14:06:33 -0800 Subject: mm: ptep_clear() page table helper We have ptep_get_and_clear() and ptep_get_and_clear_full() helpers to clear PTE from user page tables, but there is no variant for simple clear of a present PTE from user page tables without using a low level pte_clear() which can be either native or para-virtualised. Add a new ptep_clear() that can be used in common code to clear PTEs from page table. We will need this call later in order to add a hook for page table check. Link: https://lkml.kernel.org/r/20211221154650.1047963-3-pasha.tatashin@soleen.com Signed-off-by: Pasha Tatashin Cc: Aneesh Kumar K.V Cc: Dave Hansen Cc: David Rientjes Cc: Frederic Weisbecker Cc: Greg Thelen Cc: "H. Peter Anvin" Cc: Hugh Dickins Cc: Ingo Molnar Cc: Jiri Slaby Cc: Jonathan Corbet Cc: Kees Cook Cc: Masahiro Yamada Cc: Mike Rapoport Cc: Muchun Song Cc: Paul Turner Cc: Peter Zijlstra Cc: Sami Tolvanen Cc: Thomas Gleixner Cc: Wei Xu Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/arch_pgtable_helpers.rst | 6 ++++-- include/linux/pgtable.h | 8 ++++++++ mm/debug_vm_pgtable.c | 2 +- mm/khugepaged.c | 12 ++---------- 4 files changed, 15 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/Documentation/vm/arch_pgtable_helpers.rst b/Documentation/vm/arch_pgtable_helpers.rst index b3166c33db39..f8b225fc9190 100644 --- a/Documentation/vm/arch_pgtable_helpers.rst +++ b/Documentation/vm/arch_pgtable_helpers.rst @@ -66,9 +66,11 @@ PTE Page Table Helpers +---------------------------+--------------------------------------------------+ | pte_mknotpresent | Invalidates a mapped PTE | +---------------------------+--------------------------------------------------+ -| ptep_get_and_clear | Clears a PTE | +| ptep_clear | Clears a PTE | +---------------------------+--------------------------------------------------+ -| ptep_get_and_clear_full | Clears a PTE | +| ptep_get_and_clear | Clears and returns PTE | ++---------------------------+--------------------------------------------------+ +| ptep_get_and_clear_full | Clears and returns PTE (batched PTE unmap) | +---------------------------+--------------------------------------------------+ | ptep_test_and_clear_young | Clears young from a PTE | +---------------------------+--------------------------------------------------+ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index e24d2c992b11..bc8713a76e03 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -258,6 +258,14 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif +#ifndef __HAVE_ARCH_PTEP_CLEAR +static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + pte_clear(mm, addr, ptep); +} +#endif + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 2a2b24e87877..a7ac97c76762 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -652,7 +652,7 @@ static void __init pte_clear_tests(struct pgtable_debug_args *args) set_pte_at(args->mm, args->vaddr, args->ptep, pte); flush_dcache_page(page); barrier(); - pte_clear(args->mm, args->vaddr, args->ptep); + ptep_clear(args->mm, args->vaddr, args->ptep); pte = ptep_get(args->ptep); WARN_ON(!pte_none(pte)); } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index e99101162f1a..9d40dd8890e5 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -756,11 +756,7 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, * ptl mostly unnecessary. */ spin_lock(ptl); - /* - * paravirt calls inside pte_clear here are - * superfluous. - */ - pte_clear(vma->vm_mm, address, _pte); + ptep_clear(vma->vm_mm, address, _pte); spin_unlock(ptl); } } else { @@ -774,11 +770,7 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, * inside page_remove_rmap(). */ spin_lock(ptl); - /* - * paravirt calls inside pte_clear here are - * superfluous. - */ - pte_clear(vma->vm_mm, address, _pte); + ptep_clear(vma->vm_mm, address, _pte); page_remove_rmap(src_page, false); spin_unlock(ptl); free_page_and_swap_cache(src_page); -- cgit v1.2.3-59-g8ed1b From df4e817b710809425d899340dbfa8504a3ca4ba5 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Fri, 14 Jan 2022 14:06:37 -0800 Subject: mm: page table check Check user page table entries at the time they are added and removed. Allows to synchronously catch memory corruption issues related to double mapping. When a pte for an anonymous page is added into page table, we verify that this pte does not already point to a file backed page, and vice versa if this is a file backed page that is being added we verify that this page does not have an anonymous mapping We also enforce that read-only sharing for anonymous pages is allowed (i.e. cow after fork). All other sharing must be for file pages. Page table check allows to protect and debug cases where "struct page" metadata became corrupted for some reason. For example, when refcnt or mapcount become invalid. Link: https://lkml.kernel.org/r/20211221154650.1047963-4-pasha.tatashin@soleen.com Signed-off-by: Pasha Tatashin Cc: Aneesh Kumar K.V Cc: Dave Hansen Cc: David Rientjes Cc: Frederic Weisbecker Cc: Greg Thelen Cc: "H. Peter Anvin" Cc: Hugh Dickins Cc: Ingo Molnar Cc: Jiri Slaby Cc: Jonathan Corbet Cc: Kees Cook Cc: Masahiro Yamada Cc: Mike Rapoport Cc: Muchun Song Cc: Paul Turner Cc: Peter Zijlstra Cc: Sami Tolvanen Cc: Thomas Gleixner Cc: Wei Xu Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/index.rst | 1 + Documentation/vm/page_table_check.rst | 56 +++++++ MAINTAINERS | 9 ++ arch/Kconfig | 3 + include/linux/page_table_check.h | 147 ++++++++++++++++++ mm/Kconfig.debug | 24 +++ mm/Makefile | 1 + mm/page_alloc.c | 4 + mm/page_ext.c | 4 + mm/page_table_check.c | 270 ++++++++++++++++++++++++++++++++++ 10 files changed, 519 insertions(+) create mode 100644 Documentation/vm/page_table_check.rst create mode 100644 include/linux/page_table_check.h create mode 100644 mm/page_table_check.c (limited to 'include/linux') diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst index b1826ca2c576..932440805453 100644 --- a/Documentation/vm/index.rst +++ b/Documentation/vm/index.rst @@ -31,6 +31,7 @@ algorithms. If you are looking for advice on simply allocating memory, see the page_migration page_frags page_owner + page_table_check remap_file_pages slub split_page_table_lock diff --git a/Documentation/vm/page_table_check.rst b/Documentation/vm/page_table_check.rst new file mode 100644 index 000000000000..81f521ff7ea7 --- /dev/null +++ b/Documentation/vm/page_table_check.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _page_table_check: + +================ +Page Table Check +================ + +Introduction +============ + +Page table check allows to hardern the kernel by ensuring that some types of +the memory corruptions are prevented. + +Page table check performs extra verifications at the time when new pages become +accessible from the userspace by getting their page table entries (PTEs PMDs +etc.) added into the table. + +In case of detected corruption, the kernel is crashed. There is a small +performance and memory overhead associated with the page table check. Therefore, +it is disabled by default, but can be optionally enabled on systems where the +extra hardening outweighs the performance costs. Also, because page table check +is synchronous, it can help with debugging double map memory corruption issues, +by crashing kernel at the time wrong mapping occurs instead of later which is +often the case with memory corruptions bugs. + +Double mapping detection logic +============================== + ++-------------------+-------------------+-------------------+------------------+ +| Current Mapping | New mapping | Permissions | Rule | ++===================+===================+===================+==================+ +| Anonymous | Anonymous | Read | Allow | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Anonymous | Read / Write | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Anonymous | Named | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Anonymous | Any | Prohibit | ++-------------------+-------------------+-------------------+------------------+ +| Named | Named | Any | Allow | ++-------------------+-------------------+-------------------+------------------+ + +Enabling Page Table Check +========================= + +Build kernel with: + +- PAGE_TABLE_CHECK=y + Note, it can only be enabled on platforms where ARCH_SUPPORTS_PAGE_TABLE_CHECK + is available. + +- Boot with 'page_table_check=on' kernel parameter. + +Optionally, build kernel with PAGE_TABLE_CHECK_ENFORCED in order to have page +table support without extra kernel parameter. diff --git a/MAINTAINERS b/MAINTAINERS index dd36acc87ce6..fbdb860c0b8b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14387,6 +14387,15 @@ F: include/net/page_pool.h F: include/trace/events/page_pool.h F: net/core/page_pool.c +PAGE TABLE CHECK +M: Pasha Tatashin +M: Andrew Morton +L: linux-mm@kvack.org +S: Maintained +F: Documentation/vm/page_table_check.rst +F: include/linux/page_table_check.h +F: mm/page_table_check.c + PANASONIC LAPTOP ACPI EXTRAS DRIVER M: Kenneth Chan L: platform-driver-x86@vger.kernel.org diff --git a/arch/Kconfig b/arch/Kconfig index d3c4ab249e9c..4568b6b70b5d 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1297,6 +1297,9 @@ config HAVE_ARCH_PFN_VALID config ARCH_SUPPORTS_DEBUG_PAGEALLOC bool +config ARCH_SUPPORTS_PAGE_TABLE_CHECK + bool + config ARCH_SPLIT_ARG64 bool help diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h new file mode 100644 index 000000000000..38cace1da7b6 --- /dev/null +++ b/include/linux/page_table_check.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2021, Google LLC. + * Pasha Tatashin + */ +#ifndef __LINUX_PAGE_TABLE_CHECK_H +#define __LINUX_PAGE_TABLE_CHECK_H + +#ifdef CONFIG_PAGE_TABLE_CHECK +#include + +extern struct static_key_true page_table_check_disabled; +extern struct page_ext_operations page_table_check_ops; + +void __page_table_check_zero(struct page *page, unsigned int order); +void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t pte); +void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, + pmd_t pmd); +void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, + pud_t pud); +void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); +void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); +void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud); + +static inline void page_table_check_alloc(struct page *page, unsigned int order) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_zero(page, order); +} + +static inline void page_table_check_free(struct page *page, unsigned int order) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_zero(page, order); +} + +static inline void page_table_check_pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t pte) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pte_clear(mm, addr, pte); +} + +static inline void page_table_check_pmd_clear(struct mm_struct *mm, + unsigned long addr, pmd_t pmd) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pmd_clear(mm, addr, pmd); +} + +static inline void page_table_check_pud_clear(struct mm_struct *mm, + unsigned long addr, pud_t pud) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pud_clear(mm, addr, pud); +} + +static inline void page_table_check_pte_set(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + pte_t pte) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pte_set(mm, addr, ptep, pte); +} + +static inline void page_table_check_pmd_set(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp, + pmd_t pmd) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pmd_set(mm, addr, pmdp, pmd); +} + +static inline void page_table_check_pud_set(struct mm_struct *mm, + unsigned long addr, pud_t *pudp, + pud_t pud) +{ + if (static_branch_likely(&page_table_check_disabled)) + return; + + __page_table_check_pud_set(mm, addr, pudp, pud); +} + +#else + +static inline void page_table_check_alloc(struct page *page, unsigned int order) +{ +} + +static inline void page_table_check_free(struct page *page, unsigned int order) +{ +} + +static inline void page_table_check_pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t pte) +{ +} + +static inline void page_table_check_pmd_clear(struct mm_struct *mm, + unsigned long addr, pmd_t pmd) +{ +} + +static inline void page_table_check_pud_clear(struct mm_struct *mm, + unsigned long addr, pud_t pud) +{ +} + +static inline void page_table_check_pte_set(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + pte_t pte) +{ +} + +static inline void page_table_check_pmd_set(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp, + pmd_t pmd) +{ +} + +static inline void page_table_check_pud_set(struct mm_struct *mm, + unsigned long addr, pud_t *pudp, + pud_t pud) +{ +} + +#endif /* CONFIG_PAGE_TABLE_CHECK */ +#endif /* __LINUX_PAGE_TABLE_CHECK_H */ diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 1e73717802f8..5bd5bb097252 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -62,6 +62,30 @@ config PAGE_OWNER If unsure, say N. +config PAGE_TABLE_CHECK + bool "Check for invalid mappings in user page tables" + depends on ARCH_SUPPORTS_PAGE_TABLE_CHECK + select PAGE_EXTENSION + help + Check that anonymous page is not being mapped twice with read write + permissions. Check that anonymous and file pages are not being + erroneously shared. Since the checking is performed at the time + entries are added and removed to user page tables, leaking, corruption + and double mapping problems are detected synchronously. + + If unsure say "n". + +config PAGE_TABLE_CHECK_ENFORCED + bool "Enforce the page table checking by default" + depends on PAGE_TABLE_CHECK + help + Always enable page table checking. By default the page table checking + is disabled, and can be optionally enabled via page_table_check=on + kernel parameter. This config enforces that page table check is always + enabled. + + If unsure say "n". + config PAGE_POISONING bool "Poison pages after freeing" help diff --git a/mm/Makefile b/mm/Makefile index d6c0042e3aa0..5c5a3a480fa6 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -112,6 +112,7 @@ obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o obj-$(CONFIG_CMA) += cma.o obj-$(CONFIG_MEMORY_BALLOON) += balloon_compaction.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o +obj-$(CONFIG_PAGE_TABLE_CHECK) += page_table_check.o obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o obj-$(CONFIG_SECRETMEM) += secretmem.o obj-$(CONFIG_CMA_SYSFS) += cma_sysfs.o diff --git a/mm/page_alloc.c b/mm/page_alloc.c index d59023a676ed..806f317c2e7e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -63,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -1307,6 +1308,7 @@ static __always_inline bool free_pages_prepare(struct page *page, if (memcg_kmem_enabled() && PageMemcgKmem(page)) __memcg_kmem_uncharge_page(page, order); reset_page_owner(page, order); + page_table_check_free(page, order); return false; } @@ -1346,6 +1348,7 @@ static __always_inline bool free_pages_prepare(struct page *page, page_cpupid_reset_last(page); page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; reset_page_owner(page, order); + page_table_check_free(page, order); if (!PageHighMem(page)) { debug_check_no_locks_freed(page_address(page), @@ -2420,6 +2423,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order, } set_page_owner(page, order, gfp_flags); + page_table_check_alloc(page, order); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, diff --git a/mm/page_ext.c b/mm/page_ext.c index 6242afb24d84..bee3240604dc 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -8,6 +8,7 @@ #include #include #include +#include /* * struct page extension @@ -75,6 +76,9 @@ static struct page_ext_operations *page_ext_ops[] = { #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) &page_idle_ops, #endif +#ifdef CONFIG_PAGE_TABLE_CHECK + &page_table_check_ops, +#endif }; unsigned long page_ext_size = sizeof(struct page_ext); diff --git a/mm/page_table_check.c b/mm/page_table_check.c new file mode 100644 index 000000000000..7504e7caa2a1 --- /dev/null +++ b/mm/page_table_check.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2021, Google LLC. + * Pasha Tatashin + */ +#include +#include + +#undef pr_fmt +#define pr_fmt(fmt) "page_table_check: " fmt + +struct page_table_check { + atomic_t anon_map_count; + atomic_t file_map_count; +}; + +static bool __page_table_check_enabled __initdata = + IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); + +DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); +EXPORT_SYMBOL(page_table_check_disabled); + +static int __init early_page_table_check_param(char *buf) +{ + if (!buf) + return -EINVAL; + + if (strcmp(buf, "on") == 0) + __page_table_check_enabled = true; + else if (strcmp(buf, "off") == 0) + __page_table_check_enabled = false; + + return 0; +} + +early_param("page_table_check", early_page_table_check_param); + +static bool __init need_page_table_check(void) +{ + return __page_table_check_enabled; +} + +static void __init init_page_table_check(void) +{ + if (!__page_table_check_enabled) + return; + static_branch_disable(&page_table_check_disabled); +} + +struct page_ext_operations page_table_check_ops = { + .size = sizeof(struct page_table_check), + .need = need_page_table_check, + .init = init_page_table_check, +}; + +static struct page_table_check *get_page_table_check(struct page_ext *page_ext) +{ + BUG_ON(!page_ext); + return (void *)(page_ext) + page_table_check_ops.offset; +} + +static inline bool pte_user_accessible_page(pte_t pte) +{ + return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER); +} + +static inline bool pmd_user_accessible_page(pmd_t pmd) +{ + return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && + (pmd_val(pmd) & _PAGE_USER); +} + +static inline bool pud_user_accessible_page(pud_t pud) +{ + return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && + (pud_val(pud) & _PAGE_USER); +} + +/* + * An enty is removed from the page table, decrement the counters for that page + * verify that it is of correct type and counters do not become negative. + */ +static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, + unsigned long pfn, unsigned long pgcnt) +{ + struct page_ext *page_ext; + struct page *page; + bool anon; + int i; + + if (!pfn_valid(pfn)) + return; + + page = pfn_to_page(pfn); + page_ext = lookup_page_ext(page); + anon = PageAnon(page); + + for (i = 0; i < pgcnt; i++) { + struct page_table_check *ptc = get_page_table_check(page_ext); + + if (anon) { + BUG_ON(atomic_read(&ptc->file_map_count)); + BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); + } else { + BUG_ON(atomic_read(&ptc->anon_map_count)); + BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); + } + page_ext = page_ext_next(page_ext); + } +} + +/* + * A new enty is added to the page table, increment the counters for that page + * verify that it is of correct type and is not being mapped with a different + * type to a different process. + */ +static void page_table_check_set(struct mm_struct *mm, unsigned long addr, + unsigned long pfn, unsigned long pgcnt, + bool rw) +{ + struct page_ext *page_ext; + struct page *page; + bool anon; + int i; + + if (!pfn_valid(pfn)) + return; + + page = pfn_to_page(pfn); + page_ext = lookup_page_ext(page); + anon = PageAnon(page); + + for (i = 0; i < pgcnt; i++) { + struct page_table_check *ptc = get_page_table_check(page_ext); + + if (anon) { + BUG_ON(atomic_read(&ptc->file_map_count)); + BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&ptc->anon_map_count)); + BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); + } + page_ext = page_ext_next(page_ext); + } +} + +/* + * page is on free list, or is being allocated, verify that counters are zeroes + * crash if they are not. + */ +void __page_table_check_zero(struct page *page, unsigned int order) +{ + struct page_ext *page_ext = lookup_page_ext(page); + int i; + + BUG_ON(!page_ext); + for (i = 0; i < (1 << order); i++) { + struct page_table_check *ptc = get_page_table_check(page_ext); + + BUG_ON(atomic_read(&ptc->anon_map_count)); + BUG_ON(atomic_read(&ptc->file_map_count)); + page_ext = page_ext_next(page_ext); + } +} + +void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t pte) +{ + if (&init_mm == mm) + return; + + if (pte_user_accessible_page(pte)) { + page_table_check_clear(mm, addr, pte_pfn(pte), + PAGE_SIZE >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL(__page_table_check_pte_clear); + +void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, + pmd_t pmd) +{ + if (&init_mm == mm) + return; + + if (pmd_user_accessible_page(pmd)) { + page_table_check_clear(mm, addr, pmd_pfn(pmd), + PMD_PAGE_SIZE >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL(__page_table_check_pmd_clear); + +void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, + pud_t pud) +{ + if (&init_mm == mm) + return; + + if (pud_user_accessible_page(pud)) { + page_table_check_clear(mm, addr, pud_pfn(pud), + PUD_PAGE_SIZE >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL(__page_table_check_pud_clear); + +void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + pte_t old_pte; + + if (&init_mm == mm) + return; + + old_pte = *ptep; + if (pte_user_accessible_page(old_pte)) { + page_table_check_clear(mm, addr, pte_pfn(old_pte), + PAGE_SIZE >> PAGE_SHIFT); + } + + if (pte_user_accessible_page(pte)) { + page_table_check_set(mm, addr, pte_pfn(pte), + PAGE_SIZE >> PAGE_SHIFT, + pte_write(pte)); + } +} +EXPORT_SYMBOL(__page_table_check_pte_set); + +void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old_pmd; + + if (&init_mm == mm) + return; + + old_pmd = *pmdp; + if (pmd_user_accessible_page(old_pmd)) { + page_table_check_clear(mm, addr, pmd_pfn(old_pmd), + PMD_PAGE_SIZE >> PAGE_SHIFT); + } + + if (pmd_user_accessible_page(pmd)) { + page_table_check_set(mm, addr, pmd_pfn(pmd), + PMD_PAGE_SIZE >> PAGE_SHIFT, + pmd_write(pmd)); + } +} +EXPORT_SYMBOL(__page_table_check_pmd_set); + +void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, + pud_t *pudp, pud_t pud) +{ + pud_t old_pud; + + if (&init_mm == mm) + return; + + old_pud = *pudp; + if (pud_user_accessible_page(old_pud)) { + page_table_check_clear(mm, addr, pud_pfn(old_pud), + PUD_PAGE_SIZE >> PAGE_SHIFT); + } + + if (pud_user_accessible_page(pud)) { + page_table_check_set(mm, addr, pud_pfn(pud), + PUD_PAGE_SIZE >> PAGE_SHIFT, + pud_write(pud)); + } +} +EXPORT_SYMBOL(__page_table_check_pud_set); -- cgit v1.2.3-59-g8ed1b From 020e87650af9f43683546729f959fdc78422a4b7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 14 Jan 2022 14:06:44 -0800 Subject: mm: remove last argument of reuse_swap_page() None of the callers care about the total_map_swapcount() any more. Link: https://lkml.kernel.org/r/20211220205943.456187-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Linus Torvalds Reviewed-by: William Kucharski Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/swap.h | 6 +++--- mm/huge_memory.c | 2 +- mm/khugepaged.c | 2 +- mm/memory.c | 2 +- mm/swapfile.c | 8 +------- 5 files changed, 7 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index d1ea44b31f19..bdccbf1efa61 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -514,7 +514,7 @@ extern int __swp_swapcount(swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); extern struct swap_info_struct *page_swap_info(struct page *); extern struct swap_info_struct *swp_swap_info(swp_entry_t entry); -extern bool reuse_swap_page(struct page *, int *); +extern bool reuse_swap_page(struct page *); extern int try_to_free_swap(struct page *); struct backing_dev_info; extern int init_swap_address_space(unsigned int type, unsigned long nr_pages); @@ -680,8 +680,8 @@ static inline int swp_swapcount(swp_entry_t entry) return 0; } -#define reuse_swap_page(page, total_map_swapcount) \ - (page_trans_huge_mapcount(page, total_map_swapcount) == 1) +#define reuse_swap_page(page) \ + (page_trans_huge_mapcount(page, NULL) == 1) static inline int try_to_free_swap(struct page *page) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index e5483347291c..b61fbe95c856 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1322,7 +1322,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) * We can only reuse the page if nobody else maps the huge page or it's * part. */ - if (reuse_swap_page(page, NULL)) { + if (reuse_swap_page(page)) { pmd_t entry; entry = pmd_mkyoung(orig_pmd); entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 9d40dd8890e5..698ea19775ac 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -681,7 +681,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, goto out; } if (!pte_write(pteval) && PageSwapCache(page) && - !reuse_swap_page(page, NULL)) { + !reuse_swap_page(page)) { /* * Page is in the swap cache and cannot be re-used. * It cannot be collapsed into a THP. diff --git a/mm/memory.c b/mm/memory.c index 5fea331b1560..571d02f419ba 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3627,7 +3627,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS); pte = mk_pte(page, vma->vm_page_prot); - if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) { + if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) { pte = maybe_mkwrite(pte_mkdirty(pte), vma); vmf->flags &= ~FAULT_FLAG_WRITE; ret |= VM_FAULT_WRITE; diff --git a/mm/swapfile.c b/mm/swapfile.c index e64207e2ef1d..31d13a393cf0 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1668,12 +1668,8 @@ static int page_trans_huge_map_swapcount(struct page *page, int *total_mapcount, * to it. And as a side-effect, free up its swap: because the old content * on disk will never be read, and seeking back there to write new content * later would only waste time away from clustering. - * - * NOTE: total_map_swapcount should not be relied upon by the caller if - * reuse_swap_page() returns false, but it may be always overwritten - * (see the other implementation for CONFIG_SWAP=n). */ -bool reuse_swap_page(struct page *page, int *total_map_swapcount) +bool reuse_swap_page(struct page *page) { int count, total_mapcount, total_swapcount; @@ -1682,8 +1678,6 @@ bool reuse_swap_page(struct page *page, int *total_map_swapcount) return false; count = page_trans_huge_map_swapcount(page, &total_mapcount, &total_swapcount); - if (total_map_swapcount) - *total_map_swapcount = total_mapcount + total_swapcount; if (count == 1 && PageSwapCache(page) && (likely(!PageTransCompound(page)) || /* The remaining swap count will be freed soon */ -- cgit v1.2.3-59-g8ed1b From d08d2b62510e2407cf939e693aefd179dc114913 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 14 Jan 2022 14:06:51 -0800 Subject: mm: remove the total_mapcount argument from page_trans_huge_mapcount() All callers pass NULL, so we can stop calculating the value we would store in it. Link: https://lkml.kernel.org/r/20211220205943.456187-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski Acked-by: Linus Torvalds Cc: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 10 +++------- include/linux/swap.h | 2 +- mm/huge_memory.c | 30 ++++++++++-------------------- mm/swapfile.c | 2 +- 4 files changed, 15 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 4d7245e6802a..cef65f9cbdf2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -799,19 +799,15 @@ static inline int page_mapcount(struct page *page) #ifdef CONFIG_TRANSPARENT_HUGEPAGE int total_mapcount(struct page *page); -int page_trans_huge_mapcount(struct page *page, int *total_mapcount); +int page_trans_huge_mapcount(struct page *page); #else static inline int total_mapcount(struct page *page) { return page_mapcount(page); } -static inline int page_trans_huge_mapcount(struct page *page, - int *total_mapcount) +static inline int page_trans_huge_mapcount(struct page *page) { - int mapcount = page_mapcount(page); - if (total_mapcount) - *total_mapcount = mapcount; - return mapcount; + return page_mapcount(page); } #endif diff --git a/include/linux/swap.h b/include/linux/swap.h index bdccbf1efa61..1d38d9475c4d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -681,7 +681,7 @@ static inline int swp_swapcount(swp_entry_t entry) } #define reuse_swap_page(page) \ - (page_trans_huge_mapcount(page, NULL) == 1) + (page_trans_huge_mapcount(page) == 1) static inline int try_to_free_swap(struct page *page) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b61fbe95c856..6ed86a8f6a5b 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2542,38 +2542,28 @@ int total_mapcount(struct page *page) * need full accuracy to avoid breaking page pinning, because * page_trans_huge_mapcount() is slower than page_mapcount(). */ -int page_trans_huge_mapcount(struct page *page, int *total_mapcount) +int page_trans_huge_mapcount(struct page *page) { - int i, ret, _total_mapcount, mapcount; + int i, ret; /* hugetlbfs shouldn't call it */ VM_BUG_ON_PAGE(PageHuge(page), page); - if (likely(!PageTransCompound(page))) { - mapcount = atomic_read(&page->_mapcount) + 1; - if (total_mapcount) - *total_mapcount = mapcount; - return mapcount; - } + if (likely(!PageTransCompound(page))) + return atomic_read(&page->_mapcount) + 1; page = compound_head(page); - _total_mapcount = ret = 0; + ret = 0; for (i = 0; i < thp_nr_pages(page); i++) { - mapcount = atomic_read(&page[i]._mapcount) + 1; + int mapcount = atomic_read(&page[i]._mapcount) + 1; ret = max(ret, mapcount); - _total_mapcount += mapcount; } - if (PageDoubleMap(page)) { + + if (PageDoubleMap(page)) ret -= 1; - _total_mapcount -= thp_nr_pages(page); - } - mapcount = compound_mapcount(page); - ret += mapcount; - _total_mapcount += mapcount; - if (total_mapcount) - *total_mapcount = _total_mapcount; - return ret; + + return ret + compound_mapcount(page); } /* Racy check whether the huge page can be split */ diff --git a/mm/swapfile.c b/mm/swapfile.c index a93f5b5fc8b6..caa9f81a0d15 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1619,7 +1619,7 @@ static int page_trans_huge_map_swapcount(struct page *page, swapcount = page_swapcount(page); if (total_swapcount) *total_swapcount = swapcount; - return swapcount + page_trans_huge_mapcount(page, NULL); + return swapcount + page_trans_huge_mapcount(page); } page = compound_head(page); -- cgit v1.2.3-59-g8ed1b From a421ef303008b0ceee2cfc625c3246fa7654b0ca Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Fri, 14 Jan 2022 14:07:07 -0800 Subject: mm: allow !GFP_KERNEL allocations for kvmalloc Support for GFP_NO{FS,IO} and __GFP_NOFAIL has been implemented by previous patches so we can allow the support for kvmalloc. This will allow some external users to simplify or completely remove their helpers. GFP_NOWAIT semantic hasn't been supported so far but it hasn't been explicitly documented so let's add a note about that. ceph_kvmalloc is the first helper to be dropped and changed to kvmalloc. Link: https://lkml.kernel.org/r/20211122153233.9924-5-mhocko@kernel.org Signed-off-by: Michal Hocko Reviewed-by: Uladzislau Rezki (Sony) Acked-by: Vlastimil Babka Cc: Christoph Hellwig Cc: Dave Chinner Cc: Ilya Dryomov Cc: Jeff Layton Cc: Neil Brown Cc: Sebastian Andrzej Siewior Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/ceph/libceph.h | 1 - mm/util.c | 15 ++++----------- net/ceph/buffer.c | 4 ++-- net/ceph/ceph_common.c | 27 --------------------------- net/ceph/crypto.c | 2 +- net/ceph/messenger.c | 2 +- net/ceph/messenger_v2.c | 2 +- net/ceph/osdmap.c | 12 ++++++------ 8 files changed, 15 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 409d8c29bc4f..309acbcb5a8a 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -295,7 +295,6 @@ extern bool libceph_compatible(void *data); extern const char *ceph_msg_type_name(int type); extern int ceph_check_fsid(struct ceph_client *client, struct ceph_fsid *fsid); -extern void *ceph_kvmalloc(size_t size, gfp_t flags); struct fs_parameter; struct fc_log; diff --git a/mm/util.c b/mm/util.c index 741ba32a43ac..7e43369064c8 100644 --- a/mm/util.c +++ b/mm/util.c @@ -549,13 +549,10 @@ EXPORT_SYMBOL(vm_mmap); * Uses kmalloc to get the memory but if the allocation fails then falls back * to the vmalloc allocator. Use kvfree for freeing the memory. * - * Reclaim modifiers - __GFP_NORETRY and __GFP_NOFAIL are not supported. + * GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier. * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is * preferable to the vmalloc fallback, due to visible performance drawbacks. * - * Please note that any use of gfp flags outside of GFP_KERNEL is careful to not - * fall back to vmalloc. - * * Return: pointer to the allocated memory of %NULL in case of failure */ void *kvmalloc_node(size_t size, gfp_t flags, int node) @@ -563,13 +560,6 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node) gfp_t kmalloc_flags = flags; void *ret; - /* - * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables) - * so the given set of flags has to be compatible. - */ - if ((flags & GFP_KERNEL) != GFP_KERNEL) - return kmalloc_node(size, flags, node); - /* * We want to attempt a large physically contiguous block first because * it is less likely to fragment multiple larger blocks and therefore @@ -582,6 +572,9 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node) if (!(kmalloc_flags & __GFP_RETRY_MAYFAIL)) kmalloc_flags |= __GFP_NORETRY; + + /* nofail semantic is implemented by the vmalloc fallback */ + kmalloc_flags &= ~__GFP_NOFAIL; } ret = kmalloc_node(size, kmalloc_flags, node); diff --git a/net/ceph/buffer.c b/net/ceph/buffer.c index 5622763ad402..7e51f128045d 100644 --- a/net/ceph/buffer.c +++ b/net/ceph/buffer.c @@ -7,7 +7,7 @@ #include #include -#include /* for ceph_kvmalloc */ +#include /* for kvmalloc */ struct ceph_buffer *ceph_buffer_new(size_t len, gfp_t gfp) { @@ -17,7 +17,7 @@ struct ceph_buffer *ceph_buffer_new(size_t len, gfp_t gfp) if (!b) return NULL; - b->vec.iov_base = ceph_kvmalloc(len, gfp); + b->vec.iov_base = kvmalloc(len, gfp); if (!b->vec.iov_base) { kfree(b); return NULL; diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 97d6ea763e32..9441b4a4912b 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -190,33 +190,6 @@ int ceph_compare_options(struct ceph_options *new_opt, } EXPORT_SYMBOL(ceph_compare_options); -/* - * kvmalloc() doesn't fall back to the vmalloc allocator unless flags are - * compatible with (a superset of) GFP_KERNEL. This is because while the - * actual pages are allocated with the specified flags, the page table pages - * are always allocated with GFP_KERNEL. - * - * ceph_kvmalloc() may be called with GFP_KERNEL, GFP_NOFS or GFP_NOIO. - */ -void *ceph_kvmalloc(size_t size, gfp_t flags) -{ - void *p; - - if ((flags & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) { - p = kvmalloc(size, flags); - } else if ((flags & (__GFP_IO | __GFP_FS)) == __GFP_IO) { - unsigned int nofs_flag = memalloc_nofs_save(); - p = kvmalloc(size, GFP_KERNEL); - memalloc_nofs_restore(nofs_flag); - } else { - unsigned int noio_flag = memalloc_noio_save(); - p = kvmalloc(size, GFP_KERNEL); - memalloc_noio_restore(noio_flag); - } - - return p; -} - static int parse_fsid(const char *str, struct ceph_fsid *fsid) { int i = 0; diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 92d89b331645..051d22c0e4ad 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -147,7 +147,7 @@ void ceph_crypto_key_destroy(struct ceph_crypto_key *key) static const u8 *aes_iv = (u8 *)CEPH_AES_IV; /* - * Should be used for buffers allocated with ceph_kvmalloc(). + * Should be used for buffers allocated with kvmalloc(). * Currently these are encrypt out-buffer (ceph_buffer) and decrypt * in-buffer (msg front). * diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 57d043b382ed..7b891be799d2 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1920,7 +1920,7 @@ struct ceph_msg *ceph_msg_new2(int type, int front_len, int max_data_items, /* front */ if (front_len) { - m->front.iov_base = ceph_kvmalloc(front_len, flags); + m->front.iov_base = kvmalloc(front_len, flags); if (m->front.iov_base == NULL) { dout("ceph_msg_new can't allocate %d bytes\n", front_len); diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index cc40ce4e02fb..c4099b641b38 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -308,7 +308,7 @@ static void *alloc_conn_buf(struct ceph_connection *con, int len) if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs))) return NULL; - buf = ceph_kvmalloc(len, GFP_NOIO); + buf = kvmalloc(len, GFP_NOIO); if (!buf) return NULL; diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index 75b738083523..2823bb3cff55 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -980,7 +980,7 @@ static struct crush_work *alloc_workspace(const struct crush_map *c) work_size = crush_work_size(c, CEPH_PG_MAX_SIZE); dout("%s work_size %zu bytes\n", __func__, work_size); - work = ceph_kvmalloc(work_size, GFP_NOIO); + work = kvmalloc(work_size, GFP_NOIO); if (!work) return NULL; @@ -1190,9 +1190,9 @@ static int osdmap_set_max_osd(struct ceph_osdmap *map, u32 max) if (max == map->max_osd) return 0; - state = ceph_kvmalloc(array_size(max, sizeof(*state)), GFP_NOFS); - weight = ceph_kvmalloc(array_size(max, sizeof(*weight)), GFP_NOFS); - addr = ceph_kvmalloc(array_size(max, sizeof(*addr)), GFP_NOFS); + state = kvmalloc(array_size(max, sizeof(*state)), GFP_NOFS); + weight = kvmalloc(array_size(max, sizeof(*weight)), GFP_NOFS); + addr = kvmalloc(array_size(max, sizeof(*addr)), GFP_NOFS); if (!state || !weight || !addr) { kvfree(state); kvfree(weight); @@ -1222,7 +1222,7 @@ static int osdmap_set_max_osd(struct ceph_osdmap *map, u32 max) if (map->osd_primary_affinity) { u32 *affinity; - affinity = ceph_kvmalloc(array_size(max, sizeof(*affinity)), + affinity = kvmalloc(array_size(max, sizeof(*affinity)), GFP_NOFS); if (!affinity) return -ENOMEM; @@ -1503,7 +1503,7 @@ static int set_primary_affinity(struct ceph_osdmap *map, int osd, u32 aff) if (!map->osd_primary_affinity) { int i; - map->osd_primary_affinity = ceph_kvmalloc( + map->osd_primary_affinity = kvmalloc( array_size(map->max_osd, sizeof(*map->osd_primary_affinity)), GFP_NOFS); if (!map->osd_primary_affinity) -- cgit v1.2.3-59-g8ed1b From 4034247a0d6ab281ba3293798ce67af494d86129 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 14 Jan 2022 14:07:14 -0800 Subject: mm: introduce memalloc_retry_wait() Various places in the kernel - largely in filesystems - respond to a memory allocation failure by looping around and re-trying. Some of these cannot conveniently use __GFP_NOFAIL, for reasons such as: - a GFP_ATOMIC allocation, which __GFP_NOFAIL doesn't work on - a need to check for the process being signalled between failures - the possibility that other recovery actions could be performed - the allocation is quite deep in support code, and passing down an extra flag to say if __GFP_NOFAIL is wanted would be clumsy. Many of these currently use congestion_wait() which (in almost all cases) simply waits the given timeout - congestion isn't tracked for most devices. It isn't clear what the best delay is for loops, but it is clear that the various filesystems shouldn't be responsible for choosing a timeout. This patch introduces memalloc_retry_wait() with takes on that responsibility. Code that wants to retry a memory allocation can call this function passing the GFP flags that were used. It will wait however is appropriate. For now, it only considers __GFP_NORETRY and whatever gfpflags_allow_blocking() tests. If blocking is allowed without __GFP_NORETRY, then alloc_page either made some reclaim progress, or waited for a while, before failing. So there is no need for much further waiting. memalloc_retry_wait() will wait until the current jiffie ends. If this condition is not met, then alloc_page() won't have waited much if at all. In that case memalloc_retry_wait() waits about 200ms. This is the delay that most current loops uses. linux/sched/mm.h needs to be included in some files now, but linux/backing-dev.h does not. Link: https://lkml.kernel.org/r/163754371968.13692.1277530886009912421@noble.neil.brown.name Signed-off-by: NeilBrown Cc: Dave Chinner Cc: Michal Hocko Cc: "Theodore Ts'o" Cc: Jaegeuk Kim Cc: Chao Yu Cc: Darrick J. Wong Cc: Chuck Lever Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ext4/extents.c | 8 +++----- fs/ext4/inline.c | 5 ++--- fs/ext4/page-io.c | 9 +++++---- fs/f2fs/data.c | 4 ++-- fs/f2fs/gc.c | 5 ++--- fs/f2fs/inode.c | 4 ++-- fs/f2fs/node.c | 4 ++-- fs/f2fs/recovery.c | 6 +++--- fs/f2fs/segment.c | 9 +++------ fs/f2fs/super.c | 5 ++--- fs/xfs/kmem.c | 3 +-- fs/xfs/xfs_buf.c | 2 +- include/linux/sched/mm.h | 26 ++++++++++++++++++++++++++ net/sunrpc/svc_xprt.c | 3 ++- 14 files changed, 56 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index 0ecf819bf189..5582fba36b44 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -27,8 +27,8 @@ #include #include #include -#include #include +#include #include "ext4_jbd2.h" #include "ext4_extents.h" #include "xattr.h" @@ -4407,8 +4407,7 @@ retry: err = ext4_es_remove_extent(inode, last_block, EXT_MAX_BLOCKS - last_block); if (err == -ENOMEM) { - cond_resched(); - congestion_wait(BLK_RW_ASYNC, HZ/50); + memalloc_retry_wait(GFP_ATOMIC); goto retry; } if (err) @@ -4416,8 +4415,7 @@ retry: retry_remove_space: err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1); if (err == -ENOMEM) { - cond_resched(); - congestion_wait(BLK_RW_ASYNC, HZ/50); + memalloc_retry_wait(GFP_ATOMIC); goto retry_remove_space; } return err; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 39a1ab129fdc..635bcf68a67e 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include "ext4_jbd2.h" #include "ext4.h" @@ -1929,8 +1929,7 @@ int ext4_inline_data_truncate(struct inode *inode, int *has_inline) retry: err = ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); if (err == -ENOMEM) { - cond_resched(); - congestion_wait(BLK_RW_ASYNC, HZ/50); + memalloc_retry_wait(GFP_ATOMIC); goto retry; } if (err) diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 9cb261714991..1d370364230e 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include "ext4_jbd2.h" #include "xattr.h" @@ -523,12 +523,13 @@ int ext4_bio_write_page(struct ext4_io_submit *io, ret = PTR_ERR(bounce_page); if (ret == -ENOMEM && (io->io_bio || wbc->sync_mode == WB_SYNC_ALL)) { - gfp_flags = GFP_NOFS; + gfp_t new_gfp_flags = GFP_NOFS; if (io->io_bio) ext4_io_submit(io); else - gfp_flags |= __GFP_NOFAIL; - congestion_wait(BLK_RW_ASYNC, HZ/50); + new_gfp_flags |= __GFP_NOFAIL; + memalloc_retry_wait(gfp_flags); + gfp_flags = new_gfp_flags; goto retry_encrypt; } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9f754aaef558..aacf5e4dcc57 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -8,9 +8,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -2542,7 +2542,7 @@ retry_encrypt: /* flush pending IOs and wait for a while in the ENOMEM case */ if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { f2fs_flush_merged_writes(fio->sbi); - congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); gfp_flags |= __GFP_NOFAIL; goto retry_encrypt; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index a946ce0ead34..374bbb5294d9 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -7,7 +7,6 @@ */ #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include #include #include +#include #include "f2fs.h" #include "node.h" @@ -1375,8 +1375,7 @@ retry: if (err) { clear_page_private_gcing(page); if (err == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, - DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto retry; } if (is_dirty) diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 0f8b2df3e1e0..4c11254a07d4 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -8,8 +8,8 @@ #include #include #include -#include #include +#include #include "f2fs.h" #include "node.h" @@ -562,7 +562,7 @@ retry: inode = f2fs_iget(sb, ino); if (IS_ERR(inode)) { if (PTR_ERR(inode) == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto retry; } } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 556fcd8457f3..219506ca9a97 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -2750,7 +2750,7 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct page *page) retry: ipage = f2fs_grab_cache_page(NODE_MAPPING(sbi), ino, false); if (!ipage) { - congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto retry; } diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 6a1b4668d933..d1664a0567ef 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "f2fs.h" #include "node.h" #include "segment.h" @@ -587,7 +588,7 @@ retry_dn: err = f2fs_get_dnode_of_data(&dn, start, ALLOC_NODE); if (err) { if (err == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto retry_dn; } goto out; @@ -670,8 +671,7 @@ retry_prev: err = check_index_in_prev_nodes(sbi, dest, &dn); if (err) { if (err == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, - DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto retry_prev; } goto err; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index df9ed75f0b7a..40fdb4a8daeb 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -245,9 +246,7 @@ retry: LOOKUP_NODE); if (err) { if (err == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, - DEFAULT_IO_TIMEOUT); - cond_resched(); + memalloc_retry_wait(GFP_NOFS); goto retry; } err = -EAGAIN; @@ -424,9 +423,7 @@ retry: err = f2fs_do_write_data_page(&fio); if (err) { if (err == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, - DEFAULT_IO_TIMEOUT); - cond_resched(); + memalloc_retry_wait(GFP_NOFS); goto retry; } unlock_page(page); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 040b6d02e1d8..3bace24f8800 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -8,9 +8,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -2415,8 +2415,7 @@ repeat: page = read_cache_page_gfp(mapping, blkidx, GFP_NOFS); if (IS_ERR(page)) { if (PTR_ERR(page) == -ENOMEM) { - congestion_wait(BLK_RW_ASYNC, - DEFAULT_IO_TIMEOUT); + memalloc_retry_wait(GFP_NOFS); goto repeat; } set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR); diff --git a/fs/xfs/kmem.c b/fs/xfs/kmem.c index 6f49bf39183c..c557a030acfe 100644 --- a/fs/xfs/kmem.c +++ b/fs/xfs/kmem.c @@ -4,7 +4,6 @@ * All Rights Reserved. */ #include "xfs.h" -#include #include "xfs_message.h" #include "xfs_trace.h" @@ -26,6 +25,6 @@ kmem_alloc(size_t size, xfs_km_flags_t flags) "%s(%u) possible memory allocation deadlock size %u in %s (mode:0x%x)", current->comm, current->pid, (unsigned int)size, __func__, lflags); - congestion_wait(BLK_RW_ASYNC, HZ/50); + memalloc_retry_wait(lflags); } while (1); } diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 631c5a61d89b..6c45e3fa56f4 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -394,7 +394,7 @@ xfs_buf_alloc_pages( } XFS_STATS_INC(bp->b_mount, xb_page_retries); - congestion_wait(BLK_RW_ASYNC, HZ / 50); + memalloc_retry_wait(gfp_mask); } return 0; } diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index aca874d33fe6..aa5f09ca5bcf 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -214,6 +214,32 @@ static inline void fs_reclaim_acquire(gfp_t gfp_mask) { } static inline void fs_reclaim_release(gfp_t gfp_mask) { } #endif +/* Any memory-allocation retry loop should use + * memalloc_retry_wait(), and pass the flags for the most + * constrained allocation attempt that might have failed. + * This provides useful documentation of where loops are, + * and a central place to fine tune the waiting as the MM + * implementation changes. + */ +static inline void memalloc_retry_wait(gfp_t gfp_flags) +{ + /* We use io_schedule_timeout because waiting for memory + * typically included waiting for dirty pages to be + * written out, which requires IO. + */ + __set_current_state(TASK_UNINTERRUPTIBLE); + gfp_flags = current_gfp_context(gfp_flags); + if (gfpflags_allow_blocking(gfp_flags) && + !(gfp_flags & __GFP_NORETRY)) + /* Probably waited already, no need for much more */ + io_schedule_timeout(1); + else + /* Probably didn't wait, and has now released a lock, + * so now is a good time to wait + */ + io_schedule_timeout(HZ/50); +} + /** * might_alloc - Mark possible allocation sites * @gfp_mask: gfp_t flags that would be used to allocate diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 1e99ba1b9d72..9cb18b822ab2 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -688,7 +689,7 @@ static int svc_alloc_arg(struct svc_rqst *rqstp) return -EINTR; } trace_svc_alloc_arg_err(pages); - schedule_timeout(msecs_to_jiffies(500)); + memalloc_retry_wait(GFP_KERNEL); } rqstp->rq_page_end = &rqstp->rq_pages[pages]; rqstp->rq_pages[pages] = NULL; /* this might be seen in nfsd_splice_actor() */ -- cgit v1.2.3-59-g8ed1b From 1611f74a94ba2e0f2d25b75008ed8e76e122097a Mon Sep 17 00:00:00 2001 From: Changcheng Deng Date: Fri, 14 Jan 2022 14:07:21 -0800 Subject: mm: fix boolreturn.cocci warning Return statements in functions returning bool should use true/false instead of 1/0. Link: https://lkml.kernel.org/r/20211126073327.74815-1-deng.changcheng@zte.com.cn Signed-off-by: Changcheng Deng Reported-by: Zeal Robot Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/page-flags.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index b5f14d581113..18423c2157e8 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -383,7 +383,7 @@ static __always_inline int TestClearPage##uname(struct page *page) \ TESTCLEARFLAG(uname, lname, policy) #define TESTPAGEFLAG_FALSE(uname, lname) \ -static inline bool folio_test_##lname(const struct folio *folio) { return 0; } \ +static inline bool folio_test_##lname(const struct folio *folio) { return false; } \ static inline int Page##uname(const struct page *page) { return 0; } #define SETPAGEFLAG_NOOP(uname, lname) \ -- cgit v1.2.3-59-g8ed1b From be1a13eb51077b2ec5f7f4306f93dfece503a3f1 Mon Sep 17 00:00:00 2001 From: Michal Hocko Date: Fri, 14 Jan 2022 14:07:27 -0800 Subject: mm: drop node from alloc_pages_vma alloc_pages_vma is meant to allocate a page with a vma specific memory policy. The initial node parameter is always a local node so it is pointless to waste a function argument for this. Drop the parameter. Link: https://lkml.kernel.org/r/YaSnlv4QpryEpesG@dhcp22.suse.cz Signed-off-by: Michal Hocko Cc: Aneesh Kumar K.V Cc: Ben Widawsky Cc: Dave Hansen Cc: Feng Tang Cc: Andrea Arcangeli Cc: Mel Gorman Cc: Mike Kravetz Cc: Randy Dunlap Cc: Vlastimil Babka Cc: Andi Kleen Cc: Dan Williams Cc: "Huang, Ying" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/gfp.h | 8 ++++---- mm/mempolicy.c | 3 ++- mm/shmem.c | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 8fcc38467af6..78b58448f796 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -598,9 +598,9 @@ struct page *alloc_pages(gfp_t gfp, unsigned int order); struct folio *folio_alloc(gfp_t gfp, unsigned order); extern struct page *alloc_pages_vma(gfp_t gfp_mask, int order, struct vm_area_struct *vma, unsigned long addr, - int node, bool hugepage); + bool hugepage); #define alloc_hugepage_vma(gfp_mask, vma, addr, order) \ - alloc_pages_vma(gfp_mask, order, vma, addr, numa_node_id(), true) + alloc_pages_vma(gfp_mask, order, vma, addr, true) #else static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order) { @@ -610,14 +610,14 @@ static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order) { return __folio_alloc_node(gfp, order, numa_node_id()); } -#define alloc_pages_vma(gfp_mask, order, vma, addr, node, false)\ +#define alloc_pages_vma(gfp_mask, order, vma, addr, false)\ alloc_pages(gfp_mask, order) #define alloc_hugepage_vma(gfp_mask, vma, addr, order) \ alloc_pages(gfp_mask, order) #endif #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0) #define alloc_page_vma(gfp_mask, vma, addr) \ - alloc_pages_vma(gfp_mask, 0, vma, addr, numa_node_id(), false) + alloc_pages_vma(gfp_mask, 0, vma, addr, false) extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order); extern unsigned long get_zeroed_page(gfp_t gfp_mask); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 679f47b3a079..ed7d15acb6a2 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2084,9 +2084,10 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, * Return: The page on success or NULL if allocation fails. */ struct page *alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, - unsigned long addr, int node, bool hugepage) + unsigned long addr, bool hugepage) { struct mempolicy *pol; + int node = numa_node_id(); struct page *page; int preferred_nid; nodemask_t *nmask; diff --git a/mm/shmem.c b/mm/shmem.c index 8f940552c182..0700e9acf53b 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1564,8 +1564,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, return NULL; shmem_pseudo_vma_init(&pvma, info, hindex); - page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), - true); + page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, true); shmem_pseudo_vma_destroy(&pvma); if (page) prep_transhuge_page(page); -- cgit v1.2.3-59-g8ed1b From 04a536bfbd0f885338eecc2a4503dfca50ac94dd Mon Sep 17 00:00:00 2001 From: Miles Chen Date: Fri, 14 Jan 2022 14:07:30 -0800 Subject: include/linux/gfp.h: further document GFP_DMA32 kmalloc(..., GFP_DMA32) does not return DMA32 memory because the DMA32 kmalloc cache array is not implemented. (Reason: there is no such user in kernel). Put a short comment about this so people can understand this by reading the comment. [1] https://lists.linuxfoundation.org/pipermail/iommu/2018-December/031696.html Link: https://lkml.kernel.org/r/20211207093610.6406-1-miles.chen@mediatek.com Signed-off-by: Miles Chen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/gfp.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 78b58448f796..80f63c862be5 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -302,7 +302,9 @@ struct vm_area_struct; * lowest zone as a type of emergency reserve. * * %GFP_DMA32 is similar to %GFP_DMA except that the caller requires a 32-bit - * address. + * address. Note that kmalloc(..., GFP_DMA32) does not return DMA32 memory + * because the DMA32 kmalloc cache array is not implemented. + * (Reason: there is no such user in kernel). * * %GFP_HIGHUSER is for userspace allocations that may be mapped to userspace, * do not need to be directly accessible by the kernel but that cannot -- cgit v1.2.3-59-g8ed1b From 62b3107073646e0946bd97ff926832bafb846d17 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Fri, 14 Jan 2022 14:07:37 -0800 Subject: mm_zone: add function to check if managed dma zone exists Patch series "Handle warning of allocation failure on DMA zone w/o managed pages", v4. **Problem observed: On x86_64, when crash is triggered and entering into kdump kernel, page allocation failure can always be seen. --------------------------------- DMA: preallocated 128 KiB GFP_KERNEL pool for atomic allocations swapper/0: page allocation failure: order:5, mode:0xcc1(GFP_KERNEL|GFP_DMA), nodemask=(null),cpuset=/,mems_allowed=0 CPU: 0 PID: 1 Comm: swapper/0 Call Trace: dump_stack+0x7f/0xa1 warn_alloc.cold+0x72/0xd6 ...... __alloc_pages+0x24d/0x2c0 ...... dma_atomic_pool_init+0xdb/0x176 do_one_initcall+0x67/0x320 ? rcu_read_lock_sched_held+0x3f/0x80 kernel_init_freeable+0x290/0x2dc ? rest_init+0x24f/0x24f kernel_init+0xa/0x111 ret_from_fork+0x22/0x30 Mem-Info: ------------------------------------ ***Root cause: In the current kernel, it assumes that DMA zone must have managed pages and try to request pages if CONFIG_ZONE_DMA is enabled. While this is not always true. E.g in kdump kernel of x86_64, only low 1M is presented and locked down at very early stage of boot, so that this low 1M won't be added into buddy allocator to become managed pages of DMA zone. This exception will always cause page allocation failure if page is requested from DMA zone. ***Investigation: This failure happens since below commit merged into linus's tree. 1a6a9044b967 x86/setup: Remove CONFIG_X86_RESERVE_LOW and reservelow= options 23721c8e92f7 x86/crash: Remove crash_reserve_low_1M() f1d4d47c5851 x86/setup: Always reserve the first 1M of RAM 7c321eb2b843 x86/kdump: Remove the backup region handling 6f599d84231f x86/kdump: Always reserve the low 1M when the crashkernel option is specified Before them, on x86_64, the low 640K area will be reused by kdump kernel. So in kdump kernel, the content of low 640K area is copied into a backup region for dumping before jumping into kdump. Then except of those firmware reserved region in [0, 640K], the left area will be added into buddy allocator to become available managed pages of DMA zone. However, after above commits applied, in kdump kernel of x86_64, the low 1M is reserved by memblock, but not released to buddy allocator. So any later page allocation requested from DMA zone will fail. At the beginning, if crashkernel is reserved, the low 1M need be locked down because AMD SME encrypts memory making the old backup region mechanims impossible when switching into kdump kernel. Later, it was also observed that there are BIOSes corrupting memory under 1M. To solve this, in commit f1d4d47c5851, the entire region of low 1M is always reserved after the real mode trampoline is allocated. Besides, recently, Intel engineer mentioned their TDX (Trusted domain extensions) which is under development in kernel also needs to lock down the low 1M. So we can't simply revert above commits to fix the page allocation failure from DMA zone as someone suggested. ***Solution: Currently, only DMA atomic pool and dma-kmalloc will initialize and request page allocation with GFP_DMA during bootup. So only initializ DMA atomic pool when DMA zone has available managed pages, otherwise just skip the initialization. For dma-kmalloc(), for the time being, let's mute the warning of allocation failure if requesting pages from DMA zone while no manged pages. Meanwhile, change code to use dma_alloc_xx/dma_map_xx API to replace kmalloc(GFP_DMA), or do not use GFP_DMA when calling kmalloc() if not necessary. Christoph is posting patches to fix those under drivers/scsi/. Finally, we can remove the need of dma-kmalloc() as people suggested. This patch (of 3): In some places of the current kernel, it assumes that dma zone must have managed pages if CONFIG_ZONE_DMA is enabled. While this is not always true. E.g in kdump kernel of x86_64, only low 1M is presented and locked down at very early stage of boot, so that there's no managed pages at all in DMA zone. This exception will always cause page allocation failure if page is requested from DMA zone. Here add function has_managed_dma() and the relevant helper functions to check if there's DMA zone with managed pages. It will be used in later patches. Link: https://lkml.kernel.org/r/20211223094435.248523-1-bhe@redhat.com Link: https://lkml.kernel.org/r/20211223094435.248523-2-bhe@redhat.com Fixes: 6f599d84231f ("x86/kdump: Always reserve the low 1M when the crashkernel option is specified") Signed-off-by: Baoquan He Reviewed-by: David Hildenbrand Acked-by: John Donnelly Cc: Christoph Hellwig Cc: Christoph Lameter Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Vlastimil Babka Cc: David Laight Cc: Borislav Petkov Cc: Marek Szyprowski Cc: Robin Murphy Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 9 +++++++++ mm/page_alloc.c | 15 +++++++++++++++ 2 files changed, 24 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 936dc0b6c226..aed44e9b5d89 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1047,6 +1047,15 @@ static inline int is_highmem_idx(enum zone_type idx) #endif } +#ifdef CONFIG_ZONE_DMA +bool has_managed_dma(void); +#else +static inline bool has_managed_dma(void) +{ + return false; +} +#endif + /** * is_highmem - helper function to quickly check if a struct zone is a * highmem zone or not. This is an attempt to keep references diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3eace0065ecc..e2ad2303f634 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -9518,3 +9518,18 @@ bool take_page_off_buddy(struct page *page) return ret; } #endif + +#ifdef CONFIG_ZONE_DMA +bool has_managed_dma(void) +{ + struct pglist_data *pgdat; + + for_each_online_pgdat(pgdat) { + struct zone *zone = &pgdat->node_zones[ZONE_DMA]; + + if (managed_zone(zone)) + return true; + } + return false; +} +#endif /* CONFIG_ZONE_DMA */ -- cgit v1.2.3-59-g8ed1b From f47761999052b1cc987dd3e3d3adf47997358fc0 Mon Sep 17 00:00:00 2001 From: Mina Almasry Date: Fri, 14 Jan 2022 14:07:48 -0800 Subject: hugetlb: add hugetlb.*.numa_stat file For hugetlb backed jobs/VMs it's critical to understand the numa information for the memory backing these jobs to deliver optimal performance. Currently this technically can be queried from /proc/self/numa_maps, but there are significant issues with that. Namely: 1. Memory can be mapped or unmapped. 2. numa_maps are per process and need to be aggregated across all processes in the cgroup. For shared memory this is more involved as the userspace needs to make sure it doesn't double count shared mappings. 3. I believe querying numa_maps needs to hold the mmap_lock which adds to the contention on this lock. For these reasons I propose simply adding hugetlb.*.numa_stat file, which shows the numa information of the cgroup similarly to memory.numa_stat. On cgroup-v2: cat /sys/fs/cgroup/unified/test/hugetlb.2MB.numa_stat total=2097152 N0=2097152 N1=0 On cgroup-v1: cat /sys/fs/cgroup/hugetlb/test/hugetlb.2MB.numa_stat total=2097152 N0=2097152 N1=0 hierarichal_total=2097152 N0=2097152 N1=0 This patch was tested manually by allocating hugetlb memory and querying the hugetlb.*.numa_stat file of the cgroup and its parents. [colin.i.king@googlemail.com: fix spelling mistake "hierarichal" -> "hierarchical"] Link: https://lkml.kernel.org/r/20211125090635.23508-1-colin.i.king@gmail.com [keescook@chromium.org: fix copy/paste array assignment] Link: https://lkml.kernel.org/r/20211203065647.2819707-1-keescook@chromium.org Link: https://lkml.kernel.org/r/20211123001020.4083653-1-almasrymina@google.com Signed-off-by: Mina Almasry Signed-off-by: Colin Ian King Signed-off-by: Kees Cook Reviewed-by: Shakeel Butt Reviewed-by: Muchun Song Reviewed-by: Mike Kravetz Cc: Shuah Khan Cc: Miaohe Lin Cc: Oscar Salvador Cc: Michal Hocko Cc: David Rientjes Cc: Jue Wang Cc: Yang Yao Cc: Joanna Li Cc: Cannon Matthews Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/cgroup-v1/hugetlb.rst | 4 + Documentation/admin-guide/cgroup-v2.rst | 5 + include/linux/hugetlb.h | 4 +- include/linux/hugetlb_cgroup.h | 7 ++ mm/hugetlb_cgroup.c | 133 ++++++++++++++++++++++-- 5 files changed, 141 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/cgroup-v1/hugetlb.rst b/Documentation/admin-guide/cgroup-v1/hugetlb.rst index 338f2c7d7a1c..0fa724d82abb 100644 --- a/Documentation/admin-guide/cgroup-v1/hugetlb.rst +++ b/Documentation/admin-guide/cgroup-v1/hugetlb.rst @@ -29,12 +29,14 @@ Brief summary of control files:: hugetlb..max_usage_in_bytes # show max "hugepagesize" hugetlb usage recorded hugetlb..usage_in_bytes # show current usage for "hugepagesize" hugetlb hugetlb..failcnt # show the number of allocation failure due to HugeTLB usage limit + hugetlb..numa_stat # show the numa information of the hugetlb memory charged to this cgroup For a system supporting three hugepage sizes (64k, 32M and 1G), the control files include:: hugetlb.1GB.limit_in_bytes hugetlb.1GB.max_usage_in_bytes + hugetlb.1GB.numa_stat hugetlb.1GB.usage_in_bytes hugetlb.1GB.failcnt hugetlb.1GB.rsvd.limit_in_bytes @@ -43,6 +45,7 @@ files include:: hugetlb.1GB.rsvd.failcnt hugetlb.64KB.limit_in_bytes hugetlb.64KB.max_usage_in_bytes + hugetlb.64KB.numa_stat hugetlb.64KB.usage_in_bytes hugetlb.64KB.failcnt hugetlb.64KB.rsvd.limit_in_bytes @@ -51,6 +54,7 @@ files include:: hugetlb.64KB.rsvd.failcnt hugetlb.32MB.limit_in_bytes hugetlb.32MB.max_usage_in_bytes + hugetlb.32MB.numa_stat hugetlb.32MB.usage_in_bytes hugetlb.32MB.failcnt hugetlb.32MB.rsvd.limit_in_bytes diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 4f400b03dddf..5aa368d165da 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2266,6 +2266,11 @@ HugeTLB Interface Files are local to the cgroup i.e. not hierarchical. The file modified event generated on this file reflects only the local events. + hugetlb..numa_stat + Similar to memory.numa_stat, it shows the numa information of the + hugetlb pages of in this cgroup. Only active in + use hugetlb pages are included. The per-node values are in bytes. + Misc ---- diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 00351ccb49a3..d1897a69c540 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -622,8 +622,8 @@ struct hstate { #endif #ifdef CONFIG_CGROUP_HUGETLB /* cgroup control files */ - struct cftype cgroup_files_dfl[7]; - struct cftype cgroup_files_legacy[9]; + struct cftype cgroup_files_dfl[8]; + struct cftype cgroup_files_legacy[10]; #endif char name[HSTATE_NAME_LEN]; }; diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index ba025ae27882..379344828e78 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -36,6 +36,11 @@ enum hugetlb_memory_event { HUGETLB_NR_MEMORY_EVENTS, }; +struct hugetlb_cgroup_per_node { + /* hugetlb usage in pages over all hstates. */ + unsigned long usage[HUGE_MAX_HSTATE]; +}; + struct hugetlb_cgroup { struct cgroup_subsys_state css; @@ -57,6 +62,8 @@ struct hugetlb_cgroup { /* Handle for "hugetlb.events.local" */ struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; + + struct hugetlb_cgroup_per_node *nodeinfo[]; }; static inline struct hugetlb_cgroup * diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 79d93534ef1e..f9942841df18 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -123,29 +123,58 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, } } +static void hugetlb_cgroup_free(struct hugetlb_cgroup *h_cgroup) +{ + int node; + + for_each_node(node) + kfree(h_cgroup->nodeinfo[node]); + kfree(h_cgroup); +} + static struct cgroup_subsys_state * hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) { struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css); struct hugetlb_cgroup *h_cgroup; + int node; + + h_cgroup = kzalloc(struct_size(h_cgroup, nodeinfo, nr_node_ids), + GFP_KERNEL); - h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL); if (!h_cgroup) return ERR_PTR(-ENOMEM); if (!parent_h_cgroup) root_h_cgroup = h_cgroup; + /* + * TODO: this routine can waste much memory for nodes which will + * never be onlined. It's better to use memory hotplug callback + * function. + */ + for_each_node(node) { + /* Set node_to_alloc to -1 for offline nodes. */ + int node_to_alloc = + node_state(node, N_NORMAL_MEMORY) ? node : -1; + h_cgroup->nodeinfo[node] = + kzalloc_node(sizeof(struct hugetlb_cgroup_per_node), + GFP_KERNEL, node_to_alloc); + if (!h_cgroup->nodeinfo[node]) + goto fail_alloc_nodeinfo; + } + hugetlb_cgroup_init(h_cgroup, parent_h_cgroup); return &h_cgroup->css; + +fail_alloc_nodeinfo: + hugetlb_cgroup_free(h_cgroup); + return ERR_PTR(-ENOMEM); } static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) { - struct hugetlb_cgroup *h_cgroup; - - h_cgroup = hugetlb_cgroup_from_css(css); - kfree(h_cgroup); + hugetlb_cgroup_free(hugetlb_cgroup_from_css(css)); } /* @@ -289,7 +318,17 @@ static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, return; __set_hugetlb_cgroup(page, h_cg, rsvd); - return; + if (!rsvd) { + unsigned long usage = + h_cg->nodeinfo[page_to_nid(page)]->usage[idx]; + /* + * This write is not atomic due to fetching usage and writing + * to it, but that's fine because we call this with + * hugetlb_lock held anyway. + */ + WRITE_ONCE(h_cg->nodeinfo[page_to_nid(page)]->usage[idx], + usage + nr_pages); + } } void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, @@ -328,8 +367,17 @@ static void __hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, if (rsvd) css_put(&h_cg->css); - - return; + else { + unsigned long usage = + h_cg->nodeinfo[page_to_nid(page)]->usage[idx]; + /* + * This write is not atomic due to fetching usage and writing + * to it, but that's fine because we call this with + * hugetlb_lock held anyway. + */ + WRITE_ONCE(h_cg->nodeinfo[page_to_nid(page)]->usage[idx], + usage - nr_pages); + } } void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, @@ -418,6 +466,59 @@ enum { RES_RSVD_FAILCNT, }; +static int hugetlb_cgroup_read_numa_stat(struct seq_file *seq, void *dummy) +{ + int nid; + struct cftype *cft = seq_cft(seq); + int idx = MEMFILE_IDX(cft->private); + bool legacy = MEMFILE_ATTR(cft->private); + struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); + struct cgroup_subsys_state *css; + unsigned long usage; + + if (legacy) { + /* Add up usage across all nodes for the non-hierarchical total. */ + usage = 0; + for_each_node_state(nid, N_MEMORY) + usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]); + seq_printf(seq, "total=%lu", usage * PAGE_SIZE); + + /* Simply print the per-node usage for the non-hierarchical total. */ + for_each_node_state(nid, N_MEMORY) + seq_printf(seq, " N%d=%lu", nid, + READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) * + PAGE_SIZE); + seq_putc(seq, '\n'); + } + + /* + * The hierarchical total is pretty much the value recorded by the + * counter, so use that. + */ + seq_printf(seq, "%stotal=%lu", legacy ? "hierarchical_" : "", + page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE); + + /* + * For each node, transverse the css tree to obtain the hierarchical + * node usage. + */ + for_each_node_state(nid, N_MEMORY) { + usage = 0; + rcu_read_lock(); + css_for_each_descendant_pre(css, &h_cg->css) { + usage += READ_ONCE(hugetlb_cgroup_from_css(css) + ->nodeinfo[nid] + ->usage[idx]); + } + rcu_read_unlock(); + seq_printf(seq, " N%d=%lu", nid, usage * PAGE_SIZE); + } + + seq_putc(seq, '\n'); + + return 0; +} + static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -668,8 +769,14 @@ static void __init __hugetlb_cgroup_file_dfl_init(int idx) events_local_file[idx]); cft->flags = CFTYPE_NOT_ON_ROOT; - /* NULL terminate the last cft */ + /* Add the numa stat file */ cft = &h->cgroup_files_dfl[6]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.numa_stat", buf); + cft->seq_show = hugetlb_cgroup_read_numa_stat; + cft->flags = CFTYPE_NOT_ON_ROOT; + + /* NULL terminate the last cft */ + cft = &h->cgroup_files_dfl[7]; memset(cft, 0, sizeof(*cft)); WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys, @@ -739,8 +846,14 @@ static void __init __hugetlb_cgroup_file_legacy_init(int idx) cft->write = hugetlb_cgroup_reset; cft->read_u64 = hugetlb_cgroup_read_u64; - /* NULL terminate the last cft */ + /* Add the numa stat file */ cft = &h->cgroup_files_legacy[8]; + snprintf(cft->name, MAX_CFTYPE_NAME, "%s.numa_stat", buf); + cft->private = MEMFILE_PRIVATE(idx, 1); + cft->seq_show = hugetlb_cgroup_read_numa_stat; + + /* NULL terminate the last cft */ + cft = &h->cgroup_files_legacy[9]; memset(cft, 0, sizeof(*cft)); WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, -- cgit v1.2.3-59-g8ed1b From e9ea874a8ffb0f8ebed4f4981531a32c5b663d79 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Fri, 14 Jan 2022 14:07:55 -0800 Subject: mm/vmstat: add events for THP max_ptes_* exceeds There are interfaces to adjust max_ptes_none, max_ptes_swap, max_ptes_shared values, see /sys/kernel/mm/transparent_hugepage/khugepaged/. But system administrator may not know which value is the best. So Add those events to support adjusting max_ptes_* to suitable values. For example, if default max_ptes_swap value causes too much failures, and system uses zram whose IO is fast, administrator could increase max_ptes_swap until THP_SCAN_EXCEED_SWAP_PTE not increase anymore. Link: https://lkml.kernel.org/r/20211225094036.574157-1-yang.yang29@zte.com.cn Signed-off-by: Yang Yang Cc: "Huang, Ying" Cc: Dave Hansen Cc: Minchan Kim Cc: Saravanan D Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/vm_event_item.h | 3 +++ mm/khugepaged.c | 7 +++++++ mm/vmstat.c | 3 +++ 3 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index a185cc75ff52..7b2363388bfa 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -98,6 +98,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, THP_SPLIT_PAGE_FAILED, THP_DEFERRED_SPLIT_PAGE, THP_SPLIT_PMD, + THP_SCAN_EXCEED_NONE_PTE, + THP_SCAN_EXCEED_SWAP_PTE, + THP_SCAN_EXCEED_SHARED_PTE, #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD THP_SPLIT_PUD, #endif diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 698ea19775ac..02071f213c58 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -618,6 +618,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, continue; } else { result = SCAN_EXCEED_NONE_PTE; + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); goto out; } } @@ -636,6 +637,7 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, if (page_mapcount(page) > 1 && ++shared > khugepaged_max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; + count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); goto out; } @@ -1253,6 +1255,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, continue; } else { result = SCAN_EXCEED_SWAP_PTE; + count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); goto out_unmap; } } @@ -1262,6 +1265,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, continue; } else { result = SCAN_EXCEED_NONE_PTE; + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); goto out_unmap; } } @@ -1290,6 +1294,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, if (page_mapcount(page) > 1 && ++shared > khugepaged_max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; + count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); goto out_unmap; } @@ -2000,6 +2005,7 @@ static void khugepaged_scan_file(struct mm_struct *mm, if (xa_is_value(page)) { if (++swap > khugepaged_max_ptes_swap) { result = SCAN_EXCEED_SWAP_PTE; + count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); break; } continue; @@ -2046,6 +2052,7 @@ static void khugepaged_scan_file(struct mm_struct *mm, if (result == SCAN_SUCCEED) { if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { node = khugepaged_find_target_node(); collapse_file(mm, file, start, hpage, node); diff --git a/mm/vmstat.c b/mm/vmstat.c index d701c335628c..4057372745d0 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1353,6 +1353,9 @@ const char * const vmstat_text[] = { "thp_split_page_failed", "thp_deferred_split_page", "thp_split_pmd", + "thp_scan_exceed_none_pte", + "thp_scan_exceed_swap_pte", + "thp_scan_exceed_share_pte", #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD "thp_split_pud", #endif -- cgit v1.2.3-59-g8ed1b From e4b424b7ec8791087375bb1f2480a3ba05d21e0b Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 14 Jan 2022 14:08:07 -0800 Subject: vmscan: make drop_slab_node static drop_slab_node is only used in drop_slab. So remove it's declaration from header file and add keyword static for it's definition. Link: https://lkml.kernel.org/r/20211111062445.5236-1-ligang.bdlg@bytedance.com Signed-off-by: Gang Li Reviewed-by: David Hildenbrand Reviewed-by: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 1 - mm/vmscan.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index cef65f9cbdf2..eb67eb699b78 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3122,7 +3122,6 @@ int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *, #endif void drop_slab(void); -void drop_slab_node(int nid); #ifndef CONFIG_MMU #define randomize_va_space 0 diff --git a/mm/vmscan.c b/mm/vmscan.c index 700434db5735..090bfb605ecf 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -951,7 +951,7 @@ out: return freed; } -void drop_slab_node(int nid) +static void drop_slab_node(int nid) { unsigned long freed; int shift = 0; -- cgit v1.2.3-59-g8ed1b From c6018b4b254971863bd0ad36bb5e7d0fa0f0ddb0 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Fri, 14 Jan 2022 14:08:17 -0800 Subject: mm/mempolicy: add set_mempolicy_home_node syscall This syscall can be used to set a home node for the MPOL_BIND and MPOL_PREFERRED_MANY memory policy. Users should use this syscall after setting up a memory policy for the specified range as shown below. mbind(p, nr_pages * page_size, MPOL_BIND, new_nodes->maskp, new_nodes->size + 1, 0); sys_set_mempolicy_home_node((unsigned long)p, nr_pages * page_size, home_node, 0); The syscall allows specifying a home node/preferred node from which kernel will fulfill memory allocation requests first. For address range with MPOL_BIND memory policy, if nodemask specifies more than one node, page allocations will come from the node in the nodemask with sufficient free memory that is closest to the home node/preferred node. For MPOL_PREFERRED_MANY if the nodemask specifies more than one node, page allocation will come from the node in the nodemask with sufficient free memory that is closest to the home node/preferred node. If there is not enough memory in all the nodes specified in the nodemask, the allocation will be attempted from the closest numa node to the home node in the system. This helps applications to hint at a memory allocation preference node and fallback to _only_ a set of nodes if the memory is not available on the preferred node. Fallback allocation is attempted from the node which is nearest to the preferred node. This helps applications to have control on memory allocation numa nodes and avoids default fallback to slow memory NUMA nodes. For example a system with NUMA nodes 1,2 and 3 with DRAM memory and 10, 11 and 12 of slow memory new_nodes = numa_bitmask_alloc(nr_nodes); numa_bitmask_setbit(new_nodes, 1); numa_bitmask_setbit(new_nodes, 2); numa_bitmask_setbit(new_nodes, 3); p = mmap(NULL, nr_pages * page_size, protflag, mapflag, -1, 0); mbind(p, nr_pages * page_size, MPOL_BIND, new_nodes->maskp, new_nodes->size + 1, 0); sys_set_mempolicy_home_node(p, nr_pages * page_size, 2, 0); This will allocate from nodes closer to node 2 and will make sure the kernel will only allocate from nodes 1, 2, and 3. Memory will not be allocated from slow memory nodes 10, 11, and 12. This differs from default MPOL_BIND behavior in that with default MPOL_BIND the allocation will be attempted from node closer to the local node. One of the reasons to specify a home node is to allow allocations from cpu less NUMA node and its nearby NUMA nodes. With MPOL_PREFERRED_MANY on the other hand will first try to allocate from the closest node to node 2 from the node list 1, 2 and 3. If those nodes don't have enough memory, kernel will allocate from slow memory node 10, 11 and 12 which ever is closer to node 2. Link: https://lkml.kernel.org/r/20211202123810.267175-3-aneesh.kumar@linux.ibm.com Signed-off-by: Aneesh Kumar K.V Cc: Ben Widawsky Cc: Dave Hansen Cc: Feng Tang Cc: Michal Hocko Cc: Andrea Arcangeli Cc: Mel Gorman Cc: Mike Kravetz Cc: Randy Dunlap Cc: Vlastimil Babka Cc: Andi Kleen Cc: Dan Williams Cc: Huang Ying Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- .../admin-guide/mm/numa_memory_policy.rst | 16 ++++- include/linux/mempolicy.h | 1 + mm/mempolicy.c | 79 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/mm/numa_memory_policy.rst b/Documentation/admin-guide/mm/numa_memory_policy.rst index 64fd0ba0d057..5a6afecbb0d0 100644 --- a/Documentation/admin-guide/mm/numa_memory_policy.rst +++ b/Documentation/admin-guide/mm/numa_memory_policy.rst @@ -408,7 +408,7 @@ follows: Memory Policy APIs ================== -Linux supports 3 system calls for controlling memory policy. These APIS +Linux supports 4 system calls for controlling memory policy. These APIS always affect only the calling task, the calling task's address space, or some shared object mapped into the calling task's address space. @@ -460,6 +460,20 @@ requested via the 'flags' argument. See the mbind(2) man page for more details. +Set home node for a Range of Task's Address Spacec:: + + long sys_set_mempolicy_home_node(unsigned long start, unsigned long len, + unsigned long home_node, + unsigned long flags); + +sys_set_mempolicy_home_node set the home node for a VMA policy present in the +task's address range. The system call updates the home node only for the existing +mempolicy range. Other address ranges are ignored. A home node is the NUMA node +closest to which page allocation will come from. Specifying the home node override +the default allocation policy to allocate memory close to the local node for an +executing CPU. + + Memory Policy Command Line Interface ==================================== diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 3c7595e81150..668389b4b53d 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -46,6 +46,7 @@ struct mempolicy { unsigned short mode; /* See MPOL_* above */ unsigned short flags; /* See set_mempolicy() MPOL_F_* above */ nodemask_t nodes; /* interleave/bind/perfer */ + int home_node; /* Home node to use for MPOL_BIND and MPOL_PREFERRED_MANY */ union { nodemask_t cpuset_mems_allowed; /* relative to these nodes */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index de70f119984a..fc6cae7926f3 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -296,6 +296,7 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, atomic_set(&policy->refcnt, 1); policy->mode = mode; policy->flags = flags; + policy->home_node = NUMA_NO_NODE; return policy; } @@ -1478,6 +1479,77 @@ static long kernel_mbind(unsigned long start, unsigned long len, return do_mbind(start, len, lmode, mode_flags, &nodes, flags); } +SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len, + unsigned long, home_node, unsigned long, flags) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + struct mempolicy *new; + unsigned long vmstart; + unsigned long vmend; + unsigned long end; + int err = -ENOENT; + + start = untagged_addr(start); + if (start & ~PAGE_MASK) + return -EINVAL; + /* + * flags is used for future extension if any. + */ + if (flags != 0) + return -EINVAL; + + /* + * Check home_node is online to avoid accessing uninitialized + * NODE_DATA. + */ + if (home_node >= MAX_NUMNODES || !node_online(home_node)) + return -EINVAL; + + len = (len + PAGE_SIZE - 1) & PAGE_MASK; + end = start + len; + + if (end < start) + return -EINVAL; + if (end == start) + return 0; + mmap_write_lock(mm); + vma = find_vma(mm, start); + for (; vma && vma->vm_start < end; vma = vma->vm_next) { + + vmstart = max(start, vma->vm_start); + vmend = min(end, vma->vm_end); + new = mpol_dup(vma_policy(vma)); + if (IS_ERR(new)) { + err = PTR_ERR(new); + break; + } + /* + * Only update home node if there is an existing vma policy + */ + if (!new) + continue; + + /* + * If any vma in the range got policy other than MPOL_BIND + * or MPOL_PREFERRED_MANY we return error. We don't reset + * the home node for vmas we already updated before. + */ + if (new->mode != MPOL_BIND && new->mode != MPOL_PREFERRED_MANY) { + err = -EOPNOTSUPP; + break; + } + + new->home_node = home_node; + err = mbind_range(mm, vmstart, vmend, new); + mpol_put(new); + if (err) + break; + } + mmap_write_unlock(mm); + return err; +} + SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len, unsigned long, mode, const unsigned long __user *, nmask, unsigned long, maxnode, unsigned int, flags) @@ -1802,6 +1874,11 @@ static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd) WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE)); } + if ((policy->mode == MPOL_BIND || + policy->mode == MPOL_PREFERRED_MANY) && + policy->home_node != NUMA_NO_NODE) + return policy->home_node; + return nd; } @@ -2344,6 +2421,8 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) return false; if (a->flags != b->flags) return false; + if (a->home_node != b->home_node) + return false; if (mpol_store_user_nodemask(a)) if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask)) return false; -- cgit v1.2.3-59-g8ed1b From 21b084fdf2a49ca1634e8e360e9ab6f9ff0dee11 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Fri, 14 Jan 2022 14:08:21 -0800 Subject: mm/mempolicy: wire up syscall set_mempolicy_home_node Link: https://lkml.kernel.org/r/20211202123810.267175-4-aneesh.kumar@linux.ibm.com Signed-off-by: Aneesh Kumar K.V Cc: Ben Widawsky Cc: Dave Hansen Cc: Feng Tang Cc: Michal Hocko Cc: Andrea Arcangeli Cc: Mel Gorman Cc: Mike Kravetz Cc: Randy Dunlap Cc: Vlastimil Babka Cc: Andi Kleen Cc: Dan Williams Cc: Huang Ying Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/kernel/syscalls/syscall.tbl | 1 + arch/arm/tools/syscall.tbl | 1 + arch/arm64/include/asm/unistd.h | 2 +- arch/arm64/include/asm/unistd32.h | 2 ++ arch/ia64/kernel/syscalls/syscall.tbl | 1 + arch/m68k/kernel/syscalls/syscall.tbl | 1 + arch/microblaze/kernel/syscalls/syscall.tbl | 1 + arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + arch/parisc/kernel/syscalls/syscall.tbl | 1 + arch/powerpc/kernel/syscalls/syscall.tbl | 1 + arch/s390/kernel/syscalls/syscall.tbl | 1 + arch/sh/kernel/syscalls/syscall.tbl | 1 + arch/sparc/kernel/syscalls/syscall.tbl | 1 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + arch/xtensa/kernel/syscalls/syscall.tbl | 1 + include/linux/syscalls.h | 3 +++ include/uapi/asm-generic/unistd.h | 5 ++++- kernel/sys_ni.c | 1 + 21 files changed, 27 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index ca5a32228cd6..3515bc4f16a4 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -489,3 +489,4 @@ # 557 reserved for memfd_secret 558 common process_mrelease sys_process_mrelease 559 common futex_waitv sys_futex_waitv +560 common set_mempolicy_home_node sys_ni_syscall diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 543100151f2b..ac964612d8b0 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -463,3 +463,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 6bdb5f5db438..4e65da3445c7 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 450 +#define __NR_compat_syscalls 451 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 41ea1195e44b..604a2053d006 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -905,6 +905,8 @@ __SYSCALL(__NR_landlock_restrict_self, sys_landlock_restrict_self) __SYSCALL(__NR_process_mrelease, sys_process_mrelease) #define __NR_futex_waitv 449 __SYSCALL(__NR_futex_waitv, sys_futex_waitv) +#define __NR_set_mempolicy_home_node 450 +__SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) /* * Please add new compat syscalls above this comment and update diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index 707ae121f6d3..78b1d03e86e1 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -370,3 +370,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index 45bc32a41b90..b1f3940bc298 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -449,3 +449,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index 2204bde3ce4a..820145e47350 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -455,3 +455,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 72d02d363f36..253ff994ed2e 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -388,3 +388,4 @@ # 447 reserved for memfd_secret 448 n32 process_mrelease sys_process_mrelease 449 n32 futex_waitv sys_futex_waitv +450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index e2c481fcede6..3f1886ad9d80 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -364,3 +364,4 @@ # 447 reserved for memfd_secret 448 n64 process_mrelease sys_process_mrelease 449 n64 futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 3714c97b2643..8f243e35a7b2 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -437,3 +437,4 @@ # 447 reserved for memfd_secret 448 o32 process_mrelease sys_process_mrelease 449 o32 futex_waitv sys_futex_waitv +450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 358c00000755..68b46fe2f17c 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -447,3 +447,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index 15109af9d075..2600b4237292 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -529,3 +529,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index ed9c5c2eafad..799147658dee 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -452,3 +452,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index d9539d28bdaa..2de85c977f54 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -452,3 +452,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 46adabcb1720..4398cc6fb68d 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -495,3 +495,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 7e25543693de..320480a8db4f 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -454,3 +454,4 @@ 447 i386 memfd_secret sys_memfd_secret 448 i386 process_mrelease sys_process_mrelease 449 i386 futex_waitv sys_futex_waitv +450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index fe8f8dd157b4..c84d12608cd2 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -371,6 +371,7 @@ 447 common memfd_secret sys_memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 3e3e1a506bed..52c94ab5c205 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -420,3 +420,4 @@ # 447 reserved for memfd_secret 448 common process_mrelease sys_process_mrelease 449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 528a478dbda8..819c0cb00b6d 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1057,6 +1057,9 @@ asmlinkage long sys_landlock_add_rule(int ruleset_fd, enum landlock_rule_type ru const void __user *rule_attr, __u32 flags); asmlinkage long sys_landlock_restrict_self(int ruleset_fd, __u32 flags); asmlinkage long sys_memfd_secret(unsigned int flags); +asmlinkage long sys_set_mempolicy_home_node(unsigned long start, unsigned long len, + unsigned long home_node, + unsigned long flags); /* * Architecture-specific system calls diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 4557a8b6086f..1c48b0ae3ba3 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -883,8 +883,11 @@ __SYSCALL(__NR_process_mrelease, sys_process_mrelease) #define __NR_futex_waitv 449 __SYSCALL(__NR_futex_waitv, sys_futex_waitv) +#define __NR_set_mempolicy_home_node 450 +__SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) + #undef __NR_syscalls -#define __NR_syscalls 450 +#define __NR_syscalls 451 /* * 32 bit systems traditionally used different diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index d1944258cfc0..a492f159624f 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -297,6 +297,7 @@ COND_SYSCALL(get_mempolicy); COND_SYSCALL(set_mempolicy); COND_SYSCALL(migrate_pages); COND_SYSCALL(move_pages); +COND_SYSCALL(set_mempolicy_home_node); COND_SYSCALL(perf_event_open); COND_SYSCALL(accept4); -- cgit v1.2.3-59-g8ed1b From c9fdc4d5487a16bd1f003fc8b66e91f88efb50e6 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Fri, 14 Jan 2022 14:09:06 -0800 Subject: mm/hwpoison: remove MF_MSG_BUDDY_2ND and MF_MSG_POISONED_HUGE These action_page_types are no longer used, so remove them. Link: https://lkml.kernel.org/r/20211115084006.3728254-3-naoya.horiguchi@linux.dev Signed-off-by: Naoya Horiguchi Acked-by: Yang Shi Cc: "Aneesh Kumar K.V" Cc: David Hildenbrand Cc: Ding Hui Cc: Miaohe Lin Cc: Michal Hocko Cc: Oscar Salvador Cc: Peter Xu Cc: Tony Luck Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 2 -- include/ras/ras_event.h | 2 -- mm/memory-failure.c | 2 -- 3 files changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index eb67eb699b78..7f594da84aca 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3201,7 +3201,6 @@ enum mf_action_page_type { MF_MSG_KERNEL_HIGH_ORDER, MF_MSG_SLAB, MF_MSG_DIFFERENT_COMPOUND, - MF_MSG_POISONED_HUGE, MF_MSG_HUGE, MF_MSG_FREE_HUGE, MF_MSG_NON_PMD_HUGE, @@ -3216,7 +3215,6 @@ enum mf_action_page_type { MF_MSG_CLEAN_LRU, MF_MSG_TRUNCATED_LRU, MF_MSG_BUDDY, - MF_MSG_BUDDY_2ND, MF_MSG_DAX, MF_MSG_UNSPLIT_THP, MF_MSG_UNKNOWN, diff --git a/include/ras/ras_event.h b/include/ras/ras_event.h index 0bdbc0d17d2f..d0337a41141c 100644 --- a/include/ras/ras_event.h +++ b/include/ras/ras_event.h @@ -358,7 +358,6 @@ TRACE_EVENT(aer_event, EM ( MF_MSG_KERNEL_HIGH_ORDER, "high-order kernel page" ) \ EM ( MF_MSG_SLAB, "kernel slab page" ) \ EM ( MF_MSG_DIFFERENT_COMPOUND, "different compound page after locking" ) \ - EM ( MF_MSG_POISONED_HUGE, "huge page already hardware poisoned" ) \ EM ( MF_MSG_HUGE, "huge page" ) \ EM ( MF_MSG_FREE_HUGE, "free huge page" ) \ EM ( MF_MSG_NON_PMD_HUGE, "non-pmd-sized huge page" ) \ @@ -373,7 +372,6 @@ TRACE_EVENT(aer_event, EM ( MF_MSG_CLEAN_LRU, "clean LRU page" ) \ EM ( MF_MSG_TRUNCATED_LRU, "already truncated LRU page" ) \ EM ( MF_MSG_BUDDY, "free buddy page" ) \ - EM ( MF_MSG_BUDDY_2ND, "free buddy page (2nd try)" ) \ EM ( MF_MSG_DAX, "dax page" ) \ EM ( MF_MSG_UNSPLIT_THP, "unsplit thp" ) \ EMe ( MF_MSG_UNKNOWN, "unknown page" ) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 607785491a52..810328fe8adb 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -723,7 +723,6 @@ static const char * const action_page_types[] = { [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", [MF_MSG_SLAB] = "kernel slab page", [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking", - [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned", [MF_MSG_HUGE] = "huge page", [MF_MSG_FREE_HUGE] = "free huge page", [MF_MSG_NON_PMD_HUGE] = "non-pmd-sized huge page", @@ -738,7 +737,6 @@ static const char * const action_page_types[] = { [MF_MSG_CLEAN_LRU] = "clean LRU page", [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", [MF_MSG_BUDDY] = "free buddy page", - [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)", [MF_MSG_DAX] = "dax page", [MF_MSG_UNSPLIT_THP] = "unsplit thp", [MF_MSG_UNKNOWN] = "unknown page", -- cgit v1.2.3-59-g8ed1b From bf181c582588f8f7406d52f2ee228539b465f173 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Fri, 14 Jan 2022 14:09:09 -0800 Subject: mm/hwpoison: fix unpoison_memory() After recent soft-offline rework, error pages can be taken off from buddy allocator, but the existing unpoison_memory() does not properly undo the operation. Moreover, due to the recent change on __get_hwpoison_page(), get_page_unless_zero() is hardly called for hwpoisoned pages. So __get_hwpoison_page() highly likely returns -EBUSY (meaning to fail to grab page refcount) and unpoison just clears PG_hwpoison without releasing a refcount. That does not lead to a critical issue like kernel panic, but unpoisoned pages never get back to buddy (leaked permanently), which is not good. To (partially) fix this, we need to identify "taken off" pages from other types of hwpoisoned pages. We can't use refcount or page flags for this purpose, so a pseudo flag is defined by hacking ->private field. Someone might think that put_page() is enough to cancel taken-off pages, but the normal free path contains some operations not suitable for the current purpose, and can fire VM_BUG_ON(). Note that unpoison_memory() is now supposed to be cancel hwpoison events injected only by madvise() or /sys/devices/system/memory/{hard,soft}_offline_page, not by MCE injection, so please don't try to use unpoison when testing with MCE injection. [lkp@intel.com: report build failure for ARCH=i386] Link: https://lkml.kernel.org/r/20211115084006.3728254-4-naoya.horiguchi@linux.dev Signed-off-by: Naoya Horiguchi Reviewed-by: Yang Shi Cc: David Hildenbrand Cc: Oscar Salvador Cc: Michal Hocko Cc: Ding Hui Cc: Tony Luck Cc: "Aneesh Kumar K.V" Cc: Miaohe Lin Cc: Peter Xu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 1 + include/linux/page-flags.h | 4 ++ mm/memory-failure.c | 109 +++++++++++++++++++++++++++++++++++++-------- mm/page_alloc.c | 27 +++++++++++ 4 files changed, 122 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 7f594da84aca..d4fb49a5d60d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3174,6 +3174,7 @@ enum mf_flags { MF_ACTION_REQUIRED = 1 << 1, MF_MUST_KILL = 1 << 2, MF_SOFT_OFFLINE = 1 << 3, + MF_UNPOISON = 1 << 4, }; extern int memory_failure(unsigned long pfn, int flags); extern void memory_failure_queue(unsigned long pfn, int flags); diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 18423c2157e8..7e2b90dc7d3f 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -522,7 +522,11 @@ PAGEFLAG_FALSE(Uncached, uncached) PAGEFLAG(HWPoison, hwpoison, PF_ANY) TESTSCFLAG(HWPoison, hwpoison, PF_ANY) #define __PG_HWPOISON (1UL << PG_hwpoison) +#define MAGIC_HWPOISON 0x48575053U /* HWPS */ +extern void SetPageHWPoisonTakenOff(struct page *page); +extern void ClearPageHWPoisonTakenOff(struct page *page); extern bool take_page_off_buddy(struct page *page); +extern bool put_page_back_buddy(struct page *page); #else PAGEFLAG_FALSE(HWPoison, hwpoison) #define __PG_HWPOISON 0 diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 810328fe8adb..6a2b4b86b679 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1160,6 +1160,22 @@ static int page_action(struct page_state *ps, struct page *p, return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; } +static inline bool PageHWPoisonTakenOff(struct page *page) +{ + return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON; +} + +void SetPageHWPoisonTakenOff(struct page *page) +{ + set_page_private(page, MAGIC_HWPOISON); +} + +void ClearPageHWPoisonTakenOff(struct page *page) +{ + if (PageHWPoison(page)) + set_page_private(page, 0); +} + /* * Return true if a page type of a given page is supported by hwpoison * mechanism (while handling could fail), otherwise false. This function @@ -1262,6 +1278,27 @@ out: return ret; } +static int __get_unpoison_page(struct page *page) +{ + struct page *head = compound_head(page); + int ret = 0; + bool hugetlb = false; + + ret = get_hwpoison_huge_page(head, &hugetlb); + if (hugetlb) + return ret; + + /* + * PageHWPoisonTakenOff pages are not only marked as PG_hwpoison, + * but also isolated from buddy freelist, so need to identify the + * state and have to cancel both operations to unpoison. + */ + if (PageHWPoisonTakenOff(page)) + return -EHWPOISON; + + return get_page_unless_zero(page) ? 1 : 0; +} + /** * get_hwpoison_page() - Get refcount for memory error handling * @p: Raw error page (hit by memory error) @@ -1278,18 +1315,26 @@ out: * extra care for the error page's state (as done in __get_hwpoison_page()), * and has some retry logic in get_any_page(). * + * When called from unpoison_memory(), the caller should already ensure that + * the given page has PG_hwpoison. So it's never reused for other page + * allocations, and __get_unpoison_page() never races with them. + * * Return: 0 on failure, * 1 on success for in-use pages in a well-defined state, * -EIO for pages on which we can not handle memory errors, * -EBUSY when get_hwpoison_page() has raced with page lifecycle - * operations like allocation and free. + * operations like allocation and free, + * -EHWPOISON when the page is hwpoisoned and taken off from buddy. */ static int get_hwpoison_page(struct page *p, unsigned long flags) { int ret; zone_pcp_disable(page_zone(p)); - ret = get_any_page(p, flags); + if (flags & MF_UNPOISON) + ret = __get_unpoison_page(p); + else + ret = get_any_page(p, flags); zone_pcp_enable(page_zone(p)); return ret; @@ -1937,6 +1982,28 @@ core_initcall(memory_failure_init); pr_info(fmt, pfn); \ }) +static inline int clear_page_hwpoison(struct ratelimit_state *rs, struct page *p) +{ + if (TestClearPageHWPoison(p)) { + unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", + page_to_pfn(p), rs); + num_poisoned_pages_dec(); + return 1; + } + return 0; +} + +static inline int unpoison_taken_off_page(struct ratelimit_state *rs, + struct page *p) +{ + if (put_page_back_buddy(p)) { + unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", + page_to_pfn(p), rs); + return 0; + } + return -EBUSY; +} + /** * unpoison_memory - Unpoison a previously poisoned page * @pfn: Page number of the to be unpoisoned page @@ -1953,9 +2020,7 @@ int unpoison_memory(unsigned long pfn) { struct page *page; struct page *p; - int freeit = 0; - int ret = 0; - unsigned long flags = 0; + int ret = -EBUSY; static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); @@ -1991,24 +2056,30 @@ int unpoison_memory(unsigned long pfn) goto unlock_mutex; } - if (!get_hwpoison_page(p, flags)) { - if (TestClearPageHWPoison(p)) - num_poisoned_pages_dec(); - unpoison_pr_info("Unpoison: Software-unpoisoned free page %#lx\n", - pfn, &unpoison_rs); + if (PageSlab(page) || PageTable(page)) goto unlock_mutex; - } - if (TestClearPageHWPoison(page)) { - unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", - pfn, &unpoison_rs); - num_poisoned_pages_dec(); - freeit = 1; - } + ret = get_hwpoison_page(p, MF_UNPOISON); + if (!ret) { + if (clear_page_hwpoison(&unpoison_rs, page)) + ret = 0; + else + ret = -EBUSY; + } else if (ret < 0) { + if (ret == -EHWPOISON) { + ret = unpoison_taken_off_page(&unpoison_rs, p); + } else + unpoison_pr_info("Unpoison: failed to grab page %#lx\n", + pfn, &unpoison_rs); + } else { + int freeit = clear_page_hwpoison(&unpoison_rs, p); - put_page(page); - if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) put_page(page); + if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) { + put_page(page); + ret = 0; + } + } unlock_mutex: mutex_unlock(&mf_mutex); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 99fc65c532f0..d4205e5e41d1 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -9508,6 +9509,7 @@ bool take_page_off_buddy(struct page *page) del_page_from_free_list(page_head, zone, page_order); break_down_buddy_pages(zone, page_head, page, 0, page_order, migratetype); + SetPageHWPoisonTakenOff(page); if (!is_migrate_isolate(migratetype)) __mod_zone_freepage_state(zone, -1, migratetype); ret = true; @@ -9519,6 +9521,31 @@ bool take_page_off_buddy(struct page *page) spin_unlock_irqrestore(&zone->lock, flags); return ret; } + +/* + * Cancel takeoff done by take_page_off_buddy(). + */ +bool put_page_back_buddy(struct page *page) +{ + struct zone *zone = page_zone(page); + unsigned long pfn = page_to_pfn(page); + unsigned long flags; + int migratetype = get_pfnblock_migratetype(page, pfn); + bool ret = false; + + spin_lock_irqsave(&zone->lock, flags); + if (put_page_testzero(page)) { + ClearPageHWPoisonTakenOff(page); + __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); + if (TestClearPageHWPoison(page)) { + num_poisoned_pages_dec(); + ret = true; + } + } + spin_unlock_irqrestore(&zone->lock, flags); + + return ret; +} #endif #ifdef CONFIG_ZONE_DMA -- cgit v1.2.3-59-g8ed1b From 5ee2fa2f063649570c702164f47a558a3432dd9e Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Fri, 14 Jan 2022 14:09:16 -0800 Subject: mm/rmap: fix potential batched TLB flush race In theory, the following race is possible for batched TLB flushing. CPU0 CPU1 ---- ---- shrink_page_list() unmap zap_pte_range() flush_tlb_batched_pending() flush_tlb_mm() try_to_unmap() set_tlb_ubc_flush_pending() mm->tlb_flush_batched = true mm->tlb_flush_batched = false After the TLB is flushed on CPU1 via flush_tlb_mm() and before mm->tlb_flush_batched is set to false, some PTE is unmapped on CPU0 and the TLB flushing is pended. Then the pended TLB flushing will be lost. Although both set_tlb_ubc_flush_pending() and flush_tlb_batched_pending() are called with PTL locked, different PTL instances may be used. Because the race window is really small, and the lost TLB flushing will cause problem only if a TLB entry is inserted before the unmapping in the race window, the race is only theoretical. But the fix is simple and cheap too. Syzbot has reported this too as follows: ================================================================== BUG: KCSAN: data-race in flush_tlb_batched_pending / try_to_unmap_one write to 0xffff8881072cfbbc of 1 bytes by task 17406 on cpu 1: flush_tlb_batched_pending+0x5f/0x80 mm/rmap.c:691 madvise_free_pte_range+0xee/0x7d0 mm/madvise.c:594 walk_pmd_range mm/pagewalk.c:128 [inline] walk_pud_range mm/pagewalk.c:205 [inline] walk_p4d_range mm/pagewalk.c:240 [inline] walk_pgd_range mm/pagewalk.c:277 [inline] __walk_page_range+0x981/0x1160 mm/pagewalk.c:379 walk_page_range+0x131/0x300 mm/pagewalk.c:475 madvise_free_single_vma mm/madvise.c:734 [inline] madvise_dontneed_free mm/madvise.c:822 [inline] madvise_vma mm/madvise.c:996 [inline] do_madvise+0xe4a/0x1140 mm/madvise.c:1202 __do_sys_madvise mm/madvise.c:1228 [inline] __se_sys_madvise mm/madvise.c:1226 [inline] __x64_sys_madvise+0x5d/0x70 mm/madvise.c:1226 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x44/0xd0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae write to 0xffff8881072cfbbc of 1 bytes by task 71 on cpu 0: set_tlb_ubc_flush_pending mm/rmap.c:636 [inline] try_to_unmap_one+0x60e/0x1220 mm/rmap.c:1515 rmap_walk_anon+0x2fb/0x470 mm/rmap.c:2301 try_to_unmap+0xec/0x110 shrink_page_list+0xe91/0x2620 mm/vmscan.c:1719 shrink_inactive_list+0x3fb/0x730 mm/vmscan.c:2394 shrink_list mm/vmscan.c:2621 [inline] shrink_lruvec+0x3c9/0x710 mm/vmscan.c:2940 shrink_node_memcgs+0x23e/0x410 mm/vmscan.c:3129 shrink_node+0x8f6/0x1190 mm/vmscan.c:3252 kswapd_shrink_node mm/vmscan.c:4022 [inline] balance_pgdat+0x702/0xd30 mm/vmscan.c:4213 kswapd+0x200/0x340 mm/vmscan.c:4473 kthread+0x2c7/0x2e0 kernel/kthread.c:327 ret_from_fork+0x1f/0x30 value changed: 0x01 -> 0x00 Reported by Kernel Concurrency Sanitizer on: CPU: 0 PID: 71 Comm: kswapd0 Not tainted 5.16.0-rc1-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 ================================================================== [akpm@linux-foundation.org: tweak comments] Link: https://lkml.kernel.org/r/20211201021104.126469-1-ying.huang@intel.com Signed-off-by: "Huang, Ying" Reported-by: syzbot+aa5bebed695edaccf0df@syzkaller.appspotmail.com Cc: Nadav Amit Cc: Mel Gorman Cc: Andrea Arcangeli Cc: Andy Lutomirski Cc: Dave Hansen Cc: Will Deacon Cc: Yu Zhao Cc: Marco Elver Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_types.h | 2 +- mm/rmap.c | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6a89f128c990..e3b0476a4fda 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -647,7 +647,7 @@ struct mm_struct { atomic_t tlb_flush_pending; #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH /* See flush_tlb_batched_pending() */ - bool tlb_flush_batched; + atomic_t tlb_flush_batched; #endif struct uprobes_state uprobes_state; #ifdef CONFIG_PREEMPT_RT diff --git a/mm/rmap.c b/mm/rmap.c index 163ac4e6bcee..6a1e8c7f6213 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -621,9 +621,20 @@ void try_to_unmap_flush_dirty(void) try_to_unmap_flush(); } +/* + * Bits 0-14 of mm->tlb_flush_batched record pending generations. + * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations. + */ +#define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16 +#define TLB_FLUSH_BATCH_PENDING_MASK \ + ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1) +#define TLB_FLUSH_BATCH_PENDING_LARGE \ + (TLB_FLUSH_BATCH_PENDING_MASK / 2) + static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) { struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; + int batch, nbatch; arch_tlbbatch_add_mm(&tlb_ubc->arch, mm); tlb_ubc->flush_required = true; @@ -633,7 +644,22 @@ static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable) * before the PTE is cleared. */ barrier(); - mm->tlb_flush_batched = true; + batch = atomic_read(&mm->tlb_flush_batched); +retry: + if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) { + /* + * Prevent `pending' from catching up with `flushed' because of + * overflow. Reset `pending' and `flushed' to be 1 and 0 if + * `pending' becomes large. + */ + nbatch = atomic_cmpxchg(&mm->tlb_flush_batched, batch, 1); + if (nbatch != batch) { + batch = nbatch; + goto retry; + } + } else { + atomic_inc(&mm->tlb_flush_batched); + } /* * If the PTE was dirty then it's best to assume it's writable. The @@ -680,15 +706,18 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) */ void flush_tlb_batched_pending(struct mm_struct *mm) { - if (data_race(mm->tlb_flush_batched)) { - flush_tlb_mm(mm); + int batch = atomic_read(&mm->tlb_flush_batched); + int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK; + int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT; + if (pending != flushed) { + flush_tlb_mm(mm); /* - * Do not allow the compiler to re-order the clearing of - * tlb_flush_batched before the tlb is flushed. + * If the new TLB flushing is pending during flushing, leave + * mm->tlb_flush_batched as is, to avoid losing flushing. */ - barrier(); - mm->tlb_flush_batched = false; + atomic_cmpxchg(&mm->tlb_flush_batched, batch, + pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT)); } } #else -- cgit v1.2.3-59-g8ed1b From cab0a7c115546a4865fb7439558af9077a569574 Mon Sep 17 00:00:00 2001 From: Ting Liu Date: Fri, 14 Jan 2022 14:09:28 -0800 Subject: mm: make some vars and functions static or __init "page_idle_ops" as a global var, but its scope of use within this document. So it should be static. "page_ext_ops" is a var used in the kernel initial phase. And other functions are aslo used in the kernel initial phase. So they should be __init or __initdata to reclaim memory. Link: https://lkml.kernel.org/r/20211217095023.67293-1-liuting.0x7c00@bytedance.com Signed-off-by: Ting Liu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/page_idle.h | 1 - mm/page_ext.c | 4 ++-- mm/page_owner.c | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 83abf95e9fa7..4663dfed1293 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -13,7 +13,6 @@ * If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. */ -extern struct page_ext_operations page_idle_ops; static inline bool folio_test_young(struct folio *folio) { diff --git a/mm/page_ext.c b/mm/page_ext.c index bee3240604dc..2e66d934d63f 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -64,12 +64,12 @@ static bool need_page_idle(void) { return true; } -struct page_ext_operations page_idle_ops = { +static struct page_ext_operations page_idle_ops __initdata = { .need = need_page_idle, }; #endif -static struct page_ext_operations *page_ext_ops[] = { +static struct page_ext_operations *page_ext_ops[] __initdata = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/page_owner.c b/mm/page_owner.c index 4f924957ce7a..5eea061bb1e5 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -46,7 +46,7 @@ static int __init early_page_owner_param(char *buf) } early_param("page_owner", early_page_owner_param); -static bool need_page_owner(void) +static __init bool need_page_owner(void) { return page_owner_enabled; } @@ -75,7 +75,7 @@ static noinline void register_early_stack(void) early_handle = create_dummy_stack(); } -static void init_page_owner(void) +static __init void init_page_owner(void) { if (!page_owner_enabled) return; -- cgit v1.2.3-59-g8ed1b From cdeed009f3bceee41f73f0137db785fd29a05cb8 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:44 -0800 Subject: mm/damon: remove some unneeded function definitions in damon.h In damon.h some func definitions about VA & PA can only be used in its own file, so there no need to define in the header file, and the header file will look cleaner. If other files later need these functions, the prototypes can be added to damon.h at that time. [sj@kernel.org: remove unnecessary function prototype position changes] Link: https://lkml.kernel.org/r/20211118114827.20052-1-sj@kernel.org Link: https://lkml.kernel.org/r/45fd5b3ef6cce8e28dbc1c92f9dc845ccfc949d7.1636989871.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Signed-off-by: SeongJae Park Reviewed-by: SeongJae Park Cc: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 21 --------------------- mm/damon/paddr.c | 11 ++++++----- mm/damon/vaddr.c | 18 ++++++++++-------- 3 files changed, 16 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index b4d4be3cc987..1d1be348f506 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -461,34 +461,13 @@ int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); #endif /* CONFIG_DAMON */ #ifdef CONFIG_DAMON_VADDR - -/* Monitoring primitives for virtual memory address spaces */ -void damon_va_init(struct damon_ctx *ctx); -void damon_va_update(struct damon_ctx *ctx); -void damon_va_prepare_access_checks(struct damon_ctx *ctx); -unsigned int damon_va_check_accesses(struct damon_ctx *ctx); bool damon_va_target_valid(void *t); -void damon_va_cleanup(struct damon_ctx *ctx); -int damon_va_apply_scheme(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); -int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); void damon_va_set_primitives(struct damon_ctx *ctx); - #endif /* CONFIG_DAMON_VADDR */ #ifdef CONFIG_DAMON_PADDR - -/* Monitoring primitives for the physical memory address space */ -void damon_pa_prepare_access_checks(struct damon_ctx *ctx); -unsigned int damon_pa_check_accesses(struct damon_ctx *ctx); bool damon_pa_target_valid(void *t); -int damon_pa_apply_scheme(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); -int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); void damon_pa_set_primitives(struct damon_ctx *ctx); - #endif /* CONFIG_DAMON_PADDR */ #endif /* _DAMON_H */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index a496d6f203d6..4318134cbc4c 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -73,7 +73,7 @@ static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, damon_pa_mkold(r->sampling_addr); } -void damon_pa_prepare_access_checks(struct damon_ctx *ctx) +static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) { struct damon_target *t; struct damon_region *r; @@ -192,7 +192,7 @@ static void __damon_pa_check_access(struct damon_ctx *ctx, last_addr = r->sampling_addr; } -unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) +static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) { struct damon_target *t; struct damon_region *r; @@ -213,7 +213,7 @@ bool damon_pa_target_valid(void *t) return true; } -int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, +static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) { unsigned long addr; @@ -246,8 +246,9 @@ int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return 0; } -int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static int damon_pa_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { switch (scheme->action) { case DAMOS_PAGEOUT: diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 73c5d1aafda6..a9d3b4d96e29 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -272,7 +272,7 @@ static void __damon_va_init_regions(struct damon_ctx *ctx, } /* Initialize '->regions_list' of every target (task) */ -void damon_va_init(struct damon_ctx *ctx) +static void damon_va_init(struct damon_ctx *ctx) { struct damon_target *t; @@ -292,7 +292,8 @@ void damon_va_init(struct damon_ctx *ctx) * * Returns true if it is. */ -static bool damon_intersect(struct damon_region *r, struct damon_addr_range *re) +static bool damon_intersect(struct damon_region *r, + struct damon_addr_range *re) { return !(r->ar.end <= re->start || re->end <= r->ar.start); } @@ -356,7 +357,7 @@ static void damon_va_apply_three_regions(struct damon_target *t, /* * Update regions for current memory mappings */ -void damon_va_update(struct damon_ctx *ctx) +static void damon_va_update(struct damon_ctx *ctx) { struct damon_addr_range three_regions[3]; struct damon_target *t; @@ -418,7 +419,7 @@ static void __damon_va_prepare_access_check(struct damon_ctx *ctx, damon_va_mkold(mm, r->sampling_addr); } -void damon_va_prepare_access_checks(struct damon_ctx *ctx) +static void damon_va_prepare_access_checks(struct damon_ctx *ctx) { struct damon_target *t; struct mm_struct *mm; @@ -539,7 +540,7 @@ static void __damon_va_check_access(struct damon_ctx *ctx, last_addr = r->sampling_addr; } -unsigned int damon_va_check_accesses(struct damon_ctx *ctx) +static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) { struct damon_target *t; struct mm_struct *mm; @@ -603,7 +604,7 @@ out: } #endif /* CONFIG_ADVISE_SYSCALLS */ -int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, +static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) { int madv_action; @@ -633,8 +634,9 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return damos_madvise(t, r, madv_action); } -int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static int damon_va_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { switch (scheme->action) { -- cgit v1.2.3-59-g8ed1b From 9b2a38d6ef25c1748e3964b0ff30a89e4ed26583 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:53 -0800 Subject: mm/damon: move damon_rand() definition into damon.h damon_rand() is called in three files:damon/core.c, damon/ paddr.c, damon/vaddr.c, i think there is no need to redefine this twice, So move it to damon.h will be a good choice. Link: https://lkml.kernel.org/r/20211202075859.51341-1-xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 4 ++++ mm/damon/core.c | 4 ---- mm/damon/prmtv-common.h | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 1d1be348f506..3e91a597a1aa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -11,12 +11,16 @@ #include #include #include +#include /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION PAGE_SIZE /* Max priority score for DAMON-based operation schemes */ #define DAMOS_MAX_SCORE (99) +/* Get a random number in [l, r) */ +#define damon_rand(l, r) (l + prandom_u32_max(r - l)) + /** * struct damon_addr_range - Represents an address region of [@start, @end). * @start: Start address of the region (inclusive). diff --git a/mm/damon/core.c b/mm/damon/core.c index 04b8df7fd9e9..61e844d15b13 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -23,9 +22,6 @@ #define DAMON_MIN_REGION 1 #endif -/* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) - static DEFINE_MUTEX(damon_lock); static int nr_running_ctxs; diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h index 61f27037603e..e790cb5f8fe0 100644 --- a/mm/damon/prmtv-common.h +++ b/mm/damon/prmtv-common.h @@ -6,10 +6,6 @@ */ #include -#include - -/* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) struct page *damon_get_page(unsigned long pfn); -- cgit v1.2.3-59-g8ed1b From 234d68732b6c135087bdebfa0630a43ae8c27758 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:56 -0800 Subject: mm/damon: modify damon_rand() macro to static inline function damon_rand() cannot be implemented as a macro. Example: damon_rand(a++, b); The value of 'a' will be incremented twice, This is obviously unreasonable, So there fix it. Link: https://lkml.kernel.org/r/110ffcd4e420c86c42b41ce2bc9f0fe6a4f32cd3.1638795127.git.xhao@linux.alibaba.com Fixes: b9a6ac4e4ede ("mm/damon: adaptively adjust regions") Signed-off-by: Xin Hao Reported-by: Andrew Morton Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 3e91a597a1aa..e2c8152985b7 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -19,7 +19,10 @@ #define DAMOS_MAX_SCORE (99) /* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) +static inline unsigned long damon_rand(unsigned long l, unsigned long r) +{ + return l + prandom_u32_max(r - l); +} /** * struct damon_addr_range - Represents an address region of [@start, @end). -- cgit v1.2.3-59-g8ed1b From 88f86dcfa454784f7de550966c60fc78a3e95d6d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:09:59 -0800 Subject: mm/damon: convert macro functions to static inline functions Patch series "mm/damon: Misc cleanups". This patchset contains miscellaneous cleanups for DAMON's macro functions and documentation. This patch (of 6): This commit converts macro functions in DAMON to static inline functions, for better type checking, code documentation, etc[1]. [1] https://lore.kernel.org/linux-mm/20211202151213.6ec830863342220da4141bc5@linux-foundation.org/ Link: https://lkml.kernel.org/r/20211209131806.19317-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211209131806.19317-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 18 ++++++++++++------ mm/damon/core.c | 5 ++++- mm/damon/vaddr.c | 6 ++++-- 3 files changed, 20 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index e2c8152985b7..2dbc1f545da2 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -399,14 +399,20 @@ struct damon_ctx { struct list_head schemes; }; -#define damon_next_region(r) \ - (container_of(r->list.next, struct damon_region, list)) +static inline struct damon_region *damon_next_region(struct damon_region *r) +{ + return container_of(r->list.next, struct damon_region, list); +} -#define damon_prev_region(r) \ - (container_of(r->list.prev, struct damon_region, list)) +static inline struct damon_region *damon_prev_region(struct damon_region *r) +{ + return container_of(r->list.prev, struct damon_region, list); +} -#define damon_last_region(t) \ - (list_last_entry(&t->regions_list, struct damon_region, list)) +static inline struct damon_region *damon_last_region(struct damon_target *t) +{ + return list_last_entry(&t->regions_list, struct damon_region, list); +} #define damon_for_each_region(r, t) \ list_for_each_entry(r, &t->regions_list, list) diff --git a/mm/damon/core.c b/mm/damon/core.c index 61e844d15b13..4515cf82c433 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -729,7 +729,10 @@ static void kdamond_apply_schemes(struct damon_ctx *c) } } -#define sz_damon_region(r) (r->ar.end - r->ar.start) +static inline unsigned long sz_damon_region(struct damon_region *r) +{ + return r->ar.end - r->ar.start; +} /* * Merge two adjacent regions into one region diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 78ff2bcb66eb..68d9e4134816 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -26,8 +26,10 @@ * 't->id' should be the pointer to the relevant 'struct pid' having reference * count. Caller must put the returned task, unless it is NULL. */ -#define damon_get_task_struct(t) \ - (get_pid_task((struct pid *)t->id, PIDTYPE_PID)) +static inline struct task_struct *damon_get_task_struct(struct damon_target *t) +{ + return get_pid_task((struct pid *)t->id, PIDTYPE_PID); +} /* * Get the mm_struct of the given target -- cgit v1.2.3-59-g8ed1b From f4c6d22c6cf282ef7d24a724b9bd978ee2b74fc6 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:14 -0800 Subject: mm/damon: remove a mistakenly added comment for a future feature Due to a mistake in patches reordering, a comment for a future feature called 'arbitrary monitoring target support'[1], which is still under development, has added. Because it only introduces confusion and we don't have a plan to post the patches soon, this commit removes the mistakenly added part. [1] https://lore.kernel.org/linux-mm/20201215115448.25633-3-sjpark@amazon.com/ Link: https://lkml.kernel.org/r/20211209131806.19317-7-sj@kernel.org Fixes: 1f366e421c8f ("mm/damon/core: implement DAMON-based Operation Schemes (DAMOS)") Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 2dbc1f545da2..97f4a224e950 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -281,7 +281,7 @@ struct damon_ctx; * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action - * to the region. This is not used for &DAMON_ARBITRARY_TARGET case. + * to the region. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination. -- cgit v1.2.3-59-g8ed1b From 0e92c2ee9f459542c5384d9cfab24873c3dd6398 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:17 -0800 Subject: mm/damon/schemes: account scheme actions that successfully applied Patch series "mm/damon/schemes: Extend stats for better online analysis and tuning". To help online access pattern analysis and tuning of DAMON-based Operation Schemes (DAMOS), DAMOS provides simple statistics for each scheme. Introduction of DAMOS time/space quota further made the tuning easier by making the risk management easier. However, that also made understanding of the working schemes a little bit more difficult. For an example, progress of a given scheme can now be throttled by not only the aggressiveness of the target access pattern, but also the time/space quotas. So, when a scheme is showing unexpectedly slow progress, it's difficult to know by what the progress of the scheme is throttled, with currently provided statistics. This patchset extends the statistics to contain some metrics that can be helpful for such online schemes analysis and tuning (patches 1-2), exports those to users (patches 3 and 5), and add documents (patches 4 and 6). This patch (of 6): DAMON-based operation schemes (DAMOS) stats provide only the number and the amount of regions that the action of the scheme has tried to be applied. Because the action could be failed for some reasons, the currently provided information is sometimes not useful or convenient enough for schemes profiling and tuning. To improve this situation, this commit extends the DAMOS stats to provide the number and the amount of regions that the action has successfully applied. Link: https://lkml.kernel.org/r/20211210150016.35349-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211210150016.35349-2-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 28 +++++++++++++++++++++------- mm/damon/core.c | 13 ++++++++----- mm/damon/dbgfs.c | 2 +- mm/damon/paddr.c | 13 +++++++------ mm/damon/vaddr.c | 30 ++++++++++++++++-------------- 5 files changed, 53 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index 97f4a224e950..e0ad3d9aaeed 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -192,6 +192,20 @@ struct damos_watermarks { bool activated; }; +/** + * struct damos_stat - Statistics on a given scheme. + * @nr_tried: Total number of regions that the scheme is tried to be applied. + * @sz_tried: Total size of regions that the scheme is tried to be applied. + * @nr_applied: Total number of regions that the scheme is applied. + * @sz_applied: Total size of regions that the scheme is applied. + */ +struct damos_stat { + unsigned long nr_tried; + unsigned long sz_tried; + unsigned long nr_applied; + unsigned long sz_applied; +}; + /** * struct damos - Represents a Data Access Monitoring-based Operation Scheme. * @min_sz_region: Minimum size of target regions. @@ -203,8 +217,7 @@ struct damos_watermarks { * @action: &damo_action to be applied to the target regions. * @quota: Control the aggressiveness of this scheme. * @wmarks: Watermarks for automated (in)activation of this scheme. - * @stat_count: Total number of regions that this scheme is applied. - * @stat_sz: Total size of regions that this scheme is applied. + * @stat: Statistics of this scheme. * @list: List head for siblings. * * For each aggregation interval, DAMON finds regions which fit in the @@ -235,8 +248,7 @@ struct damos { enum damos_action action; struct damos_quota quota; struct damos_watermarks wmarks; - unsigned long stat_count; - unsigned long stat_sz; + struct damos_stat stat; struct list_head list; }; @@ -281,7 +293,8 @@ struct damon_ctx; * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action - * to the region. + * to the region and return bytes of the region that the action is successfully + * applied. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination. @@ -295,8 +308,9 @@ struct damon_primitive { int (*get_scheme_score)(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); - int (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); + unsigned long (*apply_scheme)(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme); bool (*target_valid)(void *target); void (*cleanup)(struct damon_ctx *context); }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 4515cf82c433..d745bf28509f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -102,8 +102,7 @@ struct damos *damon_new_scheme( scheme->min_age_region = min_age_region; scheme->max_age_region = max_age_region; scheme->action = action; - scheme->stat_count = 0; - scheme->stat_sz = 0; + scheme->stat = (struct damos_stat){}; INIT_LIST_HEAD(&scheme->list); scheme->quota.ms = quota->ms; @@ -574,6 +573,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, struct damos_quota *quota = &s->quota; unsigned long sz = r->ar.end - r->ar.start; struct timespec64 begin, end; + unsigned long sz_applied = 0; if (!s->wmarks.activated) continue; @@ -627,7 +627,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, damon_split_region_at(c, t, r, sz); } ktime_get_coarse_ts64(&begin); - c->primitive.apply_scheme(c, t, r, s); + sz_applied = c->primitive.apply_scheme(c, t, r, s); ktime_get_coarse_ts64(&end); quota->total_charged_ns += timespec64_to_ns(&end) - timespec64_to_ns(&begin); @@ -641,8 +641,11 @@ static void damon_do_apply_schemes(struct damon_ctx *c, r->age = 0; update_stat: - s->stat_count++; - s->stat_sz += sz; + s->stat.nr_tried++; + s->stat.sz_tried += sz; + if (sz_applied) + s->stat.nr_applied++; + s->stat.sz_applied += sz_applied; } } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index bf36a2756cfb..9318b52d0b46 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -117,7 +117,7 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) s->quota.weight_age, s->wmarks.metric, s->wmarks.interval, s->wmarks.high, s->wmarks.mid, s->wmarks.low, - s->stat_count, s->stat_sz); + s->stat.nr_tried, s->stat.sz_tried); if (!rc) return -ENOMEM; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 4318134cbc4c..5e8244f65a1a 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -213,14 +213,15 @@ bool damon_pa_target_valid(void *t) return true; } -static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { - unsigned long addr; + unsigned long addr, applied; LIST_HEAD(page_list); if (scheme->action != DAMOS_PAGEOUT) - return -EINVAL; + return 0; for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { struct page *page = damon_get_page(PHYS_PFN(addr)); @@ -241,9 +242,9 @@ static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, put_page(page); } } - reclaim_pages(&page_list); + applied = reclaim_pages(&page_list); cond_resched(); - return 0; + return applied * PAGE_SIZE; } static int damon_pa_scheme_score(struct damon_ctx *context, diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 68d9e4134816..a10df3fd3d02 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -572,32 +572,34 @@ bool damon_va_target_valid(void *target) } #ifndef CONFIG_ADVISE_SYSCALLS -static int damos_madvise(struct damon_target *target, struct damon_region *r, - int behavior) +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) { - return -EINVAL; + return 0; } #else -static int damos_madvise(struct damon_target *target, struct damon_region *r, - int behavior) +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) { struct mm_struct *mm; - int ret = -ENOMEM; + unsigned long start = PAGE_ALIGN(r->ar.start); + unsigned long len = PAGE_ALIGN(r->ar.end - r->ar.start); + unsigned long applied; mm = damon_get_mm(target); if (!mm) - goto out; + return 0; - ret = do_madvise(mm, PAGE_ALIGN(r->ar.start), - PAGE_ALIGN(r->ar.end - r->ar.start), behavior); + applied = do_madvise(mm, start, len, behavior) ? 0 : len; mmput(mm); -out: - return ret; + + return applied; } #endif /* CONFIG_ADVISE_SYSCALLS */ -static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { int madv_action; @@ -620,7 +622,7 @@ static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, case DAMOS_STAT: return 0; default: - return -EINVAL; + return 0; } return damos_madvise(t, r, madv_action); -- cgit v1.2.3-59-g8ed1b From 6268eac34ca30af7f6313504d556ec7fcd295621 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:20 -0800 Subject: mm/damon/schemes: account how many times quota limit has exceeded If the time/space quotas of a given DAMON-based operation scheme is too small, the scheme could show unexpectedly slow progress. However, there is no good way to notice the case in runtime. This commit extends the DAMOS stat to provide how many times the quota limits exceeded so that the users can easily notice the case and tune the scheme. Link: https://lkml.kernel.org/r/20211210150016.35349-3-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 2 ++ mm/damon/core.c | 2 ++ 2 files changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index e0ad3d9aaeed..af648388e759 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -198,12 +198,14 @@ struct damos_watermarks { * @sz_tried: Total size of regions that the scheme is tried to be applied. * @nr_applied: Total number of regions that the scheme is applied. * @sz_applied: Total size of regions that the scheme is applied. + * @qt_exceeds: Total number of times the quota of the scheme has exceeded. */ struct damos_stat { unsigned long nr_tried; unsigned long sz_tried; unsigned long nr_applied; unsigned long sz_applied; + unsigned long qt_exceeds; }; /** diff --git a/mm/damon/core.c b/mm/damon/core.c index d745bf28509f..d5120b326e1b 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -693,6 +693,8 @@ static void kdamond_apply_schemes(struct damon_ctx *c) if (time_after_eq(jiffies, quota->charged_from + msecs_to_jiffies( quota->reset_interval))) { + if (quota->esz && quota->charged_sz >= quota->esz) + s->stat.qt_exceeds++; quota->total_charged_sz += quota->charged_sz; quota->charged_from = jiffies; quota->charged_sz = 0; -- cgit v1.2.3-59-g8ed1b From 2cd4b8e10cc31eadb5b10b1d73b3f28156f3776c Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Fri, 14 Jan 2022 14:10:38 -0800 Subject: mm/damon: move the implementation of damon_insert_region to damon.h Usually, inline function is declared static since it should sit between storage and type. And implement it in a header file if used by multiple files. And this change also fixes compile issue when backport damon to 5.10. mm/damon/vaddr.c: In function `damon_va_evenly_split_region': ./include/linux/damon.h:425:13: error: inlining failed in call to `always_inline' `damon_insert_region': function body not available 425 | inline void damon_insert_region(struct damon_region *r, | ^~~~~~~~~~~~~~~~~~~ mm/damon/vaddr.c:86:3: note: called from here 86 | damon_insert_region(n, r, next, t); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Link: https://lkml.kernel.org/r/20211223085703.6142-1-guoqing.jiang@linux.dev Signed-off-by: Guoqing Jiang Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/damon.h | 13 +++++++++++-- mm/damon/core.c | 11 ----------- 2 files changed, 11 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/damon.h b/include/linux/damon.h index af648388e759..5e1e3a128b77 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -451,9 +451,18 @@ static inline struct damon_region *damon_last_region(struct damon_target *t) #ifdef CONFIG_DAMON struct damon_region *damon_new_region(unsigned long start, unsigned long end); -inline void damon_insert_region(struct damon_region *r, + +/* + * Add a region between two other regions + */ +static inline void damon_insert_region(struct damon_region *r, struct damon_region *prev, struct damon_region *next, - struct damon_target *t); + struct damon_target *t) +{ + __list_add(&r->list, &prev->list, &next->list); + t->nr_regions++; +} + void damon_add_region(struct damon_region *r, struct damon_target *t); void damon_destroy_region(struct damon_region *r, struct damon_target *t); diff --git a/mm/damon/core.c b/mm/damon/core.c index d5120b326e1b..6482d510dcbe 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -49,17 +49,6 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) return region; } -/* - * Add a region between two other regions - */ -inline void damon_insert_region(struct damon_region *r, - struct damon_region *prev, struct damon_region *next, - struct damon_target *t) -{ - __list_add(&r->list, &prev->list, &next->list); - t->nr_regions++; -} - void damon_add_region(struct damon_region *r, struct damon_target *t) { list_add_tail(&r->list, &t->regions_list); -- cgit v1.2.3-59-g8ed1b From 47d8c15615c0a2046d2d90b04cb80b81ddf31fb1 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:16:59 -0700 Subject: include: move find.h from asm_generic to linux find_bit API and bitmap API are closely related, but inclusion paths are different - include/asm-generic and include/linux, correspondingly. In the past it made a lot of troubles due to circular dependencies and/or undefined symbols. Fix this by moving find.h under include/linux. Signed-off-by: Yury Norov Tested-by: Wolfram Sang Acked-by: Geert Uytterhoeven --- MAINTAINERS | 2 +- arch/alpha/include/asm/bitops.h | 2 - arch/arc/include/asm/bitops.h | 1 - arch/arm/include/asm/bitops.h | 1 - arch/arm64/include/asm/bitops.h | 1 - arch/csky/include/asm/bitops.h | 1 - arch/h8300/include/asm/bitops.h | 1 - arch/hexagon/include/asm/bitops.h | 1 - arch/ia64/include/asm/bitops.h | 2 - arch/m68k/include/asm/bitops.h | 2 - arch/mips/include/asm/bitops.h | 1 - arch/openrisc/include/asm/bitops.h | 1 - arch/parisc/include/asm/bitops.h | 1 - arch/powerpc/include/asm/bitops.h | 2 - arch/riscv/include/asm/bitops.h | 1 - arch/s390/include/asm/bitops.h | 1 - arch/sh/include/asm/bitops.h | 1 - arch/sparc/include/asm/bitops_32.h | 1 - arch/sparc/include/asm/bitops_64.h | 2 - arch/x86/include/asm/bitops.h | 2 - arch/xtensa/include/asm/bitops.h | 1 - include/asm-generic/bitops.h | 1 - include/asm-generic/bitops/find.h | 262 ------------------------------------ include/linux/bitmap.h | 1 + include/linux/find.h | 268 +++++++++++++++++++++++++++++++++++++ 25 files changed, 270 insertions(+), 290 deletions(-) delete mode 100644 include/asm-generic/bitops/find.h create mode 100644 include/linux/find.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index dd36acc87ce6..4646786f5302 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3359,8 +3359,8 @@ M: Yury Norov R: Andy Shevchenko R: Rasmus Villemoes S: Maintained -F: include/asm-generic/bitops/find.h F: include/linux/bitmap.h +F: include/linux/find.h F: lib/bitmap.c F: lib/find_bit.c F: lib/find_bit_benchmark.c diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index 5adca78830b5..e1d8483a45f2 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -430,8 +430,6 @@ static inline unsigned int __arch_hweight8(unsigned int w) #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ /* diff --git a/arch/arc/include/asm/bitops.h b/arch/arc/include/asm/bitops.h index a7daaf64ae34..bdb7e190a294 100644 --- a/arch/arc/include/asm/bitops.h +++ b/arch/arc/include/asm/bitops.h @@ -189,7 +189,6 @@ static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x) #include #include -#include #include #include diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index c92e42a5c8f7..8e94fe7ab5eb 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -264,7 +264,6 @@ static inline int find_next_bit_le(const void *p, int size, int offset) #endif -#include #include /* diff --git a/arch/arm64/include/asm/bitops.h b/arch/arm64/include/asm/bitops.h index 81a3e519b07d..9b3c787132d2 100644 --- a/arch/arm64/include/asm/bitops.h +++ b/arch/arm64/include/asm/bitops.h @@ -18,7 +18,6 @@ #include #include -#include #include #include diff --git a/arch/csky/include/asm/bitops.h b/arch/csky/include/asm/bitops.h index 02b72a000767..72e1b2aa29a0 100644 --- a/arch/csky/include/asm/bitops.h +++ b/arch/csky/include/asm/bitops.h @@ -59,7 +59,6 @@ static __always_inline unsigned long __fls(unsigned long x) #include #include -#include #ifndef _LINUX_BITOPS_H #error only can be included directly diff --git a/arch/h8300/include/asm/bitops.h b/arch/h8300/include/asm/bitops.h index c867a80cab5b..4489e3d6edd3 100644 --- a/arch/h8300/include/asm/bitops.h +++ b/arch/h8300/include/asm/bitops.h @@ -168,7 +168,6 @@ static inline unsigned long __ffs(unsigned long word) return result; } -#include #include #include #include diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index 71429f756af0..75d6ba3643b8 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -271,7 +271,6 @@ static inline unsigned long __fls(unsigned long word) } #include -#include #include #include diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index 2f24ee6459d2..577be93c0818 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -441,8 +441,6 @@ static __inline__ unsigned long __arch_hweight64(unsigned long x) #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ #include diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 7b93e1fd8ffa..51283db53667 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -529,6 +529,4 @@ static inline int __fls(int x) #include #endif /* __KERNEL__ */ -#include - #endif /* _M68K_BITOPS_H */ diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h index dc2a6234dd3c..c09d57f907f7 100644 --- a/arch/mips/include/asm/bitops.h +++ b/arch/mips/include/asm/bitops.h @@ -446,7 +446,6 @@ static inline int ffs(int word) } #include -#include #ifdef __KERNEL__ diff --git a/arch/openrisc/include/asm/bitops.h b/arch/openrisc/include/asm/bitops.h index 7f1ca35213d8..d773ed938acb 100644 --- a/arch/openrisc/include/asm/bitops.h +++ b/arch/openrisc/include/asm/bitops.h @@ -30,7 +30,6 @@ #include #include #include -#include #ifndef _LINUX_BITOPS_H #error only can be included directly diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index daa2afd974fb..0ec9cfc5131f 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -203,7 +203,6 @@ static __inline__ int fls(unsigned int x) #include #include #include -#include #include #include diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index 11847b6a244e..abc8f2c7c570 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -255,8 +255,6 @@ unsigned long __arch_hweight64(__u64 w); #include #endif -#include - /* wrappers that deal with KASAN instrumentation */ #include #include diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index 396a3303c537..3540b690944b 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 5a530c552c23..1d40630128a5 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -387,7 +387,6 @@ static inline int fls(unsigned int word) #endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */ #include -#include #include #include #include diff --git a/arch/sh/include/asm/bitops.h b/arch/sh/include/asm/bitops.h index 3b6c7b5b7ec9..10ceb0d6b5a9 100644 --- a/arch/sh/include/asm/bitops.h +++ b/arch/sh/include/asm/bitops.h @@ -68,6 +68,5 @@ static inline unsigned long __ffs(unsigned long word) #include #include -#include #endif /* __ASM_SH_BITOPS_H */ diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 0ceff3b915a8..889afa9f990f 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -100,7 +100,6 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include #include #include -#include #include #include diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index ca7ea5913494..005a8ae858f1 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -52,8 +52,6 @@ unsigned int __arch_hweight8(unsigned int w); #include #endif /* __KERNEL__ */ -#include - #ifdef __KERNEL__ #include diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 0367efdc5b7a..a288ecd230ab 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -380,8 +380,6 @@ static __always_inline int fls64(__u64 x) #include #endif -#include - #include #include diff --git a/arch/xtensa/include/asm/bitops.h b/arch/xtensa/include/asm/bitops.h index 3f71d364ba90..cd225896c40f 100644 --- a/arch/xtensa/include/asm/bitops.h +++ b/arch/xtensa/include/asm/bitops.h @@ -205,7 +205,6 @@ BIT_OPS(change, "xor", ) #undef BIT_OP #undef TEST_AND_BIT_OP -#include #include #include diff --git a/include/asm-generic/bitops.h b/include/asm-generic/bitops.h index df9b5bc3d282..a47b8a71d6fe 100644 --- a/include/asm-generic/bitops.h +++ b/include/asm-generic/bitops.h @@ -20,7 +20,6 @@ #include #include #include -#include #ifndef _LINUX_BITOPS_H #error only can be included directly diff --git a/include/asm-generic/bitops/find.h b/include/asm-generic/bitops/find.h deleted file mode 100644 index 91b1b23f2b0c..000000000000 --- a/include/asm-generic/bitops/find.h +++ /dev/null @@ -1,262 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_BITOPS_FIND_H_ -#define _ASM_GENERIC_BITOPS_FIND_H_ - -extern unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le); -extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); -extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); -extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); - -#ifndef find_next_bit -/** - * find_next_bit - find the next set bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. - */ -static inline -unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - unsigned long offset) -{ - if (small_const_nbits(size)) { - unsigned long val; - - if (unlikely(offset >= size)) - return size; - - val = *addr & GENMASK(size - 1, offset); - return val ? __ffs(val) : size; - } - - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); -} -#endif - -#ifndef find_next_and_bit -/** - * find_next_and_bit - find the next set bit in both memory regions - * @addr1: The first address to base the search on - * @addr2: The second address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. - */ -static inline -unsigned long find_next_and_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long size, - unsigned long offset) -{ - if (small_const_nbits(size)) { - unsigned long val; - - if (unlikely(offset >= size)) - return size; - - val = *addr1 & *addr2 & GENMASK(size - 1, offset); - return val ? __ffs(val) : size; - } - - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); -} -#endif - -#ifndef find_next_zero_bit -/** - * find_next_zero_bit - find the next cleared bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits - * - * Returns the bit number of the next zero bit - * If no bits are zero, returns @size. - */ -static inline -unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, - unsigned long offset) -{ - if (small_const_nbits(size)) { - unsigned long val; - - if (unlikely(offset >= size)) - return size; - - val = *addr | ~GENMASK(size - 1, offset); - return val == ~0UL ? size : ffz(val); - } - - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); -} -#endif - -#ifdef CONFIG_GENERIC_FIND_FIRST_BIT - -#ifndef find_first_bit -/** - * find_first_bit - find the first set bit in a memory region - * @addr: The address to start the search at - * @size: The maximum number of bits to search - * - * Returns the bit number of the first set bit. - * If no bits are set, returns @size. - */ -static inline -unsigned long find_first_bit(const unsigned long *addr, unsigned long size) -{ - if (small_const_nbits(size)) { - unsigned long val = *addr & GENMASK(size - 1, 0); - - return val ? __ffs(val) : size; - } - - return _find_first_bit(addr, size); -} -#endif - -#ifndef find_first_zero_bit -/** - * find_first_zero_bit - find the first cleared bit in a memory region - * @addr: The address to start the search at - * @size: The maximum number of bits to search - * - * Returns the bit number of the first cleared bit. - * If no bits are zero, returns @size. - */ -static inline -unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) -{ - if (small_const_nbits(size)) { - unsigned long val = *addr | ~GENMASK(size - 1, 0); - - return val == ~0UL ? size : ffz(val); - } - - return _find_first_zero_bit(addr, size); -} -#endif - -#else /* CONFIG_GENERIC_FIND_FIRST_BIT */ - -#ifndef find_first_bit -#define find_first_bit(addr, size) find_next_bit((addr), (size), 0) -#endif -#ifndef find_first_zero_bit -#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) -#endif - -#endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ - -#ifndef find_last_bit -/** - * find_last_bit - find the last set bit in a memory region - * @addr: The address to start the search at - * @size: The number of bits to search - * - * Returns the bit number of the last set bit, or size. - */ -static inline -unsigned long find_last_bit(const unsigned long *addr, unsigned long size) -{ - if (small_const_nbits(size)) { - unsigned long val = *addr & GENMASK(size - 1, 0); - - return val ? __fls(val) : size; - } - - return _find_last_bit(addr, size); -} -#endif - -/** - * find_next_clump8 - find next 8-bit clump with set bits in a memory region - * @clump: location to store copy of found clump - * @addr: address to base the search on - * @size: bitmap size in number of bits - * @offset: bit offset at which to start searching - * - * Returns the bit offset for the next set clump; the found clump value is - * copied to the location pointed by @clump. If no bits are set, returns @size. - */ -extern unsigned long find_next_clump8(unsigned long *clump, - const unsigned long *addr, - unsigned long size, unsigned long offset); - -#define find_first_clump8(clump, bits, size) \ - find_next_clump8((clump), (bits), (size), 0) - -#if defined(__LITTLE_ENDIAN) - -static inline unsigned long find_next_zero_bit_le(const void *addr, - unsigned long size, unsigned long offset) -{ - return find_next_zero_bit(addr, size, offset); -} - -static inline unsigned long find_next_bit_le(const void *addr, - unsigned long size, unsigned long offset) -{ - return find_next_bit(addr, size, offset); -} - -static inline unsigned long find_first_zero_bit_le(const void *addr, - unsigned long size) -{ - return find_first_zero_bit(addr, size); -} - -#elif defined(__BIG_ENDIAN) - -#ifndef find_next_zero_bit_le -static inline -unsigned long find_next_zero_bit_le(const void *addr, unsigned - long size, unsigned long offset) -{ - if (small_const_nbits(size)) { - unsigned long val = *(const unsigned long *)addr; - - if (unlikely(offset >= size)) - return size; - - val = swab(val) | ~GENMASK(size - 1, offset); - return val == ~0UL ? size : ffz(val); - } - - return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); -} -#endif - -#ifndef find_next_bit_le -static inline -unsigned long find_next_bit_le(const void *addr, unsigned - long size, unsigned long offset) -{ - if (small_const_nbits(size)) { - unsigned long val = *(const unsigned long *)addr; - - if (unlikely(offset >= size)) - return size; - - val = swab(val) & GENMASK(size - 1, offset); - return val ? __ffs(val) : size; - } - - return _find_next_bit(addr, NULL, size, offset, 0UL, 1); -} -#endif - -#ifndef find_first_zero_bit_le -#define find_first_zero_bit_le(addr, size) \ - find_next_zero_bit_le((addr), (size), 0) -#endif - -#else -#error "Please fix " -#endif - -#endif /*_ASM_GENERIC_BITOPS_FIND_H_ */ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index a241dcf50f39..ead4a150bd7f 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/include/linux/find.h b/include/linux/find.h new file mode 100644 index 000000000000..c5410c243e04 --- /dev/null +++ b/include/linux/find.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_FIND_H_ +#define __LINUX_FIND_H_ + +#ifndef __LINUX_BITMAP_H +#error only can be included directly +#endif + +#include + +extern unsigned long _find_next_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long nbits, + unsigned long start, unsigned long invert, unsigned long le); +extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); +extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); +extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); + +#ifndef find_next_bit +/** + * find_next_bit - find the next set bit in a memory region + * @addr: The address to base the search on + * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. + */ +static inline +unsigned long find_next_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr & GENMASK(size - 1, offset); + return val ? __ffs(val) : size; + } + + return _find_next_bit(addr, NULL, size, offset, 0UL, 0); +} +#endif + +#ifndef find_next_and_bit +/** + * find_next_and_bit - find the next set bit in both memory regions + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. + */ +static inline +unsigned long find_next_and_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long size, + unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr1 & *addr2 & GENMASK(size - 1, offset); + return val ? __ffs(val) : size; + } + + return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); +} +#endif + +#ifndef find_next_zero_bit +/** + * find_next_zero_bit - find the next cleared bit in a memory region + * @addr: The address to base the search on + * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits + * + * Returns the bit number of the next zero bit + * If no bits are zero, returns @size. + */ +static inline +unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr | ~GENMASK(size - 1, offset); + return val == ~0UL ? size : ffz(val); + } + + return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); +} +#endif + +#ifdef CONFIG_GENERIC_FIND_FIRST_BIT + +#ifndef find_first_bit +/** + * find_first_bit - find the first set bit in a memory region + * @addr: The address to start the search at + * @size: The maximum number of bits to search + * + * Returns the bit number of the first set bit. + * If no bits are set, returns @size. + */ +static inline +unsigned long find_first_bit(const unsigned long *addr, unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = *addr & GENMASK(size - 1, 0); + + return val ? __ffs(val) : size; + } + + return _find_first_bit(addr, size); +} +#endif + +#ifndef find_first_zero_bit +/** + * find_first_zero_bit - find the first cleared bit in a memory region + * @addr: The address to start the search at + * @size: The maximum number of bits to search + * + * Returns the bit number of the first cleared bit. + * If no bits are zero, returns @size. + */ +static inline +unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = *addr | ~GENMASK(size - 1, 0); + + return val == ~0UL ? size : ffz(val); + } + + return _find_first_zero_bit(addr, size); +} +#endif + +#else /* CONFIG_GENERIC_FIND_FIRST_BIT */ + +#ifndef find_first_bit +#define find_first_bit(addr, size) find_next_bit((addr), (size), 0) +#endif +#ifndef find_first_zero_bit +#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) +#endif + +#endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ + +#ifndef find_last_bit +/** + * find_last_bit - find the last set bit in a memory region + * @addr: The address to start the search at + * @size: The number of bits to search + * + * Returns the bit number of the last set bit, or size. + */ +static inline +unsigned long find_last_bit(const unsigned long *addr, unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = *addr & GENMASK(size - 1, 0); + + return val ? __fls(val) : size; + } + + return _find_last_bit(addr, size); +} +#endif + +/** + * find_next_clump8 - find next 8-bit clump with set bits in a memory region + * @clump: location to store copy of found clump + * @addr: address to base the search on + * @size: bitmap size in number of bits + * @offset: bit offset at which to start searching + * + * Returns the bit offset for the next set clump; the found clump value is + * copied to the location pointed by @clump. If no bits are set, returns @size. + */ +extern unsigned long find_next_clump8(unsigned long *clump, + const unsigned long *addr, + unsigned long size, unsigned long offset); + +#define find_first_clump8(clump, bits, size) \ + find_next_clump8((clump), (bits), (size), 0) + +#if defined(__LITTLE_ENDIAN) + +static inline unsigned long find_next_zero_bit_le(const void *addr, + unsigned long size, unsigned long offset) +{ + return find_next_zero_bit(addr, size, offset); +} + +static inline unsigned long find_next_bit_le(const void *addr, + unsigned long size, unsigned long offset) +{ + return find_next_bit(addr, size, offset); +} + +static inline unsigned long find_first_zero_bit_le(const void *addr, + unsigned long size) +{ + return find_first_zero_bit(addr, size); +} + +#elif defined(__BIG_ENDIAN) + +#ifndef find_next_zero_bit_le +static inline +unsigned long find_next_zero_bit_le(const void *addr, unsigned + long size, unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val = *(const unsigned long *)addr; + + if (unlikely(offset >= size)) + return size; + + val = swab(val) | ~GENMASK(size - 1, offset); + return val == ~0UL ? size : ffz(val); + } + + return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); +} +#endif + +#ifndef find_next_bit_le +static inline +unsigned long find_next_bit_le(const void *addr, unsigned + long size, unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val = *(const unsigned long *)addr; + + if (unlikely(offset >= size)) + return size; + + val = swab(val) & GENMASK(size - 1, offset); + return val ? __ffs(val) : size; + } + + return _find_next_bit(addr, NULL, size, offset, 0UL, 1); +} +#endif + +#ifndef find_first_zero_bit_le +#define find_first_zero_bit_le(addr, size) \ + find_next_zero_bit_le((addr), (size), 0) +#endif + +#else +#error "Please fix " +#endif + +#endif /*__LINUX_FIND_H_ */ -- cgit v1.2.3-59-g8ed1b From c126a53c276048125b4a950072bab37ad0fea120 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:00 -0700 Subject: arch: remove GENERIC_FIND_FIRST_BIT entirely In 5.12 cycle we enabled GENERIC_FIND_FIRST_BIT config option for ARM64 and MIPS. It increased performance and shrunk .text size; and so far I didn't receive any negative feedback on the change. https://lore.kernel.org/linux-arch/20210225135700.1381396-1-yury.norov@gmail.com/ Now I think it's a good time to switch all architectures to use find_{first,last}_bit() unconditionally, and so remove corresponding config option. The patch does't introduce functioal changes for arc, arm, arm64, mips, m68k, s390 and x86, for other architectures I expect improvement both in performance and .text size. Signed-off-by: Yury Norov Tested-by: Alexander Lobakin (mips) Reviewed-by: Alexander Lobakin (mips) Reviewed-by: Andy Shevchenko Acked-by: Will Deacon Tested-by: Wolfram Sang --- arch/arc/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/x86/Kconfig | 1 - arch/x86/um/Kconfig | 1 - include/linux/find.h | 13 ------------- lib/Kconfig | 3 --- 8 files changed, 22 deletions(-) (limited to 'include/linux') diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index b4ae6058902a..4bec4b0b6ce1 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -20,7 +20,6 @@ config ARC select COMMON_CLK select DMA_DIRECT_REMAP select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC) - select GENERIC_FIND_FIRST_BIT # for now, we don't need GENERIC_IRQ_PROBE, CONFIG_GENERIC_IRQ_CHIP select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c4207cf9bb17..517d26c8002d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -120,7 +120,6 @@ config ARM64 select GENERIC_CPU_AUTOPROBE select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP - select GENERIC_FIND_FIRST_BIT select GENERIC_IDLE_POLL_SETUP select GENERIC_IRQ_IPI select GENERIC_IRQ_PROBE diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 0215dc1529e9..00951bfdbab0 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -32,7 +32,6 @@ config MIPS select GENERIC_ATOMIC64 if !64BIT select GENERIC_CMOS_UPDATE select GENERIC_CPU_AUTOPROBE - select GENERIC_FIND_FIRST_BIT select GENERIC_GETTIMEOFDAY select GENERIC_IOMAP select GENERIC_IRQ_PROBE diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 2a5bb4f29cfe..4f80f1c95468 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -127,7 +127,6 @@ config S390 select GENERIC_CPU_AUTOPROBE select GENERIC_CPU_VULNERABILITIES select GENERIC_ENTRY - select GENERIC_FIND_FIRST_BIT select GENERIC_GETTIMEOFDAY select GENERIC_PTDUMP select GENERIC_SMP_IDLE_THREAD diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5c2ccb85f2ef..60484b39257c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -136,7 +136,6 @@ config X86 select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_ENTRY - select GENERIC_FIND_FIRST_BIT select GENERIC_IOMAP select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_MATRIX_ALLOCATOR if X86_LOCAL_APIC diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig index 95d26a69088b..40d6a06e41c8 100644 --- a/arch/x86/um/Kconfig +++ b/arch/x86/um/Kconfig @@ -8,7 +8,6 @@ endmenu config UML_X86 def_bool y - select GENERIC_FIND_FIRST_BIT config 64BIT bool "64-bit kernel" if "$(SUBARCH)" = "x86" diff --git a/include/linux/find.h b/include/linux/find.h index c5410c243e04..ea57f7f38c49 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -101,8 +101,6 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, } #endif -#ifdef CONFIG_GENERIC_FIND_FIRST_BIT - #ifndef find_first_bit /** * find_first_bit - find the first set bit in a memory region @@ -147,17 +145,6 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) } #endif -#else /* CONFIG_GENERIC_FIND_FIRST_BIT */ - -#ifndef find_first_bit -#define find_first_bit(addr, size) find_next_bit((addr), (size), 0) -#endif -#ifndef find_first_zero_bit -#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) -#endif - -#endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ - #ifndef find_last_bit /** * find_last_bit - find the last set bit in a memory region diff --git a/lib/Kconfig b/lib/Kconfig index 5e7165e6a346..6a6ae5312fa0 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -65,9 +65,6 @@ config GENERIC_STRNLEN_USER config GENERIC_NET_UTILS bool -config GENERIC_FIND_FIRST_BIT - bool - source "lib/math/Kconfig" config NO_GENERIC_PCI_IOPORT_MAP -- cgit v1.2.3-59-g8ed1b From f68edc9297bf3f7c94abb54b9b0b053607f7587b Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:01 -0700 Subject: lib: add find_first_and_bit() Currently find_first_and_bit() is an alias to find_next_and_bit(). However, it is widely used in cpumask, so it worth to optimize it. This patch adds its own implementation for find_first_and_bit(). On x86_64 find_bit_benchmark says: Before (#define find_first_and_bit(...) find_next_and_bit(..., 0): Start testing find_bit() with random-filled bitmap [ 140.291468] find_first_and_bit: 46890919 ns, 32671 iterations Start testing find_bit() with sparse bitmap [ 140.295028] find_first_and_bit: 7103 ns, 1 iterations After: Start testing find_bit() with random-filled bitmap [ 162.574907] find_first_and_bit: 25045813 ns, 32846 iterations Start testing find_bit() with sparse bitmap [ 162.578458] find_first_and_bit: 4900 ns, 1 iterations (Thanks to Alexey Klimov for thorough testing.) Signed-off-by: Yury Norov Tested-by: Wolfram Sang Tested-by: Alexey Klimov --- include/linux/find.h | 27 +++++++++++++++++++++++++++ lib/find_bit.c | 21 +++++++++++++++++++++ lib/find_bit_benchmark.c | 21 +++++++++++++++++++++ 3 files changed, 69 insertions(+) (limited to 'include/linux') diff --git a/include/linux/find.h b/include/linux/find.h index ea57f7f38c49..6048f8c97418 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -12,6 +12,8 @@ extern unsigned long _find_next_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long nbits, unsigned long start, unsigned long invert, unsigned long le); extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); +extern unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long size); extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); @@ -123,6 +125,31 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) } #endif +#ifndef find_first_and_bit +/** + * find_first_and_bit - find the first set bit in both memory regions + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @size: The bitmap size in bits + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. + */ +static inline +unsigned long find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); + + return val ? __ffs(val) : size; + } + + return _find_first_and_bit(addr1, addr2, size); +} +#endif + #ifndef find_first_zero_bit /** * find_first_zero_bit - find the first cleared bit in a memory region diff --git a/lib/find_bit.c b/lib/find_bit.c index 0f8e2e369b1d..1b8e4b2a9cba 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -89,6 +89,27 @@ unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) EXPORT_SYMBOL(_find_first_bit); #endif +#ifndef find_first_and_bit +/* + * Find the first set bit in two memory regions. + */ +unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) +{ + unsigned long idx, val; + + for (idx = 0; idx * BITS_PER_LONG < size; idx++) { + val = addr1[idx] & addr2[idx]; + if (val) + return min(idx * BITS_PER_LONG + __ffs(val), size); + } + + return size; +} +EXPORT_SYMBOL(_find_first_and_bit); +#endif + #ifndef find_first_zero_bit /* * Find the first cleared bit in a memory region. diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c index 5637c5711db9..db904b57d4b8 100644 --- a/lib/find_bit_benchmark.c +++ b/lib/find_bit_benchmark.c @@ -49,6 +49,25 @@ static int __init test_find_first_bit(void *bitmap, unsigned long len) return 0; } +static int __init test_find_first_and_bit(void *bitmap, const void *bitmap2, unsigned long len) +{ + static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata; + unsigned long i, cnt; + ktime_t time; + + bitmap_copy(cp, bitmap, BITMAP_LEN); + + time = ktime_get(); + for (cnt = i = 0; i < len; cnt++) { + i = find_first_and_bit(cp, bitmap2, len); + __clear_bit(i, cp); + } + time = ktime_get() - time; + pr_err("find_first_and_bit: %18llu ns, %6ld iterations\n", time, cnt); + + return 0; +} + static int __init test_find_next_bit(const void *bitmap, unsigned long len) { unsigned long i, cnt; @@ -129,6 +148,7 @@ static int __init find_bit_test(void) * traverse only part of bitmap to avoid soft lockup. */ test_find_first_bit(bitmap, BITMAP_LEN / 10); + test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN / 2); test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN); pr_err("\nStart testing find_bit() with sparse bitmap\n"); @@ -145,6 +165,7 @@ static int __init find_bit_test(void) test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); test_find_first_bit(bitmap, BITMAP_LEN); + test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN); test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN); /* -- cgit v1.2.3-59-g8ed1b From 93ba139ba8190c33009c5353ca43c8519443f467 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:02 -0700 Subject: cpumask: use find_first_and_bit() Now we have an efficient implementation for find_first_and_bit(), so switch cpumask to use it where appropriate. Signed-off-by: Yury Norov Tested-by: Wolfram Sang --- include/linux/cpumask.h | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 1e7399fc69c0..c4e1b9ea0ba4 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -123,6 +123,12 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) return 0; } +static inline unsigned int cpumask_first_and(const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return 0; +} + static inline unsigned int cpumask_last(const struct cpumask *srcp) { return 0; @@ -167,7 +173,7 @@ static inline unsigned int cpumask_local_spread(unsigned int i, int node) static inline int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) { - return cpumask_next_and(-1, src1p, src2p); + return cpumask_first_and(src1p, src2p); } static inline int cpumask_any_distribute(const struct cpumask *srcp) @@ -195,6 +201,19 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); } +/** + * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 + * @src1p: the first input + * @src2p: the second input + * + * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). + */ +static inline +unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) +{ + return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); +} + /** * cpumask_last - get the last CPU in a cpumask * @srcp: - the cpumask pointer @@ -585,15 +604,6 @@ static inline void cpumask_copy(struct cpumask *dstp, */ #define cpumask_any(srcp) cpumask_first(srcp) -/** - * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 - * @src1p: the first input - * @src2p: the second input - * - * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). - */ -#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p)) - /** * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2 * @mask1: the first input cpumask -- cgit v1.2.3-59-g8ed1b From 9b51d9d866482a703646fd4c07e433c3d9d88efd Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:05 -0700 Subject: cpumask: replace cpumask_next_* with cpumask_first_* where appropriate cpumask_first() is a more effective analogue of 'next' version if n == -1 (which means start == 0). This patch replaces 'next' with 'first' where things look trivial. There's no cpumask_first_zero() function, so create it. Signed-off-by: Yury Norov Tested-by: Wolfram Sang --- block/blk-mq.c | 2 +- drivers/net/virtio_net.c | 2 +- drivers/soc/fsl/qbman/bman_portal.c | 2 +- drivers/soc/fsl/qbman/qman_portal.c | 2 +- include/linux/cpumask.h | 16 ++++++++++++++++ kernel/time/clocksource.c | 4 ++-- 6 files changed, 22 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index 8874a63ae952..ef56e753ab38 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2967,7 +2967,7 @@ static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu, struct blk_mq_hw_ctx *hctx) { - if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu) + if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu) return false; if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids) return false; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index b107835242ad..8c70ab3c436a 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2101,7 +2101,7 @@ static void virtnet_set_affinity(struct virtnet_info *vi) stragglers = num_cpu >= vi->curr_queue_pairs ? num_cpu % vi->curr_queue_pairs : 0; - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); for (i = 0; i < vi->curr_queue_pairs; i++) { group_size = stride + (i < stragglers ? 1 : 0); diff --git a/drivers/soc/fsl/qbman/bman_portal.c b/drivers/soc/fsl/qbman/bman_portal.c index acda8a5637c5..4d7b9caee1c4 100644 --- a/drivers/soc/fsl/qbman/bman_portal.c +++ b/drivers/soc/fsl/qbman/bman_portal.c @@ -155,7 +155,7 @@ static int bman_portal_probe(struct platform_device *pdev) } spin_lock(&bman_lock); - cpu = cpumask_next_zero(-1, &portal_cpus); + cpu = cpumask_first_zero(&portal_cpus); if (cpu >= nr_cpu_ids) { __bman_portals_probed = 1; /* unassigned portal, skip init */ diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 96f74a1dc603..e23b60618c1a 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -248,7 +248,7 @@ static int qman_portal_probe(struct platform_device *pdev) pcfg->pools = qm_get_pools_sdqcr(); spin_lock(&qman_lock); - cpu = cpumask_next_zero(-1, &portal_cpus); + cpu = cpumask_first_zero(&portal_cpus); if (cpu >= nr_cpu_ids) { __qman_portals_probed = 1; /* unassigned portal, skip init */ diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index c4e1b9ea0ba4..64dae70d31f5 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -123,6 +123,11 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) return 0; } +static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) +{ + return 0; +} + static inline unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) { @@ -201,6 +206,17 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); } +/** + * cpumask_first_zero - get the first unset cpu in a cpumask + * @srcp: the cpumask pointer + * + * Returns >= nr_cpu_ids if all cpus are set. + */ +static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) +{ + return find_first_zero_bit(cpumask_bits(srcp), nr_cpumask_bits); +} + /** * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 * @src1p: the first input diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index b8a14d2fb5ba..f29d1a524fa5 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -257,7 +257,7 @@ static void clocksource_verify_choose_cpus(void) return; /* Make sure to select at least one CPU other than the current CPU. */ - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); if (cpu == smp_processor_id()) cpu = cpumask_next(cpu, cpu_online_mask); if (WARN_ON_ONCE(cpu >= nr_cpu_ids)) @@ -279,7 +279,7 @@ static void clocksource_verify_choose_cpus(void) cpu = prandom_u32() % nr_cpu_ids; cpu = cpumask_next(cpu - 1, cpu_online_mask); if (cpu >= nr_cpu_ids) - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); if (!WARN_ON_ONCE(cpu >= nr_cpu_ids)) cpumask_set_cpu(cpu, &cpus_chosen); } -- cgit v1.2.3-59-g8ed1b From bc9d6635c293a2ac30c6319f7cfd08860ab7948a Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:06 -0700 Subject: include/linux: move for_each_bit() macros from bitops.h to find.h for_each_bit() macros depend on find_bit() machinery, and so the proper place for them is the find.h header. Signed-off-by: Yury Norov Tested-by: Wolfram Sang --- include/linux/bitops.h | 34 ---------------------------------- include/linux/find.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 5e62e2383b7f..7aaed501f768 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -32,40 +32,6 @@ extern unsigned long __sw_hweight64(__u64 w); */ #include -#define for_each_set_bit(bit, addr, size) \ - for ((bit) = find_first_bit((addr), (size)); \ - (bit) < (size); \ - (bit) = find_next_bit((addr), (size), (bit) + 1)) - -/* same as for_each_set_bit() but use bit as value to start with */ -#define for_each_set_bit_from(bit, addr, size) \ - for ((bit) = find_next_bit((addr), (size), (bit)); \ - (bit) < (size); \ - (bit) = find_next_bit((addr), (size), (bit) + 1)) - -#define for_each_clear_bit(bit, addr, size) \ - for ((bit) = find_first_zero_bit((addr), (size)); \ - (bit) < (size); \ - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) - -/* same as for_each_clear_bit() but use bit as value to start with */ -#define for_each_clear_bit_from(bit, addr, size) \ - for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ - (bit) < (size); \ - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) - -/** - * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits - * @start: bit offset to start search and to store the current iteration offset - * @clump: location to store copy of current 8-bit clump - * @bits: bitmap address to base the search on - * @size: bitmap size in number of bits - */ -#define for_each_set_clump8(start, clump, bits, size) \ - for ((start) = find_first_clump8(&(clump), (bits), (size)); \ - (start) < (size); \ - (start) = find_next_clump8(&(clump), (bits), (size), (start) + 8)) - static inline int get_bitmask_order(unsigned int count) { int order; diff --git a/include/linux/find.h b/include/linux/find.h index 6048f8c97418..4500e8ab93e2 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -279,4 +279,38 @@ unsigned long find_next_bit_le(const void *addr, unsigned #error "Please fix " #endif +#define for_each_set_bit(bit, addr, size) \ + for ((bit) = find_first_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1)) + +/* same as for_each_set_bit() but use bit as value to start with */ +#define for_each_set_bit_from(bit, addr, size) \ + for ((bit) = find_next_bit((addr), (size), (bit)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1)) + +#define for_each_clear_bit(bit, addr, size) \ + for ((bit) = find_first_zero_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) + +/* same as for_each_clear_bit() but use bit as value to start with */ +#define for_each_clear_bit_from(bit, addr, size) \ + for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ + (bit) < (size); \ + (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) + +/** + * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits + * @start: bit offset to start search and to store the current iteration offset + * @clump: location to store copy of current 8-bit clump + * @bits: bitmap address to base the search on + * @size: bitmap size in number of bits + */ +#define for_each_set_clump8(start, clump, bits, size) \ + for ((start) = find_first_clump8(&(clump), (bits), (size)); \ + (start) < (size); \ + (start) = find_next_clump8(&(clump), (bits), (size), (start) + 8)) + #endif /*__LINUX_FIND_H_ */ -- cgit v1.2.3-59-g8ed1b From 7516be9931b8bc8bcaac8531f490b42ab11ded1e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:07 -0700 Subject: find: micro-optimize for_each_{set,clear}_bit() The macros iterate thru all set/clear bits in a bitmap. They search a first bit using find_first_bit(), and the rest bits using find_next_bit(). Since find_next_bit() is called shortly after find_first_bit(), we can save few lines of I-cache by not using find_first_bit(). Signed-off-by: Yury Norov Tested-by: Wolfram Sang --- include/linux/find.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/find.h b/include/linux/find.h index 4500e8ab93e2..ae9ed52b52b8 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -280,7 +280,7 @@ unsigned long find_next_bit_le(const void *addr, unsigned #endif #define for_each_set_bit(bit, addr, size) \ - for ((bit) = find_first_bit((addr), (size)); \ + for ((bit) = find_next_bit((addr), (size), 0); \ (bit) < (size); \ (bit) = find_next_bit((addr), (size), (bit) + 1)) @@ -291,7 +291,7 @@ unsigned long find_next_bit_le(const void *addr, unsigned (bit) = find_next_bit((addr), (size), (bit) + 1)) #define for_each_clear_bit(bit, addr, size) \ - for ((bit) = find_first_zero_bit((addr), (size)); \ + for ((bit) = find_next_zero_bit((addr), (size), 0); \ (bit) < (size); \ (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) -- cgit v1.2.3-59-g8ed1b From ec288a2cf7ca40a939316b6df206ab845bb112d1 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 14 Aug 2021 14:17:11 -0700 Subject: bitmap: unify find_bit operations bitmap_for_each_{set,clear}_region() are similar to for_each_bit() macros in include/linux/find.h, but interface and implementation of them are different. This patch adds for_each_bitrange() macros and drops unused bitmap_*_region() API in sake of unification. Signed-off-by: Yury Norov Tested-by: Wolfram Sang Acked-by: Dennis Zhou Acked-by: Ulf Hansson # For MMC --- drivers/mmc/host/renesas_sdhi_core.c | 2 +- include/linux/bitmap.h | 33 --------------------- include/linux/find.h | 56 ++++++++++++++++++++++++++++++++++++ mm/percpu.c | 20 ++++++------- 4 files changed, 65 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c index f5b2684ad805..6de30a2719a3 100644 --- a/drivers/mmc/host/renesas_sdhi_core.c +++ b/drivers/mmc/host/renesas_sdhi_core.c @@ -628,7 +628,7 @@ static int renesas_sdhi_select_tuning(struct tmio_mmc_host *host) * is at least SH_MOBILE_SDHI_MIN_TAP_ROW probes long then use the * center index as the tap, otherwise bail out. */ - bitmap_for_each_set_region(bitmap, rs, re, 0, taps_size) { + for_each_set_bitrange(rs, re, bitmap, taps_size) { if (re - rs > tap_cnt) { tap_end = re; tap_start = rs; diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index ead4a150bd7f..7dba0847510c 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -55,12 +55,6 @@ struct device; * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area * bitmap_find_next_zero_area_off(buf, len, pos, n, mask, mask_off) as above - * bitmap_next_clear_region(map, &start, &end, nbits) Find next clear region - * bitmap_next_set_region(map, &start, &end, nbits) Find next set region - * bitmap_for_each_clear_region(map, rs, re, start, end) - * Iterate over all clear regions - * bitmap_for_each_set_region(map, rs, re, start, end) - * Iterate over all set regions * bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n * bitmap_shift_left(dst, src, n, nbits) *dst = *src << n * bitmap_cut(dst, src, first, n, nbits) Cut n bits from first, copy rest @@ -467,14 +461,6 @@ static inline void bitmap_replace(unsigned long *dst, __bitmap_replace(dst, old, new, mask, nbits); } -static inline void bitmap_next_clear_region(unsigned long *bitmap, - unsigned int *rs, unsigned int *re, - unsigned int end) -{ - *rs = find_next_zero_bit(bitmap, end, *rs); - *re = find_next_bit(bitmap, end, *rs + 1); -} - static inline void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs, unsigned int *re, unsigned int end) @@ -483,25 +469,6 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, *re = find_next_zero_bit(bitmap, end, *rs + 1); } -/* - * Bitmap region iterators. Iterates over the bitmap between [@start, @end). - * @rs and @re should be integer variables and will be set to start and end - * index of the current clear or set region. - */ -#define bitmap_for_each_clear_region(bitmap, rs, re, start, end) \ - for ((rs) = (start), \ - bitmap_next_clear_region((bitmap), &(rs), &(re), (end)); \ - (rs) < (re); \ - (rs) = (re) + 1, \ - bitmap_next_clear_region((bitmap), &(rs), &(re), (end))) - -#define bitmap_for_each_set_region(bitmap, rs, re, start, end) \ - for ((rs) = (start), \ - bitmap_next_set_region((bitmap), &(rs), &(re), (end)); \ - (rs) < (re); \ - (rs) = (re) + 1, \ - bitmap_next_set_region((bitmap), &(rs), &(re), (end))) - /** * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap. * @n: u64 value diff --git a/include/linux/find.h b/include/linux/find.h index ae9ed52b52b8..5bb6db213bcb 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -301,6 +301,62 @@ unsigned long find_next_bit_le(const void *addr, unsigned (bit) < (size); \ (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) +/** + * for_each_set_bitrange - iterate over all set bit ranges [b; e) + * @b: bit offset of start of current bitrange (first set bit) + * @e: bit offset of end of current bitrange (first unset bit) + * @addr: bitmap address to base the search on + * @size: bitmap size in number of bits + */ +#define for_each_set_bitrange(b, e, addr, size) \ + for ((b) = find_next_bit((addr), (size), 0), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1); \ + (b) < (size); \ + (b) = find_next_bit((addr), (size), (e) + 1), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1)) + +/** + * for_each_set_bitrange_from - iterate over all set bit ranges [b; e) + * @b: bit offset of start of current bitrange (first set bit); must be initialized + * @e: bit offset of end of current bitrange (first unset bit) + * @addr: bitmap address to base the search on + * @size: bitmap size in number of bits + */ +#define for_each_set_bitrange_from(b, e, addr, size) \ + for ((b) = find_next_bit((addr), (size), (b)), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1); \ + (b) < (size); \ + (b) = find_next_bit((addr), (size), (e) + 1), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1)) + +/** + * for_each_clear_bitrange - iterate over all unset bit ranges [b; e) + * @b: bit offset of start of current bitrange (first unset bit) + * @e: bit offset of end of current bitrange (first set bit) + * @addr: bitmap address to base the search on + * @size: bitmap size in number of bits + */ +#define for_each_clear_bitrange(b, e, addr, size) \ + for ((b) = find_next_zero_bit((addr), (size), 0), \ + (e) = find_next_bit((addr), (size), (b) + 1); \ + (b) < (size); \ + (b) = find_next_zero_bit((addr), (size), (e) + 1), \ + (e) = find_next_bit((addr), (size), (b) + 1)) + +/** + * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) + * @b: bit offset of start of current bitrange (first set bit); must be initialized + * @e: bit offset of end of current bitrange (first unset bit) + * @addr: bitmap address to base the search on + * @size: bitmap size in number of bits + */ +#define for_each_clear_bitrange_from(b, e, addr, size) \ + for ((b) = find_next_zero_bit((addr), (size), (b)), \ + (e) = find_next_bit((addr), (size), (b) + 1); \ + (b) < (size); \ + (b) = find_next_zero_bit((addr), (size), (e) + 1), \ + (e) = find_next_bit((addr), (size), (b) + 1)) + /** * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits * @start: bit offset to start search and to store the current iteration offset diff --git a/mm/percpu.c b/mm/percpu.c index 0c6f85c5514f..293009cc03ef 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -779,7 +779,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) { struct pcpu_block_md *block = chunk->md_blocks + index; unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); - unsigned int rs, re, start; /* region start, region end */ + unsigned int start, end; /* region start, region end */ /* promote scan_hint to contig_hint */ if (block->scan_hint) { @@ -795,9 +795,8 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) block->right_free = 0; /* iterate over free areas and update the contig hints */ - bitmap_for_each_clear_region(alloc_map, rs, re, start, - PCPU_BITMAP_BLOCK_BITS) - pcpu_block_update(block, rs, re); + for_each_clear_bitrange_from(start, end, alloc_map, PCPU_BITMAP_BLOCK_BITS) + pcpu_block_update(block, start, end); } /** @@ -1852,13 +1851,12 @@ area_found: /* populate if not all pages are already there */ if (!is_atomic) { - unsigned int page_start, page_end, rs, re; + unsigned int page_end, rs, re; - page_start = PFN_DOWN(off); + rs = PFN_DOWN(off); page_end = PFN_UP(off + size); - bitmap_for_each_clear_region(chunk->populated, rs, re, - page_start, page_end) { + for_each_clear_bitrange_from(rs, re, chunk->populated, page_end) { WARN_ON(chunk->immutable); ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp); @@ -2014,8 +2012,7 @@ static void pcpu_balance_free(bool empty_only) list_for_each_entry_safe(chunk, next, &to_free, list) { unsigned int rs, re; - bitmap_for_each_set_region(chunk->populated, rs, re, 0, - chunk->nr_pages) { + for_each_set_bitrange(rs, re, chunk->populated, chunk->nr_pages) { pcpu_depopulate_chunk(chunk, rs, re); spin_lock_irq(&pcpu_lock); pcpu_chunk_depopulated(chunk, rs, re); @@ -2085,8 +2082,7 @@ retry_pop: continue; /* @chunk can't go away while pcpu_alloc_mutex is held */ - bitmap_for_each_clear_region(chunk->populated, rs, re, 0, - chunk->nr_pages) { + for_each_clear_bitrange(rs, re, chunk->populated, chunk->nr_pages) { int nr = min_t(int, re - rs, nr_to_pop); spin_unlock_irq(&pcpu_lock); -- cgit v1.2.3-59-g8ed1b From 7372971c1be5b7d4fdd8ad237798bdc1d1d54162 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 11 Jan 2022 10:19:22 +0300 Subject: rtc: mc146818-lib: fix signedness bug in mc146818_get_time() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mc146818_get_time() function returns zero on success or negative a error code on failure. It needs to be type int. Fixes: d35786b3a28d ("rtc: mc146818-lib: change return values of mc146818_get_time()") Signed-off-by: Dan Carpenter Reviewed-by: Mateusz Jończyk Signed-off-by: Alexandre Belloni Link: https://lore.kernel.org/r/20220111071922.GE11243@kili --- drivers/rtc/rtc-mc146818-lib.c | 2 +- include/linux/mc146818rtc.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/rtc/rtc-mc146818-lib.c b/drivers/rtc/rtc-mc146818-lib.c index f62e658cbe23..7f689f1bafc5 100644 --- a/drivers/rtc/rtc-mc146818-lib.c +++ b/drivers/rtc/rtc-mc146818-lib.c @@ -130,7 +130,7 @@ static void mc146818_get_time_callback(unsigned char seconds, void *param_in) p->ctrl = CMOS_READ(RTC_CONTROL); } -unsigned int mc146818_get_time(struct rtc_time *time) +int mc146818_get_time(struct rtc_time *time) { struct mc146818_get_time_callback_param p = { .time = time diff --git a/include/linux/mc146818rtc.h b/include/linux/mc146818rtc.h index 67fb0a12becc..808bb4cee230 100644 --- a/include/linux/mc146818rtc.h +++ b/include/linux/mc146818rtc.h @@ -124,7 +124,7 @@ struct cmos_rtc_board_info { #endif /* ARCH_RTC_LOCATION */ bool mc146818_does_rtc_work(void); -unsigned int mc146818_get_time(struct rtc_time *time); +int mc146818_get_time(struct rtc_time *time); int mc146818_set_time(struct rtc_time *time); bool mc146818_avoid_UIP(void (*callback)(unsigned char seconds, void *param), -- cgit v1.2.3-59-g8ed1b From 3fe7fa5843d204e235d92902190fecb972a3f9cc Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 10 Dec 2021 15:09:21 -0500 Subject: mm: Add folio_put_refs() This is like folio_put(), but puts N references at once instead of just one. It's like put_page_refs(), but does one atomic operation instead of two, and is available to more than just gup.c. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: John Hubbard Reviewed-by: Jason Gunthorpe Reviewed-by: William Kucharski --- include/linux/mm.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index c768a7c81b0b..cb98f75b245e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1244,6 +1244,26 @@ static inline void folio_put(struct folio *folio) __put_page(&folio->page); } +/** + * folio_put_refs - Reduce the reference count on a folio. + * @folio: The folio. + * @refs: The amount to subtract from the folio's reference count. + * + * If the folio's reference count reaches zero, the memory will be + * released back to the page allocator and may be used by another + * allocation immediately. Do not access the memory or the struct folio + * after calling folio_put_refs() unless you can be sure that these weren't + * the last references. + * + * Context: May be called in process or interrupt context, but not in NMI + * context. May be called while holding a spinlock. + */ +static inline void folio_put_refs(struct folio *folio, int refs) +{ + if (folio_ref_sub_and_test(folio, refs)) + __put_page(&folio->page); +} + static inline void put_page(struct page *page) { struct folio *folio = page_folio(page); -- cgit v1.2.3-59-g8ed1b From a6097180d884ddab769fb25588ea8598589c218c Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Mon, 17 Jan 2022 09:07:26 +1100 Subject: devtmpfs regression fix: reconfigure on each mount Prior to Linux v5.4 devtmpfs used mount_single() which treats the given mount options as "remount" options, so it updates the configuration of the single super_block on each mount. Since that was changed, the mount options used for devtmpfs are ignored. This is a regression which affect systemd - which mounts devtmpfs with "-o mode=755,size=4m,nr_inodes=1m". This patch restores the "remount" effect by calling reconfigure_single() Fixes: d401727ea0d7 ("devtmpfs: don't mix {ramfs,shmem}_fill_super() with mount_single()") Acked-by: Christian Brauner Cc: Al Viro Signed-off-by: NeilBrown Signed-off-by: Linus Torvalds --- drivers/base/devtmpfs.c | 7 +++++++ fs/super.c | 4 ++-- include/linux/fs_context.h | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index 1e2c2d3882e2..f41063ac1aee 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -65,8 +65,15 @@ static struct dentry *public_dev_mount(struct file_system_type *fs_type, int fla const char *dev_name, void *data) { struct super_block *s = mnt->mnt_sb; + int err; + atomic_inc(&s->s_active); down_write(&s->s_umount); + err = reconfigure_single(s, flags, data); + if (err < 0) { + deactivate_locked_super(s); + return ERR_PTR(err); + } return dget(s->s_root); } diff --git a/fs/super.c b/fs/super.c index 3bfc0f8fbd5b..a6405d44d4ca 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1423,8 +1423,8 @@ struct dentry *mount_nodev(struct file_system_type *fs_type, } EXPORT_SYMBOL(mount_nodev); -static int reconfigure_single(struct super_block *s, - int flags, void *data) +int reconfigure_single(struct super_block *s, + int flags, void *data) { struct fs_context *fc; int ret; diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h index 6b54982fc5f3..13fa6f3df8e4 100644 --- a/include/linux/fs_context.h +++ b/include/linux/fs_context.h @@ -142,6 +142,8 @@ extern void put_fs_context(struct fs_context *fc); extern int vfs_parse_fs_param_source(struct fs_context *fc, struct fs_parameter *param); extern void fc_drop_locked(struct fs_context *fc); +int reconfigure_single(struct super_block *s, + int flags, void *data); /* * sget() wrappers to be called from the ->get_tree() op. -- cgit v1.2.3-59-g8ed1b From 09f5e7dc7ad705289e1b1ec065439aa3c42951c4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 20 Dec 2021 13:19:52 +0100 Subject: perf: Fix perf_event_read_local() time Time readers that cannot take locks (due to NMI etc..) currently make use of perf_event::shadow_ctx_time, which, for that event gives: time' = now + (time - timestamp) or, alternatively arranged: time' = time + (now - timestamp) IOW, the progression of time since the last time the shadow_ctx_time was updated. There's problems with this: A) the shadow_ctx_time is per-event, even though the ctx_time it reflects is obviously per context. The direct concequence of this is that the context needs to iterate all events all the time to keep the shadow_ctx_time in sync. B) even with the prior point, the context itself might not be active meaning its time should not advance to begin with. C) shadow_ctx_time isn't consistently updated when ctx_time is There are 3 users of this stuff, that suffer differently from this: - calc_timer_values() - perf_output_read() - perf_event_update_userpage() /* A */ - perf_event_read_local() /* A,B */ In particular, perf_output_read() doesn't suffer at all, because it's sample driven and hence only relevant when the event is actually running. This same was supposed to be true for perf_event_update_userpage(), after all self-monitoring implies the context is active *HOWEVER*, as per commit f79256532682 ("perf/core: fix userpage->time_enabled of inactive events") this goes wrong when combined with counter overcommit, in that case those events that do not get scheduled when the context becomes active (task events typically) miss out on the EVENT_TIME update and ENABLED time is inflated (for a little while) with the time the context was inactive. Once the event gets rotated in, this gets corrected, leading to a non-monotonic timeflow. perf_event_read_local() made things even worse, it can request time at any point, suffering all the problems perf_event_update_userpage() does and more. Because while perf_event_update_userpage() is limited by the context being active, perf_event_read_local() users have no such constraint. Therefore, completely overhaul things and do away with perf_event::shadow_ctx_time. Instead have regular context time updates keep track of this offset directly and provide perf_event_time_now() to complement perf_event_time(). perf_event_time_now() will, in adition to being context wide, also take into account if the context is active. For inactive context, it will not advance time. This latter property means the cgroup perf_cgroup_info context needs to grow addition state to track this. Additionally, since all this is strictly per-cpu, we can use barrier() to order context activity vs context time. Fixes: 7d9285e82db5 ("perf/bpf: Extend the perf_event_read_local() interface, a.k.a. "bpf: perf event change needed for subsequent bpf helpers"") Signed-off-by: Peter Zijlstra (Intel) Tested-by: Song Liu Tested-by: Namhyung Kim Link: https://lkml.kernel.org/r/YcB06DasOBtU0b00@hirez.programming.kicks-ass.net --- include/linux/perf_event.h | 15 +-- kernel/events/core.c | 246 +++++++++++++++++++++++++++------------------ 2 files changed, 149 insertions(+), 112 deletions(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 117f230bcdfd..733649184b27 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -693,18 +693,6 @@ struct perf_event { u64 total_time_running; u64 tstamp; - /* - * timestamp shadows the actual context timing but it can - * be safely used in NMI interrupt context. It reflects the - * context time as it was when the event was last scheduled in, - * or when ctx_sched_in failed to schedule the event because we - * run out of PMC. - * - * ctx_time already accounts for ctx->timestamp. Therefore to - * compute ctx_time for a sample, simply add perf_clock(). - */ - u64 shadow_ctx_time; - struct perf_event_attr attr; u16 header_size; u16 id_header_size; @@ -852,6 +840,7 @@ struct perf_event_context { */ u64 time; u64 timestamp; + u64 timeoffset; /* * These fields let us detect when two contexts have both @@ -934,6 +923,8 @@ struct bpf_perf_event_data_kern { struct perf_cgroup_info { u64 time; u64 timestamp; + u64 timeoffset; + int active; }; struct perf_cgroup { diff --git a/kernel/events/core.c b/kernel/events/core.c index fc18664f49b0..479c9e672ec4 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -674,6 +674,23 @@ perf_event_set_state(struct perf_event *event, enum perf_event_state state) WRITE_ONCE(event->state, state); } +/* + * UP store-release, load-acquire + */ + +#define __store_release(ptr, val) \ +do { \ + barrier(); \ + WRITE_ONCE(*(ptr), (val)); \ +} while (0) + +#define __load_acquire(ptr) \ +({ \ + __unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \ + barrier(); \ + ___p; \ +}) + #ifdef CONFIG_CGROUP_PERF static inline bool @@ -719,34 +736,51 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event) return t->time; } -static inline void __update_cgrp_time(struct perf_cgroup *cgrp) +static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) { - struct perf_cgroup_info *info; - u64 now; - - now = perf_clock(); + struct perf_cgroup_info *t; - info = this_cpu_ptr(cgrp->info); + t = per_cpu_ptr(event->cgrp->info, event->cpu); + if (!__load_acquire(&t->active)) + return t->time; + now += READ_ONCE(t->timeoffset); + return now; +} - info->time += now - info->timestamp; +static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv) +{ + if (adv) + info->time += now - info->timestamp; info->timestamp = now; + /* + * see update_context_time() + */ + WRITE_ONCE(info->timeoffset, info->time - info->timestamp); } -static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) +static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) { struct perf_cgroup *cgrp = cpuctx->cgrp; struct cgroup_subsys_state *css; + struct perf_cgroup_info *info; if (cgrp) { + u64 now = perf_clock(); + for (css = &cgrp->css; css; css = css->parent) { cgrp = container_of(css, struct perf_cgroup, css); - __update_cgrp_time(cgrp); + info = this_cpu_ptr(cgrp->info); + + __update_cgrp_time(info, now, true); + if (final) + __store_release(&info->active, 0); } } } static inline void update_cgrp_time_from_event(struct perf_event *event) { + struct perf_cgroup_info *info; struct perf_cgroup *cgrp; /* @@ -760,8 +794,10 @@ static inline void update_cgrp_time_from_event(struct perf_event *event) /* * Do not update time when cgroup is not active */ - if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) - __update_cgrp_time(event->cgrp); + if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) { + info = this_cpu_ptr(event->cgrp->info); + __update_cgrp_time(info, perf_clock(), true); + } } static inline void @@ -785,7 +821,8 @@ perf_cgroup_set_timestamp(struct task_struct *task, for (css = &cgrp->css; css; css = css->parent) { cgrp = container_of(css, struct perf_cgroup, css); info = this_cpu_ptr(cgrp->info); - info->timestamp = ctx->timestamp; + __update_cgrp_time(info, ctx->timestamp, false); + __store_release(&info->active, 1); } } @@ -981,14 +1018,6 @@ out: return ret; } -static inline void -perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) -{ - struct perf_cgroup_info *t; - t = per_cpu_ptr(event->cgrp->info, event->cpu); - event->shadow_ctx_time = now - t->timestamp; -} - static inline void perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx) { @@ -1066,7 +1095,8 @@ static inline void update_cgrp_time_from_event(struct perf_event *event) { } -static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) +static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, + bool final) { } @@ -1098,12 +1128,12 @@ perf_cgroup_switch(struct task_struct *task, struct task_struct *next) { } -static inline void -perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) +static inline u64 perf_cgroup_event_time(struct perf_event *event) { + return 0; } -static inline u64 perf_cgroup_event_time(struct perf_event *event) +static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) { return 0; } @@ -1525,22 +1555,59 @@ static void perf_unpin_context(struct perf_event_context *ctx) /* * Update the record of the current time in a context. */ -static void update_context_time(struct perf_event_context *ctx) +static void __update_context_time(struct perf_event_context *ctx, bool adv) { u64 now = perf_clock(); - ctx->time += now - ctx->timestamp; + if (adv) + ctx->time += now - ctx->timestamp; ctx->timestamp = now; + + /* + * The above: time' = time + (now - timestamp), can be re-arranged + * into: time` = now + (time - timestamp), which gives a single value + * offset to compute future time without locks on. + * + * See perf_event_time_now(), which can be used from NMI context where + * it's (obviously) not possible to acquire ctx->lock in order to read + * both the above values in a consistent manner. + */ + WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp); +} + +static void update_context_time(struct perf_event_context *ctx) +{ + __update_context_time(ctx, true); } static u64 perf_event_time(struct perf_event *event) { struct perf_event_context *ctx = event->ctx; + if (unlikely(!ctx)) + return 0; + if (is_cgroup_event(event)) return perf_cgroup_event_time(event); - return ctx ? ctx->time : 0; + return ctx->time; +} + +static u64 perf_event_time_now(struct perf_event *event, u64 now) +{ + struct perf_event_context *ctx = event->ctx; + + if (unlikely(!ctx)) + return 0; + + if (is_cgroup_event(event)) + return perf_cgroup_event_time_now(event, now); + + if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) + return ctx->time; + + now += READ_ONCE(ctx->timeoffset); + return now; } static enum event_type_t get_event_type(struct perf_event *event) @@ -2350,7 +2417,7 @@ __perf_remove_from_context(struct perf_event *event, if (ctx->is_active & EVENT_TIME) { update_context_time(ctx); - update_cgrp_time_from_cpuctx(cpuctx); + update_cgrp_time_from_cpuctx(cpuctx, false); } event_sched_out(event, cpuctx, ctx); @@ -2361,6 +2428,9 @@ __perf_remove_from_context(struct perf_event *event, list_del_event(event, ctx); if (!ctx->nr_events && ctx->is_active) { + if (ctx == &cpuctx->ctx) + update_cgrp_time_from_cpuctx(cpuctx, true); + ctx->is_active = 0; ctx->rotate_necessary = 0; if (ctx->task) { @@ -2482,40 +2552,6 @@ void perf_event_disable_inatomic(struct perf_event *event) irq_work_queue(&event->pending); } -static void perf_set_shadow_time(struct perf_event *event, - struct perf_event_context *ctx) -{ - /* - * use the correct time source for the time snapshot - * - * We could get by without this by leveraging the - * fact that to get to this function, the caller - * has most likely already called update_context_time() - * and update_cgrp_time_xx() and thus both timestamp - * are identical (or very close). Given that tstamp is, - * already adjusted for cgroup, we could say that: - * tstamp - ctx->timestamp - * is equivalent to - * tstamp - cgrp->timestamp. - * - * Then, in perf_output_read(), the calculation would - * work with no changes because: - * - event is guaranteed scheduled in - * - no scheduled out in between - * - thus the timestamp would be the same - * - * But this is a bit hairy. - * - * So instead, we have an explicit cgroup call to remain - * within the time source all along. We believe it - * is cleaner and simpler to understand. - */ - if (is_cgroup_event(event)) - perf_cgroup_set_shadow_time(event, event->tstamp); - else - event->shadow_ctx_time = event->tstamp - ctx->timestamp; -} - #define MAX_INTERRUPTS (~0ULL) static void perf_log_throttle(struct perf_event *event, int enable); @@ -2556,8 +2592,6 @@ event_sched_in(struct perf_event *event, perf_pmu_disable(event->pmu); - perf_set_shadow_time(event, ctx); - perf_log_itrace_start(event); if (event->pmu->add(event, PERF_EF_START)) { @@ -3251,16 +3285,6 @@ static void ctx_sched_out(struct perf_event_context *ctx, return; } - ctx->is_active &= ~event_type; - if (!(ctx->is_active & EVENT_ALL)) - ctx->is_active = 0; - - if (ctx->task) { - WARN_ON_ONCE(cpuctx->task_ctx != ctx); - if (!ctx->is_active) - cpuctx->task_ctx = NULL; - } - /* * Always update time if it was set; not only when it changes. * Otherwise we can 'forget' to update time for any but the last @@ -3274,7 +3298,22 @@ static void ctx_sched_out(struct perf_event_context *ctx, if (is_active & EVENT_TIME) { /* update (and stop) ctx time */ update_context_time(ctx); - update_cgrp_time_from_cpuctx(cpuctx); + update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx); + /* + * CPU-release for the below ->is_active store, + * see __load_acquire() in perf_event_time_now() + */ + barrier(); + } + + ctx->is_active &= ~event_type; + if (!(ctx->is_active & EVENT_ALL)) + ctx->is_active = 0; + + if (ctx->task) { + WARN_ON_ONCE(cpuctx->task_ctx != ctx); + if (!ctx->is_active) + cpuctx->task_ctx = NULL; } is_active ^= ctx->is_active; /* changed bits */ @@ -3711,13 +3750,19 @@ static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx, return 0; } +/* + * Because the userpage is strictly per-event (there is no concept of context, + * so there cannot be a context indirection), every userpage must be updated + * when context time starts :-( + * + * IOW, we must not miss EVENT_TIME edges. + */ static inline bool event_update_userpage(struct perf_event *event) { if (likely(!atomic_read(&event->mmap_count))) return false; perf_event_update_time(event); - perf_set_shadow_time(event, event->ctx); perf_event_update_userpage(event); return true; @@ -3801,13 +3846,23 @@ ctx_sched_in(struct perf_event_context *ctx, struct task_struct *task) { int is_active = ctx->is_active; - u64 now; lockdep_assert_held(&ctx->lock); if (likely(!ctx->nr_events)) return; + if (is_active ^ EVENT_TIME) { + /* start ctx time */ + __update_context_time(ctx, false); + perf_cgroup_set_timestamp(task, ctx); + /* + * CPU-release for the below ->is_active store, + * see __load_acquire() in perf_event_time_now() + */ + barrier(); + } + ctx->is_active |= (event_type | EVENT_TIME); if (ctx->task) { if (!is_active) @@ -3818,13 +3873,6 @@ ctx_sched_in(struct perf_event_context *ctx, is_active ^= ctx->is_active; /* changed bits */ - if (is_active & EVENT_TIME) { - /* start ctx time */ - now = perf_clock(); - ctx->timestamp = now; - perf_cgroup_set_timestamp(task, ctx); - } - /* * First go through the list and put on any pinned groups * in order to give them the best chance of going on. @@ -4418,6 +4466,18 @@ static inline u64 perf_event_count(struct perf_event *event) return local64_read(&event->count) + atomic64_read(&event->child_count); } +static void calc_timer_values(struct perf_event *event, + u64 *now, + u64 *enabled, + u64 *running) +{ + u64 ctx_time; + + *now = perf_clock(); + ctx_time = perf_event_time_now(event, *now); + __perf_update_times(event, ctx_time, enabled, running); +} + /* * NMI-safe method to read a local event, that is an event that * is: @@ -4477,10 +4537,9 @@ int perf_event_read_local(struct perf_event *event, u64 *value, *value = local64_read(&event->count); if (enabled || running) { - u64 now = event->shadow_ctx_time + perf_clock(); - u64 __enabled, __running; + u64 __enabled, __running, __now;; - __perf_update_times(event, now, &__enabled, &__running); + calc_timer_values(event, &__now, &__enabled, &__running); if (enabled) *enabled = __enabled; if (running) @@ -5802,18 +5861,6 @@ static int perf_event_index(struct perf_event *event) return event->pmu->event_idx(event); } -static void calc_timer_values(struct perf_event *event, - u64 *now, - u64 *enabled, - u64 *running) -{ - u64 ctx_time; - - *now = perf_clock(); - ctx_time = event->shadow_ctx_time + *now; - __perf_update_times(event, ctx_time, enabled, running); -} - static void perf_event_init_userpage(struct perf_event *event) { struct perf_event_mmap_page *userpg; @@ -6353,7 +6400,6 @@ accounting: ring_buffer_attach(event, rb); perf_event_update_time(event); - perf_set_shadow_time(event, event->ctx); perf_event_init_userpage(event); perf_event_update_userpage(event); } else { -- cgit v1.2.3-59-g8ed1b From a06247c6804f1a7c86a2e5398a4c1f1db1471848 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Tue, 11 Jan 2022 15:23:09 -0800 Subject: psi: Fix uaf issue when psi trigger is destroyed while being polled With write operation on psi files replacing old trigger with a new one, the lifetime of its waitqueue is totally arbitrary. Overwriting an existing trigger causes its waitqueue to be freed and pending poll() will stumble on trigger->event_wait which was destroyed. Fix this by disallowing to redefine an existing psi trigger. If a write operation is used on a file descriptor with an already existing psi trigger, the operation will fail with EBUSY error. Also bypass a check for psi_disabled in the psi_trigger_destroy as the flag can be flipped after the trigger is created, leading to a memory leak. Fixes: 0e94682b73bf ("psi: introduce psi monitor") Reported-by: syzbot+cdb5dd11c97cc532efad@syzkaller.appspotmail.com Suggested-by: Linus Torvalds Analyzed-by: Eric Biggers Signed-off-by: Suren Baghdasaryan Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Eric Biggers Acked-by: Johannes Weiner Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20220111232309.1786347-1-surenb@google.com --- Documentation/accounting/psi.rst | 3 +- include/linux/psi.h | 2 +- include/linux/psi_types.h | 3 -- kernel/cgroup/cgroup.c | 11 +++++-- kernel/sched/psi.c | 66 ++++++++++++++++++---------------------- 5 files changed, 40 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/Documentation/accounting/psi.rst b/Documentation/accounting/psi.rst index f2b3439edcc2..860fe651d645 100644 --- a/Documentation/accounting/psi.rst +++ b/Documentation/accounting/psi.rst @@ -92,7 +92,8 @@ Triggers can be set on more than one psi metric and more than one trigger for the same psi metric can be specified. However for each trigger a separate file descriptor is required to be able to poll it separately from others, therefore for each trigger a separate open() syscall should be made even -when opening the same psi interface file. +when opening the same psi interface file. Write operations to a file descriptor +with an already existing psi trigger will fail with EBUSY. Monitors activate only when system enters stall state for the monitored psi metric and deactivates upon exit from the stall state. While system is diff --git a/include/linux/psi.h b/include/linux/psi.h index a70ca833c6d7..f8ce53bfdb2a 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -33,7 +33,7 @@ void cgroup_move_task(struct task_struct *p, struct css_set *to); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, size_t nbytes, enum psi_res res); -void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *t); +void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait); diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index 516c0fe836fd..1a3cef26d129 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -141,9 +141,6 @@ struct psi_trigger { * events to one per window */ u64 last_event_time; - - /* Refcounting to prevent premature destruction */ - struct kref refcount; }; struct psi_group { diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index b31e1465868a..9d05c3ca2d5e 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3643,6 +3643,12 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, cgroup_get(cgrp); cgroup_kn_unlock(of->kn); + /* Allow only one trigger per file descriptor */ + if (ctx->psi.trigger) { + cgroup_put(cgrp); + return -EBUSY; + } + psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; new = psi_trigger_create(psi, buf, nbytes, res); if (IS_ERR(new)) { @@ -3650,8 +3656,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, return PTR_ERR(new); } - psi_trigger_replace(&ctx->psi.trigger, new); - + smp_store_release(&ctx->psi.trigger, new); cgroup_put(cgrp); return nbytes; @@ -3690,7 +3695,7 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) { struct cgroup_file_ctx *ctx = of->priv; - psi_trigger_replace(&ctx->psi.trigger, NULL); + psi_trigger_destroy(ctx->psi.trigger); } bool cgroup_psi_enabled(void) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index a679613a7cb7..c137c4d6983e 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1162,7 +1162,6 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, t->event = 0; t->last_event_time = 0; init_waitqueue_head(&t->event_wait); - kref_init(&t->refcount); mutex_lock(&group->trigger_lock); @@ -1191,15 +1190,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, return t; } -static void psi_trigger_destroy(struct kref *ref) +void psi_trigger_destroy(struct psi_trigger *t) { - struct psi_trigger *t = container_of(ref, struct psi_trigger, refcount); - struct psi_group *group = t->group; + struct psi_group *group; struct task_struct *task_to_destroy = NULL; - if (static_branch_likely(&psi_disabled)) + /* + * We do not check psi_disabled since it might have been disabled after + * the trigger got created. + */ + if (!t) return; + group = t->group; /* * Wakeup waiters to stop polling. Can happen if cgroup is deleted * from under a polling process. @@ -1235,9 +1238,9 @@ static void psi_trigger_destroy(struct kref *ref) mutex_unlock(&group->trigger_lock); /* - * Wait for both *trigger_ptr from psi_trigger_replace and - * poll_task RCUs to complete their read-side critical sections - * before destroying the trigger and optionally the poll_task + * Wait for psi_schedule_poll_work RCU to complete its read-side + * critical section before destroying the trigger and optionally the + * poll_task. */ synchronize_rcu(); /* @@ -1254,18 +1257,6 @@ static void psi_trigger_destroy(struct kref *ref) kfree(t); } -void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *new) -{ - struct psi_trigger *old = *trigger_ptr; - - if (static_branch_likely(&psi_disabled)) - return; - - rcu_assign_pointer(*trigger_ptr, new); - if (old) - kref_put(&old->refcount, psi_trigger_destroy); -} - __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait) { @@ -1275,24 +1266,15 @@ __poll_t psi_trigger_poll(void **trigger_ptr, if (static_branch_likely(&psi_disabled)) return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI; - rcu_read_lock(); - - t = rcu_dereference(*(void __rcu __force **)trigger_ptr); - if (!t) { - rcu_read_unlock(); + t = smp_load_acquire(trigger_ptr); + if (!t) return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI; - } - kref_get(&t->refcount); - - rcu_read_unlock(); poll_wait(file, &t->event_wait, wait); if (cmpxchg(&t->event, 1, 0) == 1) ret |= EPOLLPRI; - kref_put(&t->refcount, psi_trigger_destroy); - return ret; } @@ -1316,14 +1298,24 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, buf[buf_size - 1] = '\0'; - new = psi_trigger_create(&psi_system, buf, nbytes, res); - if (IS_ERR(new)) - return PTR_ERR(new); - seq = file->private_data; + /* Take seq->lock to protect seq->private from concurrent writes */ mutex_lock(&seq->lock); - psi_trigger_replace(&seq->private, new); + + /* Allow only one trigger per file descriptor */ + if (seq->private) { + mutex_unlock(&seq->lock); + return -EBUSY; + } + + new = psi_trigger_create(&psi_system, buf, nbytes, res); + if (IS_ERR(new)) { + mutex_unlock(&seq->lock); + return PTR_ERR(new); + } + + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); return nbytes; @@ -1358,7 +1350,7 @@ static int psi_fop_release(struct inode *inode, struct file *file) { struct seq_file *seq = file->private_data; - psi_trigger_replace(&seq->private, NULL); + psi_trigger_destroy(seq->private); return single_release(inode, file); } -- cgit v1.2.3-59-g8ed1b From 0e3872499de1a1230cef5221607d71aa09264bd5 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Fri, 7 Jan 2022 17:52:54 +0800 Subject: kernel/sched: Remove dl_boosted flag comment since commit 2279f540ea7d ("sched/deadline: Fix priority inheritance with multiple scheduling classes"), we should not keep it here. Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Daniel Bristot de Oliveira Link: https://lore.kernel.org/r/20220107095254.GA49258@localhost.localdomain --- include/linux/sched.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 7a1f16df66e3..7f8b4495e243 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -614,10 +614,6 @@ struct sched_dl_entity { * task has to wait for a replenishment to be performed at the * next firing of dl_timer. * - * @dl_boosted tells if we are boosted due to DI. If so we are - * outside bandwidth enforcement mechanism (but only until we - * exit the critical section); - * * @dl_yielded tells if task gave up the CPU before consuming * all its available runtime during the last job. * -- cgit v1.2.3-59-g8ed1b From be80a1d3f9dbe5aee79a325964f7037fe2d92f30 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Mon, 10 Jan 2022 14:05:49 +0000 Subject: bpf: Generalize check_ctx_reg for reuse with other types Generalize the check_ctx_reg() helper function into a more generic named one so that it can be reused for other register types as well to check whether their offset is non-zero. No functional change. Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Acked-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 4 ++-- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 21 +++++++++++---------- 3 files changed, 14 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 143401d4c9d9..e9993172f892 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -519,8 +519,8 @@ bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, void bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); -int check_ctx_reg(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno); +int check_ptr_off_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, int regno); int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, u32 mem_size); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 33bb8ae4a804..e16dafeb2450 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5686,7 +5686,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, i, btf_type_str(t)); return -EINVAL; } - if (check_ctx_reg(env, reg, regno)) + if (check_ptr_off_reg(env, reg, regno)) return -EINVAL; } else if (is_kfunc && (reg->type == PTR_TO_BTF_ID || reg2btf_ids[reg->type])) { const struct btf_type *reg_ref_t; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a8587210907d..9b8334068e71 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3969,16 +3969,16 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif -int check_ctx_reg(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno) +int check_ptr_off_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, int regno) { - /* Access to ctx or passing it to a helper is only allowed in - * its original, unmodified form. + /* Access to this pointer-typed register or passing it to a helper + * is only allowed in its original, unmodified form. */ if (reg->off) { - verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", - regno, reg->off); + verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", + reg_type_str(env, reg->type), regno, reg->off); return -EACCES; } @@ -3986,7 +3986,8 @@ int check_ctx_reg(struct bpf_verifier_env *env, char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); + verbose(env, "variable %s access var_off=%s disallowed\n", + reg_type_str(env, reg->type), tn_buf); return -EACCES; } @@ -4437,7 +4438,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn return -EACCES; } - err = check_ctx_reg(env, reg, regno); + err = check_ptr_off_reg(env, reg, regno); if (err < 0) return err; @@ -5305,7 +5306,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; if (type == PTR_TO_CTX) { - err = check_ctx_reg(env, reg, regno); + err = check_ptr_off_reg(env, reg, regno); if (err < 0) return err; } @@ -9651,7 +9652,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; } - err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); + err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); if (err < 0) return err; -- cgit v1.2.3-59-g8ed1b From a672b2e36a648afb04ad3bda93b6bda947a479a5 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 13 Jan 2022 11:11:30 +0000 Subject: bpf: Fix ringbuf memory type confusion when passing to helpers The bpf_ringbuf_submit() and bpf_ringbuf_discard() have ARG_PTR_TO_ALLOC_MEM in their bpf_func_proto definition as their first argument, and thus both expect the result from a prior bpf_ringbuf_reserve() call which has a return type of RET_PTR_TO_ALLOC_MEM_OR_NULL. While the non-NULL memory from bpf_ringbuf_reserve() can be passed to other helpers, the two sinks (bpf_ringbuf_submit(), bpf_ringbuf_discard()) right now only enforce a register type of PTR_TO_MEM. This can lead to potential type confusion since it would allow other PTR_TO_MEM memory to be passed into the two sinks which did not come from bpf_ringbuf_reserve(). Add a new MEM_ALLOC composable type attribute for PTR_TO_MEM, and enforce that: - bpf_ringbuf_reserve() returns NULL or PTR_TO_MEM | MEM_ALLOC - bpf_ringbuf_submit() and bpf_ringbuf_discard() only take PTR_TO_MEM | MEM_ALLOC but not plain PTR_TO_MEM arguments via ARG_PTR_TO_ALLOC_MEM - however, other helpers might treat PTR_TO_MEM | MEM_ALLOC as plain PTR_TO_MEM to populate the memory area when they use ARG_PTR_TO_{UNINIT_,}MEM in their func proto description Fixes: 457f44363a88 ("bpf: Implement BPF ring buffer and verifier support for it") Reported-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Acked-by: Alexei Starovoitov --- include/linux/bpf.h | 9 +++++++-- kernel/bpf/verifier.c | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 6e947cd91152..fa517ae604ad 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -316,7 +316,12 @@ enum bpf_type_flag { */ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), - __BPF_TYPE_LAST_FLAG = MEM_RDONLY, + /* MEM was "allocated" from a different helper, and cannot be mixed + * with regular non-MEM_ALLOC'ed MEM types. + */ + MEM_ALLOC = BIT(2 + BPF_BASE_TYPE_BITS), + + __BPF_TYPE_LAST_FLAG = MEM_ALLOC, }; /* Max number of base types. */ @@ -400,7 +405,7 @@ enum bpf_return_type { RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, - RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM, + RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_ALLOC | RET_PTR_TO_ALLOC_MEM, RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, /* This must be the last entry. Its purpose is to ensure the enum is diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c72c57a6684f..a39eedecc93a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -570,6 +570,8 @@ static const char *reg_type_str(struct bpf_verifier_env *env, if (type & MEM_RDONLY) strncpy(prefix, "rdonly_", 16); + if (type & MEM_ALLOC) + strncpy(prefix, "alloc_", 16); snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", prefix, str[base_type(type)], postfix); @@ -5135,6 +5137,7 @@ static const struct bpf_reg_types mem_types = { PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE, PTR_TO_MEM, + PTR_TO_MEM | MEM_ALLOC, PTR_TO_BUF, }, }; @@ -5152,7 +5155,7 @@ static const struct bpf_reg_types int_ptr_types = { static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; -static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; +static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM | MEM_ALLOC } }; static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; @@ -5315,6 +5318,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, case PTR_TO_MAP_VALUE: case PTR_TO_MEM: case PTR_TO_MEM | MEM_RDONLY: + case PTR_TO_MEM | MEM_ALLOC: case PTR_TO_BUF: case PTR_TO_BUF | MEM_RDONLY: case PTR_TO_STACK: -- cgit v1.2.3-59-g8ed1b From 1ca3fb3abd2b615c4b61728de545760a6e2c2d8b Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 19 Jan 2022 18:07:45 -0800 Subject: mm: percpu: add pcpu_fc_cpu_to_node_fn_t typedef Add pcpu_fc_cpu_to_node_fn_t and pass it into pcpu_fc_alloc_fn_t, pcpu first chunk allocation will call it to alloc memblock on the corresponding node by it, this is prepare for the next patch. Link: https://lkml.kernel.org/r/20211216112359.103822-3-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Thomas Bogendoerfer Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter Cc: Albert Ou Cc: Catalin Marinas Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/mips/mm/init.c | 12 +++++++++--- arch/powerpc/kernel/setup_64.c | 15 +++++++++++---- arch/sparc/kernel/smp_64.c | 13 ++++++++++--- arch/x86/kernel/setup_percpu.c | 18 +++++++++++++----- drivers/base/arch_numa.c | 8 +++++--- include/linux/percpu.h | 7 +++++-- mm/percpu.c | 14 +++++++++----- 7 files changed, 62 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 325e1552cbea..1d8f2844704c 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -519,12 +519,17 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return node_distance(cpu_to_node(from), cpu_to_node(to)); } -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, - size_t align) +static int __init pcpu_cpu_to_node(int cpu) +{ + return cpu_to_node(cpu); +} + +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, - cpu_to_node(cpu)); + cpu_to_nd_fn(cpu)); } static void __init pcpu_fc_free(void *ptr, size_t size) @@ -545,6 +550,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free); if (rc < 0) panic("Failed to initialize percpu areas."); diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 6052f5d5ded3..b79b10ae466f 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -784,12 +784,12 @@ void __init emergency_stack_init(void) * RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, - size_t align) +static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { const unsigned long goal = __pa(MAX_DMA_ADDRESS); #ifdef CONFIG_NUMA - int node = early_cpu_to_node(cpu); + int node = cpu_to_nd_fun(cpu); void *ptr; if (!node_online(node) || !NODE_DATA(node)) { @@ -823,6 +823,11 @@ static int pcpu_cpu_distance(unsigned int from, unsigned int to) return REMOTE_DISTANCE; } +static __init int pcpu_cpu_to_node(int cpu) +{ + return early_cpu_to_node(cpu); +} + unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); @@ -891,6 +896,7 @@ void __init setup_per_cpu_areas(void) if (pcpu_chosen_fc != PCPU_FC_PAGE) { rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem); if (rc) pr_warn("PERCPU: %s allocator failed (%d), " @@ -899,7 +905,8 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) - rc = pcpu_page_first_chunk(0, pcpu_alloc_bootmem, pcpu_free_bootmem, + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, + pcpu_alloc_bootmem, pcpu_free_bootmem, pcpu_populate_pte); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index b98a7bbe6728..14d719aa318d 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1539,12 +1539,12 @@ void smp_send_stop(void) * RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, - size_t align) +static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { const unsigned long goal = __pa(MAX_DMA_ADDRESS); #ifdef CONFIG_NUMA - int node = cpu_to_node(cpu); + int node = cpu_to_nd_fn(cpu); void *ptr; if (!node_online(node) || !NODE_DATA(node)) { @@ -1578,6 +1578,11 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return REMOTE_DISTANCE; } +static int __init pcpu_cpu_to_node(int cpu) +{ + return cpu_to_node(cpu); +} + static void __init pcpu_populate_pte(unsigned long addr) { pgd_t *pgd = pgd_offset_k(addr); @@ -1641,6 +1646,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, 4 << 20, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem); if (rc) @@ -1650,6 +1656,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem, pcpu_populate_pte); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 7b65275544b2..1d41f4844149 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -97,12 +97,12 @@ static bool __init pcpu_need_numa(void) * RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, - unsigned long align) +static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, unsigned long align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { const unsigned long goal = __pa(MAX_DMA_ADDRESS); #ifdef CONFIG_NUMA - int node = early_cpu_to_node(cpu); + int node = cpu_to_nd_fn(cpu); void *ptr; if (!node_online(node) || !NODE_DATA(node)) { @@ -128,9 +128,10 @@ static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, /* * Helpers for first chunk memory allocation */ -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align) +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { - return pcpu_alloc_bootmem(cpu, size, align); + return pcpu_alloc_bootmem(cpu, size, align, cpu_to_nd_fn); } static void __init pcpu_fc_free(void *ptr, size_t size) @@ -150,6 +151,11 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) #endif } +static int __init pcpu_cpu_to_node(int cpu) +{ + return early_cpu_to_node(cpu); +} + static void __init pcpup_populate_pte(unsigned long addr) { populate_extra_pte(addr); @@ -205,6 +211,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, dyn_size, atom_size, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free); if (rc < 0) pr_warn("%s allocator failed (%d), falling back to page size\n", @@ -212,6 +219,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free, pcpup_populate_pte); if (rc < 0) diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index bc1876915457..dae861838535 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -155,10 +155,10 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); } -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, - size_t align) +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { - int nid = early_cpu_to_node(cpu); + int nid = cpu_to_nd_fn(cpu); return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); @@ -229,6 +229,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, pcpu_cpu_distance, + early_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free); #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc < 0) @@ -240,6 +241,7 @@ void __init setup_per_cpu_areas(void) #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, + early_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free, pcpu_populate_pte); diff --git a/include/linux/percpu.h b/include/linux/percpu.h index ae4004e7957e..e4078bf45fd5 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -94,8 +94,9 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR]; extern enum pcpu_fc pcpu_chosen_fc; -typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, - size_t align); +typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); +typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); @@ -111,12 +112,14 @@ extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn); #endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK extern int __init pcpu_page_first_chunk(size_t reserved_size, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn); diff --git a/mm/percpu.c b/mm/percpu.c index f5b2c2ea5a54..267a4d295fcf 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3001,6 +3001,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( * @dyn_size: minimum free size for dynamic allocation in bytes * @atom_size: allocation atom size * @cpu_distance_fn: callback to determine distance between cpus, optional + * @cpu_to_nd_fn: callback to convert cpu to it's node, optional * @alloc_fn: function to allocate percpu page * @free_fn: function to free percpu page * @@ -3030,6 +3031,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn) { @@ -3066,7 +3068,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, BUG_ON(cpu == NR_CPUS); /* allocate space for the whole group */ - ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size); + ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); if (!ptr) { rc = -ENOMEM; goto out_free_areas; @@ -3143,6 +3145,7 @@ out_free: /** * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages * @reserved_size: the size of reserved percpu area in bytes + * @cpu_to_nd_fn: callback to convert cpu to it's node, optional * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE * @free_fn: function to free percpu page, always called with PAGE_SIZE * @populate_pte_fn: function to populate pte @@ -3157,6 +3160,7 @@ out_free: * 0 on success, -errno on failure. */ int __init pcpu_page_first_chunk(size_t reserved_size, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn) @@ -3201,7 +3205,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, for (i = 0; i < unit_pages; i++) { void *ptr; - ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE); + ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); if (!ptr) { pr_warn("failed to allocate %s page for cpu%u\n", psize_str, cpu); @@ -3278,8 +3282,8 @@ out_free_ar: unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); -static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, - size_t align) +static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { return memblock_alloc_from(size, align, __pa(MAX_DMA_ADDRESS)); } @@ -3300,7 +3304,7 @@ void __init setup_per_cpu_areas(void) * what the legacy allocator did. */ rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, - PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, NULL, pcpu_dfl_fc_alloc, pcpu_dfl_fc_free); if (rc < 0) panic("Failed to initialize percpu areas."); -- cgit v1.2.3-59-g8ed1b From 23f917169ef157aa7a6bf80d8c4aad6f1282852c Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 19 Jan 2022 18:07:49 -0800 Subject: mm: percpu: add generic pcpu_fc_alloc/free funciton With the previous patch, we could add a generic pcpu first chunk allocate and free function to cleanup the duplicated definations on each architecture. Link: https://lkml.kernel.org/r/20211216112359.103822-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Thomas Bogendoerfer Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Greg Kroah-Hartman Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter Cc: Albert Ou Cc: Catalin Marinas Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: "Rafael J. Wysocki" Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/mips/mm/init.c | 16 +-------- arch/powerpc/kernel/setup_64.c | 51 ++------------------------- arch/sparc/kernel/smp_64.c | 50 +-------------------------- arch/x86/kernel/setup_percpu.c | 59 +------------------------------- drivers/base/arch_numa.c | 19 +--------- include/linux/percpu.h | 9 +---- mm/percpu.c | 78 +++++++++++++++++++++++++----------------- 7 files changed, 54 insertions(+), 228 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 1d8f2844704c..5a8002839550 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -524,19 +524,6 @@ static int __init pcpu_cpu_to_node(int cpu) return cpu_to_node(cpu); } -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), - MEMBLOCK_ALLOC_ACCESSIBLE, - cpu_to_nd_fn(cpu)); -} - -static void __init pcpu_fc_free(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - void __init setup_per_cpu_areas(void) { unsigned long delta; @@ -550,8 +537,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, pcpu_cpu_distance, - pcpu_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_cpu_to_node); if (rc < 0) panic("Failed to initialize percpu areas."); diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index b79b10ae466f..a0c55c6e3023 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -771,50 +771,6 @@ void __init emergency_stack_init(void) } #ifdef CONFIG_SMP -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = cpu_to_nd_fun(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, node); - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " - "%016lx\n", cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -static void __init pcpu_free_bootmem(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - static int pcpu_cpu_distance(unsigned int from, unsigned int to) { if (early_cpu_to_node(from) == early_cpu_to_node(to)) @@ -896,8 +852,7 @@ void __init setup_per_cpu_areas(void) if (pcpu_chosen_fc != PCPU_FC_PAGE) { rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, - pcpu_cpu_to_node, - pcpu_alloc_bootmem, pcpu_free_bootmem); + pcpu_cpu_to_node); if (rc) pr_warn("PERCPU: %s allocator failed (%d), " "falling back to page size\n", @@ -905,9 +860,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) - rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, - pcpu_alloc_bootmem, pcpu_free_bootmem, - pcpu_populate_pte); + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 14d719aa318d..ef815b3f0592 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1526,50 +1526,6 @@ void smp_send_stop(void) smp_call_function(stop_this_cpu, NULL, 0); } -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = cpu_to_nd_fn(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, node); - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " - "%016lx\n", cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -static void __init pcpu_free_bootmem(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) { if (cpu_to_node(from) == cpu_to_node(to)) @@ -1646,9 +1602,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, 4 << 20, pcpu_cpu_distance, - pcpu_cpu_to_node, - pcpu_alloc_bootmem, - pcpu_free_bootmem); + pcpu_cpu_to_node); if (rc) pr_warn("PERCPU: %s allocator failed (%d), " "falling back to page size\n", @@ -1657,8 +1611,6 @@ void __init setup_per_cpu_areas(void) if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node, - pcpu_alloc_bootmem, - pcpu_free_bootmem, pcpu_populate_pte); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 1d41f4844149..15c5bf3cbe5f 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -84,61 +84,6 @@ static bool __init pcpu_need_numa(void) } #endif -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, unsigned long align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NUMA - int node = cpu_to_nd_fn(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = memblock_alloc_try_nid(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, - node); - - pr_debug("per cpu data for cpu%d %lu bytes on node%d at %016lx\n", - cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return memblock_alloc_from(size, align, goal); -#endif -} - -/* - * Helpers for first chunk memory allocation - */ -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - return pcpu_alloc_bootmem(cpu, size, align, cpu_to_nd_fn); -} - -static void __init pcpu_fc_free(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) { #ifdef CONFIG_NUMA @@ -211,8 +156,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, dyn_size, atom_size, pcpu_cpu_distance, - pcpu_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_cpu_to_node); if (rc < 0) pr_warn("%s allocator failed (%d), falling back to page size\n", pcpu_fc_names[pcpu_chosen_fc], rc); @@ -220,7 +164,6 @@ void __init setup_per_cpu_areas(void) if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, pcpu_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free, pcpup_populate_pte); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index dae861838535..23a10cc36165 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -155,20 +155,6 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); } -static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - int nid = cpu_to_nd_fn(cpu); - - return memblock_alloc_try_nid(size, align, - __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); -} - -static void __init pcpu_fc_free(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK static void __init pcpu_populate_pte(unsigned long addr) { @@ -229,8 +215,7 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, pcpu_cpu_distance, - early_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free); + early_cpu_to_node); #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc < 0) pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", @@ -242,8 +227,6 @@ void __init setup_per_cpu_areas(void) if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node, - pcpu_fc_alloc, - pcpu_fc_free, pcpu_populate_pte); #endif if (rc < 0) diff --git a/include/linux/percpu.h b/include/linux/percpu.h index e4078bf45fd5..d73c97ef4ff4 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -95,9 +95,6 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR]; extern enum pcpu_fc pcpu_chosen_fc; typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); -typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); -typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); @@ -112,16 +109,12 @@ extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn); + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); #endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK extern int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn); #endif diff --git a/mm/percpu.c b/mm/percpu.c index 267a4d295fcf..0f79b6d9a6d6 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2992,6 +2992,42 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( return ai; } + +static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) +{ + const unsigned long goal = __pa(MAX_DMA_ADDRESS); +#ifdef CONFIG_NUMA + int node = NUMA_NO_NODE; + void *ptr; + + if (cpu_to_nd_fn) + node = cpu_to_nd_fn(cpu); + + if (node == NUMA_NO_NODE || !node_online(node) || !NODE_DATA(node)) { + ptr = memblock_alloc_from(size, align, goal); + pr_info("cpu %d has no node %d or node-local memory\n", + cpu, node); + pr_debug("per cpu data for cpu%d %zu bytes at 0x%llx\n", + cpu, size, (u64)__pa(ptr)); + } else { + ptr = memblock_alloc_try_nid(size, align, goal, + MEMBLOCK_ALLOC_ACCESSIBLE, + node); + + pr_debug("per cpu data for cpu%d %zu bytes on node%d at 0x%llx\n", + cpu, size, node, (u64)__pa(ptr)); + } + return ptr; +#else + return memblock_alloc_from(size, align, goal); +#endif +} + +static void __init pcpu_fc_free(void *ptr, size_t size) +{ + memblock_free(ptr, size); +} #endif /* BUILD_EMBED_FIRST_CHUNK || BUILD_PAGE_FIRST_CHUNK */ #if defined(BUILD_EMBED_FIRST_CHUNK) @@ -3002,14 +3038,12 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( * @atom_size: allocation atom size * @cpu_distance_fn: callback to determine distance between cpus, optional * @cpu_to_nd_fn: callback to convert cpu to it's node, optional - * @alloc_fn: function to allocate percpu page - * @free_fn: function to free percpu page * * This is a helper to ease setting up embedded first percpu chunk and * can be called where pcpu_setup_first_chunk() is expected. * * If this function is used to setup the first chunk, it is allocated - * by calling @alloc_fn and used as-is without being mapped into + * by calling pcpu_fc_alloc and used as-is without being mapped into * vmalloc area. Allocations are always whole multiples of @atom_size * aligned to @atom_size. * @@ -3023,7 +3057,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( * @dyn_size specifies the minimum dynamic area size. * * If the needed size is smaller than the minimum or specified unit - * size, the leftover is returned using @free_fn. + * size, the leftover is returned using pcpu_fc_free. * * RETURNS: * 0 on success, -errno on failure. @@ -3031,9 +3065,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn) + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { void *base = (void *)ULONG_MAX; void **areas = NULL; @@ -3068,7 +3100,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, BUG_ON(cpu == NR_CPUS); /* allocate space for the whole group */ - ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); + ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); if (!ptr) { rc = -ENOMEM; goto out_free_areas; @@ -3107,12 +3139,12 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) { if (gi->cpu_map[i] == NR_CPUS) { /* unused unit, free whole */ - free_fn(ptr, ai->unit_size); + pcpu_fc_free(ptr, ai->unit_size); continue; } /* copy and return the unused part */ memcpy(ptr, __per_cpu_load, ai->static_size); - free_fn(ptr + size_sum, ai->unit_size - size_sum); + pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum); } } @@ -3131,7 +3163,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, out_free_areas: for (group = 0; group < ai->nr_groups; group++) if (areas[group]) - free_fn(areas[group], + pcpu_fc_free(areas[group], ai->groups[group].nr_units * ai->unit_size); out_free: pcpu_free_alloc_info(ai); @@ -3146,8 +3178,6 @@ out_free: * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages * @reserved_size: the size of reserved percpu area in bytes * @cpu_to_nd_fn: callback to convert cpu to it's node, optional - * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE - * @free_fn: function to free percpu page, always called with PAGE_SIZE * @populate_pte_fn: function to populate pte * * This is a helper to ease setting up page-remapped first percpu @@ -3161,8 +3191,6 @@ out_free: */ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn) { static struct vm_struct vm; @@ -3205,7 +3233,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, for (i = 0; i < unit_pages; i++) { void *ptr; - ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); + ptr = pcpu_fc_alloc(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); if (!ptr) { pr_warn("failed to allocate %s page for cpu%u\n", psize_str, cpu); @@ -3257,7 +3285,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, enomem: while (--j >= 0) - free_fn(page_address(pages[j]), PAGE_SIZE); + pcpu_fc_free(page_address(pages[j]), PAGE_SIZE); rc = -ENOMEM; out_free_ar: memblock_free(pages, pages_size); @@ -3282,17 +3310,6 @@ out_free_ar: unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); -static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ - return memblock_alloc_from(size, align, __pa(MAX_DMA_ADDRESS)); -} - -static void __init pcpu_dfl_fc_free(void *ptr, size_t size) -{ - memblock_free(ptr, size); -} - void __init setup_per_cpu_areas(void) { unsigned long delta; @@ -3303,9 +3320,8 @@ void __init setup_per_cpu_areas(void) * Always reserve area for module percpu variables. That's * what the legacy allocator did. */ - rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, - PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, NULL, - pcpu_dfl_fc_alloc, pcpu_dfl_fc_free); + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, + PAGE_SIZE, NULL, NULL); if (rc < 0) panic("Failed to initialize percpu areas."); -- cgit v1.2.3-59-g8ed1b From 20c035764626c56c4f6514936b9ee4be0f4cd962 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 19 Jan 2022 18:07:53 -0800 Subject: mm: percpu: add generic pcpu_populate_pte() function With NEED_PER_CPU_PAGE_FIRST_CHUNK enabled, we need a function to populate pte, this patch adds a generic pcpu populate pte function, pcpu_populate_pte(), which is marked __weak and used on most architectures, but it is overridden on x86, which has its own implementation. Link: https://lkml.kernel.org/r/20211216112359.103822-5-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: "David S. Miller" Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter Cc: Albert Ou Cc: Catalin Marinas Cc: Palmer Dabbelt Cc: Paul Walmsley Cc: Thomas Bogendoerfer Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/powerpc/kernel/setup_64.c | 47 +------------------------- arch/sparc/kernel/smp_64.c | 56 +------------------------------ arch/x86/kernel/setup_percpu.c | 5 ++- drivers/base/arch_numa.c | 51 +--------------------------- include/linux/percpu.h | 5 ++- mm/percpu.c | 76 +++++++++++++++++++++++++++++++++++++++--- 6 files changed, 78 insertions(+), 162 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index a0c55c6e3023..f7cf408217c5 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -787,51 +787,6 @@ static __init int pcpu_cpu_to_node(int cpu) unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); -static void __init pcpu_populate_pte(unsigned long addr) -{ - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud_t *new; - - new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); - if (!new) - goto err_alloc; - p4d_populate(&init_mm, p4d, new); - } - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); - if (!new) - goto err_alloc; - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); - if (!new) - goto err_alloc; - pmd_populate_kernel(&init_mm, pmd, new); - } - - return; - -err_alloc: - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); -} - - void __init setup_per_cpu_areas(void) { const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; @@ -860,7 +815,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) - rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte); + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index ef815b3f0592..a1f78e9ddaf3 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1539,59 +1539,6 @@ static int __init pcpu_cpu_to_node(int cpu) return cpu_to_node(cpu); } -static void __init pcpu_populate_pte(unsigned long addr) -{ - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - if (pgd_none(*pgd)) { - pud_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pgd_populate(&init_mm, pgd, new); - } - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - p4d_populate(&init_mm, p4d, new); - } - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pmd_populate_kernel(&init_mm, pmd, new); - } - - return; - -err_alloc: - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); -} - void __init setup_per_cpu_areas(void) { unsigned long delta; @@ -1610,8 +1557,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, - pcpu_cpu_to_node, - pcpu_populate_pte); + pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 15c5bf3cbe5f..49325caa7307 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -101,7 +101,7 @@ static int __init pcpu_cpu_to_node(int cpu) return early_cpu_to_node(cpu); } -static void __init pcpup_populate_pte(unsigned long addr) +void __init pcpu_populate_pte(unsigned long addr) { populate_extra_pte(addr); } @@ -163,8 +163,7 @@ void __init setup_per_cpu_areas(void) } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, - pcpu_cpu_to_node, - pcpup_populate_pte); + pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index 23a10cc36165..eaa31e567d1e 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -14,7 +14,6 @@ #include #include -#include struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); @@ -155,52 +154,6 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); } -#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK -static void __init pcpu_populate_pte(unsigned long addr) -{ - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { - pud_t *new; - - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - p4d_populate(&init_mm, p4d, new); - } - - pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); - if (!new) - goto err_alloc; - pmd_populate_kernel(&init_mm, pmd, new); - } - - return; - -err_alloc: - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); -} -#endif - void __init setup_per_cpu_areas(void) { unsigned long delta; @@ -225,9 +178,7 @@ void __init setup_per_cpu_areas(void) #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc < 0) - rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, - early_cpu_to_node, - pcpu_populate_pte); + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); #endif if (rc < 0) panic("Failed to initialize percpu areas (err=%d).", rc); diff --git a/include/linux/percpu.h b/include/linux/percpu.h index d73c97ef4ff4..f1ec5ad1351c 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -95,7 +95,6 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR]; extern enum pcpu_fc pcpu_chosen_fc; typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); -typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, @@ -113,9 +112,9 @@ extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, #endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK +void __init pcpu_populate_pte(unsigned long addr); extern int __init pcpu_page_first_chunk(size_t reserved_size, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_populate_pte_fn_t populate_pte_fn); + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); #endif extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align) __alloc_size(1); diff --git a/mm/percpu.c b/mm/percpu.c index 0f79b6d9a6d6..fc6f591cb54f 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3174,11 +3174,79 @@ out_free: #endif /* BUILD_EMBED_FIRST_CHUNK */ #ifdef BUILD_PAGE_FIRST_CHUNK +#include + +#ifndef P4D_TABLE_SIZE +#define P4D_TABLE_SIZE PAGE_SIZE +#endif + +#ifndef PUD_TABLE_SIZE +#define PUD_TABLE_SIZE PAGE_SIZE +#endif + +#ifndef PMD_TABLE_SIZE +#define PMD_TABLE_SIZE PAGE_SIZE +#endif + +#ifndef PTE_TABLE_SIZE +#define PTE_TABLE_SIZE PAGE_SIZE +#endif +void __init __weak pcpu_populate_pte(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd)) { + p4d_t *new; + + new = memblock_alloc(P4D_TABLE_SIZE, P4D_TABLE_SIZE); + if (!new) + goto err_alloc; + pgd_populate(&init_mm, pgd, new); + } + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + pud_t *new; + + new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); + if (!new) + goto err_alloc; + p4d_populate(&init_mm, p4d, new); + } + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd_t *new; + + new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); + if (!new) + goto err_alloc; + pud_populate(&init_mm, pud, new); + } + + pmd = pmd_offset(pud, addr); + if (!pmd_present(*pmd)) { + pte_t *new; + + new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); + if (!new) + goto err_alloc; + pmd_populate_kernel(&init_mm, pmd, new); + } + + return; + +err_alloc: + panic("%s: Failed to allocate memory\n", __func__); +} + /** * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages * @reserved_size: the size of reserved percpu area in bytes * @cpu_to_nd_fn: callback to convert cpu to it's node, optional - * @populate_pte_fn: function to populate pte * * This is a helper to ease setting up page-remapped first percpu * chunk and can be called where pcpu_setup_first_chunk() is expected. @@ -3189,9 +3257,7 @@ out_free: * RETURNS: * 0 on success, -errno on failure. */ -int __init pcpu_page_first_chunk(size_t reserved_size, - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, - pcpu_fc_populate_pte_fn_t populate_pte_fn) +int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { static struct vm_struct vm; struct pcpu_alloc_info *ai; @@ -3255,7 +3321,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, (unsigned long)vm.addr + unit * ai->unit_size; for (i = 0; i < unit_pages; i++) - populate_pte_fn(unit_addr + (i << PAGE_SHIFT)); + pcpu_populate_pte(unit_addr + (i << PAGE_SHIFT)); /* pte already populated, the following shouldn't fail */ rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], -- cgit v1.2.3-59-g8ed1b From ae62fbe299629d3b2fa61d4cf5146258c4d99fdf Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Wed, 19 Jan 2022 18:08:00 -0800 Subject: proc: make the proc_create[_data]() stubs static inlines Change the proc_create[_data]() stubs which are used when CONFIG_PROC_FS is not set from #defines to a static inline stubs. This should fix clang -Werror builds failing due to errors like this: drivers/platform/x86/thinkpad_acpi.c:918:30: error: unused variable 'dispatch_proc_ops' [-Werror,-Wunused-const-variable] Fixing this in include/linux/proc_fs.h should ensure that the same issue is also fixed in any other drivers hitting the same -Werror issue. [akpm@linux-foundation.org: fix CONFIG_PROC_FS=n] [akpm@linux-foundation.org: fix arch/sparc/kernel/led.c] [akpm@linux-foundation.org: fix build] Link: https://lkml.kernel.org/r/20211116131112.508304-1-hdegoede@redhat.com Signed-off-by: Hans de Goede Reported-by: kernel test robot Acked-by: Christian Brauner Cc: Alexander Viro Cc: Hans de Goede Cc: David Howells Cc: Christoph Hellwig Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sparc/kernel/led.c | 8 +++----- include/linux/proc_fs.h | 12 ++++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index 3a66e62eb2a0..ab657b359789 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -114,18 +114,16 @@ static const struct proc_ops led_proc_ops = { }; #endif -static struct proc_dir_entry *led; - #define LED_VERSION "0.1" static int __init led_init(void) { timer_setup(&led_blink_timer, led_blink, 0); - led = proc_create("led", 0, NULL, &led_proc_ops); - if (!led) +#ifdef CONFIG_PROC_FS + if (!proc_create("led", 0, NULL, &led_proc_ops)) return -ENOMEM; - +#endif printk(KERN_INFO "led: version %s, Lars Kotthoff \n", LED_VERSION); diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 069c7fd95396..01b9268451a8 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -178,8 +178,16 @@ static inline struct proc_dir_entry *proc_mkdir_mode(const char *name, #define proc_create_seq(name, mode, parent, ops) ({NULL;}) #define proc_create_single(name, mode, parent, show) ({NULL;}) #define proc_create_single_data(name, mode, parent, show, data) ({NULL;}) -#define proc_create(name, mode, parent, proc_ops) ({NULL;}) -#define proc_create_data(name, mode, parent, proc_ops, data) ({NULL;}) + +static inline struct proc_dir_entry * +proc_create(const char *name, umode_t mode, struct proc_dir_entry *parent, + const struct proc_ops *proc_ops) +{ return NULL; } + +static inline struct proc_dir_entry * +proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, + const struct proc_ops *proc_ops, void *data) +{ return NULL; } static inline void proc_set_size(struct proc_dir_entry *de, loff_t size) {} static inline void proc_set_user(struct proc_dir_entry *de, kuid_t uid, kgid_t gid) {} -- cgit v1.2.3-59-g8ed1b From 22c033989c3eb9731ad0c497dfab4231b8e367d6 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 19 Jan 2022 18:08:12 -0800 Subject: include/linux/unaligned: replace kernel.h with the necessary inclusions When kernel.h is used in the headers it adds a lot into dependency hell, especially when there are circular dependencies are involved. Replace kernel.h inclusion with the list of what is really being used. The rest of the changes are induced by the above and may not be split. Link: https://lkml.kernel.org/r/20211209123823.20425-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Acked-by: Arend van Spriel [brcmfmac] Acked-by: Kalle Valo Cc: Arend van Spriel Cc: Franky Lin Cc: Hante Meuleman Cc: Chi-hsien Lin Cc: Wright Feng Cc: Chung-hsien Hsu Cc: Kalle Valo Cc: David S. Miller Cc: Jakub Kicinski Cc: Heikki Krogerus Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/xtlv.c | 2 ++ include/linux/unaligned/packed_struct.h | 2 +- lib/lz4/lz4defs.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/xtlv.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/xtlv.c index 2f3c451148db..2f8908074303 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/xtlv.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/xtlv.c @@ -4,6 +4,8 @@ */ #include + +#include #include #include diff --git a/include/linux/unaligned/packed_struct.h b/include/linux/unaligned/packed_struct.h index c0d817de4df2..f4c8eaf4d012 100644 --- a/include/linux/unaligned/packed_struct.h +++ b/include/linux/unaligned/packed_struct.h @@ -1,7 +1,7 @@ #ifndef _LINUX_UNALIGNED_PACKED_STRUCT_H #define _LINUX_UNALIGNED_PACKED_STRUCT_H -#include +#include struct __una_u16 { u16 x; } __packed; struct __una_u32 { u32 x; } __packed; diff --git a/lib/lz4/lz4defs.h b/lib/lz4/lz4defs.h index 673bd206aa98..330aa539b46e 100644 --- a/lib/lz4/lz4defs.h +++ b/lib/lz4/lz4defs.h @@ -36,6 +36,8 @@ */ #include + +#include #include /* memset, memcpy */ #define FORCE_INLINE __always_inline -- cgit v1.2.3-59-g8ed1b From 40cbf09f060c8febef64541c463d4dd526abe445 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 19 Jan 2022 18:08:16 -0800 Subject: kernel.h: include a note to discourage people from including it in headers Include a note at the top to discourage people from including it in headers. Link: https://lkml.kernel.org/r/20211209150803.4473-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kernel.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 77755ac3e189..36a612d82956 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -1,4 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ +/* + * NOTE: + * + * This header has combined a lot of unrelated to each other stuff. + * The process of splitting its content is in progress while keeping + * backward compatibility. That's why it's highly recommended NOT to + * include this header inside another header file, especially under + * generic or architectural include/ directory. + */ #ifndef _LINUX_KERNEL_H #define _LINUX_KERNEL_H -- cgit v1.2.3-59-g8ed1b From 95af469c4f609de011debc08e7a35b45201623a8 Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Wed, 19 Jan 2022 18:08:29 -0800 Subject: fs/binfmt_elf: replace open-coded string copy with get_task_comm It is better to use get_task_comm() instead of the open coded string copy as we do in other places. struct elf_prpsinfo is used to dump the task information in userspace coredump or kernel vmcore. Below is the verification of vmcore, crash> ps PID PPID CPU TASK ST %MEM VSZ RSS COMM 0 0 0 ffffffff9d21a940 RU 0.0 0 0 [swapper/0] > 0 0 1 ffffa09e40f85e80 RU 0.0 0 0 [swapper/1] > 0 0 2 ffffa09e40f81f80 RU 0.0 0 0 [swapper/2] > 0 0 3 ffffa09e40f83f00 RU 0.0 0 0 [swapper/3] > 0 0 4 ffffa09e40f80000 RU 0.0 0 0 [swapper/4] > 0 0 5 ffffa09e40f89f80 RU 0.0 0 0 [swapper/5] 0 0 6 ffffa09e40f8bf00 RU 0.0 0 0 [swapper/6] > 0 0 7 ffffa09e40f88000 RU 0.0 0 0 [swapper/7] > 0 0 8 ffffa09e40f8de80 RU 0.0 0 0 [swapper/8] > 0 0 9 ffffa09e40f95e80 RU 0.0 0 0 [swapper/9] > 0 0 10 ffffa09e40f91f80 RU 0.0 0 0 [swapper/10] > 0 0 11 ffffa09e40f93f00 RU 0.0 0 0 [swapper/11] > 0 0 12 ffffa09e40f90000 RU 0.0 0 0 [swapper/12] > 0 0 13 ffffa09e40f9bf00 RU 0.0 0 0 [swapper/13] > 0 0 14 ffffa09e40f98000 RU 0.0 0 0 [swapper/14] > 0 0 15 ffffa09e40f9de80 RU 0.0 0 0 [swapper/15] It works well as expected. Some comments are added to explain why we use the hard-coded 16. Link: https://lkml.kernel.org/r/20211120112738.45980-5-laoar.shao@gmail.com Suggested-by: Kees Cook Signed-off-by: Yafang Shao Reviewed-by: David Hildenbrand Cc: Mathieu Desnoyers Cc: Arnaldo Carvalho de Melo Cc: Andrii Nakryiko Cc: Michal Miroslaw Cc: Peter Zijlstra Cc: Steven Rostedt Cc: Matthew Wilcox Cc: David Hildenbrand Cc: Al Viro Cc: Kees Cook Cc: Petr Mladek Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Dennis Dalessandro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/binfmt_elf.c | 2 +- include/linux/elfcore-compat.h | 5 +++++ include/linux/elfcore.h | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index f8c7f26f1fbb..b9a33cc34d6b 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1585,7 +1585,7 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, SET_UID(psinfo->pr_uid, from_kuid_munged(cred->user_ns, cred->uid)); SET_GID(psinfo->pr_gid, from_kgid_munged(cred->user_ns, cred->gid)); rcu_read_unlock(); - strncpy(psinfo->pr_fname, p->comm, sizeof(psinfo->pr_fname)); + get_task_comm(psinfo->pr_fname, p); return 0; } diff --git a/include/linux/elfcore-compat.h b/include/linux/elfcore-compat.h index e272c3d452ce..54feb64e9b5d 100644 --- a/include/linux/elfcore-compat.h +++ b/include/linux/elfcore-compat.h @@ -43,6 +43,11 @@ struct compat_elf_prpsinfo __compat_uid_t pr_uid; __compat_gid_t pr_gid; compat_pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; + /* + * The hard-coded 16 is derived from TASK_COMM_LEN, but it can't be + * changed as it is exposed to userspace. We'd better make it hard-coded + * here. + */ char pr_fname[16]; char pr_psargs[ELF_PRARGSZ]; }; diff --git a/include/linux/elfcore.h b/include/linux/elfcore.h index 957ebec35aad..746e081879a5 100644 --- a/include/linux/elfcore.h +++ b/include/linux/elfcore.h @@ -65,6 +65,11 @@ struct elf_prpsinfo __kernel_gid_t pr_gid; pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; /* Lots missing */ + /* + * The hard-coded 16 is derived from TASK_COMM_LEN, but it can't be + * changed as it is exposed to userspace. We'd better make it hard-coded + * here. + */ char pr_fname[16]; /* filename of executable */ char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ }; -- cgit v1.2.3-59-g8ed1b From 3087c61ed2c48548b74dd343a5209b87082c682d Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Wed, 19 Jan 2022 18:08:40 -0800 Subject: tools/testing/selftests/bpf: replace open-coded 16 with TASK_COMM_LEN As the sched:sched_switch tracepoint args are derived from the kernel, we'd better make it same with the kernel. So the macro TASK_COMM_LEN is converted to type enum, then all the BPF programs can get it through BTF. The BPF program which wants to use TASK_COMM_LEN should include the header vmlinux.h. Regarding the test_stacktrace_map and test_tracepoint, as the type defined in linux/bpf.h are also defined in vmlinux.h, so we don't need to include linux/bpf.h again. Link: https://lkml.kernel.org/r/20211120112738.45980-8-laoar.shao@gmail.com Signed-off-by: Yafang Shao Acked-by: Andrii Nakryiko Acked-by: David Hildenbrand Cc: Mathieu Desnoyers Cc: Arnaldo Carvalho de Melo Cc: Andrii Nakryiko Cc: Michal Miroslaw Cc: Peter Zijlstra Cc: Steven Rostedt Cc: Matthew Wilcox Cc: David Hildenbrand Cc: Al Viro Cc: Kees Cook Cc: Petr Mladek Cc: Alexei Starovoitov Cc: Dennis Dalessandro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/sched.h | 9 +++++++-- tools/testing/selftests/bpf/progs/test_stacktrace_map.c | 6 +++--- tools/testing/selftests/bpf/progs/test_tracepoint.c | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 78c351e35fec..cecd4806edc6 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -274,8 +274,13 @@ struct task_group; #define get_current_state() READ_ONCE(current->__state) -/* Task command name length: */ -#define TASK_COMM_LEN 16 +/* + * Define the task command name length as enum, then it can be visible to + * BPF programs. + */ +enum { + TASK_COMM_LEN = 16, +}; extern void scheduler_tick(void); diff --git a/tools/testing/selftests/bpf/progs/test_stacktrace_map.c b/tools/testing/selftests/bpf/progs/test_stacktrace_map.c index a8233e7f173b..728dbd39eff0 100644 --- a/tools/testing/selftests/bpf/progs/test_stacktrace_map.c +++ b/tools/testing/selftests/bpf/progs/test_stacktrace_map.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2018 Facebook -#include +#include #include #ifndef PERF_MAX_STACK_DEPTH @@ -41,11 +41,11 @@ struct { /* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */ struct sched_switch_args { unsigned long long pad; - char prev_comm[16]; + char prev_comm[TASK_COMM_LEN]; int prev_pid; int prev_prio; long long prev_state; - char next_comm[16]; + char next_comm[TASK_COMM_LEN]; int next_pid; int next_prio; }; diff --git a/tools/testing/selftests/bpf/progs/test_tracepoint.c b/tools/testing/selftests/bpf/progs/test_tracepoint.c index ce6974016f53..43bd7a20cc50 100644 --- a/tools/testing/selftests/bpf/progs/test_tracepoint.c +++ b/tools/testing/selftests/bpf/progs/test_tracepoint.c @@ -1,17 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2017 Facebook -#include +#include #include /* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */ struct sched_switch_args { unsigned long long pad; - char prev_comm[16]; + char prev_comm[TASK_COMM_LEN]; int prev_pid; int prev_prio; long long prev_state; - char next_comm[16]; + char next_comm[TASK_COMM_LEN]; int next_pid; int next_prio; }; -- cgit v1.2.3-59-g8ed1b From d6986ce24fc00b0638bd29efe8fb7ba7619ed2aa Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Wed, 19 Jan 2022 18:08:43 -0800 Subject: kthread: dynamically allocate memory to store kthread's full name When I was implementing a new per-cpu kthread cfs_migration, I found the comm of it "cfs_migration/%u" is truncated due to the limitation of TASK_COMM_LEN. For example, the comm of the percpu thread on CPU10~19 all have the same name "cfs_migration/1", which will confuse the user. This issue is not critical, because we can get the corresponding CPU from the task's Cpus_allowed. But for kthreads corresponding to other hardware devices, it is not easy to get the detailed device info from task comm, for example, jbd2/nvme0n1p2- xfs-reclaim/sdf Currently there are so many truncated kthreads: rcu_tasks_kthre rcu_tasks_rude_ rcu_tasks_trace poll_mpt3sas0_s ext4-rsv-conver xfs-reclaim/sd{a, b, c, ...} xfs-blockgc/sd{a, b, c, ...} xfs-inodegc/sd{a, b, c, ...} audit_send_repl ecryptfs-kthrea vfio-irqfd-clea jbd2/nvme0n1p2- ... We can shorten these names to work around this problem, but it may be not applied to all of the truncated kthreads. Take 'jbd2/nvme0n1p2-' for example, it is a nice name, and it is not a good idea to shorten it. One possible way to fix this issue is extending the task comm size, but as task->comm is used in lots of places, that may cause some potential buffer overflows. Another more conservative approach is introducing a new pointer to store kthread's full name if it is truncated, which won't introduce too much overhead as it is in the non-critical path. Finally we make a dicision to use the second approach. See also the discussions in this thread: https://lore.kernel.org/lkml/20211101060419.4682-1-laoar.shao@gmail.com/ After this change, the full name of these truncated kthreads will be displayed via /proc/[pid]/comm: rcu_tasks_kthread rcu_tasks_rude_kthread rcu_tasks_trace_kthread poll_mpt3sas0_statu ext4-rsv-conversion xfs-reclaim/sdf1 xfs-blockgc/sdf1 xfs-inodegc/sdf1 audit_send_reply ecryptfs-kthread vfio-irqfd-cleanup jbd2/nvme0n1p2-8 Link: https://lkml.kernel.org/r/20211120112850.46047-1-laoar.shao@gmail.com Signed-off-by: Yafang Shao Reviewed-by: David Hildenbrand Reviewed-by: Petr Mladek Suggested-by: Petr Mladek Suggested-by: Steven Rostedt Cc: Mathieu Desnoyers Cc: Arnaldo Carvalho de Melo Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Michal Miroslaw Cc: Peter Zijlstra Cc: Steven Rostedt Cc: Matthew Wilcox Cc: Al Viro Cc: Kees Cook Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/array.c | 3 +++ include/linux/kthread.h | 1 + kernel/kthread.c | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/array.c b/fs/proc/array.c index ff869a66b34e..4321aa63835d 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -92,6 +92,7 @@ #include #include #include +#include #include #include "internal.h" @@ -102,6 +103,8 @@ void proc_task_name(struct seq_file *m, struct task_struct *p, bool escape) if (p->flags & PF_WQ_WORKER) wq_worker_comm(tcomm, sizeof(tcomm), p); + else if (p->flags & PF_KTHREAD) + get_kthread_comm(tcomm, sizeof(tcomm), p); else __get_task_comm(tcomm, sizeof(tcomm), p); diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 346b0f269161..2a5c04494663 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -33,6 +33,7 @@ struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), unsigned int cpu, const char *namefmt); +void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk); void set_kthread_struct(struct task_struct *p); void kthread_set_per_cpu(struct task_struct *k, int cpu); diff --git a/kernel/kthread.c b/kernel/kthread.c index 7113003fab63..a70cd5dc94e3 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -60,6 +60,8 @@ struct kthread { #ifdef CONFIG_BLK_CGROUP struct cgroup_subsys_state *blkcg_css; #endif + /* To store the full name if task comm is truncated. */ + char *full_name; }; enum KTHREAD_BITS { @@ -93,6 +95,18 @@ static inline struct kthread *__to_kthread(struct task_struct *p) return kthread; } +void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk) +{ + struct kthread *kthread = to_kthread(tsk); + + if (!kthread || !kthread->full_name) { + __get_task_comm(buf, buf_size, tsk); + return; + } + + strscpy_pad(buf, kthread->full_name, buf_size); +} + void set_kthread_struct(struct task_struct *p) { struct kthread *kthread; @@ -118,9 +132,13 @@ void free_kthread_struct(struct task_struct *k) * or if kmalloc() in kthread() failed. */ kthread = to_kthread(k); + if (!kthread) + return; + #ifdef CONFIG_BLK_CGROUP - WARN_ON_ONCE(kthread && kthread->blkcg_css); + WARN_ON_ONCE(kthread->blkcg_css); #endif + kfree(kthread->full_name); kfree(kthread); } @@ -406,12 +424,22 @@ struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data), task = create->result; if (!IS_ERR(task)) { char name[TASK_COMM_LEN]; + va_list aq; + int len; /* * task is already visible to other tasks, so updating * COMM must be protected. */ - vsnprintf(name, sizeof(name), namefmt, args); + va_copy(aq, args); + len = vsnprintf(name, sizeof(name), namefmt, aq); + va_end(aq); + if (len >= TASK_COMM_LEN) { + struct kthread *kthread = to_kthread(task); + + /* leave it truncated when out of memory. */ + kthread->full_name = kvasprintf(GFP_KERNEL, namefmt, args); + } set_task_comm(task, name); } kfree(create); -- cgit v1.2.3-59-g8ed1b From 0425473037db40d9e322631f2d4dc6ef51f97e88 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 19 Jan 2022 18:08:56 -0800 Subject: list: introduce list_is_head() helper and re-use it in list.h Introduce list_is_head() in the similar (*) way as it's done for list_entry_is_head(). Make use of it in the list.h. *) it's done as inliner and not a macro to be aligned with other list_is_*() APIs; while at it, make all three to have the same style. Link: https://lkml.kernel.org/r/20211201141824.81400-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Heikki Krogerus Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/list.h | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/list.h b/include/linux/list.h index 6636fc07f918..dd6c2041d09c 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -258,8 +258,7 @@ static inline void list_bulk_move_tail(struct list_head *head, * @list: the entry to test * @head: the head of the list */ -static inline int list_is_first(const struct list_head *list, - const struct list_head *head) +static inline int list_is_first(const struct list_head *list, const struct list_head *head) { return list->prev == head; } @@ -269,12 +268,21 @@ static inline int list_is_first(const struct list_head *list, * @list: the entry to test * @head: the head of the list */ -static inline int list_is_last(const struct list_head *list, - const struct list_head *head) +static inline int list_is_last(const struct list_head *list, const struct list_head *head) { return list->next == head; } +/** + * list_is_head - tests whether @list is the list @head + * @list: the entry to test + * @head: the head of the list + */ +static inline int list_is_head(const struct list_head *list, const struct list_head *head) +{ + return list == head; +} + /** * list_empty - tests whether a list is empty * @head: the list to test. @@ -318,7 +326,7 @@ static inline void list_del_init_careful(struct list_head *entry) static inline int list_empty_careful(const struct list_head *head) { struct list_head *next = smp_load_acquire(&head->next); - return (next == head) && (next == head->prev); + return list_is_head(next, head) && (next == head->prev); } /** @@ -393,10 +401,9 @@ static inline void list_cut_position(struct list_head *list, { if (list_empty(head)) return; - if (list_is_singular(head) && - (head->next != entry && head != entry)) + if (list_is_singular(head) && !list_is_head(entry, head) && (entry != head->next)) return; - if (entry == head) + if (list_is_head(entry, head)) INIT_LIST_HEAD(list); else __list_cut_position(list, head, entry); @@ -570,7 +577,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @head: the head for your list. */ #define list_for_each(pos, head) \ - for (pos = (head)->next; pos != (head); pos = pos->next) + for (pos = (head)->next; !list_is_head(pos, (head)); pos = pos->next) /** * list_for_each_continue - continue iteration over a list @@ -580,7 +587,7 @@ static inline void list_splice_tail_init(struct list_head *list, * Continue to iterate over a list, continuing after the current position. */ #define list_for_each_continue(pos, head) \ - for (pos = pos->next; pos != (head); pos = pos->next) + for (pos = pos->next; !list_is_head(pos, (head)); pos = pos->next) /** * list_for_each_prev - iterate over a list backwards @@ -588,7 +595,7 @@ static inline void list_splice_tail_init(struct list_head *list, * @head: the head for your list. */ #define list_for_each_prev(pos, head) \ - for (pos = (head)->prev; pos != (head); pos = pos->prev) + for (pos = (head)->prev; !list_is_head(pos, (head)); pos = pos->prev) /** * list_for_each_safe - iterate over a list safe against removal of list entry @@ -597,8 +604,9 @@ static inline void list_splice_tail_init(struct list_head *list, * @head: the head for your list. */ #define list_for_each_safe(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = n, n = pos->next) + for (pos = (head)->next, n = pos->next; \ + !list_is_head(pos, (head)); \ + pos = n, n = pos->next) /** * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry @@ -608,7 +616,7 @@ static inline void list_splice_tail_init(struct list_head *list, */ #define list_for_each_prev_safe(pos, n, head) \ for (pos = (head)->prev, n = pos->prev; \ - pos != (head); \ + !list_is_head(pos, (head)); \ pos = n, n = pos->prev) /** -- cgit v1.2.3-59-g8ed1b From fd0a1462405b087377e59b84e119fe7e2d08499a Mon Sep 17 00:00:00 2001 From: Isabella Basso Date: Wed, 19 Jan 2022 18:09:02 -0800 Subject: hash.h: remove unused define directive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "test_hash.c: refactor into KUnit", v3. We refactored the lib/test_hash.c file into KUnit as part of the student group LKCAMP [1] introductory hackathon for kernel development. This test was pointed to our group by Daniel Latypov [2], so its full conversion into a pure KUnit test was our goal in this patch series, but we ran into many problems relating to it not being split as unit tests, which complicated matters a bit, as the reasoning behind the original tests is quite cryptic for those unfamiliar with hash implementations. Some interesting developments we'd like to highlight are: - In patch 1/5 we noticed that there was an unused define directive that could be removed. - In patch 4/5 we noticed how stringhash and hash tests are all under the lib/test_hash.c file, which might cause some confusion, and we also broke those kernel config entries up. Overall KUnit developments have been made in the other patches in this series: In patches 2/5, 3/5 and 5/5 we refactored the lib/test_hash.c file so as to make it more compatible with the KUnit style, whilst preserving the original idea of the maintainer who designed it (i.e. George Spelvin), which might be undesirable for unit tests, but we assume it is enough for a first patch. This patch (of 5): Currently, there exist hash_32() and __hash_32() functions, which were introduced in a patch [1] targeting architecture specific optimizations. These functions can be overridden on a per-architecture basis to achieve such optimizations. They must set their corresponding define directive (HAVE_ARCH_HASH_32 and HAVE_ARCH__HASH_32, respectively) so that header files can deal with these overrides properly. As the supported 32-bit architectures that have their own hash function implementation (i.e. m68k, Microblaze, H8/300, pa-risc) have only been making use of the (more general) __hash_32() function (which only lacks a right shift operation when compared to the hash_32() function), remove the define directive corresponding to the arch-specific hash_32() implementation. [1] https://lore.kernel.org/lkml/20160525073311.5600.qmail@ns.sciencehorizons.net/ [akpm@linux-foundation.org: hash_32_generic() becomes hash_32()] Link: https://lkml.kernel.org/r/20211208183711.390454-1-isabbasso@riseup.net Link: https://lkml.kernel.org/r/20211208183711.390454-2-isabbasso@riseup.net Reviewed-by: David Gow Tested-by: David Gow Co-developed-by: Augusto Durães Camargo Signed-off-by: Augusto Durães Camargo Co-developed-by: Enzo Ferreira Signed-off-by: Enzo Ferreira Signed-off-by: Isabella Basso Cc: Geert Uytterhoeven Cc: Brendan Higgins Cc: Daniel Latypov Cc: Shuah Khan Cc: Rodrigo Siqueira Cc: kernel test robot Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/infiniband/sw/rxe/rxe_qp.c | 3 +-- include/linux/hash.h | 5 +---- lib/test_hash.c | 24 +----------------------- tools/include/linux/hash.h | 5 +---- 4 files changed, 4 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 54b8711321c1..44c9ea601bff 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -217,8 +217,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, * the port number must be in the Dynamic Ports range * (0xc000 - 0xffff). */ - qp->src_port = RXE_ROCE_V2_SPORT + - (hash_32_generic(qp_num(qp), 14) & 0x3fff); + qp->src_port = RXE_ROCE_V2_SPORT + (hash_32(qp_num(qp), 14) & 0x3fff); qp->sq.max_wr = init->cap.max_send_wr; /* These caps are limited by rxe_qp_chk_cap() done by the caller */ diff --git a/include/linux/hash.h b/include/linux/hash.h index ad6fa21d977b..38edaa08f862 100644 --- a/include/linux/hash.h +++ b/include/linux/hash.h @@ -62,10 +62,7 @@ static inline u32 __hash_32_generic(u32 val) return val * GOLDEN_RATIO_32; } -#ifndef HAVE_ARCH_HASH_32 -#define hash_32 hash_32_generic -#endif -static inline u32 hash_32_generic(u32 val, unsigned int bits) +static inline u32 hash_32(u32 val, unsigned int bits) { /* High bits are more random, so use them. */ return __hash_32(val) >> (32 - bits); diff --git a/lib/test_hash.c b/lib/test_hash.c index 0ee40b4a56dd..d4b0cfdb0377 100644 --- a/lib/test_hash.c +++ b/lib/test_hash.c @@ -94,22 +94,7 @@ test_int_hash(unsigned long long h64, u32 hash_or[2][33]) pr_err("hash_32(%#x, %d) = %#x > %#x", h0, k, h1, m); return false; } -#ifdef HAVE_ARCH_HASH_32 - h2 = hash_32_generic(h0, k); -#if HAVE_ARCH_HASH_32 == 1 - if (h1 != h2) { - pr_err("hash_32(%#x, %d) = %#x != hash_32_generic() " - " = %#x", h0, k, h1, h2); - return false; - } -#else - if (h2 > m) { - pr_err("hash_32_generic(%#x, %d) = %#x > %#x", - h0, k, h1, m); - return false; - } -#endif -#endif + /* Test hash_64 */ hash_or[1][k] |= h1 = hash_64(h64, k); if (h1 > m) { @@ -227,13 +212,6 @@ test_hash_init(void) #else pr_info("__hash_32() has no arch implementation to test."); #endif -#ifdef HAVE_ARCH_HASH_32 -#if HAVE_ARCH_HASH_32 != 1 - pr_info("hash_32() is arch-specific; not compared to generic."); -#endif -#else - pr_info("hash_32() has no arch implementation to test."); -#endif #ifdef HAVE_ARCH_HASH_64 #if HAVE_ARCH_HASH_64 != 1 pr_info("hash_64() is arch-specific; not compared to generic."); diff --git a/tools/include/linux/hash.h b/tools/include/linux/hash.h index ad6fa21d977b..38edaa08f862 100644 --- a/tools/include/linux/hash.h +++ b/tools/include/linux/hash.h @@ -62,10 +62,7 @@ static inline u32 __hash_32_generic(u32 val) return val * GOLDEN_RATIO_32; } -#ifndef HAVE_ARCH_HASH_32 -#define hash_32 hash_32_generic -#endif -static inline u32 hash_32_generic(u32 val, unsigned int bits) +static inline u32 hash_32(u32 val, unsigned int bits) { /* High bits are more random, so use them. */ return __hash_32(val) >> (32 - bits); -- cgit v1.2.3-59-g8ed1b From a3d5dc908a5f572ce3e31fe83fd2459a1c3c5422 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 19 Jan 2022 18:10:02 -0800 Subject: delayacct: support swapin delay accounting for swapping without blkio Currently delayacct accounts swapin delay only for swapping that cause blkio. If we use zram for swapping, tools/accounting/getdelays can't get any SWAP delay. It's useful to get zram swapin delay information, for example to adjust compress algorithm or /proc/sys/vm/swappiness. Reference to PSI, it accounts any kind of swapping by doing its work in swap_readpage(), no matter whether swapping causes blkio. Let delayacct do the similar work. Link: https://lkml.kernel.org/r/20211112083813.8559-1-yang.yang29@zte.com.cn Signed-off-by: Yang Yang Reported-by: Zeal Robot Cc: Balbir Singh Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/delayacct.h | 44 ++++++++++++++++++++++---------------------- kernel/delayacct.c | 33 ++++++++++++++++++--------------- mm/memory.c | 4 ---- mm/page_io.c | 3 +++ 4 files changed, 43 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index af7e6eb50283..b96d68f310a2 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -9,14 +9,6 @@ #include -/* - * Per-task flags relevant to delay accounting - * maintained privately to avoid exhausting similar flags in sched.h:PF_* - * Used to set current->delays->flags - */ -#define DELAYACCT_PF_SWAPIN 0x00000001 /* I am doing a swapin */ -#define DELAYACCT_PF_BLKIO 0x00000002 /* I am waiting on IO */ - #ifdef CONFIG_TASK_DELAY_ACCT struct task_delay_info { raw_spinlock_t lock; @@ -37,13 +29,13 @@ struct task_delay_info { * associated with the operation is added to XXX_delay. * XXX_delay contains the accumulated delay time in nanoseconds. */ - u64 blkio_start; /* Shared by blkio, swapin */ + u64 blkio_start; u64 blkio_delay; /* wait for sync block io completion */ - u64 swapin_delay; /* wait for swapin block io completion */ + u64 swapin_start; + u64 swapin_delay; /* wait for swapin */ u32 blkio_count; /* total count of the number of sync block */ /* io operations performed */ - u32 swapin_count; /* total count of the number of swapin block */ - /* io operations performed */ + u32 swapin_count; /* total count of swapin */ u64 freepages_start; u64 freepages_delay; /* wait for memory reclaim */ @@ -79,14 +71,8 @@ extern void __delayacct_freepages_start(void); extern void __delayacct_freepages_end(void); extern void __delayacct_thrashing_start(void); extern void __delayacct_thrashing_end(void); - -static inline int delayacct_is_task_waiting_on_io(struct task_struct *p) -{ - if (p->delays) - return (p->delays->flags & DELAYACCT_PF_BLKIO); - else - return 0; -} +extern void __delayacct_swapin_start(void); +extern void __delayacct_swapin_end(void); static inline void delayacct_set_flag(struct task_struct *p, int flag) { @@ -123,7 +109,6 @@ static inline void delayacct_blkio_start(void) if (!static_branch_unlikely(&delayacct_key)) return; - delayacct_set_flag(current, DELAYACCT_PF_BLKIO); if (current->delays) __delayacct_blkio_start(); } @@ -135,7 +120,6 @@ static inline void delayacct_blkio_end(struct task_struct *p) if (p->delays) __delayacct_blkio_end(p); - delayacct_clear_flag(p, DELAYACCT_PF_BLKIO); } static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk) @@ -169,6 +153,18 @@ static inline void delayacct_thrashing_end(void) __delayacct_thrashing_end(); } +static inline void delayacct_swapin_start(void) +{ + if (current->delays) + __delayacct_swapin_start(); +} + +static inline void delayacct_swapin_end(void) +{ + if (current->delays) + __delayacct_swapin_end(); +} + #else static inline void delayacct_set_flag(struct task_struct *p, int flag) {} @@ -199,6 +195,10 @@ static inline void delayacct_thrashing_start(void) {} static inline void delayacct_thrashing_end(void) {} +static inline void delayacct_swapin_start(void) +{} +static inline void delayacct_swapin_end(void) +{} #endif /* CONFIG_TASK_DELAY_ACCT */ diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 51530d5b15a8..97699848c1f0 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -100,19 +100,10 @@ void __delayacct_blkio_start(void) */ void __delayacct_blkio_end(struct task_struct *p) { - struct task_delay_info *delays = p->delays; - u64 *total; - u32 *count; - - if (p->delays->flags & DELAYACCT_PF_SWAPIN) { - total = &delays->swapin_delay; - count = &delays->swapin_count; - } else { - total = &delays->blkio_delay; - count = &delays->blkio_count; - } - - delayacct_end(&delays->lock, &delays->blkio_start, total, count); + delayacct_end(&p->delays->lock, + &p->delays->blkio_start, + &p->delays->blkio_delay, + &p->delays->blkio_count); } int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) @@ -179,8 +170,7 @@ __u64 __delayacct_blkio_ticks(struct task_struct *tsk) unsigned long flags; raw_spin_lock_irqsave(&tsk->delays->lock, flags); - ret = nsec_to_clock_t(tsk->delays->blkio_delay + - tsk->delays->swapin_delay); + ret = nsec_to_clock_t(tsk->delays->blkio_delay); raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); return ret; } @@ -210,3 +200,16 @@ void __delayacct_thrashing_end(void) ¤t->delays->thrashing_delay, ¤t->delays->thrashing_count); } + +void __delayacct_swapin_start(void) +{ + current->delays->swapin_start = local_clock(); +} + +void __delayacct_swapin_end(void) +{ + delayacct_end(¤t->delays->lock, + ¤t->delays->swapin_start, + ¤t->delays->swapin_delay, + ¤t->delays->swapin_count); +} diff --git a/mm/memory.c b/mm/memory.c index 8f1de811a1dc..ced3274c3deb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3507,7 +3507,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (unlikely(!si)) goto out; - delayacct_set_flag(current, DELAYACCT_PF_SWAPIN); page = lookup_swap_cache(entry, vma, vmf->address); swapcache = page; @@ -3555,7 +3554,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vmf->address, &vmf->ptl); if (likely(pte_same(*vmf->pte, vmf->orig_pte))) ret = VM_FAULT_OOM; - delayacct_clear_flag(current, DELAYACCT_PF_SWAPIN); goto unlock; } @@ -3569,13 +3567,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * owner processes (which may be unknown at hwpoison time) */ ret = VM_FAULT_HWPOISON; - delayacct_clear_flag(current, DELAYACCT_PF_SWAPIN); goto out_release; } locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags); - delayacct_clear_flag(current, DELAYACCT_PF_SWAPIN); if (!locked) { ret |= VM_FAULT_RETRY; goto out_release; diff --git a/mm/page_io.c b/mm/page_io.c index 9725c7e1eeea..0bf8e40f4e57 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include void end_swap_bio_write(struct bio *bio) { @@ -370,6 +371,7 @@ int swap_readpage(struct page *page, bool synchronous) * significant part of overall IO time. */ psi_memstall_enter(&pflags); + delayacct_swapin_start(); if (frontswap_load(page) == 0) { SetPageUptodate(page); @@ -432,6 +434,7 @@ int swap_readpage(struct page *page, bool synchronous) out: psi_memstall_leave(&pflags); + delayacct_swapin_end(); return ret; } -- cgit v1.2.3-59-g8ed1b From 82065b7266899fbdce4c7394d7dd02688161f0cf Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 19 Jan 2022 18:10:06 -0800 Subject: delayacct: fix incomplete disable operation when switch enable to disable When a task is created after delayacct is enabled, kernel will do all the delay accountings for that task. The problems is if user disables delayacct by set /proc/sys/kernel/task_delayacct to zero, only blkio delay accounting is disabled. Now disable all the kinds of delay accountings when /proc/sys/kernel/task_delayacct sets to zero. Link: https://lkml.kernel.org/r/20211123140342.32962-1-ran.xiaokai@zte.com.cn Signed-off-by: Yang Yang Reported-by: Zeal Robot Cc: Balbir Singh Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/delayacct.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index b96d68f310a2..c675cfb6437e 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -131,36 +131,54 @@ static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk) static inline void delayacct_freepages_start(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_freepages_start(); } static inline void delayacct_freepages_end(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_freepages_end(); } static inline void delayacct_thrashing_start(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_thrashing_start(); } static inline void delayacct_thrashing_end(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_thrashing_end(); } static inline void delayacct_swapin_start(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_swapin_start(); } static inline void delayacct_swapin_end(void) { + if (!static_branch_unlikely(&delayacct_key)) + return; + if (current->delays) __delayacct_swapin_end(); } -- cgit v1.2.3-59-g8ed1b From 1193829da1a6728249cd02577a020bd64fd9c160 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 19 Jan 2022 18:10:09 -0800 Subject: delayacct: cleanup flags in struct task_delay_info and functions use it Flags in struct task_delay_info is used to distinguish the difference between swapin and blkio delay acountings. But after patch "delayacct: support swapin delay accounting for swapping without blkio", there is no need to do that since swapin and blkio delay accounting use their own functions. Link: https://lkml.kernel.org/r/20211124065958.36703-1-yang.yang29@zte.com.cn Signed-off-by: Yang Yang Cc: Balbir Singh Cc: Ingo Molnar Cc: Johannes Weiner Cc: Peter Zijlstra Cc: Zeal Robot Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/delayacct.h | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index c675cfb6437e..435c3654a0ff 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -12,7 +12,6 @@ #ifdef CONFIG_TASK_DELAY_ACCT struct task_delay_info { raw_spinlock_t lock; - unsigned int flags; /* Private per-task flags */ /* For each stat XXX, add following, aligned appropriately * @@ -74,18 +73,6 @@ extern void __delayacct_thrashing_end(void); extern void __delayacct_swapin_start(void); extern void __delayacct_swapin_end(void); -static inline void delayacct_set_flag(struct task_struct *p, int flag) -{ - if (p->delays) - p->delays->flags |= flag; -} - -static inline void delayacct_clear_flag(struct task_struct *p, int flag) -{ - if (p->delays) - p->delays->flags &= ~flag; -} - static inline void delayacct_tsk_init(struct task_struct *tsk) { /* reinitialize in case parent's non-null pointer was dup'ed*/ @@ -184,10 +171,6 @@ static inline void delayacct_swapin_end(void) } #else -static inline void delayacct_set_flag(struct task_struct *p, int flag) -{} -static inline void delayacct_clear_flag(struct task_struct *p, int flag) -{} static inline void delayacct_init(void) {} static inline void delayacct_tsk_init(struct task_struct *tsk) -- cgit v1.2.3-59-g8ed1b From 5bf18281534451bf1ad56a45a3085cd7ad46860d Mon Sep 17 00:00:00 2001 From: wangyong Date: Wed, 19 Jan 2022 18:10:15 -0800 Subject: delayacct: track delays from memory compact Delay accounting does not track the delay of memory compact. When there is not enough free memory, tasks can spend a amount of their time waiting for compact. To get the impact of tasks in direct memory compact, measure the delay when allocating memory through memory compact. Also update tools/accounting/getdelays.c: / # ./getdelays_next -di -p 304 print delayacct stats ON printing IO accounting PID 304 CPU count real total virtual total delay total delay average 277 780000000 849039485 18877296 0.068ms IO count delay total delay average 0 0 0ms SWAP count delay total delay average 0 0 0ms RECLAIM count delay total delay average 5 11088812685 2217ms THRASHING count delay total delay average 0 0 0ms COMPACT count delay total delay average 3 72758 0ms watch: read=0, write=0, cancelled_write=0 Link: https://lkml.kernel.org/r/1638619795-71451-1-git-send-email-wang.yong12@zte.com.cn Signed-off-by: wangyong Reviewed-by: Jiang Xuexin Reviewed-by: Zhang Wenya Reviewed-by: Yang Yang Reviewed-by: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/delayacct.h | 28 ++++++++++++++++++++++++++++ include/uapi/linux/taskstats.h | 6 +++++- kernel/delayacct.c | 16 ++++++++++++++++ mm/page_alloc.c | 3 +++ tools/accounting/getdelays.c | 8 +++++++- 5 files changed, 59 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index 435c3654a0ff..3e03d010bd2e 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -42,8 +42,12 @@ struct task_delay_info { u64 thrashing_start; u64 thrashing_delay; /* wait for thrashing page */ + u64 compact_start; + u64 compact_delay; /* wait for memory compact */ + u32 freepages_count; /* total count of memory reclaim */ u32 thrashing_count; /* total count of thrash waits */ + u32 compact_count; /* total count of memory compact */ }; #endif @@ -72,6 +76,8 @@ extern void __delayacct_thrashing_start(void); extern void __delayacct_thrashing_end(void); extern void __delayacct_swapin_start(void); extern void __delayacct_swapin_end(void); +extern void __delayacct_compact_start(void); +extern void __delayacct_compact_end(void); static inline void delayacct_tsk_init(struct task_struct *tsk) { @@ -170,6 +176,24 @@ static inline void delayacct_swapin_end(void) __delayacct_swapin_end(); } +static inline void delayacct_compact_start(void) +{ + if (!static_branch_unlikely(&delayacct_key)) + return; + + if (current->delays) + __delayacct_compact_start(); +} + +static inline void delayacct_compact_end(void) +{ + if (!static_branch_unlikely(&delayacct_key)) + return; + + if (current->delays) + __delayacct_compact_end(); +} + #else static inline void delayacct_init(void) {} @@ -200,6 +224,10 @@ static inline void delayacct_swapin_start(void) {} static inline void delayacct_swapin_end(void) {} +static inline void delayacct_compact_start(void) +{} +static inline void delayacct_compact_end(void) +{} #endif /* CONFIG_TASK_DELAY_ACCT */ diff --git a/include/uapi/linux/taskstats.h b/include/uapi/linux/taskstats.h index ccbd08709321..12327d32378f 100644 --- a/include/uapi/linux/taskstats.h +++ b/include/uapi/linux/taskstats.h @@ -34,7 +34,7 @@ */ -#define TASKSTATS_VERSION 10 +#define TASKSTATS_VERSION 11 #define TS_COMM_LEN 32 /* should be >= TASK_COMM_LEN * in linux/sched.h */ @@ -172,6 +172,10 @@ struct taskstats { /* v10: 64-bit btime to avoid overflow */ __u64 ac_btime64; /* 64-bit begin time */ + + /* Delay waiting for memory compact */ + __u64 compact_count; + __u64 compact_delay_total; }; diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 97699848c1f0..c5e8cea9e05f 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -155,10 +155,13 @@ int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp; tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay; d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp; + tmp = d->compact_delay_total + tsk->delays->compact_delay; + d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp; d->blkio_count += tsk->delays->blkio_count; d->swapin_count += tsk->delays->swapin_count; d->freepages_count += tsk->delays->freepages_count; d->thrashing_count += tsk->delays->thrashing_count; + d->compact_count += tsk->delays->compact_count; raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); return 0; @@ -213,3 +216,16 @@ void __delayacct_swapin_end(void) ¤t->delays->swapin_delay, ¤t->delays->swapin_count); } + +void __delayacct_compact_start(void) +{ + current->delays->compact_start = local_clock(); +} + +void __delayacct_compact_end(void) +{ + delayacct_end(¤t->delays->lock, + ¤t->delays->compact_start, + ¤t->delays->compact_delay, + ¤t->delays->compact_count); +} diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c5952749ad40..635063f49671 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -72,6 +72,7 @@ #include #include #include +#include #include #include #include @@ -4348,6 +4349,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, return NULL; psi_memstall_enter(&pflags); + delayacct_compact_start(); noreclaim_flag = memalloc_noreclaim_save(); *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, @@ -4355,6 +4357,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, memalloc_noreclaim_restore(noreclaim_flag); psi_memstall_leave(&pflags); + delayacct_compact_end(); if (*compact_result == COMPACT_SKIPPED) return NULL; diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 5ef1c15e88ad..11e86739456d 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -205,6 +205,8 @@ static void print_delayacct(struct taskstats *t) "RECLAIM %12s%15s%15s\n" " %15llu%15llu%15llums\n" "THRASHING%12s%15s%15s\n" + " %15llu%15llu%15llums\n" + "COMPACT %12s%15s%15s\n" " %15llu%15llu%15llums\n", "count", "real total", "virtual total", "delay total", "delay average", @@ -228,7 +230,11 @@ static void print_delayacct(struct taskstats *t) "count", "delay total", "delay average", (unsigned long long)t->thrashing_count, (unsigned long long)t->thrashing_delay_total, - average_ms(t->thrashing_delay_total, t->thrashing_count)); + average_ms(t->thrashing_delay_total, t->thrashing_count), + "count", "delay total", "delay average", + (unsigned long long)t->compact_count, + (unsigned long long)t->compact_delay_total, + average_ms(t->compact_delay_total, t->compact_count)); } static void task_context_switch_counts(struct taskstats *t) -- cgit v1.2.3-59-g8ed1b From 47934e06b65637c88a762d9c98329ae6e3238888 Mon Sep 17 00:00:00 2001 From: Congyu Liu Date: Tue, 18 Jan 2022 14:20:13 -0500 Subject: net: fix information leakage in /proc/net/ptype In one net namespace, after creating a packet socket without binding it to a device, users in other net namespaces can observe the new `packet_type` added by this packet socket by reading `/proc/net/ptype` file. This is minor information leakage as packet socket is namespace aware. Add a net pointer in `packet_type` to keep the net namespace of of corresponding packet socket. In `ptype_seq_show`, this net pointer must be checked when it is not NULL. Fixes: 2feb27dbe00c ("[NETNS]: Minor information leak via /proc/net/ptype file.") Signed-off-by: Congyu Liu Signed-off-by: David S. Miller --- include/linux/netdevice.h | 1 + net/core/net-procfs.c | 3 ++- net/packet/af_packet.c | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3213c7227b59..e490b84732d1 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2548,6 +2548,7 @@ struct packet_type { struct net_device *); bool (*id_match)(struct packet_type *ptype, struct sock *sk); + struct net *af_packet_net; void *af_packet_priv; struct list_head list; }; diff --git a/net/core/net-procfs.c b/net/core/net-procfs.c index d8b9dbabd4a4..5b8016335aca 100644 --- a/net/core/net-procfs.c +++ b/net/core/net-procfs.c @@ -260,7 +260,8 @@ static int ptype_seq_show(struct seq_file *seq, void *v) if (v == SEQ_START_TOKEN) seq_puts(seq, "Type Device Function\n"); - else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) { + else if ((!pt->af_packet_net || net_eq(pt->af_packet_net, seq_file_net(seq))) && + (!pt->dev || net_eq(dev_net(pt->dev), seq_file_net(seq)))) { if (pt->type == htons(ETH_P_ALL)) seq_puts(seq, "ALL "); else diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 5bd409ab4cc2..85ea7ddb48db 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1774,6 +1774,7 @@ static int fanout_add(struct sock *sk, struct fanout_args *args) match->prot_hook.dev = po->prot_hook.dev; match->prot_hook.func = packet_rcv_fanout; match->prot_hook.af_packet_priv = match; + match->prot_hook.af_packet_net = read_pnet(&match->net); match->prot_hook.id_match = match_fanout_group; match->max_num_members = args->max_num_members; list_add(&match->list, &fanout_list); @@ -3353,6 +3354,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, po->prot_hook.func = packet_rcv_spkt; po->prot_hook.af_packet_priv = sk; + po->prot_hook.af_packet_net = sock_net(sk); if (proto) { po->prot_hook.type = proto; -- cgit v1.2.3-59-g8ed1b From e2f08207c558bc0bc8abaa557cdb29bad776ac7b Mon Sep 17 00:00:00 2001 From: Moshe Tal Date: Thu, 20 Jan 2022 11:55:50 +0200 Subject: ethtool: Fix link extended state for big endian The link extended sub-states are assigned as enum that is an integer size but read from a union as u8, this is working for small values on little endian systems but for big endian this always give 0. Fix the variable in the union to match the enum size. Fixes: ecc31c60240b ("ethtool: Add link extended state") Signed-off-by: Moshe Tal Reviewed-by: Ido Schimmel Tested-by: Ido Schimmel Reviewed-by: Gal Pressman Reviewed-by: Amit Cohen Signed-off-by: David S. Miller --- include/linux/ethtool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index a26f37a27167..11efc45de66a 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -111,7 +111,7 @@ struct ethtool_link_ext_state_info { enum ethtool_link_ext_substate_bad_signal_integrity bad_signal_integrity; enum ethtool_link_ext_substate_cable_issue cable_issue; enum ethtool_link_ext_substate_module module; - u8 __link_ext_substate; + u32 __link_ext_substate; }; }; -- cgit v1.2.3-59-g8ed1b From 66a8f7f04979f4ad739085f01d99c8caf620b4f5 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 18 Jan 2022 18:35:02 +0100 Subject: of: base: make small of_parse_phandle() variants static inline Make all the smaller variants of the of_parse_phandle() static inline. This also let us remove the empty function stubs if CONFIG_OF is not defined. Suggested-by: Rob Herring Signed-off-by: Michael Walle [robh: move index < 0 check into __of_parse_phandle_with_args] Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220118173504.2867523-2-michael@walle.cc --- drivers/of/base.c | 131 ++++------------------------------------------- include/linux/of.h | 148 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 129 insertions(+), 150 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/base.c b/drivers/of/base.c index 8a24d37153b4..e7d92b67cb8a 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1420,15 +1420,18 @@ int of_phandle_iterator_args(struct of_phandle_iterator *it, return count; } -static int __of_parse_phandle_with_args(const struct device_node *np, - const char *list_name, - const char *cells_name, - int cell_count, int index, - struct of_phandle_args *out_args) +int __of_parse_phandle_with_args(const struct device_node *np, + const char *list_name, + const char *cells_name, + int cell_count, int index, + struct of_phandle_args *out_args) { struct of_phandle_iterator it; int rc, cur_index = 0; + if (index < 0) + return -EINVAL; + /* Loop over the phandles until all the requested entry is found */ of_for_each_phandle(&it, rc, np, list_name, cells_name, cell_count) { /* @@ -1471,82 +1474,7 @@ static int __of_parse_phandle_with_args(const struct device_node *np, of_node_put(it.node); return rc; } - -/** - * of_parse_phandle - Resolve a phandle property to a device_node pointer - * @np: Pointer to device node holding phandle property - * @phandle_name: Name of property holding a phandle value - * @index: For properties holding a table of phandles, this is the index into - * the table - * - * Return: The device_node pointer with refcount incremented. Use - * of_node_put() on it when done. - */ -struct device_node *of_parse_phandle(const struct device_node *np, - const char *phandle_name, int index) -{ - struct of_phandle_args args; - - if (index < 0) - return NULL; - - if (__of_parse_phandle_with_args(np, phandle_name, NULL, 0, - index, &args)) - return NULL; - - return args.np; -} -EXPORT_SYMBOL(of_parse_phandle); - -/** - * of_parse_phandle_with_args() - Find a node pointed by phandle in a list - * @np: pointer to a device tree node containing a list - * @list_name: property name that contains a list - * @cells_name: property name that specifies phandles' arguments count - * @index: index of a phandle to parse out - * @out_args: optional pointer to output arguments structure (will be filled) - * - * This function is useful to parse lists of phandles and their arguments. - * Returns 0 on success and fills out_args, on error returns appropriate - * errno value. - * - * Caller is responsible to call of_node_put() on the returned out_args->np - * pointer. - * - * Example:: - * - * phandle1: node1 { - * #list-cells = <2>; - * }; - * - * phandle2: node2 { - * #list-cells = <1>; - * }; - * - * node3 { - * list = <&phandle1 1 2 &phandle2 3>; - * }; - * - * To get a device_node of the ``node2`` node you may call this: - * of_parse_phandle_with_args(node3, "list", "#list-cells", 1, &args); - */ -int of_parse_phandle_with_args(const struct device_node *np, const char *list_name, - const char *cells_name, int index, - struct of_phandle_args *out_args) -{ - int cell_count = -1; - - if (index < 0) - return -EINVAL; - - /* If cells_name is NULL we assume a cell count of 0 */ - if (!cells_name) - cell_count = 0; - - return __of_parse_phandle_with_args(np, list_name, cells_name, - cell_count, index, out_args); -} -EXPORT_SYMBOL(of_parse_phandle_with_args); +EXPORT_SYMBOL(__of_parse_phandle_with_args); /** * of_parse_phandle_with_args_map() - Find a node pointed by phandle in a list and remap it @@ -1732,47 +1660,6 @@ free: } EXPORT_SYMBOL(of_parse_phandle_with_args_map); -/** - * of_parse_phandle_with_fixed_args() - Find a node pointed by phandle in a list - * @np: pointer to a device tree node containing a list - * @list_name: property name that contains a list - * @cell_count: number of argument cells following the phandle - * @index: index of a phandle to parse out - * @out_args: optional pointer to output arguments structure (will be filled) - * - * This function is useful to parse lists of phandles and their arguments. - * Returns 0 on success and fills out_args, on error returns appropriate - * errno value. - * - * Caller is responsible to call of_node_put() on the returned out_args->np - * pointer. - * - * Example:: - * - * phandle1: node1 { - * }; - * - * phandle2: node2 { - * }; - * - * node3 { - * list = <&phandle1 0 2 &phandle2 2 3>; - * }; - * - * To get a device_node of the ``node2`` node you may call this: - * of_parse_phandle_with_fixed_args(node3, "list", 2, 1, &args); - */ -int of_parse_phandle_with_fixed_args(const struct device_node *np, - const char *list_name, int cell_count, - int index, struct of_phandle_args *out_args) -{ - if (index < 0) - return -EINVAL; - return __of_parse_phandle_with_args(np, list_name, NULL, cell_count, - index, out_args); -} -EXPORT_SYMBOL(of_parse_phandle_with_fixed_args); - /** * of_count_phandle_with_args() - Find the number of phandles references in a property * @np: pointer to a device tree node containing a list diff --git a/include/linux/of.h b/include/linux/of.h index ff143a027abc..16d76c92fbe0 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -364,18 +364,12 @@ extern const struct of_device_id *of_match_node( const struct of_device_id *matches, const struct device_node *node); extern int of_modalias_node(struct device_node *node, char *modalias, int len); extern void of_print_phandle_args(const char *msg, const struct of_phandle_args *args); -extern struct device_node *of_parse_phandle(const struct device_node *np, - const char *phandle_name, - int index); -extern int of_parse_phandle_with_args(const struct device_node *np, - const char *list_name, const char *cells_name, int index, - struct of_phandle_args *out_args); +extern int __of_parse_phandle_with_args(const struct device_node *np, + const char *list_name, const char *cells_name, int cell_count, + int index, struct of_phandle_args *out_args); extern int of_parse_phandle_with_args_map(const struct device_node *np, const char *list_name, const char *stem_name, int index, struct of_phandle_args *out_args); -extern int of_parse_phandle_with_fixed_args(const struct device_node *np, - const char *list_name, int cells_count, int index, - struct of_phandle_args *out_args); extern int of_count_phandle_with_args(const struct device_node *np, const char *list_name, const char *cells_name); @@ -865,18 +859,12 @@ static inline int of_property_read_string_helper(const struct device_node *np, return -ENOSYS; } -static inline struct device_node *of_parse_phandle(const struct device_node *np, - const char *phandle_name, - int index) -{ - return NULL; -} - -static inline int of_parse_phandle_with_args(const struct device_node *np, - const char *list_name, - const char *cells_name, - int index, - struct of_phandle_args *out_args) +static inline int __of_parse_phandle_with_args(const struct device_node *np, + const char *list_name, + const char *cells_name, + int cell_count, + int index, + struct of_phandle_args *out_args) { return -ENOSYS; } @@ -890,13 +878,6 @@ static inline int of_parse_phandle_with_args_map(const struct device_node *np, return -ENOSYS; } -static inline int of_parse_phandle_with_fixed_args(const struct device_node *np, - const char *list_name, int cells_count, int index, - struct of_phandle_args *out_args) -{ - return -ENOSYS; -} - static inline int of_count_phandle_with_args(const struct device_node *np, const char *list_name, const char *cells_name) @@ -1077,6 +1058,117 @@ static inline bool of_node_is_type(const struct device_node *np, const char *typ return np && match && type && !strcmp(match, type); } +/** + * of_parse_phandle - Resolve a phandle property to a device_node pointer + * @np: Pointer to device node holding phandle property + * @phandle_name: Name of property holding a phandle value + * @index: For properties holding a table of phandles, this is the index into + * the table + * + * Return: The device_node pointer with refcount incremented. Use + * of_node_put() on it when done. + */ +static inline struct device_node *of_parse_phandle(const struct device_node *np, + const char *phandle_name, + int index) +{ + struct of_phandle_args args; + + if (__of_parse_phandle_with_args(np, phandle_name, NULL, 0, + index, &args)) + return NULL; + + return args.np; +} + +/** + * of_parse_phandle_with_args() - Find a node pointed by phandle in a list + * @np: pointer to a device tree node containing a list + * @list_name: property name that contains a list + * @cells_name: property name that specifies phandles' arguments count + * @index: index of a phandle to parse out + * @out_args: optional pointer to output arguments structure (will be filled) + * + * This function is useful to parse lists of phandles and their arguments. + * Returns 0 on success and fills out_args, on error returns appropriate + * errno value. + * + * Caller is responsible to call of_node_put() on the returned out_args->np + * pointer. + * + * Example:: + * + * phandle1: node1 { + * #list-cells = <2>; + * }; + * + * phandle2: node2 { + * #list-cells = <1>; + * }; + * + * node3 { + * list = <&phandle1 1 2 &phandle2 3>; + * }; + * + * To get a device_node of the ``node2`` node you may call this: + * of_parse_phandle_with_args(node3, "list", "#list-cells", 1, &args); + */ +static inline int of_parse_phandle_with_args(const struct device_node *np, + const char *list_name, + const char *cells_name, + int index, + struct of_phandle_args *out_args) +{ + int cell_count = -1; + + /* If cells_name is NULL we assume a cell count of 0 */ + if (!cells_name) + cell_count = 0; + + return __of_parse_phandle_with_args(np, list_name, cells_name, + cell_count, index, out_args); +} + +/** + * of_parse_phandle_with_fixed_args() - Find a node pointed by phandle in a list + * @np: pointer to a device tree node containing a list + * @list_name: property name that contains a list + * @cell_count: number of argument cells following the phandle + * @index: index of a phandle to parse out + * @out_args: optional pointer to output arguments structure (will be filled) + * + * This function is useful to parse lists of phandles and their arguments. + * Returns 0 on success and fills out_args, on error returns appropriate + * errno value. + * + * Caller is responsible to call of_node_put() on the returned out_args->np + * pointer. + * + * Example:: + * + * phandle1: node1 { + * }; + * + * phandle2: node2 { + * }; + * + * node3 { + * list = <&phandle1 0 2 &phandle2 2 3>; + * }; + * + * To get a device_node of the ``node2`` node you may call this: + * of_parse_phandle_with_fixed_args(node3, "list", 2, 1, &args); + */ +static inline int of_parse_phandle_with_fixed_args(const struct device_node *np, + const char *list_name, + int cell_count, + int index, + struct of_phandle_args *out_args) +{ + return __of_parse_phandle_with_args(np, list_name, NULL, cell_count, + index, out_args); +} + /** * of_property_count_u8_elems - Count the number of u8 elements in a property * -- cgit v1.2.3-59-g8ed1b From 2ca42c3ad9ed875b136065b010753a4caaaa1d38 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 18 Jan 2022 18:35:03 +0100 Subject: of: property: define of_property_read_u{8,16,32,64}_array() unconditionally We can get rid of all the empty stubs because all these functions call of_property_read_variable_u{8,16,32,64}_array() which already have an empty stub if CONFIG_OF is not defined. Signed-off-by: Michael Walle Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20220118173504.2867523-3-michael@walle.cc --- include/linux/of.h | 274 ++++++++++++++++++++++++----------------------------- 1 file changed, 124 insertions(+), 150 deletions(-) (limited to 'include/linux') diff --git a/include/linux/of.h b/include/linux/of.h index 16d76c92fbe0..2dc77430a91a 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -410,130 +410,6 @@ extern int of_detach_node(struct device_node *); #define of_match_ptr(_ptr) (_ptr) -/** - * of_property_read_u8_array - Find and read an array of u8 from a property. - * - * @np: device node from which the property value is to be read. - * @propname: name of the property to be searched. - * @out_values: pointer to return value, modified only if return value is 0. - * @sz: number of array elements to read - * - * Search for a property in a device node and read 8-bit value(s) from - * it. - * - * dts entry of array should be like: - * ``property = /bits/ 8 <0x50 0x60 0x70>;`` - * - * Return: 0 on success, -EINVAL if the property does not exist, - * -ENODATA if property does not have a value, and -EOVERFLOW if the - * property data isn't large enough. - * - * The out_values is modified only if a valid u8 value can be decoded. - */ -static inline int of_property_read_u8_array(const struct device_node *np, - const char *propname, - u8 *out_values, size_t sz) -{ - int ret = of_property_read_variable_u8_array(np, propname, out_values, - sz, 0); - if (ret >= 0) - return 0; - else - return ret; -} - -/** - * of_property_read_u16_array - Find and read an array of u16 from a property. - * - * @np: device node from which the property value is to be read. - * @propname: name of the property to be searched. - * @out_values: pointer to return value, modified only if return value is 0. - * @sz: number of array elements to read - * - * Search for a property in a device node and read 16-bit value(s) from - * it. - * - * dts entry of array should be like: - * ``property = /bits/ 16 <0x5000 0x6000 0x7000>;`` - * - * Return: 0 on success, -EINVAL if the property does not exist, - * -ENODATA if property does not have a value, and -EOVERFLOW if the - * property data isn't large enough. - * - * The out_values is modified only if a valid u16 value can be decoded. - */ -static inline int of_property_read_u16_array(const struct device_node *np, - const char *propname, - u16 *out_values, size_t sz) -{ - int ret = of_property_read_variable_u16_array(np, propname, out_values, - sz, 0); - if (ret >= 0) - return 0; - else - return ret; -} - -/** - * of_property_read_u32_array - Find and read an array of 32 bit integers - * from a property. - * - * @np: device node from which the property value is to be read. - * @propname: name of the property to be searched. - * @out_values: pointer to return value, modified only if return value is 0. - * @sz: number of array elements to read - * - * Search for a property in a device node and read 32-bit value(s) from - * it. - * - * Return: 0 on success, -EINVAL if the property does not exist, - * -ENODATA if property does not have a value, and -EOVERFLOW if the - * property data isn't large enough. - * - * The out_values is modified only if a valid u32 value can be decoded. - */ -static inline int of_property_read_u32_array(const struct device_node *np, - const char *propname, - u32 *out_values, size_t sz) -{ - int ret = of_property_read_variable_u32_array(np, propname, out_values, - sz, 0); - if (ret >= 0) - return 0; - else - return ret; -} - -/** - * of_property_read_u64_array - Find and read an array of 64 bit integers - * from a property. - * - * @np: device node from which the property value is to be read. - * @propname: name of the property to be searched. - * @out_values: pointer to return value, modified only if return value is 0. - * @sz: number of array elements to read - * - * Search for a property in a device node and read 64-bit value(s) from - * it. - * - * Return: 0 on success, -EINVAL if the property does not exist, - * -ENODATA if property does not have a value, and -EOVERFLOW if the - * property data isn't large enough. - * - * The out_values is modified only if a valid u64 value can be decoded. - */ -static inline int of_property_read_u64_array(const struct device_node *np, - const char *propname, - u64 *out_values, size_t sz) -{ - int ret = of_property_read_variable_u64_array(np, propname, out_values, - sz, 0); - if (ret >= 0) - return 0; - else - return ret; -} - /* * struct property *prop; * const __be32 *p; @@ -728,32 +604,6 @@ static inline int of_property_count_elems_of_size(const struct device_node *np, return -ENOSYS; } -static inline int of_property_read_u8_array(const struct device_node *np, - const char *propname, u8 *out_values, size_t sz) -{ - return -ENOSYS; -} - -static inline int of_property_read_u16_array(const struct device_node *np, - const char *propname, u16 *out_values, size_t sz) -{ - return -ENOSYS; -} - -static inline int of_property_read_u32_array(const struct device_node *np, - const char *propname, - u32 *out_values, size_t sz) -{ - return -ENOSYS; -} - -static inline int of_property_read_u64_array(const struct device_node *np, - const char *propname, - u64 *out_values, size_t sz) -{ - return -ENOSYS; -} - static inline int of_property_read_u32_index(const struct device_node *np, const char *propname, u32 index, u32 *out_value) { @@ -1328,6 +1178,130 @@ static inline bool of_property_read_bool(const struct device_node *np, return prop ? true : false; } +/** + * of_property_read_u8_array - Find and read an array of u8 from a property. + * + * @np: device node from which the property value is to be read. + * @propname: name of the property to be searched. + * @out_values: pointer to return value, modified only if return value is 0. + * @sz: number of array elements to read + * + * Search for a property in a device node and read 8-bit value(s) from + * it. + * + * dts entry of array should be like: + * ``property = /bits/ 8 <0x50 0x60 0x70>;`` + * + * Return: 0 on success, -EINVAL if the property does not exist, + * -ENODATA if property does not have a value, and -EOVERFLOW if the + * property data isn't large enough. + * + * The out_values is modified only if a valid u8 value can be decoded. + */ +static inline int of_property_read_u8_array(const struct device_node *np, + const char *propname, + u8 *out_values, size_t sz) +{ + int ret = of_property_read_variable_u8_array(np, propname, out_values, + sz, 0); + if (ret >= 0) + return 0; + else + return ret; +} + +/** + * of_property_read_u16_array - Find and read an array of u16 from a property. + * + * @np: device node from which the property value is to be read. + * @propname: name of the property to be searched. + * @out_values: pointer to return value, modified only if return value is 0. + * @sz: number of array elements to read + * + * Search for a property in a device node and read 16-bit value(s) from + * it. + * + * dts entry of array should be like: + * ``property = /bits/ 16 <0x5000 0x6000 0x7000>;`` + * + * Return: 0 on success, -EINVAL if the property does not exist, + * -ENODATA if property does not have a value, and -EOVERFLOW if the + * property data isn't large enough. + * + * The out_values is modified only if a valid u16 value can be decoded. + */ +static inline int of_property_read_u16_array(const struct device_node *np, + const char *propname, + u16 *out_values, size_t sz) +{ + int ret = of_property_read_variable_u16_array(np, propname, out_values, + sz, 0); + if (ret >= 0) + return 0; + else + return ret; +} + +/** + * of_property_read_u32_array - Find and read an array of 32 bit integers + * from a property. + * + * @np: device node from which the property value is to be read. + * @propname: name of the property to be searched. + * @out_values: pointer to return value, modified only if return value is 0. + * @sz: number of array elements to read + * + * Search for a property in a device node and read 32-bit value(s) from + * it. + * + * Return: 0 on success, -EINVAL if the property does not exist, + * -ENODATA if property does not have a value, and -EOVERFLOW if the + * property data isn't large enough. + * + * The out_values is modified only if a valid u32 value can be decoded. + */ +static inline int of_property_read_u32_array(const struct device_node *np, + const char *propname, + u32 *out_values, size_t sz) +{ + int ret = of_property_read_variable_u32_array(np, propname, out_values, + sz, 0); + if (ret >= 0) + return 0; + else + return ret; +} + +/** + * of_property_read_u64_array - Find and read an array of 64 bit integers + * from a property. + * + * @np: device node from which the property value is to be read. + * @propname: name of the property to be searched. + * @out_values: pointer to return value, modified only if return value is 0. + * @sz: number of array elements to read + * + * Search for a property in a device node and read 64-bit value(s) from + * it. + * + * Return: 0 on success, -EINVAL if the property does not exist, + * -ENODATA if property does not have a value, and -EOVERFLOW if the + * property data isn't large enough. + * + * The out_values is modified only if a valid u64 value can be decoded. + */ +static inline int of_property_read_u64_array(const struct device_node *np, + const char *propname, + u64 *out_values, size_t sz) +{ + int ret = of_property_read_variable_u64_array(np, propname, out_values, + sz, 0); + if (ret >= 0) + return 0; + else + return ret; +} + static inline int of_property_read_u8(const struct device_node *np, const char *propname, u8 *out_value) -- cgit v1.2.3-59-g8ed1b From c522e3ad296b7b692ed3960dfde467f2a34b434f Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 7 Jan 2022 09:28:41 +0000 Subject: fscache: Add a comment explaining how page-release optimisation works Add a comment into fscache_note_page_release() to explain how the page-release optimisation logic works[1]. It's not entirely obvious as it has nothing to do with whether or not the netfs file contains data. FSCACHE_COOKIE_NO_DATA_TO_READ is set if we have no data in the cache yet (ie. the backing file lookup was negative, the file is 0 length or the cookie got invalidated). It means that we have no data in the cache, not that the file is necessarily empty on the server. FSCACHE_COOKIE_HAVE_DATA is set once we've stored data in the backing file. From that point on, we have data we *could* read - however, it's covered by pages in the netfs pagecache until at such time one of those covering pages is released. So if we've written data to the cache (HAVE_DATA) and there wasn't any data in the cache when we started (NO_DATA_TO_READ), it may no longer be true that we can skip reading from the cache. Read skipping is done by cachefiles_prepare_read(). Note that tracking is not done on a per-page basis, but only on a per-file basis. Signed-off-by: David Howells Reviewed-by: Jeff Layton cc: linux-cachefs@redhat.com Link: https://lore.kernel.org/r/043a206f03929c2667a465314144e518070a9b2d.camel@kernel.org/ [1] Link: https://lore.kernel.org/r/164251408479.3435901.9540165422908194636.stgit@warthog.procyon.org.uk/ # v1 --- include/linux/fscache.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fscache.h b/include/linux/fscache.h index ede50406bcb0..296c5f1d9f35 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -665,6 +665,11 @@ static inline void fscache_clear_inode_writeback(struct fscache_cookie *cookie, static inline void fscache_note_page_release(struct fscache_cookie *cookie) { + /* If we've written data to the cache (HAVE_DATA) and there wasn't any + * data in the cache when we started (NO_DATA_TO_READ), it may no + * longer be true that we can skip reading from the cache - so clear + * the flag that causes reads to be skipped. + */ if (cookie && test_bit(FSCACHE_COOKIE_HAVE_DATA, &cookie->flags) && test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) -- cgit v1.2.3-59-g8ed1b From ffa65753c43142f3b803486442813744da71cff2 Mon Sep 17 00:00:00 2001 From: Alistair Popple Date: Fri, 21 Jan 2022 22:10:46 -0800 Subject: mm/migrate.c: rework migration_entry_wait() to not take a pageref This fixes the FIXME in migrate_vma_check_page(). Before migrating a page migration code will take a reference and check there are no unexpected page references, failing the migration if there are. When a thread faults on a migration entry it will take a temporary reference to the page to wait for the page to become unlocked signifying the migration entry has been removed. This reference is dropped just prior to waiting on the page lock, however the extra reference can cause migration failures so it is desirable to avoid taking it. As migration code already has a reference to the migrating page an extra reference to wait on PG_locked is unnecessary so long as the reference can't be dropped whilst setting up the wait. When faulting on a migration entry the ptl is taken to check the migration entry. Removing a migration entry also requires the ptl, and migration code won't drop its page reference until after the migration entry has been removed. Therefore retaining the ptl of a migration entry is sufficient to ensure the page has a reference. Reworking migration_entry_wait() to hold the ptl until the wait setup is complete means the extra page reference is no longer needed. [apopple@nvidia.com: v5] Link: https://lkml.kernel.org/r/20211213033848.1973946-1-apopple@nvidia.com Link: https://lkml.kernel.org/r/20211118020754.954425-1-apopple@nvidia.com Signed-off-by: Alistair Popple Acked-by: David Hildenbrand Cc: David Howells Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: John Hubbard Cc: Matthew Wilcox (Oracle) Cc: Ralph Campbell Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/migrate.h | 2 ++ mm/filemap.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ mm/migrate.c | 38 +++------------------ 3 files changed, 97 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 4850cc5bf813..db96e10eb8da 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -40,6 +40,8 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page); extern int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, int extra_count); +void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep, + spinlock_t *ptl); void folio_migrate_flags(struct folio *newfolio, struct folio *folio); void folio_migrate_copy(struct folio *newfolio, struct folio *folio); int folio_migrate_mapping(struct address_space *mapping, diff --git a/mm/filemap.c b/mm/filemap.c index 2fd9b2f24025..60866ae711e2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include "internal.h" @@ -1388,6 +1390,95 @@ repeat: return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; } +#ifdef CONFIG_MIGRATION +/** + * migration_entry_wait_on_locked - Wait for a migration entry to be removed + * @entry: migration swap entry. + * @ptep: mapped pte pointer. Will return with the ptep unmapped. Only required + * for pte entries, pass NULL for pmd entries. + * @ptl: already locked ptl. This function will drop the lock. + * + * Wait for a migration entry referencing the given page to be removed. This is + * equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except + * this can be called without taking a reference on the page. Instead this + * should be called while holding the ptl for the migration entry referencing + * the page. + * + * Returns after unmapping and unlocking the pte/ptl with pte_unmap_unlock(). + * + * This follows the same logic as folio_wait_bit_common() so see the comments + * there. + */ +void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep, + spinlock_t *ptl) +{ + struct wait_page_queue wait_page; + wait_queue_entry_t *wait = &wait_page.wait; + bool thrashing = false; + bool delayacct = false; + unsigned long pflags; + wait_queue_head_t *q; + struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); + + q = folio_waitqueue(folio); + if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { + if (!folio_test_swapbacked(folio)) { + delayacct_thrashing_start(); + delayacct = true; + } + psi_memstall_enter(&pflags); + thrashing = true; + } + + init_wait(wait); + wait->func = wake_page_function; + wait_page.folio = folio; + wait_page.bit_nr = PG_locked; + wait->flags = 0; + + spin_lock_irq(&q->lock); + folio_set_waiters(folio); + if (!folio_trylock_flag(folio, PG_locked, wait)) + __add_wait_queue_entry_tail(q, wait); + spin_unlock_irq(&q->lock); + + /* + * If a migration entry exists for the page the migration path must hold + * a valid reference to the page, and it must take the ptl to remove the + * migration entry. So the page is valid until the ptl is dropped. + */ + if (ptep) + pte_unmap_unlock(ptep, ptl); + else + spin_unlock(ptl); + + for (;;) { + unsigned int flags; + + set_current_state(TASK_UNINTERRUPTIBLE); + + /* Loop until we've been woken or interrupted */ + flags = smp_load_acquire(&wait->flags); + if (!(flags & WQ_FLAG_WOKEN)) { + if (signal_pending_state(TASK_UNINTERRUPTIBLE, current)) + break; + + io_schedule(); + continue; + } + break; + } + + finish_wait(q, wait); + + if (thrashing) { + if (delayacct) + delayacct_thrashing_end(); + psi_memstall_leave(&pflags); + } +} +#endif + void folio_wait_bit(struct folio *folio, int bit_nr) { folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); diff --git a/mm/migrate.c b/mm/migrate.c index 18ce840914f0..c7da064b4781 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -291,7 +291,6 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, { pte_t pte; swp_entry_t entry; - struct folio *folio; spin_lock(ptl); pte = *ptep; @@ -302,17 +301,7 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, if (!is_migration_entry(entry)) goto out; - folio = page_folio(pfn_swap_entry_to_page(entry)); - - /* - * Once page cache replacement of page migration started, page_count - * is zero; but we must not call folio_put_wait_locked() without - * a ref. Use folio_try_get(), and just fault again if it fails. - */ - if (!folio_try_get(folio)) - goto out; - pte_unmap_unlock(ptep, ptl); - folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); + migration_entry_wait_on_locked(entry, ptep, ptl); return; out: pte_unmap_unlock(ptep, ptl); @@ -337,16 +326,11 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; - struct folio *folio; ptl = pmd_lock(mm, pmd); if (!is_pmd_migration_entry(*pmd)) goto unlock; - folio = page_folio(pfn_swap_entry_to_page(pmd_to_swp_entry(*pmd))); - if (!folio_try_get(folio)) - goto unlock; - spin_unlock(ptl); - folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); + migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl); return; unlock: spin_unlock(ptl); @@ -2431,22 +2415,8 @@ static bool migrate_vma_check_page(struct page *page) return false; /* Page from ZONE_DEVICE have one extra reference */ - if (is_zone_device_page(page)) { - /* - * Private page can never be pin as they have no valid pte and - * GUP will fail for those. Yet if there is a pending migration - * a thread might try to wait on the pte migration entry and - * will bump the page reference count. Sadly there is no way to - * differentiate a regular pin from migration wait. Hence to - * avoid 2 racing thread trying to migrate back to CPU to enter - * infinite loop (one stopping migration because the other is - * waiting on pte migration entry). We always return true here. - * - * FIXME proper solution is to rework migration_entry_wait() so - * it does not need to take a reference on page. - */ - return is_device_private_page(page); - } + if (is_zone_device_page(page)) + extra++; /* For file back page */ if (page_mapping(page)) -- cgit v1.2.3-59-g8ed1b From 3ddd9a808cee7284931312f2f3e854c9617f44b2 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:10:50 -0800 Subject: sysctl: add a new register_sysctl_init() interface Patch series "sysctl: first set of kernel/sysctl cleanups", v2. Finally had time to respin the series of the work we had started last year on cleaning up the kernel/sysct.c kitchen sink. People keeps stuffing their sysctls in that file and this creates a maintenance burden. So this effort is aimed at placing sysctls where they actually belong. I'm going to split patches up into series as there is quite a bit of work. This first set adds register_sysctl_init() for uses of registerting a sysctl on the init path, adds const where missing to a few places, generalizes common values so to be more easy to share, and starts the move of a few kernel/sysctl.c out where they belong. The majority of rework on v2 in this first patch set is 0-day fixes. Eric Biederman's feedback is later addressed in subsequent patch sets. I'll only post the first two patch sets for now. We can address the rest once the first two patch sets get completely reviewed / Acked. This patch (of 9): The kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. Today though folks heavily rely on tables on kernel/sysctl.c so they can easily just extend this table with their needed sysctls. In order to help users move their sysctls out we need to provide a helper which can be used during code initialization. We special-case the initialization use of register_sysctl() since it *is* safe to fail, given all that sysctls do is provide a dynamic interface to query or modify at runtime an existing variable. So the use case of register_sysctl() on init should *not* stop if the sysctls don't end up getting registered. It would be counter productive to stop boot if a simple sysctl registration failed. Provide a helper for init then, and document the recommended init levels to use for callers of this routine. We will later use this in subsequent patches to start slimming down kernel/sysctl.c tables and moving sysctl registration to the code which actually needs these sysctls. [mcgrof@kernel.org: major commit log and documentation rephrasing also moved to fs/proc/proc_sysctl.c ] Link: https://lkml.kernel.org/r/20211123202347.818157-1-mcgrof@kernel.org Link: https://lkml.kernel.org/r/20211123202347.818157-2-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Reviewed-by: Kees Cook Cc: Iurii Zaikin Cc: "Eric W. Biederman" Cc: Peter Zijlstra Cc: Greg Kroah-Hartman Cc: Paul Turner Cc: Andy Shevchenko Cc: Sebastian Reichel Cc: Tetsuo Handa Cc: Petr Mladek Cc: Sergey Senozhatsky Cc: Qing Wang Cc: Benjamin LaHaise Cc: Al Viro Cc: Jan Kara Cc: Amir Goldstein Cc: Stephen Kitt Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Phillip Potter Cc: Rodrigo Vivi Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 33 +++++++++++++++++++++++++++++++++ include/linux/sysctl.h | 3 +++ 2 files changed, 36 insertions(+) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 389e1e42e7d9..55a54973f061 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "internal.h" static const struct dentry_operations proc_sys_dentry_operations; @@ -1383,6 +1384,38 @@ struct ctl_table_header *register_sysctl(const char *path, struct ctl_table *tab } EXPORT_SYMBOL(register_sysctl); +/** + * __register_sysctl_init() - register sysctl table to path + * @path: path name for sysctl base + * @table: This is the sysctl table that needs to be registered to the path + * @table_name: The name of sysctl table, only used for log printing when + * registration fails + * + * The sysctl interface is used by userspace to query or modify at runtime + * a predefined value set on a variable. These variables however have default + * values pre-set. Code which depends on these variables will always work even + * if register_sysctl() fails. If register_sysctl() fails you'd just loose the + * ability to query or modify the sysctls dynamically at run time. Chances of + * register_sysctl() failing on init are extremely low, and so for both reasons + * this function does not return any error as it is used by initialization code. + * + * Context: Can only be called after your respective sysctl base path has been + * registered. So for instance, most base directories are registered early on + * init before init levels are processed through proc_sys_init() and + * sysctl_init(). + */ +void __init __register_sysctl_init(const char *path, struct ctl_table *table, + const char *table_name) +{ + struct ctl_table_header *hdr = register_sysctl(path, table); + + if (unlikely(!hdr)) { + pr_err("failed when register_sysctl %s to %s\n", table_name, path); + return; + } + kmemleak_not_leak(hdr); +} + static char *append_path(const char *path, char *pos, const char *name) { int namelen; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 1fa2b69c6fc3..d3ab7969b6b5 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -199,6 +199,9 @@ struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, void unregister_sysctl_table(struct ctl_table_header * table); extern int sysctl_init(void); +extern void __register_sysctl_init(const char *path, struct ctl_table *table, + const char *table_name); +#define register_sysctl_init(path, table) __register_sysctl_init(path, table, #table) void do_sysctl_args(void); extern int pwrsw_enabled; -- cgit v1.2.3-59-g8ed1b From 78e36f3b0dae586f623c4a37ec5eb5496f5abbe1 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:10:55 -0800 Subject: sysctl: move some boundary constants from sysctl.c to sysctl_vals sysctl has helpers which let us specify boundary values for a min or max int value. Since these are used for a boundary check only they don't change, so move these variables to sysctl_vals to avoid adding duplicate variables. This will help with our cleanup of kernel/sysctl.c. [akpm@linux-foundation.org: update it for "mm/pagealloc: sysctl: change watermark_scale_factor max limit to 30%"] [mcgrof@kernel.org: major rebase] Link: https://lkml.kernel.org/r/20211123202347.818157-3-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Reviewed-by: Kees Cook Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Benjamin LaHaise Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Jan Kara Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Qing Wang Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Phillip Potter Cc: Rodrigo Vivi Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 2 +- include/linux/sysctl.h | 13 ++++++++++--- kernel/sysctl.c | 45 +++++++++++++++++++-------------------------- 3 files changed, 30 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 55a54973f061..cd8bbf13d0f0 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -26,7 +26,7 @@ static const struct file_operations proc_sys_dir_file_operations; static const struct inode_operations proc_sys_dir_operations; /* shared constants to be used in various sysctls */ -const int sysctl_vals[] = { 0, 1, INT_MAX }; +const int sysctl_vals[] = { -1, 0, 1, 2, 4, 100, 200, 1000, 3000, INT_MAX }; EXPORT_SYMBOL(sysctl_vals); /* Support for permanently empty directories */ diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index d3ab7969b6b5..47cf70c8eb93 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -38,9 +38,16 @@ struct ctl_table_header; struct ctl_dir; /* Keep the same order as in fs/proc/proc_sysctl.c */ -#define SYSCTL_ZERO ((void *)&sysctl_vals[0]) -#define SYSCTL_ONE ((void *)&sysctl_vals[1]) -#define SYSCTL_INT_MAX ((void *)&sysctl_vals[2]) +#define SYSCTL_NEG_ONE ((void *)&sysctl_vals[0]) +#define SYSCTL_ZERO ((void *)&sysctl_vals[1]) +#define SYSCTL_ONE ((void *)&sysctl_vals[2]) +#define SYSCTL_TWO ((void *)&sysctl_vals[3]) +#define SYSCTL_FOUR ((void *)&sysctl_vals[4]) +#define SYSCTL_ONE_HUNDRED ((void *)&sysctl_vals[5]) +#define SYSCTL_TWO_HUNDRED ((void *)&sysctl_vals[6]) +#define SYSCTL_ONE_THOUSAND ((void *)&sysctl_vals[7]) +#define SYSCTL_THREE_THOUSAND ((void *)&sysctl_vals[8]) +#define SYSCTL_INT_MAX ((void *)&sysctl_vals[9]) extern const int sysctl_vals[]; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index ef77be575d87..901a9dfe4d62 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -114,16 +114,9 @@ static int sixty = 60; #endif -static int __maybe_unused neg_one = -1; -static int __maybe_unused two = 2; -static int __maybe_unused four = 4; static unsigned long zero_ul; static unsigned long one_ul = 1; static unsigned long long_max = LONG_MAX; -static int one_hundred = 100; -static int two_hundred = 200; -static int one_thousand = 1000; -static int three_thousand = 3000; #ifdef CONFIG_PRINTK static int ten_thousand = 10000; #endif @@ -1964,7 +1957,7 @@ static struct ctl_table kern_table[] = { .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = &neg_one, + .extra1 = SYSCTL_NEG_ONE, .extra2 = SYSCTL_ONE, }, #endif @@ -2306,7 +2299,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax_sysadmin, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, #endif { @@ -2566,7 +2559,7 @@ static struct ctl_table kern_table[] = { .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = &neg_one, + .extra1 = SYSCTL_NEG_ONE, }, #endif #ifdef CONFIG_RT_MUTEXES @@ -2628,7 +2621,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = perf_cpu_time_max_percent_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, { .procname = "perf_event_max_stack", @@ -2646,7 +2639,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = perf_event_max_stack_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_thousand, + .extra2 = SYSCTL_ONE_THOUSAND, }, #endif { @@ -2677,7 +2670,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = bpf_unpriv_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, { .procname = "bpf_stats_enabled", @@ -2731,7 +2724,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = overcommit_policy_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, { .procname = "panic_on_oom", @@ -2740,7 +2733,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, { .procname = "oom_kill_allocating_task", @@ -2785,7 +2778,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = dirty_background_ratio_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, { .procname = "dirty_background_bytes", @@ -2802,7 +2795,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = dirty_ratio_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, { .procname = "dirty_bytes", @@ -2842,7 +2835,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = &two_hundred, + .extra2 = SYSCTL_TWO_HUNDRED, }, #ifdef CONFIG_HUGETLB_PAGE { @@ -2899,7 +2892,7 @@ static struct ctl_table vm_table[] = { .mode = 0200, .proc_handler = drop_caches_sysctl_handler, .extra1 = SYSCTL_ONE, - .extra2 = &four, + .extra2 = SYSCTL_FOUR, }, #ifdef CONFIG_COMPACTION { @@ -2916,7 +2909,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = compaction_proactiveness_sysctl_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, { .procname = "extfrag_threshold", @@ -2961,7 +2954,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = watermark_scale_factor_sysctl_handler, .extra1 = SYSCTL_ONE, - .extra2 = &three_thousand, + .extra2 = SYSCTL_THREE_THOUSAND, }, { .procname = "percpu_pagelist_high_fraction", @@ -3040,7 +3033,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = sysctl_min_unmapped_ratio_sysctl_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, { .procname = "min_slab_ratio", @@ -3049,7 +3042,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = sysctl_min_slab_ratio_sysctl_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &one_hundred, + .extra2 = SYSCTL_ONE_HUNDRED, }, #endif #ifdef CONFIG_SMP @@ -3339,7 +3332,7 @@ static struct ctl_table fs_table[] = { .mode = 0600, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, { .procname = "protected_regular", @@ -3348,7 +3341,7 @@ static struct ctl_table fs_table[] = { .mode = 0600, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, { .procname = "suid_dumpable", @@ -3357,7 +3350,7 @@ static struct ctl_table fs_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax_coredump, .extra1 = SYSCTL_ZERO, - .extra2 = &two, + .extra2 = SYSCTL_TWO, }, #if defined(CONFIG_BINFMT_MISC) || defined(CONFIG_BINFMT_MISC_MODULE) { -- cgit v1.2.3-59-g8ed1b From bbe7a10ed83a5fa0b0ff6161ecdc4e65a0e9c993 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:11:00 -0800 Subject: hung_task: move hung_task sysctl interface to hung_task.c The kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move hung_task sysctl interface to hung_task.c and use register_sysctl() to register the sysctl interface. [mcgrof@kernel.org: commit log refresh and fixed 2-3 0day reported compile issues] Link: https://lkml.kernel.org/r/20211123202347.818157-4-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Reviewed-by: Kees Cook Reviewed-by: Petr Mladek Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Benjamin LaHaise Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Jan Kara Cc: Paul Turner Cc: Peter Zijlstra Cc: Qing Wang Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Phillip Potter Cc: Rodrigo Vivi Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/sched/sysctl.h | 14 +------- kernel/hung_task.c | 81 ++++++++++++++++++++++++++++++++++++++++++-- kernel/sysctl.c | 61 --------------------------------- 3 files changed, 79 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 304f431178fd..c19dd5a2c05c 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -7,20 +7,8 @@ struct ctl_table; #ifdef CONFIG_DETECT_HUNG_TASK - -#ifdef CONFIG_SMP -extern unsigned int sysctl_hung_task_all_cpu_backtrace; -#else -#define sysctl_hung_task_all_cpu_backtrace 0 -#endif /* CONFIG_SMP */ - -extern int sysctl_hung_task_check_count; -extern unsigned int sysctl_hung_task_panic; +/* used for hung_task and block/ */ extern unsigned long sysctl_hung_task_timeout_secs; -extern unsigned long sysctl_hung_task_check_interval_secs; -extern int sysctl_hung_task_warnings; -int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); #else /* Avoid need for ifdefs elsewhere in the code */ enum { sysctl_hung_task_timeout_secs = 0 }; diff --git a/kernel/hung_task.c b/kernel/hung_task.c index 9888e2bc8c76..52501e5f7655 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c @@ -63,7 +63,9 @@ static struct task_struct *watchdog_task; * Should we dump all CPUs backtraces in a hung task event? * Defaults to 0, can be changed via sysctl. */ -unsigned int __read_mostly sysctl_hung_task_all_cpu_backtrace; +static unsigned int __read_mostly sysctl_hung_task_all_cpu_backtrace; +#else +#define sysctl_hung_task_all_cpu_backtrace 0 #endif /* CONFIG_SMP */ /* @@ -222,11 +224,13 @@ static long hung_timeout_jiffies(unsigned long last_checked, MAX_SCHEDULE_TIMEOUT; } +#ifdef CONFIG_SYSCTL /* * Process updating of timeout sysctl */ -int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, + void __user *buffer, + size_t *lenp, loff_t *ppos) { int ret; @@ -241,6 +245,76 @@ int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, return ret; } +/* + * This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs + * and hung_task_check_interval_secs + */ +static const unsigned long hung_task_timeout_max = (LONG_MAX / HZ); +static struct ctl_table hung_task_sysctls[] = { +#ifdef CONFIG_SMP + { + .procname = "hung_task_all_cpu_backtrace", + .data = &sysctl_hung_task_all_cpu_backtrace, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif /* CONFIG_SMP */ + { + .procname = "hung_task_panic", + .data = &sysctl_hung_task_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "hung_task_check_count", + .data = &sysctl_hung_task_check_count, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, + { + .procname = "hung_task_timeout_secs", + .data = &sysctl_hung_task_timeout_secs, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_dohung_task_timeout_secs, + .extra2 = (void *)&hung_task_timeout_max, + }, + { + .procname = "hung_task_check_interval_secs", + .data = &sysctl_hung_task_check_interval_secs, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_dohung_task_timeout_secs, + .extra2 = (void *)&hung_task_timeout_max, + }, + { + .procname = "hung_task_warnings", + .data = &sysctl_hung_task_warnings, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_NEG_ONE, + }, + {} +}; + +static void __init hung_task_sysctl_init(void) +{ + register_sysctl_init("kernel", hung_task_sysctls); +} +#else +#define hung_task_sysctl_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + + static atomic_t reset_hung_task = ATOMIC_INIT(0); void reset_hung_task_detector(void) @@ -310,6 +384,7 @@ static int __init hung_task_init(void) pm_notifier(hungtask_pm_notify, 0); watchdog_task = kthread_run(watchdog, NULL, "khungtaskd"); + hung_task_sysctl_init(); return 0; } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 901a9dfe4d62..a4e00abe6f78 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -134,13 +134,6 @@ static int minolduid; static int ngroups_max = NGROUPS_MAX; static const int cap_last_cap = CAP_LAST_CAP; -/* - * This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs - * and hung_task_check_interval_secs - */ -#ifdef CONFIG_DETECT_HUNG_TASK -static unsigned long hung_task_timeout_max = (LONG_MAX/HZ); -#endif #ifdef CONFIG_INOTIFY_USER #include @@ -2508,60 +2501,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_DETECT_HUNG_TASK -#ifdef CONFIG_SMP - { - .procname = "hung_task_all_cpu_backtrace", - .data = &sysctl_hung_task_all_cpu_backtrace, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif /* CONFIG_SMP */ - { - .procname = "hung_task_panic", - .data = &sysctl_hung_task_panic, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, - { - .procname = "hung_task_check_count", - .data = &sysctl_hung_task_check_count, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - }, - { - .procname = "hung_task_timeout_secs", - .data = &sysctl_hung_task_timeout_secs, - .maxlen = sizeof(unsigned long), - .mode = 0644, - .proc_handler = proc_dohung_task_timeout_secs, - .extra2 = &hung_task_timeout_max, - }, - { - .procname = "hung_task_check_interval_secs", - .data = &sysctl_hung_task_check_interval_secs, - .maxlen = sizeof(unsigned long), - .mode = 0644, - .proc_handler = proc_dohung_task_timeout_secs, - .extra2 = &hung_task_timeout_max, - }, - { - .procname = "hung_task_warnings", - .data = &sysctl_hung_task_warnings, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_NEG_ONE, - }, -#endif #ifdef CONFIG_RT_MUTEXES { .procname = "max_lock_depth", -- cgit v1.2.3-59-g8ed1b From 86b12b6c5d6b46e64bf2e8080528781032e4bd90 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:11:24 -0800 Subject: aio: move aio sysctl to aio.c The kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. Move aio sysctl to aio.c and use the new register_sysctl_init() to register the sysctl interface for aio. [mcgrof@kernel.org: adjust commit log to justify the move] Link: https://lkml.kernel.org/r/20211123202347.818157-9-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Reviewed-by: Jan Kara Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Benjamin LaHaise Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Kees Cook Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Qing Wang Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Phillip Potter Cc: Rodrigo Vivi Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 31 +++++++++++++++++++++++++++++-- include/linux/aio.h | 4 ---- kernel/sysctl.c | 17 ----------------- 3 files changed, 29 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/fs/aio.c b/fs/aio.c index f6f1cbffef9e..4ceba13a7db0 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -220,9 +220,35 @@ struct aio_kiocb { /*------ sysctl variables----*/ static DEFINE_SPINLOCK(aio_nr_lock); -unsigned long aio_nr; /* current system wide number of aio requests */ -unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ +static unsigned long aio_nr; /* current system wide number of aio requests */ +static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ /*----end sysctl variables---*/ +#ifdef CONFIG_SYSCTL +static struct ctl_table aio_sysctls[] = { + { + .procname = "aio-nr", + .data = &aio_nr, + .maxlen = sizeof(aio_nr), + .mode = 0444, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "aio-max-nr", + .data = &aio_max_nr, + .maxlen = sizeof(aio_max_nr), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + {} +}; + +static void __init aio_sysctl_init(void) +{ + register_sysctl_init("fs", aio_sysctls); +} +#else +#define aio_sysctl_init() do { } while (0) +#endif static struct kmem_cache *kiocb_cachep; static struct kmem_cache *kioctx_cachep; @@ -275,6 +301,7 @@ static int __init aio_setup(void) kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); + aio_sysctl_init(); return 0; } __initcall(aio_setup); diff --git a/include/linux/aio.h b/include/linux/aio.h index b83e68dd006f..86892a4fe7c8 100644 --- a/include/linux/aio.h +++ b/include/linux/aio.h @@ -20,8 +20,4 @@ static inline void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel) { } #endif /* CONFIG_AIO */ -/* for sysctl: */ -extern unsigned long aio_nr; -extern unsigned long aio_max_nr; - #endif /* __LINUX__AIO_H */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index a2175a305f10..822555a0ec76 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -20,7 +20,6 @@ */ #include -#include #include #include #include @@ -3111,22 +3110,6 @@ static struct ctl_table fs_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_AIO - { - .procname = "aio-nr", - .data = &aio_nr, - .maxlen = sizeof(aio_nr), - .mode = 0444, - .proc_handler = proc_doulongvec_minmax, - }, - { - .procname = "aio-max-nr", - .data = &aio_max_nr, - .maxlen = sizeof(aio_max_nr), - .mode = 0644, - .proc_handler = proc_doulongvec_minmax, - }, -#endif /* CONFIG_AIO */ #ifdef CONFIG_INOTIFY_USER { .procname = "inotify", -- cgit v1.2.3-59-g8ed1b From 49a4de75719b6c0f1f375df9908a95cef1e34945 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:11:29 -0800 Subject: dnotify: move dnotify sysctl to dnotify.c The kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move dnotify sysctls to dnotify.c and use the new register_sysctl_init() to register the sysctl interface. [mcgrof@kernel.org: adjust the commit log to justify the move] Link: https://lkml.kernel.org/r/20211123202347.818157-10-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Acked-by: Jan Kara Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Benjamin LaHaise Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Kees Cook Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Qing Wang Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Phillip Potter Cc: Rodrigo Vivi Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/notify/dnotify/dnotify.c | 21 ++++++++++++++++++++- include/linux/dnotify.h | 1 - kernel/sysctl.c | 10 ---------- 3 files changed, 20 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c index d5ebebb034ff..829dd4a61b66 100644 --- a/fs/notify/dnotify/dnotify.c +++ b/fs/notify/dnotify/dnotify.c @@ -19,7 +19,25 @@ #include #include -int dir_notify_enable __read_mostly = 1; +static int dir_notify_enable __read_mostly = 1; +#ifdef CONFIG_SYSCTL +static struct ctl_table dnotify_sysctls[] = { + { + .procname = "dir-notify-enable", + .data = &dir_notify_enable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + {} +}; +static void __init dnotify_sysctl_init(void) +{ + register_sysctl_init("fs", dnotify_sysctls); +} +#else +#define dnotify_sysctl_init() do { } while (0) +#endif static struct kmem_cache *dnotify_struct_cache __read_mostly; static struct kmem_cache *dnotify_mark_cache __read_mostly; @@ -386,6 +404,7 @@ static int __init dnotify_init(void) dnotify_group = fsnotify_alloc_group(&dnotify_fsnotify_ops); if (IS_ERR(dnotify_group)) panic("unable to allocate fsnotify group for dnotify\n"); + dnotify_sysctl_init(); return 0; } diff --git a/include/linux/dnotify.h b/include/linux/dnotify.h index b87c3b85a166..b1d26f9f1c9f 100644 --- a/include/linux/dnotify.h +++ b/include/linux/dnotify.h @@ -29,7 +29,6 @@ struct dnotify_struct { FS_CREATE | FS_RENAME |\ FS_MOVED_FROM | FS_MOVED_TO) -extern int dir_notify_enable; extern void dnotify_flush(struct file *, fl_owner_t); extern int fcntl_dirnotify(int, struct file *, unsigned long); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 822555a0ec76..fb37825cd3cd 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -49,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -3091,15 +3090,6 @@ static struct ctl_table fs_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_DNOTIFY - { - .procname = "dir-notify-enable", - .data = &dir_notify_enable, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif #ifdef CONFIG_MMU #ifdef CONFIG_FILE_LOCKING { -- cgit v1.2.3-59-g8ed1b From 7b9ad122b52c9839e1f68f16c907990a6ad6f793 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:11:59 -0800 Subject: inotify: simplify subdirectory registration with register_sysctl() There is no need to user boiler plate code to specify a set of base directories we're going to stuff sysctls under. Simplify this by using register_sysctl() and specifying the directory path directly. Move inotify_user sysctl to inotify_user.c while at it to remove clutter from kernel/sysctl.c. [mcgrof@kernel.org: remember to register fanotify_table] Link: https://lkml.kernel.org/r/YZ5A6iWLb0h3N3RC@bombadil.infradead.org [mcgrof@kernel.org: update commit log to reflect new path we decided to take] Link: https://lkml.kernel.org/r/20211123202422.819032-7-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/notify/fanotify/fanotify_user.c | 10 +++++++++- fs/notify/inotify/inotify_user.c | 11 ++++++++++- include/linux/fanotify.h | 2 -- include/linux/inotify.h | 3 --- kernel/sysctl.c | 21 --------------------- 5 files changed, 19 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 73a3e939c921..73b1615f9d96 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -59,7 +59,7 @@ static int fanotify_max_queued_events __read_mostly; static long ft_zero = 0; static long ft_int_max = INT_MAX; -struct ctl_table fanotify_table[] = { +static struct ctl_table fanotify_table[] = { { .procname = "max_user_groups", .data = &init_user_ns.ucount_max[UCOUNT_FANOTIFY_GROUPS], @@ -88,6 +88,13 @@ struct ctl_table fanotify_table[] = { }, { } }; + +static void __init fanotify_sysctls_init(void) +{ + register_sysctl("fs/fanotify", fanotify_table); +} +#else +#define fanotify_sysctls_init() do { } while (0) #endif /* CONFIG_SYSCTL */ /* @@ -1743,6 +1750,7 @@ static int __init fanotify_user_setup(void) init_user_ns.ucount_max[UCOUNT_FANOTIFY_GROUPS] = FANOTIFY_DEFAULT_MAX_GROUPS; init_user_ns.ucount_max[UCOUNT_FANOTIFY_MARKS] = max_marks; + fanotify_sysctls_init(); return 0; } diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 29fca3284bb5..54583f62dc44 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -58,7 +58,7 @@ struct kmem_cache *inotify_inode_mark_cachep __read_mostly; static long it_zero = 0; static long it_int_max = INT_MAX; -struct ctl_table inotify_table[] = { +static struct ctl_table inotify_table[] = { { .procname = "max_user_instances", .data = &init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES], @@ -87,6 +87,14 @@ struct ctl_table inotify_table[] = { }, { } }; + +static void __init inotify_sysctls_init(void) +{ + register_sysctl("fs/inotify", inotify_table); +} + +#else +#define inotify_sysctls_init() do { } while (0) #endif /* CONFIG_SYSCTL */ static inline __u32 inotify_arg_to_mask(struct inode *inode, u32 arg) @@ -849,6 +857,7 @@ static int __init inotify_user_setup(void) inotify_max_queued_events = 16384; init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES] = 128; init_user_ns.ucount_max[UCOUNT_INOTIFY_WATCHES] = watches_max; + inotify_sysctls_init(); return 0; } diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h index 3afdf339d53c..419cadcd7ff5 100644 --- a/include/linux/fanotify.h +++ b/include/linux/fanotify.h @@ -5,8 +5,6 @@ #include #include -extern struct ctl_table fanotify_table[]; /* for sysctl */ - #define FAN_GROUP_FLAG(group, flag) \ ((group)->fanotify_data.flags & (flag)) diff --git a/include/linux/inotify.h b/include/linux/inotify.h index 6a24905f6e1e..8d20caa1b268 100644 --- a/include/linux/inotify.h +++ b/include/linux/inotify.h @@ -7,11 +7,8 @@ #ifndef _LINUX_INOTIFY_H #define _LINUX_INOTIFY_H -#include #include -extern struct ctl_table inotify_table[]; /* for sysctl */ - #define ALL_INOTIFY_BITS (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | \ IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | \ IN_MOVED_TO | IN_CREATE | IN_DELETE | \ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index fb37825cd3cd..2ee0e76888d8 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -126,13 +126,6 @@ static const int maxolduid = 65535; static const int ngroups_max = NGROUPS_MAX; static const int cap_last_cap = CAP_LAST_CAP; -#ifdef CONFIG_INOTIFY_USER -#include -#endif -#ifdef CONFIG_FANOTIFY -#include -#endif - #ifdef CONFIG_PROC_SYSCTL /** @@ -3100,20 +3093,6 @@ static struct ctl_table fs_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_INOTIFY_USER - { - .procname = "inotify", - .mode = 0555, - .child = inotify_table, - }, -#endif -#ifdef CONFIG_FANOTIFY - { - .procname = "fanotify", - .mode = 0555, - .child = fanotify_table, - }, -#endif #ifdef CONFIG_EPOLL { .procname = "epoll", -- cgit v1.2.3-59-g8ed1b From a8f5de894f76f1c73f4a068d04897a5e2f873825 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:12:09 -0800 Subject: eventpoll: simplify sysctl declaration with register_sysctl() The kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the epoll_table sysctl to fs/eventpoll.c and use register_sysctl(). Link: https://lkml.kernel.org/r/20211123202422.819032-9-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: "Eric W. Biederman" Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Tetsuo Handa Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Jani Nikula Cc: John Ogness Cc: Martin K. Petersen Cc: "Rafael J. Wysocki" Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/eventpoll.c | 10 +++++++++- include/linux/poll.h | 2 -- include/linux/sysctl.h | 1 - kernel/sysctl.c | 7 ------- 4 files changed, 9 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 06f4c5ae1451..e2daa940ebce 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -307,7 +307,7 @@ static void unlist_file(struct epitems_head *head) static long long_zero; static long long_max = LONG_MAX; -struct ctl_table epoll_table[] = { +static struct ctl_table epoll_table[] = { { .procname = "max_user_watches", .data = &max_user_watches, @@ -319,6 +319,13 @@ struct ctl_table epoll_table[] = { }, { } }; + +static void __init epoll_sysctls_init(void) +{ + register_sysctl("fs/epoll", epoll_table); +} +#else +#define epoll_sysctls_init() do { } while (0) #endif /* CONFIG_SYSCTL */ static const struct file_operations eventpoll_fops; @@ -2378,6 +2385,7 @@ static int __init eventpoll_init(void) /* Allocates slab cache used to allocate "struct eppoll_entry" */ pwq_cache = kmem_cache_create("eventpoll_pwq", sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL); + epoll_sysctls_init(); ephead_cache = kmem_cache_create("ep_head", sizeof(struct epitems_head), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL); diff --git a/include/linux/poll.h b/include/linux/poll.h index 1cdc32b1f1b0..a9e0e1c2d1f2 100644 --- a/include/linux/poll.h +++ b/include/linux/poll.h @@ -8,12 +8,10 @@ #include #include #include -#include #include #include #include -extern struct ctl_table epoll_table[]; /* for sysctl */ /* ~832 bytes of stack space used max in sys_select/sys_poll before allocating additional memory. */ #ifdef __clang__ diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 47cf70c8eb93..6dd0f277f844 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -219,7 +219,6 @@ extern int no_unaligned_warning; extern struct ctl_table sysctl_mount_point[]; extern struct ctl_table random_table[]; extern struct ctl_table firmware_config_table[]; -extern struct ctl_table epoll_table[]; #else /* CONFIG_SYSCTL */ static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * table) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2ee0e76888d8..09fa72299f18 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3093,13 +3093,6 @@ static struct ctl_table fs_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_EPOLL - { - .procname = "epoll", - .mode = 0555, - .child = epoll_table, - }, -#endif #endif { .procname = "protected_symlinks", -- cgit v1.2.3-59-g8ed1b From 6aad36d421d8bfe156508fa4edfe67827234cf0f Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:12:13 -0800 Subject: firmware_loader: move firmware sysctl to its own files Patch series "sysctl: 3rd set of kernel/sysctl cleanups", v2. This is the third set of patches to help address cleaning the kitchen seink in kernel/sysctl.c and to move sysctls away to where they are actually implemented / used. This patch (of 8): kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the firmware configuration sysctl table to the only place where it is used, and make it clear that if sysctls are disabled this is not used. [akpm@linux-foundation.org: export register_firmware_config_sysctl and unregister_firmware_config_sysctl to modules] [akpm@linux-foundation.org: use EXPORT_SYMBOL_NS_GPL instead] [sfr@canb.auug.org.au: fix that so it compiles] Link: https://lkml.kernel.org/r/20211201160626.401d828d@canb.auug.org.au [mcgrof@kernel.org: major commit log update to justify the move] Link: https://lkml.kernel.org/r/20211124231435.1445213-1-mcgrof@kernel.org Link: https://lkml.kernel.org/r/20211124231435.1445213-2-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Signed-off-by: Stephen Rothwell Cc: Kees Cook Cc: Iurii Zaikin Cc: Eric Biederman Cc: Stephen Kitt Cc: Greg Kroah-Hartman Cc: "Rafael J. Wysocki" Cc: "Theodore Ts'o" Cc: Al Viro Cc: Petr Mladek Cc: Sergey Senozhatsky Cc: Steven Rostedt (VMware) Cc: John Ogness Cc: Douglas Gilbert Cc: James E.J. Bottomley Cc: Martin K. Petersen Cc: Lukas Middendorf Cc: Antti Palosaari Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: Jani Nikula Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Mark Fasheh Cc: Paul Turner Cc: Peter Zijlstra Cc: Phillip Potter Cc: Qing Wang Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Suren Baghdasaryan Cc: Tetsuo Handa Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/firmware_loader/fallback.c | 7 ++++++- drivers/base/firmware_loader/fallback.h | 11 +++++++++++ drivers/base/firmware_loader/fallback_table.c | 25 +++++++++++++++++++++++-- include/linux/sysctl.h | 1 - kernel/sysctl.c | 7 ------- 5 files changed, 40 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c index d7d63c1aa993..4afb0e9312c0 100644 --- a/drivers/base/firmware_loader/fallback.c +++ b/drivers/base/firmware_loader/fallback.c @@ -199,11 +199,16 @@ static struct class firmware_class = { int register_sysfs_loader(void) { - return class_register(&firmware_class); + int ret = class_register(&firmware_class); + + if (ret != 0) + return ret; + return register_firmware_config_sysctl(); } void unregister_sysfs_loader(void) { + unregister_firmware_config_sysctl(); class_unregister(&firmware_class); } diff --git a/drivers/base/firmware_loader/fallback.h b/drivers/base/firmware_loader/fallback.h index 3af7205b302f..9f3055d3b4ca 100644 --- a/drivers/base/firmware_loader/fallback.h +++ b/drivers/base/firmware_loader/fallback.h @@ -42,6 +42,17 @@ void fw_fallback_set_default_timeout(void); int register_sysfs_loader(void); void unregister_sysfs_loader(void); +#ifdef CONFIG_SYSCTL +extern int register_firmware_config_sysctl(void); +extern void unregister_firmware_config_sysctl(void); +#else +static inline int register_firmware_config_sysctl(void) +{ + return 0; +} +static inline void unregister_firmware_config_sysctl(void) { } +#endif /* CONFIG_SYSCTL */ + #else /* CONFIG_FW_LOADER_USER_HELPER */ static inline int firmware_fallback_sysfs(struct firmware *fw, const char *name, struct device *device, diff --git a/drivers/base/firmware_loader/fallback_table.c b/drivers/base/firmware_loader/fallback_table.c index 46a731dede6f..e5ac098d0742 100644 --- a/drivers/base/firmware_loader/fallback_table.c +++ b/drivers/base/firmware_loader/fallback_table.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,7 @@ struct firmware_fallback_config fw_fallback_config = { EXPORT_SYMBOL_NS_GPL(fw_fallback_config, FIRMWARE_LOADER_PRIVATE); #ifdef CONFIG_SYSCTL -struct ctl_table firmware_config_table[] = { +static struct ctl_table firmware_config_table[] = { { .procname = "force_sysfs_fallback", .data = &fw_fallback_config.force_sysfs_fallback, @@ -45,4 +46,24 @@ struct ctl_table firmware_config_table[] = { }, { } }; -#endif + +static struct ctl_table_header *firmware_config_sysct_table_header; +int register_firmware_config_sysctl(void) +{ + firmware_config_sysct_table_header = + register_sysctl("kernel/firmware_config", + firmware_config_table); + if (!firmware_config_sysct_table_header) + return -ENOMEM; + return 0; +} +EXPORT_SYMBOL_NS_GPL(register_firmware_config_sysctl, FIRMWARE_LOADER_PRIVATE); + +void unregister_firmware_config_sysctl(void) +{ + unregister_sysctl_table(firmware_config_sysct_table_header); + firmware_config_sysct_table_header = NULL; +} +EXPORT_SYMBOL_NS_GPL(unregister_firmware_config_sysctl, FIRMWARE_LOADER_PRIVATE); + +#endif /* CONFIG_SYSCTL */ diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 6dd0f277f844..3985e9c80155 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -218,7 +218,6 @@ extern int no_unaligned_warning; extern struct ctl_table sysctl_mount_point[]; extern struct ctl_table random_table[]; -extern struct ctl_table firmware_config_table[]; #else /* CONFIG_SYSCTL */ static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * table) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 09fa72299f18..5fc3ae16e995 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2154,13 +2154,6 @@ static struct ctl_table kern_table[] = { .mode = 0555, .child = usermodehelper_table, }, -#ifdef CONFIG_FW_LOADER_USER_HELPER - { - .procname = "firmware_config", - .mode = 0555, - .child = firmware_config_table, - }, -#endif { .procname = "overflowuid", .data = &overflowuid, -- cgit v1.2.3-59-g8ed1b From 5475e8f03c80bbce7b43a57d861f5acc44a60b22 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:12:18 -0800 Subject: random: move the random sysctl declarations to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the random sysctls to their own file and use register_sysctl_init(). [mcgrof@kernel.org: commit log update to justify the move] Link: https://lkml.kernel.org/r/20211124231435.1445213-3-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: Douglas Gilbert Cc: Eric Biederman Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: James E.J. Bottomley Cc: Jani Nikula Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: John Ogness Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Martin K. Petersen Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: "Rafael J. Wysocki" Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: Tetsuo Handa Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/random.c | 14 ++++++++++++-- include/linux/sysctl.h | 1 - kernel/sysctl.c | 5 ----- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/char/random.c b/drivers/char/random.c index b411182df6f6..68613f0b6887 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1992,8 +1992,7 @@ static int proc_do_entropy(struct ctl_table *table, int write, void *buffer, } static int sysctl_poolsize = POOL_BITS; -extern struct ctl_table random_table[]; -struct ctl_table random_table[] = { +static struct ctl_table random_table[] = { { .procname = "poolsize", .data = &sysctl_poolsize, @@ -2055,6 +2054,17 @@ struct ctl_table random_table[] = { #endif { } }; + +/* + * rand_initialize() is called before sysctl_init(), + * so we cannot call register_sysctl_init() in rand_initialize() + */ +static int __init random_sysctls_init(void) +{ + register_sysctl_init("kernel/random", random_table); + return 0; +} +device_initcall(random_sysctls_init); #endif /* CONFIG_SYSCTL */ struct batched_entropy { diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 3985e9c80155..fce05a060bc5 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -217,7 +217,6 @@ extern int unaligned_dump_stack; extern int no_unaligned_warning; extern struct ctl_table sysctl_mount_point[]; -extern struct ctl_table random_table[]; #else /* CONFIG_SYSCTL */ static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * table) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5fc3ae16e995..67a5b04f2f84 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2144,11 +2144,6 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sysctl_max_threads, }, - { - .procname = "random", - .mode = 0555, - .child = random_table, - }, { .procname = "usermodehelper", .mode = 0555, -- cgit v1.2.3-59-g8ed1b From ee9efac48a082904d17a20131aa73d82f058cdd6 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:12:23 -0800 Subject: sysctl: add helper to register a sysctl mount point The way to create a subdirectory on top of sysctl_mount_point is a bit obscure, and *why* we do that even so more. Provide a helper which makes it clear why we do this. [akpm@linux-foundation.org: export register_sysctl_mount_point() to modules] Link: https://lkml.kernel.org/r/20211124231435.1445213-4-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Suggested-by: "Eric W. Biederman" Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: Douglas Gilbert Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: James E.J. Bottomley Cc: Jani Nikula Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: John Ogness Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Martin K. Petersen Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: "Rafael J. Wysocki" Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: Tetsuo Handa Cc: "Theodore Ts'o" Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 14 ++++++++++++++ include/linux/sysctl.h | 7 +++++++ 2 files changed, 21 insertions(+) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index cd8bbf13d0f0..6bb2d9a3513d 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -35,6 +35,20 @@ struct ctl_table sysctl_mount_point[] = { { } }; +/** + * register_sysctl_mount_point() - registers a sysctl mount point + * @path: path for the mount point + * + * Used to create a permanently empty directory to serve as mount point. + * There are some subtle but important permission checks this allows in the + * case of unprivileged mounts. + */ +struct ctl_table_header *register_sysctl_mount_point(const char *path) +{ + return register_sysctl(path, sysctl_mount_point); +} +EXPORT_SYMBOL(register_sysctl_mount_point); + static bool is_empty_dir(struct ctl_table_header *head) { return head->ctl_table[0].child == sysctl_mount_point; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index fce05a060bc5..746c098a6ff5 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -209,6 +209,8 @@ extern int sysctl_init(void); extern void __register_sysctl_init(const char *path, struct ctl_table *table, const char *table_name); #define register_sysctl_init(path, table) __register_sysctl_init(path, table, #table) +extern struct ctl_table_header *register_sysctl_mount_point(const char *path); + void do_sysctl_args(void); extern int pwrsw_enabled; @@ -224,6 +226,11 @@ static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * return NULL; } +static inline struct sysctl_header *register_sysctl_mount_point(const char *path) +{ + return NULL; +} + static inline struct ctl_table_header *register_sysctl_paths( const struct ctl_path *path, struct ctl_table *table) { -- cgit v1.2.3-59-g8ed1b From 0df8bdd5e3b3e557ce2c2575fce0c64c5dd1045a Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:12:43 -0800 Subject: stackleak: move stack_erasing sysctl to stackleak.c kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the stack_erasing sysctl from kernel/sysctl.c to kernel/stackleak.c and use register_sysctl() to register the sysctl interface. [mcgrof@kernel.org: commit log update] Link: https://lkml.kernel.org/r/20211124231435.1445213-8-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: Douglas Gilbert Cc: Eric Biederman Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: James E.J. Bottomley Cc: Jani Nikula Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: John Ogness Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Martin K. Petersen Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: "Rafael J. Wysocki" Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: Tetsuo Handa Cc: "Theodore Ts'o" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/stackleak.h | 5 ----- kernel/stackleak.c | 26 ++++++++++++++++++++++++-- kernel/sysctl.c | 14 -------------- 3 files changed, 24 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/stackleak.h b/include/linux/stackleak.h index a59db2f08e76..ccaab2043fcd 100644 --- a/include/linux/stackleak.h +++ b/include/linux/stackleak.h @@ -23,11 +23,6 @@ static inline void stackleak_task_init(struct task_struct *t) # endif } -#ifdef CONFIG_STACKLEAK_RUNTIME_DISABLE -int stack_erasing_sysctl(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); -#endif - #else /* !CONFIG_GCC_PLUGIN_STACKLEAK */ static inline void stackleak_task_init(struct task_struct *t) { } #endif diff --git a/kernel/stackleak.c b/kernel/stackleak.c index ce161a8e8d97..66b8af394e58 100644 --- a/kernel/stackleak.c +++ b/kernel/stackleak.c @@ -16,11 +16,13 @@ #ifdef CONFIG_STACKLEAK_RUNTIME_DISABLE #include #include +#include static DEFINE_STATIC_KEY_FALSE(stack_erasing_bypass); -int stack_erasing_sysctl(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +#ifdef CONFIG_SYSCTL +static int stack_erasing_sysctl(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) { int ret = 0; int state = !static_branch_unlikely(&stack_erasing_bypass); @@ -42,6 +44,26 @@ int stack_erasing_sysctl(struct ctl_table *table, int write, state ? "enabled" : "disabled"); return ret; } +static struct ctl_table stackleak_sysctls[] = { + { + .procname = "stack_erasing", + .data = NULL, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = stack_erasing_sysctl, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static int __init stackleak_sysctls_init(void) +{ + register_sysctl_init("kernel", stackleak_sysctls); + return 0; +} +late_initcall(stackleak_sysctls_init); +#endif /* CONFIG_SYSCTL */ #define skip_erasing() static_branch_unlikely(&stack_erasing_bypass) #else diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b0cced3808d4..3e06dafdd2c3 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -95,9 +95,6 @@ #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_LOCK_STAT) #include #endif -#ifdef CONFIG_STACKLEAK_RUNTIME_DISABLE -#include -#endif #if defined(CONFIG_SYSCTL) @@ -2442,17 +2439,6 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, }, -#endif -#ifdef CONFIG_STACKLEAK_RUNTIME_DISABLE - { - .procname = "stack_erasing", - .data = NULL, - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = stack_erasing_sysctl, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #endif { } }; -- cgit v1.2.3-59-g8ed1b From b1f2aff888af54a057c2c3c0d88a13ef5d37b52a Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:12:48 -0800 Subject: sysctl: share unsigned long const values Provide a way to share unsigned long values. This will allow others to not have to re-invent these values. Link: https://lkml.kernel.org/r/20211124231435.1445213-9-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Amir Goldstein Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Benjamin LaHaise Cc: Clemens Ladisch Cc: David Airlie Cc: Douglas Gilbert Cc: Eric Biederman Cc: Greg Kroah-Hartman Cc: Iurii Zaikin Cc: James E.J. Bottomley Cc: Jani Nikula Cc: Jani Nikula Cc: Jan Kara Cc: Joel Becker Cc: John Ogness Cc: Joonas Lahtinen Cc: Joseph Qi Cc: Julia Lawall Cc: Kees Cook Cc: Lukas Middendorf Cc: Mark Fasheh Cc: Martin K. Petersen Cc: Paul Turner Cc: Peter Zijlstra Cc: Petr Mladek Cc: Phillip Potter Cc: Qing Wang Cc: "Rafael J. Wysocki" Cc: Rodrigo Vivi Cc: Sebastian Reichel Cc: Sergey Senozhatsky Cc: Stephen Kitt Cc: Steven Rostedt (VMware) Cc: Suren Baghdasaryan Cc: Tetsuo Handa Cc: "Theodore Ts'o" Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 3 +++ include/linux/sysctl.h | 6 ++++++ kernel/sysctl.c | 9 +++------ 3 files changed, 12 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 6bb2d9a3513d..2eb5987091ea 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -29,6 +29,9 @@ static const struct inode_operations proc_sys_dir_operations; const int sysctl_vals[] = { -1, 0, 1, 2, 4, 100, 200, 1000, 3000, INT_MAX }; EXPORT_SYMBOL(sysctl_vals); +const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; +EXPORT_SYMBOL_GPL(sysctl_long_vals); + /* Support for permanently empty directories */ struct ctl_table sysctl_mount_point[] = { diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 746c098a6ff5..2de6d20d191b 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -51,6 +51,12 @@ struct ctl_dir; extern const int sysctl_vals[]; +#define SYSCTL_LONG_ZERO ((void *)&sysctl_long_vals[0]) +#define SYSCTL_LONG_ONE ((void *)&sysctl_long_vals[1]) +#define SYSCTL_LONG_MAX ((void *)&sysctl_long_vals[2]) + +extern const unsigned long sysctl_long_vals[]; + typedef int proc_handler(struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3e06dafdd2c3..1b8a1cb8aac9 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -100,9 +100,6 @@ /* Constants used for minimum and maximum */ -static const unsigned long zero_ul; -static const unsigned long one_ul = 1; -static const unsigned long long_max = LONG_MAX; #ifdef CONFIG_PRINTK static const int ten_thousand = 10000; #endif @@ -2513,7 +2510,7 @@ static struct ctl_table vm_table[] = { .maxlen = sizeof(dirty_background_bytes), .mode = 0644, .proc_handler = dirty_background_bytes_handler, - .extra1 = (void *)&one_ul, + .extra1 = SYSCTL_LONG_ONE, }, { .procname = "dirty_ratio", @@ -2931,8 +2928,8 @@ static struct ctl_table fs_table[] = { .maxlen = sizeof(files_stat.max_files), .mode = 0644, .proc_handler = proc_doulongvec_minmax, - .extra1 = (void *)&zero_ul, - .extra2 = (void *)&long_max, + .extra1 = SYSCTL_LONG_ZERO, + .extra2 = SYSCTL_LONG_MAX, }, { .procname = "nr_open", -- cgit v1.2.3-59-g8ed1b From 1d67fe585049d3e2448b997af78c68cbf90ada09 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:12:52 -0800 Subject: fs: move inode sysctls to its own file Patch series "sysctl: 4th set of kernel/sysctl cleanups". This is slimming down the fs uses of kernel/sysctl.c to the point that the next step is to just get rid of the fs base directory for it and move that elsehwere, so that next patch series starts dealing with that to demo how we can end up cleaning up a full base directory from kernel/sysctl.c, one at a time. This patch (of 9): kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the inode sysctls to its own file. Since we are no longer using this outside of fs/ remove the extern declaration of its respective proc helper. We use early_initcall() as it is the earliest we can use. [arnd@arndb.de: avoid unused-variable warning] Link: https://lkml.kernel.org/r/20211203190123.874239-1-arnd@kernel.org Link: https://lkml.kernel.org/r/20211129205548.605569-1-mcgrof@kernel.org Link: https://lkml.kernel.org/r/20211129205548.605569-2-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Signed-off-by: Arnd Bergmann Cc: Al Viro Cc: Kees Cook Cc: Iurii Zaikin Cc: Xiaoming Ni Cc: Eric Biederman Cc: Stephen Kitt Cc: Lukas Middendorf Cc: Antti Palosaari Cc: Andy Shevchenko Cc: Jeff Layton Cc: "J. Bruce Fields" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/inode.c | 39 ++++++++++++++++++++++++++++++++------- include/linux/fs.h | 3 --- kernel/sysctl.c | 14 -------------- 3 files changed, 32 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index 980e7b7a5460..63324df6fa27 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -67,11 +67,6 @@ const struct address_space_operations empty_aops = { }; EXPORT_SYMBOL(empty_aops); -/* - * Statistics gathering.. - */ -struct inodes_stat_t inodes_stat; - static DEFINE_PER_CPU(unsigned long, nr_inodes); static DEFINE_PER_CPU(unsigned long, nr_unused); @@ -106,13 +101,43 @@ long get_nr_dirty_inodes(void) * Handle nr_inode sysctl */ #ifdef CONFIG_SYSCTL -int proc_nr_inodes(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +/* + * Statistics gathering.. + */ +static struct inodes_stat_t inodes_stat; + +static int proc_nr_inodes(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) { inodes_stat.nr_inodes = get_nr_inodes(); inodes_stat.nr_unused = get_nr_inodes_unused(); return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); } + +static struct ctl_table inodes_sysctls[] = { + { + .procname = "inode-nr", + .data = &inodes_stat, + .maxlen = 2*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_inodes, + }, + { + .procname = "inode-state", + .data = &inodes_stat, + .maxlen = 7*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_inodes, + }, + { } +}; + +static int __init init_fs_inode_sysctls(void) +{ + register_sysctl_init("fs", inodes_sysctls); + return 0; +} +early_initcall(init_fs_inode_sysctls); #endif static int no_open(struct inode *inode, struct file *file) diff --git a/include/linux/fs.h b/include/linux/fs.h index c8510da6cc6d..044de67c8167 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -82,7 +82,6 @@ extern void __init files_maxfiles_init(void); extern struct files_stat_struct files_stat; extern unsigned long get_max_files(void); extern unsigned int sysctl_nr_open; -extern struct inodes_stat_t inodes_stat; extern int leases_enable, lease_break_time; extern int sysctl_protected_symlinks; extern int sysctl_protected_hardlinks; @@ -3537,8 +3536,6 @@ int proc_nr_files(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int proc_nr_dentry(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -int proc_nr_inodes(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); int __init list_bdev_fs_names(char *buf, size_t size); #define __FMODE_EXEC ((__force int) FMODE_EXEC) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 1b8a1cb8aac9..402c9d3246bf 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2901,20 +2901,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { - { - .procname = "inode-nr", - .data = &inodes_stat, - .maxlen = 2*sizeof(long), - .mode = 0444, - .proc_handler = proc_nr_inodes, - }, - { - .procname = "inode-state", - .data = &inodes_stat, - .maxlen = 7*sizeof(long), - .mode = 0444, - .proc_handler = proc_nr_inodes, - }, { .procname = "file-nr", .data = &files_stat, -- cgit v1.2.3-59-g8ed1b From 204d5a24e15562b2816825c0f9b49d26814b77be Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:12:56 -0800 Subject: fs: move fs stat sysctls to file_table.c kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. We can create the sysctl dynamically on early init for fs stat to help with this clutter. This dusts off the fs stat syctls knobs and puts them into where they are declared. Link: https://lkml.kernel.org/r/20211129205548.605569-3-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/file_table.c | 47 +++++++++++++++++++++++++++++++++++++++-------- include/linux/fs.h | 3 --- kernel/sysctl.c | 25 ------------------------- 3 files changed, 39 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/fs/file_table.c b/fs/file_table.c index 45437f8e1003..57edef16dce4 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -33,7 +33,7 @@ #include "internal.h" /* sysctl tunables... */ -struct files_stat_struct files_stat = { +static struct files_stat_struct files_stat = { .max_files = NR_FILE }; @@ -75,22 +75,53 @@ unsigned long get_max_files(void) } EXPORT_SYMBOL_GPL(get_max_files); +#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) + /* * Handle nr_files sysctl */ -#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) -int proc_nr_files(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) +static int proc_nr_files(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) { files_stat.nr_files = get_nr_files(); return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); } -#else -int proc_nr_files(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) + +static struct ctl_table fs_stat_sysctls[] = { + { + .procname = "file-nr", + .data = &files_stat, + .maxlen = sizeof(files_stat), + .mode = 0444, + .proc_handler = proc_nr_files, + }, + { + .procname = "file-max", + .data = &files_stat.max_files, + .maxlen = sizeof(files_stat.max_files), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_LONG_ZERO, + .extra2 = SYSCTL_LONG_MAX, + }, + { + .procname = "nr_open", + .data = &sysctl_nr_open, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &sysctl_nr_open_min, + .extra2 = &sysctl_nr_open_max, + }, + { } +}; + +static int __init init_fs_stat_sysctls(void) { - return -ENOSYS; + register_sysctl_init("fs", fs_stat_sysctls); + return 0; } +fs_initcall(init_fs_stat_sysctls); #endif static struct file *__alloc_file(int flags, const struct cred *cred) diff --git a/include/linux/fs.h b/include/linux/fs.h index 044de67c8167..1e6761966120 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -79,7 +79,6 @@ extern void __init inode_init_early(void); extern void __init files_init(void); extern void __init files_maxfiles_init(void); -extern struct files_stat_struct files_stat; extern unsigned long get_max_files(void); extern unsigned int sysctl_nr_open; extern int leases_enable, lease_break_time; @@ -3532,8 +3531,6 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos); struct ctl_table; -int proc_nr_files(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); int proc_nr_dentry(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int __init list_bdev_fs_names(char *buf, size_t size); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 402c9d3246bf..d2c411b15854 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2901,31 +2901,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { - { - .procname = "file-nr", - .data = &files_stat, - .maxlen = sizeof(files_stat), - .mode = 0444, - .proc_handler = proc_nr_files, - }, - { - .procname = "file-max", - .data = &files_stat.max_files, - .maxlen = sizeof(files_stat.max_files), - .mode = 0644, - .proc_handler = proc_doulongvec_minmax, - .extra1 = SYSCTL_LONG_ZERO, - .extra2 = SYSCTL_LONG_MAX, - }, - { - .procname = "nr_open", - .data = &sysctl_nr_open, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &sysctl_nr_open_min, - .extra2 = &sysctl_nr_open_max, - }, { .procname = "dentry-state", .data = &dentry_stat, -- cgit v1.2.3-59-g8ed1b From c8c0c239d5ab1e3e8d2bb0453ce642fe2c6357ec Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:12:59 -0800 Subject: fs: move dcache sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the dcache sysctl clutter out of kernel/sysctl.c. This is a small one-off entry, perhaps later we can simplify this representation, but for now we use the helpers we have. We won't know how we can simplify this further untl we're fully done with the cleanup. [arnd@arndb.de: avoid unused-function warning] Link: https://lkml.kernel.org/r/20211203190123.874239-2-arnd@kernel.org Link: https://lkml.kernel.org/r/20211129205548.605569-4-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Signed-off-by: Arnd Bergmann Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/dcache.c | 37 +++++++++++++++++++++++++++++++------ include/linux/dcache.h | 10 ---------- include/linux/fs.h | 2 -- kernel/sysctl.c | 7 ------- 4 files changed, 31 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/fs/dcache.c b/fs/dcache.c index cf871a81f4fd..c84269c6e8bf 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -115,10 +115,13 @@ static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT); } - -/* Statistics gathering. */ -struct dentry_stat_t dentry_stat = { - .age_limit = 45, +struct dentry_stat_t { + long nr_dentry; + long nr_unused; + long age_limit; /* age in seconds */ + long want_pages; /* pages requested by system */ + long nr_negative; /* # of unused negative dentries */ + long dummy; /* Reserved for future use */ }; static DEFINE_PER_CPU(long, nr_dentry); @@ -126,6 +129,10 @@ static DEFINE_PER_CPU(long, nr_dentry_unused); static DEFINE_PER_CPU(long, nr_dentry_negative); #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) +/* Statistics gathering. */ +static struct dentry_stat_t dentry_stat = { + .age_limit = 45, +}; /* * Here we resort to our own counters instead of using generic per-cpu counters @@ -167,14 +174,32 @@ static long get_nr_dentry_negative(void) return sum < 0 ? 0 : sum; } -int proc_nr_dentry(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos) +static int proc_nr_dentry(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) { dentry_stat.nr_dentry = get_nr_dentry(); dentry_stat.nr_unused = get_nr_dentry_unused(); dentry_stat.nr_negative = get_nr_dentry_negative(); return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); } + +static struct ctl_table fs_dcache_sysctls[] = { + { + .procname = "dentry-state", + .data = &dentry_stat, + .maxlen = 6*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_dentry, + }, + { } +}; + +static int __init init_fs_dcache_sysctls(void) +{ + register_sysctl_init("fs", fs_dcache_sysctls); + return 0; +} +fs_initcall(init_fs_dcache_sysctls); #endif /* diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 9e23d33bb6f1..f5bba51480b2 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -61,16 +61,6 @@ extern const struct qstr empty_name; extern const struct qstr slash_name; extern const struct qstr dotdot_name; -struct dentry_stat_t { - long nr_dentry; - long nr_unused; - long age_limit; /* age in seconds */ - long want_pages; /* pages requested by system */ - long nr_negative; /* # of unused negative dentries */ - long dummy; /* Reserved for future use */ -}; -extern struct dentry_stat_t dentry_stat; - /* * Try to keep struct dentry aligned on 64 byte cachelines (this will * give reasonable cacheline footprint with larger lines without the diff --git a/include/linux/fs.h b/include/linux/fs.h index 1e6761966120..9b856d5da9e2 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3531,8 +3531,6 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, size_t len, loff_t *ppos); struct ctl_table; -int proc_nr_dentry(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); int __init list_bdev_fs_names(char *buf, size_t size); #define __FMODE_EXEC ((__force int) FMODE_EXEC) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index d2c411b15854..4d182fb5c22e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2901,13 +2901,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { - { - .procname = "dentry-state", - .data = &dentry_stat, - .maxlen = 6*sizeof(long), - .mode = 0444, - .proc_handler = proc_nr_dentry, - }, { .procname = "overflowuid", .data = &fs_overflowuid, -- cgit v1.2.3-59-g8ed1b From 54771613e8a7dbbba2a205ddf1b33e25a290b3fd Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:03 -0800 Subject: sysctl: move maxolduid as a sysctl specific const MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maxolduid value is only shared for sysctl purposes for use on a max range. Just stuff this into our shared const array. [akpm@linux-foundation.org: fix sysctl_vals[], per Mickaël] Link: https://lkml.kernel.org/r/20211129205548.605569-5-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Signed-off-by: Mickaël Salaün Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 2 +- include/linux/sysctl.h | 3 +++ kernel/sysctl.c | 12 ++++-------- 3 files changed, 8 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 2eb5987091ea..b9d53b53d769 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -26,7 +26,7 @@ static const struct file_operations proc_sys_dir_file_operations; static const struct inode_operations proc_sys_dir_operations; /* shared constants to be used in various sysctls */ -const int sysctl_vals[] = { -1, 0, 1, 2, 4, 100, 200, 1000, 3000, INT_MAX }; +const int sysctl_vals[] = { -1, 0, 1, 2, 4, 100, 200, 1000, 3000, INT_MAX, 65535 }; EXPORT_SYMBOL(sysctl_vals); const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 2de6d20d191b..bb921eb8a02d 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -49,6 +49,9 @@ struct ctl_dir; #define SYSCTL_THREE_THOUSAND ((void *)&sysctl_vals[8]) #define SYSCTL_INT_MAX ((void *)&sysctl_vals[9]) +/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ +#define SYSCTL_MAXOLDUID ((void *)&sysctl_vals[10]) + extern const int sysctl_vals[]; #define SYSCTL_LONG_ZERO ((void *)&sysctl_long_vals[0]) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 4d182fb5c22e..5bbb4c59dd1a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -110,10 +110,6 @@ static const int six_hundred_forty_kb = 640 * 1024; /* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; -/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */ -static const int maxolduid = 65535; -/* minolduid is SYSCTL_ZERO */ - static const int ngroups_max = NGROUPS_MAX; static const int cap_last_cap = CAP_LAST_CAP; @@ -2127,7 +2123,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&maxolduid, + .extra2 = SYSCTL_MAXOLDUID, }, { .procname = "overflowgid", @@ -2136,7 +2132,7 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&maxolduid, + .extra2 = SYSCTL_MAXOLDUID, }, #ifdef CONFIG_S390 { @@ -2908,7 +2904,7 @@ static struct ctl_table fs_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&maxolduid, + .extra2 = SYSCTL_MAXOLDUID, }, { .procname = "overflowgid", @@ -2917,7 +2913,7 @@ static struct ctl_table fs_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&maxolduid, + .extra2 = SYSCTL_MAXOLDUID, }, #ifdef CONFIG_FILE_LOCKING { -- cgit v1.2.3-59-g8ed1b From dd81faa88340a1fe8cd81c8ecbadd8e95c58549c Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:10 -0800 Subject: fs: move locking sysctls where they are used kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. The locking fs sysctls are only used on fs/locks.c, so move them there. Link: https://lkml.kernel.org/r/20211129205548.605569-7-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/locks.c | 34 ++++++++++++++++++++++++++++++++-- include/linux/fs.h | 4 ---- kernel/sysctl.c | 20 -------------------- 3 files changed, 32 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/fs/locks.c b/fs/locks.c index 0fca9d680978..8c6df10cd9ed 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -62,6 +62,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -88,8 +89,37 @@ static int target_leasetype(struct file_lock *fl) return fl->fl_type; } -int leases_enable = 1; -int lease_break_time = 45; +static int leases_enable = 1; +static int lease_break_time = 45; + +#ifdef CONFIG_SYSCTL +static struct ctl_table locks_sysctls[] = { + { + .procname = "leases-enable", + .data = &leases_enable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_MMU + { + .procname = "lease-break-time", + .data = &lease_break_time, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_MMU */ + {} +}; + +static int __init init_fs_locks_sysctls(void) +{ + register_sysctl_init("fs", locks_sysctls); + return 0; +} +early_initcall(init_fs_locks_sysctls); +#endif /* CONFIG_SYSCTL */ /* * The global file_lock_list is only used for displaying /proc/locks, so we diff --git a/include/linux/fs.h b/include/linux/fs.h index 9b856d5da9e2..0e08c3dd8f75 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -82,10 +82,6 @@ extern void __init files_maxfiles_init(void); extern unsigned long get_max_files(void); extern unsigned int sysctl_nr_open; extern int leases_enable, lease_break_time; -extern int sysctl_protected_symlinks; -extern int sysctl_protected_hardlinks; -extern int sysctl_protected_fifos; -extern int sysctl_protected_regular; typedef __kernel_rwf_t rwf_t; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 6d9d2001b790..073948e9d165 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2897,26 +2897,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { -#ifdef CONFIG_FILE_LOCKING - { - .procname = "leases-enable", - .data = &leases_enable, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif -#ifdef CONFIG_MMU -#ifdef CONFIG_FILE_LOCKING - { - .procname = "lease-break-time", - .data = &lease_break_time, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif -#endif { .procname = "protected_symlinks", .data = &sysctl_protected_symlinks, -- cgit v1.2.3-59-g8ed1b From 9c011be132972ff94bde2ae99064e29f94e85c68 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:13 -0800 Subject: fs: move namei sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move namei's own sysctl knobs to its own file. Other than the move we also avoid initializing two static variables to 0 as this is not needed: * sysctl_protected_symlinks * sysctl_protected_hardlinks Link: https://lkml.kernel.org/r/20211129205548.605569-8-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/namei.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++---- include/linux/fs.h | 1 - kernel/sysctl.c | 36 --------------------------------- 3 files changed, 54 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index d81f04f8d818..b867a92c078e 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1020,10 +1020,60 @@ static inline void put_link(struct nameidata *nd) path_put(&last->link); } -int sysctl_protected_symlinks __read_mostly = 0; -int sysctl_protected_hardlinks __read_mostly = 0; -int sysctl_protected_fifos __read_mostly; -int sysctl_protected_regular __read_mostly; +static int sysctl_protected_symlinks __read_mostly; +static int sysctl_protected_hardlinks __read_mostly; +static int sysctl_protected_fifos __read_mostly; +static int sysctl_protected_regular __read_mostly; + +#ifdef CONFIG_SYSCTL +static struct ctl_table namei_sysctls[] = { + { + .procname = "protected_symlinks", + .data = &sysctl_protected_symlinks, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "protected_hardlinks", + .data = &sysctl_protected_hardlinks, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "protected_fifos", + .data = &sysctl_protected_fifos, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, + { + .procname = "protected_regular", + .data = &sysctl_protected_regular, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, + { } +}; + +static int __init init_fs_namei_sysctls(void) +{ + register_sysctl_init("fs", namei_sysctls); + return 0; +} +fs_initcall(init_fs_namei_sysctls); + +#endif /* CONFIG_SYSCTL */ /** * may_follow_link - Check symlink following for unsafe situations diff --git a/include/linux/fs.h b/include/linux/fs.h index 0e08c3dd8f75..9617dea24978 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -81,7 +81,6 @@ extern void __init files_maxfiles_init(void); extern unsigned long get_max_files(void); extern unsigned int sysctl_nr_open; -extern int leases_enable, lease_break_time; typedef __kernel_rwf_t rwf_t; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 073948e9d165..53fb4692facc 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2897,42 +2897,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { - { - .procname = "protected_symlinks", - .data = &sysctl_protected_symlinks, - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, - { - .procname = "protected_hardlinks", - .data = &sysctl_protected_hardlinks, - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, - { - .procname = "protected_fifos", - .data = &sysctl_protected_fifos, - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, - { - .procname = "protected_regular", - .data = &sysctl_protected_regular, - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, { .procname = "suid_dumpable", .data = &suid_dumpable, -- cgit v1.2.3-59-g8ed1b From 1998f19324d24df7de4e74d81503b4299eb99e7d Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:20 -0800 Subject: fs: move pipe sysctls to is own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the pipe sysctls to its own file. Link: https://lkml.kernel.org/r/20211129205548.605569-10-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Andy Shevchenko Cc: Antti Palosaari Cc: Eric Biederman Cc: Iurii Zaikin Cc: "J. Bruce Fields" Cc: Jeff Layton Cc: Kees Cook Cc: Lukas Middendorf Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/pipe.c | 64 ++++++++++++++++++++++++++++++++++++++++++++--- include/linux/pipe_fs_i.h | 4 --- include/linux/sysctl.h | 6 +++++ kernel/sysctl.c | 61 +++++--------------------------------------- 4 files changed, 73 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/fs/pipe.c b/fs/pipe.c index 6d4342bad9f1..cc28623a67b6 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -50,13 +51,13 @@ * The max size that a non-root user is allowed to grow the pipe. Can * be set by root in /proc/sys/fs/pipe-max-size */ -unsigned int pipe_max_size = 1048576; +static unsigned int pipe_max_size = 1048576; /* Maximum allocatable pages per user. Hard limit is unset by default, soft * matches default values. */ -unsigned long pipe_user_pages_hard; -unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; +static unsigned long pipe_user_pages_hard; +static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; /* * We use head and tail indices that aren't masked off, except at the point of @@ -1428,6 +1429,60 @@ static struct file_system_type pipe_fs_type = { .kill_sb = kill_anon_super, }; +#ifdef CONFIG_SYSCTL +static int do_proc_dopipe_max_size_conv(unsigned long *lvalp, + unsigned int *valp, + int write, void *data) +{ + if (write) { + unsigned int val; + + val = round_pipe_size(*lvalp); + if (val == 0) + return -EINVAL; + + *valp = val; + } else { + unsigned int val = *valp; + *lvalp = (unsigned long) val; + } + + return 0; +} + +static int proc_dopipe_max_size(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + return do_proc_douintvec(table, write, buffer, lenp, ppos, + do_proc_dopipe_max_size_conv, NULL); +} + +static struct ctl_table fs_pipe_sysctls[] = { + { + .procname = "pipe-max-size", + .data = &pipe_max_size, + .maxlen = sizeof(pipe_max_size), + .mode = 0644, + .proc_handler = proc_dopipe_max_size, + }, + { + .procname = "pipe-user-pages-hard", + .data = &pipe_user_pages_hard, + .maxlen = sizeof(pipe_user_pages_hard), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "pipe-user-pages-soft", + .data = &pipe_user_pages_soft, + .maxlen = sizeof(pipe_user_pages_soft), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { } +}; +#endif + static int __init init_pipe_fs(void) { int err = register_filesystem(&pipe_fs_type); @@ -1439,6 +1494,9 @@ static int __init init_pipe_fs(void) unregister_filesystem(&pipe_fs_type); } } +#ifdef CONFIG_SYSCTL + register_sysctl_init("fs", fs_pipe_sysctls); +#endif return err; } diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index fc5642431b92..c00c618ef290 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -238,10 +238,6 @@ void pipe_lock(struct pipe_inode_info *); void pipe_unlock(struct pipe_inode_info *); void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *); -extern unsigned int pipe_max_size; -extern unsigned long pipe_user_pages_hard; -extern unsigned long pipe_user_pages_soft; - /* Wait for a pipe to be readable/writable while dropping the pipe lock */ void pipe_wait_readable(struct pipe_inode_info *); void pipe_wait_writable(struct pipe_inode_info *); diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index bb921eb8a02d..4294e9668bd5 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -221,6 +221,12 @@ extern void __register_sysctl_init(const char *path, struct ctl_table *table, extern struct ctl_table_header *register_sysctl_mount_point(const char *path); void do_sysctl_args(void); +int do_proc_douintvec(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos, + int (*conv)(unsigned long *lvalp, + unsigned int *valp, + int write, void *data), + void *data); extern int pwrsw_enabled; extern int unaligned_enabled; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 8bab82aa0192..f79ec2cf8721 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -57,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -761,12 +760,12 @@ static int __do_proc_douintvec(void *tbl_data, struct ctl_table *table, return do_proc_douintvec_r(i, buffer, lenp, ppos, conv, data); } -static int do_proc_douintvec(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos, - int (*conv)(unsigned long *lvalp, - unsigned int *valp, - int write, void *data), - void *data) +int do_proc_douintvec(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos, + int (*conv)(unsigned long *lvalp, + unsigned int *valp, + int write, void *data), + void *data) { return __do_proc_douintvec(table->data, table, write, buffer, lenp, ppos, conv, data); @@ -1090,33 +1089,6 @@ int proc_dou8vec_minmax(struct ctl_table *table, int write, } EXPORT_SYMBOL_GPL(proc_dou8vec_minmax); -static int do_proc_dopipe_max_size_conv(unsigned long *lvalp, - unsigned int *valp, - int write, void *data) -{ - if (write) { - unsigned int val; - - val = round_pipe_size(*lvalp); - if (val == 0) - return -EINVAL; - - *valp = val; - } else { - unsigned int val = *valp; - *lvalp = (unsigned long) val; - } - - return 0; -} - -static int proc_dopipe_max_size(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - return do_proc_douintvec(table, write, buffer, lenp, ppos, - do_proc_dopipe_max_size_conv, NULL); -} - #ifdef CONFIG_MAGIC_SYSRQ static int sysrq_sysctl_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) @@ -2840,27 +2812,6 @@ static struct ctl_table vm_table[] = { }; static struct ctl_table fs_table[] = { - { - .procname = "pipe-max-size", - .data = &pipe_max_size, - .maxlen = sizeof(pipe_max_size), - .mode = 0644, - .proc_handler = proc_dopipe_max_size, - }, - { - .procname = "pipe-user-pages-hard", - .data = &pipe_user_pages_hard, - .maxlen = sizeof(pipe_user_pages_hard), - .mode = 0644, - .proc_handler = proc_doulongvec_minmax, - }, - { - .procname = "pipe-user-pages-soft", - .data = &pipe_user_pages_soft, - .maxlen = sizeof(pipe_user_pages_soft), - .mode = 0644, - .proc_handler = proc_doulongvec_minmax, - }, { .procname = "mount-max", .data = &sysctl_mount_max, -- cgit v1.2.3-59-g8ed1b From 51cb8dfc5a5c39e6c70376b9dc9a14d624a9d271 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:24 -0800 Subject: sysctl: add and use base directory declarer and registration helper Patch series "sysctl: add and use base directory declarer and registration helper". In this patch series we start addressing base directories, and so we start with the "fs" sysctls. The end goal is we end up completely moving all "fs" sysctl knobs out from kernel/sysctl. This patch (of 6): Add a set of helpers which can be used to declare and register base directory sysctls on their own. We do this so we can later move each of the base sysctl directories like "fs", "kernel", etc, to their own respective files instead of shoving the declarations and registrations all on kernel/sysctl.c. The lazy approach has caught up and with this, we just end up extending the list of base directories / sysctls on one file and this makes maintenance difficult due to merge conflicts from many developers. The declarations are used first by kernel/sysctl.c for registration its own base which over time we'll try to clean up. It will be used in the next patch to demonstrate how to cleanly deal with base sysctl directories. [mcgrof@kernel.org: null-terminate the ctl_table arrays] Link: https://lkml.kernel.org/r/YafJY3rXDYnjK/gs@bombadil.infradead.org Link: https://lkml.kernel.org/r/20211129211943.640266-1-mcgrof@kernel.org Link: https://lkml.kernel.org/r/20211129211943.640266-2-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Kees Cook Cc: Iurii Zaikin Cc: Xiaoming Ni Cc: Eric Biederman Cc: Stephen Kitt Cc: Lukas Middendorf Cc: Antti Palosaari Cc: Christian Brauner Cc: Eric Biggers Cc: "Naveen N. Rao" Cc: "David S. Miller" Cc: Masami Hiramatsu Cc: Anil S Keshavamurthy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/proc_sysctl.c | 9 +++++++++ include/linux/sysctl.h | 24 ++++++++++++++++++++++++ kernel/sysctl.c | 41 ++++++++++------------------------------- 3 files changed, 43 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index b9d53b53d769..95dfd195e04e 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -1646,6 +1646,15 @@ struct ctl_table_header *register_sysctl_table(struct ctl_table *table) } EXPORT_SYMBOL(register_sysctl_table); +int __register_sysctl_base(struct ctl_table *base_table) +{ + struct ctl_table_header *hdr; + + hdr = register_sysctl_table(base_table); + kmemleak_not_leak(hdr); + return 0; +} + static void put_links(struct ctl_table_header *header) { struct ctl_table_set *root_set = &sysctl_table_root.default_set; diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 4294e9668bd5..02134a8abad7 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -194,6 +194,20 @@ struct ctl_path { #ifdef CONFIG_SYSCTL +#define DECLARE_SYSCTL_BASE(_name, _table) \ +static struct ctl_table _name##_base_table[] = { \ + { \ + .procname = #_name, \ + .mode = 0555, \ + .child = _table, \ + }, \ + { }, \ +} + +extern int __register_sysctl_base(struct ctl_table *base_table); + +#define register_sysctl_base(_name) __register_sysctl_base(_name##_base_table) + void proc_sys_poll_notify(struct ctl_table_poll *poll); extern void setup_sysctl_set(struct ctl_table_set *p, @@ -236,6 +250,16 @@ extern int no_unaligned_warning; extern struct ctl_table sysctl_mount_point[]; #else /* CONFIG_SYSCTL */ + +#define DECLARE_SYSCTL_BASE(_name, _table) + +static inline int __register_sysctl_base(struct ctl_table *base_table) +{ + return 0; +} + +#define register_sysctl_base(table) __register_sysctl_base(table) + static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * table) { return NULL; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f79ec2cf8721..12456a535059 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2851,41 +2851,20 @@ static struct ctl_table dev_table[] = { { } }; -static struct ctl_table sysctl_base_table[] = { - { - .procname = "kernel", - .mode = 0555, - .child = kern_table, - }, - { - .procname = "vm", - .mode = 0555, - .child = vm_table, - }, - { - .procname = "fs", - .mode = 0555, - .child = fs_table, - }, - { - .procname = "debug", - .mode = 0555, - .child = debug_table, - }, - { - .procname = "dev", - .mode = 0555, - .child = dev_table, - }, - { } -}; +DECLARE_SYSCTL_BASE(kernel, kern_table); +DECLARE_SYSCTL_BASE(vm, vm_table); +DECLARE_SYSCTL_BASE(fs, fs_table); +DECLARE_SYSCTL_BASE(debug, debug_table); +DECLARE_SYSCTL_BASE(dev, dev_table); int __init sysctl_init(void) { - struct ctl_table_header *hdr; + register_sysctl_base(kernel); + register_sysctl_base(vm); + register_sysctl_base(fs); + register_sysctl_base(debug); + register_sysctl_base(dev); - hdr = register_sysctl_table(sysctl_base_table); - kmemleak_not_leak(hdr); return 0; } #endif /* CONFIG_SYSCTL */ -- cgit v1.2.3-59-g8ed1b From ab171b952c6e065779687b44041038efdadb3915 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:27 -0800 Subject: fs: move namespace sysctls and declare fs base directory This moves the namespace sysctls to its own file as part of the kernel/sysctl.c spring cleaning Since we have now removed all sysctls for "fs", we now have to declare it on the filesystem code, we do that using the new helper, which reduces boiler plate code. We rename init_fs_shared_sysctls() to init_fs_sysctls() to reflect that now fs/sysctls.c is taking on the burden of being the first to register the base directory as well. Lastly, since init code will load in the order in which we link it we have to move the sysctl code to be linked in early, so that its early init routine runs prior to other fs code. This way, other filesystem code can register their own sysctls using the helpers after this: * register_sysctl_init() * register_sysctl() Link: https://lkml.kernel.org/r/20211129211943.640266-3-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Anil S Keshavamurthy Cc: Antti Palosaari Cc: Christian Brauner Cc: "David S. Miller" Cc: Eric Biederman Cc: Eric Biggers Cc: Iurii Zaikin Cc: Kees Cook Cc: Lukas Middendorf Cc: Masami Hiramatsu Cc: "Naveen N. Rao" Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/Makefile | 3 ++- fs/namespace.c | 24 +++++++++++++++++++++++- fs/sysctls.c | 9 +++++---- include/linux/mount.h | 3 --- kernel/sysctl.c | 14 -------------- 5 files changed, 30 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/fs/Makefile b/fs/Makefile index ea8770d124da..dab324aea08f 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -6,6 +6,8 @@ # Rewritten to use lists instead of if-statements. # +obj-$(CONFIG_SYSCTL) += sysctls.o + obj-y := open.o read_write.o file_table.o super.o \ char_dev.o stat.o exec.o pipe.o namei.o fcntl.o \ ioctl.o readdir.o select.o dcache.o inode.o \ @@ -28,7 +30,6 @@ obj-y += notify/ obj-$(CONFIG_EPOLL) += eventpoll.o obj-y += anon_inodes.o obj-$(CONFIG_SIGNALFD) += signalfd.o -obj-$(CONFIG_SYSCTL) += sysctls.o obj-$(CONFIG_TIMERFD) += timerfd.o obj-$(CONFIG_EVENTFD) += eventfd.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o diff --git a/fs/namespace.c b/fs/namespace.c index dc31ad6b370f..40b994a29e90 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -37,7 +37,7 @@ #include "internal.h" /* Maximum number of mounts in a mount namespace */ -unsigned int sysctl_mount_max __read_mostly = 100000; +static unsigned int sysctl_mount_max __read_mostly = 100000; static unsigned int m_hash_mask __read_mostly; static unsigned int m_hash_shift __read_mostly; @@ -4620,3 +4620,25 @@ const struct proc_ns_operations mntns_operations = { .install = mntns_install, .owner = mntns_owner, }; + +#ifdef CONFIG_SYSCTL +static struct ctl_table fs_namespace_sysctls[] = { + { + .procname = "mount-max", + .data = &sysctl_mount_max, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + }, + { } +}; + +static int __init init_fs_namespace_sysctls(void) +{ + register_sysctl_init("fs", fs_namespace_sysctls); + return 0; +} +fs_initcall(init_fs_namespace_sysctls); + +#endif /* CONFIG_SYSCTL */ diff --git a/fs/sysctls.c b/fs/sysctls.c index 54216cd1ecd7..c701273c9432 100644 --- a/fs/sysctls.c +++ b/fs/sysctls.c @@ -29,10 +29,11 @@ static struct ctl_table fs_shared_sysctls[] = { { } }; -static int __init init_fs_shared_sysctls(void) +DECLARE_SYSCTL_BASE(fs, fs_shared_sysctls); + +static int __init init_fs_sysctls(void) { - register_sysctl_init("fs", fs_shared_sysctls); - return 0; + return register_sysctl_base(fs); } -early_initcall(init_fs_shared_sysctls); +early_initcall(init_fs_sysctls); diff --git a/include/linux/mount.h b/include/linux/mount.h index 5d92a7e1a742..7f18a7555dff 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -113,9 +113,6 @@ extern void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list); extern void mark_mounts_for_expiry(struct list_head *mounts); extern dev_t name_to_dev_t(const char *name); - -extern unsigned int sysctl_mount_max; - extern bool path_is_mountpoint(const struct path *path); extern void kern_unmount_array(struct vfsmount *mnt[], unsigned int num); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 12456a535059..cdc3dc5f0005 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2811,18 +2811,6 @@ static struct ctl_table vm_table[] = { { } }; -static struct ctl_table fs_table[] = { - { - .procname = "mount-max", - .data = &sysctl_mount_max, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ONE, - }, - { } -}; - static struct ctl_table debug_table[] = { #ifdef CONFIG_SYSCTL_EXCEPTION_TRACE { @@ -2853,7 +2841,6 @@ static struct ctl_table dev_table[] = { DECLARE_SYSCTL_BASE(kernel, kern_table); DECLARE_SYSCTL_BASE(vm, vm_table); -DECLARE_SYSCTL_BASE(fs, fs_table); DECLARE_SYSCTL_BASE(debug, debug_table); DECLARE_SYSCTL_BASE(dev, dev_table); @@ -2861,7 +2848,6 @@ int __init sysctl_init(void) { register_sysctl_base(kernel); register_sysctl_base(vm); - register_sysctl_base(fs); register_sysctl_base(debug); register_sysctl_base(dev); -- cgit v1.2.3-59-g8ed1b From d8c0418aac78e661b5283c9d6a1dfc61d44f26fd Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 21 Jan 2022 22:13:31 -0800 Subject: kernel/sysctl.c: rename sysctl_init() to sysctl_init_bases() Rename sysctl_init() to sysctl_init_bases() so to reflect exactly what this is doing. Link: https://lkml.kernel.org/r/20211129211943.640266-4-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Anil S Keshavamurthy Cc: Antti Palosaari Cc: Christian Brauner Cc: "David S. Miller" Cc: Eric Biederman Cc: Eric Biggers Cc: Iurii Zaikin Cc: Kees Cook Cc: Lukas Middendorf Cc: Masami Hiramatsu Cc: "Naveen N. Rao" Cc: Stephen Kitt Cc: Xiaoming Ni Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm/mm/alignment.c | 2 +- arch/sh/mm/alignment.c | 2 +- fs/proc/proc_sysctl.c | 4 ++-- include/linux/sysctl.h | 2 +- kernel/sysctl.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index adbb3817d0be..6f499559d193 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -1005,7 +1005,7 @@ static int __init noalign_setup(char *__unused) __setup("noalign", noalign_setup); /* - * This needs to be done after sysctl_init, otherwise sys/ will be + * This needs to be done after sysctl_init_bases(), otherwise sys/ will be * overwritten. Actually, this shouldn't be in sys/ at all since * it isn't a sysctl, and it doesn't contain sysctl information. * We now locate it in /proc/cpu/alignment instead. diff --git a/arch/sh/mm/alignment.c b/arch/sh/mm/alignment.c index fb517b82a87b..d802801ceb93 100644 --- a/arch/sh/mm/alignment.c +++ b/arch/sh/mm/alignment.c @@ -161,7 +161,7 @@ static const struct proc_ops alignment_proc_ops = { }; /* - * This needs to be done after sysctl_init, otherwise sys/ will be + * This needs to be done after sysctl_init_bases(), otherwise sys/ will be * overwritten. Actually, this shouldn't be in sys/ at all since * it isn't a sysctl, and it doesn't contain sysctl information. * We now locate it in /proc/cpu/alignment instead. diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 95dfd195e04e..7d9cfc730bd4 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -1419,7 +1419,7 @@ EXPORT_SYMBOL(register_sysctl); * Context: Can only be called after your respective sysctl base path has been * registered. So for instance, most base directories are registered early on * init before init levels are processed through proc_sys_init() and - * sysctl_init(). + * sysctl_init_bases(). */ void __init __register_sysctl_init(const char *path, struct ctl_table *table, const char *table_name) @@ -1768,7 +1768,7 @@ int __init proc_sys_init(void) proc_sys_root->proc_dir_ops = &proc_sys_dir_file_operations; proc_sys_root->nlink = 0; - return sysctl_init(); + return sysctl_init_bases(); } struct sysctl_alias { diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 02134a8abad7..180adf7da785 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -228,7 +228,7 @@ struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, void unregister_sysctl_table(struct ctl_table_header * table); -extern int sysctl_init(void); +extern int sysctl_init_bases(void); extern void __register_sysctl_init(const char *path, struct ctl_table *table, const char *table_name); #define register_sysctl_init(path, table) __register_sysctl_init(path, table, #table) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index cdc3dc5f0005..bb07183cd305 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2844,7 +2844,7 @@ DECLARE_SYSCTL_BASE(vm, vm_table); DECLARE_SYSCTL_BASE(debug, debug_table); DECLARE_SYSCTL_BASE(dev, dev_table); -int __init sysctl_init(void) +int __init sysctl_init_bases(void) { register_sysctl_base(kernel); register_sysctl_base(vm); -- cgit v1.2.3-59-g8ed1b From fdcd4073fccc6f989308be3f1d61d8a68cd990ce Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:13:34 -0800 Subject: printk: fix build warning when CONFIG_PRINTK=n build warning when CONFIG_PRINTK=n kernel/printk/printk.c:175:5: warning: no previous prototype for 'devkmsg_sysctl_set_loglvl' [-Wmissing-prototypes] devkmsg_sysctl_set_loglvl() is only used in sysctl.c when CONFIG_PRINTK=y, but it participates in the build when CONFIG_PRINTK=n. So add compile dependency CONFIG_PRINTK=y && CONFIG_SYSCTL=y to fix the build warning. Link: https://lkml.kernel.org/r/20211129211943.640266-5-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Anil S Keshavamurthy Cc: Antti Palosaari Cc: Christian Brauner Cc: "David S. Miller" Cc: Eric Biederman Cc: Eric Biggers Cc: Iurii Zaikin Cc: Kees Cook Cc: Lukas Middendorf Cc: Masami Hiramatsu Cc: "Naveen N. Rao" Cc: Stephen Kitt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/printk.h | 4 ---- kernel/printk/internal.h | 2 ++ kernel/printk/printk.c | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/printk.h b/include/linux/printk.h index 9497f6b98339..1522df223c0f 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -183,10 +183,6 @@ extern bool printk_timed_ratelimit(unsigned long *caller_jiffies, extern int printk_delay_msec; extern int dmesg_restrict; -extern int -devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, void *buf, - size_t *lenp, loff_t *ppos); - extern void wake_up_klogd(void); char *log_buf_addr_get(void); diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 6b1c4b399845..d947ca6c84f9 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -6,6 +6,8 @@ #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) void __init printk_sysctl_init(void); +int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos); #else #define printk_sysctl_init() do { } while (0) #endif diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 363a493bded8..82abfaf3c2aa 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -171,7 +171,7 @@ static int __init control_devkmsg(char *str) __setup("printk.devkmsg=", control_devkmsg); char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit"; - +#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -210,6 +210,7 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, return 0; } +#endif /* CONFIG_PRINTK && CONFIG_SYSCTL */ /* Number of registered extended console drivers. */ static int nr_ext_console_drivers; -- cgit v1.2.3-59-g8ed1b From f0bc21b268c1464603192a00851cdbbf7c2cdc36 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:13:38 -0800 Subject: fs/coredump: move coredump sysctls into its own file This moves the fs/coredump.c respective sysctls to its own file. Link: https://lkml.kernel.org/r/20211129211943.640266-6-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Anil S Keshavamurthy Cc: Antti Palosaari Cc: Christian Brauner Cc: "David S. Miller" Cc: Eric Biederman Cc: Eric Biggers Cc: Iurii Zaikin Cc: Kees Cook Cc: Lukas Middendorf Cc: Masami Hiramatsu Cc: "Naveen N. Rao" Cc: Stephen Kitt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/coredump.c | 66 ++++++++++++++++++++++++++++++++++++++++++++---- fs/exec.c | 55 ---------------------------------------- include/linux/coredump.h | 10 +++++--- kernel/sysctl.c | 2 -- 4 files changed, 67 insertions(+), 66 deletions(-) (limited to 'include/linux') diff --git a/fs/coredump.c b/fs/coredump.c index 7dece20b162b..1c060c0a2d72 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -52,9 +53,9 @@ #include -int core_uses_pid; -unsigned int core_pipe_limit; -char core_pattern[CORENAME_MAX_SIZE] = "core"; +static int core_uses_pid; +static unsigned int core_pipe_limit; +static char core_pattern[CORENAME_MAX_SIZE] = "core"; static int core_name_size = CORENAME_MAX_SIZE; struct core_name { @@ -62,8 +63,6 @@ struct core_name { int used, size; }; -/* The maximal length of core_pattern is also specified in sysctl.c */ - static int expand_corename(struct core_name *cn, int size) { char *corename = krealloc(cn->corename, size, GFP_KERNEL); @@ -893,6 +892,63 @@ int dump_align(struct coredump_params *cprm, int align) } EXPORT_SYMBOL(dump_align); +#ifdef CONFIG_SYSCTL + +void validate_coredump_safety(void) +{ + if (suid_dumpable == SUID_DUMP_ROOT && + core_pattern[0] != '/' && core_pattern[0] != '|') { + pr_warn( +"Unsafe core_pattern used with fs.suid_dumpable=2.\n" +"Pipe handler or fully qualified core dump path required.\n" +"Set kernel.core_pattern before fs.suid_dumpable.\n" + ); + } +} + +static int proc_dostring_coredump(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int error = proc_dostring(table, write, buffer, lenp, ppos); + + if (!error) + validate_coredump_safety(); + return error; +} + +static struct ctl_table coredump_sysctls[] = { + { + .procname = "core_uses_pid", + .data = &core_uses_pid, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "core_pattern", + .data = core_pattern, + .maxlen = CORENAME_MAX_SIZE, + .mode = 0644, + .proc_handler = proc_dostring_coredump, + }, + { + .procname = "core_pipe_limit", + .data = &core_pipe_limit, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { } +}; + +static int __init init_fs_coredump_sysctls(void) +{ + register_sysctl_init("kernel", coredump_sysctls); + return 0; +} +fs_initcall(init_fs_coredump_sysctls); +#endif /* CONFIG_SYSCTL */ + /* * The purpose of always_dump_vma() is to make sure that special kernel mappings * that are useful for post-mortem analysis are included in every core dump. diff --git a/fs/exec.c b/fs/exec.c index 310750340e4a..79f2c9483302 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -2103,20 +2103,6 @@ COMPAT_SYSCALL_DEFINE5(execveat, int, fd, #ifdef CONFIG_SYSCTL -static void validate_coredump_safety(void) -{ -#ifdef CONFIG_COREDUMP - if (suid_dumpable == SUID_DUMP_ROOT && - core_pattern[0] != '/' && core_pattern[0] != '|') { - pr_warn( -"Unsafe core_pattern used with fs.suid_dumpable=2.\n" -"Pipe handler or fully qualified core dump path required.\n" -"Set kernel.core_pattern before fs.suid_dumpable.\n" - ); - } -#endif -} - static int proc_dointvec_minmax_coredump(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -2140,50 +2126,9 @@ static struct ctl_table fs_exec_sysctls[] = { { } }; -#ifdef CONFIG_COREDUMP - -static int proc_dostring_coredump(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - int error = proc_dostring(table, write, buffer, lenp, ppos); - - if (!error) - validate_coredump_safety(); - return error; -} - -static struct ctl_table kernel_exec_sysctls[] = { - { - .procname = "core_uses_pid", - .data = &core_uses_pid, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "core_pattern", - .data = core_pattern, - .maxlen = CORENAME_MAX_SIZE, - .mode = 0644, - .proc_handler = proc_dostring_coredump, - }, - { - .procname = "core_pipe_limit", - .data = &core_pipe_limit, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { } -}; -#endif - static int __init init_fs_exec_sysctls(void) { register_sysctl_init("fs", fs_exec_sysctls); -#ifdef CONFIG_COREDUMP - register_sysctl_init("kernel", kernel_exec_sysctls); -#endif return 0; } diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 78fcd776b185..248a68c668b4 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -14,10 +14,6 @@ struct core_vma_metadata { unsigned long dump_size; }; -extern int core_uses_pid; -extern char core_pattern[]; -extern unsigned int core_pipe_limit; - /* * These are the only things you should do on a core-file: use only these * functions to write out all the necessary info. @@ -37,4 +33,10 @@ extern void do_coredump(const kernel_siginfo_t *siginfo); static inline void do_coredump(const kernel_siginfo_t *siginfo) {} #endif +#if defined(CONFIG_COREDUMP) && defined(CONFIG_SYSCTL) +extern void validate_coredump_safety(void); +#else +static inline void validate_coredump_safety(void) {} +#endif + #endif /* _LINUX_COREDUMP_H */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index bb07183cd305..c78585292005 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -62,12 +62,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include -- cgit v1.2.3-59-g8ed1b From a737a3c6744bc822d1e6a837fef550e665ddf877 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Fri, 21 Jan 2022 22:13:41 -0800 Subject: kprobe: move sysctl_kprobes_optimization to kprobes.c kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. Move sysctl_kprobes_optimization from kernel/sysctl.c to kernel/kprobes.c. Use register_sysctl() to register the sysctl interface. [mcgrof@kernel.org: fix compile issue when CONFIG_OPTPROBES is disabled] Link: https://lkml.kernel.org/r/20211129211943.640266-7-mcgrof@kernel.org Signed-off-by: Xiaoming Ni Signed-off-by: Luis Chamberlain Cc: Al Viro Cc: Anil S Keshavamurthy Cc: Antti Palosaari Cc: Christian Brauner Cc: "David S. Miller" Cc: Eric Biederman Cc: Eric Biggers Cc: Iurii Zaikin Cc: Kees Cook Cc: Lukas Middendorf Cc: Masami Hiramatsu Cc: "Naveen N. Rao" Cc: Stephen Kitt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kprobes.h | 6 ------ kernel/kprobes.c | 30 ++++++++++++++++++++++++++---- kernel/sysctl.c | 12 ------------ 3 files changed, 26 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 8c8f7a4d93af..19b884353b15 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -348,12 +348,6 @@ extern void opt_pre_handler(struct kprobe *p, struct pt_regs *regs); DEFINE_INSN_CACHE_OPS(optinsn); -#ifdef CONFIG_SYSCTL -extern int sysctl_kprobes_optimization; -extern int proc_kprobes_optimization_handler(struct ctl_table *table, - int write, void *buffer, - size_t *length, loff_t *ppos); -#endif /* CONFIG_SYSCTL */ extern void wait_for_kprobe_optimizer(void); #else /* !CONFIG_OPTPROBES */ static inline void wait_for_kprobe_optimizer(void) { } diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 21eccc961bba..94cab8c9ce56 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -48,6 +48,9 @@ #define KPROBE_HASH_BITS 6 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) +#if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL) +#define kprobe_sysctls_init() do { } while (0) +#endif static int kprobes_initialized; /* kprobe_table can be accessed by @@ -938,10 +941,10 @@ static void unoptimize_all_kprobes(void) } static DEFINE_MUTEX(kprobe_sysctl_mutex); -int sysctl_kprobes_optimization; -int proc_kprobes_optimization_handler(struct ctl_table *table, int write, - void *buffer, size_t *length, - loff_t *ppos) +static int sysctl_kprobes_optimization; +static int proc_kprobes_optimization_handler(struct ctl_table *table, + int write, void *buffer, + size_t *length, loff_t *ppos) { int ret; @@ -957,6 +960,24 @@ int proc_kprobes_optimization_handler(struct ctl_table *table, int write, return ret; } + +static struct ctl_table kprobe_sysctls[] = { + { + .procname = "kprobes-optimization", + .data = &sysctl_kprobes_optimization, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_kprobes_optimization_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static void __init kprobe_sysctls_init(void) +{ + register_sysctl_init("debug", kprobe_sysctls); +} #endif /* CONFIG_SYSCTL */ /* Put a breakpoint for a probe. */ @@ -2584,6 +2605,7 @@ static int __init init_kprobes(void) err = register_module_notifier(&kprobe_module_nb); kprobes_initialized = (err == 0); + kprobe_sysctls_init(); return err; } early_initcall(init_kprobes); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c78585292005..c322bb629d10 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -2818,17 +2817,6 @@ static struct ctl_table debug_table[] = { .mode = 0644, .proc_handler = proc_dointvec }, -#endif -#if defined(CONFIG_OPTPROBES) - { - .procname = "kprobes-optimization", - .data = &sysctl_kprobes_optimization, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_kprobes_optimization_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #endif { } }; -- cgit v1.2.3-59-g8ed1b From 4a57d6bbaecd28c8175dc5da013009e4158018c2 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Fri, 21 Jan 2022 22:14:10 -0800 Subject: locking/rwlocks: introduce write_lock_nested In preparation for converting bit_spin_lock to rwlock in zsmalloc so that multiple writers of zspages can run at the same time but those zspages are supposed to be different zspage instance. Thus, it's not deadlock. This patch adds write_lock_nested to support the case for LOCKDEP. [minchan@kernel.org: fix write_lock_nested for RT] Link: https://lkml.kernel.org/r/YZfrMTAXV56HFWJY@google.com [bigeasy@linutronix.de: fixup write_lock_nested() implementation] Link: https://lkml.kernel.org/r/20211123170134.y6xb7pmpgdn4m3bn@linutronix.de Link: https://lkml.kernel.org/r/20211115185909.3949505-8-minchan@kernel.org Signed-off-by: Minchan Kim Signed-off-by: Sebastian Andrzej Siewior Acked-by: Peter Zijlstra (Intel) Acked-by: Sebastian Andrzej Siewior Tested-by: Sebastian Andrzej Siewior Cc: Mike Galbraith Cc: Sergey Senozhatsky Cc: Thomas Gleixner Cc: Naresh Kamboju Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rwlock.h | 6 ++++++ include/linux/rwlock_api_smp.h | 8 ++++++++ include/linux/rwlock_rt.h | 10 ++++++++++ include/linux/spinlock_api_up.h | 1 + kernel/locking/spinlock.c | 10 ++++++++++ kernel/locking/spinlock_rt.c | 12 ++++++++++++ 6 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 2c0ad417ce3c..8f416c5e929e 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -55,6 +55,12 @@ do { \ #define write_lock(lock) _raw_write_lock(lock) #define read_lock(lock) _raw_read_lock(lock) +#ifdef CONFIG_DEBUG_LOCK_ALLOC +#define write_lock_nested(lock, subclass) _raw_write_lock_nested(lock, subclass) +#else +#define write_lock_nested(lock, subclass) _raw_write_lock(lock) +#endif + #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) #define read_lock_irqsave(lock, flags) \ diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index f1db6f17c4fb..dceb0a59b692 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -17,6 +17,7 @@ void __lockfunc _raw_read_lock(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_write_lock(rwlock_t *lock) __acquires(lock); +void __lockfunc _raw_write_lock_nested(rwlock_t *lock, int subclass) __acquires(lock); void __lockfunc _raw_read_lock_bh(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_write_lock_bh(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires(lock); @@ -209,6 +210,13 @@ static inline void __raw_write_lock(rwlock_t *lock) LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); } +static inline void __raw_write_lock_nested(rwlock_t *lock, int subclass) +{ + preempt_disable(); + rwlock_acquire(&lock->dep_map, subclass, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_write_trylock, do_raw_write_lock); +} + #endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */ static inline void __raw_write_unlock(rwlock_t *lock) diff --git a/include/linux/rwlock_rt.h b/include/linux/rwlock_rt.h index 49c1f3842ed5..8544ff05e594 100644 --- a/include/linux/rwlock_rt.h +++ b/include/linux/rwlock_rt.h @@ -28,6 +28,7 @@ extern void rt_read_lock(rwlock_t *rwlock); extern int rt_read_trylock(rwlock_t *rwlock); extern void rt_read_unlock(rwlock_t *rwlock); extern void rt_write_lock(rwlock_t *rwlock); +extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass); extern int rt_write_trylock(rwlock_t *rwlock); extern void rt_write_unlock(rwlock_t *rwlock); @@ -83,6 +84,15 @@ static __always_inline void write_lock(rwlock_t *rwlock) rt_write_lock(rwlock); } +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static __always_inline void write_lock_nested(rwlock_t *rwlock, int subclass) +{ + rt_write_lock_nested(rwlock, subclass); +} +#else +#define write_lock_nested(lock, subclass) rt_write_lock(((void)(subclass), (lock))) +#endif + static __always_inline void write_lock_bh(rwlock_t *rwlock) { local_bh_disable(); diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index d0d188861ad6..b8ba00ccccde 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -59,6 +59,7 @@ #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock) #define _raw_write_lock(lock) __LOCK(lock) +#define _raw_write_lock_nested(lock, subclass) __LOCK(lock) #define _raw_spin_lock_bh(lock) __LOCK_BH(lock) #define _raw_read_lock_bh(lock) __LOCK_BH(lock) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b562f9289372..7f49baaa4979 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -300,6 +300,16 @@ void __lockfunc _raw_write_lock(rwlock_t *lock) __raw_write_lock(lock); } EXPORT_SYMBOL(_raw_write_lock); + +#ifndef CONFIG_DEBUG_LOCK_ALLOC +#define __raw_write_lock_nested(lock, subclass) __raw_write_lock(((void)(subclass), (lock))) +#endif + +void __lockfunc _raw_write_lock_nested(rwlock_t *lock, int subclass) +{ + __raw_write_lock_nested(lock, subclass); +} +EXPORT_SYMBOL(_raw_write_lock_nested); #endif #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c index 9e396a09fe0f..48a19ed8486d 100644 --- a/kernel/locking/spinlock_rt.c +++ b/kernel/locking/spinlock_rt.c @@ -239,6 +239,18 @@ void __sched rt_write_lock(rwlock_t *rwlock) } EXPORT_SYMBOL(rt_write_lock); +#ifdef CONFIG_DEBUG_LOCK_ALLOC +void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) +{ + rtlock_might_resched(); + rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_); + rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); + rcu_read_lock(); + migrate_disable(); +} +EXPORT_SYMBOL(rt_write_lock_nested); +#endif + void __sched rt_read_unlock(rwlock_t *rwlock) { rwlock_release(&rwlock->dep_map, _RET_IP_); -- cgit v1.2.3-59-g8ed1b From 6dfbbae14a7b961f41d80a106e1ab60e86d061c5 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 21 Jan 2022 22:14:20 -0800 Subject: fs: proc: store PDE()->data into inode->i_private PDE_DATA(inode) is introduced to get user private data and hide the layout of struct proc_dir_entry. The inode->i_private is used to do the same thing as well. Save a copy of user private data to inode-> i_private when proc inode is allocated. This means the user also can get their private data by inode->i_private. Introduce pde_data() to wrap inode->i_private so that we can remove PDE_DATA() from fs/proc/generic.c and make PTE_DATE() as a wrapper of pde_data(). It will be easier if we decide to remove PDE_DATE() in the future. Link: https://lkml.kernel.org/r/20211124081956.87711-1-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Christian Brauner Cc: Alexey Dobriyan Cc: Alexey Gladkov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/generic.c | 6 ------ fs/proc/inode.c | 1 + fs/proc/internal.h | 5 ----- include/linux/proc_fs.h | 13 ++++++++++++- 4 files changed, 13 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 5b78739e60e4..f2132407e133 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -791,12 +791,6 @@ void proc_remove(struct proc_dir_entry *de) } EXPORT_SYMBOL(proc_remove); -void *PDE_DATA(const struct inode *inode) -{ - return __PDE_DATA(inode); -} -EXPORT_SYMBOL(PDE_DATA); - /* * Pull a user buffer into memory and pass it to the file's write handler if * one is supplied. The ->write() method is permitted to modify the diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 599eb724ff2d..f84355c5a36d 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -650,6 +650,7 @@ struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de) return NULL; } + inode->i_private = de->data; inode->i_ino = de->low_ino; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); PROC_I(inode)->pde = de; diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 03415f3fb3a8..06a80f78433d 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -115,11 +115,6 @@ static inline struct proc_dir_entry *PDE(const struct inode *inode) return PROC_I(inode)->pde; } -static inline void *__PDE_DATA(const struct inode *inode) -{ - return PDE(inode)->data; -} - static inline struct pid *proc_pid(const struct inode *inode) { return PROC_I(inode)->pid; diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 01b9268451a8..b6e7005cc1b2 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -110,7 +110,18 @@ extern struct proc_dir_entry *proc_create_data(const char *, umode_t, struct proc_dir_entry *proc_create(const char *name, umode_t mode, struct proc_dir_entry *parent, const struct proc_ops *proc_ops); extern void proc_set_size(struct proc_dir_entry *, loff_t); extern void proc_set_user(struct proc_dir_entry *, kuid_t, kgid_t); -extern void *PDE_DATA(const struct inode *); + +/* + * Obtain the private data passed by user through proc_create_data() or + * related. + */ +static inline void *pde_data(const struct inode *inode) +{ + return inode->i_private; +} + +#define PDE_DATA(i) pde_data(i) + extern void *proc_get_parent_data(const struct inode *); extern void proc_remove(struct proc_dir_entry *); extern void remove_proc_entry(const char *, struct proc_dir_entry *); -- cgit v1.2.3-59-g8ed1b From 359745d78351c6f5442435f81549f0207ece28aa Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 21 Jan 2022 22:14:23 -0800 Subject: proc: remove PDE_DATA() completely Remove PDE_DATA() completely and replace it with pde_data(). [akpm@linux-foundation.org: fix naming clash in drivers/nubus/proc.c] [akpm@linux-foundation.org: now fix it properly] Link: https://lkml.kernel.org/r/20211124081956.87711-2-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Christian Brauner Cc: Alexey Dobriyan Cc: Alexey Gladkov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/kernel/srm_env.c | 4 +-- arch/arm/kernel/atags_proc.c | 2 +- arch/ia64/kernel/salinfo.c | 10 +++--- arch/powerpc/kernel/proc_powerpc.c | 4 +-- arch/sh/mm/alignment.c | 2 +- arch/xtensa/platforms/iss/simdisk.c | 4 +-- drivers/acpi/proc.c | 2 +- drivers/hwmon/dell-smm-hwmon.c | 4 +-- drivers/net/bonding/bond_procfs.c | 8 ++--- drivers/net/wireless/cisco/airo.c | 22 ++++++------- drivers/net/wireless/intersil/hostap/hostap_ap.c | 16 +++++----- .../net/wireless/intersil/hostap/hostap_download.c | 2 +- drivers/net/wireless/intersil/hostap/hostap_proc.c | 24 +++++++-------- drivers/net/wireless/ray_cs.c | 2 +- drivers/nubus/proc.c | 36 +++++++++++----------- drivers/parisc/led.c | 4 +-- drivers/pci/proc.c | 10 +++--- drivers/platform/x86/thinkpad_acpi.c | 4 +-- drivers/platform/x86/toshiba_acpi.c | 16 +++++----- drivers/pnp/isapnp/proc.c | 2 +- drivers/pnp/pnpbios/proc.c | 4 +-- drivers/scsi/scsi_proc.c | 4 +-- drivers/usb/gadget/function/rndis.c | 4 +-- drivers/zorro/proc.c | 2 +- fs/afs/proc.c | 6 ++-- fs/ext4/mballoc.c | 14 ++++----- fs/jbd2/journal.c | 2 +- fs/proc/proc_net.c | 8 ++--- include/linux/proc_fs.h | 4 +-- include/linux/seq_file.h | 2 +- ipc/util.c | 2 +- kernel/irq/proc.c | 8 ++--- kernel/resource.c | 4 +-- net/atm/proc.c | 4 +-- net/bluetooth/af_bluetooth.c | 8 ++--- net/can/bcm.c | 2 +- net/can/proc.c | 2 +- net/core/neighbour.c | 6 ++-- net/core/pktgen.c | 6 ++-- net/ipv4/netfilter/ipt_CLUSTERIP.c | 6 ++-- net/ipv4/raw.c | 8 ++--- net/ipv4/tcp_ipv4.c | 2 +- net/ipv4/udp.c | 6 ++-- net/netfilter/x_tables.c | 10 +++--- net/netfilter/xt_hashlimit.c | 18 +++++------ net/netfilter/xt_recent.c | 4 +-- net/sunrpc/auth_gss/svcauth_gss.c | 4 +-- net/sunrpc/cache.c | 24 +++++++-------- net/sunrpc/stats.c | 2 +- sound/core/info.c | 4 +-- 50 files changed, 178 insertions(+), 180 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/kernel/srm_env.c b/arch/alpha/kernel/srm_env.c index 528d2be58182..217b4dca51dd 100644 --- a/arch/alpha/kernel/srm_env.c +++ b/arch/alpha/kernel/srm_env.c @@ -83,14 +83,14 @@ static int srm_env_proc_show(struct seq_file *m, void *v) static int srm_env_proc_open(struct inode *inode, struct file *file) { - return single_open(file, srm_env_proc_show, PDE_DATA(inode)); + return single_open(file, srm_env_proc_show, pde_data(inode)); } static ssize_t srm_env_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos) { int res; - unsigned long id = (unsigned long)PDE_DATA(file_inode(file)); + unsigned long id = (unsigned long)pde_data(file_inode(file)); char *buf = (char *) __get_free_page(GFP_USER); unsigned long ret1, ret2; diff --git a/arch/arm/kernel/atags_proc.c b/arch/arm/kernel/atags_proc.c index 3c2faf2bd124..3ec2afe78423 100644 --- a/arch/arm/kernel/atags_proc.c +++ b/arch/arm/kernel/atags_proc.c @@ -13,7 +13,7 @@ struct buffer { static ssize_t atags_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - struct buffer *b = PDE_DATA(file_inode(file)); + struct buffer *b = pde_data(file_inode(file)); return simple_read_from_buffer(buf, count, ppos, b->data, b->size); } diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index a25ab9b37953..bd3ba276e69c 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -282,7 +282,7 @@ salinfo_event_open(struct inode *inode, struct file *file) static ssize_t salinfo_event_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos) { - struct salinfo_data *data = PDE_DATA(file_inode(file)); + struct salinfo_data *data = pde_data(file_inode(file)); char cmd[32]; size_t size; int i, n, cpu = -1; @@ -340,7 +340,7 @@ static const struct proc_ops salinfo_event_proc_ops = { static int salinfo_log_open(struct inode *inode, struct file *file) { - struct salinfo_data *data = PDE_DATA(inode); + struct salinfo_data *data = pde_data(inode); if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -365,7 +365,7 @@ salinfo_log_open(struct inode *inode, struct file *file) static int salinfo_log_release(struct inode *inode, struct file *file) { - struct salinfo_data *data = PDE_DATA(inode); + struct salinfo_data *data = pde_data(inode); if (data->state == STATE_NO_DATA) { vfree(data->log_buffer); @@ -433,7 +433,7 @@ retry: static ssize_t salinfo_log_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos) { - struct salinfo_data *data = PDE_DATA(file_inode(file)); + struct salinfo_data *data = pde_data(file_inode(file)); u8 *buf; u64 bufsize; @@ -494,7 +494,7 @@ salinfo_log_clear(struct salinfo_data *data, int cpu) static ssize_t salinfo_log_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { - struct salinfo_data *data = PDE_DATA(file_inode(file)); + struct salinfo_data *data = pde_data(file_inode(file)); char cmd[32]; size_t size; u32 offset; diff --git a/arch/powerpc/kernel/proc_powerpc.c b/arch/powerpc/kernel/proc_powerpc.c index 877817471e3c..6a029f2378e1 100644 --- a/arch/powerpc/kernel/proc_powerpc.c +++ b/arch/powerpc/kernel/proc_powerpc.c @@ -25,7 +25,7 @@ static ssize_t page_map_read( struct file *file, char __user *buf, size_t nbytes loff_t *ppos) { return simple_read_from_buffer(buf, nbytes, ppos, - PDE_DATA(file_inode(file)), PAGE_SIZE); + pde_data(file_inode(file)), PAGE_SIZE); } static int page_map_mmap( struct file *file, struct vm_area_struct *vma ) @@ -34,7 +34,7 @@ static int page_map_mmap( struct file *file, struct vm_area_struct *vma ) return -EINVAL; remap_pfn_range(vma, vma->vm_start, - __pa(PDE_DATA(file_inode(file))) >> PAGE_SHIFT, + __pa(pde_data(file_inode(file))) >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot); return 0; } diff --git a/arch/sh/mm/alignment.c b/arch/sh/mm/alignment.c index d802801ceb93..3a76a766f423 100644 --- a/arch/sh/mm/alignment.c +++ b/arch/sh/mm/alignment.c @@ -140,7 +140,7 @@ static int alignment_proc_open(struct inode *inode, struct file *file) static ssize_t alignment_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos) { - int *data = PDE_DATA(file_inode(file)); + int *data = pde_data(file_inode(file)); char mode; if (count > 0) { diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 07b642c1916a..8eb6ad1a3a1d 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -208,7 +208,7 @@ static int simdisk_detach(struct simdisk *dev) static ssize_t proc_read_simdisk(struct file *file, char __user *buf, size_t size, loff_t *ppos) { - struct simdisk *dev = PDE_DATA(file_inode(file)); + struct simdisk *dev = pde_data(file_inode(file)); const char *s = dev->filename; if (s) { ssize_t n = simple_read_from_buffer(buf, size, ppos, @@ -225,7 +225,7 @@ static ssize_t proc_write_simdisk(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char *tmp = memdup_user_nul(buf, count); - struct simdisk *dev = PDE_DATA(file_inode(file)); + struct simdisk *dev = pde_data(file_inode(file)); int err; if (IS_ERR(tmp)) diff --git a/drivers/acpi/proc.c b/drivers/acpi/proc.c index 0cca7991f186..4322f2da6d10 100644 --- a/drivers/acpi/proc.c +++ b/drivers/acpi/proc.c @@ -127,7 +127,7 @@ static int acpi_system_wakeup_device_open_fs(struct inode *inode, struct file *file) { return single_open(file, acpi_system_wakeup_device_seq_show, - PDE_DATA(inode)); + pde_data(inode)); } static const struct proc_ops acpi_system_wakeup_device_proc_ops = { diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index d401f9acf450..9949eeb79378 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -451,7 +451,7 @@ static int i8k_get_power_status(void) static long i8k_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) { - struct dell_smm_data *data = PDE_DATA(file_inode(fp)); + struct dell_smm_data *data = pde_data(file_inode(fp)); int __user *argp = (int __user *)arg; int speed, err; int val = 0; @@ -585,7 +585,7 @@ static int i8k_proc_show(struct seq_file *seq, void *offset) static int i8k_open_fs(struct inode *inode, struct file *file) { - return single_open(file, i8k_proc_show, PDE_DATA(inode)); + return single_open(file, i8k_proc_show, pde_data(inode)); } static const struct proc_ops i8k_proc_ops = { diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c index 2ec11af5f0cc..46b150e6289e 100644 --- a/drivers/net/bonding/bond_procfs.c +++ b/drivers/net/bonding/bond_procfs.c @@ -11,7 +11,7 @@ static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos) __acquires(RCU) { - struct bonding *bond = PDE_DATA(file_inode(seq->file)); + struct bonding *bond = pde_data(file_inode(seq->file)); struct list_head *iter; struct slave *slave; loff_t off = 0; @@ -30,7 +30,7 @@ static void *bond_info_seq_start(struct seq_file *seq, loff_t *pos) static void *bond_info_seq_next(struct seq_file *seq, void *v, loff_t *pos) { - struct bonding *bond = PDE_DATA(file_inode(seq->file)); + struct bonding *bond = pde_data(file_inode(seq->file)); struct list_head *iter; struct slave *slave; bool found = false; @@ -57,7 +57,7 @@ static void bond_info_seq_stop(struct seq_file *seq, void *v) static void bond_info_show_master(struct seq_file *seq) { - struct bonding *bond = PDE_DATA(file_inode(seq->file)); + struct bonding *bond = pde_data(file_inode(seq->file)); const struct bond_opt_value *optval; struct slave *curr, *primary; int i; @@ -175,7 +175,7 @@ static void bond_info_show_master(struct seq_file *seq) static void bond_info_show_slave(struct seq_file *seq, const struct slave *slave) { - struct bonding *bond = PDE_DATA(file_inode(seq->file)); + struct bonding *bond = pde_data(file_inode(seq->file)); seq_printf(seq, "\nSlave Interface: %s\n", slave->dev->name); seq_printf(seq, "MII Status: %s\n", bond_slave_link_status(slave->link)); diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c index 45594f003ef7..452d08545d31 100644 --- a/drivers/net/wireless/cisco/airo.c +++ b/drivers/net/wireless/cisco/airo.c @@ -4672,7 +4672,7 @@ static ssize_t proc_write(struct file *file, static int proc_status_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *apriv = dev->ml_priv; CapabilityRid cap_rid; StatusRid status_rid; @@ -4756,7 +4756,7 @@ static int proc_stats_rid_open(struct inode *inode, u16 rid) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *apriv = dev->ml_priv; StatsRid stats; int i, j; @@ -4819,7 +4819,7 @@ static inline int sniffing_mode(struct airo_info *ai) static void proc_config_on_close(struct inode *inode, struct file *file) { struct proc_data *data = file->private_data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; char *line; @@ -5030,7 +5030,7 @@ static const char *get_rmode(__le16 mode) static int proc_config_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; int i; __le16 mode; @@ -5120,7 +5120,7 @@ static int proc_config_open(struct inode *inode, struct file *file) static void proc_SSID_on_close(struct inode *inode, struct file *file) { struct proc_data *data = file->private_data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; SsidRid SSID_rid; int i; @@ -5156,7 +5156,7 @@ static void proc_SSID_on_close(struct inode *inode, struct file *file) static void proc_APList_on_close(struct inode *inode, struct file *file) { struct proc_data *data = file->private_data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; APListRid *APList_rid = &ai->APList; int i; @@ -5280,7 +5280,7 @@ static int set_wep_tx_idx(struct airo_info *ai, u16 index, int perm, int lock) static void proc_wepkey_on_close(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; int i, rc; char key[16]; @@ -5331,7 +5331,7 @@ static void proc_wepkey_on_close(struct inode *inode, struct file *file) static int proc_wepkey_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; char *ptr; WepKeyRid wkr; @@ -5379,7 +5379,7 @@ static int proc_wepkey_open(struct inode *inode, struct file *file) static int proc_SSID_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; int i; char *ptr; @@ -5423,7 +5423,7 @@ static int proc_SSID_open(struct inode *inode, struct file *file) static int proc_APList_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; int i; char *ptr; @@ -5462,7 +5462,7 @@ static int proc_APList_open(struct inode *inode, struct file *file) static int proc_BSSList_open(struct inode *inode, struct file *file) { struct proc_data *data; - struct net_device *dev = PDE_DATA(inode); + struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; char *ptr; BSSListRid BSSList_rid; diff --git a/drivers/net/wireless/intersil/hostap/hostap_ap.c b/drivers/net/wireless/intersil/hostap/hostap_ap.c index 8bcc1cdcb75b..462ccc7d7d1a 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_ap.c +++ b/drivers/net/wireless/intersil/hostap/hostap_ap.c @@ -69,7 +69,7 @@ static void prism2_send_mgmt(struct net_device *dev, #if !defined(PRISM2_NO_PROCFS_DEBUG) && defined(CONFIG_PROC_FS) static int ap_debug_proc_show(struct seq_file *m, void *v) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); seq_printf(m, "BridgedUnicastFrames=%u\n", ap->bridged_unicast); seq_printf(m, "BridgedMulticastFrames=%u\n", ap->bridged_multicast); @@ -320,7 +320,7 @@ void hostap_deauth_all_stas(struct net_device *dev, struct ap_data *ap, static int ap_control_proc_show(struct seq_file *m, void *v) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); char *policy_txt; struct mac_entry *entry; @@ -352,20 +352,20 @@ static int ap_control_proc_show(struct seq_file *m, void *v) static void *ap_control_proc_start(struct seq_file *m, loff_t *_pos) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); spin_lock_bh(&ap->mac_restrictions.lock); return seq_list_start_head(&ap->mac_restrictions.mac_list, *_pos); } static void *ap_control_proc_next(struct seq_file *m, void *v, loff_t *_pos) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); return seq_list_next(v, &ap->mac_restrictions.mac_list, _pos); } static void ap_control_proc_stop(struct seq_file *m, void *v) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); spin_unlock_bh(&ap->mac_restrictions.lock); } @@ -554,20 +554,20 @@ static int prism2_ap_proc_show(struct seq_file *m, void *v) static void *prism2_ap_proc_start(struct seq_file *m, loff_t *_pos) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); spin_lock_bh(&ap->sta_table_lock); return seq_list_start_head(&ap->sta_list, *_pos); } static void *prism2_ap_proc_next(struct seq_file *m, void *v, loff_t *_pos) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); return seq_list_next(v, &ap->sta_list, _pos); } static void prism2_ap_proc_stop(struct seq_file *m, void *v) { - struct ap_data *ap = PDE_DATA(file_inode(m->file)); + struct ap_data *ap = pde_data(file_inode(m->file)); spin_unlock_bh(&ap->sta_table_lock); } diff --git a/drivers/net/wireless/intersil/hostap/hostap_download.c b/drivers/net/wireless/intersil/hostap/hostap_download.c index 7c6a5a6d1d45..3672291ced5c 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_download.c +++ b/drivers/net/wireless/intersil/hostap/hostap_download.c @@ -227,7 +227,7 @@ static int prism2_download_aux_dump_proc_open(struct inode *inode, struct file * sizeof(struct prism2_download_aux_dump)); if (ret == 0) { struct seq_file *m = file->private_data; - m->private = PDE_DATA(inode); + m->private = pde_data(inode); } return ret; } diff --git a/drivers/net/wireless/intersil/hostap/hostap_proc.c b/drivers/net/wireless/intersil/hostap/hostap_proc.c index 51c847d98755..61f68786056f 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_proc.c +++ b/drivers/net/wireless/intersil/hostap/hostap_proc.c @@ -97,20 +97,20 @@ static int prism2_wds_proc_show(struct seq_file *m, void *v) static void *prism2_wds_proc_start(struct seq_file *m, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); read_lock_bh(&local->iface_lock); return seq_list_start(&local->hostap_interfaces, *_pos); } static void *prism2_wds_proc_next(struct seq_file *m, void *v, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); return seq_list_next(v, &local->hostap_interfaces, _pos); } static void prism2_wds_proc_stop(struct seq_file *m, void *v) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); read_unlock_bh(&local->iface_lock); } @@ -123,7 +123,7 @@ static const struct seq_operations prism2_wds_proc_seqops = { static int prism2_bss_list_proc_show(struct seq_file *m, void *v) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); struct list_head *ptr = v; struct hostap_bss_info *bss; @@ -151,21 +151,21 @@ static int prism2_bss_list_proc_show(struct seq_file *m, void *v) static void *prism2_bss_list_proc_start(struct seq_file *m, loff_t *_pos) __acquires(&local->lock) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); spin_lock_bh(&local->lock); return seq_list_start_head(&local->bss_list, *_pos); } static void *prism2_bss_list_proc_next(struct seq_file *m, void *v, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); return seq_list_next(v, &local->bss_list, _pos); } static void prism2_bss_list_proc_stop(struct seq_file *m, void *v) __releases(&local->lock) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); spin_unlock_bh(&local->lock); } @@ -198,7 +198,7 @@ static int prism2_crypt_proc_show(struct seq_file *m, void *v) static ssize_t prism2_pda_proc_read(struct file *file, char __user *buf, size_t count, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(file)); + local_info_t *local = pde_data(file_inode(file)); size_t off; if (local->pda == NULL || *_pos >= PRISM2_PDA_SIZE) @@ -272,7 +272,7 @@ static int prism2_io_debug_proc_read(char *page, char **start, off_t off, #ifndef PRISM2_NO_STATION_MODES static int prism2_scan_results_proc_show(struct seq_file *m, void *v) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); unsigned long entry; int i, len; struct hfa384x_hostscan_result *scanres; @@ -322,7 +322,7 @@ static int prism2_scan_results_proc_show(struct seq_file *m, void *v) static void *prism2_scan_results_proc_start(struct seq_file *m, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); spin_lock_bh(&local->lock); /* We have a header (pos 0) + N results to show (pos 1...N) */ @@ -333,7 +333,7 @@ static void *prism2_scan_results_proc_start(struct seq_file *m, loff_t *_pos) static void *prism2_scan_results_proc_next(struct seq_file *m, void *v, loff_t *_pos) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); ++*_pos; if (*_pos > local->last_scan_results_count) @@ -343,7 +343,7 @@ static void *prism2_scan_results_proc_next(struct seq_file *m, void *v, loff_t * static void prism2_scan_results_proc_stop(struct seq_file *m, void *v) { - local_info_t *local = PDE_DATA(file_inode(m->file)); + local_info_t *local = pde_data(file_inode(m->file)); spin_unlock_bh(&local->lock); } diff --git a/drivers/net/wireless/ray_cs.c b/drivers/net/wireless/ray_cs.c index e3a3dc3e45b4..2987ad9271f6 100644 --- a/drivers/net/wireless/ray_cs.c +++ b/drivers/net/wireless/ray_cs.c @@ -2746,7 +2746,7 @@ static ssize_t int_proc_write(struct file *file, const char __user *buffer, nr = nr * 10 + c; p++; } while (--len); - *(int *)PDE_DATA(file_inode(file)) = nr; + *(int *)pde_data(file_inode(file)) = nr; return count; } diff --git a/drivers/nubus/proc.c b/drivers/nubus/proc.c index 88e1f9a0faaf..1fd667852271 100644 --- a/drivers/nubus/proc.c +++ b/drivers/nubus/proc.c @@ -93,30 +93,30 @@ struct nubus_proc_pde_data { static struct nubus_proc_pde_data * nubus_proc_alloc_pde_data(unsigned char *ptr, unsigned int size) { - struct nubus_proc_pde_data *pde_data; + struct nubus_proc_pde_data *pded; - pde_data = kmalloc(sizeof(*pde_data), GFP_KERNEL); - if (!pde_data) + pded = kmalloc(sizeof(*pded), GFP_KERNEL); + if (!pded) return NULL; - pde_data->res_ptr = ptr; - pde_data->res_size = size; - return pde_data; + pded->res_ptr = ptr; + pded->res_size = size; + return pded; } static int nubus_proc_rsrc_show(struct seq_file *m, void *v) { struct inode *inode = m->private; - struct nubus_proc_pde_data *pde_data; + struct nubus_proc_pde_data *pded; - pde_data = PDE_DATA(inode); - if (!pde_data) + pded = pde_data(inode); + if (!pded) return 0; - if (pde_data->res_size > m->size) + if (pded->res_size > m->size) return -EFBIG; - if (pde_data->res_size) { + if (pded->res_size) { int lanes = (int)proc_get_parent_data(inode); struct nubus_dirent ent; @@ -124,11 +124,11 @@ static int nubus_proc_rsrc_show(struct seq_file *m, void *v) return 0; ent.mask = lanes; - ent.base = pde_data->res_ptr; + ent.base = pded->res_ptr; ent.data = 0; - nubus_seq_write_rsrc_mem(m, &ent, pde_data->res_size); + nubus_seq_write_rsrc_mem(m, &ent, pded->res_size); } else { - unsigned int data = (unsigned int)pde_data->res_ptr; + unsigned int data = (unsigned int)pded->res_ptr; seq_putc(m, data >> 16); seq_putc(m, data >> 8); @@ -142,18 +142,18 @@ void nubus_proc_add_rsrc_mem(struct proc_dir_entry *procdir, unsigned int size) { char name[9]; - struct nubus_proc_pde_data *pde_data; + struct nubus_proc_pde_data *pded; if (!procdir) return; snprintf(name, sizeof(name), "%x", ent->type); if (size) - pde_data = nubus_proc_alloc_pde_data(nubus_dirptr(ent), size); + pded = nubus_proc_alloc_pde_data(nubus_dirptr(ent), size); else - pde_data = NULL; + pded = NULL; proc_create_single_data(name, S_IFREG | 0444, procdir, - nubus_proc_rsrc_show, pde_data); + nubus_proc_rsrc_show, pded); } void nubus_proc_add_rsrc(struct proc_dir_entry *procdir, diff --git a/drivers/parisc/led.c b/drivers/parisc/led.c index cf91cb024be3..1e4a5663d011 100644 --- a/drivers/parisc/led.c +++ b/drivers/parisc/led.c @@ -168,14 +168,14 @@ static int led_proc_show(struct seq_file *m, void *v) static int led_proc_open(struct inode *inode, struct file *file) { - return single_open(file, led_proc_show, PDE_DATA(inode)); + return single_open(file, led_proc_show, pde_data(inode)); } static ssize_t led_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - void *data = PDE_DATA(file_inode(file)); + void *data = pde_data(file_inode(file)); char *cur, lbuf[32]; int d; diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index cb18f8a13ab6..9c7edec64f7e 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -21,14 +21,14 @@ static int proc_initialized; /* = 0 */ static loff_t proc_bus_pci_lseek(struct file *file, loff_t off, int whence) { - struct pci_dev *dev = PDE_DATA(file_inode(file)); + struct pci_dev *dev = pde_data(file_inode(file)); return fixed_size_llseek(file, off, whence, dev->cfg_size); } static ssize_t proc_bus_pci_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { - struct pci_dev *dev = PDE_DATA(file_inode(file)); + struct pci_dev *dev = pde_data(file_inode(file)); unsigned int pos = *ppos; unsigned int cnt, size; @@ -114,7 +114,7 @@ static ssize_t proc_bus_pci_write(struct file *file, const char __user *buf, size_t nbytes, loff_t *ppos) { struct inode *ino = file_inode(file); - struct pci_dev *dev = PDE_DATA(ino); + struct pci_dev *dev = pde_data(ino); int pos = *ppos; int size = dev->cfg_size; int cnt, ret; @@ -196,7 +196,7 @@ struct pci_filp_private { static long proc_bus_pci_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - struct pci_dev *dev = PDE_DATA(file_inode(file)); + struct pci_dev *dev = pde_data(file_inode(file)); #ifdef HAVE_PCI_MMAP struct pci_filp_private *fpriv = file->private_data; #endif /* HAVE_PCI_MMAP */ @@ -244,7 +244,7 @@ static long proc_bus_pci_ioctl(struct file *file, unsigned int cmd, #ifdef HAVE_PCI_MMAP static int proc_bus_pci_mmap(struct file *file, struct vm_area_struct *vma) { - struct pci_dev *dev = PDE_DATA(file_inode(file)); + struct pci_dev *dev = pde_data(file_inode(file)); struct pci_filp_private *fpriv = file->private_data; int i, ret, write_combine = 0, res_bit = IORESOURCE_MEM; diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index 82fa6148216c..098180fb1cfc 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -880,14 +880,14 @@ static int dispatch_proc_show(struct seq_file *m, void *v) static int dispatch_proc_open(struct inode *inode, struct file *file) { - return single_open(file, dispatch_proc_show, PDE_DATA(inode)); + return single_open(file, dispatch_proc_show, pde_data(inode)); } static ssize_t dispatch_proc_write(struct file *file, const char __user *userbuf, size_t count, loff_t *pos) { - struct ibm_struct *ibm = PDE_DATA(file_inode(file)); + struct ibm_struct *ibm = pde_data(file_inode(file)); char *kernbuf; int ret; diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c index 352508d30467..f113dec98e21 100644 --- a/drivers/platform/x86/toshiba_acpi.c +++ b/drivers/platform/x86/toshiba_acpi.c @@ -1368,7 +1368,7 @@ static int lcd_proc_show(struct seq_file *m, void *v) static int lcd_proc_open(struct inode *inode, struct file *file) { - return single_open(file, lcd_proc_show, PDE_DATA(inode)); + return single_open(file, lcd_proc_show, pde_data(inode)); } static int set_lcd_brightness(struct toshiba_acpi_dev *dev, int value) @@ -1404,7 +1404,7 @@ static int set_lcd_status(struct backlight_device *bd) static ssize_t lcd_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - struct toshiba_acpi_dev *dev = PDE_DATA(file_inode(file)); + struct toshiba_acpi_dev *dev = pde_data(file_inode(file)); char cmd[42]; size_t len; int levels; @@ -1469,13 +1469,13 @@ static int video_proc_show(struct seq_file *m, void *v) static int video_proc_open(struct inode *inode, struct file *file) { - return single_open(file, video_proc_show, PDE_DATA(inode)); + return single_open(file, video_proc_show, pde_data(inode)); } static ssize_t video_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - struct toshiba_acpi_dev *dev = PDE_DATA(file_inode(file)); + struct toshiba_acpi_dev *dev = pde_data(file_inode(file)); char *buffer; char *cmd; int lcd_out = -1, crt_out = -1, tv_out = -1; @@ -1580,13 +1580,13 @@ static int fan_proc_show(struct seq_file *m, void *v) static int fan_proc_open(struct inode *inode, struct file *file) { - return single_open(file, fan_proc_show, PDE_DATA(inode)); + return single_open(file, fan_proc_show, pde_data(inode)); } static ssize_t fan_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - struct toshiba_acpi_dev *dev = PDE_DATA(file_inode(file)); + struct toshiba_acpi_dev *dev = pde_data(file_inode(file)); char cmd[42]; size_t len; int value; @@ -1628,13 +1628,13 @@ static int keys_proc_show(struct seq_file *m, void *v) static int keys_proc_open(struct inode *inode, struct file *file) { - return single_open(file, keys_proc_show, PDE_DATA(inode)); + return single_open(file, keys_proc_show, pde_data(inode)); } static ssize_t keys_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - struct toshiba_acpi_dev *dev = PDE_DATA(file_inode(file)); + struct toshiba_acpi_dev *dev = pde_data(file_inode(file)); char cmd[42]; size_t len; int value; diff --git a/drivers/pnp/isapnp/proc.c b/drivers/pnp/isapnp/proc.c index 1ae458c02656..55ae72a2818b 100644 --- a/drivers/pnp/isapnp/proc.c +++ b/drivers/pnp/isapnp/proc.c @@ -22,7 +22,7 @@ static loff_t isapnp_proc_bus_lseek(struct file *file, loff_t off, int whence) static ssize_t isapnp_proc_bus_read(struct file *file, char __user * buf, size_t nbytes, loff_t * ppos) { - struct pnp_dev *dev = PDE_DATA(file_inode(file)); + struct pnp_dev *dev = pde_data(file_inode(file)); int pos = *ppos; int cnt, size = 256; diff --git a/drivers/pnp/pnpbios/proc.c b/drivers/pnp/pnpbios/proc.c index a806830e3a40..0f0d819b157f 100644 --- a/drivers/pnp/pnpbios/proc.c +++ b/drivers/pnp/pnpbios/proc.c @@ -173,13 +173,13 @@ static int pnpbios_proc_show(struct seq_file *m, void *v) static int pnpbios_proc_open(struct inode *inode, struct file *file) { - return single_open(file, pnpbios_proc_show, PDE_DATA(inode)); + return single_open(file, pnpbios_proc_show, pde_data(inode)); } static ssize_t pnpbios_proc_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - void *data = PDE_DATA(file_inode(file)); + void *data = pde_data(file_inode(file)); struct pnp_bios_node *node; int boot = (long)data >> 8; u8 nodenum = (long)data; diff --git a/drivers/scsi/scsi_proc.c b/drivers/scsi/scsi_proc.c index d6982d355739..95aee1ad1383 100644 --- a/drivers/scsi/scsi_proc.c +++ b/drivers/scsi/scsi_proc.c @@ -49,7 +49,7 @@ static DEFINE_MUTEX(global_host_template_mutex); static ssize_t proc_scsi_host_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - struct Scsi_Host *shost = PDE_DATA(file_inode(file)); + struct Scsi_Host *shost = pde_data(file_inode(file)); ssize_t ret = -ENOMEM; char *page; @@ -79,7 +79,7 @@ static int proc_scsi_show(struct seq_file *m, void *v) static int proc_scsi_host_open(struct inode *inode, struct file *file) { - return single_open_size(file, proc_scsi_show, PDE_DATA(inode), + return single_open_size(file, proc_scsi_show, pde_data(inode), 4 * PAGE_SIZE); } diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c index 64de9f1b874c..431d5a7d737e 100644 --- a/drivers/usb/gadget/function/rndis.c +++ b/drivers/usb/gadget/function/rndis.c @@ -1117,7 +1117,7 @@ static int rndis_proc_show(struct seq_file *m, void *v) static ssize_t rndis_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { - rndis_params *p = PDE_DATA(file_inode(file)); + rndis_params *p = pde_data(file_inode(file)); u32 speed = 0; int i, fl_speed = 0; @@ -1161,7 +1161,7 @@ static ssize_t rndis_proc_write(struct file *file, const char __user *buffer, static int rndis_proc_open(struct inode *inode, struct file *file) { - return single_open(file, rndis_proc_show, PDE_DATA(inode)); + return single_open(file, rndis_proc_show, pde_data(inode)); } static const struct proc_ops rndis_proc_ops = { diff --git a/drivers/zorro/proc.c b/drivers/zorro/proc.c index 1c9ae08225d8..f916bf60b312 100644 --- a/drivers/zorro/proc.c +++ b/drivers/zorro/proc.c @@ -30,7 +30,7 @@ proc_bus_zorro_lseek(struct file *file, loff_t off, int whence) static ssize_t proc_bus_zorro_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { - struct zorro_dev *z = PDE_DATA(file_inode(file)); + struct zorro_dev *z = pde_data(file_inode(file)); struct ConfigDev cd; loff_t pos = *ppos; diff --git a/fs/afs/proc.c b/fs/afs/proc.c index 065a28bfa3f1..e1b863449296 100644 --- a/fs/afs/proc.c +++ b/fs/afs/proc.c @@ -227,7 +227,7 @@ static int afs_proc_cell_volumes_show(struct seq_file *m, void *v) static void *afs_proc_cell_volumes_start(struct seq_file *m, loff_t *_pos) __acquires(cell->proc_lock) { - struct afs_cell *cell = PDE_DATA(file_inode(m->file)); + struct afs_cell *cell = pde_data(file_inode(m->file)); rcu_read_lock(); return seq_hlist_start_head_rcu(&cell->proc_volumes, *_pos); @@ -236,7 +236,7 @@ static void *afs_proc_cell_volumes_start(struct seq_file *m, loff_t *_pos) static void *afs_proc_cell_volumes_next(struct seq_file *m, void *v, loff_t *_pos) { - struct afs_cell *cell = PDE_DATA(file_inode(m->file)); + struct afs_cell *cell = pde_data(file_inode(m->file)); return seq_hlist_next_rcu(v, &cell->proc_volumes, _pos); } @@ -322,7 +322,7 @@ static void *afs_proc_cell_vlservers_start(struct seq_file *m, loff_t *_pos) { struct afs_vl_seq_net_private *priv = m->private; struct afs_vlserver_list *vllist; - struct afs_cell *cell = PDE_DATA(file_inode(m->file)); + struct afs_cell *cell = pde_data(file_inode(m->file)); loff_t pos = *_pos; rcu_read_lock(); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index cf2fd9fc7d98..9f86dd947032 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -2834,7 +2834,7 @@ out: static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); ext4_group_t group; if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) @@ -2845,7 +2845,7 @@ static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); ext4_group_t group; ++*pos; @@ -2857,7 +2857,7 @@ static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); ext4_group_t group = (ext4_group_t) ((unsigned long) v); int i; int err, buddy_loaded = 0; @@ -2985,7 +2985,7 @@ int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos) __acquires(&EXT4_SB(sb)->s_mb_rb_lock) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); unsigned long position; read_lock(&EXT4_SB(sb)->s_mb_rb_lock); @@ -2998,7 +2998,7 @@ __acquires(&EXT4_SB(sb)->s_mb_rb_lock) static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, loff_t *pos) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); unsigned long position; ++*pos; @@ -3010,7 +3010,7 @@ static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, lof static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); struct ext4_sb_info *sbi = EXT4_SB(sb); unsigned long position = ((unsigned long) v); struct ext4_group_info *grp; @@ -3058,7 +3058,7 @@ static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v) __releases(&EXT4_SB(sb)->s_mb_rb_lock) { - struct super_block *sb = PDE_DATA(file_inode(seq->file)); + struct super_block *sb = pde_data(file_inode(seq->file)); read_unlock(&EXT4_SB(sb)->s_mb_rb_lock); } diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 0b86a4365b66..f13d548e4a7f 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1212,7 +1212,7 @@ static const struct seq_operations jbd2_seq_info_ops = { static int jbd2_seq_info_open(struct inode *inode, struct file *file) { - journal_t *journal = PDE_DATA(inode); + journal_t *journal = pde_data(inode); struct jbd2_stats_proc_session *s; int rc, size; diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c index 39b823ab2564..e1cfeda397f3 100644 --- a/fs/proc/proc_net.c +++ b/fs/proc/proc_net.c @@ -138,7 +138,7 @@ EXPORT_SYMBOL_GPL(proc_create_net_data); * @parent: The parent directory in which to create. * @ops: The seq_file ops with which to read the file. * @write: The write method with which to 'modify' the file. - * @data: Data for retrieval by PDE_DATA(). + * @data: Data for retrieval by pde_data(). * * Create a network namespaced proc file in the @parent directory with the * specified @name and @mode that allows reading of a file that displays a @@ -153,7 +153,7 @@ EXPORT_SYMBOL_GPL(proc_create_net_data); * modified by the @write function. @write should return 0 on success. * * The @data value is accessible from the @show and @write functions by calling - * PDE_DATA() on the file inode. The network namespace must be accessed by + * pde_data() on the file inode. The network namespace must be accessed by * calling seq_file_net() on the seq_file struct. */ struct proc_dir_entry *proc_create_net_data_write(const char *name, umode_t mode, @@ -230,7 +230,7 @@ EXPORT_SYMBOL_GPL(proc_create_net_single); * @parent: The parent directory in which to create. * @show: The seqfile show method with which to read the file. * @write: The write method with which to 'modify' the file. - * @data: Data for retrieval by PDE_DATA(). + * @data: Data for retrieval by pde_data(). * * Create a network-namespaced proc file in the @parent directory with the * specified @name and @mode that allows reading of a file that displays a @@ -245,7 +245,7 @@ EXPORT_SYMBOL_GPL(proc_create_net_single); * modified by the @write function. @write should return 0 on success. * * The @data value is accessible from the @show and @write functions by calling - * PDE_DATA() on the file inode. The network namespace must be accessed by + * pde_data() on the file inode. The network namespace must be accessed by * calling seq_file_single_net() on the seq_file struct. */ struct proc_dir_entry *proc_create_net_single_write(const char *name, umode_t mode, diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index b6e7005cc1b2..81d6e4ec2294 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -120,8 +120,6 @@ static inline void *pde_data(const struct inode *inode) return inode->i_private; } -#define PDE_DATA(i) pde_data(i) - extern void *proc_get_parent_data(const struct inode *); extern void proc_remove(struct proc_dir_entry *); extern void remove_proc_entry(const char *, struct proc_dir_entry *); @@ -202,7 +200,7 @@ proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, static inline void proc_set_size(struct proc_dir_entry *de, loff_t size) {} static inline void proc_set_user(struct proc_dir_entry *de, kuid_t uid, kgid_t gid) {} -static inline void *PDE_DATA(const struct inode *inode) {BUG(); return NULL;} +static inline void *pde_data(const struct inode *inode) {BUG(); return NULL;} static inline void *proc_get_parent_data(const struct inode *inode) { BUG(); return NULL; } static inline void proc_remove(struct proc_dir_entry *de) {} diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index 72dbb44a4573..88cc16444b43 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -209,7 +209,7 @@ static const struct file_operations __name ## _fops = { \ #define DEFINE_PROC_SHOW_ATTRIBUTE(__name) \ static int __name ## _open(struct inode *inode, struct file *file) \ { \ - return single_open(file, __name ## _show, PDE_DATA(inode)); \ + return single_open(file, __name ## _show, pde_data(inode)); \ } \ \ static const struct proc_ops __name ## _proc_ops = { \ diff --git a/ipc/util.c b/ipc/util.c index fa2d86ef3fb8..a2208d0f26b2 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -894,7 +894,7 @@ static int sysvipc_proc_open(struct inode *inode, struct file *file) if (!iter) return -ENOMEM; - iter->iface = PDE_DATA(inode); + iter->iface = pde_data(inode); iter->ns = get_ipc_ns(current->nsproxy->ipc_ns); iter->pid_ns = get_pid_ns(task_active_pid_ns(current)); diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index ee595ec09778..623b8136e9af 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -137,7 +137,7 @@ static inline int irq_select_affinity_usr(unsigned int irq) static ssize_t write_irq_affinity(int type, struct file *file, const char __user *buffer, size_t count, loff_t *pos) { - unsigned int irq = (int)(long)PDE_DATA(file_inode(file)); + unsigned int irq = (int)(long)pde_data(file_inode(file)); cpumask_var_t new_value; int err; @@ -190,12 +190,12 @@ static ssize_t irq_affinity_list_proc_write(struct file *file, static int irq_affinity_proc_open(struct inode *inode, struct file *file) { - return single_open(file, irq_affinity_proc_show, PDE_DATA(inode)); + return single_open(file, irq_affinity_proc_show, pde_data(inode)); } static int irq_affinity_list_proc_open(struct inode *inode, struct file *file) { - return single_open(file, irq_affinity_list_proc_show, PDE_DATA(inode)); + return single_open(file, irq_affinity_list_proc_show, pde_data(inode)); } static const struct proc_ops irq_affinity_proc_ops = { @@ -265,7 +265,7 @@ out: static int default_affinity_open(struct inode *inode, struct file *file) { - return single_open(file, default_affinity_show, PDE_DATA(inode)); + return single_open(file, default_affinity_show, pde_data(inode)); } static const struct proc_ops default_affinity_proc_ops = { diff --git a/kernel/resource.c b/kernel/resource.c index 5ad3eba619ba..9c08d6e9eef2 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -99,7 +99,7 @@ enum { MAX_IORES_LEVEL = 5 }; static void *r_start(struct seq_file *m, loff_t *pos) __acquires(resource_lock) { - struct resource *p = PDE_DATA(file_inode(m->file)); + struct resource *p = pde_data(file_inode(m->file)); loff_t l = 0; read_lock(&resource_lock); for (p = p->child; p && l < *pos; p = r_next(m, p, &l)) @@ -115,7 +115,7 @@ static void r_stop(struct seq_file *m, void *v) static int r_show(struct seq_file *m, void *v) { - struct resource *root = PDE_DATA(file_inode(m->file)); + struct resource *root = pde_data(file_inode(m->file)); struct resource *r = v, *p; unsigned long long start, end; int width = root->end < 0x10000 ? 4 : 8; diff --git a/net/atm/proc.c b/net/atm/proc.c index 4369ffa3302a..9bf736290e48 100644 --- a/net/atm/proc.c +++ b/net/atm/proc.c @@ -108,7 +108,7 @@ out: static inline void *vcc_walk(struct seq_file *seq, loff_t l) { struct vcc_state *state = seq->private; - int family = (uintptr_t)(PDE_DATA(file_inode(seq->file))); + int family = (uintptr_t)(pde_data(file_inode(seq->file))); return __vcc_walk(&state->sk, family, &state->bucket, l) ? state : NULL; @@ -324,7 +324,7 @@ static ssize_t proc_dev_atm_read(struct file *file, char __user *buf, page = get_zeroed_page(GFP_KERNEL); if (!page) return -ENOMEM; - dev = PDE_DATA(file_inode(file)); + dev = pde_data(file_inode(file)); if (!dev->ops->proc_read) length = -EINVAL; else { diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index 1661979b6a6e..ee319779781e 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -611,7 +611,7 @@ EXPORT_SYMBOL(bt_sock_wait_ready); static void *bt_seq_start(struct seq_file *seq, loff_t *pos) __acquires(seq->private->l->lock) { - struct bt_sock_list *l = PDE_DATA(file_inode(seq->file)); + struct bt_sock_list *l = pde_data(file_inode(seq->file)); read_lock(&l->lock); return seq_hlist_start_head(&l->head, *pos); @@ -619,7 +619,7 @@ static void *bt_seq_start(struct seq_file *seq, loff_t *pos) static void *bt_seq_next(struct seq_file *seq, void *v, loff_t *pos) { - struct bt_sock_list *l = PDE_DATA(file_inode(seq->file)); + struct bt_sock_list *l = pde_data(file_inode(seq->file)); return seq_hlist_next(v, &l->head, pos); } @@ -627,14 +627,14 @@ static void *bt_seq_next(struct seq_file *seq, void *v, loff_t *pos) static void bt_seq_stop(struct seq_file *seq, void *v) __releases(seq->private->l->lock) { - struct bt_sock_list *l = PDE_DATA(file_inode(seq->file)); + struct bt_sock_list *l = pde_data(file_inode(seq->file)); read_unlock(&l->lock); } static int bt_seq_show(struct seq_file *seq, void *v) { - struct bt_sock_list *l = PDE_DATA(file_inode(seq->file)); + struct bt_sock_list *l = pde_data(file_inode(seq->file)); if (v == SEQ_START_TOKEN) { seq_puts(seq, "sk RefCnt Rmem Wmem User Inode Parent"); diff --git a/net/can/bcm.c b/net/can/bcm.c index bc88d901a1c0..95d209b52e6a 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -193,7 +193,7 @@ static int bcm_proc_show(struct seq_file *m, void *v) { char ifname[IFNAMSIZ]; struct net *net = m->private; - struct sock *sk = (struct sock *)PDE_DATA(m->file->f_inode); + struct sock *sk = (struct sock *)pde_data(m->file->f_inode); struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; diff --git a/net/can/proc.c b/net/can/proc.c index b3099f0a3cb8..bbce97825f13 100644 --- a/net/can/proc.c +++ b/net/can/proc.c @@ -305,7 +305,7 @@ static inline void can_rcvlist_proc_show_one(struct seq_file *m, int idx, static int can_rcvlist_proc_show(struct seq_file *m, void *v) { /* double cast to prevent GCC warning */ - int idx = (int)(long)PDE_DATA(m->file->f_inode); + int idx = (int)(long)pde_data(m->file->f_inode); struct net_device *dev; struct can_dev_rcv_lists *dev_rcv_lists; struct net *net = m->private; diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 213cb7b26b7a..6c2016f7f3d1 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -3364,7 +3364,7 @@ EXPORT_SYMBOL(neigh_seq_stop); static void *neigh_stat_seq_start(struct seq_file *seq, loff_t *pos) { - struct neigh_table *tbl = PDE_DATA(file_inode(seq->file)); + struct neigh_table *tbl = pde_data(file_inode(seq->file)); int cpu; if (*pos == 0) @@ -3381,7 +3381,7 @@ static void *neigh_stat_seq_start(struct seq_file *seq, loff_t *pos) static void *neigh_stat_seq_next(struct seq_file *seq, void *v, loff_t *pos) { - struct neigh_table *tbl = PDE_DATA(file_inode(seq->file)); + struct neigh_table *tbl = pde_data(file_inode(seq->file)); int cpu; for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { @@ -3401,7 +3401,7 @@ static void neigh_stat_seq_stop(struct seq_file *seq, void *v) static int neigh_stat_seq_show(struct seq_file *seq, void *v) { - struct neigh_table *tbl = PDE_DATA(file_inode(seq->file)); + struct neigh_table *tbl = pde_data(file_inode(seq->file)); struct neigh_statistics *st = v; if (v == SEQ_START_TOKEN) { diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 560a5e712dc3..84b62cd7bc57 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -546,7 +546,7 @@ static ssize_t pgctrl_write(struct file *file, const char __user *buf, static int pgctrl_open(struct inode *inode, struct file *file) { - return single_open(file, pgctrl_show, PDE_DATA(inode)); + return single_open(file, pgctrl_show, pde_data(inode)); } static const struct proc_ops pktgen_proc_ops = { @@ -1811,7 +1811,7 @@ static ssize_t pktgen_if_write(struct file *file, static int pktgen_if_open(struct inode *inode, struct file *file) { - return single_open(file, pktgen_if_show, PDE_DATA(inode)); + return single_open(file, pktgen_if_show, pde_data(inode)); } static const struct proc_ops pktgen_if_proc_ops = { @@ -1948,7 +1948,7 @@ out: static int pktgen_thread_open(struct inode *inode, struct file *file) { - return single_open(file, pktgen_thread_show, PDE_DATA(inode)); + return single_open(file, pktgen_thread_show, pde_data(inode)); } static const struct proc_ops pktgen_thread_proc_ops = { diff --git a/net/ipv4/netfilter/ipt_CLUSTERIP.c b/net/ipv4/netfilter/ipt_CLUSTERIP.c index b518f20c9a24..f8e176c77d1c 100644 --- a/net/ipv4/netfilter/ipt_CLUSTERIP.c +++ b/net/ipv4/netfilter/ipt_CLUSTERIP.c @@ -776,7 +776,7 @@ static int clusterip_proc_open(struct inode *inode, struct file *file) if (!ret) { struct seq_file *sf = file->private_data; - struct clusterip_config *c = PDE_DATA(inode); + struct clusterip_config *c = pde_data(inode); sf->private = c; @@ -788,7 +788,7 @@ static int clusterip_proc_open(struct inode *inode, struct file *file) static int clusterip_proc_release(struct inode *inode, struct file *file) { - struct clusterip_config *c = PDE_DATA(inode); + struct clusterip_config *c = pde_data(inode); int ret; ret = seq_release(inode, file); @@ -802,7 +802,7 @@ static int clusterip_proc_release(struct inode *inode, struct file *file) static ssize_t clusterip_proc_write(struct file *file, const char __user *input, size_t size, loff_t *ofs) { - struct clusterip_config *c = PDE_DATA(file_inode(file)); + struct clusterip_config *c = pde_data(file_inode(file)); #define PROC_WRITELEN 10 char buffer[PROC_WRITELEN+1]; unsigned long nodenum; diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index a53f256bf9d3..9eb5fc247868 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -971,7 +971,7 @@ struct proto raw_prot = { static struct sock *raw_get_first(struct seq_file *seq) { struct sock *sk; - struct raw_hashinfo *h = PDE_DATA(file_inode(seq->file)); + struct raw_hashinfo *h = pde_data(file_inode(seq->file)); struct raw_iter_state *state = raw_seq_private(seq); for (state->bucket = 0; state->bucket < RAW_HTABLE_SIZE; @@ -987,7 +987,7 @@ found: static struct sock *raw_get_next(struct seq_file *seq, struct sock *sk) { - struct raw_hashinfo *h = PDE_DATA(file_inode(seq->file)); + struct raw_hashinfo *h = pde_data(file_inode(seq->file)); struct raw_iter_state *state = raw_seq_private(seq); do { @@ -1016,7 +1016,7 @@ static struct sock *raw_get_idx(struct seq_file *seq, loff_t pos) void *raw_seq_start(struct seq_file *seq, loff_t *pos) __acquires(&h->lock) { - struct raw_hashinfo *h = PDE_DATA(file_inode(seq->file)); + struct raw_hashinfo *h = pde_data(file_inode(seq->file)); read_lock(&h->lock); return *pos ? raw_get_idx(seq, *pos - 1) : SEQ_START_TOKEN; @@ -1039,7 +1039,7 @@ EXPORT_SYMBOL_GPL(raw_seq_next); void raw_seq_stop(struct seq_file *seq, void *v) __releases(&h->lock) { - struct raw_hashinfo *h = PDE_DATA(file_inode(seq->file)); + struct raw_hashinfo *h = pde_data(file_inode(seq->file)); read_unlock(&h->lock); } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index b3f34e366b27..b53476e78c84 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -3002,7 +3002,7 @@ static unsigned short seq_file_family(const struct seq_file *seq) #endif /* Iterated from proc fs */ - afinfo = PDE_DATA(file_inode(seq->file)); + afinfo = pde_data(file_inode(seq->file)); return afinfo->family; } diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 464590ea922e..090360939401 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -2960,7 +2960,7 @@ static struct sock *udp_get_first(struct seq_file *seq, int start) if (state->bpf_seq_afinfo) afinfo = state->bpf_seq_afinfo; else - afinfo = PDE_DATA(file_inode(seq->file)); + afinfo = pde_data(file_inode(seq->file)); for (state->bucket = start; state->bucket <= afinfo->udp_table->mask; ++state->bucket) { @@ -2993,7 +2993,7 @@ static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) if (state->bpf_seq_afinfo) afinfo = state->bpf_seq_afinfo; else - afinfo = PDE_DATA(file_inode(seq->file)); + afinfo = pde_data(file_inode(seq->file)); do { sk = sk_next(sk); @@ -3050,7 +3050,7 @@ void udp_seq_stop(struct seq_file *seq, void *v) if (state->bpf_seq_afinfo) afinfo = state->bpf_seq_afinfo; else - afinfo = PDE_DATA(file_inode(seq->file)); + afinfo = pde_data(file_inode(seq->file)); if (state->bucket <= afinfo->udp_table->mask) spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 25524e393349..54a489f16b17 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1517,7 +1517,7 @@ EXPORT_SYMBOL_GPL(xt_unregister_table); #ifdef CONFIG_PROC_FS static void *xt_table_seq_start(struct seq_file *seq, loff_t *pos) { - u8 af = (unsigned long)PDE_DATA(file_inode(seq->file)); + u8 af = (unsigned long)pde_data(file_inode(seq->file)); struct net *net = seq_file_net(seq); struct xt_pernet *xt_net; @@ -1529,7 +1529,7 @@ static void *xt_table_seq_start(struct seq_file *seq, loff_t *pos) static void *xt_table_seq_next(struct seq_file *seq, void *v, loff_t *pos) { - u8 af = (unsigned long)PDE_DATA(file_inode(seq->file)); + u8 af = (unsigned long)pde_data(file_inode(seq->file)); struct net *net = seq_file_net(seq); struct xt_pernet *xt_net; @@ -1540,7 +1540,7 @@ static void *xt_table_seq_next(struct seq_file *seq, void *v, loff_t *pos) static void xt_table_seq_stop(struct seq_file *seq, void *v) { - u_int8_t af = (unsigned long)PDE_DATA(file_inode(seq->file)); + u_int8_t af = (unsigned long)pde_data(file_inode(seq->file)); mutex_unlock(&xt[af].mutex); } @@ -1584,7 +1584,7 @@ static void *xt_mttg_seq_next(struct seq_file *seq, void *v, loff_t *ppos, [MTTG_TRAV_NFP_UNSPEC] = MTTG_TRAV_NFP_SPEC, [MTTG_TRAV_NFP_SPEC] = MTTG_TRAV_DONE, }; - uint8_t nfproto = (unsigned long)PDE_DATA(file_inode(seq->file)); + uint8_t nfproto = (unsigned long)pde_data(file_inode(seq->file)); struct nf_mttg_trav *trav = seq->private; if (ppos != NULL) @@ -1633,7 +1633,7 @@ static void *xt_mttg_seq_start(struct seq_file *seq, loff_t *pos, static void xt_mttg_seq_stop(struct seq_file *seq, void *v) { - uint8_t nfproto = (unsigned long)PDE_DATA(file_inode(seq->file)); + uint8_t nfproto = (unsigned long)pde_data(file_inode(seq->file)); struct nf_mttg_trav *trav = seq->private; switch (trav->class) { diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c index 9c5cfd74a0ee..0859b8f76764 100644 --- a/net/netfilter/xt_hashlimit.c +++ b/net/netfilter/xt_hashlimit.c @@ -1052,7 +1052,7 @@ static struct xt_match hashlimit_mt_reg[] __read_mostly = { static void *dl_seq_start(struct seq_file *s, loff_t *pos) __acquires(htable->lock) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket; spin_lock_bh(&htable->lock); @@ -1069,7 +1069,7 @@ static void *dl_seq_start(struct seq_file *s, loff_t *pos) static void *dl_seq_next(struct seq_file *s, void *v, loff_t *pos) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket = v; *pos = ++(*bucket); @@ -1083,7 +1083,7 @@ static void *dl_seq_next(struct seq_file *s, void *v, loff_t *pos) static void dl_seq_stop(struct seq_file *s, void *v) __releases(htable->lock) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket = v; if (!IS_ERR(bucket)) @@ -1125,7 +1125,7 @@ static void dl_seq_print(struct dsthash_ent *ent, u_int8_t family, static int dl_seq_real_show_v2(struct dsthash_ent *ent, u_int8_t family, struct seq_file *s) { - struct xt_hashlimit_htable *ht = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *ht = pde_data(file_inode(s->file)); spin_lock(&ent->lock); /* recalculate to show accurate numbers */ @@ -1140,7 +1140,7 @@ static int dl_seq_real_show_v2(struct dsthash_ent *ent, u_int8_t family, static int dl_seq_real_show_v1(struct dsthash_ent *ent, u_int8_t family, struct seq_file *s) { - struct xt_hashlimit_htable *ht = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *ht = pde_data(file_inode(s->file)); spin_lock(&ent->lock); /* recalculate to show accurate numbers */ @@ -1155,7 +1155,7 @@ static int dl_seq_real_show_v1(struct dsthash_ent *ent, u_int8_t family, static int dl_seq_real_show(struct dsthash_ent *ent, u_int8_t family, struct seq_file *s) { - struct xt_hashlimit_htable *ht = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *ht = pde_data(file_inode(s->file)); spin_lock(&ent->lock); /* recalculate to show accurate numbers */ @@ -1169,7 +1169,7 @@ static int dl_seq_real_show(struct dsthash_ent *ent, u_int8_t family, static int dl_seq_show_v2(struct seq_file *s, void *v) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket = (unsigned int *)v; struct dsthash_ent *ent; @@ -1183,7 +1183,7 @@ static int dl_seq_show_v2(struct seq_file *s, void *v) static int dl_seq_show_v1(struct seq_file *s, void *v) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket = v; struct dsthash_ent *ent; @@ -1197,7 +1197,7 @@ static int dl_seq_show_v1(struct seq_file *s, void *v) static int dl_seq_show(struct seq_file *s, void *v) { - struct xt_hashlimit_htable *htable = PDE_DATA(file_inode(s->file)); + struct xt_hashlimit_htable *htable = pde_data(file_inode(s->file)); unsigned int *bucket = v; struct dsthash_ent *ent; diff --git a/net/netfilter/xt_recent.c b/net/netfilter/xt_recent.c index 0446307516cd..7ddb9a78e3fc 100644 --- a/net/netfilter/xt_recent.c +++ b/net/netfilter/xt_recent.c @@ -551,7 +551,7 @@ static int recent_seq_open(struct inode *inode, struct file *file) if (st == NULL) return -ENOMEM; - st->table = PDE_DATA(inode); + st->table = pde_data(inode); return 0; } @@ -559,7 +559,7 @@ static ssize_t recent_mt_proc_write(struct file *file, const char __user *input, size_t size, loff_t *loff) { - struct recent_table *t = PDE_DATA(file_inode(file)); + struct recent_table *t = pde_data(file_inode(file)); struct recent_entry *e; char buf[sizeof("+b335:1d35:1e55:dead:c0de:1715:5afe:c0de")]; const char *c = buf; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index b87565b64928..c2ba9d4cd2c7 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -1433,7 +1433,7 @@ static bool use_gss_proxy(struct net *net) static ssize_t write_gssp(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - struct net *net = PDE_DATA(file_inode(file)); + struct net *net = pde_data(file_inode(file)); char tbuf[20]; unsigned long i; int res; @@ -1461,7 +1461,7 @@ static ssize_t write_gssp(struct file *file, const char __user *buf, static ssize_t read_gssp(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - struct net *net = PDE_DATA(file_inode(file)); + struct net *net = pde_data(file_inode(file)); struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); unsigned long p = *ppos; char tbuf[10]; diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 59641803472c..bb1177395b99 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -1536,7 +1536,7 @@ static ssize_t write_flush(struct file *file, const char __user *buf, static ssize_t cache_read_procfs(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - struct cache_detail *cd = PDE_DATA(file_inode(filp)); + struct cache_detail *cd = pde_data(file_inode(filp)); return cache_read(filp, buf, count, ppos, cd); } @@ -1544,14 +1544,14 @@ static ssize_t cache_read_procfs(struct file *filp, char __user *buf, static ssize_t cache_write_procfs(struct file *filp, const char __user *buf, size_t count, loff_t *ppos) { - struct cache_detail *cd = PDE_DATA(file_inode(filp)); + struct cache_detail *cd = pde_data(file_inode(filp)); return cache_write(filp, buf, count, ppos, cd); } static __poll_t cache_poll_procfs(struct file *filp, poll_table *wait) { - struct cache_detail *cd = PDE_DATA(file_inode(filp)); + struct cache_detail *cd = pde_data(file_inode(filp)); return cache_poll(filp, wait, cd); } @@ -1560,21 +1560,21 @@ static long cache_ioctl_procfs(struct file *filp, unsigned int cmd, unsigned long arg) { struct inode *inode = file_inode(filp); - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return cache_ioctl(inode, filp, cmd, arg, cd); } static int cache_open_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return cache_open(inode, filp, cd); } static int cache_release_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return cache_release(inode, filp, cd); } @@ -1591,14 +1591,14 @@ static const struct proc_ops cache_channel_proc_ops = { static int content_open_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return content_open(inode, filp, cd); } static int content_release_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return content_release(inode, filp, cd); } @@ -1612,14 +1612,14 @@ static const struct proc_ops content_proc_ops = { static int open_flush_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return open_flush(inode, filp, cd); } static int release_flush_procfs(struct inode *inode, struct file *filp) { - struct cache_detail *cd = PDE_DATA(inode); + struct cache_detail *cd = pde_data(inode); return release_flush(inode, filp, cd); } @@ -1627,7 +1627,7 @@ static int release_flush_procfs(struct inode *inode, struct file *filp) static ssize_t read_flush_procfs(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - struct cache_detail *cd = PDE_DATA(file_inode(filp)); + struct cache_detail *cd = pde_data(file_inode(filp)); return read_flush(filp, buf, count, ppos, cd); } @@ -1636,7 +1636,7 @@ static ssize_t write_flush_procfs(struct file *filp, const char __user *buf, size_t count, loff_t *ppos) { - struct cache_detail *cd = PDE_DATA(file_inode(filp)); + struct cache_detail *cd = pde_data(file_inode(filp)); return write_flush(filp, buf, count, ppos, cd); } diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c index c964b48eaaba..52908f9e6eab 100644 --- a/net/sunrpc/stats.c +++ b/net/sunrpc/stats.c @@ -66,7 +66,7 @@ static int rpc_proc_show(struct seq_file *seq, void *v) { static int rpc_proc_open(struct inode *inode, struct file *file) { - return single_open(file, rpc_proc_show, PDE_DATA(inode)); + return single_open(file, rpc_proc_show, pde_data(inode)); } static const struct proc_ops rpc_proc_ops = { diff --git a/sound/core/info.c b/sound/core/info.c index a451b24199c3..782fba87cc04 100644 --- a/sound/core/info.c +++ b/sound/core/info.c @@ -234,7 +234,7 @@ static int snd_info_entry_mmap(struct file *file, struct vm_area_struct *vma) static int snd_info_entry_open(struct inode *inode, struct file *file) { - struct snd_info_entry *entry = PDE_DATA(inode); + struct snd_info_entry *entry = pde_data(inode); struct snd_info_private_data *data; int mode, err; @@ -365,7 +365,7 @@ static int snd_info_seq_show(struct seq_file *seq, void *p) static int snd_info_text_entry_open(struct inode *inode, struct file *file) { - struct snd_info_entry *entry = PDE_DATA(inode); + struct snd_info_entry *entry = pde_data(inode); struct snd_info_private_data *data; int err; -- cgit v1.2.3-59-g8ed1b From 2dba5eb1c73b6ba2988ced07250edeac0f8cbf5a Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 21 Jan 2022 22:14:27 -0800 Subject: lib/stackdepot: allow optional init and stack_table allocation by kvmalloc() Currently, enabling CONFIG_STACKDEPOT means its stack_table will be allocated from memblock, even if stack depot ends up not actually used. The default size of stack_table is 4MB on 32-bit, 8MB on 64-bit. This is fine for use-cases such as KASAN which is also a config option and has overhead on its own. But it's an issue for functionality that has to be actually enabled on boot (page_owner) or depends on hardware (GPU drivers) and thus the memory might be wasted. This was raised as an issue [1] when attempting to add stackdepot support for SLUB's debug object tracking functionality. It's common to build kernels with CONFIG_SLUB_DEBUG and enable slub_debug on boot only when needed, or create only specific kmem caches with debugging for testing purposes. It would thus be more efficient if stackdepot's table was allocated only when actually going to be used. This patch thus makes the allocation (and whole stack_depot_init() call) optional: - Add a CONFIG_STACKDEPOT_ALWAYS_INIT flag to keep using the current well-defined point of allocation as part of mem_init(). Make CONFIG_KASAN select this flag. - Other users have to call stack_depot_init() as part of their own init when it's determined that stack depot will actually be used. This may depend on both config and runtime conditions. Convert current users which are page_owner and several in the DRM subsystem. Same will be done for SLUB later. - Because the init might now be called after the boot-time memblock allocation has given all memory to the buddy allocator, change stack_depot_init() to allocate stack_table with kvmalloc() when memblock is no longer available. Also handle allocation failure by disabling stackdepot (could have theoretically happened even with memblock allocation previously), and don't unnecessarily align the memblock allocation to its own size anymore. [1] https://lore.kernel.org/all/CAMuHMdW=eoVzM1Re5FVoEN87nKfiLmM2+Ah7eNu2KXEhCvbZyA@mail.gmail.com/ Link: https://lkml.kernel.org/r/20211013073005.11351-1-vbabka@suse.cz Signed-off-by: Vlastimil Babka Acked-by: Dmitry Vyukov Reviewed-by: Marco Elver # stackdepot Cc: Marco Elver Cc: Vijayanand Jitta Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Cc: David Airlie Cc: Daniel Vetter Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Geert Uytterhoeven Cc: Oliver Glitta Cc: Imran Khan From: Colin Ian King Subject: lib/stackdepot: fix spelling mistake and grammar in pr_err message There is a spelling mistake of the work allocation so fix this and re-phrase the message to make it easier to read. Link: https://lkml.kernel.org/r/20211015104159.11282-1-colin.king@canonical.com Signed-off-by: Colin Ian King Cc: Vlastimil Babka From: Vlastimil Babka Subject: lib/stackdepot: allow optional init and stack_table allocation by kvmalloc() - fixup On FLATMEM, we call page_ext_init_flatmem_late() just before kmem_cache_init() which means stack_depot_init() (called by page owner init) will not recognize properly it should use kvmalloc() and not memblock_alloc(). memblock_alloc() will also not issue a warning and return a block memory that can be invalid and cause kernel page fault when saving stacks, as reported by the kernel test robot [1]. Fix this by moving page_ext_init_flatmem_late() below kmem_cache_init() so that slab_is_available() is true during stack_depot_init(). SPARSEMEM doesn't have this issue, as it doesn't do page_ext_init_flatmem_late(), but a different page_ext_init() even later in the boot process. Thanks to Mike Rapoport for pointing out the FLATMEM init ordering issue. While at it, also actually resolve a checkpatch warning in stack_depot_init() from DRM CI, which was supposed to be in the original patch already. [1] https://lore.kernel.org/all/20211014085450.GC18719@xsang-OptiPlex-9020/ Link: https://lkml.kernel.org/r/6abd9213-19a9-6d58-cedc-2414386d2d81@suse.cz Signed-off-by: Vlastimil Babka Reported-by: kernel test robot Cc: Mike Rapoport Cc: Stephen Rothwell From: Vlastimil Babka Subject: lib/stackdepot: allow optional init and stack_table allocation by kvmalloc() - fixup3 Due to cd06ab2fd48f ("drm/locking: add backtrace for locking contended locks without backoff") landing recently to -next adding a new stack depot user in drivers/gpu/drm/drm_modeset_lock.c we need to add an appropriate call to stack_depot_init() there as well. Link: https://lkml.kernel.org/r/2a692365-cfa1-64f2-34e0-8aa5674dce5e@suse.cz Signed-off-by: Vlastimil Babka Cc: Jani Nikula Cc: Naresh Kamboju Cc: Marco Elver Cc: Vijayanand Jitta Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Cc: David Airlie Cc: Daniel Vetter Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Geert Uytterhoeven Cc: Oliver Glitta Cc: Imran Khan Cc: Stephen Rothwell From: Vlastimil Babka Subject: lib/stackdepot: allow optional init and stack_table allocation by kvmalloc() - fixup4 Due to 4e66934eaadc ("lib: add reference counting tracking infrastructure") landing recently to net-next adding a new stack depot user in lib/ref_tracker.c we need to add an appropriate call to stack_depot_init() there as well. Link: https://lkml.kernel.org/r/45c1b738-1a2f-5b5f-2f6d-86fab206d01c@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: Eric Dumazet Cc: Jiri Slab Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/gpu/drm/drm_dp_mst_topology.c | 1 + drivers/gpu/drm/drm_mm.c | 4 ++++ drivers/gpu/drm/drm_modeset_lock.c | 9 +++++++++ drivers/gpu/drm/i915/intel_runtime_pm.c | 3 +++ include/linux/ref_tracker.h | 2 ++ include/linux/stackdepot.h | 25 ++++++++++++++++--------- init/main.c | 9 ++++++--- lib/Kconfig | 4 ++++ lib/Kconfig.kasan | 2 +- lib/stackdepot.c | 33 ++++++++++++++++++++++++++++----- mm/page_owner.c | 2 ++ 11 files changed, 76 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index f3d79eda94bb..8b3822142fed 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -5511,6 +5511,7 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr, mutex_init(&mgr->probe_lock); #if IS_ENABLED(CONFIG_DRM_DEBUG_DP_MST_TOPOLOGY_REFS) mutex_init(&mgr->topology_ref_history_lock); + stack_depot_init(); #endif INIT_LIST_HEAD(&mgr->tx_msg_downq); INIT_LIST_HEAD(&mgr->destroy_port_list); diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index 7d1c578388d3..8257f9d4f619 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -980,6 +980,10 @@ void drm_mm_init(struct drm_mm *mm, u64 start, u64 size) add_hole(&mm->head_node); mm->scan_active = 0; + +#ifdef CONFIG_DRM_DEBUG_MM + stack_depot_init(); +#endif } EXPORT_SYMBOL(drm_mm_init); diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index c97323365675..918065982db4 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -107,6 +107,11 @@ static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) kfree(buf); } + +static void __drm_stack_depot_init(void) +{ + stack_depot_init(); +} #else /* CONFIG_DRM_DEBUG_MODESET_LOCK */ static depot_stack_handle_t __drm_stack_depot_save(void) { @@ -115,6 +120,9 @@ static depot_stack_handle_t __drm_stack_depot_save(void) static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) { } +static void __drm_stack_depot_init(void) +{ +} #endif /* CONFIG_DRM_DEBUG_MODESET_LOCK */ /** @@ -359,6 +367,7 @@ void drm_modeset_lock_init(struct drm_modeset_lock *lock) { ww_mutex_init(&lock->mutex, &crtc_ww_class); INIT_LIST_HEAD(&lock->head); + __drm_stack_depot_init(); } EXPORT_SYMBOL(drm_modeset_lock_init); diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 22dab36afcb6..53f1ccb78849 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -68,6 +68,9 @@ static noinline depot_stack_handle_t __save_depot_stack(void) static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) { spin_lock_init(&rpm->debug.lock); + + if (rpm->available) + stack_depot_init(); } static noinline depot_stack_handle_t diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h index c11c9db5825c..60f3453be23e 100644 --- a/include/linux/ref_tracker.h +++ b/include/linux/ref_tracker.h @@ -4,6 +4,7 @@ #include #include #include +#include struct ref_tracker; @@ -26,6 +27,7 @@ static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, spin_lock_init(&dir->lock); dir->quarantine_avail = quarantine_count; refcount_set(&dir->untracked, 1); + stack_depot_init(); } void ref_tracker_dir_exit(struct ref_tracker_dir *dir); diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index c34b55a6e554..17f992fe6355 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -19,6 +19,22 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t gfp_flags, bool can_alloc); +/* + * Every user of stack depot has to call this during its own init when it's + * decided that it will be calling stack_depot_save() later. + * + * The alternative is to select STACKDEPOT_ALWAYS_INIT to have stack depot + * enabled as part of mm_init(), for subsystems where it's known at compile time + * that stack depot will be used. + */ +int stack_depot_init(void); + +#ifdef CONFIG_STACKDEPOT_ALWAYS_INIT +static inline int stack_depot_early_init(void) { return stack_depot_init(); } +#else +static inline int stack_depot_early_init(void) { return 0; } +#endif + depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t gfp_flags); @@ -30,13 +46,4 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, void stack_depot_print(depot_stack_handle_t stack); -#ifdef CONFIG_STACKDEPOT -int stack_depot_init(void); -#else -static inline int stack_depot_init(void) -{ - return 0; -} -#endif /* CONFIG_STACKDEPOT */ - #endif diff --git a/init/main.c b/init/main.c index bb984ed79de0..65fa2e41a9c0 100644 --- a/init/main.c +++ b/init/main.c @@ -834,12 +834,15 @@ static void __init mm_init(void) init_mem_debugging_and_hardening(); kfence_alloc_pool(); report_meminit(); - stack_depot_init(); + stack_depot_early_init(); mem_init(); mem_init_print_info(); - /* page_owner must be initialized after buddy is ready */ - page_ext_init_flatmem_late(); kmem_cache_init(); + /* + * page_owner must be initialized after buddy is ready, and also after + * slab is ready so that stack_depot_init() works properly + */ + page_ext_init_flatmem_late(); kmemleak_init(); pgtable_init(); debug_objects_mem_init(); diff --git a/lib/Kconfig b/lib/Kconfig index c20b68ad2bc3..51c368a50b16 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -673,6 +673,10 @@ config STACKDEPOT bool select STACKTRACE +config STACKDEPOT_ALWAYS_INIT + bool + select STACKDEPOT + config STACK_HASH_ORDER int "stack depot hash size (12 => 4KB, 20 => 1024KB)" range 12 20 diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index cdc842d090db..879757b6dd14 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -38,7 +38,7 @@ menuconfig KASAN CC_HAS_WORKING_NOSANITIZE_ADDRESS) || \ HAVE_ARCH_KASAN_HW_TAGS depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) - select STACKDEPOT + select STACKDEPOT_ALWAYS_INIT help Enables KASAN (KernelAddressSANitizer) - runtime memory debugger, designed to find out-of-bounds accesses and use-after-free bugs. diff --git a/lib/stackdepot.c b/lib/stackdepot.c index b437ae79aca1..00ccb106f1a8 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -161,18 +162,40 @@ static int __init is_stack_depot_disabled(char *str) } early_param("stack_depot_disable", is_stack_depot_disabled); -int __init stack_depot_init(void) +/* + * __ref because of memblock_alloc(), which will not be actually called after + * the __init code is gone, because at that point slab_is_available() is true + */ +__ref int stack_depot_init(void) { - if (!stack_depot_disable) { + static DEFINE_MUTEX(stack_depot_init_mutex); + + mutex_lock(&stack_depot_init_mutex); + if (!stack_depot_disable && !stack_table) { size_t size = (STACK_HASH_SIZE * sizeof(struct stack_record *)); int i; - stack_table = memblock_alloc(size, size); - for (i = 0; i < STACK_HASH_SIZE; i++) - stack_table[i] = NULL; + if (slab_is_available()) { + pr_info("Stack Depot allocating hash table with kvmalloc\n"); + stack_table = kvmalloc(size, GFP_KERNEL); + } else { + pr_info("Stack Depot allocating hash table with memblock_alloc\n"); + stack_table = memblock_alloc(size, SMP_CACHE_BYTES); + } + if (stack_table) { + for (i = 0; i < STACK_HASH_SIZE; i++) + stack_table[i] = NULL; + } else { + pr_err("Stack Depot hash table allocation failed, disabling\n"); + stack_depot_disable = true; + mutex_unlock(&stack_depot_init_mutex); + return -ENOMEM; + } } + mutex_unlock(&stack_depot_init_mutex); return 0; } +EXPORT_SYMBOL_GPL(stack_depot_init); /* Calculate hash for a stack */ static inline u32 hash_stack(unsigned long *entries, unsigned int size) diff --git a/mm/page_owner.c b/mm/page_owner.c index 5eea061bb1e5..99e360df9465 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -80,6 +80,8 @@ static __init void init_page_owner(void) if (!page_owner_enabled) return; + stack_depot_init(); + register_dummy_stack(); register_failure_stack(); register_early_stack(); -- cgit v1.2.3-59-g8ed1b From 0a4ee518185e902758191d968600399f3bc2be31 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:34 -0800 Subject: mm: remove cleancache Patch series "remove Xen tmem leftovers". Since the removal of the Xen tmem driver in 2019, the cleancache hooks are entirely unused, as are large parts of frontswap. This series against linux-next (with the folio changes included) removes cleancaches, and cuts down frontswap to the bits actually used by zswap. This patch (of 13): The cleancache subsystem is unused since the removal of Xen tmem driver in commit 814bbf49dcd0 ("xen: remove tmem driver"). [akpm@linux-foundation.org: remove now-unreachable code] Link: https://lkml.kernel.org/r/20211224062246.1258487-1-hch@lst.de Link: https://lkml.kernel.org/r/20211224062246.1258487-2-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Acked-by: Geert Uytterhoeven Cc: Konrad Rzeszutek Wilk Cc: Hugh Dickins Cc: Seth Jennings Cc: Dan Streetman Cc: Vitaly Wool Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/cleancache.rst | 296 -------------------------------- Documentation/vm/frontswap.rst | 12 +- Documentation/vm/index.rst | 1 - MAINTAINERS | 7 - arch/arm/configs/bcm2835_defconfig | 1 - arch/arm/configs/qcom_defconfig | 1 - arch/m68k/configs/amiga_defconfig | 1 - arch/m68k/configs/apollo_defconfig | 1 - arch/m68k/configs/atari_defconfig | 1 - arch/m68k/configs/bvme6000_defconfig | 1 - arch/m68k/configs/hp300_defconfig | 1 - arch/m68k/configs/mac_defconfig | 1 - arch/m68k/configs/multi_defconfig | 1 - arch/m68k/configs/mvme147_defconfig | 1 - arch/m68k/configs/mvme16x_defconfig | 1 - arch/m68k/configs/q40_defconfig | 1 - arch/m68k/configs/sun3_defconfig | 1 - arch/m68k/configs/sun3x_defconfig | 1 - arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - block/bdev.c | 5 - fs/btrfs/extent_io.c | 10 -- fs/btrfs/super.c | 2 - fs/ext4/readpage.c | 6 - fs/ext4/super.c | 3 - fs/f2fs/data.c | 13 -- fs/mpage.c | 7 - fs/ntfs3/ntfs_fs.h | 1 - fs/ocfs2/super.c | 2 - fs/super.c | 3 - include/linux/cleancache.h | 124 -------------- include/linux/fs.h | 5 - mm/Kconfig | 22 --- mm/Makefile | 1 - mm/cleancache.c | 315 ----------------------------------- mm/filemap.c | 11 -- mm/truncate.c | 15 +- 37 files changed, 4 insertions(+), 873 deletions(-) delete mode 100644 Documentation/vm/cleancache.rst delete mode 100644 include/linux/cleancache.h delete mode 100644 mm/cleancache.c (limited to 'include/linux') diff --git a/Documentation/vm/cleancache.rst b/Documentation/vm/cleancache.rst deleted file mode 100644 index 68cba9131c31..000000000000 --- a/Documentation/vm/cleancache.rst +++ /dev/null @@ -1,296 +0,0 @@ -.. _cleancache: - -========== -Cleancache -========== - -Motivation -========== - -Cleancache is a new optional feature provided by the VFS layer that -potentially dramatically increases page cache effectiveness for -many workloads in many environments at a negligible cost. - -Cleancache can be thought of as a page-granularity victim cache for clean -pages that the kernel's pageframe replacement algorithm (PFRA) would like -to keep around, but can't since there isn't enough memory. So when the -PFRA "evicts" a page, it first attempts to use cleancache code to -put the data contained in that page into "transcendent memory", memory -that is not directly accessible or addressable by the kernel and is -of unknown and possibly time-varying size. - -Later, when a cleancache-enabled filesystem wishes to access a page -in a file on disk, it first checks cleancache to see if it already -contains it; if it does, the page of data is copied into the kernel -and a disk access is avoided. - -Transcendent memory "drivers" for cleancache are currently implemented -in Xen (using hypervisor memory) and zcache (using in-kernel compressed -memory) and other implementations are in development. - -:ref:`FAQs ` are included below. - -Implementation Overview -======================= - -A cleancache "backend" that provides transcendent memory registers itself -to the kernel's cleancache "frontend" by calling cleancache_register_ops, -passing a pointer to a cleancache_ops structure with funcs set appropriately. -The functions provided must conform to certain semantics as follows: - -Most important, cleancache is "ephemeral". Pages which are copied into -cleancache have an indefinite lifetime which is completely unknowable -by the kernel and so may or may not still be in cleancache at any later time. -Thus, as its name implies, cleancache is not suitable for dirty pages. -Cleancache has complete discretion over what pages to preserve and what -pages to discard and when. - -Mounting a cleancache-enabled filesystem should call "init_fs" to obtain a -pool id which, if positive, must be saved in the filesystem's superblock; -a negative return value indicates failure. A "put_page" will copy a -(presumably about-to-be-evicted) page into cleancache and associate it with -the pool id, a file key, and a page index into the file. (The combination -of a pool id, a file key, and an index is sometimes called a "handle".) -A "get_page" will copy the page, if found, from cleancache into kernel memory. -An "invalidate_page" will ensure the page no longer is present in cleancache; -an "invalidate_inode" will invalidate all pages associated with the specified -file; and, when a filesystem is unmounted, an "invalidate_fs" will invalidate -all pages in all files specified by the given pool id and also surrender -the pool id. - -An "init_shared_fs", like init_fs, obtains a pool id but tells cleancache -to treat the pool as shared using a 128-bit UUID as a key. On systems -that may run multiple kernels (such as hard partitioned or virtualized -systems) that may share a clustered filesystem, and where cleancache -may be shared among those kernels, calls to init_shared_fs that specify the -same UUID will receive the same pool id, thus allowing the pages to -be shared. Note that any security requirements must be imposed outside -of the kernel (e.g. by "tools" that control cleancache). Or a -cleancache implementation can simply disable shared_init by always -returning a negative value. - -If a get_page is successful on a non-shared pool, the page is invalidated -(thus making cleancache an "exclusive" cache). On a shared pool, the page -is NOT invalidated on a successful get_page so that it remains accessible to -other sharers. The kernel is responsible for ensuring coherency between -cleancache (shared or not), the page cache, and the filesystem, using -cleancache invalidate operations as required. - -Note that cleancache must enforce put-put-get coherency and get-get -coherency. For the former, if two puts are made to the same handle but -with different data, say AAA by the first put and BBB by the second, a -subsequent get can never return the stale data (AAA). For get-get coherency, -if a get for a given handle fails, subsequent gets for that handle will -never succeed unless preceded by a successful put with that handle. - -Last, cleancache provides no SMP serialization guarantees; if two -different Linux threads are simultaneously putting and invalidating a page -with the same handle, the results are indeterminate. Callers must -lock the page to ensure serial behavior. - -Cleancache Performance Metrics -============================== - -If properly configured, monitoring of cleancache is done via debugfs in -the `/sys/kernel/debug/cleancache` directory. The effectiveness of cleancache -can be measured (across all filesystems) with: - -``succ_gets`` - number of gets that were successful - -``failed_gets`` - number of gets that failed - -``puts`` - number of puts attempted (all "succeed") - -``invalidates`` - number of invalidates attempted - -A backend implementation may provide additional metrics. - -.. _faq: - -FAQ -=== - -* Where's the value? (Andrew Morton) - -Cleancache provides a significant performance benefit to many workloads -in many environments with negligible overhead by improving the -effectiveness of the pagecache. Clean pagecache pages are -saved in transcendent memory (RAM that is otherwise not directly -addressable to the kernel); fetching those pages later avoids "refaults" -and thus disk reads. - -Cleancache (and its sister code "frontswap") provide interfaces for -this transcendent memory (aka "tmem"), which conceptually lies between -fast kernel-directly-addressable RAM and slower DMA/asynchronous devices. -Disallowing direct kernel or userland reads/writes to tmem -is ideal when data is transformed to a different form and size (such -as with compression) or secretly moved (as might be useful for write- -balancing for some RAM-like devices). Evicted page-cache pages (and -swap pages) are a great use for this kind of slower-than-RAM-but-much- -faster-than-disk transcendent memory, and the cleancache (and frontswap) -"page-object-oriented" specification provides a nice way to read and -write -- and indirectly "name" -- the pages. - -In the virtual case, the whole point of virtualization is to statistically -multiplex physical resources across the varying demands of multiple -virtual machines. This is really hard to do with RAM and efforts to -do it well with no kernel change have essentially failed (except in some -well-publicized special-case workloads). Cleancache -- and frontswap -- -with a fairly small impact on the kernel, provide a huge amount -of flexibility for more dynamic, flexible RAM multiplexing. -Specifically, the Xen Transcendent Memory backend allows otherwise -"fallow" hypervisor-owned RAM to not only be "time-shared" between multiple -virtual machines, but the pages can be compressed and deduplicated to -optimize RAM utilization. And when guest OS's are induced to surrender -underutilized RAM (e.g. with "self-ballooning"), page cache pages -are the first to go, and cleancache allows those pages to be -saved and reclaimed if overall host system memory conditions allow. - -And the identical interface used for cleancache can be used in -physical systems as well. The zcache driver acts as a memory-hungry -device that stores pages of data in a compressed state. And -the proposed "RAMster" driver shares RAM across multiple physical -systems. - -* Why does cleancache have its sticky fingers so deep inside the - filesystems and VFS? (Andrew Morton and Christoph Hellwig) - -The core hooks for cleancache in VFS are in most cases a single line -and the minimum set are placed precisely where needed to maintain -coherency (via cleancache_invalidate operations) between cleancache, -the page cache, and disk. All hooks compile into nothingness if -cleancache is config'ed off and turn into a function-pointer- -compare-to-NULL if config'ed on but no backend claims the ops -functions, or to a compare-struct-element-to-negative if a -backend claims the ops functions but a filesystem doesn't enable -cleancache. - -Some filesystems are built entirely on top of VFS and the hooks -in VFS are sufficient, so don't require an "init_fs" hook; the -initial implementation of cleancache didn't provide this hook. -But for some filesystems (such as btrfs), the VFS hooks are -incomplete and one or more hooks in fs-specific code are required. -And for some other filesystems, such as tmpfs, cleancache may -be counterproductive. So it seemed prudent to require a filesystem -to "opt in" to use cleancache, which requires adding a hook in -each filesystem. Not all filesystems are supported by cleancache -only because they haven't been tested. The existing set should -be sufficient to validate the concept, the opt-in approach means -that untested filesystems are not affected, and the hooks in the -existing filesystems should make it very easy to add more -filesystems in the future. - -The total impact of the hooks to existing fs and mm files is only -about 40 lines added (not counting comments and blank lines). - -* Why not make cleancache asynchronous and batched so it can more - easily interface with real devices with DMA instead of copying each - individual page? (Minchan Kim) - -The one-page-at-a-time copy semantics simplifies the implementation -on both the frontend and backend and also allows the backend to -do fancy things on-the-fly like page compression and -page deduplication. And since the data is "gone" (copied into/out -of the pageframe) before the cleancache get/put call returns, -a great deal of race conditions and potential coherency issues -are avoided. While the interface seems odd for a "real device" -or for real kernel-addressable RAM, it makes perfect sense for -transcendent memory. - -* Why is non-shared cleancache "exclusive"? And where is the - page "invalidated" after a "get"? (Minchan Kim) - -The main reason is to free up space in transcendent memory and -to avoid unnecessary cleancache_invalidate calls. If you want inclusive, -the page can be "put" immediately following the "get". If -put-after-get for inclusive becomes common, the interface could -be easily extended to add a "get_no_invalidate" call. - -The invalidate is done by the cleancache backend implementation. - -* What's the performance impact? - -Performance analysis has been presented at OLS'09 and LCA'10. -Briefly, performance gains can be significant on most workloads, -especially when memory pressure is high (e.g. when RAM is -overcommitted in a virtual workload); and because the hooks are -invoked primarily in place of or in addition to a disk read/write, -overhead is negligible even in worst case workloads. Basically -cleancache replaces I/O with memory-copy-CPU-overhead; on older -single-core systems with slow memory-copy speeds, cleancache -has little value, but in newer multicore machines, especially -consolidated/virtualized machines, it has great value. - -* How do I add cleancache support for filesystem X? (Boaz Harrash) - -Filesystems that are well-behaved and conform to certain -restrictions can utilize cleancache simply by making a call to -cleancache_init_fs at mount time. Unusual, misbehaving, or -poorly layered filesystems must either add additional hooks -and/or undergo extensive additional testing... or should just -not enable the optional cleancache. - -Some points for a filesystem to consider: - - - The FS should be block-device-based (e.g. a ram-based FS such - as tmpfs should not enable cleancache) - - To ensure coherency/correctness, the FS must ensure that all - file removal or truncation operations either go through VFS or - add hooks to do the equivalent cleancache "invalidate" operations - - To ensure coherency/correctness, either inode numbers must - be unique across the lifetime of the on-disk file OR the - FS must provide an "encode_fh" function. - - The FS must call the VFS superblock alloc and deactivate routines - or add hooks to do the equivalent cleancache calls done there. - - To maximize performance, all pages fetched from the FS should - go through the do_mpag_readpage routine or the FS should add - hooks to do the equivalent (cf. btrfs) - - Currently, the FS blocksize must be the same as PAGESIZE. This - is not an architectural restriction, but no backends currently - support anything different. - - A clustered FS should invoke the "shared_init_fs" cleancache - hook to get best performance for some backends. - -* Why not use the KVA of the inode as the key? (Christoph Hellwig) - -If cleancache would use the inode virtual address instead of -inode/filehandle, the pool id could be eliminated. But, this -won't work because cleancache retains pagecache data pages -persistently even when the inode has been pruned from the -inode unused list, and only invalidates the data page if the file -gets removed/truncated. So if cleancache used the inode kva, -there would be potential coherency issues if/when the inode -kva is reused for a different file. Alternately, if cleancache -invalidated the pages when the inode kva was freed, much of the value -of cleancache would be lost because the cache of pages in cleanache -is potentially much larger than the kernel pagecache and is most -useful if the pages survive inode cache removal. - -* Why is a global variable required? - -The cleancache_enabled flag is checked in all of the frequently-used -cleancache hooks. The alternative is a function call to check a static -variable. Since cleancache is enabled dynamically at runtime, systems -that don't enable cleancache would suffer thousands (possibly -tens-of-thousands) of unnecessary function calls per second. So the -global variable allows cleancache to be enabled by default at compile -time, but have insignificant performance impact when cleancache remains -disabled at runtime. - -* Does cleanache work with KVM? - -The memory model of KVM is sufficiently different that a cleancache -backend may have less value for KVM. This remains to be tested, -especially in an overcommitted system. - -* Does cleancache work in userspace? It sounds useful for - memory hungry caches like web browsers. (Jamie Lokier) - -No plans yet, though we agree it sounds useful, at least for -apps that bypass the page cache (e.g. O_DIRECT). - -Last updated: Dan Magenheimer, April 13 2011 diff --git a/Documentation/vm/frontswap.rst b/Documentation/vm/frontswap.rst index 1979f430c1c5..e2e5ab3e375e 100644 --- a/Documentation/vm/frontswap.rst +++ b/Documentation/vm/frontswap.rst @@ -8,12 +8,6 @@ Frontswap provides a "transcendent memory" interface for swap pages. In some environments, dramatic performance savings may be obtained because swapped pages are saved in RAM (or a RAM-like device) instead of a swap disk. -(Note, frontswap -- and :ref:`cleancache` (merged at 3.0) -- are the "frontends" -and the only necessary changes to the core kernel for transcendent memory; -all other supporting code -- the "backends" -- is implemented as drivers. -See the LWN.net article `Transcendent memory in a nutshell`_ -for a detailed overview of frontswap and related kernel parts) - .. _Transcendent memory in a nutshell: https://lwn.net/Articles/454795/ Frontswap is so named because it can be thought of as the opposite of @@ -87,11 +81,9 @@ This interface is ideal when data is transformed to a different form and size (such as with compression) or secretly moved (as might be useful for write-balancing for some RAM-like devices). Swap pages (and evicted page-cache pages) are a great use for this kind of slower-than-RAM- -but-much-faster-than-disk "pseudo-RAM device" and the frontswap (and -cleancache) interface to transcendent memory provides a nice way to read -and write -- and indirectly "name" -- the pages. +but-much-faster-than-disk "pseudo-RAM device". -Frontswap -- and cleancache -- with a fairly small impact on the kernel, +Frontswap with a fairly small impact on the kernel, provides a huge amount of flexibility for more dynamic, flexible RAM utilization in various system configurations: diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst index 932440805453..44365c4574a3 100644 --- a/Documentation/vm/index.rst +++ b/Documentation/vm/index.rst @@ -15,7 +15,6 @@ algorithms. If you are looking for advice on simply allocating memory, see the active_mm arch_pgtable_helpers balance - cleancache damon/index free_page_reporting frontswap diff --git a/MAINTAINERS b/MAINTAINERS index 6c08cbe953fe..45f8750f6e30 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4705,13 +4705,6 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/cla F: include/linux/cfi.h F: kernel/cfi.c -CLEANCACHE API -M: Konrad Rzeszutek Wilk -L: linux-kernel@vger.kernel.org -S: Maintained -F: include/linux/cleancache.h -F: mm/cleancache.c - CLK API M: Russell King L: linux-clk@vger.kernel.org diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig index 383c632eba7b..a9ed79b7f871 100644 --- a/arch/arm/configs/bcm2835_defconfig +++ b/arch/arm/configs/bcm2835_defconfig @@ -31,7 +31,6 @@ CONFIG_ARCH_BCM2835=y CONFIG_PREEMPT_VOLUNTARY=y CONFIG_AEABI=y CONFIG_KSM=y -CONFIG_CLEANCACHE=y CONFIG_CMA=y CONFIG_SECCOMP=y CONFIG_KEXEC=y diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig index 0daa9c0d298e..9981566f2096 100644 --- a/arch/arm/configs/qcom_defconfig +++ b/arch/arm/configs/qcom_defconfig @@ -27,7 +27,6 @@ CONFIG_PCIE_QCOM=y CONFIG_SMP=y CONFIG_PREEMPT=y CONFIG_HIGHMEM=y -CONFIG_CLEANCACHE=y CONFIG_ARM_APPENDED_DTB=y CONFIG_ARM_ATAG_DTB_COMPAT=y CONFIG_CPU_IDLE=y diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index a4b6c7108465..bc9952f8be66 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -45,7 +45,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 2db721965520..a77269c6e5ba 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -41,7 +41,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index c266a704eecd..7a74efa6b9a1 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -48,7 +48,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index f644f08dd6ed..a5323bf2eb33 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -38,7 +38,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index e4924650b687..5e80aa0869d5 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -40,7 +40,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 24113871ea76..e84326a3f62d 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -39,7 +39,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 6a7e4be70eea..337552f43339 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -59,7 +59,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 1d223247aff0..7b688f7d272a 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -37,7 +37,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 961f789f96c9..7c2cb31d63dd 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -38,7 +38,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index ff4b5e469390..ca43897af26d 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -39,7 +39,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 5f228621d0cc..e3d515f37144 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -35,7 +35,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index a600cb9e68c2..d601606c969b 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -35,7 +35,6 @@ CONFIG_IOSCHED_BFQ=m CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set -CONFIG_CLEANCACHE=y CONFIG_ZPOOL=m CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 354e51dcb3e2..7fe8975b49ec 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -96,7 +96,6 @@ CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_CLEANCACHE=y CONFIG_FRONTSWAP=y CONFIG_CMA_DEBUG=y CONFIG_CMA_DEBUGFS=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 8dee6c3782f3..466780c465f5 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -91,7 +91,6 @@ CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_CLEANCACHE=y CONFIG_FRONTSWAP=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=7 diff --git a/block/bdev.c b/block/bdev.c index 8bf93a19041b..102837a37051 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include "../fs/internal.h" @@ -88,10 +87,6 @@ void invalidate_bdev(struct block_device *bdev) lru_add_drain_all(); /* make sure all lru add caches are flushed */ invalidate_mapping_pages(mapping, 0, -1); } - /* 99% of the time, we don't need to flush the cleancache on the bdev. - * But, for the strange corners, lets be cautious - */ - cleancache_invalidate_inode(mapping); } EXPORT_SYMBOL(invalidate_bdev); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index d6d48ecf823c..409bad3928db 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include "misc.h" #include "extent_io.h" @@ -3578,15 +3577,6 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, goto out; } - if (!PageUptodate(page)) { - if (cleancache_get_page(page) == 0) { - BUG_ON(blocksize != PAGE_SIZE); - unlock_extent(tree, start, end); - unlock_page(page); - goto out; - } - } - if (page->index == last_byte >> PAGE_SHIFT) { size_t zero_offset = offset_in_page(last_byte); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 0ec09fe01be6..4d947ba32da9 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -1374,7 +1373,6 @@ static int btrfs_fill_super(struct super_block *sb, goto fail_close; } - cleancache_init_fs(sb); sb->s_flags |= SB_ACTIVE; return 0; diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 3db923403505..4cd62f1d848c 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -43,7 +43,6 @@ #include #include #include -#include #include "ext4.h" @@ -350,11 +349,6 @@ int ext4_mpage_readpages(struct inode *inode, } else if (fully_mapped) { SetPageMappedToDisk(page); } - if (fully_mapped && blocks_per_page == 1 && - !PageUptodate(page) && cleancache_get_page(page) == 0) { - SetPageUptodate(page); - goto confused; - } /* * This page will go to BIO. Do we need to send this diff --git a/fs/ext4/super.c b/fs/ext4/super.c index db9fe4843529..eee0d9ebfa6c 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -3149,8 +3148,6 @@ done: EXT4_BLOCKS_PER_GROUP(sb), EXT4_INODES_PER_GROUP(sb), sbi->s_mount_opt, sbi->s_mount_opt2); - - cleancache_init_fs(sb); return err; } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0a1d236212f8..8c417864c66a 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -2035,12 +2034,6 @@ got_it: block_nr = map->m_pblk + block_in_file - map->m_lblk; SetPageMappedToDisk(page); - if (!PageUptodate(page) && (!PageSwapCache(page) && - !cleancache_get_page(page))) { - SetPageUptodate(page); - goto confused; - } - if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, DATA_GENERIC_ENHANCE_READ)) { ret = -EFSCORRUPTED; @@ -2096,12 +2089,6 @@ submit_and_realloc: ClearPageError(page); *last_block_in_bio = block_nr; goto out; -confused: - if (bio) { - __submit_bio(F2FS_I_SB(inode), bio, DATA); - bio = NULL; - } - unlock_page(page); out: *bio_ret = bio; return ret; diff --git a/fs/mpage.c b/fs/mpage.c index 334e7d09aa65..87f5cfef6caa 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -29,7 +29,6 @@ #include #include #include -#include #include "internal.h" /* @@ -284,12 +283,6 @@ static struct bio *do_mpage_readpage(struct mpage_readpage_args *args) SetPageMappedToDisk(page); } - if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) && - cleancache_get_page(page) == 0) { - SetPageUptodate(page); - goto confused; - } - /* * This page will go to BIO. Do we need to send this BIO off first? */ diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h index 8aaec7e0804e..fb825059d488 100644 --- a/fs/ntfs3/ntfs_fs.h +++ b/fs/ntfs3/ntfs_fs.h @@ -11,7 +11,6 @@ #include #include -#include #include #include #include diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 1286b88b6fa1..2772dec9dcea 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #define CREATE_TRACE_POINTS @@ -2283,7 +2282,6 @@ static int ocfs2_initialize_super(struct super_block *sb, mlog_errno(status); goto bail; } - cleancache_init_shared_fs(sb); osb->ocfs2_wq = alloc_ordered_workqueue("ocfs2_wq", WQ_MEM_RECLAIM); if (!osb->ocfs2_wq) { diff --git a/fs/super.c b/fs/super.c index a6405d44d4ca..7af820ba5ad5 100644 --- a/fs/super.c +++ b/fs/super.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -260,7 +259,6 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, s->s_time_gran = 1000000000; s->s_time_min = TIME64_MIN; s->s_time_max = TIME64_MAX; - s->cleancache_poolid = CLEANCACHE_NO_POOL; s->s_shrink.seeks = DEFAULT_SEEKS; s->s_shrink.scan_objects = super_cache_scan; @@ -330,7 +328,6 @@ void deactivate_locked_super(struct super_block *s) { struct file_system_type *fs = s->s_type; if (atomic_dec_and_test(&s->s_active)) { - cleancache_invalidate_fs(s); unregister_shrinker(&s->s_shrink); fs->kill_sb(s); diff --git a/include/linux/cleancache.h b/include/linux/cleancache.h deleted file mode 100644 index 5f5730c1d324..000000000000 --- a/include/linux/cleancache.h +++ /dev/null @@ -1,124 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_CLEANCACHE_H -#define _LINUX_CLEANCACHE_H - -#include -#include -#include - -#define CLEANCACHE_NO_POOL -1 -#define CLEANCACHE_NO_BACKEND -2 -#define CLEANCACHE_NO_BACKEND_SHARED -3 - -#define CLEANCACHE_KEY_MAX 6 - -/* - * cleancache requires every file with a page in cleancache to have a - * unique key unless/until the file is removed/truncated. For some - * filesystems, the inode number is unique, but for "modern" filesystems - * an exportable filehandle is required (see exportfs.h) - */ -struct cleancache_filekey { - union { - ino_t ino; - __u32 fh[CLEANCACHE_KEY_MAX]; - u32 key[CLEANCACHE_KEY_MAX]; - } u; -}; - -struct cleancache_ops { - int (*init_fs)(size_t); - int (*init_shared_fs)(uuid_t *uuid, size_t); - int (*get_page)(int, struct cleancache_filekey, - pgoff_t, struct page *); - void (*put_page)(int, struct cleancache_filekey, - pgoff_t, struct page *); - void (*invalidate_page)(int, struct cleancache_filekey, pgoff_t); - void (*invalidate_inode)(int, struct cleancache_filekey); - void (*invalidate_fs)(int); -}; - -extern int cleancache_register_ops(const struct cleancache_ops *ops); -extern void __cleancache_init_fs(struct super_block *); -extern void __cleancache_init_shared_fs(struct super_block *); -extern int __cleancache_get_page(struct page *); -extern void __cleancache_put_page(struct page *); -extern void __cleancache_invalidate_page(struct address_space *, struct page *); -extern void __cleancache_invalidate_inode(struct address_space *); -extern void __cleancache_invalidate_fs(struct super_block *); - -#ifdef CONFIG_CLEANCACHE -#define cleancache_enabled (1) -static inline bool cleancache_fs_enabled_mapping(struct address_space *mapping) -{ - return mapping->host->i_sb->cleancache_poolid >= 0; -} -static inline bool cleancache_fs_enabled(struct page *page) -{ - return cleancache_fs_enabled_mapping(page->mapping); -} -#else -#define cleancache_enabled (0) -#define cleancache_fs_enabled(_page) (0) -#define cleancache_fs_enabled_mapping(_page) (0) -#endif - -/* - * The shim layer provided by these inline functions allows the compiler - * to reduce all cleancache hooks to nothingness if CONFIG_CLEANCACHE - * is disabled, to a single global variable check if CONFIG_CLEANCACHE - * is enabled but no cleancache "backend" has dynamically enabled it, - * and, for the most frequent cleancache ops, to a single global variable - * check plus a superblock element comparison if CONFIG_CLEANCACHE is enabled - * and a cleancache backend has dynamically enabled cleancache, but the - * filesystem referenced by that cleancache op has not enabled cleancache. - * As a result, CONFIG_CLEANCACHE can be enabled by default with essentially - * no measurable performance impact. - */ - -static inline void cleancache_init_fs(struct super_block *sb) -{ - if (cleancache_enabled) - __cleancache_init_fs(sb); -} - -static inline void cleancache_init_shared_fs(struct super_block *sb) -{ - if (cleancache_enabled) - __cleancache_init_shared_fs(sb); -} - -static inline int cleancache_get_page(struct page *page) -{ - if (cleancache_enabled && cleancache_fs_enabled(page)) - return __cleancache_get_page(page); - return -1; -} - -static inline void cleancache_put_page(struct page *page) -{ - if (cleancache_enabled && cleancache_fs_enabled(page)) - __cleancache_put_page(page); -} - -static inline void cleancache_invalidate_page(struct address_space *mapping, - struct page *page) -{ - /* careful... page->mapping is NULL sometimes when this is called */ - if (cleancache_enabled && cleancache_fs_enabled_mapping(mapping)) - __cleancache_invalidate_page(mapping, page); -} - -static inline void cleancache_invalidate_inode(struct address_space *mapping) -{ - if (cleancache_enabled && cleancache_fs_enabled_mapping(mapping)) - __cleancache_invalidate_inode(mapping); -} - -static inline void cleancache_invalidate_fs(struct super_block *sb) -{ - if (cleancache_enabled) - __cleancache_invalidate_fs(sb); -} - -#endif /* _LINUX_CLEANCACHE_H */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 9617dea24978..f3daaea16554 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1535,11 +1535,6 @@ struct super_block { const struct dentry_operations *s_d_op; /* default d_op for dentries */ - /* - * Saved pool identifier for cleancache (-1 means none) - */ - int cleancache_poolid; - struct shrinker s_shrink; /* per-sb shrinker handle */ /* Number of inodes with nlink == 0 but still referenced */ diff --git a/mm/Kconfig b/mm/Kconfig index a99bd499ef51..430240289b02 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -444,28 +444,6 @@ config USE_PERCPU_NUMA_NODE_ID config HAVE_SETUP_PER_CPU_AREA bool -config CLEANCACHE - bool "Enable cleancache driver to cache clean pages if tmem is present" - help - Cleancache can be thought of as a page-granularity victim cache - for clean pages that the kernel's pageframe replacement algorithm - (PFRA) would like to keep around, but can't since there isn't enough - memory. So when the PFRA "evicts" a page, it first attempts to use - cleancache code to put the data contained in that page into - "transcendent memory", memory that is not directly accessible or - addressable by the kernel and is of unknown and possibly - time-varying size. And when a cleancache-enabled - filesystem wishes to access a page in a file on disk, it first - checks cleancache to see if it already contains it; if it does, - the page is copied into the kernel and a disk access is avoided. - When a transcendent memory driver is available (such as zcache or - Xen transcendent memory), a significant I/O reduction - may be achieved. When none is available, all cleancache calls - are reduced to a single pointer-compare-against-NULL resulting - in a negligible performance hit. - - If unsure, say Y to enable cleancache - config FRONTSWAP bool "Enable frontswap to cache swap pages if tmem is present" depends on SWAP diff --git a/mm/Makefile b/mm/Makefile index 588d3113f3b0..70d4309c9ce3 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -104,7 +104,6 @@ obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o obj-$(CONFIG_DEBUG_RODATA_TEST) += rodata_test.o obj-$(CONFIG_DEBUG_VM_PGTABLE) += debug_vm_pgtable.o obj-$(CONFIG_PAGE_OWNER) += page_owner.o -obj-$(CONFIG_CLEANCACHE) += cleancache.o obj-$(CONFIG_MEMORY_ISOLATION) += page_isolation.o obj-$(CONFIG_ZPOOL) += zpool.o obj-$(CONFIG_ZBUD) += zbud.o diff --git a/mm/cleancache.c b/mm/cleancache.c deleted file mode 100644 index db7eee9c0886..000000000000 --- a/mm/cleancache.c +++ /dev/null @@ -1,315 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cleancache frontend - * - * This code provides the generic "frontend" layer to call a matching - * "backend" driver implementation of cleancache. See - * Documentation/vm/cleancache.rst for more information. - * - * Copyright (C) 2009-2010 Oracle Corp. All rights reserved. - * Author: Dan Magenheimer - */ - -#include -#include -#include -#include -#include -#include - -/* - * cleancache_ops is set by cleancache_register_ops to contain the pointers - * to the cleancache "backend" implementation functions. - */ -static const struct cleancache_ops *cleancache_ops __read_mostly; - -/* - * Counters available via /sys/kernel/debug/cleancache (if debugfs is - * properly configured. These are for information only so are not protected - * against increment races. - */ -static u64 cleancache_succ_gets; -static u64 cleancache_failed_gets; -static u64 cleancache_puts; -static u64 cleancache_invalidates; - -static void cleancache_register_ops_sb(struct super_block *sb, void *unused) -{ - switch (sb->cleancache_poolid) { - case CLEANCACHE_NO_BACKEND: - __cleancache_init_fs(sb); - break; - case CLEANCACHE_NO_BACKEND_SHARED: - __cleancache_init_shared_fs(sb); - break; - } -} - -/* - * Register operations for cleancache. Returns 0 on success. - */ -int cleancache_register_ops(const struct cleancache_ops *ops) -{ - if (cmpxchg(&cleancache_ops, NULL, ops)) - return -EBUSY; - - /* - * A cleancache backend can be built as a module and hence loaded after - * a cleancache enabled filesystem has called cleancache_init_fs. To - * handle such a scenario, here we call ->init_fs or ->init_shared_fs - * for each active super block. To differentiate between local and - * shared filesystems, we temporarily initialize sb->cleancache_poolid - * to CLEANCACHE_NO_BACKEND or CLEANCACHE_NO_BACKEND_SHARED - * respectively in case there is no backend registered at the time - * cleancache_init_fs or cleancache_init_shared_fs is called. - * - * Since filesystems can be mounted concurrently with cleancache - * backend registration, we have to be careful to guarantee that all - * cleancache enabled filesystems that has been mounted by the time - * cleancache_register_ops is called has got and all mounted later will - * get cleancache_poolid. This is assured by the following statements - * tied together: - * - * a) iterate_supers skips only those super blocks that has started - * ->kill_sb - * - * b) if iterate_supers encounters a super block that has not finished - * ->mount yet, it waits until it is finished - * - * c) cleancache_init_fs is called from ->mount and - * cleancache_invalidate_fs is called from ->kill_sb - * - * d) we call iterate_supers after cleancache_ops has been set - * - * From a) it follows that if iterate_supers skips a super block, then - * either the super block is already dead, in which case we do not need - * to bother initializing cleancache for it, or it was mounted after we - * initiated iterate_supers. In the latter case, it must have seen - * cleancache_ops set according to d) and initialized cleancache from - * ->mount by itself according to c). This proves that we call - * ->init_fs at least once for each active super block. - * - * From b) and c) it follows that if iterate_supers encounters a super - * block that has already started ->init_fs, it will wait until ->mount - * and hence ->init_fs has finished, then check cleancache_poolid, see - * that it has already been set and therefore do nothing. This proves - * that we call ->init_fs no more than once for each super block. - * - * Combined together, the last two paragraphs prove the function - * correctness. - * - * Note that various cleancache callbacks may proceed before this - * function is called or even concurrently with it, but since - * CLEANCACHE_NO_BACKEND is negative, they will all result in a noop - * until the corresponding ->init_fs has been actually called and - * cleancache_ops has been set. - */ - iterate_supers(cleancache_register_ops_sb, NULL); - return 0; -} -EXPORT_SYMBOL(cleancache_register_ops); - -/* Called by a cleancache-enabled filesystem at time of mount */ -void __cleancache_init_fs(struct super_block *sb) -{ - int pool_id = CLEANCACHE_NO_BACKEND; - - if (cleancache_ops) { - pool_id = cleancache_ops->init_fs(PAGE_SIZE); - if (pool_id < 0) - pool_id = CLEANCACHE_NO_POOL; - } - sb->cleancache_poolid = pool_id; -} -EXPORT_SYMBOL(__cleancache_init_fs); - -/* Called by a cleancache-enabled clustered filesystem at time of mount */ -void __cleancache_init_shared_fs(struct super_block *sb) -{ - int pool_id = CLEANCACHE_NO_BACKEND_SHARED; - - if (cleancache_ops) { - pool_id = cleancache_ops->init_shared_fs(&sb->s_uuid, PAGE_SIZE); - if (pool_id < 0) - pool_id = CLEANCACHE_NO_POOL; - } - sb->cleancache_poolid = pool_id; -} -EXPORT_SYMBOL(__cleancache_init_shared_fs); - -/* - * If the filesystem uses exportable filehandles, use the filehandle as - * the key, else use the inode number. - */ -static int cleancache_get_key(struct inode *inode, - struct cleancache_filekey *key) -{ - int (*fhfn)(struct inode *, __u32 *fh, int *, struct inode *); - int len = 0, maxlen = CLEANCACHE_KEY_MAX; - struct super_block *sb = inode->i_sb; - - key->u.ino = inode->i_ino; - if (sb->s_export_op != NULL) { - fhfn = sb->s_export_op->encode_fh; - if (fhfn) { - len = (*fhfn)(inode, &key->u.fh[0], &maxlen, NULL); - if (len <= FILEID_ROOT || len == FILEID_INVALID) - return -1; - if (maxlen > CLEANCACHE_KEY_MAX) - return -1; - } - } - return 0; -} - -/* - * "Get" data from cleancache associated with the poolid/inode/index - * that were specified when the data was put to cleanache and, if - * successful, use it to fill the specified page with data and return 0. - * The pageframe is unchanged and returns -1 if the get fails. - * Page must be locked by caller. - * - * The function has two checks before any action is taken - whether - * a backend is registered and whether the sb->cleancache_poolid - * is correct. - */ -int __cleancache_get_page(struct page *page) -{ - int ret = -1; - int pool_id; - struct cleancache_filekey key = { .u.key = { 0 } }; - - if (!cleancache_ops) { - cleancache_failed_gets++; - goto out; - } - - VM_BUG_ON_PAGE(!PageLocked(page), page); - pool_id = page->mapping->host->i_sb->cleancache_poolid; - if (pool_id < 0) - goto out; - - if (cleancache_get_key(page->mapping->host, &key) < 0) - goto out; - - ret = cleancache_ops->get_page(pool_id, key, page->index, page); - if (ret == 0) - cleancache_succ_gets++; - else - cleancache_failed_gets++; -out: - return ret; -} -EXPORT_SYMBOL(__cleancache_get_page); - -/* - * "Put" data from a page to cleancache and associate it with the - * (previously-obtained per-filesystem) poolid and the page's, - * inode and page index. Page must be locked. Note that a put_page - * always "succeeds", though a subsequent get_page may succeed or fail. - * - * The function has two checks before any action is taken - whether - * a backend is registered and whether the sb->cleancache_poolid - * is correct. - */ -void __cleancache_put_page(struct page *page) -{ - int pool_id; - struct cleancache_filekey key = { .u.key = { 0 } }; - - if (!cleancache_ops) { - cleancache_puts++; - return; - } - - VM_BUG_ON_PAGE(!PageLocked(page), page); - pool_id = page->mapping->host->i_sb->cleancache_poolid; - if (pool_id >= 0 && - cleancache_get_key(page->mapping->host, &key) >= 0) { - cleancache_ops->put_page(pool_id, key, page->index, page); - cleancache_puts++; - } -} -EXPORT_SYMBOL(__cleancache_put_page); - -/* - * Invalidate any data from cleancache associated with the poolid and the - * page's inode and page index so that a subsequent "get" will fail. - * - * The function has two checks before any action is taken - whether - * a backend is registered and whether the sb->cleancache_poolid - * is correct. - */ -void __cleancache_invalidate_page(struct address_space *mapping, - struct page *page) -{ - /* careful... page->mapping is NULL sometimes when this is called */ - int pool_id = mapping->host->i_sb->cleancache_poolid; - struct cleancache_filekey key = { .u.key = { 0 } }; - - if (!cleancache_ops) - return; - - if (pool_id >= 0) { - VM_BUG_ON_PAGE(!PageLocked(page), page); - if (cleancache_get_key(mapping->host, &key) >= 0) { - cleancache_ops->invalidate_page(pool_id, - key, page->index); - cleancache_invalidates++; - } - } -} -EXPORT_SYMBOL(__cleancache_invalidate_page); - -/* - * Invalidate all data from cleancache associated with the poolid and the - * mappings's inode so that all subsequent gets to this poolid/inode - * will fail. - * - * The function has two checks before any action is taken - whether - * a backend is registered and whether the sb->cleancache_poolid - * is correct. - */ -void __cleancache_invalidate_inode(struct address_space *mapping) -{ - int pool_id = mapping->host->i_sb->cleancache_poolid; - struct cleancache_filekey key = { .u.key = { 0 } }; - - if (!cleancache_ops) - return; - - if (pool_id >= 0 && cleancache_get_key(mapping->host, &key) >= 0) - cleancache_ops->invalidate_inode(pool_id, key); -} -EXPORT_SYMBOL(__cleancache_invalidate_inode); - -/* - * Called by any cleancache-enabled filesystem at time of unmount; - * note that pool_id is surrendered and may be returned by a subsequent - * cleancache_init_fs or cleancache_init_shared_fs. - */ -void __cleancache_invalidate_fs(struct super_block *sb) -{ - int pool_id; - - pool_id = sb->cleancache_poolid; - sb->cleancache_poolid = CLEANCACHE_NO_POOL; - - if (cleancache_ops && pool_id >= 0) - cleancache_ops->invalidate_fs(pool_id); -} -EXPORT_SYMBOL(__cleancache_invalidate_fs); - -static int __init init_cleancache(void) -{ -#ifdef CONFIG_DEBUG_FS - struct dentry *root = debugfs_create_dir("cleancache", NULL); - - debugfs_create_u64("succ_gets", 0444, root, &cleancache_succ_gets); - debugfs_create_u64("failed_gets", 0444, root, &cleancache_failed_gets); - debugfs_create_u64("puts", 0444, root, &cleancache_puts); - debugfs_create_u64("invalidates", 0444, root, &cleancache_invalidates); -#endif - return 0; -} -module_init(init_cleancache) diff --git a/mm/filemap.c b/mm/filemap.c index 60866ae711e2..b50910ac2c88 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -151,16 +150,6 @@ static void filemap_unaccount_folio(struct address_space *mapping, { long nr; - /* - * if we're uptodate, flush out into the cleancache, otherwise - * invalidate any existing cleancache entries. We can't leave - * stale data around in the cleancache once our page is gone - */ - if (folio_test_uptodate(folio) && folio_test_mappedtodisk(folio)) - cleancache_put_page(&folio->page); - else - cleancache_invalidate_page(mapping, &folio->page); - VM_BUG_ON_FOLIO(folio_mapped(folio), folio); if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { int mapcount; diff --git a/mm/truncate.c b/mm/truncate.c index 5e243d7269c0..9dbf0b75da5d 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -22,7 +22,6 @@ #include /* grr. try_to_release_page, do_invalidatepage */ #include -#include #include #include "internal.h" @@ -264,7 +263,6 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) */ folio_zero_range(folio, offset, length); - cleancache_invalidate_page(folio->mapping, &folio->page); if (folio_has_private(folio)) do_invalidatepage(&folio->page, offset, length); if (!folio_test_large(folio)) @@ -351,7 +349,7 @@ void truncate_inode_pages_range(struct address_space *mapping, bool same_folio; if (mapping_empty(mapping)) - goto out; + return; /* * 'start' and 'end' always covers the range of pages to be fully @@ -442,9 +440,6 @@ void truncate_inode_pages_range(struct address_space *mapping, folio_batch_release(&fbatch); index++; } - -out: - cleancache_invalidate_inode(mapping); } EXPORT_SYMBOL(truncate_inode_pages_range); @@ -498,10 +493,6 @@ void truncate_inode_pages_final(struct address_space *mapping) xa_unlock_irq(&mapping->i_pages); } - /* - * Cleancache needs notification even if there are no pages or shadow - * entries. - */ truncate_inode_pages(mapping, 0); } EXPORT_SYMBOL(truncate_inode_pages_final); @@ -661,7 +652,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping, int did_range_unmap = 0; if (mapping_empty(mapping)) - goto out; + return 0; folio_batch_init(&fbatch); index = start; @@ -725,8 +716,6 @@ int invalidate_inode_pages2_range(struct address_space *mapping, if (dax_mapping(mapping)) { unmap_mapping_pages(mapping, start, end - start + 1, false); } -out: - cleancache_invalidate_inode(mapping); return ret; } EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range); -- cgit v1.2.3-59-g8ed1b From 3d6035f136009f9cae380022754cba31f32570c5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:38 -0800 Subject: frontswap: remove frontswap_writethrough frontswap_writethrough is never called, so remove it. Link: https://lkml.kernel.org/r/20211224062246.1258487-3-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/frontswap.rst | 6 ------ include/linux/frontswap.h | 1 - mm/frontswap.c | 23 +---------------------- 3 files changed, 1 insertion(+), 29 deletions(-) (limited to 'include/linux') diff --git a/Documentation/vm/frontswap.rst b/Documentation/vm/frontswap.rst index e2e5ab3e375e..2ab660651d04 100644 --- a/Documentation/vm/frontswap.rst +++ b/Documentation/vm/frontswap.rst @@ -39,12 +39,6 @@ a disk write and, if the data is later read back, a disk read are avoided. If a store returns failure, transcendent memory has rejected the data, and the page can be written to swap as usual. -If a backend chooses, frontswap can be configured as a "writethrough -cache" by calling frontswap_writethrough(). In this mode, the reduction -in swap device writes is lost (and also a non-trivial performance advantage) -in order to allow the backend to arbitrarily "reclaim" space used to -store frontswap pages to more completely manage its memory usage. - Note that if a page is stored and the page already exists in transcendent memory (a "duplicate" store), either the store succeeds and the data is overwritten, or the store fails AND the page is invalidated. This ensures stale data may diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index b07d88c92bb2..4a03fda41572 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -26,7 +26,6 @@ struct frontswap_ops { extern void frontswap_register_ops(struct frontswap_ops *ops); extern void frontswap_shrink(unsigned long); extern unsigned long frontswap_curr_pages(void); -extern void frontswap_writethrough(bool); #define FRONTSWAP_HAS_EXCLUSIVE_GETS extern void frontswap_tmem_exclusive_gets(bool); diff --git a/mm/frontswap.c b/mm/frontswap.c index 6bed12260dea..51a662a83955 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -32,16 +32,6 @@ static struct frontswap_ops *frontswap_ops __read_mostly; #define for_each_frontswap_ops(ops) \ for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next) -/* - * If enabled, frontswap_store will return failure even on success. As - * a result, the swap subsystem will always write the page to swap, in - * effect converting frontswap into a writethrough cache. In this mode, - * there is no direct reduction in swap writes, but a frontswap backend - * can unilaterally "reclaim" any pages in use with no data loss, thus - * providing increases control over maximum memory usage due to frontswap. - */ -static bool frontswap_writethrough_enabled __read_mostly; - /* * If enabled, the underlying tmem implementation is capable of doing * exclusive gets, so frontswap_load, on a successful tmem_get must @@ -170,15 +160,6 @@ void frontswap_register_ops(struct frontswap_ops *ops) } EXPORT_SYMBOL(frontswap_register_ops); -/* - * Enable/disable frontswap writethrough (see above). - */ -void frontswap_writethrough(bool enable) -{ - frontswap_writethrough_enabled = enable; -} -EXPORT_SYMBOL(frontswap_writethrough); - /* * Enable/disable frontswap exclusive gets (see above). */ @@ -283,9 +264,7 @@ int __frontswap_store(struct page *page) } else { inc_frontswap_failed_stores(); } - if (frontswap_writethrough_enabled) - /* report failure so swap also writes to swap device */ - ret = -1; + return ret; } EXPORT_SYMBOL(__frontswap_store); -- cgit v1.2.3-59-g8ed1b From 71024cb4a0bfe7767aec7a128d0a1a13a37b7fcd Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:41 -0800 Subject: frontswap: remove frontswap_tmem_exclusive_gets frontswap_tmem_exclusive_gets is never called, so remove it. Link: https://lkml.kernel.org/r/20211224062246.1258487-4-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 2 -- mm/frontswap.c | 23 +---------------------- 2 files changed, 1 insertion(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index 4a03fda41572..83a56392cc7f 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -26,8 +26,6 @@ struct frontswap_ops { extern void frontswap_register_ops(struct frontswap_ops *ops); extern void frontswap_shrink(unsigned long); extern unsigned long frontswap_curr_pages(void); -#define FRONTSWAP_HAS_EXCLUSIVE_GETS -extern void frontswap_tmem_exclusive_gets(bool); extern bool __frontswap_test(struct swap_info_struct *, pgoff_t); extern void __frontswap_init(unsigned type, unsigned long *map); diff --git a/mm/frontswap.c b/mm/frontswap.c index 51a662a83955..dba7f087ee86 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -32,13 +32,6 @@ static struct frontswap_ops *frontswap_ops __read_mostly; #define for_each_frontswap_ops(ops) \ for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next) -/* - * If enabled, the underlying tmem implementation is capable of doing - * exclusive gets, so frontswap_load, on a successful tmem_get must - * mark the page as no longer in frontswap AND mark it dirty. - */ -static bool frontswap_tmem_exclusive_gets_enabled __read_mostly; - #ifdef CONFIG_DEBUG_FS /* * Counters available via /sys/kernel/debug/frontswap (if debugfs is @@ -160,15 +153,6 @@ void frontswap_register_ops(struct frontswap_ops *ops) } EXPORT_SYMBOL(frontswap_register_ops); -/* - * Enable/disable frontswap exclusive gets (see above). - */ -void frontswap_tmem_exclusive_gets(bool enable) -{ - frontswap_tmem_exclusive_gets_enabled = enable; -} -EXPORT_SYMBOL(frontswap_tmem_exclusive_gets); - /* * Called when a swap device is swapon'd. */ @@ -296,13 +280,8 @@ int __frontswap_load(struct page *page) if (!ret) /* successful load */ break; } - if (ret == 0) { + if (ret == 0) inc_frontswap_loads(); - if (frontswap_tmem_exclusive_gets_enabled) { - SetPageDirty(page); - __frontswap_clear(sis, offset); - } - } return ret; } EXPORT_SYMBOL(__frontswap_load); -- cgit v1.2.3-59-g8ed1b From 0b364446d734da76e421dbfb09e5268270cefaf0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:44 -0800 Subject: frontswap: remove frontswap_shrink frontswap_shrink is never called, so remove it. Link: https://lkml.kernel.org/r/20211224062246.1258487-5-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/frontswap.rst | 13 ------- include/linux/frontswap.h | 1 - mm/frontswap.c | 83 ------------------------------------------ 3 files changed, 97 deletions(-) (limited to 'include/linux') diff --git a/Documentation/vm/frontswap.rst b/Documentation/vm/frontswap.rst index 2ab660651d04..feecc5e24477 100644 --- a/Documentation/vm/frontswap.rst +++ b/Documentation/vm/frontswap.rst @@ -255,19 +255,6 @@ the old data and ensure that it is no longer accessible. Since the swap subsystem then writes the new data to the read swap device, this is the correct course of action to ensure coherency. -* What is frontswap_shrink for? - -When the (non-frontswap) swap subsystem swaps out a page to a real -swap device, that page is only taking up low-value pre-allocated disk -space. But if frontswap has placed a page in transcendent memory, that -page may be taking up valuable real estate. The frontswap_shrink -routine allows code outside of the swap subsystem to force pages out -of the memory managed by frontswap and back into kernel-addressable memory. -For example, in RAMster, a "suction driver" thread will attempt -to "repatriate" pages sent to a remote machine back to the local machine; -this is driven using the frontswap_shrink mechanism when memory pressure -subsides. - * Why does the frontswap patch create the new include file swapfile.h? The frontswap code depends on some swap-subsystem-internal data diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index 83a56392cc7f..d268d7bb6513 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -24,7 +24,6 @@ struct frontswap_ops { }; extern void frontswap_register_ops(struct frontswap_ops *ops); -extern void frontswap_shrink(unsigned long); extern unsigned long frontswap_curr_pages(void); extern bool __frontswap_test(struct swap_info_struct *, pgoff_t); diff --git a/mm/frontswap.c b/mm/frontswap.c index dba7f087ee86..a77ebba6101b 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -341,89 +341,6 @@ static unsigned long __frontswap_curr_pages(void) return totalpages; } -static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused, - int *swapid) -{ - int ret = -EINVAL; - struct swap_info_struct *si = NULL; - int si_frontswap_pages; - unsigned long total_pages_to_unuse = total; - unsigned long pages = 0, pages_to_unuse = 0; - - assert_spin_locked(&swap_lock); - plist_for_each_entry(si, &swap_active_head, list) { - si_frontswap_pages = atomic_read(&si->frontswap_pages); - if (total_pages_to_unuse < si_frontswap_pages) { - pages = pages_to_unuse = total_pages_to_unuse; - } else { - pages = si_frontswap_pages; - pages_to_unuse = 0; /* unuse all */ - } - /* ensure there is enough RAM to fetch pages from frontswap */ - if (security_vm_enough_memory_mm(current->mm, pages)) { - ret = -ENOMEM; - continue; - } - vm_unacct_memory(pages); - *unused = pages_to_unuse; - *swapid = si->type; - ret = 0; - break; - } - - return ret; -} - -/* - * Used to check if it's necessary and feasible to unuse pages. - * Return 1 when nothing to do, 0 when need to shrink pages, - * error code when there is an error. - */ -static int __frontswap_shrink(unsigned long target_pages, - unsigned long *pages_to_unuse, - int *type) -{ - unsigned long total_pages = 0, total_pages_to_unuse; - - assert_spin_locked(&swap_lock); - - total_pages = __frontswap_curr_pages(); - if (total_pages <= target_pages) { - /* Nothing to do */ - *pages_to_unuse = 0; - return 1; - } - total_pages_to_unuse = total_pages - target_pages; - return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type); -} - -/* - * Frontswap, like a true swap device, may unnecessarily retain pages - * under certain circumstances; "shrink" frontswap is essentially a - * "partial swapoff" and works by calling try_to_unuse to attempt to - * unuse enough frontswap pages to attempt to -- subject to memory - * constraints -- reduce the number of pages in frontswap to the - * number given in the parameter target_pages. - */ -void frontswap_shrink(unsigned long target_pages) -{ - unsigned long pages_to_unuse = 0; - int type, ret; - - /* - * we don't want to hold swap_lock while doing a very - * lengthy try_to_unuse, but swap_list may change - * so restart scan from swap_active_head each time - */ - spin_lock(&swap_lock); - ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type); - spin_unlock(&swap_lock); - if (ret == 0) - try_to_unuse(type, true, pages_to_unuse); - return; -} -EXPORT_SYMBOL(frontswap_shrink); - /* * Count and return the number of frontswap pages across all * swap devices. This is exported so that backend drivers can -- cgit v1.2.3-59-g8ed1b From 3e8e1af63d7a831f576477c25d9b89049bd2d53d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:47 -0800 Subject: frontswap: remove frontswap_curr_pages frontswap_curr_pages is never called, so remove it. Link: https://lkml.kernel.org/r/20211224062246.1258487-6-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 1 - mm/frontswap.c | 28 ---------------------------- 2 files changed, 29 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index d268d7bb6513..5205c2977b20 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -24,7 +24,6 @@ struct frontswap_ops { }; extern void frontswap_register_ops(struct frontswap_ops *ops); -extern unsigned long frontswap_curr_pages(void); extern bool __frontswap_test(struct swap_info_struct *, pgoff_t); extern void __frontswap_init(unsigned type, unsigned long *map); diff --git a/mm/frontswap.c b/mm/frontswap.c index a77ebba6101b..af8f68d0e5cc 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -330,34 +330,6 @@ void __frontswap_invalidate_area(unsigned type) } EXPORT_SYMBOL(__frontswap_invalidate_area); -static unsigned long __frontswap_curr_pages(void) -{ - unsigned long totalpages = 0; - struct swap_info_struct *si = NULL; - - assert_spin_locked(&swap_lock); - plist_for_each_entry(si, &swap_active_head, list) - totalpages += atomic_read(&si->frontswap_pages); - return totalpages; -} - -/* - * Count and return the number of frontswap pages across all - * swap devices. This is exported so that backend drivers can - * determine current usage without reading debugfs. - */ -unsigned long frontswap_curr_pages(void) -{ - unsigned long totalpages = 0; - - spin_lock(&swap_lock); - totalpages = __frontswap_curr_pages(); - spin_unlock(&swap_lock); - - return totalpages; -} -EXPORT_SYMBOL(frontswap_curr_pages); - static int __init init_frontswap(void) { #ifdef CONFIG_DEBUG_FS -- cgit v1.2.3-59-g8ed1b From 1cf53c894d15dd4b73397a56fa055d76d3db66b4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:51 -0800 Subject: frontswap: simplify frontswap_init Just use IS_ENABLED() and remove the __frontswap_init indirection. Also remove the unused export. Link: https://lkml.kernel.org/r/20211224062246.1258487-7-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 9 +-------- mm/frontswap.c | 3 +-- mm/swapfile.c | 3 ++- 3 files changed, 4 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index 5205c2977b20..73d7beb44f2b 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -26,7 +26,7 @@ struct frontswap_ops { extern void frontswap_register_ops(struct frontswap_ops *ops); extern bool __frontswap_test(struct swap_info_struct *, pgoff_t); -extern void __frontswap_init(unsigned type, unsigned long *map); +extern void frontswap_init(unsigned type, unsigned long *map); extern int __frontswap_store(struct page *page); extern int __frontswap_load(struct page *page); extern void __frontswap_invalidate_page(unsigned, pgoff_t); @@ -107,11 +107,4 @@ static inline void frontswap_invalidate_area(unsigned type) __frontswap_invalidate_area(type); } -static inline void frontswap_init(unsigned type, unsigned long *map) -{ -#ifdef CONFIG_FRONTSWAP - __frontswap_init(type, map); -#endif -} - #endif /* _LINUX_FRONTSWAP_H */ diff --git a/mm/frontswap.c b/mm/frontswap.c index af8f68d0e5cc..132d6ad6d70b 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -156,7 +156,7 @@ EXPORT_SYMBOL(frontswap_register_ops); /* * Called when a swap device is swapon'd. */ -void __frontswap_init(unsigned type, unsigned long *map) +void frontswap_init(unsigned type, unsigned long *map) { struct swap_info_struct *sis = swap_info[type]; struct frontswap_ops *ops; @@ -179,7 +179,6 @@ void __frontswap_init(unsigned type, unsigned long *map) for_each_frontswap_ops(ops) ops->init(type); } -EXPORT_SYMBOL(__frontswap_init); bool __frontswap_test(struct swap_info_struct *sis, pgoff_t offset) diff --git a/mm/swapfile.c b/mm/swapfile.c index caa9f81a0d15..df5930ccd93d 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2463,7 +2463,8 @@ static void enable_swap_info(struct swap_info_struct *p, int prio, struct swap_cluster_info *cluster_info, unsigned long *frontswap_map) { - frontswap_init(p->type, frontswap_map); + if (IS_ENABLED(CONFIG_FRONTSWAP)) + frontswap_init(p->type, frontswap_map); spin_lock(&swap_lock); spin_lock(&p->lock); setup_swap_info(p, prio, swap_map, cluster_info); -- cgit v1.2.3-59-g8ed1b From 10a9c496789fe2098bfc018650fc77b23ba08a54 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:14:57 -0800 Subject: mm: simplify try_to_unuse Remove the unused frontswap and pages_to_unuse arguments, and mark the function static now that the caller in frontswap is gone. [akpm@linux-foundation.org: fix shmem_unuse() stub, per Matthew] Link: https://lkml.kernel.org/r/20211224062246.1258487-9-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Cc: Naresh Kamboju Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 7 ---- include/linux/shmem_fs.h | 3 +- include/linux/swapfile.h | 1 - mm/shmem.c | 33 ++++--------------- mm/swapfile.c | 83 +++++++++++++---------------------------------- 5 files changed, 30 insertions(+), 97 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index 73d7beb44f2b..a9817d4fa74c 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -7,13 +7,6 @@ #include #include -/* - * Return code to denote that requested number of - * frontswap pages are unused(moved to page cache). - * Used in shmem_unuse and try_to_unuse. - */ -#define FRONTSWAP_PAGES_UNUSED 2 - struct frontswap_ops { void (*init)(unsigned); /* this swap type was just swapon'ed */ int (*store)(unsigned, pgoff_t, struct page *); /* store a page */ diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 166158b6e917..e65b80ed09e7 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -83,8 +83,7 @@ extern void shmem_unlock_mapping(struct address_space *mapping); extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end); -extern int shmem_unuse(unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse); +int shmem_unuse(unsigned int type); extern bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, pgoff_t index); diff --git a/include/linux/swapfile.h b/include/linux/swapfile.h index e06febf62978..809cd01ef2c5 100644 --- a/include/linux/swapfile.h +++ b/include/linux/swapfile.h @@ -9,7 +9,6 @@ extern spinlock_t swap_lock; extern struct plist_head swap_active_head; extern struct swap_info_struct *swap_info[]; -extern int try_to_unuse(unsigned int, bool, unsigned long); extern unsigned long generic_max_swapfile_size(void); extern unsigned long max_swapfile_size(void); diff --git a/mm/shmem.c b/mm/shmem.c index 66909efd0a1b..a09b29ec2b45 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -1152,7 +1151,7 @@ static void shmem_evict_inode(struct inode *inode) static int shmem_find_swap_entries(struct address_space *mapping, pgoff_t start, unsigned int nr_entries, struct page **entries, pgoff_t *indices, - unsigned int type, bool frontswap) + unsigned int type) { XA_STATE(xas, &mapping->i_pages, start); struct page *page; @@ -1173,9 +1172,6 @@ static int shmem_find_swap_entries(struct address_space *mapping, entry = radix_to_swp_entry(page); if (swp_type(entry) != type) continue; - if (frontswap && - !frontswap_test(swap_info[type], swp_offset(entry))) - continue; indices[ret] = xas.xa_index; entries[ret] = page; @@ -1228,26 +1224,20 @@ static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec, /* * If swap found in inode, free it and move page from swapcache to filecache. */ -static int shmem_unuse_inode(struct inode *inode, unsigned int type, - bool frontswap, unsigned long *fs_pages_to_unuse) +static int shmem_unuse_inode(struct inode *inode, unsigned int type) { struct address_space *mapping = inode->i_mapping; pgoff_t start = 0; struct pagevec pvec; pgoff_t indices[PAGEVEC_SIZE]; - bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0); int ret = 0; pagevec_init(&pvec); do { unsigned int nr_entries = PAGEVEC_SIZE; - if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE) - nr_entries = *fs_pages_to_unuse; - pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries, - pvec.pages, indices, - type, frontswap); + pvec.pages, indices, type); if (pvec.nr == 0) { ret = 0; break; @@ -1257,14 +1247,6 @@ static int shmem_unuse_inode(struct inode *inode, unsigned int type, if (ret < 0) break; - if (frontswap_partial) { - *fs_pages_to_unuse -= ret; - if (*fs_pages_to_unuse == 0) { - ret = FRONTSWAP_PAGES_UNUSED; - break; - } - } - start = indices[pvec.nr - 1]; } while (true); @@ -1276,8 +1258,7 @@ static int shmem_unuse_inode(struct inode *inode, unsigned int type, * device 'type' back into memory, so the swap device can be * unused. */ -int shmem_unuse(unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) +int shmem_unuse(unsigned int type) { struct shmem_inode_info *info, *next; int error = 0; @@ -1300,8 +1281,7 @@ int shmem_unuse(unsigned int type, bool frontswap, atomic_inc(&info->stop_eviction); mutex_unlock(&shmem_swaplist_mutex); - error = shmem_unuse_inode(&info->vfs_inode, type, frontswap, - fs_pages_to_unuse); + error = shmem_unuse_inode(&info->vfs_inode, type); cond_resched(); mutex_lock(&shmem_swaplist_mutex); @@ -4015,8 +3995,7 @@ int __init shmem_init(void) return 0; } -int shmem_unuse(unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) +int shmem_unuse(unsigned int type) { return 0; } diff --git a/mm/swapfile.c b/mm/swapfile.c index df5930ccd93d..82342c77791b 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1923,8 +1923,7 @@ out: static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) + unsigned int type) { struct page *page; swp_entry_t entry; @@ -1945,9 +1944,6 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, continue; offset = swp_offset(entry); - if (frontswap && !frontswap_test(si, offset)) - continue; - pte_unmap(pte); swap_map = &si->swap_map[offset]; page = lookup_swap_cache(entry, vma, addr); @@ -1979,11 +1975,6 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, try_to_free_swap(page); unlock_page(page); put_page(page); - - if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) { - ret = FRONTSWAP_PAGES_UNUSED; - goto out; - } try_next: pte = pte_offset_map(pmd, addr); } while (pte++, addr += PAGE_SIZE, addr != end); @@ -1996,8 +1987,7 @@ out: static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, - unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) + unsigned int type) { pmd_t *pmd; unsigned long next; @@ -2009,8 +1999,7 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, next = pmd_addr_end(addr, end); if (pmd_none_or_trans_huge_or_clear_bad(pmd)) continue; - ret = unuse_pte_range(vma, pmd, addr, next, type, - frontswap, fs_pages_to_unuse); + ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; } while (pmd++, addr = next, addr != end); @@ -2019,8 +2008,7 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) + unsigned int type) { pud_t *pud; unsigned long next; @@ -2031,8 +2019,7 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; - ret = unuse_pmd_range(vma, pud, addr, next, type, - frontswap, fs_pages_to_unuse); + ret = unuse_pmd_range(vma, pud, addr, next, type); if (ret) return ret; } while (pud++, addr = next, addr != end); @@ -2041,8 +2028,7 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned int type, bool frontswap, - unsigned long *fs_pages_to_unuse) + unsigned int type) { p4d_t *p4d; unsigned long next; @@ -2053,16 +2039,14 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) continue; - ret = unuse_pud_range(vma, p4d, addr, next, type, - frontswap, fs_pages_to_unuse); + ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) return ret; } while (p4d++, addr = next, addr != end); return 0; } -static int unuse_vma(struct vm_area_struct *vma, unsigned int type, - bool frontswap, unsigned long *fs_pages_to_unuse) +static int unuse_vma(struct vm_area_struct *vma, unsigned int type) { pgd_t *pgd; unsigned long addr, end, next; @@ -2076,16 +2060,14 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type, next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; - ret = unuse_p4d_range(vma, pgd, addr, next, type, - frontswap, fs_pages_to_unuse); + ret = unuse_p4d_range(vma, pgd, addr, next, type); if (ret) return ret; } while (pgd++, addr = next, addr != end); return 0; } -static int unuse_mm(struct mm_struct *mm, unsigned int type, - bool frontswap, unsigned long *fs_pages_to_unuse) +static int unuse_mm(struct mm_struct *mm, unsigned int type) { struct vm_area_struct *vma; int ret = 0; @@ -2093,8 +2075,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type, mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (vma->anon_vma) { - ret = unuse_vma(vma, type, frontswap, - fs_pages_to_unuse); + ret = unuse_vma(vma, type); if (ret) break; } @@ -2110,7 +2091,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type, * if there are no inuse entries after prev till end of the map. */ static unsigned int find_next_to_unuse(struct swap_info_struct *si, - unsigned int prev, bool frontswap) + unsigned int prev) { unsigned int i; unsigned char count; @@ -2124,8 +2105,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si, for (i = prev + 1; i < si->max; i++) { count = READ_ONCE(si->swap_map[i]); if (count && swap_count(count) != SWAP_MAP_BAD) - if (!frontswap || frontswap_test(si, i)) - break; + break; if ((i % LATENCY_LIMIT) == 0) cond_resched(); } @@ -2136,12 +2116,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si, return i; } -/* - * If the boolean frontswap is true, only unuse pages_to_unuse pages; - * pages_to_unuse==0 means all pages; ignored if frontswap is false - */ -int try_to_unuse(unsigned int type, bool frontswap, - unsigned long pages_to_unuse) +static int try_to_unuse(unsigned int type) { struct mm_struct *prev_mm; struct mm_struct *mm; @@ -2155,13 +2130,10 @@ int try_to_unuse(unsigned int type, bool frontswap, if (!READ_ONCE(si->inuse_pages)) return 0; - if (!frontswap) - pages_to_unuse = 0; - retry: - retval = shmem_unuse(type, frontswap, &pages_to_unuse); + retval = shmem_unuse(type); if (retval) - goto out; + return retval; prev_mm = &init_mm; mmget(prev_mm); @@ -2178,11 +2150,10 @@ retry: spin_unlock(&mmlist_lock); mmput(prev_mm); prev_mm = mm; - retval = unuse_mm(mm, type, frontswap, &pages_to_unuse); - + retval = unuse_mm(mm, type); if (retval) { mmput(prev_mm); - goto out; + return retval; } /* @@ -2199,7 +2170,7 @@ retry: i = 0; while (READ_ONCE(si->inuse_pages) && !signal_pending(current) && - (i = find_next_to_unuse(si, i, frontswap)) != 0) { + (i = find_next_to_unuse(si, i)) != 0) { entry = swp_entry(type, i); page = find_get_page(swap_address_space(entry), i); @@ -2217,14 +2188,6 @@ retry: try_to_free_swap(page); unlock_page(page); put_page(page); - - /* - * For frontswap, we just need to unuse pages_to_unuse, if - * it was specified. Need not check frontswap again here as - * we already zeroed out pages_to_unuse if not frontswap. - */ - if (pages_to_unuse && --pages_to_unuse == 0) - goto out; } /* @@ -2242,10 +2205,10 @@ retry: if (READ_ONCE(si->inuse_pages)) { if (!signal_pending(current)) goto retry; - retval = -EINTR; + return -EINTR; } -out: - return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval; + + return 0; } /* @@ -2577,7 +2540,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) disable_swap_slots_cache_lock(); set_current_oom_origin(); - err = try_to_unuse(p->type, false, 0); /* force unuse all pages */ + err = try_to_unuse(p->type); clear_current_oom_origin(); if (err) { -- cgit v1.2.3-59-g8ed1b From bd9cd521496ba8d537d8f46f4167bf4221aba9a3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:15:01 -0800 Subject: frontswap: remove frontswap_test frontswap_test is unused now, remove it. Link: https://lkml.kernel.org/r/20211224062246.1258487-10-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 11 ----------- mm/frontswap.c | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index a9817d4fa74c..c5b2848d2240 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -18,7 +18,6 @@ struct frontswap_ops { extern void frontswap_register_ops(struct frontswap_ops *ops); -extern bool __frontswap_test(struct swap_info_struct *, pgoff_t); extern void frontswap_init(unsigned type, unsigned long *map); extern int __frontswap_store(struct page *page); extern int __frontswap_load(struct page *page); @@ -33,11 +32,6 @@ static inline bool frontswap_enabled(void) return static_branch_unlikely(&frontswap_enabled_key); } -static inline bool frontswap_test(struct swap_info_struct *sis, pgoff_t offset) -{ - return __frontswap_test(sis, offset); -} - static inline void frontswap_map_set(struct swap_info_struct *p, unsigned long *map) { @@ -56,11 +50,6 @@ static inline bool frontswap_enabled(void) return false; } -static inline bool frontswap_test(struct swap_info_struct *sis, pgoff_t offset) -{ - return false; -} - static inline void frontswap_map_set(struct swap_info_struct *p, unsigned long *map) { diff --git a/mm/frontswap.c b/mm/frontswap.c index 42d554da53bb..f51159f0d75d 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -179,7 +179,7 @@ void frontswap_init(unsigned type, unsigned long *map) ops->init(type); } -bool __frontswap_test(struct swap_info_struct *sis, +static bool __frontswap_test(struct swap_info_struct *sis, pgoff_t offset) { if (sis->frontswap_map) -- cgit v1.2.3-59-g8ed1b From 633423a09cb5cfe61438283e1ce49c23cf4a0611 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:15:07 -0800 Subject: mm: mark swap_lock and swap_active_head static swap_lock and swap_active_head are only used in swapfile.c, so mark them static. Link: https://lkml.kernel.org/r/20211224062246.1258487-12-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/swapfile.h | 2 -- mm/swapfile.c | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swapfile.h b/include/linux/swapfile.h index 809cd01ef2c5..54078542134c 100644 --- a/include/linux/swapfile.h +++ b/include/linux/swapfile.h @@ -6,8 +6,6 @@ * these were static in swapfile.c but frontswap.c needs them and we don't * want to expose them to the dozens of source files that include swap.h */ -extern spinlock_t swap_lock; -extern struct plist_head swap_active_head; extern struct swap_info_struct *swap_info[]; extern unsigned long generic_max_swapfile_size(void); extern unsigned long max_swapfile_size(void); diff --git a/mm/swapfile.c b/mm/swapfile.c index 82342c77791b..bf0df7aa7158 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -49,7 +49,7 @@ static bool swap_count_continued(struct swap_info_struct *, pgoff_t, unsigned char); static void free_swap_count_continuations(struct swap_info_struct *); -DEFINE_SPINLOCK(swap_lock); +static DEFINE_SPINLOCK(swap_lock); static unsigned int nr_swapfiles; atomic_long_t nr_swap_pages; /* @@ -71,7 +71,7 @@ static const char Unused_offset[] = "Unused swap offset entry "; * all active swap_info_structs * protected with swap_lock, and ordered by priority. */ -PLIST_HEAD(swap_active_head); +static PLIST_HEAD(swap_active_head); /* * all available (active, not full) swap_info_structs -- cgit v1.2.3-59-g8ed1b From 1da0d94a3ec8c5f3793b7be8538b55e60ebeefe3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 21 Jan 2022 22:15:10 -0800 Subject: frontswap: remove support for multiple ops There is only a single instance of frontswap ops in the kernel, so simplify the frontswap code by removing support for multiple operations. Link: https://lkml.kernel.org/r/20211224062246.1258487-13-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Juergen Gross Cc: Dan Streetman Cc: Geert Uytterhoeven Cc: Hugh Dickins Cc: Konrad Rzeszutek Wilk Cc: Matthew Wilcox (Oracle) Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/frontswap.h | 3 +-- mm/frontswap.c | 50 ++++++++++++----------------------------------- mm/zswap.c | 8 ++++++-- 3 files changed, 19 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index c5b2848d2240..a631bac12220 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -13,10 +13,9 @@ struct frontswap_ops { int (*load)(unsigned, pgoff_t, struct page *); /* load a page */ void (*invalidate_page)(unsigned, pgoff_t); /* page no longer needed */ void (*invalidate_area)(unsigned); /* swap type just swapoff'ed */ - struct frontswap_ops *next; /* private pointer to next ops */ }; -extern void frontswap_register_ops(struct frontswap_ops *ops); +int frontswap_register_ops(const struct frontswap_ops *ops); extern void frontswap_init(unsigned type, unsigned long *map); extern int __frontswap_store(struct page *page); diff --git a/mm/frontswap.c b/mm/frontswap.c index 35040fa4eba8..6f69b044a8cc 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -27,10 +27,7 @@ DEFINE_STATIC_KEY_FALSE(frontswap_enabled_key); * may be registered, but implementations can never deregister. This * is a simple singly-linked list of all registered implementations. */ -static struct frontswap_ops *frontswap_ops __read_mostly; - -#define for_each_frontswap_ops(ops) \ - for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next) +static const struct frontswap_ops *frontswap_ops __read_mostly; #ifdef CONFIG_DEBUG_FS /* @@ -97,18 +94,14 @@ static inline void inc_frontswap_invalidates(void) { } /* * Register operations for frontswap */ -void frontswap_register_ops(struct frontswap_ops *ops) +int frontswap_register_ops(const struct frontswap_ops *ops) { - /* - * Setting frontswap_ops must happen after the ops->init() calls - * above; cmpxchg implies smp_mb() which will ensure the init is - * complete at this point. - */ - do { - ops->next = frontswap_ops; - } while (cmpxchg(&frontswap_ops, ops->next, ops) != ops->next); + if (frontswap_ops) + return -EINVAL; + frontswap_ops = ops; static_branch_inc(&frontswap_enabled_key); + return 0; } /* @@ -117,7 +110,6 @@ void frontswap_register_ops(struct frontswap_ops *ops) void frontswap_init(unsigned type, unsigned long *map) { struct swap_info_struct *sis = swap_info[type]; - struct frontswap_ops *ops; VM_BUG_ON(sis == NULL); @@ -133,9 +125,7 @@ void frontswap_init(unsigned type, unsigned long *map) * p->frontswap set to something valid to work properly. */ frontswap_map_set(sis, map); - - for_each_frontswap_ops(ops) - ops->init(type); + frontswap_ops->init(type); } static bool __frontswap_test(struct swap_info_struct *sis, @@ -174,7 +164,6 @@ int __frontswap_store(struct page *page) int type = swp_type(entry); struct swap_info_struct *sis = swap_info[type]; pgoff_t offset = swp_offset(entry); - struct frontswap_ops *ops; VM_BUG_ON(!frontswap_ops); VM_BUG_ON(!PageLocked(page)); @@ -188,16 +177,10 @@ int __frontswap_store(struct page *page) */ if (__frontswap_test(sis, offset)) { __frontswap_clear(sis, offset); - for_each_frontswap_ops(ops) - ops->invalidate_page(type, offset); + frontswap_ops->invalidate_page(type, offset); } - /* Try to store in each implementation, until one succeeds. */ - for_each_frontswap_ops(ops) { - ret = ops->store(type, offset, page); - if (!ret) /* successful store */ - break; - } + ret = frontswap_ops->store(type, offset, page); if (ret == 0) { __frontswap_set(sis, offset); inc_frontswap_succ_stores(); @@ -220,7 +203,6 @@ int __frontswap_load(struct page *page) int type = swp_type(entry); struct swap_info_struct *sis = swap_info[type]; pgoff_t offset = swp_offset(entry); - struct frontswap_ops *ops; VM_BUG_ON(!frontswap_ops); VM_BUG_ON(!PageLocked(page)); @@ -230,11 +212,7 @@ int __frontswap_load(struct page *page) return -1; /* Try loading from each implementation, until one succeeds. */ - for_each_frontswap_ops(ops) { - ret = ops->load(type, offset, page); - if (!ret) /* successful load */ - break; - } + ret = frontswap_ops->load(type, offset, page); if (ret == 0) inc_frontswap_loads(); return ret; @@ -247,7 +225,6 @@ int __frontswap_load(struct page *page) void __frontswap_invalidate_page(unsigned type, pgoff_t offset) { struct swap_info_struct *sis = swap_info[type]; - struct frontswap_ops *ops; VM_BUG_ON(!frontswap_ops); VM_BUG_ON(sis == NULL); @@ -255,8 +232,7 @@ void __frontswap_invalidate_page(unsigned type, pgoff_t offset) if (!__frontswap_test(sis, offset)) return; - for_each_frontswap_ops(ops) - ops->invalidate_page(type, offset); + frontswap_ops->invalidate_page(type, offset); __frontswap_clear(sis, offset); inc_frontswap_invalidates(); } @@ -268,7 +244,6 @@ void __frontswap_invalidate_page(unsigned type, pgoff_t offset) void __frontswap_invalidate_area(unsigned type) { struct swap_info_struct *sis = swap_info[type]; - struct frontswap_ops *ops; VM_BUG_ON(!frontswap_ops); VM_BUG_ON(sis == NULL); @@ -276,8 +251,7 @@ void __frontswap_invalidate_area(unsigned type) if (sis->frontswap_map == NULL) return; - for_each_frontswap_ops(ops) - ops->invalidate_area(type); + frontswap_ops->invalidate_area(type); atomic_set(&sis->frontswap_pages, 0); bitmap_zero(sis->frontswap_map, sis->max); } diff --git a/mm/zswap.c b/mm/zswap.c index 7944e3e57e78..cdf6950fcb2e 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1378,7 +1378,7 @@ static void zswap_frontswap_init(unsigned type) zswap_trees[type] = tree; } -static struct frontswap_ops zswap_frontswap_ops = { +static const struct frontswap_ops zswap_frontswap_ops = { .store = zswap_frontswap_store, .load = zswap_frontswap_load, .invalidate_page = zswap_frontswap_invalidate_page, @@ -1475,11 +1475,15 @@ static int __init init_zswap(void) if (!shrink_wq) goto fallback_fail; - frontswap_register_ops(&zswap_frontswap_ops); + ret = frontswap_register_ops(&zswap_frontswap_ops); + if (ret) + goto destroy_wq; if (zswap_debugfs_init()) pr_warn("debugfs initialization failed\n"); return 0; +destroy_wq: + destroy_workqueue(shrink_wq); fallback_fail: if (pool) zswap_pool_destroy(pool); -- cgit v1.2.3-59-g8ed1b From a37d9a17f099072fe4d3a9048b0321978707a918 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 20 Jan 2022 23:53:04 +0200 Subject: fsnotify: invalidate dcache before IN_DELETE event Apparently, there are some applications that use IN_DELETE event as an invalidation mechanism and expect that if they try to open a file with the name reported with the delete event, that it should not contain the content of the deleted file. Commit 49246466a989 ("fsnotify: move fsnotify_nameremove() hook out of d_delete()") moved the fsnotify delete hook before d_delete() so fsnotify will have access to a positive dentry. This allowed a race where opening the deleted file via cached dentry is now possible after receiving the IN_DELETE event. To fix the regression, create a new hook fsnotify_delete() that takes the unlinked inode as an argument and use a helper d_delete_notify() to pin the inode, so we can pass it to fsnotify_delete() after d_delete(). Backporting hint: this regression is from v5.3. Although patch will apply with only trivial conflicts to v5.4 and v5.10, it won't build, because fsnotify_delete() implementation is different in each of those versions (see fsnotify_link()). A follow up patch will fix the fsnotify_unlink/rmdir() calls in pseudo filesystem that do not need to call d_delete(). Link: https://lore.kernel.org/r/20220120215305.282577-1-amir73il@gmail.com Reported-by: Ivan Delalande Link: https://lore.kernel.org/linux-fsdevel/YeNyzoDM5hP5LtGW@visor/ Fixes: 49246466a989 ("fsnotify: move fsnotify_nameremove() hook out of d_delete()") Cc: stable@vger.kernel.org # v5.3+ Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara --- fs/btrfs/ioctl.c | 6 ++---- fs/namei.c | 10 +++++----- include/linux/fsnotify.h | 49 ++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 50 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index a5bd6926f7ff..7807b28b7892 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -3086,10 +3086,8 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, btrfs_inode_lock(inode, 0); err = btrfs_delete_subvolume(dir, dentry); btrfs_inode_unlock(inode, 0); - if (!err) { - fsnotify_rmdir(dir, dentry); - d_delete(dentry); - } + if (!err) + d_delete_notify(dir, dentry); out_dput: dput(dentry); diff --git a/fs/namei.c b/fs/namei.c index d81f04f8d818..4ed0e41feab7 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -3974,13 +3974,12 @@ int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir, dentry->d_inode->i_flags |= S_DEAD; dont_mount(dentry); detach_mounts(dentry); - fsnotify_rmdir(dir, dentry); out: inode_unlock(dentry->d_inode); dput(dentry); if (!error) - d_delete(dentry); + d_delete_notify(dir, dentry); return error; } EXPORT_SYMBOL(vfs_rmdir); @@ -4102,7 +4101,6 @@ int vfs_unlink(struct user_namespace *mnt_userns, struct inode *dir, if (!error) { dont_mount(dentry); detach_mounts(dentry); - fsnotify_unlink(dir, dentry); } } } @@ -4110,9 +4108,11 @@ out: inode_unlock(target); /* We don't d_delete() NFS sillyrenamed files--they still exist. */ - if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) { + if (!error && dentry->d_flags & DCACHE_NFSFS_RENAMED) { + fsnotify_unlink(dir, dentry); + } else if (!error) { fsnotify_link_count(target); - d_delete(dentry); + d_delete_notify(dir, dentry); } return error; diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 3a2d7dc3c607..bb8467cd11ae 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -224,6 +224,43 @@ static inline void fsnotify_link(struct inode *dir, struct inode *inode, dir, &new_dentry->d_name, 0); } +/* + * fsnotify_delete - @dentry was unlinked and unhashed + * + * Caller must make sure that dentry->d_name is stable. + * + * Note: unlike fsnotify_unlink(), we have to pass also the unlinked inode + * as this may be called after d_delete() and old_dentry may be negative. + */ +static inline void fsnotify_delete(struct inode *dir, struct inode *inode, + struct dentry *dentry) +{ + __u32 mask = FS_DELETE; + + if (S_ISDIR(inode->i_mode)) + mask |= FS_ISDIR; + + fsnotify_name(mask, inode, FSNOTIFY_EVENT_INODE, dir, &dentry->d_name, + 0); +} + +/** + * d_delete_notify - delete a dentry and call fsnotify_delete() + * @dentry: The dentry to delete + * + * This helper is used to guaranty that the unlinked inode cannot be found + * by lookup of this name after fsnotify_delete() event has been delivered. + */ +static inline void d_delete_notify(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + ihold(inode); + d_delete(dentry); + fsnotify_delete(dir, inode, dentry); + iput(inode); +} + /* * fsnotify_unlink - 'name' was unlinked * @@ -231,10 +268,10 @@ static inline void fsnotify_link(struct inode *dir, struct inode *inode, */ static inline void fsnotify_unlink(struct inode *dir, struct dentry *dentry) { - /* Expected to be called before d_delete() */ - WARN_ON_ONCE(d_is_negative(dentry)); + if (WARN_ON_ONCE(d_is_negative(dentry))) + return; - fsnotify_dirent(dir, dentry, FS_DELETE); + fsnotify_delete(dir, d_inode(dentry), dentry); } /* @@ -258,10 +295,10 @@ static inline void fsnotify_mkdir(struct inode *dir, struct dentry *dentry) */ static inline void fsnotify_rmdir(struct inode *dir, struct dentry *dentry) { - /* Expected to be called before d_delete() */ - WARN_ON_ONCE(d_is_negative(dentry)); + if (WARN_ON_ONCE(d_is_negative(dentry))) + return; - fsnotify_dirent(dir, dentry, FS_DELETE | FS_ISDIR); + fsnotify_delete(dir, d_inode(dentry), dentry); } /* -- cgit v1.2.3-59-g8ed1b From 9daf0a4d32d60a57f2a2533bdf4c178be7fdff7f Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Sun, 16 Jan 2022 04:59:36 -0800 Subject: quota: cleanup double word in comment Remove the second 'handle'. Link: https://lore.kernel.org/r/20220116125936.389767-1-trix@redhat.com Signed-off-by: Tom Rix Signed-off-by: Jan Kara --- include/linux/quota.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/quota.h b/include/linux/quota.h index 18ebd39c9487..fd692b4a41d5 100644 --- a/include/linux/quota.h +++ b/include/linux/quota.h @@ -91,7 +91,7 @@ extern bool qid_valid(struct kqid qid); * * When there is no mapping defined for the user-namespace, type, * qid tuple an invalid kqid is returned. Callers are expected to - * test for and handle handle invalid kqids being returned. + * test for and handle invalid kqids being returned. * Invalid kqids may be tested for using qid_valid(). */ static inline struct kqid make_kqid(struct user_namespace *from, -- cgit v1.2.3-59-g8ed1b From 945c37ed564770c78dfe6b9f08bed57a1b4e60ef Mon Sep 17 00:00:00 2001 From: Linyu Yuan Date: Mon, 10 Jan 2022 20:43:28 +0800 Subject: usb: roles: fix include/linux/usb/role.h compile issue when CONFIG_USB_ROLE_SWITCH is not defined, add usb_role_switch_find_by_fwnode() definition which return NULL. Fixes: c6919d5e0cd1 ("usb: roles: Add usb_role_switch_find_by_fwnode()") Signed-off-by: Linyu Yuan Link: https://lore.kernel.org/r/1641818608-25039-1-git-send-email-quic_linyyuan@quicinc.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/role.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/usb/role.h b/include/linux/usb/role.h index 031f148ab373..b5deafd91f67 100644 --- a/include/linux/usb/role.h +++ b/include/linux/usb/role.h @@ -91,6 +91,12 @@ fwnode_usb_role_switch_get(struct fwnode_handle *node) static inline void usb_role_switch_put(struct usb_role_switch *sw) { } +static inline struct usb_role_switch * +usb_role_switch_find_by_fwnode(const struct fwnode_handle *fwnode) +{ + return NULL; +} + static inline struct usb_role_switch * usb_role_switch_register(struct device *parent, const struct usb_role_switch_desc *desc) -- cgit v1.2.3-59-g8ed1b From 33569ef3c754a82010f266b7b938a66a3ccf90a4 Mon Sep 17 00:00:00 2001 From: Amadeusz Sławiński Date: Wed, 19 Jan 2022 11:47:51 +0100 Subject: PM: hibernate: Remove register_nosave_region_late() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is an unused wrapper forcing kmalloc allocation for registering nosave regions. Also, rename __register_nosave_region() to register_nosave_region() now that there is no need for disambiguation. Signed-off-by: Amadeusz Sławiński Reviewed-by: Cezary Rojewski Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 11 +---------- kernel/power/snapshot.c | 21 +++++++-------------- 2 files changed, 8 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 5785d909c321..3e8ecdebe601 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -430,15 +430,7 @@ struct platform_hibernation_ops { #ifdef CONFIG_HIBERNATION /* kernel/power/snapshot.c */ -extern void __register_nosave_region(unsigned long b, unsigned long e, int km); -static inline void __init register_nosave_region(unsigned long b, unsigned long e) -{ - __register_nosave_region(b, e, 0); -} -static inline void __init register_nosave_region_late(unsigned long b, unsigned long e) -{ - __register_nosave_region(b, e, 1); -} +extern void register_nosave_region(unsigned long b, unsigned long e); extern int swsusp_page_is_forbidden(struct page *); extern void swsusp_set_page_free(struct page *); extern void swsusp_unset_page_free(struct page *); @@ -458,7 +450,6 @@ int pfn_is_nosave(unsigned long pfn); int hibernate_quiet_exec(int (*func)(void *data), void *data); #else /* CONFIG_HIBERNATION */ static inline void register_nosave_region(unsigned long b, unsigned long e) {} -static inline void register_nosave_region_late(unsigned long b, unsigned long e) {} static inline int swsusp_page_is_forbidden(struct page *p) { return 0; } static inline void swsusp_set_page_free(struct page *p) {} static inline void swsusp_unset_page_free(struct page *p) {} diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index f7a986078213..330d49937692 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -978,8 +978,7 @@ static void memory_bm_recycle(struct memory_bitmap *bm) * Register a range of page frames the contents of which should not be saved * during hibernation (to be used in the early initialization code). */ -void __init __register_nosave_region(unsigned long start_pfn, - unsigned long end_pfn, int use_kmalloc) +void __init register_nosave_region(unsigned long start_pfn, unsigned long end_pfn) { struct nosave_region *region; @@ -995,18 +994,12 @@ void __init __register_nosave_region(unsigned long start_pfn, goto Report; } } - if (use_kmalloc) { - /* During init, this shouldn't fail */ - region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL); - BUG_ON(!region); - } else { - /* This allocation cannot fail */ - region = memblock_alloc(sizeof(struct nosave_region), - SMP_CACHE_BYTES); - if (!region) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(struct nosave_region)); - } + /* This allocation cannot fail */ + region = memblock_alloc(sizeof(struct nosave_region), + SMP_CACHE_BYTES); + if (!region) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct nosave_region)); region->start_pfn = start_pfn; region->end_pfn = end_pfn; list_add_tail(®ion->list, &nosave_regions); -- cgit v1.2.3-59-g8ed1b From d7e4f8545b497b3f5687e592f1c355cbaee64c8c Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Wed, 26 Jan 2022 13:04:26 +0800 Subject: pid: Introduce helper task_is_in_init_pid_ns() Currently the kernel uses open code in multiple places to check if a task is in the root PID namespace with the kind of format: if (task_active_pid_ns(current) == &init_pid_ns) do_something(); This patch creates a new helper function, task_is_in_init_pid_ns(), it returns true if a passed task is in the root PID namespace, otherwise returns false. So it will be used to replace open codes. Suggested-by: Suzuki K Poulose Signed-off-by: Leo Yan Reviewed-by: Leon Romanovsky Acked-by: Suzuki K Poulose Acked-by: Balbir Singh Signed-off-by: Jakub Kicinski --- include/linux/pid_namespace.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pid_namespace.h b/include/linux/pid_namespace.h index 7c7e627503d2..07481bb87d4e 100644 --- a/include/linux/pid_namespace.h +++ b/include/linux/pid_namespace.h @@ -86,4 +86,9 @@ extern struct pid_namespace *task_active_pid_ns(struct task_struct *tsk); void pidhash_init(void); void pid_idr_init(void); +static inline bool task_is_in_init_pid_ns(struct task_struct *tsk) +{ + return task_active_pid_ns(tsk) == &init_pid_ns; +} + #endif /* _LINUX_PID_NS_H */ -- cgit v1.2.3-59-g8ed1b From 364df53c081d93fcfd6b91085ff2650c7f17b3c7 Mon Sep 17 00:00:00 2001 From: Menglong Dong Date: Thu, 27 Jan 2022 17:13:01 +0800 Subject: net: socket: rename SKB_DROP_REASON_SOCKET_FILTER Rename SKB_DROP_REASON_SOCKET_FILTER, which is used as the reason of skb drop out of socket filter before it's part of a released kernel. It will be used for more protocols than just TCP in future series. Signed-off-by: Menglong Dong Reviewed-by: David Ahern Link: https://lore.kernel.org/all/20220127091308.91401-2-imagedong@tencent.com/ Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 2 +- include/trace/events/skb.h | 2 +- net/ipv4/tcp_ipv4.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index bf11e1fbd69b..8a636e678902 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -318,7 +318,7 @@ enum skb_drop_reason { SKB_DROP_REASON_NO_SOCKET, SKB_DROP_REASON_PKT_TOO_SMALL, SKB_DROP_REASON_TCP_CSUM, - SKB_DROP_REASON_TCP_FILTER, + SKB_DROP_REASON_SOCKET_FILTER, SKB_DROP_REASON_UDP_CSUM, SKB_DROP_REASON_MAX, }; diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h index 3e042ca2cedb..a8a64b97504d 100644 --- a/include/trace/events/skb.h +++ b/include/trace/events/skb.h @@ -14,7 +14,7 @@ EM(SKB_DROP_REASON_NO_SOCKET, NO_SOCKET) \ EM(SKB_DROP_REASON_PKT_TOO_SMALL, PKT_TOO_SMALL) \ EM(SKB_DROP_REASON_TCP_CSUM, TCP_CSUM) \ - EM(SKB_DROP_REASON_TCP_FILTER, TCP_FILTER) \ + EM(SKB_DROP_REASON_SOCKET_FILTER, SOCKET_FILTER) \ EM(SKB_DROP_REASON_UDP_CSUM, UDP_CSUM) \ EMe(SKB_DROP_REASON_MAX, MAX) diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index b3f34e366b27..938b59636578 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2095,7 +2095,7 @@ process: nf_reset_ct(skb); if (tcp_filter(sk, skb)) { - drop_reason = SKB_DROP_REASON_TCP_FILTER; + drop_reason = SKB_DROP_REASON_SOCKET_FILTER; goto discard_and_relse; } th = (const struct tcphdr *)skb->data; -- cgit v1.2.3-59-g8ed1b From 7f5056b9e7b71149bf11073f00a57fa1ac2921a9 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Wed, 26 Jan 2022 15:35:14 -0500 Subject: security, lsm: dentry_init_security() Handle multi LSM registration A ceph user has reported that ceph is crashing with kernel NULL pointer dereference. Following is the backtrace. /proc/version: Linux version 5.16.2-arch1-1 (linux@archlinux) (gcc (GCC) 11.1.0, GNU ld (GNU Binutils) 2.36.1) #1 SMP PREEMPT Thu, 20 Jan 2022 16:18:29 +0000 distro / arch: Arch Linux / x86_64 SELinux is not enabled ceph cluster version: 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) relevant dmesg output: [ 30.947129] BUG: kernel NULL pointer dereference, address: 0000000000000000 [ 30.947206] #PF: supervisor read access in kernel mode [ 30.947258] #PF: error_code(0x0000) - not-present page [ 30.947310] PGD 0 P4D 0 [ 30.947342] Oops: 0000 [#1] PREEMPT SMP PTI [ 30.947388] CPU: 5 PID: 778 Comm: touch Not tainted 5.16.2-arch1-1 #1 86fbf2c313cc37a553d65deb81d98e9dcc2a3659 [ 30.947486] Hardware name: Gigabyte Technology Co., Ltd. B365M DS3H/B365M DS3H, BIOS F5 08/13/2019 [ 30.947569] RIP: 0010:strlen+0x0/0x20 [ 30.947616] Code: b6 07 38 d0 74 16 48 83 c7 01 84 c0 74 05 48 39 f7 75 ec 31 c0 31 d2 89 d6 89 d7 c3 48 89 f8 31 d2 89 d6 89 d7 c3 0 f 1f 40 00 <80> 3f 00 74 12 48 89 f8 48 83 c0 01 80 38 00 75 f7 48 29 f8 31 ff [ 30.947782] RSP: 0018:ffffa4ed80ffbbb8 EFLAGS: 00010246 [ 30.947836] RAX: 0000000000000000 RBX: ffffa4ed80ffbc60 RCX: 0000000000000000 [ 30.947904] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 [ 30.947971] RBP: ffff94b0d15c0ae0 R08: 0000000000000000 R09: 0000000000000000 [ 30.948040] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 [ 30.948106] R13: 0000000000000001 R14: ffffa4ed80ffbc60 R15: 0000000000000000 [ 30.948174] FS: 00007fc7520f0740(0000) GS:ffff94b7ced40000(0000) knlGS:0000000000000000 [ 30.948252] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 30.948308] CR2: 0000000000000000 CR3: 0000000104a40001 CR4: 00000000003706e0 [ 30.948376] Call Trace: [ 30.948404] [ 30.948431] ceph_security_init_secctx+0x7b/0x240 [ceph 49f9c4b9bf5be8760f19f1747e26da33920bce4b] [ 30.948582] ceph_atomic_open+0x51e/0x8a0 [ceph 49f9c4b9bf5be8760f19f1747e26da33920bce4b] [ 30.948708] ? get_cached_acl+0x4d/0xa0 [ 30.948759] path_openat+0x60d/0x1030 [ 30.948809] do_filp_open+0xa5/0x150 [ 30.948859] do_sys_openat2+0xc4/0x190 [ 30.948904] __x64_sys_openat+0x53/0xa0 [ 30.948948] do_syscall_64+0x5c/0x90 [ 30.948989] ? exc_page_fault+0x72/0x180 [ 30.949034] entry_SYSCALL_64_after_hwframe+0x44/0xae [ 30.949091] RIP: 0033:0x7fc7521e25bb [ 30.950849] Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 0 0 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 91 00 00 00 48 8b 54 24 28 64 48 2b 14 25 Core of the problem is that ceph checks for return code from security_dentry_init_security() and if return code is 0, it assumes everything is fine and continues to call strlen(name), which crashes. Typically SELinux LSM returns 0 and sets name to "security.selinux" and it is not a problem. Or if selinux is not compiled in or disabled, it returns -EOPNOTSUP and ceph deals with it. But somehow in this configuration, 0 is being returned and "name" is not being initialized and that's creating the problem. Our suspicion is that BPF LSM is registering a hook for dentry_init_security() and returns hook default of 0. LSM_HOOK(int, 0, dentry_init_security, struct dentry *dentry,...) I have not been able to reproduce it just by doing CONFIG_BPF_LSM=y. Stephen has tested the patch though and confirms it solves the problem for him. dentry_init_security() is written in such a way that it expects only one LSM to register the hook. Atleast that's the expectation with current code. If another LSM returns a hook and returns default, it will simply return 0 as of now and that will break ceph. Hence, suggestion is that change semantics of this hook a bit. If there are no LSMs or no LSM is taking ownership and initializing security context, then return -EOPNOTSUP. Also allow at max one LSM to initialize security context. This hook can't deal with multiple LSMs trying to init security context. This patch implements this new behavior. Reported-by: Stephen Muth Tested-by: Stephen Muth Suggested-by: Casey Schaufler Acked-by: Casey Schaufler Reviewed-by: Serge Hallyn Cc: Jeff Layton Cc: Christian Brauner Cc: Paul Moore Cc: # 5.16.0 Signed-off-by: Vivek Goyal Reviewed-by: Jeff Layton Acked-by: Paul Moore Acked-by: Christian Brauner Signed-off-by: James Morris --- include/linux/lsm_hook_defs.h | 2 +- security/security.c | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index df8de62f4710..f0c7b352340a 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -82,7 +82,7 @@ LSM_HOOK(int, 0, sb_add_mnt_opt, const char *option, const char *val, int len, void **mnt_opts) LSM_HOOK(int, 0, move_mount, const struct path *from_path, const struct path *to_path) -LSM_HOOK(int, 0, dentry_init_security, struct dentry *dentry, +LSM_HOOK(int, -EOPNOTSUPP, dentry_init_security, struct dentry *dentry, int mode, const struct qstr *name, const char **xattr_name, void **ctx, u32 *ctxlen) LSM_HOOK(int, 0, dentry_create_files_as, struct dentry *dentry, int mode, diff --git a/security/security.c b/security/security.c index c88167a414b4..64abdfb20bc2 100644 --- a/security/security.c +++ b/security/security.c @@ -1056,8 +1056,19 @@ int security_dentry_init_security(struct dentry *dentry, int mode, const char **xattr_name, void **ctx, u32 *ctxlen) { - return call_int_hook(dentry_init_security, -EOPNOTSUPP, dentry, mode, - name, xattr_name, ctx, ctxlen); + struct security_hook_list *hp; + int rc; + + /* + * Only one module will provide a security context. + */ + hlist_for_each_entry(hp, &security_hook_heads.dentry_init_security, list) { + rc = hp->hook.dentry_init_security(dentry, mode, name, + xattr_name, ctx, ctxlen); + if (rc != LSM_RET_DEFAULT(dentry_init_security)) + return rc; + } + return LSM_RET_DEFAULT(dentry_init_security); } EXPORT_SYMBOL(security_dentry_init_security); -- cgit v1.2.3-59-g8ed1b From e45c47d1f94e0cc7b6b079fdb4bcce2995e2adc4 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 28 Jan 2022 10:58:39 -0500 Subject: block: add bio_start_io_acct_time() to control start_time bio_start_io_acct_time() interface is like bio_start_io_acct() that allows start_time to be passed in. This gives drivers the ability to defer starting accounting until after IO is issued (but possibily not entirely due to bio splitting). Reviewed-by: Christoph Hellwig Signed-off-by: Mike Snitzer Link: https://lore.kernel.org/r/20220128155841.39644-2-snitzer@redhat.com Signed-off-by: Jens Axboe --- block/blk-core.c | 25 +++++++++++++++++++------ include/linux/blkdev.h | 1 + 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 97f8bc8d3a79..d93e3bb9a769 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1061,20 +1061,32 @@ again: } static unsigned long __part_start_io_acct(struct block_device *part, - unsigned int sectors, unsigned int op) + unsigned int sectors, unsigned int op, + unsigned long start_time) { const int sgrp = op_stat_group(op); - unsigned long now = READ_ONCE(jiffies); part_stat_lock(); - update_io_ticks(part, now, false); + update_io_ticks(part, start_time, false); part_stat_inc(part, ios[sgrp]); part_stat_add(part, sectors[sgrp], sectors); part_stat_local_inc(part, in_flight[op_is_write(op)]); part_stat_unlock(); - return now; + return start_time; +} + +/** + * bio_start_io_acct_time - start I/O accounting for bio based drivers + * @bio: bio to start account for + * @start_time: start time that should be passed back to bio_end_io_acct(). + */ +void bio_start_io_acct_time(struct bio *bio, unsigned long start_time) +{ + __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), + bio_op(bio), start_time); } +EXPORT_SYMBOL_GPL(bio_start_io_acct_time); /** * bio_start_io_acct - start I/O accounting for bio based drivers @@ -1084,14 +1096,15 @@ static unsigned long __part_start_io_acct(struct block_device *part, */ unsigned long bio_start_io_acct(struct bio *bio) { - return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio)); + return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), + bio_op(bio), jiffies); } EXPORT_SYMBOL_GPL(bio_start_io_acct); unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, unsigned int op) { - return __part_start_io_acct(disk->part0, sectors, op); + return __part_start_io_acct(disk->part0, sectors, op, jiffies); } EXPORT_SYMBOL(disk_start_io_acct); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 9c95df26fc26..f35aea98bc35 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1258,6 +1258,7 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, void disk_end_io_acct(struct gendisk *disk, unsigned int op, unsigned long start_time); +void bio_start_io_acct_time(struct bio *bio, unsigned long start_time); unsigned long bio_start_io_acct(struct bio *bio); void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, struct block_device *orig_bdev); -- cgit v1.2.3-59-g8ed1b From 6cb917411e028dcb66ce8f5db1b47361b78d7d3f Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Sat, 29 Jan 2022 13:40:52 -0800 Subject: include/linux/sysctl.h: fix register_sysctl_mount_point() return type The CONFIG_SYSCTL=n stub returns the wrong type. Fixes: ee9efac48a082 ("sysctl: add helper to register a sysctl mount point") Reported-by: kernel test robot Acked-by: Luis Chamberlain Cc: Tong Zhang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/sysctl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 180adf7da785..6353d6db69b2 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -265,7 +265,7 @@ static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * return NULL; } -static inline struct sysctl_header *register_sysctl_mount_point(const char *path) +static inline struct ctl_table_header *register_sysctl_mount_point(const char *path) { return NULL; } -- cgit v1.2.3-59-g8ed1b From 536f4217ced62b671bd759f6b549621a5654a70f Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 29 Jan 2022 13:41:04 -0800 Subject: mm: page->mapping folio->mapping should have the same offset As with the other members of folio, the offset of page->mapping and folio->mapping must be the same. The compile-time check was inadvertently removed during development. Add it back. [willy@infradead.org: changelog redo] Link: https://lkml.kernel.org/r/20220104011734.21714-1-richard.weiyang@gmail.com Signed-off-by: Wei Yang Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_types.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 9db36dc5d4cf..5140e5feb486 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -261,6 +261,7 @@ static_assert(sizeof(struct page) == sizeof(struct folio)); static_assert(offsetof(struct page, pg) == offsetof(struct folio, fl)) FOLIO_MATCH(flags, flags); FOLIO_MATCH(lru, lru); +FOLIO_MATCH(mapping, mapping); FOLIO_MATCH(compound_head, lru); FOLIO_MATCH(index, index); FOLIO_MATCH(private, private); -- cgit v1.2.3-59-g8ed1b From 27fe73394a1c6d0b07fa4d95f1bca116d1cc66e9 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Sat, 29 Jan 2022 13:41:14 -0800 Subject: mm, kasan: use compare-exchange operation to set KASAN page tag It has been reported that the tag setting operation on newly-allocated pages can cause the page flags to be corrupted when performed concurrently with other flag updates as a result of the use of non-atomic operations. Fix the problem by using a compare-exchange loop to update the tag. Link: https://lkml.kernel.org/r/20220120020148.1632253-1-pcc@google.com Link: https://linux-review.googlesource.com/id/I456b24a2b9067d93968d43b4bb3351c0cec63101 Fixes: 2813b9c02962 ("kasan, mm, arm64: tag non slab memory allocated via pagealloc") Signed-off-by: Peter Collingbourne Reviewed-by: Andrey Konovalov Cc: Peter Zijlstra Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index e1a84b1e6787..213cc569b192 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1506,11 +1506,18 @@ static inline u8 page_kasan_tag(const struct page *page) static inline void page_kasan_tag_set(struct page *page, u8 tag) { - if (kasan_enabled()) { - tag ^= 0xff; - page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); - page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; - } + unsigned long old_flags, flags; + + if (!kasan_enabled()) + return; + + tag ^= 0xff; + old_flags = READ_ONCE(page->flags); + do { + flags = old_flags; + flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); + flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; + } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags))); } static inline void page_kasan_tag_reset(struct page *page) -- cgit v1.2.3-59-g8ed1b From 51e50fbd3efc6064c30ed73a5e009018b36e290a Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Sat, 29 Jan 2022 13:41:17 -0800 Subject: psi: fix "no previous prototype" warnings when CONFIG_CGROUPS=n When CONFIG_CGROUPS is disabled psi code generates the following warnings: kernel/sched/psi.c:1112:21: warning: no previous prototype for 'psi_trigger_create' [-Wmissing-prototypes] 1112 | struct psi_trigger *psi_trigger_create(struct psi_group *group, | ^~~~~~~~~~~~~~~~~~ kernel/sched/psi.c:1182:6: warning: no previous prototype for 'psi_trigger_destroy' [-Wmissing-prototypes] 1182 | void psi_trigger_destroy(struct psi_trigger *t) | ^~~~~~~~~~~~~~~~~~~ kernel/sched/psi.c:1249:10: warning: no previous prototype for 'psi_trigger_poll' [-Wmissing-prototypes] 1249 | __poll_t psi_trigger_poll(void **trigger_ptr, | ^~~~~~~~~~~~~~~~ Change the declarations of these functions in the header to provide the prototypes even when they are unused. Link: https://lkml.kernel.org/r/20220119223940.787748-2-surenb@google.com Fixes: 0e94682b73bf ("psi: introduce psi monitor") Signed-off-by: Suren Baghdasaryan Reported-by: kernel test robot Acked-by: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/psi.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/psi.h b/include/linux/psi.h index f8ce53bfdb2a..7f7d1d88c3bb 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -25,18 +25,17 @@ void psi_memstall_enter(unsigned long *flags); void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); - -#ifdef CONFIG_CGROUPS -int psi_cgroup_alloc(struct cgroup *cgrp); -void psi_cgroup_free(struct cgroup *cgrp); -void cgroup_move_task(struct task_struct *p, struct css_set *to); - struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, size_t nbytes, enum psi_res res); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait); + +#ifdef CONFIG_CGROUPS +int psi_cgroup_alloc(struct cgroup *cgrp); +void psi_cgroup_free(struct cgroup *cgrp); +void cgroup_move_task(struct task_struct *p, struct css_set *to); #endif #else /* CONFIG_PSI */ -- cgit v1.2.3-59-g8ed1b