From 5994f58977e0123d5cb77617b3cc4676325028dd Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Fri, 10 Jun 2022 12:24:09 +0300 Subject: dt-bindings: reset: add sama7g5 definitions Add reset bindings for SAMA7G5. At the moment only USB PHYs are included. Signed-off-by: Claudiu Beznea Acked-by: Philipp Zabel Acked-by: Rob Herring Signed-off-by: Sebastian Reichel --- include/dt-bindings/reset/sama7g5-reset.h | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 include/dt-bindings/reset/sama7g5-reset.h (limited to 'include') diff --git a/include/dt-bindings/reset/sama7g5-reset.h b/include/dt-bindings/reset/sama7g5-reset.h new file mode 100644 index 000000000000..2116f41d04e0 --- /dev/null +++ b/include/dt-bindings/reset/sama7g5-reset.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef __DT_BINDINGS_RESET_SAMA7G5_H +#define __DT_BINDINGS_RESET_SAMA7G5_H + +#define SAMA7G5_RESET_USB_PHY1 4 +#define SAMA7G5_RESET_USB_PHY2 5 +#define SAMA7G5_RESET_USB_PHY3 6 + +#endif /* __DT_BINDINGS_RESET_SAMA7G5_H */ -- cgit v1.3-15-g9ac1 From ef4f7e4bf1dc26aaa86cf8e5a13013684139be51 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Thu, 28 Jul 2022 00:41:25 +0300 Subject: scsi: target: core: De-RCU of se_lun and se_lun acl se_lun and se_lun_acl are immutable pointers of struct se_dev_entry. Remove RCU usage for access to those pointers. Link: https://lore.kernel.org/r/20220727214125.19647-3-d.bogdanov@yadro.com Reviewed-by: Mike Christie Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_alua.c | 3 +-- drivers/target/target_core_device.c | 29 +++++++++++------------------ drivers/target/target_core_pr.c | 28 ++++++++-------------------- drivers/target/target_core_stat.c | 10 ++++------ drivers/target/target_core_xcopy.c | 2 +- include/target/target_core_base.h | 4 ++-- 6 files changed, 27 insertions(+), 49 deletions(-) (limited to 'include') diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c index 58df0145e8d0..fb91423a4e2e 100644 --- a/drivers/target/target_core_alua.c +++ b/drivers/target/target_core_alua.c @@ -934,8 +934,7 @@ static void core_alua_queue_state_change_ua(struct t10_alua_tg_pt_gp *tg_pt_gp) spin_lock(&lun->lun_deve_lock); list_for_each_entry(se_deve, &lun->lun_deve_list, lun_link) { - lacl = rcu_dereference_check(se_deve->se_lun_acl, - lockdep_is_held(&lun->lun_deve_lock)); + lacl = se_deve->se_lun_acl; /* * spc4r37 p.242: diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c index 88a844ae1fd0..b7f16ee8aa0e 100644 --- a/drivers/target/target_core_device.c +++ b/drivers/target/target_core_device.c @@ -75,7 +75,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd) return TCM_WRITE_PROTECTED; } - se_lun = rcu_dereference(deve->se_lun); + se_lun = deve->se_lun; if (!percpu_ref_tryget_live(&se_lun->lun_ref)) { se_lun = NULL; @@ -152,7 +152,7 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd) rcu_read_lock(); deve = target_nacl_find_deve(nacl, se_cmd->orig_fe_lun); if (deve) { - se_lun = rcu_dereference(deve->se_lun); + se_lun = deve->se_lun; if (!percpu_ref_tryget_live(&se_lun->lun_ref)) { se_lun = NULL; @@ -216,7 +216,7 @@ struct se_dev_entry *core_get_se_deve_from_rtpi( rcu_read_lock(); hlist_for_each_entry_rcu(deve, &nacl->lun_entry_hlist, link) { - lun = rcu_dereference(deve->se_lun); + lun = deve->se_lun; if (!lun) { pr_err("%s device entries device pointer is" " NULL, but Initiator has access.\n", @@ -243,11 +243,8 @@ void core_free_device_list_for_node( struct se_dev_entry *deve; mutex_lock(&nacl->lun_entry_mutex); - hlist_for_each_entry_rcu(deve, &nacl->lun_entry_hlist, link) { - struct se_lun *lun = rcu_dereference_check(deve->se_lun, - lockdep_is_held(&nacl->lun_entry_mutex)); - core_disable_device_list_for_node(lun, deve, nacl, tpg); - } + hlist_for_each_entry_rcu(deve, &nacl->lun_entry_hlist, link) + core_disable_device_list_for_node(deve->se_lun, deve, nacl, tpg); mutex_unlock(&nacl->lun_entry_mutex); } @@ -334,8 +331,7 @@ int core_enable_device_list_for_node( mutex_lock(&nacl->lun_entry_mutex); orig = target_nacl_find_deve(nacl, mapped_lun); if (orig && orig->se_lun) { - struct se_lun *orig_lun = rcu_dereference_check(orig->se_lun, - lockdep_is_held(&nacl->lun_entry_mutex)); + struct se_lun *orig_lun = orig->se_lun; if (orig_lun != lun) { pr_err("Existing orig->se_lun doesn't match new lun" @@ -355,8 +351,8 @@ int core_enable_device_list_for_node( return -EINVAL; } - rcu_assign_pointer(new->se_lun, lun); - rcu_assign_pointer(new->se_lun_acl, lun_acl); + new->se_lun = lun; + new->se_lun_acl = lun_acl; hlist_del_rcu(&orig->link); hlist_add_head_rcu(&new->link, &nacl->lun_entry_hlist); mutex_unlock(&nacl->lun_entry_mutex); @@ -374,8 +370,8 @@ int core_enable_device_list_for_node( return 0; } - rcu_assign_pointer(new->se_lun, lun); - rcu_assign_pointer(new->se_lun_acl, lun_acl); + new->se_lun = lun; + new->se_lun_acl = lun_acl; hlist_add_head_rcu(&new->link, &nacl->lun_entry_hlist); mutex_unlock(&nacl->lun_entry_mutex); @@ -454,10 +450,7 @@ void core_clear_lun_from_tpg(struct se_lun *lun, struct se_portal_group *tpg) mutex_lock(&nacl->lun_entry_mutex); hlist_for_each_entry_rcu(deve, &nacl->lun_entry_hlist, link) { - struct se_lun *tmp_lun = rcu_dereference_check(deve->se_lun, - lockdep_is_held(&nacl->lun_entry_mutex)); - - if (lun != tmp_lun) + if (lun != deve->se_lun) continue; core_disable_device_list_for_node(lun, deve, nacl, tpg); diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 3829b61b56c1..a1d67554709f 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -739,8 +739,7 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration( if (!deve_tmp->se_lun_acl) continue; - lacl_tmp = rcu_dereference_check(deve_tmp->se_lun_acl, - lockdep_is_held(&lun_tmp->lun_deve_lock)); + lacl_tmp = deve_tmp->se_lun_acl; nacl_tmp = lacl_tmp->se_lun_nacl; /* * Skip the matching struct se_node_acl that is allocated @@ -784,8 +783,7 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration( * the original *pr_reg is processed in * __core_scsi3_add_registration() */ - dest_lun = rcu_dereference_check(deve_tmp->se_lun, - kref_read(&deve_tmp->pr_kref) != 0); + dest_lun = deve_tmp->se_lun; pr_reg_atp = __core_scsi3_do_alloc_registration(dev, nacl_tmp, dest_lun, deve_tmp, @@ -1437,34 +1435,26 @@ static void core_scsi3_nodeacl_undepend_item(struct se_node_acl *nacl) static int core_scsi3_lunacl_depend_item(struct se_dev_entry *se_deve) { - struct se_lun_acl *lun_acl; - /* * For nacl->dynamic_node_acl=1 */ - lun_acl = rcu_dereference_check(se_deve->se_lun_acl, - kref_read(&se_deve->pr_kref) != 0); - if (!lun_acl) + if (!se_deve->se_lun_acl) return 0; - return target_depend_item(&lun_acl->se_lun_group.cg_item); + return target_depend_item(&se_deve->se_lun_acl->se_lun_group.cg_item); } static void core_scsi3_lunacl_undepend_item(struct se_dev_entry *se_deve) { - struct se_lun_acl *lun_acl; - /* * For nacl->dynamic_node_acl=1 */ - lun_acl = rcu_dereference_check(se_deve->se_lun_acl, - kref_read(&se_deve->pr_kref) != 0); - if (!lun_acl) { + if (!se_deve->se_lun_acl) { kref_put(&se_deve->pr_kref, target_pr_kref_release); return; } - target_undepend_item(&lun_acl->se_lun_group.cg_item); + target_undepend_item(&se_deve->se_lun_acl->se_lun_group.cg_item); kref_put(&se_deve->pr_kref, target_pr_kref_release); } @@ -1751,8 +1741,7 @@ core_scsi3_decode_spec_i_port( * and then call __core_scsi3_add_registration() in the * 2nd loop which will never fail. */ - dest_lun = rcu_dereference_check(dest_se_deve->se_lun, - kref_read(&dest_se_deve->pr_kref) != 0); + dest_lun = dest_se_deve->se_lun; dest_pr_reg = __core_scsi3_alloc_registration(cmd->se_dev, dest_node_acl, dest_lun, dest_se_deve, @@ -3446,8 +3435,7 @@ after_iport_check: dest_pr_reg = __core_scsi3_locate_pr_reg(dev, dest_node_acl, iport_ptr); if (!dest_pr_reg) { - struct se_lun *dest_lun = rcu_dereference_check(dest_se_deve->se_lun, - kref_read(&dest_se_deve->pr_kref) != 0); + struct se_lun *dest_lun = dest_se_deve->se_lun; spin_unlock(&dev->dev_reservation_lock); if (core_scsi3_alloc_registration(cmd->se_dev, dest_node_acl, diff --git a/drivers/target/target_core_stat.c b/drivers/target/target_core_stat.c index 62d15bcc3d93..f85ee5b0fd80 100644 --- a/drivers/target/target_core_stat.c +++ b/drivers/target/target_core_stat.c @@ -877,7 +877,6 @@ static ssize_t target_stat_auth_dev_show(struct config_item *item, struct se_lun_acl *lacl = auth_to_lacl(item); struct se_node_acl *nacl = lacl->se_lun_nacl; struct se_dev_entry *deve; - struct se_lun *lun; ssize_t ret; rcu_read_lock(); @@ -886,9 +885,9 @@ static ssize_t target_stat_auth_dev_show(struct config_item *item, rcu_read_unlock(); return -ENODEV; } - lun = rcu_dereference(deve->se_lun); + /* scsiDeviceIndex */ - ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_index); + ret = snprintf(page, PAGE_SIZE, "%u\n", deve->se_lun->lun_index); rcu_read_unlock(); return ret; } @@ -1217,7 +1216,6 @@ static ssize_t target_stat_iport_dev_show(struct config_item *item, struct se_lun_acl *lacl = iport_to_lacl(item); struct se_node_acl *nacl = lacl->se_lun_nacl; struct se_dev_entry *deve; - struct se_lun *lun; ssize_t ret; rcu_read_lock(); @@ -1226,9 +1224,9 @@ static ssize_t target_stat_iport_dev_show(struct config_item *item, rcu_read_unlock(); return -ENODEV; } - lun = rcu_dereference(deve->se_lun); + /* scsiDeviceIndex */ - ret = snprintf(page, PAGE_SIZE, "%u\n", lun->lun_index); + ret = snprintf(page, PAGE_SIZE, "%u\n", deve->se_lun->lun_index); rcu_read_unlock(); return ret; } diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c index 6bb20aa9c5bc..8713cda0c2fb 100644 --- a/drivers/target/target_core_xcopy.c +++ b/drivers/target/target_core_xcopy.c @@ -88,7 +88,7 @@ static int target_xcopy_locate_se_dev_e4(struct se_session *sess, struct se_device *this_dev; int rc; - this_lun = rcu_dereference(deve->se_lun); + this_lun = deve->se_lun; this_dev = rcu_dereference_raw(this_lun->lun_se_dev); rc = target_xcopy_locate_se_dev_e4_iter(this_dev, dev_wwn); diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h index c2b36f7d917d..8c920456edd9 100644 --- a/include/target/target_core_base.h +++ b/include/target/target_core_base.h @@ -665,9 +665,9 @@ struct se_dev_entry { /* Used for PR SPEC_I_PT=1 and REGISTER_AND_MOVE */ struct kref pr_kref; struct completion pr_comp; - struct se_lun_acl __rcu *se_lun_acl; + struct se_lun_acl *se_lun_acl; spinlock_t ua_lock; - struct se_lun __rcu *se_lun; + struct se_lun *se_lun; #define DEF_PR_REG_ACTIVE 1 unsigned long deve_flags; struct list_head alua_port_list; -- cgit v1.3-15-g9ac1 From fe442604199ed3e60d5411137159f9623534e956 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 28 Jul 2022 15:18:48 -0700 Subject: scsi: core: Make sure that targets outlive devices This commit prevents that the following sequence triggers a kernel crash: - Deletion of a SCSI device is requested via sysfs. Device removal takes some time because blk_cleanup_queue() is waiting for the SCSI error handler. - The SCSI target associated with that SCSI device is removed. - scsi_remove_target() returns and its caller frees the resources associated with the SCSI target. - The error handler makes progress and invokes an LLD callback that dereferences the SCSI target pointer. Link: https://lore.kernel.org/r/20220728221851.1822295-2-bvanassche@acm.org Cc: Christoph Hellwig Cc: Mike Christie Cc: Hannes Reinecke Cc: John Garry Cc: Li Zhijian Reported-by: Mike Christie Reviewed-by: Ming Lei Reviewed-by: Mike Christie Signed-off-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_scan.c | 2 ++ drivers/scsi/scsi_sysfs.c | 20 +++++++++++++++++--- include/scsi/scsi_device.h | 2 ++ 3 files changed, 21 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 91ac901a6682..4c1efd6a3b0c 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -521,6 +521,8 @@ static struct scsi_target *scsi_alloc_target(struct device *parent, starget->state = STARGET_CREATED; starget->scsi_level = SCSI_2; starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED; + init_waitqueue_head(&starget->sdev_wq); + retry: spin_lock_irqsave(shost->host_lock, flags); diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 43949798a2e4..1bc9c26fe1d4 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -443,7 +443,9 @@ static void scsi_device_cls_release(struct device *class_dev) static void scsi_device_dev_release_usercontext(struct work_struct *work) { - struct scsi_device *sdev; + struct scsi_device *sdev = container_of(work, struct scsi_device, + ew.work); + struct scsi_target *starget = sdev->sdev_target; struct device *parent; struct list_head *this, *tmp; struct scsi_vpd *vpd_pg80 = NULL, *vpd_pg83 = NULL; @@ -452,8 +454,6 @@ static void scsi_device_dev_release_usercontext(struct work_struct *work) unsigned long flags; struct module *mod; - sdev = container_of(work, struct scsi_device, ew.work); - mod = sdev->host->hostt->module; scsi_dh_release_device(sdev); @@ -516,6 +516,9 @@ static void scsi_device_dev_release_usercontext(struct work_struct *work) kfree(sdev->inquiry); kfree(sdev); + if (starget && atomic_dec_return(&starget->sdev_count) == 0) + wake_up(&starget->sdev_wq); + if (parent) put_device(parent); module_put(mod); @@ -1535,6 +1538,14 @@ static void __scsi_remove_target(struct scsi_target *starget) goto restart; } spin_unlock_irqrestore(shost->host_lock, flags); + + /* + * After scsi_remove_target() returns its caller can remove resources + * associated with @starget, e.g. an rport or session. Wait until all + * devices associated with @starget have been removed to prevent that + * a SCSI error handling callback function triggers a use-after-free. + */ + wait_event(starget->sdev_wq, atomic_read(&starget->sdev_count) == 0); } /** @@ -1645,6 +1656,9 @@ void scsi_sysfs_device_initialize(struct scsi_device *sdev) list_add_tail(&sdev->same_target_siblings, &starget->devices); list_add_tail(&sdev->siblings, &shost->__devices); spin_unlock_irqrestore(shost->host_lock, flags); + + atomic_inc(&starget->sdev_count); + /* * device can now only be removed via __scsi_remove_device() so hold * the target. Target will be held in CREATED state until something diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 7cf5f3b7589f..190d2081f4c6 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -309,6 +309,8 @@ struct scsi_target { struct list_head devices; struct device dev; struct kref reap_ref; /* last put renders target invisible */ + atomic_t sdev_count; + wait_queue_head_t sdev_wq; unsigned int channel; unsigned int id; /* target id ... replace * scsi_device.id eventually */ -- cgit v1.3-15-g9ac1 From 16728aaba62e8b3b170735fdc3d8aa972835c136 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 28 Jul 2022 15:18:49 -0700 Subject: scsi: core: Make sure that hosts outlive targets Fix the race conditions between SCSI LLD kernel module unloading and SCSI device and target removal by making sure that SCSI hosts are destroyed after all associated target and device objects have been freed. Link: https://lore.kernel.org/r/20220728221851.1822295-3-bvanassche@acm.org Cc: Christoph Hellwig Cc: Ming Lei Cc: Mike Christie Cc: Hannes Reinecke Cc: John Garry Reviewed-by: Mike Christie Signed-off-by: Ming Lei Signed-off-by: Bart Van Assche [ bvanassche: Reworked Ming's patch and split it ] Signed-off-by: Martin K. Petersen --- drivers/scsi/hosts.c | 8 ++++++++ drivers/scsi/scsi_scan.c | 7 +++++++ include/scsi/scsi_host.h | 3 +++ 3 files changed, 18 insertions(+) (limited to 'include') diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index ef6c0e37acce..8fa98c8d0ee0 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -190,6 +190,13 @@ void scsi_remove_host(struct Scsi_Host *shost) transport_unregister_device(&shost->shost_gendev); device_unregister(&shost->shost_dev); device_del(&shost->shost_gendev); + + /* + * After scsi_remove_host() has returned the scsi LLD module can be + * unloaded and/or the host resources can be released. Hence wait until + * the dependent SCSI targets and devices are gone before returning. + */ + wait_event(shost->targets_wq, atomic_read(&shost->target_count) == 0); } EXPORT_SYMBOL(scsi_remove_host); @@ -394,6 +401,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize) INIT_LIST_HEAD(&shost->starved_list); init_waitqueue_head(&shost->host_wait); mutex_init(&shost->scan_mutex); + init_waitqueue_head(&shost->targets_wq); index = ida_alloc(&host_index_ida, GFP_KERNEL); if (index < 0) { diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 4c1efd6a3b0c..ac6059702d13 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -406,9 +406,14 @@ static void scsi_target_destroy(struct scsi_target *starget) static void scsi_target_dev_release(struct device *dev) { struct device *parent = dev->parent; + struct Scsi_Host *shost = dev_to_shost(parent); struct scsi_target *starget = to_scsi_target(dev); kfree(starget); + + if (atomic_dec_return(&shost->target_count) == 0) + wake_up(&shost->targets_wq); + put_device(parent); } @@ -523,6 +528,8 @@ static struct scsi_target *scsi_alloc_target(struct device *parent, starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED; init_waitqueue_head(&starget->sdev_wq); + atomic_inc(&shost->target_count); + retry: spin_lock_irqsave(shost->host_lock, flags); diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 667d889b92b5..339f975d356e 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -689,6 +689,9 @@ struct Scsi_Host { /* ldm bits */ struct device shost_gendev, shost_dev; + atomic_t target_count; + wait_queue_head_t targets_wq; + /* * Points to the transport data (if any) which is allocated * separately -- cgit v1.3-15-g9ac1 From f482aa98652795846cc55da98ebe331eb74f3d0b Mon Sep 17 00:00:00 2001 From: Peilin Ye Date: Wed, 3 Aug 2022 15:23:43 -0700 Subject: audit, io_uring, io-wq: Fix memory leak in io_sq_thread() and io_wqe_worker() Currently @audit_context is allocated twice for io_uring workers: 1. copy_process() calls audit_alloc(); 2. io_sq_thread() or io_wqe_worker() calls audit_alloc_kernel() (which is effectively audit_alloc()) and overwrites @audit_context, causing: BUG: memory leak unreferenced object 0xffff888144547400 (size 1024): <...> hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [] audit_alloc+0x133/0x210 [] copy_process+0xcd3/0x2340 [] create_io_thread+0x63/0x90 [] create_io_worker+0xb4/0x230 [] io_wqe_enqueue+0x248/0x3b0 [] io_queue_iowq+0xba/0x200 [] io_queue_async+0x113/0x180 [] io_req_task_submit+0x18f/0x1a0 [] io_apoll_task_func+0xdd/0x120 [] tctx_task_work+0x11f/0x570 [] task_work_run+0x7e/0xc0 [] get_signal+0xc18/0xf10 [] arch_do_signal_or_restart+0x2b/0x730 [] exit_to_user_mode_prepare+0x5e/0x180 [] syscall_exit_to_user_mode+0x12/0x20 [] do_syscall_64+0x40/0x80 Then, 3. io_sq_thread() or io_wqe_worker() frees @audit_context using audit_free(); 4. do_exit() eventually calls audit_free() again, which is okay because audit_free() does a NULL check. As suggested by Paul Moore, fix it by deleting audit_alloc_kernel() and redundant audit_free() calls. Fixes: 5bd2182d58e9 ("audit,io_uring,io-wq: add some basic audit support to io_uring") Suggested-by: Paul Moore Cc: stable@vger.kernel.org Signed-off-by: Peilin Ye Acked-by: Paul Moore Link: https://lore.kernel.org/r/20220803222343.31673-1-yepeilin.cs@gmail.com Signed-off-by: Jens Axboe --- include/linux/audit.h | 5 ----- io_uring/io-wq.c | 3 --- io_uring/sqpoll.c | 4 ---- kernel/auditsc.c | 25 ------------------------- 4 files changed, 37 deletions(-) (limited to 'include') diff --git a/include/linux/audit.h b/include/linux/audit.h index 00f7a80f1a3e..3608992848d3 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -285,7 +285,6 @@ static inline int audit_signal_info(int sig, struct task_struct *t) /* These are defined in auditsc.c */ /* Public API */ extern int audit_alloc(struct task_struct *task); -extern int audit_alloc_kernel(struct task_struct *task); extern void __audit_free(struct task_struct *task); extern void __audit_uring_entry(u8 op); extern void __audit_uring_exit(int success, long code); @@ -578,10 +577,6 @@ static inline int audit_alloc(struct task_struct *task) { return 0; } -static inline int audit_alloc_kernel(struct task_struct *task) -{ - return 0; -} static inline void audit_free(struct task_struct *task) { } static inline void audit_uring_entry(u8 op) diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 77df5b43bf52..c6536d4b2da0 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -624,8 +624,6 @@ static int io_wqe_worker(void *data) snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid); set_task_comm(current, buf); - audit_alloc_kernel(current); - while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { long ret; @@ -660,7 +658,6 @@ static int io_wqe_worker(void *data) if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) io_worker_handle_work(worker); - audit_free(current); io_worker_exit(worker); return 0; } diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 76d4d70c733a..559652380672 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -235,8 +235,6 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; - audit_alloc_kernel(current); - mutex_lock(&sqd->lock); while (1) { bool cap_entries, sqt_spin = false; @@ -310,8 +308,6 @@ static int io_sq_thread(void *data) io_run_task_work(); mutex_unlock(&sqd->lock); - audit_free(current); - complete(&sqd->exited); do_exit(0); } diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 3a8c9d744800..dd8d9ab747c3 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1073,31 +1073,6 @@ int audit_alloc(struct task_struct *tsk) return 0; } -/** - * audit_alloc_kernel - allocate an audit_context for a kernel task - * @tsk: the kernel task - * - * Similar to the audit_alloc() function, but intended for kernel private - * threads. Returns zero on success, negative values on failure. - */ -int audit_alloc_kernel(struct task_struct *tsk) -{ - /* - * At the moment we are just going to call into audit_alloc() to - * simplify the code, but there two things to keep in mind with this - * approach: - * - * 1. Filtering internal kernel tasks is a bit laughable in almost all - * cases, but there is at least one case where there is a benefit: - * the '-a task,never' case allows the admin to effectively disable - * task auditing at runtime. - * - * 2. The {set,clear}_task_syscall_work() ops likely have zero effect - * on these internal kernel tasks, but they probably don't hurt either. - */ - return audit_alloc(tsk); -} - static inline void audit_free_context(struct audit_context *context) { /* resetting is extra work, but it is likely just noise */ -- cgit v1.3-15-g9ac1 From 46dae32fe625a75f549c3a70edc77b778197bb05 Mon Sep 17 00:00:00 2001 From: Youngmin Nam Date: Tue, 12 Jul 2022 18:47:15 +0900 Subject: time: Correct the prototype of ns_to_kernel_old_timeval and ns_to_timespec64 In ns_to_kernel_old_timeval() definition, the function argument is defined with const identifier in kernel/time/time.c, but the prototype in include/linux/time32.h looks different. - The function is defined in kernel/time/time.c as below: struct __kernel_old_timeval ns_to_kernel_old_timeval(const s64 nsec) - The function is decalared in include/linux/time32.h as below: extern struct __kernel_old_timeval ns_to_kernel_old_timeval(s64 nsec); Because the variable of arithmethic types isn't modified in the calling scope, there's no need to mark arguments as const, which was already mentioned during review (Link[1) of the original patch. Likewise remove the "const" keyword in both definition and declaration of ns_to_timespec64() as requested by Arnd (Link[2]). Fixes: a84d1169164b ("y2038: Introduce struct __kernel_old_timeval") Signed-off-by: Youngmin Nam Signed-off-by: Thomas Gleixner Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/all/20220712094715.2918823-1-youngmin.nam@samsung.com Link[1]: https://lore.kernel.org/all/20180310081123.thin6wphgk7tongy@gmail.com/ Link[2]: https://lore.kernel.org/all/CAK8P3a3nknJgEDESGdJH91jMj6R_xydFqWASd8r5BbesdvMBgA@mail.gmail.com/ --- include/linux/time64.h | 2 +- kernel/time/time.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/time64.h b/include/linux/time64.h index 2fb8232cff1d..f1bcea8c124a 100644 --- a/include/linux/time64.h +++ b/include/linux/time64.h @@ -145,7 +145,7 @@ static inline s64 timespec64_to_ns(const struct timespec64 *ts) * * Returns the timespec64 representation of the nsec parameter. */ -extern struct timespec64 ns_to_timespec64(const s64 nsec); +extern struct timespec64 ns_to_timespec64(s64 nsec); /** * timespec64_add_ns - Adds nanoseconds to a timespec64 diff --git a/kernel/time/time.c b/kernel/time/time.c index 29923b20e0e4..526257b3727c 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -449,7 +449,7 @@ time64_t mktime64(const unsigned int year0, const unsigned int mon0, } EXPORT_SYMBOL(mktime64); -struct __kernel_old_timeval ns_to_kernel_old_timeval(const s64 nsec) +struct __kernel_old_timeval ns_to_kernel_old_timeval(s64 nsec) { struct timespec64 ts = ns_to_timespec64(nsec); struct __kernel_old_timeval tv; @@ -503,7 +503,7 @@ EXPORT_SYMBOL(set_normalized_timespec64); * * Returns the timespec64 representation of the nsec parameter. */ -struct timespec64 ns_to_timespec64(const s64 nsec) +struct timespec64 ns_to_timespec64(s64 nsec) { struct timespec64 ts = { 0, 0 }; s32 rem; -- cgit v1.3-15-g9ac1 From c2a052a4a949df53f50a5024843432d2234cb824 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Fri, 24 Jun 2022 10:55:41 +0800 Subject: remoteproc: rename len of rpoc_vring to num Rename the member len in the structure rpoc_vring to num. And remove 'in bytes' from the comment of it. This is misleading. Because this actually refers to the size of the virtio vring to be created. The unit is not bytes. Signed-off-by: Xuan Zhuo Message-Id: <20220624025621.128843-2-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/remoteproc/remoteproc_core.c | 4 ++-- drivers/remoteproc/remoteproc_virtio.c | 10 +++++----- include/linux/remoteproc.h | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 02a04ab34a23..2d2f3bab5888 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -334,7 +334,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) size_t size; /* actual size of vring (in bytes) */ - size = PAGE_ALIGN(vring_size(rvring->len, rvring->align)); + size = PAGE_ALIGN(vring_size(rvring->num, rvring->align)); rsc = (void *)rproc->table_ptr + rvdev->rsc_offset; @@ -401,7 +401,7 @@ rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i) return -EINVAL; } - rvring->len = vring->num; + rvring->num = vring->num; rvring->align = vring->align; rvring->rvdev = rvdev; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 70ab496d0431..d43d74733f0a 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -87,7 +87,7 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, struct fw_rsc_vdev *rsc; struct virtqueue *vq; void *addr; - int len, size; + int num, size; /* we're temporarily limited to two virtqueues per rvdev */ if (id >= ARRAY_SIZE(rvdev->vring)) @@ -104,20 +104,20 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, rvring = &rvdev->vring[id]; addr = mem->va; - len = rvring->len; + num = rvring->num; /* zero vring */ - size = vring_size(len, rvring->align); + size = vring_size(num, rvring->align); memset(addr, 0, size); dev_dbg(dev, "vring%d: va %pK qsz %d notifyid %d\n", - id, addr, len, rvring->notifyid); + id, addr, num, rvring->notifyid); /* * Create the new vq, and tell virtio we're not interested in * the 'weak' smp barriers, since we're talking with a real device. */ - vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, ctx, + vq = vring_new_virtqueue(id, num, rvring->align, vdev, false, ctx, addr, rproc_virtio_notify, callback, name); if (!vq) { dev_err(dev, "vring_new_virtqueue %s failed\n", name); diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 7c943f0a2fc4..aea79c77db0f 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -597,7 +597,7 @@ struct rproc_subdev { /** * struct rproc_vring - remoteproc vring state * @va: virtual address - * @len: length, in bytes + * @num: vring size * @da: device address * @align: vring alignment * @notifyid: rproc-specific unique vring index @@ -606,7 +606,7 @@ struct rproc_subdev { */ struct rproc_vring { void *va; - int len; + int num; u32 da; u32 align; int notifyid; -- cgit v1.3-15-g9ac1 From da802961832f9852886304290135457519815497 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:21 +0800 Subject: virtio: record the maximum queue num supported by the device. virtio-net can display the maximum (supported by hardware) ring size in ethtool -g eth0. When the subsequent patch implements vring reset, it can judge whether the ring size passed by the driver is legal based on this. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-2-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- arch/um/drivers/virtio_uml.c | 1 + drivers/platform/mellanox/mlxbf-tmfifo.c | 2 ++ drivers/remoteproc/remoteproc_virtio.c | 2 ++ drivers/s390/virtio/virtio_ccw.c | 3 +++ drivers/virtio/virtio_mmio.c | 2 ++ drivers/virtio/virtio_pci_legacy.c | 2 ++ drivers/virtio/virtio_pci_modern.c | 2 ++ drivers/virtio/virtio_vdpa.c | 2 ++ include/linux/virtio.h | 2 ++ 9 files changed, 18 insertions(+) (limited to 'include') diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 82ff3785bf69..e719af8bdf56 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -958,6 +958,7 @@ static struct virtqueue *vu_setup_vq(struct virtio_device *vdev, goto error_create; } vq->priv = info; + vq->num_max = num; num = virtqueue_get_vring_size(vq); if (vu_dev->protocol_features & diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 38800e86ed8a..1ae3c56b66b0 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -959,6 +959,8 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, goto error; } + vq->num_max = vring->num; + vqs[i] = vq; vring->vq = vq; vq->priv = vring; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index d43d74733f0a..0f7706e23eb9 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -125,6 +125,8 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, return ERR_PTR(-ENOMEM); } + vq->num_max = num; + rvring->vq = vq; vq->priv = rvring; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 161d3b141f0d..6b86d0280d6b 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -530,6 +530,9 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, err = -ENOMEM; goto out_err; } + + vq->num_max = info->num; + /* it may have been reduced */ info->num = virtqueue_get_vring_size(vq); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 945cb8fb60b6..3ff746e3f24a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -403,6 +403,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_new_virtqueue; } + vq->num_max = num; + /* Activate the queue */ writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); if (vm_dev->version == 1) { diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index a5e5721145c7..2257f1b3d8ae 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -135,6 +135,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; if (q_pfn >> 32) { dev_err(&vp_dev->pci_dev->dev, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 623906b4996c..e7e0b8c850f6 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -218,6 +218,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + /* activate the queue */ vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index c40f7deb6b5a..9670cc79371d 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -183,6 +183,8 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, goto error_new_virtqueue; } + vq->num_max = max_num; + /* Setup virtqueue callback */ cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; cb.private = info; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index d8fdf170637c..129bde7521e3 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -19,6 +19,7 @@ * @priv: a pointer for the virtqueue implementation to use. * @index: the zero-based ordinal number for this queue. * @num_free: number of elements we expect to be able to fit. + * @num_max: the maximum number of elements supported by the device. * * A note on @num_free: with indirect buffers, each buffer needs one * element in the queue, otherwise a buffer will need one element per @@ -31,6 +32,7 @@ struct virtqueue { struct virtio_device *vdev; unsigned int index; unsigned int num_free; + unsigned int num_max; void *priv; }; -- cgit v1.3-15-g9ac1 From 3086e9fc9173166774652a488467e4176ee1c81b Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:22 +0800 Subject: virtio: struct virtio_config_ops add callbacks for queue_reset reset can be divided into the following four steps (example): 1. transport: notify the device to reset the queue 2. vring: recycle the buffer submitted 3. vring: reset/resize the vring (may re-alloc) 4. transport: mmap vring to device, and enable the queue In order to support queue reset, add two callbacks in struct virtio_config_ops to implement steps 1 and 4. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-3-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- include/linux/virtio_config.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include') diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index b47c2e7ed0ee..36ec7be1f480 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -78,6 +78,18 @@ struct virtio_shm_region { * @set_vq_affinity: set the affinity for a virtqueue (optional). * @get_vq_affinity: get the affinity for a virtqueue (optional). * @get_shm_region: get a shared memory region based on the index. + * @disable_vq_and_reset: reset a queue individually (optional). + * vq: the virtqueue + * Returns 0 on success or error status + * disable_vq_and_reset will guarantee that the callbacks are disabled and + * synchronized. + * Except for the callback, the caller should guarantee that the vring is + * not accessed by any functions of virtqueue. + * @enable_vq_after_reset: enable a reset queue + * vq: the virtqueue + * Returns 0 on success or error status + * If disable_vq_and_reset is set, then enable_vq_after_reset must also be + * set. */ typedef void vq_callback_t(struct virtqueue *); struct virtio_config_ops { @@ -104,6 +116,8 @@ struct virtio_config_ops { int index); bool (*get_shm_region)(struct virtio_device *vdev, struct virtio_shm_region *region, u8 id); + int (*disable_vq_and_reset)(struct virtqueue *vq); + int (*enable_vq_after_reset)(struct virtqueue *vq); }; /* If driver didn't advertise the feature, it will never appear. */ -- cgit v1.3-15-g9ac1 From 07d9629d49584b6f79faa6158cd7aef7e6919703 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:27 +0800 Subject: virtio_ring: split: stop __vring_new_virtqueue as export symbol There is currently only one place to reference __vring_new_virtqueue() directly from the outside of virtio core. And here vring_new_virtqueue() can be used instead. Subsequent patches will modify __vring_new_virtqueue, so stop it as an export symbol for now. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-8-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 25 ++++++++++++++++--------- include/linux/virtio_ring.h | 10 ---------- tools/virtio/virtio_test.c | 4 ++-- 3 files changed, 18 insertions(+), 21 deletions(-) (limited to 'include') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a63ef2d99955..8ce6cc73d814 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -204,6 +204,14 @@ struct vring_virtqueue { #endif }; +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name); /* * Helpers. @@ -2195,14 +2203,14 @@ irqreturn_t vring_interrupt(int irq, void *_vq) EXPORT_SYMBOL_GPL(vring_interrupt); /* Only available for split ring */ -struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, - struct virtio_device *vdev, - bool weak_barriers, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) { struct vring_virtqueue *vq; @@ -2277,7 +2285,6 @@ err_state: kfree(vq); return NULL; } -EXPORT_SYMBOL_GPL(__vring_new_virtqueue); struct virtqueue *vring_create_virtqueue( unsigned int index, diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index b485b13fa50b..8b8af1a38991 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -76,16 +76,6 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, void (*callback)(struct virtqueue *vq), const char *name); -/* Creates a virtqueue with a custom layout. */ -struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, - struct virtio_device *vdev, - bool weak_barriers, - bool ctx, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name); - /* * Creates a virtqueue with a standard layout but a caller-allocated * ring. diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c index 23f142af544a..86a410ddcedd 100644 --- a/tools/virtio/virtio_test.c +++ b/tools/virtio/virtio_test.c @@ -102,8 +102,8 @@ static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) memset(info->ring, 0, vring_size(num, 4096)); vring_init(&info->vring, num, info->ring, 4096); - info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true, - false, vq_notify, vq_callback, "test"); + info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false, + info->ring, vq_notify, vq_callback, "test"); assert(info->vq); info->vq->priv = info; } -- cgit v1.3-15-g9ac1 From c790e8e1817f1a17c05e64f1c4f16f231b8529d5 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:44 +0800 Subject: virtio_ring: introduce virtqueue_resize() Introduce virtqueue_resize() to implement the resize of vring. Based on these, the driver can dynamically adjust the size of the vring. For example: ethtool -G. virtqueue_resize() implements resize based on the vq reset function. In case of failure to allocate a new vring, it will give up resize and use the original vring. During this process, if the re-enable reset vq fails, the vq can no longer be used. Although the probability of this situation is not high. The parameter recycle is used to recycle the buffer that is no longer used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-25-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/virtio.h | 3 ++ 2 files changed, 72 insertions(+) (limited to 'include') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index bea5a3448217..6447a09e2e38 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2539,6 +2539,75 @@ struct virtqueue *vring_create_virtqueue( } EXPORT_SYMBOL_GPL(vring_create_virtqueue); +/** + * virtqueue_resize - resize the vring of vq + * @_vq: the struct virtqueue we're talking about. + * @num: new ring num + * @recycle: callback for recycle the useless buffer + * + * When it is really necessary to create a new vring, it will set the current vq + * into the reset state. Then call the passed callback to recycle the buffer + * that is no longer used. Only after the new vring is successfully created, the + * old vring will be released. + * + * Caller must ensure we don't call this with other virtqueue operations + * at the same time (except where noted). + * + * Returns zero or a negative error. + * 0: success. + * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size. + * vq can still work normally + * -EBUSY: Failed to sync with device, vq may not work properly + * -ENOENT: Transport or device not supported + * -E2BIG/-EINVAL: num error + * -EPERM: Operation not permitted + * + */ +int virtqueue_resize(struct virtqueue *_vq, u32 num, + void (*recycle)(struct virtqueue *vq, void *buf)) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = vq->vq.vdev; + void *buf; + int err; + + if (!vq->we_own_ring) + return -EPERM; + + if (num > vq->vq.num_max) + return -E2BIG; + + if (!num) + return -EINVAL; + + if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) + return 0; + + if (!vdev->config->disable_vq_and_reset) + return -ENOENT; + + if (!vdev->config->enable_vq_after_reset) + return -ENOENT; + + err = vdev->config->disable_vq_and_reset(_vq); + if (err) + return err; + + while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL) + recycle(_vq, buf); + + if (vq->packed_ring) + err = virtqueue_resize_packed(_vq, num); + else + err = virtqueue_resize_split(_vq, num); + + if (vdev->config->enable_vq_after_reset(_vq)) + return -EBUSY; + + return err; +} +EXPORT_SYMBOL_GPL(virtqueue_resize); + /* Only available for split ring */ struct virtqueue *vring_new_virtqueue(unsigned int index, unsigned int num, diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 129bde7521e3..62e31bca5602 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -91,6 +91,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); +int virtqueue_resize(struct virtqueue *vq, u32 num, + void (*recycle)(struct virtqueue *vq, void *buf)); + /** * virtio_device - representation of a device using virtio * @index: unique position on the virtio bus -- cgit v1.3-15-g9ac1 From ea024594b1dc5b6719c1400ae154690f5c203996 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:45 +0800 Subject: virtio_pci: struct virtio_pci_common_cfg add queue_notify_data Add queue_notify_data in struct virtio_pci_common_cfg, which comes from here https://github.com/oasis-tcs/virtio-spec/issues/89 In order not to affect the API, add a dedicated structure struct virtio_pci_modern_common_cfg to virtio_pci_modern.h. Since I want to add queue_reset after queue_notify_data, I submitted this patch first. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-26-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- include/linux/virtio_pci_modern.h | 7 +++++++ include/uapi/linux/virtio_pci.h | 1 + 2 files changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h index eb2bd9b4077d..41f5a018bd94 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -5,6 +5,13 @@ #include #include +struct virtio_pci_modern_common_cfg { + struct virtio_pci_common_cfg cfg; + + __le16 queue_notify_data; /* read-write */ + __le16 padding; +}; + struct virtio_pci_modern_device { struct pci_dev *pci_dev; diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pci.h index 3a86f36d7e3d..f5981a874481 100644 --- a/include/uapi/linux/virtio_pci.h +++ b/include/uapi/linux/virtio_pci.h @@ -202,6 +202,7 @@ struct virtio_pci_cfg_cap { #define VIRTIO_PCI_COMMON_Q_AVAILHI 44 #define VIRTIO_PCI_COMMON_Q_USEDLO 48 #define VIRTIO_PCI_COMMON_Q_USEDHI 52 +#define VIRTIO_PCI_COMMON_Q_NDATA 56 #endif /* VIRTIO_PCI_NO_MODERN */ -- cgit v1.3-15-g9ac1 From 3251063155032729b8793ac3957136ae25c0bafa Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:46 +0800 Subject: virtio: allow to unbreak/break virtqueue individually This patch allows the new introduced __virtqueue_break()/__virtqueue_unbreak() to break/unbreak the virtqueue. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-27-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++++++ include/linux/virtio.h | 3 +++ 2 files changed, 27 insertions(+) (limited to 'include') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 6447a09e2e38..accb3ae6cc95 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2724,6 +2724,30 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_break(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, true); +} +EXPORT_SYMBOL_GPL(__virtqueue_break); + +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_unbreak(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, false); +} +EXPORT_SYMBOL_GPL(__virtqueue_unbreak); + bool virtqueue_is_broken(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 62e31bca5602..d45ee82a4470 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -138,6 +138,9 @@ bool is_virtio_device(struct device *dev); void virtio_break_device(struct virtio_device *dev); void __virtio_unbreak_device(struct virtio_device *dev); +void __virtqueue_break(struct virtqueue *_vq); +void __virtqueue_unbreak(struct virtqueue *_vq); + void virtio_config_changed(struct virtio_device *dev); #ifdef CONFIG_PM_SLEEP int virtio_device_freeze(struct virtio_device *dev); -- cgit v1.3-15-g9ac1 From d94587b5bb5c4bba32fbc2bd92c86cc6de25167f Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:47 +0800 Subject: virtio: queue_reset: add VIRTIO_F_RING_RESET Added VIRTIO_F_RING_RESET, it came from here https://github.com/oasis-tcs/virtio-spec/issues/124 https://github.com/oasis-tcs/virtio-spec/issues/139 This feature indicates that the driver can reset a queue individually. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-28-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- include/uapi/linux/virtio_config.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h index f0fb0ae021c0..3c05162bc988 100644 --- a/include/uapi/linux/virtio_config.h +++ b/include/uapi/linux/virtio_config.h @@ -52,7 +52,7 @@ * rest are per-device feature bits. */ #define VIRTIO_TRANSPORT_F_START 28 -#define VIRTIO_TRANSPORT_F_END 38 +#define VIRTIO_TRANSPORT_F_END 41 #ifndef VIRTIO_CONFIG_NO_LEGACY /* Do we get callbacks when the ring is completely used, even if we've @@ -98,4 +98,9 @@ * Does the device support Single Root I/O Virtualization? */ #define VIRTIO_F_SR_IOV 37 + +/* + * This feature indicates that the driver can reset a queue individually. + */ +#define VIRTIO_F_RING_RESET 40 #endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */ -- cgit v1.3-15-g9ac1 From 4913e85441b40386c4bb093f188b955d8165f1b7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:48 +0800 Subject: virtio_ring: struct virtqueue introduce reset Introduce a new member reset to the structure virtqueue to determine whether the current vq is in the reset state. Subsequent patches will use it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-29-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 2 ++ include/linux/virtio.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'include') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index accb3ae6cc95..d66c8e6d0ef3 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1996,6 +1996,7 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = true; vq->notify = notify; vq->weak_barriers = weak_barriers; @@ -2481,6 +2482,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = false; vq->notify = notify; vq->weak_barriers = weak_barriers; diff --git a/include/linux/virtio.h b/include/linux/virtio.h index d45ee82a4470..a3f73bb6733e 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -20,6 +20,7 @@ * @index: the zero-based ordinal number for this queue. * @num_free: number of elements we expect to be able to fit. * @num_max: the maximum number of elements supported by the device. + * @reset: vq is in reset state or not. * * A note on @num_free: with indirect buffers, each buffer needs one * element in the queue, otherwise a buffer will need one element per @@ -34,6 +35,7 @@ struct virtqueue { unsigned int num_free; unsigned int num_max; void *priv; + bool reset; }; int virtqueue_add_outbuf(struct virtqueue *vq, -- cgit v1.3-15-g9ac1 From 0cdd450e70510c9e13af8099e9f6c1467e6a0b91 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:49 +0800 Subject: virtio_pci: struct virtio_pci_common_cfg add queue_reset Add queue_reset in virtio_pci_modern_common_cfg. https://github.com/oasis-tcs/virtio-spec/issues/124 https://github.com/oasis-tcs/virtio-spec/issues/139 Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-30-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- include/linux/virtio_pci_modern.h | 2 +- include/uapi/linux/virtio_pci.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h index 41f5a018bd94..05123b9a606f 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -9,7 +9,7 @@ struct virtio_pci_modern_common_cfg { struct virtio_pci_common_cfg cfg; __le16 queue_notify_data; /* read-write */ - __le16 padding; + __le16 queue_reset; /* read-write */ }; struct virtio_pci_modern_device { diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pci.h index f5981a874481..f703afc7ad31 100644 --- a/include/uapi/linux/virtio_pci.h +++ b/include/uapi/linux/virtio_pci.h @@ -203,6 +203,7 @@ struct virtio_pci_cfg_cap { #define VIRTIO_PCI_COMMON_Q_USEDLO 48 #define VIRTIO_PCI_COMMON_Q_USEDHI 52 #define VIRTIO_PCI_COMMON_Q_NDATA 56 +#define VIRTIO_PCI_COMMON_Q_RESET 58 #endif /* VIRTIO_PCI_NO_MODERN */ -- cgit v1.3-15-g9ac1 From 0b50cece0b7857732d2055f2c77f8730c10f9196 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:50 +0800 Subject: virtio_pci: introduce helper to get/set queue reset Introduce new helpers to implement queue reset and get queue reset status. https://github.com/oasis-tcs/virtio-spec/issues/124 https://github.com/oasis-tcs/virtio-spec/issues/139 Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-31-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern_dev.c | 39 ++++++++++++++++++++++++++++++++++ include/linux/virtio_pci_modern.h | 2 ++ 2 files changed, 41 insertions(+) (limited to 'include') diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index fa2a9445bb18..869cb46bef96 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -3,6 +3,7 @@ #include #include #include +#include /* * vp_modern_map_capability - map a part of virtio pci capability @@ -474,6 +475,44 @@ void vp_modern_set_status(struct virtio_pci_modern_device *mdev, } EXPORT_SYMBOL_GPL(vp_modern_set_status); +/* + * vp_modern_get_queue_reset - get the queue reset status + * @mdev: the modern virtio-pci device + * @index: queue index + */ +int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + return vp_ioread16(&cfg->queue_reset); +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset); + +/* + * vp_modern_set_queue_reset - reset the queue + * @mdev: the modern virtio-pci device + * @index: queue index + */ +void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + vp_iowrite16(1, &cfg->queue_reset); + + while (vp_ioread16(&cfg->queue_reset)) + msleep(1); + + while (vp_ioread16(&cfg->cfg.queue_enable)) + msleep(1); +} +EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset); + /* * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue * @mdev: the modern virtio-pci device diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h index 05123b9a606f..c4eeb79b0139 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -113,4 +113,6 @@ void __iomem * vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev, u16 index, resource_size_t *pa); int vp_modern_probe(struct virtio_pci_modern_device *mdev); void vp_modern_remove(struct virtio_pci_modern_device *mdev); +int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index); +void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index); #endif -- cgit v1.3-15-g9ac1 From a10fba0377145fccefea4dc4dd5915b7ed87e546 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:53 +0800 Subject: virtio: find_vqs() add arg sizes find_vqs() adds a new parameter sizes to specify the size of each vq vring. NULL as sizes means that all queues in find_vqs() use the maximum size. A value in the array is 0, which means that the corresponding queue uses the maximum size. In the split scenario, the meaning of size is the largest size, because it may be limited by memory, the virtio core will try a smaller size. And the size is power of 2. Signed-off-by: Xuan Zhuo Acked-by: Hans de Goede Reviewed-by: Mathieu Poirier Acked-by: Jason Wang Message-Id: <20220801063902.129329-34-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- arch/um/drivers/virtio_uml.c | 2 +- drivers/platform/mellanox/mlxbf-tmfifo.c | 1 + drivers/remoteproc/remoteproc_virtio.c | 1 + drivers/s390/virtio/virtio_ccw.c | 1 + drivers/virtio/virtio_mmio.c | 1 + drivers/virtio/virtio_pci_common.c | 2 +- drivers/virtio/virtio_pci_common.h | 2 +- drivers/virtio/virtio_pci_modern.c | 7 +++++-- drivers/virtio/virtio_vdpa.c | 1 + include/linux/virtio_config.h | 14 +++++++++----- 10 files changed, 22 insertions(+), 10 deletions(-) (limited to 'include') diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index e719af8bdf56..79e38afd4b91 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -1011,7 +1011,7 @@ error_kzalloc: static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc) { struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 1ae3c56b66b0..8be13d416f48 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -928,6 +928,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 0f7706e23eb9..81c4f5776109 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -158,6 +158,7 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool * ctx, struct irq_affinity *desc) { diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 6b86d0280d6b..72500cd2dbf5 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -635,6 +635,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 3ff746e3f24a..dfcecfd7aba1 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -474,6 +474,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ad258a9d3b9f..7ad734584823 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -396,7 +396,7 @@ out_del_vqs: /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc) { int err; diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 23112d84218f..a5ff838b85a5 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -110,7 +110,7 @@ void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc); const char *vp_bus_name(struct virtio_device *vdev); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index c3b9f2761849..be51ec849252 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -347,12 +347,15 @@ err: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; - int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); + int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx, + desc); if (rc) return rc; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 9670cc79371d..832d2c5b1b19 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -269,6 +269,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 36ec7be1f480..888f7e96f0c7 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -55,6 +55,7 @@ struct virtio_shm_region { * include a NULL entry for vqs that do not need a callback * names: array of virtqueue names (mainly for debugging) * include a NULL entry for vqs unused by driver + * sizes: array of virtqueue sizes * Returns 0 on success or error status * @del_vqs: free virtqueues found by find_vqs(). * @synchronize_cbs: synchronize with the virtqueue callbacks (optional) @@ -103,7 +104,9 @@ struct virtio_config_ops { void (*reset)(struct virtio_device *vdev); int (*find_vqs)(struct virtio_device *, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc); void (*del_vqs)(struct virtio_device *); void (*synchronize_cbs)(struct virtio_device *); @@ -212,7 +215,7 @@ struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev, const char *names[] = { n }; struct virtqueue *vq; int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL, - NULL); + NULL, NULL); if (err < 0) return ERR_PTR(err); return vq; @@ -224,7 +227,8 @@ int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, const char * const names[], struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, + NULL, desc); } static inline @@ -233,8 +237,8 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, const char * const names[], const bool *ctx, struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, - desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, + ctx, desc); } /** -- cgit v1.3-15-g9ac1 From fe3dc04e31aa51f91dc7f741a5f76cc4817eb5b4 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:56 +0800 Subject: virtio: add helper virtio_find_vqs_ctx_size() Introduce helper virtio_find_vqs_ctx_size() to call find_vqs and specify the maximum size of each vq ring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-37-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- include/linux/virtio_config.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include') diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 888f7e96f0c7..6adff09f7170 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -241,6 +241,18 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, ctx, desc); } +static inline +int virtio_find_vqs_ctx_size(struct virtio_device *vdev, u32 nvqs, + struct virtqueue *vqs[], + vq_callback_t *callbacks[], + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc) +{ + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, + ctx, desc); +} + /** * virtio_synchronize_cbs - synchronize with virtqueue callbacks * @vdev: the device -- cgit v1.3-15-g9ac1 From 699b045a8e43bd1063db4795be685bfd659649dc Mon Sep 17 00:00:00 2001 From: Alvaro Karsz Date: Mon, 18 Jul 2022 12:11:02 +0300 Subject: net: virtio_net: notifications coalescing support New VirtIO network feature: VIRTIO_NET_F_NOTF_COAL. Control a Virtio network device notifications coalescing parameters using the control virtqueue. A device that supports this fetature can receive VIRTIO_NET_CTRL_NOTF_COAL control commands. - VIRTIO_NET_CTRL_NOTF_COAL_TX_SET: Ask the network device to change the following parameters: - tx_usecs: Maximum number of usecs to delay a TX notification. - tx_max_packets: Maximum number of packets to send before a TX notification. - VIRTIO_NET_CTRL_NOTF_COAL_RX_SET: Ask the network device to change the following parameters: - rx_usecs: Maximum number of usecs to delay a RX notification. - rx_max_packets: Maximum number of packets to receive before a RX notification. VirtIO spec. patch: https://lists.oasis-open.org/archives/virtio-comment/202206/msg00100.html Signed-off-by: Alvaro Karsz Message-Id: <20220718091102.498774-1-alvaro.karsz@solid-run.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Jakub Kicinski Acked-by: Jason Wang --- drivers/net/virtio_net.c | 111 ++++++++++++++++++++++++++++++++++------ include/uapi/linux/virtio_net.h | 34 +++++++++++- 2 files changed, 129 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 16783ed782c5..d9c434b00e9b 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -270,6 +270,12 @@ struct virtnet_info { u8 duplex; u32 speed; + /* Interrupt coalescing settings */ + u32 tx_usecs; + u32 rx_usecs; + u32 tx_max_packets; + u32 rx_max_packets; + unsigned long guest_offloads; unsigned long guest_offloads_capable; @@ -2737,27 +2743,89 @@ static int virtnet_get_link_ksettings(struct net_device *dev, return 0; } +static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, + struct ethtool_coalesce *ec) +{ + struct scatterlist sgs_tx, sgs_rx; + struct virtio_net_ctrl_coal_tx coal_tx; + struct virtio_net_ctrl_coal_rx coal_rx; + + coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); + coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); + sg_init_one(&sgs_tx, &coal_tx, sizeof(coal_tx)); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, + VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, + &sgs_tx)) + return -EINVAL; + + /* Save parameters */ + vi->tx_usecs = ec->tx_coalesce_usecs; + vi->tx_max_packets = ec->tx_max_coalesced_frames; + + coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); + coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); + sg_init_one(&sgs_rx, &coal_rx, sizeof(coal_rx)); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, + VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, + &sgs_rx)) + return -EINVAL; + + /* Save parameters */ + vi->rx_usecs = ec->rx_coalesce_usecs; + vi->rx_max_packets = ec->rx_max_coalesced_frames; + + return 0; +} + +static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) +{ + /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL + * feature is negotiated. + */ + if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) + return -EOPNOTSUPP; + + if (ec->tx_max_coalesced_frames > 1 || + ec->rx_max_coalesced_frames != 1) + return -EINVAL; + + return 0; +} + static int virtnet_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); - int i, napi_weight; - - if (ec->tx_max_coalesced_frames > 1 || - ec->rx_max_coalesced_frames != 1) - return -EINVAL; + int ret, i, napi_weight; + bool update_napi = false; + /* Can't change NAPI weight if the link is up */ napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; if (napi_weight ^ vi->sq[0].napi.weight) { if (dev->flags & IFF_UP) return -EBUSY; + else + update_napi = true; + } + + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) + ret = virtnet_send_notf_coal_cmds(vi, ec); + else + ret = virtnet_coal_params_supported(ec); + + if (ret) + return ret; + + if (update_napi) { for (i = 0; i < vi->max_queue_pairs; i++) vi->sq[i].napi.weight = napi_weight; } - return 0; + return ret; } static int virtnet_get_coalesce(struct net_device *dev, @@ -2765,16 +2833,19 @@ static int virtnet_get_coalesce(struct net_device *dev, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { - struct ethtool_coalesce ec_default = { - .cmd = ETHTOOL_GCOALESCE, - .rx_max_coalesced_frames = 1, - }; struct virtnet_info *vi = netdev_priv(dev); - memcpy(ec, &ec_default, sizeof(ec_default)); + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { + ec->rx_coalesce_usecs = vi->rx_usecs; + ec->tx_coalesce_usecs = vi->tx_usecs; + ec->tx_max_coalesced_frames = vi->tx_max_packets; + ec->rx_max_coalesced_frames = vi->rx_max_packets; + } else { + ec->rx_max_coalesced_frames = 1; - if (vi->sq[0].napi.weight) - ec->tx_max_coalesced_frames = 1; + if (vi->sq[0].napi.weight) + ec->tx_max_coalesced_frames = 1; + } return 0; } @@ -2893,7 +2964,8 @@ static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) } static const struct ethtool_ops virtnet_ethtool_ops = { - .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES, + .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_USECS, .get_drvinfo = virtnet_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = virtnet_get_ringparam, @@ -3606,6 +3678,8 @@ static bool virtnet_validate_features(struct virtio_device *vdev) VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, "VIRTIO_NET_F_CTRL_VQ") || VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, + "VIRTIO_NET_F_CTRL_VQ") || + VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, "VIRTIO_NET_F_CTRL_VQ"))) { return false; } @@ -3742,6 +3816,13 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { + vi->rx_usecs = 0; + vi->tx_usecs = 0; + vi->tx_max_packets = 0; + vi->rx_max_packets = 0; + } + if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) vi->has_rss_hash_report = true; @@ -3977,7 +4058,7 @@ static struct virtio_device_id id_table[] = { VIRTIO_NET_F_CTRL_MAC_ADDR, \ VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ - VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT + VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL static unsigned int features[] = { VIRTNET_FEATURES, diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index 3f55a4215f11..29ced55514d4 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -56,7 +56,7 @@ #define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ - +#define VIRTIO_NET_F_NOTF_COAL 53 /* Guest can handle notifications coalescing */ #define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */ #define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */ #define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ @@ -355,4 +355,36 @@ struct virtio_net_hash_config { #define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0 +/* + * Control notifications coalescing. + * + * Request the device to change the notifications coalescing parameters. + * + * Available with the VIRTIO_NET_F_NOTF_COAL feature bit. + */ +#define VIRTIO_NET_CTRL_NOTF_COAL 6 +/* + * Set the tx-usecs/tx-max-packets patameters. + * tx-usecs - Maximum number of usecs to delay a TX notification. + * tx-max-packets - Maximum number of packets to send before a TX notification. + */ +struct virtio_net_ctrl_coal_tx { + __le32 tx_max_packets; + __le32 tx_usecs; +}; + +#define VIRTIO_NET_CTRL_NOTF_COAL_TX_SET 0 + +/* + * Set the rx-usecs/rx-max-packets patameters. + * rx-usecs - Maximum number of usecs to delay a RX notification. + * rx-max-frames - Maximum number of packets to receive before a RX notification. + */ +struct virtio_net_ctrl_coal_rx { + __le32 rx_max_packets; + __le32 rx_usecs; +}; + +#define VIRTIO_NET_CTRL_NOTF_COAL_RX_SET 1 + #endif /* _UAPI_LINUX_VIRTIO_NET_H */ -- cgit v1.3-15-g9ac1 From 79a463be9e0051997508d52cf411ed5e91d657f6 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:22 +0800 Subject: vduse: Support registering userspace memory for IOVA regions Introduce two ioctls: VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM to support registering and de-registering userspace memory for IOVA regions. Now it only supports registering userspace memory for bounce buffer region in virtio-vdpa case. Signed-off-by: Xie Yongji Acked-by: Jason Wang Message-Id: <20220803045523.23851-5-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/vduse_dev.c | 141 +++++++++++++++++++++++++++++++++++++ include/uapi/linux/vduse.h | 23 ++++++ 2 files changed, 164 insertions(+) (limited to 'include') diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 3bc27de58f46..eedff0a3885a 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -64,6 +66,13 @@ struct vduse_vdpa { struct vduse_dev *dev; }; +struct vduse_umem { + unsigned long iova; + unsigned long npages; + struct page **pages; + struct mm_struct *mm; +}; + struct vduse_dev { struct vduse_vdpa *vdev; struct device *dev; @@ -95,6 +104,8 @@ struct vduse_dev { u8 status; u32 vq_num; u32 vq_align; + struct vduse_umem *umem; + struct mutex mem_lock; }; struct vduse_dev_msg { @@ -917,6 +928,102 @@ unlock: return ret; } +static int vduse_dev_dereg_umem(struct vduse_dev *dev, + u64 iova, u64 size) +{ + int ret; + + mutex_lock(&dev->mem_lock); + ret = -ENOENT; + if (!dev->umem) + goto unlock; + + ret = -EINVAL; + if (dev->umem->iova != iova || size != dev->domain->bounce_size) + goto unlock; + + vduse_domain_remove_user_bounce_pages(dev->domain); + unpin_user_pages_dirty_lock(dev->umem->pages, + dev->umem->npages, true); + atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); + mmdrop(dev->umem->mm); + vfree(dev->umem->pages); + kfree(dev->umem); + dev->umem = NULL; + ret = 0; +unlock: + mutex_unlock(&dev->mem_lock); + return ret; +} + +static int vduse_dev_reg_umem(struct vduse_dev *dev, + u64 iova, u64 uaddr, u64 size) +{ + struct page **page_list = NULL; + struct vduse_umem *umem = NULL; + long pinned = 0; + unsigned long npages, lock_limit; + int ret; + + if (!dev->domain->bounce_map || + size != dev->domain->bounce_size || + iova != 0 || uaddr & ~PAGE_MASK) + return -EINVAL; + + mutex_lock(&dev->mem_lock); + ret = -EEXIST; + if (dev->umem) + goto unlock; + + ret = -ENOMEM; + npages = size >> PAGE_SHIFT; + page_list = __vmalloc(array_size(npages, sizeof(struct page *)), + GFP_KERNEL_ACCOUNT); + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (!page_list || !umem) + goto unlock; + + mmap_read_lock(current->mm); + + lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); + if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) + goto out; + + pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, + page_list, NULL); + if (pinned != npages) { + ret = pinned < 0 ? pinned : -ENOMEM; + goto out; + } + + ret = vduse_domain_add_user_bounce_pages(dev->domain, + page_list, pinned); + if (ret) + goto out; + + atomic64_add(npages, ¤t->mm->pinned_vm); + + umem->pages = page_list; + umem->npages = pinned; + umem->iova = iova; + umem->mm = current->mm; + mmgrab(current->mm); + + dev->umem = umem; +out: + if (ret && pinned > 0) + unpin_user_pages(page_list, pinned); + + mmap_read_unlock(current->mm); +unlock: + if (ret) { + vfree(page_list); + kfree(umem); + } + mutex_unlock(&dev->mem_lock); + return ret; +} + static long vduse_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -1089,6 +1196,38 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject); break; } + case VDUSE_IOTLB_REG_UMEM: { + struct vduse_iova_umem umem; + + ret = -EFAULT; + if (copy_from_user(&umem, argp, sizeof(umem))) + break; + + ret = -EINVAL; + if (!is_mem_zero((const char *)umem.reserved, + sizeof(umem.reserved))) + break; + + ret = vduse_dev_reg_umem(dev, umem.iova, + umem.uaddr, umem.size); + break; + } + case VDUSE_IOTLB_DEREG_UMEM: { + struct vduse_iova_umem umem; + + ret = -EFAULT; + if (copy_from_user(&umem, argp, sizeof(umem))) + break; + + ret = -EINVAL; + if (!is_mem_zero((const char *)umem.reserved, + sizeof(umem.reserved))) + break; + + ret = vduse_dev_dereg_umem(dev, umem.iova, + umem.size); + break; + } default: ret = -ENOIOCTLCMD; break; @@ -1101,6 +1240,7 @@ static int vduse_dev_release(struct inode *inode, struct file *file) { struct vduse_dev *dev = file->private_data; + vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); spin_lock(&dev->msg_lock); /* Make sure the inflight messages can processed after reconncection */ list_splice_init(&dev->recv_list, &dev->send_list); @@ -1163,6 +1303,7 @@ static struct vduse_dev *vduse_dev_create(void) return NULL; mutex_init(&dev->lock); + mutex_init(&dev->mem_lock); spin_lock_init(&dev->msg_lock); INIT_LIST_HEAD(&dev->send_list); INIT_LIST_HEAD(&dev->recv_list); diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h index 7cfe1c1280c0..9885e0571f09 100644 --- a/include/uapi/linux/vduse.h +++ b/include/uapi/linux/vduse.h @@ -210,6 +210,29 @@ struct vduse_vq_eventfd { */ #define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32) +/** + * struct vduse_iova_umem - userspace memory configuration for one IOVA region + * @uaddr: start address of userspace memory, it must be aligned to page size + * @iova: start of the IOVA region + * @size: size of the IOVA region + * @reserved: for future use, needs to be initialized to zero + * + * Structure used by VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM + * ioctls to register/de-register userspace memory for IOVA regions + */ +struct vduse_iova_umem { + __u64 uaddr; + __u64 iova; + __u64 size; + __u64 reserved[3]; +}; + +/* Register userspace memory for IOVA regions */ +#define VDUSE_IOTLB_REG_UMEM _IOW(VDUSE_BASE, 0x18, struct vduse_iova_umem) + +/* De-register the userspace memory. Caller should set iova and size field. */ +#define VDUSE_IOTLB_DEREG_UMEM _IOW(VDUSE_BASE, 0x19, struct vduse_iova_umem) + /* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */ /** -- cgit v1.3-15-g9ac1 From ad146355bfad627bd0717ece73997c6c93b1b82e Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:23 +0800 Subject: vduse: Support querying information of IOVA regions This introduces a new ioctl: VDUSE_IOTLB_GET_INFO to support querying some information of IOVA regions. Now it can be used to query whether the IOVA region supports userspace memory registration. Signed-off-by: Xie Yongji Message-Id: <20220803045523.23851-6-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/vdpa_user/vduse_dev.c | 39 ++++++++++++++++++++++++++++++++++++++ include/uapi/linux/vduse.h | 24 +++++++++++++++++++++++ 2 files changed, 63 insertions(+) (limited to 'include') diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index eedff0a3885a..41c0b29739f1 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -1228,6 +1228,45 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, umem.size); break; } + case VDUSE_IOTLB_GET_INFO: { + struct vduse_iova_info info; + struct vhost_iotlb_map *map; + struct vduse_iova_domain *domain = dev->domain; + + ret = -EFAULT; + if (copy_from_user(&info, argp, sizeof(info))) + break; + + ret = -EINVAL; + if (info.start > info.last) + break; + + if (!is_mem_zero((const char *)info.reserved, + sizeof(info.reserved))) + break; + + spin_lock(&domain->iotlb_lock); + map = vhost_iotlb_itree_first(domain->iotlb, + info.start, info.last); + if (map) { + info.start = map->start; + info.last = map->last; + info.capability = 0; + if (domain->bounce_map && map->start == 0 && + map->last == domain->bounce_size - 1) + info.capability |= VDUSE_IOVA_CAP_UMEM; + } + spin_unlock(&domain->iotlb_lock); + if (!map) + break; + + ret = -EFAULT; + if (copy_to_user(argp, &info, sizeof(info))) + break; + + ret = 0; + break; + } default: ret = -ENOIOCTLCMD; break; diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h index 9885e0571f09..11bd48c72c6c 100644 --- a/include/uapi/linux/vduse.h +++ b/include/uapi/linux/vduse.h @@ -233,6 +233,30 @@ struct vduse_iova_umem { /* De-register the userspace memory. Caller should set iova and size field. */ #define VDUSE_IOTLB_DEREG_UMEM _IOW(VDUSE_BASE, 0x19, struct vduse_iova_umem) +/** + * struct vduse_iova_info - information of one IOVA region + * @start: start of the IOVA region + * @last: last of the IOVA region + * @capability: capability of the IOVA regsion + * @reserved: for future use, needs to be initialized to zero + * + * Structure used by VDUSE_IOTLB_GET_INFO ioctl to get information of + * one IOVA region. + */ +struct vduse_iova_info { + __u64 start; + __u64 last; +#define VDUSE_IOVA_CAP_UMEM (1 << 0) + __u64 capability; + __u64 reserved[3]; +}; + +/* + * Find the first IOVA region that overlaps with the range [start, last] + * and return some information on it. Caller should set start and last fields. + */ +#define VDUSE_IOTLB_GET_INFO _IOWR(VDUSE_BASE, 0x1a, struct vduse_iova_info) + /* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */ /** -- cgit v1.3-15-g9ac1 From cae15c2ed8e6e058bd5e32de292ab7982640161e Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Thu, 14 Jul 2022 14:39:26 +0300 Subject: vdpa/mlx5: Implement susupend virtqueue callback Implement the suspend callback allowing to suspend the virtqueues so they stop processing descriptors. This is required to allow to query a consistent state of the virtqueue while live migration is taking place. Signed-off-by: Eli Cohen Message-Id: <20220714113927.85729-2-elic@nvidia.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/mlx5/net/mlx5_vnet.c | 83 ++++++++++++++++++++++++++++++++++++-- include/linux/mlx5/mlx5_ifc_vdpa.h | 8 ++++ 2 files changed, 88 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 99bbbf38c8b1..a476e06476f6 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -164,6 +164,7 @@ struct mlx5_vdpa_net { bool setup; u32 cur_num_vqs; u32 rqt_size; + bool nb_registered; struct notifier_block nb; struct vdpa_callback config_cb; struct mlx5_vdpa_wq_ent cvq_ent; @@ -895,6 +896,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque if (err) goto err_cmd; + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; kfree(in); mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); @@ -922,6 +924,7 @@ static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtq mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); return; } + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; umems_destroy(ndev, mvq); } @@ -1121,6 +1124,20 @@ err_cmd: return err; } +static bool is_valid_state_change(int oldstate, int newstate) +{ + switch (oldstate) { + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: + return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: + return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: + default: + return false; + } +} + static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) { int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); @@ -1130,6 +1147,12 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque void *in; int err; + if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) + return 0; + + if (!is_valid_state_change(mvq->fw_state, state)) + return -EINVAL; + in = kzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -1992,6 +2015,7 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct mlx5_vdpa_virtqueue *mvq; + int err; if (!mvdev->actual_features) return; @@ -2005,8 +2029,16 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready } mvq = &ndev->vqs[idx]; - if (!ready) + if (!ready) { suspend_vq(ndev, mvq); + } else { + err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); + if (err) { + mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); + ready = false; + } + } + mvq->ready = ready; } @@ -2733,6 +2765,37 @@ out_err: return err; } +static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) +{ + struct mlx5_control_vq *cvq; + + if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) + return; + + cvq = &mvdev->cvq; + cvq->ready = false; +} + +static int mlx5_vdpa_suspend(struct vdpa_device *vdev) +{ + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct mlx5_vdpa_virtqueue *mvq; + int i; + + down_write(&ndev->reslock); + mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + ndev->nb_registered = false; + flush_workqueue(ndev->mvdev.wq); + for (i = 0; i < ndev->cur_num_vqs; i++) { + mvq = &ndev->vqs[i]; + suspend_vq(ndev, mvq); + } + mlx5_vdpa_cvq_suspend(mvdev); + up_write(&ndev->reslock); + return 0; +} + static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_vq_address = mlx5_vdpa_set_vq_address, .set_vq_num = mlx5_vdpa_set_vq_num, @@ -2763,6 +2826,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { .get_generation = mlx5_vdpa_get_generation, .set_map = mlx5_vdpa_set_map, .free = mlx5_vdpa_free, + .suspend = mlx5_vdpa_suspend, }; static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) @@ -2828,6 +2892,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev) mvq->index = i; mvq->ndev = ndev; mvq->fwqp.fw = true; + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; } for (; i < ndev->mvdev.max_vqs; i++) { mvq = &ndev->vqs[i]; @@ -2902,13 +2967,21 @@ static int event_handler(struct notifier_block *nb, unsigned long event, void *p switch (eqe->sub_type) { case MLX5_PORT_CHANGE_SUBTYPE_DOWN: case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: + down_read(&ndev->reslock); + if (!ndev->nb_registered) { + up_read(&ndev->reslock); + return NOTIFY_DONE; + } wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); - if (!wqent) + if (!wqent) { + up_read(&ndev->reslock); return NOTIFY_DONE; + } wqent->mvdev = &ndev->mvdev; INIT_WORK(&wqent->work, update_carrier); queue_work(ndev->mvdev.wq, &wqent->work); + up_read(&ndev->reslock); ret = NOTIFY_OK; break; default: @@ -3062,6 +3135,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, ndev->nb.notifier_call = event_handler; mlx5_notifier_register(mdev, &ndev->nb); + ndev->nb_registered = true; mvdev->vdev.mdev = &mgtdev->mgtdev; err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); if (err) @@ -3093,7 +3167,10 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device * struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct workqueue_struct *wq; - mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + if (ndev->nb_registered) { + mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + ndev->nb_registered = false; + } wq = mvdev->wq; mvdev->wq = NULL; destroy_workqueue(wq); diff --git a/include/linux/mlx5/mlx5_ifc_vdpa.h b/include/linux/mlx5/mlx5_ifc_vdpa.h index 4414ed5b6ed2..9becdc3fa503 100644 --- a/include/linux/mlx5/mlx5_ifc_vdpa.h +++ b/include/linux/mlx5/mlx5_ifc_vdpa.h @@ -150,6 +150,14 @@ enum { MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR = 0x3, }; +/* This indicates that the object was not created or has already + * been desroyed. It is very safe to assume that this object will never + * have so many states + */ +enum { + MLX5_VIRTIO_NET_Q_OBJECT_NONE = 0xffffffff +}; + enum { MLX5_RQTC_LIST_Q_TYPE_RQ = 0x0, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q = 0x1, -- cgit v1.3-15-g9ac1 From 848ecea184e1253758423b37cbfc1ed732ccf5b4 Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:09 +0200 Subject: vdpa: Add suspend operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This operation is optional: It it's not implemented, backend feature bit will not be exposed. Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-2-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- include/linux/vdpa.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 7b4a13d3bd91..d282f464d2f1 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -218,6 +218,9 @@ struct vdpa_map_file { * @reset: Reset device * @vdev: vdpa device * Returns integer: success (0) or error (< 0) + * @suspend: Suspend or resume the device (optional) + * @vdev: vdpa device + * Returns integer: success (0) or error (< 0) * @get_config_size: Get the size of the configuration space includes * fields that are conditional on feature bits. * @vdev: vdpa device @@ -319,6 +322,7 @@ struct vdpa_config_ops { u8 (*get_status)(struct vdpa_device *vdev); void (*set_status)(struct vdpa_device *vdev, u8 status); int (*reset)(struct vdpa_device *vdev); + int (*suspend)(struct vdpa_device *vdev); size_t (*get_config_size)(struct vdpa_device *vdev); void (*get_config)(struct vdpa_device *vdev, unsigned int offset, void *buf, unsigned int len); -- cgit v1.3-15-g9ac1 From 0723f1df5c3ec8a1112d150dab98e149361ef488 Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:10 +0200 Subject: vhost-vdpa: introduce SUSPEND backend feature bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Userland knows if it can suspend the device or not by checking this feature bit. It's only offered if the vdpa driver backend implements the suspend() operation callback, and to offer it or userland to ack it if the backend does not offer that callback is an error. Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-3-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 16 +++++++++++++++- include/uapi/linux/vhost_types.h | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 2c997d77d266..092752fea8e1 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -347,6 +347,14 @@ static long vhost_vdpa_set_config(struct vhost_vdpa *v, return 0; } +static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->suspend; +} + static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; @@ -577,7 +585,11 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, if (cmd == VHOST_SET_BACKEND_FEATURES) { if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; - if (features & ~VHOST_VDPA_BACKEND_FEATURES) + if (features & ~(VHOST_VDPA_BACKEND_FEATURES | + BIT_ULL(VHOST_BACKEND_F_SUSPEND))) + return -EOPNOTSUPP; + if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) && + !vhost_vdpa_can_suspend(v)) return -EOPNOTSUPP; vhost_set_backend_features(&v->vdev, features); return 0; @@ -628,6 +640,8 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, break; case VHOST_GET_BACKEND_FEATURES: features = VHOST_VDPA_BACKEND_FEATURES; + if (vhost_vdpa_can_suspend(v)) + features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND); if (copy_to_user(featurep, &features, sizeof(features))) r = -EFAULT; break; diff --git a/include/uapi/linux/vhost_types.h b/include/uapi/linux/vhost_types.h index 634cee485abb..1bdd6e363f4c 100644 --- a/include/uapi/linux/vhost_types.h +++ b/include/uapi/linux/vhost_types.h @@ -161,5 +161,7 @@ struct vhost_vdpa_iova_range { * message */ #define VHOST_BACKEND_F_IOTLB_ASID 0x3 +/* Device can be suspended */ +#define VHOST_BACKEND_F_SUSPEND 0x4 #endif -- cgit v1.3-15-g9ac1 From f345a0143b4dd1cfc850009c6979a3801b86a06f Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:11 +0200 Subject: vhost-vdpa: uAPI to suspend the device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ioctl adds support for suspending the device from userspace. This is a must before getting virtqueue indexes (base) for live migration, since the device could modify them after userland gets them. There are individual ways to perform that action for some devices (VHOST_NET_SET_BACKEND, VHOST_VSOCK_SET_RUNNING, ...) but there was no way to perform it for any vhost device (and, in particular, vhost-vdpa). After a successful return of the ioctl call the device must not process more virtqueue descriptors. The device can answer to read or writes of config fields as if it were not suspended. In particular, writing to "queue_enable" with a value of 1 will not make the device start processing buffers of the virtqueue. Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-4-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 19 +++++++++++++++++++ include/uapi/linux/vhost.h | 9 +++++++++ 2 files changed, 28 insertions(+) (limited to 'include') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 092752fea8e1..166044642fd5 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -478,6 +478,22 @@ static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp) return 0; } +/* After a successful return of ioctl the device must not process more + * virtqueue descriptors. The device can answer to read or writes of config + * fields as if it were not suspended. In particular, writing to "queue_enable" + * with a value of 1 will not make the device start processing buffers. + */ +static long vhost_vdpa_suspend(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!ops->suspend) + return -EOPNOTSUPP; + + return ops->suspend(vdpa); +} + static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { @@ -654,6 +670,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_VDPA_GET_VQS_COUNT: r = vhost_vdpa_get_vqs_count(v, argp); break; + case VHOST_VDPA_SUSPEND: + r = vhost_vdpa_suspend(v); + break; default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index cab645d4a645..f9f115a7c75b 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -171,4 +171,13 @@ #define VHOST_VDPA_SET_GROUP_ASID _IOW(VHOST_VIRTIO, 0x7C, \ struct vhost_vring_state) +/* Suspend a device so it does not process virtqueue requests anymore + * + * After the return of ioctl the device must preserve all the necessary state + * (the virtqueue vring base plus the possible device specific states) that is + * required for restoring in the future. The device must not change its + * configuration after that point. + */ +#define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D) + #endif -- cgit v1.3-15-g9ac1 From 0b2f3212b551a87fe936701fa0813032861a3308 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 5 Aug 2022 10:59:57 +0200 Subject: netfilter: nfnetlink: re-enable conntrack expectation events To avoid allocation of the conntrack extension area when possible, the default behaviour was changed to only allocate the event extension if a userspace program is subscribed to a notification group. Problem is that while 'conntrack -E' does enable the event allocation behind the scenes, 'conntrack -E expect' does not: no expectation events are delivered unless user sets "net.netfilter.nf_conntrack_events" back to 1 (always on). Fix the autodetection to also consider EXP type group. We need to track the 6 event groups (3+3, new/update/destroy for events and for expectations each) independently, else we'd disable events again if an expectation group becomes empty while there is still an active event group. Fixes: 2794cdb0b97b ("netfilter: nfnetlink: allow to detect if ctnetlink listeners exist") Reported-by: Yi Chen Signed-off-by: Florian Westphal --- include/net/netns/conntrack.h | 2 +- net/netfilter/nfnetlink.c | 83 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 72 insertions(+), 13 deletions(-) (limited to 'include') diff --git a/include/net/netns/conntrack.h b/include/net/netns/conntrack.h index 0677cd3de034..c396a3862e80 100644 --- a/include/net/netns/conntrack.h +++ b/include/net/netns/conntrack.h @@ -95,7 +95,7 @@ struct nf_ip_net { struct netns_ct { #ifdef CONFIG_NF_CONNTRACK_EVENTS - bool ctnetlink_has_listener; + u8 ctnetlink_has_listener; bool ecache_dwork_pending; #endif u8 sysctl_log_invalid; /* Log invalid packets */ diff --git a/net/netfilter/nfnetlink.c b/net/netfilter/nfnetlink.c index c24b1240908f..9c44518cb70f 100644 --- a/net/netfilter/nfnetlink.c +++ b/net/netfilter/nfnetlink.c @@ -44,6 +44,10 @@ MODULE_DESCRIPTION("Netfilter messages via netlink socket"); static unsigned int nfnetlink_pernet_id __read_mostly; +#ifdef CONFIG_NF_CONNTRACK_EVENTS +static DEFINE_SPINLOCK(nfnl_grp_active_lock); +#endif + struct nfnl_net { struct sock *nfnl; }; @@ -654,6 +658,44 @@ static void nfnetlink_rcv(struct sk_buff *skb) netlink_rcv_skb(skb, nfnetlink_rcv_msg); } +static void nfnetlink_bind_event(struct net *net, unsigned int group) +{ +#ifdef CONFIG_NF_CONNTRACK_EVENTS + int type, group_bit; + u8 v; + + /* All NFNLGRP_CONNTRACK_* group bits fit into u8. + * The other groups are not relevant and can be ignored. + */ + if (group >= 8) + return; + + type = nfnl_group2type[group]; + + switch (type) { + case NFNL_SUBSYS_CTNETLINK: + break; + case NFNL_SUBSYS_CTNETLINK_EXP: + break; + default: + return; + } + + group_bit = (1 << group); + + spin_lock(&nfnl_grp_active_lock); + v = READ_ONCE(net->ct.ctnetlink_has_listener); + if ((v & group_bit) == 0) { + v |= group_bit; + + /* read concurrently without nfnl_grp_active_lock held. */ + WRITE_ONCE(net->ct.ctnetlink_has_listener, v); + } + + spin_unlock(&nfnl_grp_active_lock); +#endif +} + static int nfnetlink_bind(struct net *net, int group) { const struct nfnetlink_subsystem *ss; @@ -670,28 +712,45 @@ static int nfnetlink_bind(struct net *net, int group) if (!ss) request_module_nowait("nfnetlink-subsys-%d", type); -#ifdef CONFIG_NF_CONNTRACK_EVENTS - if (type == NFNL_SUBSYS_CTNETLINK) { - nfnl_lock(NFNL_SUBSYS_CTNETLINK); - WRITE_ONCE(net->ct.ctnetlink_has_listener, true); - nfnl_unlock(NFNL_SUBSYS_CTNETLINK); - } -#endif + nfnetlink_bind_event(net, group); return 0; } static void nfnetlink_unbind(struct net *net, int group) { #ifdef CONFIG_NF_CONNTRACK_EVENTS + int type, group_bit; + if (group <= NFNLGRP_NONE || group > NFNLGRP_MAX) return; - if (nfnl_group2type[group] == NFNL_SUBSYS_CTNETLINK) { - nfnl_lock(NFNL_SUBSYS_CTNETLINK); - if (!nfnetlink_has_listeners(net, group)) - WRITE_ONCE(net->ct.ctnetlink_has_listener, false); - nfnl_unlock(NFNL_SUBSYS_CTNETLINK); + type = nfnl_group2type[group]; + + switch (type) { + case NFNL_SUBSYS_CTNETLINK: + break; + case NFNL_SUBSYS_CTNETLINK_EXP: + break; + default: + return; + } + + /* ctnetlink_has_listener is u8 */ + if (group >= 8) + return; + + group_bit = (1 << group); + + spin_lock(&nfnl_grp_active_lock); + if (!nfnetlink_has_listeners(net, group)) { + u8 v = READ_ONCE(net->ct.ctnetlink_has_listener); + + v &= ~group_bit; + + /* read concurrently without nfnl_grp_active_lock held. */ + WRITE_ONCE(net->ct.ctnetlink_has_listener, v); } + spin_unlock(&nfnl_grp_active_lock); #endif } -- cgit v1.3-15-g9ac1 From b1c3497e604ddccea5af4071831ed0e4680fb35e Mon Sep 17 00:00:00 2001 From: Jane Malalane Date: Fri, 29 Jul 2022 08:04:16 +0100 Subject: x86/xen: Add support for HVMOP_set_evtchn_upcall_vector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement support for the HVMOP_set_evtchn_upcall_vector hypercall in order to set the per-vCPU event channel vector callback on Linux and use it in preference of HVM_PARAM_CALLBACK_IRQ. If the per-VCPU vector setup is successful on BSP, use this method for the APs. If not, fallback to the global vector-type callback. Also register callback_irq at per-vCPU event channel setup to trick toolstack to think the domain is enlightened. Suggested-by: "Roger Pau Monné" Signed-off-by: Jane Malalane Reviewed-by: Boris Ostrovsky Link: https://lore.kernel.org/r/20220729070416.23306-1-jane.malalane@citrix.com Signed-off-by: Juergen Gross --- arch/x86/include/asm/xen/cpuid.h | 2 ++ arch/x86/include/asm/xen/events.h | 3 ++- arch/x86/xen/enlighten.c | 2 +- arch/x86/xen/enlighten_hvm.c | 24 ++++++++++++----- arch/x86/xen/suspend_hvm.c | 10 ++++++- drivers/xen/events/events_base.c | 53 +++++++++++++++++++++++++++++++++----- include/xen/hvm.h | 2 ++ include/xen/interface/hvm/hvm_op.h | 19 ++++++++++++++ 8 files changed, 100 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/arch/x86/include/asm/xen/cpuid.h b/arch/x86/include/asm/xen/cpuid.h index 78e667a31d6c..6daa9b0c8d11 100644 --- a/arch/x86/include/asm/xen/cpuid.h +++ b/arch/x86/include/asm/xen/cpuid.h @@ -107,6 +107,8 @@ * ID field from 8 to 15 bits, allowing to target APIC IDs up 32768. */ #define XEN_HVM_CPUID_EXT_DEST_ID (1u << 5) +/* Per-vCPU event channel upcalls */ +#define XEN_HVM_CPUID_UPCALL_VECTOR (1u << 6) /* * Leaf 6 (0x40000x05) diff --git a/arch/x86/include/asm/xen/events.h b/arch/x86/include/asm/xen/events.h index 068d9b067c83..62bdceb594f1 100644 --- a/arch/x86/include/asm/xen/events.h +++ b/arch/x86/include/asm/xen/events.h @@ -23,7 +23,7 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) /* No need for a barrier -- XCHG is a barrier on x86. */ #define xchg_xen_ulong(ptr, val) xchg((ptr), (val)) -extern int xen_have_vector_callback; +extern bool xen_have_vector_callback; /* * Events delivered via platform PCI interrupts are always @@ -34,4 +34,5 @@ static inline bool xen_support_evtchn_rebind(void) return (!xen_hvm_domain() || xen_have_vector_callback); } +extern bool xen_percpu_upcall; #endif /* _ASM_X86_XEN_EVENTS_H */ diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 30c6e986a6cd..b8db2148c07d 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -51,7 +51,7 @@ EXPORT_SYMBOL_GPL(xen_start_info); struct shared_info xen_dummy_shared_info; -__read_mostly int xen_have_vector_callback; +__read_mostly bool xen_have_vector_callback = true; EXPORT_SYMBOL_GPL(xen_have_vector_callback); /* diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 28762f800596..1c1ac418484b 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include @@ -31,6 +33,9 @@ static unsigned long shared_info_pfn; +__ro_after_init bool xen_percpu_upcall; +EXPORT_SYMBOL_GPL(xen_percpu_upcall); + void xen_hvm_init_shared_info(void) { struct xen_add_to_physmap xatp; @@ -126,6 +131,9 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback) { struct pt_regs *old_regs = set_irq_regs(regs); + if (xen_percpu_upcall) + ack_APIC_irq(); + inc_irq_stat(irq_hv_callback_count); xen_hvm_evtchn_do_upcall(); @@ -169,6 +177,15 @@ static int xen_cpu_up_prepare_hvm(unsigned int cpu) if (!xen_have_vector_callback) return 0; + if (xen_percpu_upcall) { + rc = xen_set_upcall_vector(cpu); + if (rc) { + WARN(1, "HVMOP_set_evtchn_upcall_vector" + " for CPU %d failed: %d\n", cpu, rc); + return rc; + } + } + if (xen_feature(XENFEAT_hvm_safe_pvclock)) xen_setup_timer(cpu); @@ -189,8 +206,6 @@ static int xen_cpu_dead_hvm(unsigned int cpu) return 0; } -static bool no_vector_callback __initdata; - static void __init xen_hvm_guest_init(void) { if (xen_pv_domain()) @@ -213,9 +228,6 @@ static void __init xen_hvm_guest_init(void) xen_panic_handler_init(); - if (!no_vector_callback && xen_feature(XENFEAT_hvm_callback_vector)) - xen_have_vector_callback = 1; - xen_hvm_smp_init(); WARN_ON(xen_cpuhp_setup(xen_cpu_up_prepare_hvm, xen_cpu_dead_hvm)); xen_unplug_emulated_devices(); @@ -241,7 +253,7 @@ early_param("xen_nopv", xen_parse_nopv); static __init int xen_parse_no_vector_callback(char *arg) { - no_vector_callback = true; + xen_have_vector_callback = false; return 0; } early_param("xen_no_vector_callback", xen_parse_no_vector_callback); diff --git a/arch/x86/xen/suspend_hvm.c b/arch/x86/xen/suspend_hvm.c index 9d548b0c772f..0c4f7554b7cc 100644 --- a/arch/x86/xen/suspend_hvm.c +++ b/arch/x86/xen/suspend_hvm.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "xen-ops.h" @@ -14,6 +15,13 @@ void xen_hvm_post_suspend(int suspend_cancelled) xen_hvm_init_shared_info(); xen_vcpu_restore(); } - xen_setup_callback_vector(); + if (xen_percpu_upcall) { + unsigned int cpu; + + for_each_online_cpu(cpu) + BUG_ON(xen_set_upcall_vector(cpu)); + } else { + xen_setup_callback_vector(); + } xen_unplug_emulated_devices(); } diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index 46d9295d9a6e..206d4b466e44 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #endif #include @@ -2183,6 +2184,7 @@ static struct irq_chip xen_percpu_chip __read_mostly = { .irq_ack = ack_dynirq, }; +#ifdef CONFIG_X86 #ifdef CONFIG_XEN_PVHVM /* Vector callbacks are better than PCI interrupts to receive event * channel notifications because we can receive vector callbacks on any @@ -2195,11 +2197,48 @@ void xen_setup_callback_vector(void) callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); if (xen_set_callback_via(callback_via)) { pr_err("Request for Xen HVM callback vector failed\n"); - xen_have_vector_callback = 0; + xen_have_vector_callback = false; } } } +/* + * Setup per-vCPU vector-type callbacks. If this setup is unavailable, + * fallback to the global vector-type callback. + */ +static __init void xen_init_setup_upcall_vector(void) +{ + if (!xen_have_vector_callback) + return; + + if ((cpuid_eax(xen_cpuid_base() + 4) & XEN_HVM_CPUID_UPCALL_VECTOR) && + !xen_set_upcall_vector(0)) + xen_percpu_upcall = true; + else if (xen_feature(XENFEAT_hvm_callback_vector)) + xen_setup_callback_vector(); + else + xen_have_vector_callback = false; +} + +int xen_set_upcall_vector(unsigned int cpu) +{ + int rc; + xen_hvm_evtchn_upcall_vector_t op = { + .vector = HYPERVISOR_CALLBACK_VECTOR, + .vcpu = per_cpu(xen_vcpu_id, cpu), + }; + + rc = HYPERVISOR_hvm_op(HVMOP_set_evtchn_upcall_vector, &op); + if (rc) + return rc; + + /* Trick toolstack to think we are enlightened. */ + if (!cpu) + rc = xen_set_callback_via(1); + + return rc; +} + static __init void xen_alloc_callback_vector(void) { if (!xen_have_vector_callback) @@ -2210,8 +2249,11 @@ static __init void xen_alloc_callback_vector(void) } #else void xen_setup_callback_vector(void) {} +static inline void xen_init_setup_upcall_vector(void) {} +int xen_set_upcall_vector(unsigned int cpu) {} static inline void xen_alloc_callback_vector(void) {} -#endif +#endif /* CONFIG_XEN_PVHVM */ +#endif /* CONFIG_X86 */ bool xen_fifo_events = true; module_param_named(fifo_events, xen_fifo_events, bool, 0); @@ -2271,10 +2313,9 @@ void __init xen_init_IRQ(void) if (xen_initial_domain()) pci_xen_initial_domain(); } - if (xen_feature(XENFEAT_hvm_callback_vector)) { - xen_setup_callback_vector(); - xen_alloc_callback_vector(); - } + xen_init_setup_upcall_vector(); + xen_alloc_callback_vector(); + if (xen_hvm_domain()) { native_init_IRQ(); diff --git a/include/xen/hvm.h b/include/xen/hvm.h index b7fd7fc9ad41..8da7a6747058 100644 --- a/include/xen/hvm.h +++ b/include/xen/hvm.h @@ -60,4 +60,6 @@ static inline int hvm_get_parameter(int idx, uint64_t *value) void xen_setup_callback_vector(void); +int xen_set_upcall_vector(unsigned int cpu); + #endif /* XEN_HVM_H__ */ diff --git a/include/xen/interface/hvm/hvm_op.h b/include/xen/interface/hvm/hvm_op.h index f3097e79bb03..03134bf3cec1 100644 --- a/include/xen/interface/hvm/hvm_op.h +++ b/include/xen/interface/hvm/hvm_op.h @@ -46,4 +46,23 @@ struct xen_hvm_get_mem_type { }; DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_get_mem_type); +#if defined(__i386__) || defined(__x86_64__) + +/* + * HVMOP_set_evtchn_upcall_vector: Set a that should be used for event + * channel upcalls on the specified . If set, + * this vector will be used in preference to the + * domain global callback via (see + * HVM_PARAM_CALLBACK_IRQ). + */ +#define HVMOP_set_evtchn_upcall_vector 23 +struct xen_hvm_evtchn_upcall_vector { + uint32_t vcpu; + uint8_t vector; +}; +typedef struct xen_hvm_evtchn_upcall_vector xen_hvm_evtchn_upcall_vector_t; +DEFINE_GUEST_HANDLE_STRUCT(xen_hvm_evtchn_upcall_vector_t); + +#endif /* defined(__i386__) || defined(__x86_64__) */ + #endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */ -- cgit v1.3-15-g9ac1 From addebd9ac9ca0ef8b3764907bf8018e48caffc64 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 12 Aug 2022 15:56:33 -0700 Subject: fs: don't randomize struct kiocb fields This is a size sensitive structure and randomizing can introduce extra padding that breaks io_uring's fixed size expectations. There are few fields here as it is, half of which need a fixed order to optimally pack, so the randomization isn't providing much. Suggested-by: Linus Torvalds Signed-off-by: Keith Busch Link: https://lore.kernel.org/io-uring/b6f508ca-b1b2-5f40-7998-e4cff1cf7212@kernel.dk/ Signed-off-by: Jens Axboe --- include/linux/fs.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include') diff --git a/include/linux/fs.h b/include/linux/fs.h index 9f131e559d05..daf69a6504b6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -339,17 +339,12 @@ enum rw_hint { struct kiocb { struct file *ki_filp; - - /* The 'ki_filp' pointer is shared in a union for aio */ - randomized_struct_fields_start - loff_t ki_pos; void (*ki_complete)(struct kiocb *iocb, long ret); void *private; int ki_flags; u16 ki_ioprio; /* See linux/ioprio.h */ struct wait_page_queue *ki_waitq; /* for async buffered IO */ - randomized_struct_fields_end }; static inline bool is_sync_kiocb(struct kiocb *kiocb) -- cgit v1.3-15-g9ac1 From f2ccb5aed7bce1d8b3ed5b3385759a5509663028 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 11 Aug 2022 09:11:15 +0200 Subject: io_uring: make io_kiocb_to_cmd() typesafe We need to make sure (at build time) that struct io_cmd_data is not casted to a structure that's larger. Signed-off-by: Stefan Metzmacher Link: https://lore.kernel.org/r/c024cdf25ae19fc0319d4180e2298bade8ed17b8.1660201408.git.metze@samba.org Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 9 ++++++++- io_uring/advise.c | 8 ++++---- io_uring/cancel.c | 4 ++-- io_uring/epoll.c | 4 ++-- io_uring/fs.c | 28 ++++++++++++++-------------- io_uring/kbuf.c | 8 ++++---- io_uring/msg_ring.c | 8 ++++---- io_uring/net.c | 42 +++++++++++++++++++++--------------------- io_uring/notif.c | 2 -- io_uring/notif.h | 2 +- io_uring/openclose.c | 16 ++++++++-------- io_uring/poll.c | 16 ++++++++-------- io_uring/rsrc.c | 10 +++++----- io_uring/rw.c | 28 ++++++++++++++-------------- io_uring/splice.c | 8 ++++---- io_uring/statx.c | 6 +++--- io_uring/sync.c | 12 ++++++------ io_uring/timeout.c | 26 +++++++++++++------------- io_uring/uring_cmd.c | 8 ++++---- io_uring/xattr.c | 18 +++++++++--------- 20 files changed, 134 insertions(+), 129 deletions(-) (limited to 'include') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f7fab3758cb9..677a25d44d7f 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -491,7 +491,14 @@ struct io_cmd_data { __u8 data[56]; }; -#define io_kiocb_to_cmd(req) ((void *) &(req)->cmd) +static inline void io_kiocb_cmd_sz_check(size_t cmd_sz) +{ + BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data)); +} +#define io_kiocb_to_cmd(req, cmd_type) ( \ + io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \ + ((cmd_type *)&(req)->cmd) \ +) #define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) struct io_kiocb { diff --git a/io_uring/advise.c b/io_uring/advise.c index 581956934c0b..449c6f14649f 100644 --- a/io_uring/advise.c +++ b/io_uring/advise.c @@ -31,7 +31,7 @@ struct io_madvise { int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) - struct io_madvise *ma = io_kiocb_to_cmd(req); + struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise); if (sqe->buf_index || sqe->off || sqe->splice_fd_in) return -EINVAL; @@ -48,7 +48,7 @@ int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_madvise(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) - struct io_madvise *ma = io_kiocb_to_cmd(req); + struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -64,7 +64,7 @@ int io_madvise(struct io_kiocb *req, unsigned int issue_flags) int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_fadvise *fa = io_kiocb_to_cmd(req); + struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); if (sqe->buf_index || sqe->addr || sqe->splice_fd_in) return -EINVAL; @@ -77,7 +77,7 @@ int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) { - struct io_fadvise *fa = io_kiocb_to_cmd(req); + struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); int ret; if (issue_flags & IO_URING_F_NONBLOCK) { diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 8435a1eba59a..e4e1dc0325f0 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -107,7 +107,7 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_cancel *cancel = io_kiocb_to_cmd(req); + struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); if (unlikely(req->flags & REQ_F_BUFFER_SELECT)) return -EINVAL; @@ -164,7 +164,7 @@ static int __io_async_cancel(struct io_cancel_data *cd, int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { - struct io_cancel *cancel = io_kiocb_to_cmd(req); + struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); struct io_cancel_data cd = { .ctx = req->ctx, .data = cancel->addr, diff --git a/io_uring/epoll.c b/io_uring/epoll.c index a8b794471d6b..9aa74d2c80bc 100644 --- a/io_uring/epoll.c +++ b/io_uring/epoll.c @@ -23,7 +23,7 @@ struct io_epoll { int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_epoll *epoll = io_kiocb_to_cmd(req); + struct io_epoll *epoll = io_kiocb_to_cmd(req, struct io_epoll); pr_warn_once("%s: epoll_ctl support in io_uring is deprecated and will " "be removed in a future Linux kernel version.\n", @@ -49,7 +49,7 @@ int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) { - struct io_epoll *ie = io_kiocb_to_cmd(req); + struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll); int ret; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; diff --git a/io_uring/fs.c b/io_uring/fs.c index 0de4f549bb7d..7100c293c13a 100644 --- a/io_uring/fs.c +++ b/io_uring/fs.c @@ -49,7 +49,7 @@ struct io_link { int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); const char __user *oldf, *newf; if (sqe->buf_index || sqe->splice_fd_in) @@ -79,7 +79,7 @@ int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_renameat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -95,7 +95,7 @@ int io_renameat(struct io_kiocb *req, unsigned int issue_flags) void io_renameat_cleanup(struct io_kiocb *req) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); putname(ren->oldpath); putname(ren->newpath); @@ -103,7 +103,7 @@ void io_renameat_cleanup(struct io_kiocb *req) int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_unlink *un = io_kiocb_to_cmd(req); + struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink); const char __user *fname; if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in) @@ -128,7 +128,7 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_unlink *un = io_kiocb_to_cmd(req); + struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -146,14 +146,14 @@ int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) void io_unlinkat_cleanup(struct io_kiocb *req) { - struct io_unlink *ul = io_kiocb_to_cmd(req); + struct io_unlink *ul = io_kiocb_to_cmd(req, struct io_unlink); putname(ul->filename); } int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_mkdir *mkd = io_kiocb_to_cmd(req); + struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir); const char __user *fname; if (sqe->off || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -175,7 +175,7 @@ int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_mkdir *mkd = io_kiocb_to_cmd(req); + struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -190,14 +190,14 @@ int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) void io_mkdirat_cleanup(struct io_kiocb *req) { - struct io_mkdir *md = io_kiocb_to_cmd(req); + struct io_mkdir *md = io_kiocb_to_cmd(req, struct io_mkdir); putname(md->filename); } int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); const char __user *oldpath, *newpath; if (sqe->len || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -225,7 +225,7 @@ int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -240,7 +240,7 @@ int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_link *lnk = io_kiocb_to_cmd(req); + struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link); const char __user *oldf, *newf; if (sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -270,7 +270,7 @@ int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_linkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_link *lnk = io_kiocb_to_cmd(req); + struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -286,7 +286,7 @@ int io_linkat(struct io_kiocb *req, unsigned int issue_flags) void io_link_cleanup(struct io_kiocb *req) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); putname(sl->oldpath); putname(sl->newpath); diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index a73f40a4cfe6..25cd724ade18 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -272,7 +272,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx) int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); u64 tmp; if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off || @@ -291,7 +291,7 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; int ret = 0; @@ -319,7 +319,7 @@ int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { unsigned long size, tmp_check; - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); u64 tmp; if (sqe->rw_flags || sqe->splice_fd_in) @@ -421,7 +421,7 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; int ret = 0; diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 753d16734319..976c4ba68ee7 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -26,7 +26,7 @@ struct io_msg { static int io_msg_ring_data(struct io_kiocb *req) { struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); if (msg->src_fd || msg->dst_fd || msg->flags) return -EINVAL; @@ -76,7 +76,7 @@ static int io_double_lock_ctx(struct io_ring_ctx *ctx, static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); struct io_ring_ctx *ctx = req->ctx; unsigned long file_ptr; struct file *src_file; @@ -122,7 +122,7 @@ out_unlock: int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); if (unlikely(sqe->buf_index || sqe->personality)) return -EINVAL; @@ -141,7 +141,7 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) { - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); int ret; ret = -EBADFD; diff --git a/io_uring/net.c b/io_uring/net.c index e6fc9748fbd2..6d71748e2c5a 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -77,7 +77,7 @@ struct io_sendzc { int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); + struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)) @@ -89,7 +89,7 @@ int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) { - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); + struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); struct socket *sock; int ret; @@ -174,7 +174,7 @@ static int io_setup_async_msg(struct io_kiocb *req, static int io_sendmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); iomsg->msg.msg_name = &iomsg->addr; iomsg->free_iov = iomsg->fast_iov; @@ -201,7 +201,7 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); if (unlikely(sqe->file_index || sqe->addr2)) return -EINVAL; @@ -225,7 +225,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr iomsg, *kmsg; struct socket *sock; unsigned flags; @@ -284,7 +284,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) int io_send(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct iovec iov; struct socket *sock; @@ -358,7 +358,7 @@ static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct user_msghdr msg; int ret; @@ -405,7 +405,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct compat_msghdr msg; struct compat_iovec __user *uiov; int ret; @@ -483,7 +483,7 @@ int io_recvmsg_prep_async(struct io_kiocb *req) int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); if (unlikely(sqe->file_index || sqe->addr2)) return -EINVAL; @@ -518,7 +518,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static inline void io_recv_prep_retry(struct io_kiocb *req) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); sr->done_io = 0; sr->len = 0; /* get from the provided buffer */ @@ -647,7 +647,7 @@ static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr iomsg, *kmsg; struct socket *sock; unsigned int cflags; @@ -759,7 +759,7 @@ retry_multishot: int io_recv(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct socket *sock; struct iovec iov; @@ -850,7 +850,7 @@ out_free: int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sendzc *zc = io_kiocb_to_cmd(req); + struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); struct io_ring_ctx *ctx = req->ctx; if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)) @@ -946,7 +946,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) { struct sockaddr_storage address; struct io_ring_ctx *ctx = req->ctx; - struct io_sendzc *zc = io_kiocb_to_cmd(req); + struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); struct io_notif_slot *notif_slot; struct io_kiocb *notif; struct msghdr msg; @@ -1037,7 +1037,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_accept *accept = io_kiocb_to_cmd(req); + struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); unsigned flags; if (sqe->len || sqe->buf_index) @@ -1071,7 +1071,7 @@ int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_accept(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - struct io_accept *accept = io_kiocb_to_cmd(req); + struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; bool fixed = !!accept->file_slot; @@ -1129,7 +1129,7 @@ retry: int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_socket *sock = io_kiocb_to_cmd(req); + struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); if (sqe->addr || sqe->rw_flags || sqe->buf_index) return -EINVAL; @@ -1150,7 +1150,7 @@ int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_socket(struct io_kiocb *req, unsigned int issue_flags) { - struct io_socket *sock = io_kiocb_to_cmd(req); + struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); bool fixed = !!sock->file_slot; struct file *file; int ret, fd; @@ -1184,14 +1184,14 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags) int io_connect_prep_async(struct io_kiocb *req) { struct io_async_connect *io = req->async_data; - struct io_connect *conn = io_kiocb_to_cmd(req); + struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); } int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_connect *conn = io_kiocb_to_cmd(req); + struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) return -EINVAL; @@ -1203,7 +1203,7 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_connect(struct io_kiocb *req, unsigned int issue_flags) { - struct io_connect *connect = io_kiocb_to_cmd(req); + struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); struct io_async_connect __io, *io; unsigned file_flags; int ret; diff --git a/io_uring/notif.c b/io_uring/notif.c index 48d29dead62a..977736e82c1a 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -123,8 +123,6 @@ __cold int io_notif_register(struct io_ring_ctx *ctx, struct io_uring_notification_register reg; unsigned i; - BUILD_BUG_ON(sizeof(struct io_notif_data) > 64); - if (ctx->nr_notif_slots) return -EBUSY; if (size != sizeof(reg)) diff --git a/io_uring/notif.h b/io_uring/notif.h index 0819304d7e00..65f0b42f2555 100644 --- a/io_uring/notif.h +++ b/io_uring/notif.h @@ -46,7 +46,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) { - return io_kiocb_to_cmd(notif); + return io_kiocb_to_cmd(notif, struct io_notif_data); } static inline struct io_kiocb *io_get_notif(struct io_ring_ctx *ctx, diff --git a/io_uring/openclose.c b/io_uring/openclose.c index d1818ec9169b..67178e4bb282 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -33,7 +33,7 @@ struct io_close { static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); const char __user *fname; int ret; @@ -66,7 +66,7 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); u64 mode = READ_ONCE(sqe->len); u64 flags = READ_ONCE(sqe->open_flags); @@ -76,7 +76,7 @@ int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_how __user *how; size_t len; int ret; @@ -95,7 +95,7 @@ int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_openat2(struct io_kiocb *req, unsigned int issue_flags) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_flags op; struct file *file; bool resolve_nonblock, nonblock_set; @@ -167,7 +167,7 @@ int io_openat(struct io_kiocb *req, unsigned int issue_flags) void io_open_cleanup(struct io_kiocb *req) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); if (open->filename) putname(open->filename); @@ -187,14 +187,14 @@ int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags, static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) { - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1); } int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) return -EINVAL; @@ -212,7 +212,7 @@ int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_close(struct io_kiocb *req, unsigned int issue_flags) { struct files_struct *files = current->files; - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); struct fdtable *fdt; struct file *file; int ret = -EBADF; diff --git a/io_uring/poll.c b/io_uring/poll.c index dadd293749b0..d5bad0bea6e4 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -85,7 +85,7 @@ static struct io_poll *io_poll_get_double(struct io_kiocb *req) static struct io_poll *io_poll_get_single(struct io_kiocb *req) { if (req->opcode == IORING_OP_POLL_ADD) - return io_kiocb_to_cmd(req); + return io_kiocb_to_cmd(req, struct io_poll); return &req->apoll->poll; } @@ -274,7 +274,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) return; if (ret == IOU_POLL_DONE) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); req->cqe.res = mangle_poll(req->cqe.res & poll->events); } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) { req->cqe.res = ret; @@ -475,7 +475,7 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, struct poll_table_struct *p) { struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); - struct io_poll *poll = io_kiocb_to_cmd(pt->req); + struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); __io_queue_proc(poll, pt, head, (struct io_poll **) &pt->req->async_data); @@ -821,7 +821,7 @@ static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_poll_update *upd = io_kiocb_to_cmd(req); + struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update); u32 flags; if (sqe->buf_index || sqe->splice_fd_in) @@ -851,7 +851,7 @@ int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); u32 flags; if (sqe->buf_index || sqe->off || sqe->addr) @@ -868,7 +868,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); struct io_poll_table ipt; int ret; @@ -891,7 +891,7 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) { - struct io_poll_update *poll_update = io_kiocb_to_cmd(req); + struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); struct io_cancel_data cd = { .data = poll_update->old_user_data, }; struct io_ring_ctx *ctx = req->ctx; struct io_hash_bucket *bucket; @@ -930,7 +930,7 @@ found: if (poll_update->update_events || poll_update->update_user_data) { /* only mask one event flags, keep behavior flags */ if (poll_update->update_events) { - struct io_poll *poll = io_kiocb_to_cmd(preq); + struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll); poll->events &= ~0xffff; poll->events |= poll_update->events & 0xffff; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 59704b9ac537..71359a4d0bd4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -657,7 +657,7 @@ __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) return -EINVAL; @@ -676,7 +676,7 @@ int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_files_update_with_index_alloc(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); __s32 __user *fds = u64_to_user_ptr(up->arg); unsigned int done; struct file *file; @@ -714,7 +714,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update2 up2; int ret; @@ -743,7 +743,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) static int io_notif_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); struct io_ring_ctx *ctx = req->ctx; unsigned len = up->nr_args; unsigned idx_end, idx = up->offset; @@ -778,7 +778,7 @@ out: int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); switch (up->type) { case IORING_RSRC_UPDATE_FILES: diff --git a/io_uring/rw.c b/io_uring/rw.c index 2b784795103c..3d732b19b760 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -35,7 +35,7 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); unsigned ioprio; int ret; @@ -102,7 +102,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_pos != -1) return &rw->kiocb.ki_pos; @@ -186,7 +186,7 @@ static void kiocb_end_write(struct io_kiocb *req) static bool __io_complete_rw_common(struct io_kiocb *req, long res) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_flags & IOCB_WRITE) { kiocb_end_write(req); @@ -241,7 +241,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, unsigned int issue_flags) { struct io_async_rw *io = req->async_data; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); /* add previously done IO, if any */ if (req_has_async_data(req) && io->bytes_done > 0) { @@ -277,7 +277,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct compat_iovec __user *uiov; compat_ssize_t clen; void __user *buf; @@ -305,7 +305,7 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct iovec __user *uiov = u64_to_user_ptr(rw->addr); void __user *buf; ssize_t len; @@ -328,7 +328,7 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { iov[0].iov_base = u64_to_user_ptr(rw->addr); @@ -350,7 +350,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, struct io_rw_state *s, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct iov_iter *iter = &s->iter; u8 opcode = req->opcode; struct iovec *iovec; @@ -571,7 +571,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, { struct wait_page_queue *wpq; struct io_kiocb *req = wait->private; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct wait_page_key *key = arg; wpq = container_of(wait, struct wait_page_queue, wait); @@ -601,7 +601,7 @@ static bool io_rw_should_retry(struct io_kiocb *req) { struct io_async_rw *io = req->async_data; struct wait_page_queue *wait = &io->wpq; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct kiocb *kiocb = &rw->kiocb; /* never retry for NOWAIT, we just complete with -EAGAIN */ @@ -649,7 +649,7 @@ static bool need_complete_io(struct io_kiocb *req) static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct kiocb *kiocb = &rw->kiocb; struct io_ring_ctx *ctx = req->ctx; struct file *file = req->file; @@ -694,7 +694,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) int io_read(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &rw->kiocb; @@ -839,7 +839,7 @@ done: int io_write(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &rw->kiocb; @@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) wq_list_for_each(pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); int ret; /* diff --git a/io_uring/splice.c b/io_uring/splice.c index b013ba34bffa..53e4232d0866 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -26,7 +26,7 @@ struct io_splice { static int __io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; sp->len = READ_ONCE(sqe->len); @@ -46,7 +46,7 @@ int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_tee(struct io_kiocb *req, unsigned int issue_flags) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); struct file *out = sp->file_out; unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; struct file *in; @@ -78,7 +78,7 @@ done: int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); sp->off_in = READ_ONCE(sqe->splice_off_in); sp->off_out = READ_ONCE(sqe->off); @@ -87,7 +87,7 @@ int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_splice(struct io_kiocb *req, unsigned int issue_flags) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); struct file *out = sp->file_out; unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; loff_t *poff_in, *poff_out; diff --git a/io_uring/statx.c b/io_uring/statx.c index 6056cd7f4876..d8fc933d3f59 100644 --- a/io_uring/statx.c +++ b/io_uring/statx.c @@ -22,7 +22,7 @@ struct io_statx { int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); const char __user *path; if (sqe->buf_index || sqe->splice_fd_in) @@ -53,7 +53,7 @@ int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_statx(struct io_kiocb *req, unsigned int issue_flags) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -66,7 +66,7 @@ int io_statx(struct io_kiocb *req, unsigned int issue_flags) void io_statx_cleanup(struct io_kiocb *req) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); if (sx->filename) putname(sx->filename); diff --git a/io_uring/sync.c b/io_uring/sync.c index f2102afa79ca..64e87ea2b8fb 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -24,7 +24,7 @@ struct io_sync { int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in)) return -EINVAL; @@ -37,7 +37,7 @@ int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); int ret; /* sync_file_range always requires a blocking context */ @@ -51,7 +51,7 @@ int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in)) return -EINVAL; @@ -67,7 +67,7 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fsync(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); loff_t end = sync->off + sync->len; int ret; @@ -83,7 +83,7 @@ int io_fsync(struct io_kiocb *req, unsigned int issue_flags) int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) return -EINVAL; @@ -96,7 +96,7 @@ int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); int ret; /* fallocate always requiring blocking context */ diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 2f9e56935479..78ea2c64b70e 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -36,7 +36,7 @@ struct io_timeout_rem { static inline bool io_is_timeout_noseq(struct io_kiocb *req) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); return !timeout->off; } @@ -56,7 +56,7 @@ static bool io_kill_timeout(struct io_kiocb *req, int status) struct io_timeout_data *io = req->async_data; if (hrtimer_try_to_cancel(&io->timer) != -1) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); if (status) req_set_fail(req); @@ -188,7 +188,7 @@ struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, __must_hold(&req->ctx->timeout_lock) { struct io_timeout_data *io = link->async_data; - struct io_timeout *timeout = io_kiocb_to_cmd(link); + struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); io_remove_next_linked(req); timeout->head = NULL; @@ -205,7 +205,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) struct io_timeout_data *data = container_of(timer, struct io_timeout_data, timer); struct io_kiocb *req = data->req; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -252,7 +252,7 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, io = req->async_data; if (hrtimer_try_to_cancel(&io->timer) == -1) return ERR_PTR(-EALREADY); - timeout = io_kiocb_to_cmd(req); + timeout = io_kiocb_to_cmd(req, struct io_timeout); list_del_init(&timeout->list); return req; } @@ -275,7 +275,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) { unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_kiocb *prev = timeout->prev; int ret = -ENOENT; @@ -302,7 +302,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) struct io_timeout_data *data = container_of(timer, struct io_timeout_data, timer); struct io_kiocb *prev, *req = data->req; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -378,7 +378,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, { struct io_cancel_data cd = { .data = user_data, }; struct io_kiocb *req = io_timeout_extract(ctx, &cd); - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data; if (IS_ERR(req)) @@ -395,7 +395,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); + struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) return -EINVAL; @@ -435,7 +435,7 @@ static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) */ int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) { - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); + struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); struct io_ring_ctx *ctx = req->ctx; int ret; @@ -466,7 +466,7 @@ static int __io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, bool is_timeout_link) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data; unsigned flags; u32 off = READ_ONCE(sqe->off); @@ -532,7 +532,7 @@ int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_timeout(struct io_kiocb *req, unsigned int issue_flags) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; struct io_timeout_data *data = req->async_data; struct list_head *entry; @@ -583,7 +583,7 @@ add: void io_queue_linked_timeout(struct io_kiocb *req) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; spin_lock_irq(&ctx->timeout_lock); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index ee7036f2241f..478e86a9dfaf 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -11,7 +11,7 @@ static void io_uring_cmd_work(struct io_kiocb *req, bool *locked) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); ioucmd->task_work_cb(ioucmd); } @@ -55,7 +55,7 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_done); int io_uring_cmd_prep_async(struct io_kiocb *req) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); size_t cmd_size; cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128); @@ -66,7 +66,7 @@ int io_uring_cmd_prep_async(struct io_kiocb *req) int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); if (sqe->rw_flags || sqe->__pad1) return -EINVAL; @@ -77,7 +77,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); struct io_ring_ctx *ctx = req->ctx; struct file *file = req->file; int ret; diff --git a/io_uring/xattr.c b/io_uring/xattr.c index b179f9acd5ac..84180afd090b 100644 --- a/io_uring/xattr.c +++ b/io_uring/xattr.c @@ -24,7 +24,7 @@ struct io_xattr { void io_xattr_cleanup(struct io_kiocb *req) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); if (ix->filename) putname(ix->filename); @@ -44,7 +44,7 @@ static void io_xattr_finish(struct io_kiocb *req, int ret) static int __io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *name; int ret; @@ -85,7 +85,7 @@ int io_fgetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *path; int ret; @@ -106,7 +106,7 @@ int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -122,7 +122,7 @@ int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); unsigned int lookup_flags = LOOKUP_FOLLOW; struct path path; int ret; @@ -151,7 +151,7 @@ retry: static int __io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *name; int ret; @@ -181,7 +181,7 @@ static int __io_setxattr_prep(struct io_kiocb *req, int io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *path; int ret; @@ -208,7 +208,7 @@ int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int __io_setxattr(struct io_kiocb *req, unsigned int issue_flags, struct path *path) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; ret = mnt_want_write(path->mnt); @@ -234,7 +234,7 @@ int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags) int io_setxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); unsigned int lookup_flags = LOOKUP_FOLLOW; struct path path; int ret; -- cgit v1.3-15-g9ac1 From 9f162193d6e48eb4ff51c2ea3612f1daebca1b7e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 11 Aug 2022 22:34:25 -0700 Subject: radix-tree: replace gfp.h inclusion with gfp_types.h Radix tree header includes gfp.h for __GFP_BITS_SHIFT only. Now we have gfp_types.h for this. Fixes powerpc allmodconfig build: In file included from include/linux/nodemask.h:97, from include/linux/mmzone.h:17, from include/linux/gfp.h:7, from include/linux/radix-tree.h:12, from include/linux/idr.h:15, from include/linux/kernfs.h:12, from include/linux/sysfs.h:16, from include/linux/kobject.h:20, from include/linux/pci.h:35, from arch/powerpc/kernel/prom_init.c:24: include/linux/random.h: In function 'add_latent_entropy': >> include/linux/random.h:25:46: error: 'latent_entropy' undeclared (first use in this function); did you mean 'add_latent_entropy'? 25 | add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy)); | ^~~~~~~~~~~~~~ | add_latent_entropy include/linux/random.h:25:46: note: each undeclared identifier is reported only once for each function it appears in Reported-by: kernel test robot CC: Andy Shevchenko CC: Andrew Morton CC: Jason A. Donenfeld Signed-off-by: Yury Norov Signed-off-by: Linus Torvalds --- include/linux/radix-tree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index f7c1d21c2f39..eae67015ce51 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -9,7 +9,7 @@ #define _LINUX_RADIX_TREE_H #include -#include +#include #include #include #include -- cgit v1.3-15-g9ac1 From 0ff4eb3d5ebbf72a7fc355e6001a0a6740662bf9 Mon Sep 17 00:00:00 2001 From: Alexander Mikhalitsyn Date: Thu, 11 Aug 2022 18:20:12 +0300 Subject: neighbour: make proxy_queue.qlen limit per-device Right now we have a neigh_param PROXY_QLEN which specifies maximum length of neigh_table->proxy_queue. But in fact, this limitation doesn't work well because check condition looks like: tbl->proxy_queue.qlen > NEIGH_VAR(p, PROXY_QLEN) The problem is that p (struct neigh_parms) is a per-device thing, but tbl (struct neigh_table) is a system-wide global thing. It seems reasonable to make proxy_queue limit per-device based. v2: - nothing changed in this patch v3: - rebase to net tree Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Daniel Borkmann Cc: David Ahern Cc: Yajun Deng Cc: Roopa Prabhu Cc: Christian Brauner Cc: netdev@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: Alexey Kuznetsov Cc: Alexander Mikhalitsyn Cc: Konstantin Khorenko Cc: kernel@openvz.org Cc: devel@openvz.org Suggested-by: Denis V. Lunev Signed-off-by: Alexander Mikhalitsyn Reviewed-by: Denis V. Lunev Signed-off-by: David S. Miller --- include/net/neighbour.h | 1 + net/core/neighbour.c | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 9f0bab0589d9..3827a6b395fd 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -83,6 +83,7 @@ struct neigh_parms { struct rcu_head rcu_head; int reachable_time; + int qlen; int data[NEIGH_VAR_DATA_MAX]; DECLARE_BITMAP(data_state, NEIGH_VAR_DATA_MAX); }; diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 0e38a05d5b23..5b669eb80270 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -316,9 +316,18 @@ static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net) skb = skb_peek(list); while (skb != NULL) { struct sk_buff *skb_next = skb_peek_next(skb, list); - if (net == NULL || net_eq(dev_net(skb->dev), net)) { + struct net_device *dev = skb->dev; + if (net == NULL || net_eq(dev_net(dev), net)) { + struct in_device *in_dev; + + rcu_read_lock(); + in_dev = __in_dev_get_rcu(dev); + if (in_dev) + in_dev->arp_parms->qlen--; + rcu_read_unlock(); __skb_unlink(skb, list); - dev_put(skb->dev); + + dev_put(dev); kfree_skb(skb); } skb = skb_next; @@ -1606,8 +1615,15 @@ static void neigh_proxy_process(struct timer_list *t) if (tdif <= 0) { struct net_device *dev = skb->dev; + struct in_device *in_dev; + rcu_read_lock(); + in_dev = __in_dev_get_rcu(dev); + if (in_dev) + in_dev->arp_parms->qlen--; + rcu_read_unlock(); __skb_unlink(skb, &tbl->proxy_queue); + if (tbl->proxy_redo && netif_running(dev)) { rcu_read_lock(); tbl->proxy_redo(skb); @@ -1632,7 +1648,7 @@ void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p, unsigned long sched_next = jiffies + prandom_u32_max(NEIGH_VAR(p, PROXY_DELAY)); - if (tbl->proxy_queue.qlen > NEIGH_VAR(p, PROXY_QLEN)) { + if (p->qlen > NEIGH_VAR(p, PROXY_QLEN)) { kfree_skb(skb); return; } @@ -1648,6 +1664,7 @@ void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p, skb_dst_drop(skb); dev_hold(skb->dev); __skb_queue_tail(&tbl->proxy_queue, skb); + p->qlen++; mod_timer(&tbl->proxy_timer, sched_next); spin_unlock(&tbl->proxy_queue.lock); } @@ -1680,6 +1697,7 @@ struct neigh_parms *neigh_parms_alloc(struct net_device *dev, refcount_set(&p->refcnt, 1); p->reachable_time = neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME)); + p->qlen = 0; netdev_hold(dev, &p->dev_tracker, GFP_KERNEL); p->dev = dev; write_pnet(&p->net, net); @@ -1745,6 +1763,7 @@ void neigh_table_init(int index, struct neigh_table *tbl) refcount_set(&tbl->parms.refcnt, 1); tbl->parms.reachable_time = neigh_rand_reach_time(NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME)); + tbl->parms.qlen = 0; tbl->stats = alloc_percpu(struct neigh_statistics); if (!tbl->stats) -- cgit v1.3-15-g9ac1 From 484b9fa4886bd9377969aad5e9ea17efda4ecda6 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Tue, 16 Aug 2022 01:36:31 -0400 Subject: virtio: Revert "virtio: add helper virtio_find_vqs_ctx_size()" This reverts commit fe3dc04e31aa51f91dc7f741a5f76cc4817eb5b4: the API is now unused and in fact can't be implemented on top of a legacy device. Fixes: fe3dc04e31aa ("virtio: add helper virtio_find_vqs_ctx_size()") Cc: "Xuan Zhuo" Signed-off-by: Michael S. Tsirkin Message-Id: <20220816053602.173815-3-mst@redhat.com> --- include/linux/virtio_config.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include') diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 6adff09f7170..888f7e96f0c7 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -241,18 +241,6 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, ctx, desc); } -static inline -int virtio_find_vqs_ctx_size(struct virtio_device *vdev, u32 nvqs, - struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - u32 sizes[], - const bool *ctx, struct irq_affinity *desc) -{ - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, - ctx, desc); -} - /** * virtio_synchronize_cbs - synchronize with virtqueue callbacks * @vdev: the device -- cgit v1.3-15-g9ac1 From 9993a4f989c7ca5e227329b2878f65d05c9fc20f Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Tue, 16 Aug 2022 01:36:58 -0400 Subject: virtio: Revert "virtio: find_vqs() add arg sizes" This reverts commit a10fba0377145fccefea4dc4dd5915b7ed87e546: the proposed API isn't supported on all transports but no effort was made to address this. It might not be hard to fix if we want to: maybe just rename size to size_hint and make sure legacy transports ignore the hint. But it's not sure what the benefit is in any case, so let's drop it. Fixes: a10fba037714 ("virtio: find_vqs() add arg sizes") Signed-off-by: Michael S. Tsirkin Message-Id: <20220816053602.173815-8-mst@redhat.com> --- arch/um/drivers/virtio_uml.c | 2 +- drivers/platform/mellanox/mlxbf-tmfifo.c | 1 - drivers/remoteproc/remoteproc_virtio.c | 1 - drivers/s390/virtio/virtio_ccw.c | 1 - drivers/virtio/virtio_mmio.c | 1 - drivers/virtio/virtio_pci_common.c | 2 +- drivers/virtio/virtio_pci_common.h | 2 +- drivers/virtio/virtio_pci_modern.c | 7 ++----- drivers/virtio/virtio_vdpa.c | 1 - include/linux/virtio_config.h | 14 +++++--------- 10 files changed, 10 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index 79e38afd4b91..e719af8bdf56 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -1011,7 +1011,7 @@ error_kzalloc: static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], u32 sizes[], const bool *ctx, + const char * const names[], const bool *ctx, struct irq_affinity *desc) { struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 8be13d416f48..1ae3c56b66b0 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -928,7 +928,6 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], - u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 81c4f5776109..0f7706e23eb9 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -158,7 +158,6 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], - u32 sizes[], const bool * ctx, struct irq_affinity *desc) { diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 896896e32664..a10dbe632ef9 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -637,7 +637,6 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], - u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index dfcecfd7aba1..3ff746e3f24a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -474,7 +474,6 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], - u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 7ad734584823..ad258a9d3b9f 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -396,7 +396,7 @@ out_del_vqs: /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], u32 sizes[], const bool *ctx, + const char * const names[], const bool *ctx, struct irq_affinity *desc) { int err; diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index a5ff838b85a5..23112d84218f 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -110,7 +110,7 @@ void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], u32 sizes[], const bool *ctx, + const char * const names[], const bool *ctx, struct irq_affinity *desc); const char *vp_bus_name(struct virtio_device *vdev); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index be51ec849252..c3b9f2761849 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -347,15 +347,12 @@ err: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], - u32 sizes[], - const bool *ctx, + const char * const names[], const bool *ctx, struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; - int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx, - desc); + int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); if (rc) return rc; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 832d2c5b1b19..9670cc79371d 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -269,7 +269,6 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], - u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 888f7e96f0c7..36ec7be1f480 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -55,7 +55,6 @@ struct virtio_shm_region { * include a NULL entry for vqs that do not need a callback * names: array of virtqueue names (mainly for debugging) * include a NULL entry for vqs unused by driver - * sizes: array of virtqueue sizes * Returns 0 on success or error status * @del_vqs: free virtqueues found by find_vqs(). * @synchronize_cbs: synchronize with the virtqueue callbacks (optional) @@ -104,9 +103,7 @@ struct virtio_config_ops { void (*reset)(struct virtio_device *vdev); int (*find_vqs)(struct virtio_device *, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], - u32 sizes[], - const bool *ctx, + const char * const names[], const bool *ctx, struct irq_affinity *desc); void (*del_vqs)(struct virtio_device *); void (*synchronize_cbs)(struct virtio_device *); @@ -215,7 +212,7 @@ struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev, const char *names[] = { n }; struct virtqueue *vq; int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL, - NULL, NULL); + NULL); if (err < 0) return ERR_PTR(err); return vq; @@ -227,8 +224,7 @@ int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, const char * const names[], struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, - NULL, desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc); } static inline @@ -237,8 +233,8 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, const char * const names[], const bool *ctx, struct irq_affinity *desc) { - return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, - ctx, desc); + return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, + desc); } /** -- cgit v1.3-15-g9ac1 From 5c669c4a4c6aa0489848093c93b8029f5c5c75ec Mon Sep 17 00:00:00 2001 From: Ricardo Cañuelo Date: Wed, 10 Aug 2022 11:40:03 +0200 Subject: virtio: kerneldocs fixes and enhancements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix variable names in some kerneldocs, naming in others. Add kerneldocs for struct vring_desc and vring_interrupt. Signed-off-by: Ricardo Cañuelo Message-Id: <20220810094004.1250-2-ricardo.canuelo@collabora.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Cornelia Huck --- drivers/virtio/virtio_ring.c | 8 ++++++++ include/linux/virtio.h | 6 +++--- include/linux/virtio_config.h | 6 +++--- include/uapi/linux/virtio_ring.h | 16 +++++++++++----- 4 files changed, 25 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index d66c8e6d0ef3..4620e9d79dde 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2426,6 +2426,14 @@ static inline bool more_used(const struct vring_virtqueue *vq) return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); } +/** + * vring_interrupt - notify a virtqueue on an interrupt + * @irq: the IRQ number (ignored) + * @_vq: the struct virtqueue to notify + * + * Calls the callback function of @_vq to process the virtqueue + * notification. + */ irqreturn_t vring_interrupt(int irq, void *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index a3f73bb6733e..dcab9c7e8784 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -11,7 +11,7 @@ #include /** - * virtqueue - a queue to register buffers for sending or receiving. + * struct virtqueue - a queue to register buffers for sending or receiving. * @list: the chain of virtqueues for this device * @callback: the function to call when buffers are consumed (can be NULL). * @name: the name of this virtqueue (mainly for debugging) @@ -97,7 +97,7 @@ int virtqueue_resize(struct virtqueue *vq, u32 num, void (*recycle)(struct virtqueue *vq, void *buf)); /** - * virtio_device - representation of a device using virtio + * struct virtio_device - representation of a device using virtio * @index: unique position on the virtio bus * @failed: saved value for VIRTIO_CONFIG_S_FAILED bit (for restore) * @config_enabled: configuration change reporting enabled @@ -156,7 +156,7 @@ size_t virtio_max_dma_size(struct virtio_device *vdev); list_for_each_entry(vq, &vdev->vqs, list) /** - * virtio_driver - operations for a virtio I/O driver + * struct virtio_driver - operations for a virtio I/O driver * @driver: underlying device driver (populate name and owner). * @id_table: the ids serviced by this driver. * @feature_table: an array of feature numbers supported by this driver. diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 36ec7be1f480..4b517649cfe8 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -239,7 +239,7 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, /** * virtio_synchronize_cbs - synchronize with virtqueue callbacks - * @vdev: the device + * @dev: the virtio device */ static inline void virtio_synchronize_cbs(struct virtio_device *dev) @@ -258,7 +258,7 @@ void virtio_synchronize_cbs(struct virtio_device *dev) /** * virtio_device_ready - enable vq use in probe function - * @vdev: the device + * @dev: the virtio device * * Driver must call this to use vqs in the probe function. * @@ -306,7 +306,7 @@ const char *virtio_bus_name(struct virtio_device *vdev) /** * virtqueue_set_affinity - setting affinity for a virtqueue * @vq: the virtqueue - * @cpu: the cpu no. + * @cpu_mask: the cpu mask * * Pay attention the function are best-effort: the affinity hint may not be set * due to config support, irq type and sharing. diff --git a/include/uapi/linux/virtio_ring.h b/include/uapi/linux/virtio_ring.h index 476d3e5c0fe7..f8c20d3de8da 100644 --- a/include/uapi/linux/virtio_ring.h +++ b/include/uapi/linux/virtio_ring.h @@ -93,15 +93,21 @@ #define VRING_USED_ALIGN_SIZE 4 #define VRING_DESC_ALIGN_SIZE 16 -/* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ +/** + * struct vring_desc - Virtio ring descriptors, + * 16 bytes long. These can chain together via @next. + * + * @addr: buffer address (guest-physical) + * @len: buffer length + * @flags: descriptor flags + * @next: index of the next descriptor in the chain, + * if the VRING_DESC_F_NEXT flag is set. We chain unused + * descriptors via this, too. + */ struct vring_desc { - /* Address (guest-physical). */ __virtio64 addr; - /* Length. */ __virtio32 len; - /* The flags as indicated above. */ __virtio16 flags; - /* We chain unused descriptors via this, too */ __virtio16 next; }; -- cgit v1.3-15-g9ac1 From 415d832497098030241605c52ea83d4e2cfa7879 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 16 Aug 2022 16:03:11 +0900 Subject: locking/atomic: Make test_and_*_bit() ordered on failure These operations are documented as always ordered in include/asm-generic/bitops/instrumented-atomic.h, and producer-consumer type use cases where one side needs to ensure a flag is left pending after some shared data was updated rely on this ordering, even in the failure case. This is the case with the workqueue code, which currently suffers from a reproducible ordering violation on Apple M1 platforms (which are notoriously out-of-order) that ends up causing the TTY layer to fail to deliver data to userspace properly under the right conditions. This change fixes that bug. Change the documentation to restrict the "no order on failure" story to the _lock() variant (for which it makes sense), and remove the early-exit from the generic implementation, which is what causes the missing barrier semantics in that case. Without this, the remaining atomic op is fully ordered (including on ARM64 LSE, as of recent versions of the architecture spec). Suggested-by: Linus Torvalds Cc: stable@vger.kernel.org Fixes: e986a0d6cb36 ("locking/atomics, asm-generic/bitops/atomic.h: Rewrite using atomic_*() APIs") Fixes: 61e02392d3c7 ("locking/atomic/bitops: Document and clarify ordering semantics for failed test_and_{}_bit()") Signed-off-by: Hector Martin Acked-by: Will Deacon Reviewed-by: Arnd Bergmann Signed-off-by: Linus Torvalds --- Documentation/atomic_bitops.txt | 2 +- include/asm-generic/bitops/atomic.h | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'include') diff --git a/Documentation/atomic_bitops.txt b/Documentation/atomic_bitops.txt index 093cdaefdb37..d8b101c97031 100644 --- a/Documentation/atomic_bitops.txt +++ b/Documentation/atomic_bitops.txt @@ -59,7 +59,7 @@ Like with atomic_t, the rule of thumb is: - RMW operations that have a return value are fully ordered. - RMW operations that are conditional are unordered on FAILURE, - otherwise the above rules apply. In the case of test_and_{}_bit() operations, + otherwise the above rules apply. In the case of test_and_set_bit_lock(), if the bit in memory is unchanged by the operation then it is deemed to have failed. diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h index 3096f086b5a3..71ab4ba9c25d 100644 --- a/include/asm-generic/bitops/atomic.h +++ b/include/asm-generic/bitops/atomic.h @@ -39,9 +39,6 @@ arch_test_and_set_bit(unsigned int nr, volatile unsigned long *p) unsigned long mask = BIT_MASK(nr); p += BIT_WORD(nr); - if (READ_ONCE(*p) & mask) - return 1; - old = arch_atomic_long_fetch_or(mask, (atomic_long_t *)p); return !!(old & mask); } @@ -53,9 +50,6 @@ arch_test_and_clear_bit(unsigned int nr, volatile unsigned long *p) unsigned long mask = BIT_MASK(nr); p += BIT_WORD(nr); - if (!(READ_ONCE(*p) & mask)) - return 0; - old = arch_atomic_long_fetch_andnot(mask, (atomic_long_t *)p); return !!(old & mask); } -- cgit v1.3-15-g9ac1 From fc4aaf9fb3c99bcb326d52f9d320ed5680bd1cee Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 16 Aug 2022 10:34:40 +0100 Subject: net: Fix suspicious RCU usage in bpf_sk_reuseport_detach() bpf_sk_reuseport_detach() calls __rcu_dereference_sk_user_data_with_flags() to obtain the value of sk->sk_user_data, but that function is only usable if the RCU read lock is held, and neither that function nor any of its callers hold it. Fix this by adding a new helper, __locked_read_sk_user_data_with_flags() that checks to see if sk->sk_callback_lock() is held and use that here instead. Alternatively, making __rcu_dereference_sk_user_data_with_flags() use rcu_dereference_checked() might suffice. Without this, the following warning can be occasionally observed: ============================= WARNING: suspicious RCU usage 6.0.0-rc1-build2+ #563 Not tainted ----------------------------- include/net/sock.h:592 suspicious rcu_dereference_check() usage! other info that might help us debug this: rcu_scheduler_active = 2, debug_locks = 1 5 locks held by locktest/29873: #0: ffff88812734b550 (&sb->s_type->i_mutex_key#9){+.+.}-{3:3}, at: __sock_release+0x77/0x121 #1: ffff88812f5621b0 (sk_lock-AF_INET){+.+.}-{0:0}, at: tcp_close+0x1c/0x70 #2: ffff88810312f5c8 (&h->lhash2[i].lock){+.+.}-{2:2}, at: inet_unhash+0x76/0x1c0 #3: ffffffff83768bb8 (reuseport_lock){+...}-{2:2}, at: reuseport_detach_sock+0x18/0xdd #4: ffff88812f562438 (clock-AF_INET){++..}-{2:2}, at: bpf_sk_reuseport_detach+0x24/0xa4 stack backtrace: CPU: 1 PID: 29873 Comm: locktest Not tainted 6.0.0-rc1-build2+ #563 Hardware name: ASUS All Series/H97-PLUS, BIOS 2306 10/09/2014 Call Trace: dump_stack_lvl+0x4c/0x5f bpf_sk_reuseport_detach+0x6d/0xa4 reuseport_detach_sock+0x75/0xdd inet_unhash+0xa5/0x1c0 tcp_set_state+0x169/0x20f ? lockdep_sock_is_held+0x3a/0x3a ? __lock_release.isra.0+0x13e/0x220 ? reacquire_held_locks+0x1bb/0x1bb ? hlock_class+0x31/0x96 ? mark_lock+0x9e/0x1af __tcp_close+0x50/0x4b6 tcp_close+0x28/0x70 inet_release+0x8e/0xa7 __sock_release+0x95/0x121 sock_close+0x14/0x17 __fput+0x20f/0x36a task_work_run+0xa3/0xcc exit_to_user_mode_prepare+0x9c/0x14d syscall_exit_to_user_mode+0x18/0x44 entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: cf8c1e967224 ("net: refactor bpf_sk_reuseport_detach()") Signed-off-by: David Howells cc: Hawkins Jiawei Link: https://lore.kernel.org/r/166064248071.3502205.10036394558814861778.stgit@warthog.procyon.org.uk Signed-off-by: Jakub Kicinski --- include/net/sock.h | 25 +++++++++++++++++++++++++ kernel/bpf/reuseport_array.c | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/net/sock.h b/include/net/sock.h index 05a1bbdf5805..d08cfe190a78 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -577,6 +577,31 @@ static inline bool sk_user_data_is_nocopy(const struct sock *sk) #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) +/** + * __locked_read_sk_user_data_with_flags - return the pointer + * only if argument flags all has been set in sk_user_data. Otherwise + * return NULL + * + * @sk: socket + * @flags: flag bits + * + * The caller must be holding sk->sk_callback_lock. + */ +static inline void * +__locked_read_sk_user_data_with_flags(const struct sock *sk, + uintptr_t flags) +{ + uintptr_t sk_user_data = + (uintptr_t)rcu_dereference_check(__sk_user_data(sk), + lockdep_is_held(&sk->sk_callback_lock)); + + WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK); + + if ((sk_user_data & flags) == flags) + return (void *)(sk_user_data & SK_USER_DATA_PTRMASK); + return NULL; +} + /** * __rcu_dereference_sk_user_data_with_flags - return the pointer * only if argument flags all has been set in sk_user_data. Otherwise diff --git a/kernel/bpf/reuseport_array.c b/kernel/bpf/reuseport_array.c index 85fa9dbfa8bf..82c61612f382 100644 --- a/kernel/bpf/reuseport_array.c +++ b/kernel/bpf/reuseport_array.c @@ -24,7 +24,7 @@ void bpf_sk_reuseport_detach(struct sock *sk) struct sock __rcu **socks; write_lock_bh(&sk->sk_callback_lock); - socks = __rcu_dereference_sk_user_data_with_flags(sk, SK_USER_DATA_BPF); + socks = __locked_read_sk_user_data_with_flags(sk, SK_USER_DATA_BPF); if (socks) { WRITE_ONCE(sk->sk_user_data, NULL); /* -- cgit v1.3-15-g9ac1 From 5152de7b79ab0be150f5966481b0c8f996192531 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 16 Aug 2022 16:53:46 +0300 Subject: net: mscc: ocelot: fix incorrect ndo_get_stats64 packet counters Reading stats using the SYS_COUNT_* register definitions is only used by ocelot_get_stats64() from the ocelot switchdev driver, however, currently the bucket definitions are incorrect. Separately, on both RX and TX, we have the following problems: - a 256-1023 bucket which actually tracks the 256-511 packets - the 1024-1526 bucket actually tracks the 512-1023 packets - the 1527-max bucket actually tracks the 1024-1526 packets => nobody tracks the packets from the real 1527-max bucket Additionally, the RX_PAUSE, RX_CONTROL, RX_LONGS and RX_CLASSIFIED_DROPS all track the wrong thing. However this doesn't seem to have any consequence, since ocelot_get_stats64() doesn't use these. Even though this problem only manifests itself for the switchdev driver, we cannot split the fix for ocelot and for DSA, since it requires fixing the bucket definitions from enum ocelot_reg, which makes us necessarily adapt the structures from felix and seville as well. Fixes: 84705fc16552 ("net: dsa: felix: introduce support for Seville VSC9953 switch") Fixes: 56051948773e ("net: dsa: ocelot: add driver for Felix switch family") Fixes: a556c76adc05 ("net: mscc: Add initial Ocelot switch support") Signed-off-by: Vladimir Oltean Signed-off-by: Jakub Kicinski --- drivers/net/dsa/ocelot/felix_vsc9959.c | 20 ++++++++++++-------- drivers/net/dsa/ocelot/seville_vsc9953.c | 16 +++++++++------- drivers/net/ethernet/mscc/ocelot_net.c | 6 ++++-- drivers/net/ethernet/mscc/vsc7514_regs.c | 24 +++++++++++++----------- include/soc/mscc/ocelot.h | 6 ++++-- 5 files changed, 42 insertions(+), 30 deletions(-) (limited to 'include') diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 5859ef3b242c..e1ebe21cad00 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -281,19 +281,23 @@ static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_RX_64, 0x000024), REG(SYS_COUNT_RX_65_127, 0x000028), REG(SYS_COUNT_RX_128_255, 0x00002c), - REG(SYS_COUNT_RX_256_1023, 0x000030), - REG(SYS_COUNT_RX_1024_1526, 0x000034), - REG(SYS_COUNT_RX_1527_MAX, 0x000038), - REG(SYS_COUNT_RX_LONGS, 0x000044), + REG(SYS_COUNT_RX_256_511, 0x000030), + REG(SYS_COUNT_RX_512_1023, 0x000034), + REG(SYS_COUNT_RX_1024_1526, 0x000038), + REG(SYS_COUNT_RX_1527_MAX, 0x00003c), + REG(SYS_COUNT_RX_PAUSE, 0x000040), + REG(SYS_COUNT_RX_CONTROL, 0x000044), + REG(SYS_COUNT_RX_LONGS, 0x000048), REG(SYS_COUNT_TX_OCTETS, 0x000200), REG(SYS_COUNT_TX_COLLISION, 0x000210), REG(SYS_COUNT_TX_DROPS, 0x000214), REG(SYS_COUNT_TX_64, 0x00021c), REG(SYS_COUNT_TX_65_127, 0x000220), - REG(SYS_COUNT_TX_128_511, 0x000224), - REG(SYS_COUNT_TX_512_1023, 0x000228), - REG(SYS_COUNT_TX_1024_1526, 0x00022c), - REG(SYS_COUNT_TX_1527_MAX, 0x000230), + REG(SYS_COUNT_TX_128_255, 0x000224), + REG(SYS_COUNT_TX_256_511, 0x000228), + REG(SYS_COUNT_TX_512_1023, 0x00022c), + REG(SYS_COUNT_TX_1024_1526, 0x000230), + REG(SYS_COUNT_TX_1527_MAX, 0x000234), REG(SYS_COUNT_TX_AGING, 0x000278), REG(SYS_RESET_CFG, 0x000e00), REG(SYS_SR_ETYPE_CFG, 0x000e04), diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index ea0649211356..ebe9ddbbe2b7 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -277,19 +277,21 @@ static const u32 vsc9953_sys_regmap[] = { REG(SYS_COUNT_RX_64, 0x000024), REG(SYS_COUNT_RX_65_127, 0x000028), REG(SYS_COUNT_RX_128_255, 0x00002c), - REG(SYS_COUNT_RX_256_1023, 0x000030), - REG(SYS_COUNT_RX_1024_1526, 0x000034), - REG(SYS_COUNT_RX_1527_MAX, 0x000038), + REG(SYS_COUNT_RX_256_511, 0x000030), + REG(SYS_COUNT_RX_512_1023, 0x000034), + REG(SYS_COUNT_RX_1024_1526, 0x000038), + REG(SYS_COUNT_RX_1527_MAX, 0x00003c), REG(SYS_COUNT_RX_LONGS, 0x000048), REG(SYS_COUNT_TX_OCTETS, 0x000100), REG(SYS_COUNT_TX_COLLISION, 0x000110), REG(SYS_COUNT_TX_DROPS, 0x000114), REG(SYS_COUNT_TX_64, 0x00011c), REG(SYS_COUNT_TX_65_127, 0x000120), - REG(SYS_COUNT_TX_128_511, 0x000124), - REG(SYS_COUNT_TX_512_1023, 0x000128), - REG(SYS_COUNT_TX_1024_1526, 0x00012c), - REG(SYS_COUNT_TX_1527_MAX, 0x000130), + REG(SYS_COUNT_TX_128_255, 0x000124), + REG(SYS_COUNT_TX_256_511, 0x000128), + REG(SYS_COUNT_TX_512_1023, 0x00012c), + REG(SYS_COUNT_TX_1024_1526, 0x000130), + REG(SYS_COUNT_TX_1527_MAX, 0x000134), REG(SYS_COUNT_TX_AGING, 0x000178), REG(SYS_RESET_CFG, 0x000318), REG_RESERVED(SYS_SR_ETYPE_CFG), diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 5e6136e80282..9d8cea16245e 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -739,7 +739,8 @@ static void ocelot_get_stats64(struct net_device *dev, ocelot_read(ocelot, SYS_COUNT_RX_64) + ocelot_read(ocelot, SYS_COUNT_RX_65_127) + ocelot_read(ocelot, SYS_COUNT_RX_128_255) + - ocelot_read(ocelot, SYS_COUNT_RX_256_1023) + + ocelot_read(ocelot, SYS_COUNT_RX_256_511) + + ocelot_read(ocelot, SYS_COUNT_RX_512_1023) + ocelot_read(ocelot, SYS_COUNT_RX_1024_1526) + ocelot_read(ocelot, SYS_COUNT_RX_1527_MAX); stats->multicast = ocelot_read(ocelot, SYS_COUNT_RX_MULTICAST); @@ -749,7 +750,8 @@ static void ocelot_get_stats64(struct net_device *dev, stats->tx_bytes = ocelot_read(ocelot, SYS_COUNT_TX_OCTETS); stats->tx_packets = ocelot_read(ocelot, SYS_COUNT_TX_64) + ocelot_read(ocelot, SYS_COUNT_TX_65_127) + - ocelot_read(ocelot, SYS_COUNT_TX_128_511) + + ocelot_read(ocelot, SYS_COUNT_TX_128_255) + + ocelot_read(ocelot, SYS_COUNT_TX_256_511) + ocelot_read(ocelot, SYS_COUNT_TX_512_1023) + ocelot_read(ocelot, SYS_COUNT_TX_1024_1526) + ocelot_read(ocelot, SYS_COUNT_TX_1527_MAX); diff --git a/drivers/net/ethernet/mscc/vsc7514_regs.c b/drivers/net/ethernet/mscc/vsc7514_regs.c index c2af4eb8ca5d..38ab20b48cd4 100644 --- a/drivers/net/ethernet/mscc/vsc7514_regs.c +++ b/drivers/net/ethernet/mscc/vsc7514_regs.c @@ -180,13 +180,14 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_COUNT_RX_64, 0x000024), REG(SYS_COUNT_RX_65_127, 0x000028), REG(SYS_COUNT_RX_128_255, 0x00002c), - REG(SYS_COUNT_RX_256_1023, 0x000030), - REG(SYS_COUNT_RX_1024_1526, 0x000034), - REG(SYS_COUNT_RX_1527_MAX, 0x000038), - REG(SYS_COUNT_RX_PAUSE, 0x00003c), - REG(SYS_COUNT_RX_CONTROL, 0x000040), - REG(SYS_COUNT_RX_LONGS, 0x000044), - REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x000048), + REG(SYS_COUNT_RX_256_511, 0x000030), + REG(SYS_COUNT_RX_512_1023, 0x000034), + REG(SYS_COUNT_RX_1024_1526, 0x000038), + REG(SYS_COUNT_RX_1527_MAX, 0x00003c), + REG(SYS_COUNT_RX_PAUSE, 0x000040), + REG(SYS_COUNT_RX_CONTROL, 0x000044), + REG(SYS_COUNT_RX_LONGS, 0x000048), + REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x00004c), REG(SYS_COUNT_TX_OCTETS, 0x000100), REG(SYS_COUNT_TX_UNICAST, 0x000104), REG(SYS_COUNT_TX_MULTICAST, 0x000108), @@ -196,10 +197,11 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_COUNT_TX_PAUSE, 0x000118), REG(SYS_COUNT_TX_64, 0x00011c), REG(SYS_COUNT_TX_65_127, 0x000120), - REG(SYS_COUNT_TX_128_511, 0x000124), - REG(SYS_COUNT_TX_512_1023, 0x000128), - REG(SYS_COUNT_TX_1024_1526, 0x00012c), - REG(SYS_COUNT_TX_1527_MAX, 0x000130), + REG(SYS_COUNT_TX_128_255, 0x000124), + REG(SYS_COUNT_TX_256_511, 0x000128), + REG(SYS_COUNT_TX_512_1023, 0x00012c), + REG(SYS_COUNT_TX_1024_1526, 0x000130), + REG(SYS_COUNT_TX_1527_MAX, 0x000134), REG(SYS_COUNT_TX_AGING, 0x000170), REG(SYS_RESET_CFG, 0x000508), REG(SYS_CMID, 0x00050c), diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index ac151ecc7f19..e7e5b06deb2d 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -335,7 +335,8 @@ enum ocelot_reg { SYS_COUNT_RX_64, SYS_COUNT_RX_65_127, SYS_COUNT_RX_128_255, - SYS_COUNT_RX_256_1023, + SYS_COUNT_RX_256_511, + SYS_COUNT_RX_512_1023, SYS_COUNT_RX_1024_1526, SYS_COUNT_RX_1527_MAX, SYS_COUNT_RX_PAUSE, @@ -351,7 +352,8 @@ enum ocelot_reg { SYS_COUNT_TX_PAUSE, SYS_COUNT_TX_64, SYS_COUNT_TX_65_127, - SYS_COUNT_TX_128_511, + SYS_COUNT_TX_128_255, + SYS_COUNT_TX_256_511, SYS_COUNT_TX_512_1023, SYS_COUNT_TX_1024_1526, SYS_COUNT_TX_1527_MAX, -- cgit v1.3-15-g9ac1 From 22d842e3efe56402c33b5e6e303bb71ce9bf9334 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 16 Aug 2022 16:53:48 +0300 Subject: net: mscc: ocelot: turn stats_lock into a spinlock ocelot_get_stats64() currently runs unlocked and therefore may collide with ocelot_port_update_stats() which indirectly accesses the same counters. However, ocelot_get_stats64() runs in atomic context, and we cannot simply take the sleepable ocelot->stats_lock mutex. We need to convert it to an atomic spinlock first. Do that as a preparatory change. Signed-off-by: Vladimir Oltean Signed-off-by: Jakub Kicinski --- drivers/net/dsa/ocelot/felix_vsc9959.c | 4 ++-- drivers/net/ethernet/mscc/ocelot.c | 11 +++++------ include/soc/mscc/ocelot.h | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index e1ebe21cad00..46fd6cd0d8f3 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -2171,7 +2171,7 @@ static void vsc9959_psfp_sgi_table_del(struct ocelot *ocelot, static void vsc9959_psfp_counters_get(struct ocelot *ocelot, u32 index, struct felix_stream_filter_counters *counters) { - mutex_lock(&ocelot->stats_lock); + spin_lock(&ocelot->stats_lock); ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(index), SYS_STAT_CFG_STAT_VIEW_M, @@ -2188,7 +2188,7 @@ static void vsc9959_psfp_counters_get(struct ocelot *ocelot, u32 index, SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10), SYS_STAT_CFG); - mutex_unlock(&ocelot->stats_lock); + spin_unlock(&ocelot->stats_lock); } static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index d4649e4ee0e7..c67f162f8ab5 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -1906,13 +1906,13 @@ static void ocelot_check_stats_work(struct work_struct *work) stats_work); int i, err; - mutex_lock(&ocelot->stats_lock); + spin_lock(&ocelot->stats_lock); for (i = 0; i < ocelot->num_phys_ports; i++) { err = ocelot_port_update_stats(ocelot, i); if (err) break; } - mutex_unlock(&ocelot->stats_lock); + spin_unlock(&ocelot->stats_lock); if (err) dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err); @@ -1925,7 +1925,7 @@ void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data) { int i, err; - mutex_lock(&ocelot->stats_lock); + spin_lock(&ocelot->stats_lock); /* check and update now */ err = ocelot_port_update_stats(ocelot, port); @@ -1934,7 +1934,7 @@ void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data) for (i = 0; i < ocelot->num_stats; i++) *data++ = ocelot->stats[port * ocelot->num_stats + i]; - mutex_unlock(&ocelot->stats_lock); + spin_unlock(&ocelot->stats_lock); if (err) dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err); @@ -3363,7 +3363,7 @@ int ocelot_init(struct ocelot *ocelot) if (!ocelot->stats) return -ENOMEM; - mutex_init(&ocelot->stats_lock); + spin_lock_init(&ocelot->stats_lock); mutex_init(&ocelot->ptp_lock); mutex_init(&ocelot->mact_lock); mutex_init(&ocelot->fwd_domain_lock); @@ -3511,7 +3511,6 @@ void ocelot_deinit(struct ocelot *ocelot) cancel_delayed_work(&ocelot->stats_work); destroy_workqueue(ocelot->stats_queue); destroy_workqueue(ocelot->owq); - mutex_destroy(&ocelot->stats_lock); } EXPORT_SYMBOL(ocelot_deinit); diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index e7e5b06deb2d..72b9474391da 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -752,7 +752,7 @@ struct ocelot { struct ocelot_psfp_list psfp; /* Workqueue to check statistics for overflow with its lock */ - struct mutex stats_lock; + spinlock_t stats_lock; u64 *stats; struct delayed_work stats_work; struct workqueue_struct *stats_queue; -- cgit v1.3-15-g9ac1 From 9190460084ddd0e9235f55eab0fdd5456b5f2fd5 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 16 Aug 2022 16:53:50 +0300 Subject: net: mscc: ocelot: make struct ocelot_stat_layout array indexable The ocelot counters are 32-bit and require periodic reading, every 2 seconds, by ocelot_port_update_stats(), so that wraparounds are detected. Currently, the counters reported by ocelot_get_stats64() come from the 32-bit hardware counters directly, rather than from the 64-bit accumulated ocelot->stats, and this is a problem for their integrity. The strategy is to make ocelot_get_stats64() able to cherry-pick individual stats from ocelot->stats the way in which it currently reads them out from SYS_COUNT_* registers. But currently it can't, because ocelot->stats is an opaque u64 array that's used only to feed data into ethtool -S. To solve that problem, we need to make ocelot->stats indexable, and associate each element with an element of struct ocelot_stat_layout used by ethtool -S. This makes ocelot_stat_layout a fat (and possibly sparse) array, so we need to change the way in which we access it. We no longer need OCELOT_STAT_END as a sentinel, because we know the array's size (OCELOT_NUM_STATS). We just need to skip the array elements that were left unpopulated for the switch revision (ocelot, felix, seville). Signed-off-by: Vladimir Oltean Signed-off-by: Jakub Kicinski --- drivers/net/dsa/ocelot/felix_vsc9959.c | 468 +++++++++++++++++++++++------ drivers/net/dsa/ocelot/seville_vsc9953.c | 468 +++++++++++++++++++++++------ drivers/net/ethernet/mscc/ocelot.c | 40 ++- drivers/net/ethernet/mscc/ocelot_vsc7514.c | 468 +++++++++++++++++++++++------ include/soc/mscc/ocelot.h | 105 ++++++- 5 files changed, 1243 insertions(+), 306 deletions(-) (limited to 'include') diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 46fd6cd0d8f3..c9f270f24b1c 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -551,101 +551,379 @@ static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = { [SYS_PAUSE_CFG_PAUSE_ENA] = REG_FIELD_ID(SYS_PAUSE_CFG, 0, 1, 7, 4), }; -static const struct ocelot_stat_layout vsc9959_stats_layout[] = { - { .offset = 0x00, .name = "rx_octets", }, - { .offset = 0x01, .name = "rx_unicast", }, - { .offset = 0x02, .name = "rx_multicast", }, - { .offset = 0x03, .name = "rx_broadcast", }, - { .offset = 0x04, .name = "rx_shorts", }, - { .offset = 0x05, .name = "rx_fragments", }, - { .offset = 0x06, .name = "rx_jabbers", }, - { .offset = 0x07, .name = "rx_crc_align_errs", }, - { .offset = 0x08, .name = "rx_sym_errs", }, - { .offset = 0x09, .name = "rx_frames_below_65_octets", }, - { .offset = 0x0A, .name = "rx_frames_65_to_127_octets", }, - { .offset = 0x0B, .name = "rx_frames_128_to_255_octets", }, - { .offset = 0x0C, .name = "rx_frames_256_to_511_octets", }, - { .offset = 0x0D, .name = "rx_frames_512_to_1023_octets", }, - { .offset = 0x0E, .name = "rx_frames_1024_to_1526_octets", }, - { .offset = 0x0F, .name = "rx_frames_over_1526_octets", }, - { .offset = 0x10, .name = "rx_pause", }, - { .offset = 0x11, .name = "rx_control", }, - { .offset = 0x12, .name = "rx_longs", }, - { .offset = 0x13, .name = "rx_classified_drops", }, - { .offset = 0x14, .name = "rx_red_prio_0", }, - { .offset = 0x15, .name = "rx_red_prio_1", }, - { .offset = 0x16, .name = "rx_red_prio_2", }, - { .offset = 0x17, .name = "rx_red_prio_3", }, - { .offset = 0x18, .name = "rx_red_prio_4", }, - { .offset = 0x19, .name = "rx_red_prio_5", }, - { .offset = 0x1A, .name = "rx_red_prio_6", }, - { .offset = 0x1B, .name = "rx_red_prio_7", }, - { .offset = 0x1C, .name = "rx_yellow_prio_0", }, - { .offset = 0x1D, .name = "rx_yellow_prio_1", }, - { .offset = 0x1E, .name = "rx_yellow_prio_2", }, - { .offset = 0x1F, .name = "rx_yellow_prio_3", }, - { .offset = 0x20, .name = "rx_yellow_prio_4", }, - { .offset = 0x21, .name = "rx_yellow_prio_5", }, - { .offset = 0x22, .name = "rx_yellow_prio_6", }, - { .offset = 0x23, .name = "rx_yellow_prio_7", }, - { .offset = 0x24, .name = "rx_green_prio_0", }, - { .offset = 0x25, .name = "rx_green_prio_1", }, - { .offset = 0x26, .name = "rx_green_prio_2", }, - { .offset = 0x27, .name = "rx_green_prio_3", }, - { .offset = 0x28, .name = "rx_green_prio_4", }, - { .offset = 0x29, .name = "rx_green_prio_5", }, - { .offset = 0x2A, .name = "rx_green_prio_6", }, - { .offset = 0x2B, .name = "rx_green_prio_7", }, - { .offset = 0x80, .name = "tx_octets", }, - { .offset = 0x81, .name = "tx_unicast", }, - { .offset = 0x82, .name = "tx_multicast", }, - { .offset = 0x83, .name = "tx_broadcast", }, - { .offset = 0x84, .name = "tx_collision", }, - { .offset = 0x85, .name = "tx_drops", }, - { .offset = 0x86, .name = "tx_pause", }, - { .offset = 0x87, .name = "tx_frames_below_65_octets", }, - { .offset = 0x88, .name = "tx_frames_65_to_127_octets", }, - { .offset = 0x89, .name = "tx_frames_128_255_octets", }, - { .offset = 0x8A, .name = "tx_frames_256_511_octets", }, - { .offset = 0x8B, .name = "tx_frames_512_1023_octets", }, - { .offset = 0x8C, .name = "tx_frames_1024_1526_octets", }, - { .offset = 0x8D, .name = "tx_frames_over_1526_octets", }, - { .offset = 0x8E, .name = "tx_yellow_prio_0", }, - { .offset = 0x8F, .name = "tx_yellow_prio_1", }, - { .offset = 0x90, .name = "tx_yellow_prio_2", }, - { .offset = 0x91, .name = "tx_yellow_prio_3", }, - { .offset = 0x92, .name = "tx_yellow_prio_4", }, - { .offset = 0x93, .name = "tx_yellow_prio_5", }, - { .offset = 0x94, .name = "tx_yellow_prio_6", }, - { .offset = 0x95, .name = "tx_yellow_prio_7", }, - { .offset = 0x96, .name = "tx_green_prio_0", }, - { .offset = 0x97, .name = "tx_green_prio_1", }, - { .offset = 0x98, .name = "tx_green_prio_2", }, - { .offset = 0x99, .name = "tx_green_prio_3", }, - { .offset = 0x9A, .name = "tx_green_prio_4", }, - { .offset = 0x9B, .name = "tx_green_prio_5", }, - { .offset = 0x9C, .name = "tx_green_prio_6", }, - { .offset = 0x9D, .name = "tx_green_prio_7", }, - { .offset = 0x9E, .name = "tx_aged", }, - { .offset = 0x100, .name = "drop_local", }, - { .offset = 0x101, .name = "drop_tail", }, - { .offset = 0x102, .name = "drop_yellow_prio_0", }, - { .offset = 0x103, .name = "drop_yellow_prio_1", }, - { .offset = 0x104, .name = "drop_yellow_prio_2", }, - { .offset = 0x105, .name = "drop_yellow_prio_3", }, - { .offset = 0x106, .name = "drop_yellow_prio_4", }, - { .offset = 0x107, .name = "drop_yellow_prio_5", }, - { .offset = 0x108, .name = "drop_yellow_prio_6", }, - { .offset = 0x109, .name = "drop_yellow_prio_7", }, - { .offset = 0x10A, .name = "drop_green_prio_0", }, - { .offset = 0x10B, .name = "drop_green_prio_1", }, - { .offset = 0x10C, .name = "drop_green_prio_2", }, - { .offset = 0x10D, .name = "drop_green_prio_3", }, - { .offset = 0x10E, .name = "drop_green_prio_4", }, - { .offset = 0x10F, .name = "drop_green_prio_5", }, - { .offset = 0x110, .name = "drop_green_prio_6", }, - { .offset = 0x111, .name = "drop_green_prio_7", }, - OCELOT_STAT_END +static const struct ocelot_stat_layout vsc9959_stats_layout[OCELOT_NUM_STATS] = { + [OCELOT_STAT_RX_OCTETS] = { + .name = "rx_octets", + .offset = 0x00, + }, + [OCELOT_STAT_RX_UNICAST] = { + .name = "rx_unicast", + .offset = 0x01, + }, + [OCELOT_STAT_RX_MULTICAST] = { + .name = "rx_multicast", + .offset = 0x02, + }, + [OCELOT_STAT_RX_BROADCAST] = { + .name = "rx_broadcast", + .offset = 0x03, + }, + [OCELOT_STAT_RX_SHORTS] = { + .name = "rx_shorts", + .offset = 0x04, + }, + [OCELOT_STAT_RX_FRAGMENTS] = { + .name = "rx_fragments", + .offset = 0x05, + }, + [OCELOT_STAT_RX_JABBERS] = { + .name = "rx_jabbers", + .offset = 0x06, + }, + [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { + .name = "rx_crc_align_errs", + .offset = 0x07, + }, + [OCELOT_STAT_RX_SYM_ERRS] = { + .name = "rx_sym_errs", + .offset = 0x08, + }, + [OCELOT_STAT_RX_64] = { + .name = "rx_frames_below_65_octets", + .offset = 0x09, + }, + [OCELOT_STAT_RX_65_127] = { + .name = "rx_frames_65_to_127_octets", + .offset = 0x0A, + }, + [OCELOT_STAT_RX_128_255] = { + .name = "rx_frames_128_to_255_octets", + .offset = 0x0B, + }, + [OCELOT_STAT_RX_256_511] = { + .name = "rx_frames_256_to_511_octets", + .offset = 0x0C, + }, + [OCELOT_STAT_RX_512_1023] = { + .name = "rx_frames_512_to_1023_octets", + .offset = 0x0D, + }, + [OCELOT_STAT_RX_1024_1526] = { + .name = "rx_frames_1024_to_1526_octets", + .offset = 0x0E, + }, + [OCELOT_STAT_RX_1527_MAX] = { + .name = "rx_frames_over_1526_octets", + .offset = 0x0F, + }, + [OCELOT_STAT_RX_PAUSE] = { + .name = "rx_pause", + .offset = 0x10, + }, + [OCELOT_STAT_RX_CONTROL] = { + .name = "rx_control", + .offset = 0x11, + }, + [OCELOT_STAT_RX_LONGS] = { + .name = "rx_longs", + .offset = 0x12, + }, + [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { + .name = "rx_classified_drops", + .offset = 0x13, + }, + [OCELOT_STAT_RX_RED_PRIO_0] = { + .name = "rx_red_prio_0", + .offset = 0x14, + }, + [OCELOT_STAT_RX_RED_PRIO_1] = { + .name = "rx_red_prio_1", + .offset = 0x15, + }, + [OCELOT_STAT_RX_RED_PRIO_2] = { + .name = "rx_red_prio_2", + .offset = 0x16, + }, + [OCELOT_STAT_RX_RED_PRIO_3] = { + .name = "rx_red_prio_3", + .offset = 0x17, + }, + [OCELOT_STAT_RX_RED_PRIO_4] = { + .name = "rx_red_prio_4", + .offset = 0x18, + }, + [OCELOT_STAT_RX_RED_PRIO_5] = { + .name = "rx_red_prio_5", + .offset = 0x19, + }, + [OCELOT_STAT_RX_RED_PRIO_6] = { + .name = "rx_red_prio_6", + .offset = 0x1A, + }, + [OCELOT_STAT_RX_RED_PRIO_7] = { + .name = "rx_red_prio_7", + .offset = 0x1B, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_0] = { + .name = "rx_yellow_prio_0", + .offset = 0x1C, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_1] = { + .name = "rx_yellow_prio_1", + .offset = 0x1D, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_2] = { + .name = "rx_yellow_prio_2", + .offset = 0x1E, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_3] = { + .name = "rx_yellow_prio_3", + .offset = 0x1F, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_4] = { + .name = "rx_yellow_prio_4", + .offset = 0x20, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_5] = { + .name = "rx_yellow_prio_5", + .offset = 0x21, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_6] = { + .name = "rx_yellow_prio_6", + .offset = 0x22, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_7] = { + .name = "rx_yellow_prio_7", + .offset = 0x23, + }, + [OCELOT_STAT_RX_GREEN_PRIO_0] = { + .name = "rx_green_prio_0", + .offset = 0x24, + }, + [OCELOT_STAT_RX_GREEN_PRIO_1] = { + .name = "rx_green_prio_1", + .offset = 0x25, + }, + [OCELOT_STAT_RX_GREEN_PRIO_2] = { + .name = "rx_green_prio_2", + .offset = 0x26, + }, + [OCELOT_STAT_RX_GREEN_PRIO_3] = { + .name = "rx_green_prio_3", + .offset = 0x27, + }, + [OCELOT_STAT_RX_GREEN_PRIO_4] = { + .name = "rx_green_prio_4", + .offset = 0x28, + }, + [OCELOT_STAT_RX_GREEN_PRIO_5] = { + .name = "rx_green_prio_5", + .offset = 0x29, + }, + [OCELOT_STAT_RX_GREEN_PRIO_6] = { + .name = "rx_green_prio_6", + .offset = 0x2A, + }, + [OCELOT_STAT_RX_GREEN_PRIO_7] = { + .name = "rx_green_prio_7", + .offset = 0x2B, + }, + [OCELOT_STAT_TX_OCTETS] = { + .name = "tx_octets", + .offset = 0x80, + }, + [OCELOT_STAT_TX_UNICAST] = { + .name = "tx_unicast", + .offset = 0x81, + }, + [OCELOT_STAT_TX_MULTICAST] = { + .name = "tx_multicast", + .offset = 0x82, + }, + [OCELOT_STAT_TX_BROADCAST] = { + .name = "tx_broadcast", + .offset = 0x83, + }, + [OCELOT_STAT_TX_COLLISION] = { + .name = "tx_collision", + .offset = 0x84, + }, + [OCELOT_STAT_TX_DROPS] = { + .name = "tx_drops", + .offset = 0x85, + }, + [OCELOT_STAT_TX_PAUSE] = { + .name = "tx_pause", + .offset = 0x86, + }, + [OCELOT_STAT_TX_64] = { + .name = "tx_frames_below_65_octets", + .offset = 0x87, + }, + [OCELOT_STAT_TX_65_127] = { + .name = "tx_frames_65_to_127_octets", + .offset = 0x88, + }, + [OCELOT_STAT_TX_128_255] = { + .name = "tx_frames_128_255_octets", + .offset = 0x89, + }, + [OCELOT_STAT_TX_256_511] = { + .name = "tx_frames_256_511_octets", + .offset = 0x8A, + }, + [OCELOT_STAT_TX_512_1023] = { + .name = "tx_frames_512_1023_octets", + .offset = 0x8B, + }, + [OCELOT_STAT_TX_1024_1526] = { + .name = "tx_frames_1024_1526_octets", + .offset = 0x8C, + }, + [OCELOT_STAT_TX_1527_MAX] = { + .name = "tx_frames_over_1526_octets", + .offset = 0x8D, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_0] = { + .name = "tx_yellow_prio_0", + .offset = 0x8E, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_1] = { + .name = "tx_yellow_prio_1", + .offset = 0x8F, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_2] = { + .name = "tx_yellow_prio_2", + .offset = 0x90, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_3] = { + .name = "tx_yellow_prio_3", + .offset = 0x91, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_4] = { + .name = "tx_yellow_prio_4", + .offset = 0x92, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_5] = { + .name = "tx_yellow_prio_5", + .offset = 0x93, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_6] = { + .name = "tx_yellow_prio_6", + .offset = 0x94, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_7] = { + .name = "tx_yellow_prio_7", + .offset = 0x95, + }, + [OCELOT_STAT_TX_GREEN_PRIO_0] = { + .name = "tx_green_prio_0", + .offset = 0x96, + }, + [OCELOT_STAT_TX_GREEN_PRIO_1] = { + .name = "tx_green_prio_1", + .offset = 0x97, + }, + [OCELOT_STAT_TX_GREEN_PRIO_2] = { + .name = "tx_green_prio_2", + .offset = 0x98, + }, + [OCELOT_STAT_TX_GREEN_PRIO_3] = { + .name = "tx_green_prio_3", + .offset = 0x99, + }, + [OCELOT_STAT_TX_GREEN_PRIO_4] = { + .name = "tx_green_prio_4", + .offset = 0x9A, + }, + [OCELOT_STAT_TX_GREEN_PRIO_5] = { + .name = "tx_green_prio_5", + .offset = 0x9B, + }, + [OCELOT_STAT_TX_GREEN_PRIO_6] = { + .name = "tx_green_prio_6", + .offset = 0x9C, + }, + [OCELOT_STAT_TX_GREEN_PRIO_7] = { + .name = "tx_green_prio_7", + .offset = 0x9D, + }, + [OCELOT_STAT_TX_AGED] = { + .name = "tx_aged", + .offset = 0x9E, + }, + [OCELOT_STAT_DROP_LOCAL] = { + .name = "drop_local", + .offset = 0x100, + }, + [OCELOT_STAT_DROP_TAIL] = { + .name = "drop_tail", + .offset = 0x101, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { + .name = "drop_yellow_prio_0", + .offset = 0x102, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { + .name = "drop_yellow_prio_1", + .offset = 0x103, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { + .name = "drop_yellow_prio_2", + .offset = 0x104, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { + .name = "drop_yellow_prio_3", + .offset = 0x105, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { + .name = "drop_yellow_prio_4", + .offset = 0x106, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { + .name = "drop_yellow_prio_5", + .offset = 0x107, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { + .name = "drop_yellow_prio_6", + .offset = 0x108, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { + .name = "drop_yellow_prio_7", + .offset = 0x109, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_0] = { + .name = "drop_green_prio_0", + .offset = 0x10A, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_1] = { + .name = "drop_green_prio_1", + .offset = 0x10B, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_2] = { + .name = "drop_green_prio_2", + .offset = 0x10C, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_3] = { + .name = "drop_green_prio_3", + .offset = 0x10D, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_4] = { + .name = "drop_green_prio_4", + .offset = 0x10E, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_5] = { + .name = "drop_green_prio_5", + .offset = 0x10F, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_6] = { + .name = "drop_green_prio_6", + .offset = 0x110, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_7] = { + .name = "drop_green_prio_7", + .offset = 0x111, + }, }; static const struct vcap_field vsc9959_vcap_es0_keys[] = { diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index ebe9ddbbe2b7..fe5d4642d0bc 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -545,101 +545,379 @@ static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = { [SYS_PAUSE_CFG_PAUSE_ENA] = REG_FIELD_ID(SYS_PAUSE_CFG, 0, 1, 11, 4), }; -static const struct ocelot_stat_layout vsc9953_stats_layout[] = { - { .offset = 0x00, .name = "rx_octets", }, - { .offset = 0x01, .name = "rx_unicast", }, - { .offset = 0x02, .name = "rx_multicast", }, - { .offset = 0x03, .name = "rx_broadcast", }, - { .offset = 0x04, .name = "rx_shorts", }, - { .offset = 0x05, .name = "rx_fragments", }, - { .offset = 0x06, .name = "rx_jabbers", }, - { .offset = 0x07, .name = "rx_crc_align_errs", }, - { .offset = 0x08, .name = "rx_sym_errs", }, - { .offset = 0x09, .name = "rx_frames_below_65_octets", }, - { .offset = 0x0A, .name = "rx_frames_65_to_127_octets", }, - { .offset = 0x0B, .name = "rx_frames_128_to_255_octets", }, - { .offset = 0x0C, .name = "rx_frames_256_to_511_octets", }, - { .offset = 0x0D, .name = "rx_frames_512_to_1023_octets", }, - { .offset = 0x0E, .name = "rx_frames_1024_to_1526_octets", }, - { .offset = 0x0F, .name = "rx_frames_over_1526_octets", }, - { .offset = 0x10, .name = "rx_pause", }, - { .offset = 0x11, .name = "rx_control", }, - { .offset = 0x12, .name = "rx_longs", }, - { .offset = 0x13, .name = "rx_classified_drops", }, - { .offset = 0x14, .name = "rx_red_prio_0", }, - { .offset = 0x15, .name = "rx_red_prio_1", }, - { .offset = 0x16, .name = "rx_red_prio_2", }, - { .offset = 0x17, .name = "rx_red_prio_3", }, - { .offset = 0x18, .name = "rx_red_prio_4", }, - { .offset = 0x19, .name = "rx_red_prio_5", }, - { .offset = 0x1A, .name = "rx_red_prio_6", }, - { .offset = 0x1B, .name = "rx_red_prio_7", }, - { .offset = 0x1C, .name = "rx_yellow_prio_0", }, - { .offset = 0x1D, .name = "rx_yellow_prio_1", }, - { .offset = 0x1E, .name = "rx_yellow_prio_2", }, - { .offset = 0x1F, .name = "rx_yellow_prio_3", }, - { .offset = 0x20, .name = "rx_yellow_prio_4", }, - { .offset = 0x21, .name = "rx_yellow_prio_5", }, - { .offset = 0x22, .name = "rx_yellow_prio_6", }, - { .offset = 0x23, .name = "rx_yellow_prio_7", }, - { .offset = 0x24, .name = "rx_green_prio_0", }, - { .offset = 0x25, .name = "rx_green_prio_1", }, - { .offset = 0x26, .name = "rx_green_prio_2", }, - { .offset = 0x27, .name = "rx_green_prio_3", }, - { .offset = 0x28, .name = "rx_green_prio_4", }, - { .offset = 0x29, .name = "rx_green_prio_5", }, - { .offset = 0x2A, .name = "rx_green_prio_6", }, - { .offset = 0x2B, .name = "rx_green_prio_7", }, - { .offset = 0x40, .name = "tx_octets", }, - { .offset = 0x41, .name = "tx_unicast", }, - { .offset = 0x42, .name = "tx_multicast", }, - { .offset = 0x43, .name = "tx_broadcast", }, - { .offset = 0x44, .name = "tx_collision", }, - { .offset = 0x45, .name = "tx_drops", }, - { .offset = 0x46, .name = "tx_pause", }, - { .offset = 0x47, .name = "tx_frames_below_65_octets", }, - { .offset = 0x48, .name = "tx_frames_65_to_127_octets", }, - { .offset = 0x49, .name = "tx_frames_128_255_octets", }, - { .offset = 0x4A, .name = "tx_frames_256_511_octets", }, - { .offset = 0x4B, .name = "tx_frames_512_1023_octets", }, - { .offset = 0x4C, .name = "tx_frames_1024_1526_octets", }, - { .offset = 0x4D, .name = "tx_frames_over_1526_octets", }, - { .offset = 0x4E, .name = "tx_yellow_prio_0", }, - { .offset = 0x4F, .name = "tx_yellow_prio_1", }, - { .offset = 0x50, .name = "tx_yellow_prio_2", }, - { .offset = 0x51, .name = "tx_yellow_prio_3", }, - { .offset = 0x52, .name = "tx_yellow_prio_4", }, - { .offset = 0x53, .name = "tx_yellow_prio_5", }, - { .offset = 0x54, .name = "tx_yellow_prio_6", }, - { .offset = 0x55, .name = "tx_yellow_prio_7", }, - { .offset = 0x56, .name = "tx_green_prio_0", }, - { .offset = 0x57, .name = "tx_green_prio_1", }, - { .offset = 0x58, .name = "tx_green_prio_2", }, - { .offset = 0x59, .name = "tx_green_prio_3", }, - { .offset = 0x5A, .name = "tx_green_prio_4", }, - { .offset = 0x5B, .name = "tx_green_prio_5", }, - { .offset = 0x5C, .name = "tx_green_prio_6", }, - { .offset = 0x5D, .name = "tx_green_prio_7", }, - { .offset = 0x5E, .name = "tx_aged", }, - { .offset = 0x80, .name = "drop_local", }, - { .offset = 0x81, .name = "drop_tail", }, - { .offset = 0x82, .name = "drop_yellow_prio_0", }, - { .offset = 0x83, .name = "drop_yellow_prio_1", }, - { .offset = 0x84, .name = "drop_yellow_prio_2", }, - { .offset = 0x85, .name = "drop_yellow_prio_3", }, - { .offset = 0x86, .name = "drop_yellow_prio_4", }, - { .offset = 0x87, .name = "drop_yellow_prio_5", }, - { .offset = 0x88, .name = "drop_yellow_prio_6", }, - { .offset = 0x89, .name = "drop_yellow_prio_7", }, - { .offset = 0x8A, .name = "drop_green_prio_0", }, - { .offset = 0x8B, .name = "drop_green_prio_1", }, - { .offset = 0x8C, .name = "drop_green_prio_2", }, - { .offset = 0x8D, .name = "drop_green_prio_3", }, - { .offset = 0x8E, .name = "drop_green_prio_4", }, - { .offset = 0x8F, .name = "drop_green_prio_5", }, - { .offset = 0x90, .name = "drop_green_prio_6", }, - { .offset = 0x91, .name = "drop_green_prio_7", }, - OCELOT_STAT_END +static const struct ocelot_stat_layout vsc9953_stats_layout[OCELOT_NUM_STATS] = { + [OCELOT_STAT_RX_OCTETS] = { + .name = "rx_octets", + .offset = 0x00, + }, + [OCELOT_STAT_RX_UNICAST] = { + .name = "rx_unicast", + .offset = 0x01, + }, + [OCELOT_STAT_RX_MULTICAST] = { + .name = "rx_multicast", + .offset = 0x02, + }, + [OCELOT_STAT_RX_BROADCAST] = { + .name = "rx_broadcast", + .offset = 0x03, + }, + [OCELOT_STAT_RX_SHORTS] = { + .name = "rx_shorts", + .offset = 0x04, + }, + [OCELOT_STAT_RX_FRAGMENTS] = { + .name = "rx_fragments", + .offset = 0x05, + }, + [OCELOT_STAT_RX_JABBERS] = { + .name = "rx_jabbers", + .offset = 0x06, + }, + [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { + .name = "rx_crc_align_errs", + .offset = 0x07, + }, + [OCELOT_STAT_RX_SYM_ERRS] = { + .name = "rx_sym_errs", + .offset = 0x08, + }, + [OCELOT_STAT_RX_64] = { + .name = "rx_frames_below_65_octets", + .offset = 0x09, + }, + [OCELOT_STAT_RX_65_127] = { + .name = "rx_frames_65_to_127_octets", + .offset = 0x0A, + }, + [OCELOT_STAT_RX_128_255] = { + .name = "rx_frames_128_to_255_octets", + .offset = 0x0B, + }, + [OCELOT_STAT_RX_256_511] = { + .name = "rx_frames_256_to_511_octets", + .offset = 0x0C, + }, + [OCELOT_STAT_RX_512_1023] = { + .name = "rx_frames_512_to_1023_octets", + .offset = 0x0D, + }, + [OCELOT_STAT_RX_1024_1526] = { + .name = "rx_frames_1024_to_1526_octets", + .offset = 0x0E, + }, + [OCELOT_STAT_RX_1527_MAX] = { + .name = "rx_frames_over_1526_octets", + .offset = 0x0F, + }, + [OCELOT_STAT_RX_PAUSE] = { + .name = "rx_pause", + .offset = 0x10, + }, + [OCELOT_STAT_RX_CONTROL] = { + .name = "rx_control", + .offset = 0x11, + }, + [OCELOT_STAT_RX_LONGS] = { + .name = "rx_longs", + .offset = 0x12, + }, + [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { + .name = "rx_classified_drops", + .offset = 0x13, + }, + [OCELOT_STAT_RX_RED_PRIO_0] = { + .name = "rx_red_prio_0", + .offset = 0x14, + }, + [OCELOT_STAT_RX_RED_PRIO_1] = { + .name = "rx_red_prio_1", + .offset = 0x15, + }, + [OCELOT_STAT_RX_RED_PRIO_2] = { + .name = "rx_red_prio_2", + .offset = 0x16, + }, + [OCELOT_STAT_RX_RED_PRIO_3] = { + .name = "rx_red_prio_3", + .offset = 0x17, + }, + [OCELOT_STAT_RX_RED_PRIO_4] = { + .name = "rx_red_prio_4", + .offset = 0x18, + }, + [OCELOT_STAT_RX_RED_PRIO_5] = { + .name = "rx_red_prio_5", + .offset = 0x19, + }, + [OCELOT_STAT_RX_RED_PRIO_6] = { + .name = "rx_red_prio_6", + .offset = 0x1A, + }, + [OCELOT_STAT_RX_RED_PRIO_7] = { + .name = "rx_red_prio_7", + .offset = 0x1B, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_0] = { + .name = "rx_yellow_prio_0", + .offset = 0x1C, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_1] = { + .name = "rx_yellow_prio_1", + .offset = 0x1D, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_2] = { + .name = "rx_yellow_prio_2", + .offset = 0x1E, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_3] = { + .name = "rx_yellow_prio_3", + .offset = 0x1F, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_4] = { + .name = "rx_yellow_prio_4", + .offset = 0x20, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_5] = { + .name = "rx_yellow_prio_5", + .offset = 0x21, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_6] = { + .name = "rx_yellow_prio_6", + .offset = 0x22, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_7] = { + .name = "rx_yellow_prio_7", + .offset = 0x23, + }, + [OCELOT_STAT_RX_GREEN_PRIO_0] = { + .name = "rx_green_prio_0", + .offset = 0x24, + }, + [OCELOT_STAT_RX_GREEN_PRIO_1] = { + .name = "rx_green_prio_1", + .offset = 0x25, + }, + [OCELOT_STAT_RX_GREEN_PRIO_2] = { + .name = "rx_green_prio_2", + .offset = 0x26, + }, + [OCELOT_STAT_RX_GREEN_PRIO_3] = { + .name = "rx_green_prio_3", + .offset = 0x27, + }, + [OCELOT_STAT_RX_GREEN_PRIO_4] = { + .name = "rx_green_prio_4", + .offset = 0x28, + }, + [OCELOT_STAT_RX_GREEN_PRIO_5] = { + .name = "rx_green_prio_5", + .offset = 0x29, + }, + [OCELOT_STAT_RX_GREEN_PRIO_6] = { + .name = "rx_green_prio_6", + .offset = 0x2A, + }, + [OCELOT_STAT_RX_GREEN_PRIO_7] = { + .name = "rx_green_prio_7", + .offset = 0x2B, + }, + [OCELOT_STAT_TX_OCTETS] = { + .name = "tx_octets", + .offset = 0x40, + }, + [OCELOT_STAT_TX_UNICAST] = { + .name = "tx_unicast", + .offset = 0x41, + }, + [OCELOT_STAT_TX_MULTICAST] = { + .name = "tx_multicast", + .offset = 0x42, + }, + [OCELOT_STAT_TX_BROADCAST] = { + .name = "tx_broadcast", + .offset = 0x43, + }, + [OCELOT_STAT_TX_COLLISION] = { + .name = "tx_collision", + .offset = 0x44, + }, + [OCELOT_STAT_TX_DROPS] = { + .name = "tx_drops", + .offset = 0x45, + }, + [OCELOT_STAT_TX_PAUSE] = { + .name = "tx_pause", + .offset = 0x46, + }, + [OCELOT_STAT_TX_64] = { + .name = "tx_frames_below_65_octets", + .offset = 0x47, + }, + [OCELOT_STAT_TX_65_127] = { + .name = "tx_frames_65_to_127_octets", + .offset = 0x48, + }, + [OCELOT_STAT_TX_128_255] = { + .name = "tx_frames_128_255_octets", + .offset = 0x49, + }, + [OCELOT_STAT_TX_256_511] = { + .name = "tx_frames_256_511_octets", + .offset = 0x4A, + }, + [OCELOT_STAT_TX_512_1023] = { + .name = "tx_frames_512_1023_octets", + .offset = 0x4B, + }, + [OCELOT_STAT_TX_1024_1526] = { + .name = "tx_frames_1024_1526_octets", + .offset = 0x4C, + }, + [OCELOT_STAT_TX_1527_MAX] = { + .name = "tx_frames_over_1526_octets", + .offset = 0x4D, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_0] = { + .name = "tx_yellow_prio_0", + .offset = 0x4E, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_1] = { + .name = "tx_yellow_prio_1", + .offset = 0x4F, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_2] = { + .name = "tx_yellow_prio_2", + .offset = 0x50, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_3] = { + .name = "tx_yellow_prio_3", + .offset = 0x51, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_4] = { + .name = "tx_yellow_prio_4", + .offset = 0x52, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_5] = { + .name = "tx_yellow_prio_5", + .offset = 0x53, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_6] = { + .name = "tx_yellow_prio_6", + .offset = 0x54, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_7] = { + .name = "tx_yellow_prio_7", + .offset = 0x55, + }, + [OCELOT_STAT_TX_GREEN_PRIO_0] = { + .name = "tx_green_prio_0", + .offset = 0x56, + }, + [OCELOT_STAT_TX_GREEN_PRIO_1] = { + .name = "tx_green_prio_1", + .offset = 0x57, + }, + [OCELOT_STAT_TX_GREEN_PRIO_2] = { + .name = "tx_green_prio_2", + .offset = 0x58, + }, + [OCELOT_STAT_TX_GREEN_PRIO_3] = { + .name = "tx_green_prio_3", + .offset = 0x59, + }, + [OCELOT_STAT_TX_GREEN_PRIO_4] = { + .name = "tx_green_prio_4", + .offset = 0x5A, + }, + [OCELOT_STAT_TX_GREEN_PRIO_5] = { + .name = "tx_green_prio_5", + .offset = 0x5B, + }, + [OCELOT_STAT_TX_GREEN_PRIO_6] = { + .name = "tx_green_prio_6", + .offset = 0x5C, + }, + [OCELOT_STAT_TX_GREEN_PRIO_7] = { + .name = "tx_green_prio_7", + .offset = 0x5D, + }, + [OCELOT_STAT_TX_AGED] = { + .name = "tx_aged", + .offset = 0x5E, + }, + [OCELOT_STAT_DROP_LOCAL] = { + .name = "drop_local", + .offset = 0x80, + }, + [OCELOT_STAT_DROP_TAIL] = { + .name = "drop_tail", + .offset = 0x81, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { + .name = "drop_yellow_prio_0", + .offset = 0x82, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { + .name = "drop_yellow_prio_1", + .offset = 0x83, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { + .name = "drop_yellow_prio_2", + .offset = 0x84, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { + .name = "drop_yellow_prio_3", + .offset = 0x85, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { + .name = "drop_yellow_prio_4", + .offset = 0x86, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { + .name = "drop_yellow_prio_5", + .offset = 0x87, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { + .name = "drop_yellow_prio_6", + .offset = 0x88, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { + .name = "drop_yellow_prio_7", + .offset = 0x89, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_0] = { + .name = "drop_green_prio_0", + .offset = 0x8A, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_1] = { + .name = "drop_green_prio_1", + .offset = 0x8B, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_2] = { + .name = "drop_green_prio_2", + .offset = 0x8C, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_3] = { + .name = "drop_green_prio_3", + .offset = 0x8D, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_4] = { + .name = "drop_green_prio_4", + .offset = 0x8E, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_5] = { + .name = "drop_green_prio_5", + .offset = 0x8F, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_6] = { + .name = "drop_green_prio_6", + .offset = 0x90, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_7] = { + .name = "drop_green_prio_7", + .offset = 0x91, + }, }; static const struct vcap_field vsc9953_vcap_es0_keys[] = { diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index c67f162f8ab5..68991b021c56 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -1860,16 +1860,20 @@ void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data) if (sset != ETH_SS_STATS) return; - for (i = 0; i < ocelot->num_stats; i++) + for (i = 0; i < OCELOT_NUM_STATS; i++) { + if (ocelot->stats_layout[i].name[0] == '\0') + continue; + memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name, ETH_GSTRING_LEN); + } } EXPORT_SYMBOL(ocelot_get_strings); /* Caller must hold &ocelot->stats_lock */ static int ocelot_port_update_stats(struct ocelot *ocelot, int port) { - unsigned int idx = port * ocelot->num_stats; + unsigned int idx = port * OCELOT_NUM_STATS; struct ocelot_stats_region *region; int err, j; @@ -1930,9 +1934,15 @@ void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data) /* check and update now */ err = ocelot_port_update_stats(ocelot, port); - /* Copy all counters */ - for (i = 0; i < ocelot->num_stats; i++) - *data++ = ocelot->stats[port * ocelot->num_stats + i]; + /* Copy all supported counters */ + for (i = 0; i < OCELOT_NUM_STATS; i++) { + int index = port * OCELOT_NUM_STATS + i; + + if (ocelot->stats_layout[i].name[0] == '\0') + continue; + + *data++ = ocelot->stats[index]; + } spin_unlock(&ocelot->stats_lock); @@ -1943,10 +1953,16 @@ EXPORT_SYMBOL(ocelot_get_ethtool_stats); int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset) { + int i, num_stats = 0; + if (sset != ETH_SS_STATS) return -EOPNOTSUPP; - return ocelot->num_stats; + for (i = 0; i < OCELOT_NUM_STATS; i++) + if (ocelot->stats_layout[i].name[0] != '\0') + num_stats++; + + return num_stats; } EXPORT_SYMBOL(ocelot_get_sset_count); @@ -1958,7 +1974,10 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot) INIT_LIST_HEAD(&ocelot->stats_regions); - for (i = 0; i < ocelot->num_stats; i++) { + for (i = 0; i < OCELOT_NUM_STATS; i++) { + if (ocelot->stats_layout[i].name[0] == '\0') + continue; + if (region && ocelot->stats_layout[i].offset == last + 1) { region->count++; } else { @@ -3340,7 +3359,6 @@ static void ocelot_detect_features(struct ocelot *ocelot) int ocelot_init(struct ocelot *ocelot) { - const struct ocelot_stat_layout *stat; char queue_name[32]; int i, ret; u32 port; @@ -3353,12 +3371,8 @@ int ocelot_init(struct ocelot *ocelot) } } - ocelot->num_stats = 0; - for_each_stat(ocelot, stat) - ocelot->num_stats++; - ocelot->stats = devm_kcalloc(ocelot->dev, - ocelot->num_phys_ports * ocelot->num_stats, + ocelot->num_phys_ports * OCELOT_NUM_STATS, sizeof(u64), GFP_KERNEL); if (!ocelot->stats) return -ENOMEM; diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index 961f803aca19..9ff910560043 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -96,101 +96,379 @@ static const struct reg_field ocelot_regfields[REGFIELD_MAX] = { [SYS_PAUSE_CFG_PAUSE_ENA] = REG_FIELD_ID(SYS_PAUSE_CFG, 0, 1, 12, 4), }; -static const struct ocelot_stat_layout ocelot_stats_layout[] = { - { .name = "rx_octets", .offset = 0x00, }, - { .name = "rx_unicast", .offset = 0x01, }, - { .name = "rx_multicast", .offset = 0x02, }, - { .name = "rx_broadcast", .offset = 0x03, }, - { .name = "rx_shorts", .offset = 0x04, }, - { .name = "rx_fragments", .offset = 0x05, }, - { .name = "rx_jabbers", .offset = 0x06, }, - { .name = "rx_crc_align_errs", .offset = 0x07, }, - { .name = "rx_sym_errs", .offset = 0x08, }, - { .name = "rx_frames_below_65_octets", .offset = 0x09, }, - { .name = "rx_frames_65_to_127_octets", .offset = 0x0A, }, - { .name = "rx_frames_128_to_255_octets", .offset = 0x0B, }, - { .name = "rx_frames_256_to_511_octets", .offset = 0x0C, }, - { .name = "rx_frames_512_to_1023_octets", .offset = 0x0D, }, - { .name = "rx_frames_1024_to_1526_octets", .offset = 0x0E, }, - { .name = "rx_frames_over_1526_octets", .offset = 0x0F, }, - { .name = "rx_pause", .offset = 0x10, }, - { .name = "rx_control", .offset = 0x11, }, - { .name = "rx_longs", .offset = 0x12, }, - { .name = "rx_classified_drops", .offset = 0x13, }, - { .name = "rx_red_prio_0", .offset = 0x14, }, - { .name = "rx_red_prio_1", .offset = 0x15, }, - { .name = "rx_red_prio_2", .offset = 0x16, }, - { .name = "rx_red_prio_3", .offset = 0x17, }, - { .name = "rx_red_prio_4", .offset = 0x18, }, - { .name = "rx_red_prio_5", .offset = 0x19, }, - { .name = "rx_red_prio_6", .offset = 0x1A, }, - { .name = "rx_red_prio_7", .offset = 0x1B, }, - { .name = "rx_yellow_prio_0", .offset = 0x1C, }, - { .name = "rx_yellow_prio_1", .offset = 0x1D, }, - { .name = "rx_yellow_prio_2", .offset = 0x1E, }, - { .name = "rx_yellow_prio_3", .offset = 0x1F, }, - { .name = "rx_yellow_prio_4", .offset = 0x20, }, - { .name = "rx_yellow_prio_5", .offset = 0x21, }, - { .name = "rx_yellow_prio_6", .offset = 0x22, }, - { .name = "rx_yellow_prio_7", .offset = 0x23, }, - { .name = "rx_green_prio_0", .offset = 0x24, }, - { .name = "rx_green_prio_1", .offset = 0x25, }, - { .name = "rx_green_prio_2", .offset = 0x26, }, - { .name = "rx_green_prio_3", .offset = 0x27, }, - { .name = "rx_green_prio_4", .offset = 0x28, }, - { .name = "rx_green_prio_5", .offset = 0x29, }, - { .name = "rx_green_prio_6", .offset = 0x2A, }, - { .name = "rx_green_prio_7", .offset = 0x2B, }, - { .name = "tx_octets", .offset = 0x40, }, - { .name = "tx_unicast", .offset = 0x41, }, - { .name = "tx_multicast", .offset = 0x42, }, - { .name = "tx_broadcast", .offset = 0x43, }, - { .name = "tx_collision", .offset = 0x44, }, - { .name = "tx_drops", .offset = 0x45, }, - { .name = "tx_pause", .offset = 0x46, }, - { .name = "tx_frames_below_65_octets", .offset = 0x47, }, - { .name = "tx_frames_65_to_127_octets", .offset = 0x48, }, - { .name = "tx_frames_128_255_octets", .offset = 0x49, }, - { .name = "tx_frames_256_511_octets", .offset = 0x4A, }, - { .name = "tx_frames_512_1023_octets", .offset = 0x4B, }, - { .name = "tx_frames_1024_1526_octets", .offset = 0x4C, }, - { .name = "tx_frames_over_1526_octets", .offset = 0x4D, }, - { .name = "tx_yellow_prio_0", .offset = 0x4E, }, - { .name = "tx_yellow_prio_1", .offset = 0x4F, }, - { .name = "tx_yellow_prio_2", .offset = 0x50, }, - { .name = "tx_yellow_prio_3", .offset = 0x51, }, - { .name = "tx_yellow_prio_4", .offset = 0x52, }, - { .name = "tx_yellow_prio_5", .offset = 0x53, }, - { .name = "tx_yellow_prio_6", .offset = 0x54, }, - { .name = "tx_yellow_prio_7", .offset = 0x55, }, - { .name = "tx_green_prio_0", .offset = 0x56, }, - { .name = "tx_green_prio_1", .offset = 0x57, }, - { .name = "tx_green_prio_2", .offset = 0x58, }, - { .name = "tx_green_prio_3", .offset = 0x59, }, - { .name = "tx_green_prio_4", .offset = 0x5A, }, - { .name = "tx_green_prio_5", .offset = 0x5B, }, - { .name = "tx_green_prio_6", .offset = 0x5C, }, - { .name = "tx_green_prio_7", .offset = 0x5D, }, - { .name = "tx_aged", .offset = 0x5E, }, - { .name = "drop_local", .offset = 0x80, }, - { .name = "drop_tail", .offset = 0x81, }, - { .name = "drop_yellow_prio_0", .offset = 0x82, }, - { .name = "drop_yellow_prio_1", .offset = 0x83, }, - { .name = "drop_yellow_prio_2", .offset = 0x84, }, - { .name = "drop_yellow_prio_3", .offset = 0x85, }, - { .name = "drop_yellow_prio_4", .offset = 0x86, }, - { .name = "drop_yellow_prio_5", .offset = 0x87, }, - { .name = "drop_yellow_prio_6", .offset = 0x88, }, - { .name = "drop_yellow_prio_7", .offset = 0x89, }, - { .name = "drop_green_prio_0", .offset = 0x8A, }, - { .name = "drop_green_prio_1", .offset = 0x8B, }, - { .name = "drop_green_prio_2", .offset = 0x8C, }, - { .name = "drop_green_prio_3", .offset = 0x8D, }, - { .name = "drop_green_prio_4", .offset = 0x8E, }, - { .name = "drop_green_prio_5", .offset = 0x8F, }, - { .name = "drop_green_prio_6", .offset = 0x90, }, - { .name = "drop_green_prio_7", .offset = 0x91, }, - OCELOT_STAT_END +static const struct ocelot_stat_layout ocelot_stats_layout[OCELOT_NUM_STATS] = { + [OCELOT_STAT_RX_OCTETS] = { + .name = "rx_octets", + .offset = 0x00, + }, + [OCELOT_STAT_RX_UNICAST] = { + .name = "rx_unicast", + .offset = 0x01, + }, + [OCELOT_STAT_RX_MULTICAST] = { + .name = "rx_multicast", + .offset = 0x02, + }, + [OCELOT_STAT_RX_BROADCAST] = { + .name = "rx_broadcast", + .offset = 0x03, + }, + [OCELOT_STAT_RX_SHORTS] = { + .name = "rx_shorts", + .offset = 0x04, + }, + [OCELOT_STAT_RX_FRAGMENTS] = { + .name = "rx_fragments", + .offset = 0x05, + }, + [OCELOT_STAT_RX_JABBERS] = { + .name = "rx_jabbers", + .offset = 0x06, + }, + [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { + .name = "rx_crc_align_errs", + .offset = 0x07, + }, + [OCELOT_STAT_RX_SYM_ERRS] = { + .name = "rx_sym_errs", + .offset = 0x08, + }, + [OCELOT_STAT_RX_64] = { + .name = "rx_frames_below_65_octets", + .offset = 0x09, + }, + [OCELOT_STAT_RX_65_127] = { + .name = "rx_frames_65_to_127_octets", + .offset = 0x0A, + }, + [OCELOT_STAT_RX_128_255] = { + .name = "rx_frames_128_to_255_octets", + .offset = 0x0B, + }, + [OCELOT_STAT_RX_256_511] = { + .name = "rx_frames_256_to_511_octets", + .offset = 0x0C, + }, + [OCELOT_STAT_RX_512_1023] = { + .name = "rx_frames_512_to_1023_octets", + .offset = 0x0D, + }, + [OCELOT_STAT_RX_1024_1526] = { + .name = "rx_frames_1024_to_1526_octets", + .offset = 0x0E, + }, + [OCELOT_STAT_RX_1527_MAX] = { + .name = "rx_frames_over_1526_octets", + .offset = 0x0F, + }, + [OCELOT_STAT_RX_PAUSE] = { + .name = "rx_pause", + .offset = 0x10, + }, + [OCELOT_STAT_RX_CONTROL] = { + .name = "rx_control", + .offset = 0x11, + }, + [OCELOT_STAT_RX_LONGS] = { + .name = "rx_longs", + .offset = 0x12, + }, + [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { + .name = "rx_classified_drops", + .offset = 0x13, + }, + [OCELOT_STAT_RX_RED_PRIO_0] = { + .name = "rx_red_prio_0", + .offset = 0x14, + }, + [OCELOT_STAT_RX_RED_PRIO_1] = { + .name = "rx_red_prio_1", + .offset = 0x15, + }, + [OCELOT_STAT_RX_RED_PRIO_2] = { + .name = "rx_red_prio_2", + .offset = 0x16, + }, + [OCELOT_STAT_RX_RED_PRIO_3] = { + .name = "rx_red_prio_3", + .offset = 0x17, + }, + [OCELOT_STAT_RX_RED_PRIO_4] = { + .name = "rx_red_prio_4", + .offset = 0x18, + }, + [OCELOT_STAT_RX_RED_PRIO_5] = { + .name = "rx_red_prio_5", + .offset = 0x19, + }, + [OCELOT_STAT_RX_RED_PRIO_6] = { + .name = "rx_red_prio_6", + .offset = 0x1A, + }, + [OCELOT_STAT_RX_RED_PRIO_7] = { + .name = "rx_red_prio_7", + .offset = 0x1B, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_0] = { + .name = "rx_yellow_prio_0", + .offset = 0x1C, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_1] = { + .name = "rx_yellow_prio_1", + .offset = 0x1D, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_2] = { + .name = "rx_yellow_prio_2", + .offset = 0x1E, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_3] = { + .name = "rx_yellow_prio_3", + .offset = 0x1F, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_4] = { + .name = "rx_yellow_prio_4", + .offset = 0x20, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_5] = { + .name = "rx_yellow_prio_5", + .offset = 0x21, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_6] = { + .name = "rx_yellow_prio_6", + .offset = 0x22, + }, + [OCELOT_STAT_RX_YELLOW_PRIO_7] = { + .name = "rx_yellow_prio_7", + .offset = 0x23, + }, + [OCELOT_STAT_RX_GREEN_PRIO_0] = { + .name = "rx_green_prio_0", + .offset = 0x24, + }, + [OCELOT_STAT_RX_GREEN_PRIO_1] = { + .name = "rx_green_prio_1", + .offset = 0x25, + }, + [OCELOT_STAT_RX_GREEN_PRIO_2] = { + .name = "rx_green_prio_2", + .offset = 0x26, + }, + [OCELOT_STAT_RX_GREEN_PRIO_3] = { + .name = "rx_green_prio_3", + .offset = 0x27, + }, + [OCELOT_STAT_RX_GREEN_PRIO_4] = { + .name = "rx_green_prio_4", + .offset = 0x28, + }, + [OCELOT_STAT_RX_GREEN_PRIO_5] = { + .name = "rx_green_prio_5", + .offset = 0x29, + }, + [OCELOT_STAT_RX_GREEN_PRIO_6] = { + .name = "rx_green_prio_6", + .offset = 0x2A, + }, + [OCELOT_STAT_RX_GREEN_PRIO_7] = { + .name = "rx_green_prio_7", + .offset = 0x2B, + }, + [OCELOT_STAT_TX_OCTETS] = { + .name = "tx_octets", + .offset = 0x40, + }, + [OCELOT_STAT_TX_UNICAST] = { + .name = "tx_unicast", + .offset = 0x41, + }, + [OCELOT_STAT_TX_MULTICAST] = { + .name = "tx_multicast", + .offset = 0x42, + }, + [OCELOT_STAT_TX_BROADCAST] = { + .name = "tx_broadcast", + .offset = 0x43, + }, + [OCELOT_STAT_TX_COLLISION] = { + .name = "tx_collision", + .offset = 0x44, + }, + [OCELOT_STAT_TX_DROPS] = { + .name = "tx_drops", + .offset = 0x45, + }, + [OCELOT_STAT_TX_PAUSE] = { + .name = "tx_pause", + .offset = 0x46, + }, + [OCELOT_STAT_TX_64] = { + .name = "tx_frames_below_65_octets", + .offset = 0x47, + }, + [OCELOT_STAT_TX_65_127] = { + .name = "tx_frames_65_to_127_octets", + .offset = 0x48, + }, + [OCELOT_STAT_TX_128_255] = { + .name = "tx_frames_128_255_octets", + .offset = 0x49, + }, + [OCELOT_STAT_TX_256_511] = { + .name = "tx_frames_256_511_octets", + .offset = 0x4A, + }, + [OCELOT_STAT_TX_512_1023] = { + .name = "tx_frames_512_1023_octets", + .offset = 0x4B, + }, + [OCELOT_STAT_TX_1024_1526] = { + .name = "tx_frames_1024_1526_octets", + .offset = 0x4C, + }, + [OCELOT_STAT_TX_1527_MAX] = { + .name = "tx_frames_over_1526_octets", + .offset = 0x4D, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_0] = { + .name = "tx_yellow_prio_0", + .offset = 0x4E, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_1] = { + .name = "tx_yellow_prio_1", + .offset = 0x4F, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_2] = { + .name = "tx_yellow_prio_2", + .offset = 0x50, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_3] = { + .name = "tx_yellow_prio_3", + .offset = 0x51, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_4] = { + .name = "tx_yellow_prio_4", + .offset = 0x52, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_5] = { + .name = "tx_yellow_prio_5", + .offset = 0x53, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_6] = { + .name = "tx_yellow_prio_6", + .offset = 0x54, + }, + [OCELOT_STAT_TX_YELLOW_PRIO_7] = { + .name = "tx_yellow_prio_7", + .offset = 0x55, + }, + [OCELOT_STAT_TX_GREEN_PRIO_0] = { + .name = "tx_green_prio_0", + .offset = 0x56, + }, + [OCELOT_STAT_TX_GREEN_PRIO_1] = { + .name = "tx_green_prio_1", + .offset = 0x57, + }, + [OCELOT_STAT_TX_GREEN_PRIO_2] = { + .name = "tx_green_prio_2", + .offset = 0x58, + }, + [OCELOT_STAT_TX_GREEN_PRIO_3] = { + .name = "tx_green_prio_3", + .offset = 0x59, + }, + [OCELOT_STAT_TX_GREEN_PRIO_4] = { + .name = "tx_green_prio_4", + .offset = 0x5A, + }, + [OCELOT_STAT_TX_GREEN_PRIO_5] = { + .name = "tx_green_prio_5", + .offset = 0x5B, + }, + [OCELOT_STAT_TX_GREEN_PRIO_6] = { + .name = "tx_green_prio_6", + .offset = 0x5C, + }, + [OCELOT_STAT_TX_GREEN_PRIO_7] = { + .name = "tx_green_prio_7", + .offset = 0x5D, + }, + [OCELOT_STAT_TX_AGED] = { + .name = "tx_aged", + .offset = 0x5E, + }, + [OCELOT_STAT_DROP_LOCAL] = { + .name = "drop_local", + .offset = 0x80, + }, + [OCELOT_STAT_DROP_TAIL] = { + .name = "drop_tail", + .offset = 0x81, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { + .name = "drop_yellow_prio_0", + .offset = 0x82, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { + .name = "drop_yellow_prio_1", + .offset = 0x83, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { + .name = "drop_yellow_prio_2", + .offset = 0x84, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { + .name = "drop_yellow_prio_3", + .offset = 0x85, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { + .name = "drop_yellow_prio_4", + .offset = 0x86, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { + .name = "drop_yellow_prio_5", + .offset = 0x87, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { + .name = "drop_yellow_prio_6", + .offset = 0x88, + }, + [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { + .name = "drop_yellow_prio_7", + .offset = 0x89, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_0] = { + .name = "drop_green_prio_0", + .offset = 0x8A, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_1] = { + .name = "drop_green_prio_1", + .offset = 0x8B, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_2] = { + .name = "drop_green_prio_2", + .offset = 0x8C, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_3] = { + .name = "drop_green_prio_3", + .offset = 0x8D, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_4] = { + .name = "drop_green_prio_4", + .offset = 0x8E, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_5] = { + .name = "drop_green_prio_5", + .offset = 0x8F, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_6] = { + .name = "drop_green_prio_6", + .offset = 0x90, + }, + [OCELOT_STAT_DROP_GREEN_PRIO_7] = { + .name = "drop_green_prio_7", + .offset = 0x91, + }, }; static void ocelot_pll5_init(struct ocelot *ocelot) diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index 72b9474391da..2428bc64cb1d 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -105,11 +105,6 @@ #define REG_RESERVED_ADDR 0xffffffff #define REG_RESERVED(reg) REG(reg, REG_RESERVED_ADDR) -#define for_each_stat(ocelot, stat) \ - for ((stat) = (ocelot)->stats_layout; \ - ((stat)->name[0] != '\0'); \ - (stat)++) - enum ocelot_target { ANA = 1, QS, @@ -540,13 +535,108 @@ enum ocelot_ptp_pins { TOD_ACC_PIN }; +enum ocelot_stat { + OCELOT_STAT_RX_OCTETS, + OCELOT_STAT_RX_UNICAST, + OCELOT_STAT_RX_MULTICAST, + OCELOT_STAT_RX_BROADCAST, + OCELOT_STAT_RX_SHORTS, + OCELOT_STAT_RX_FRAGMENTS, + OCELOT_STAT_RX_JABBERS, + OCELOT_STAT_RX_CRC_ALIGN_ERRS, + OCELOT_STAT_RX_SYM_ERRS, + OCELOT_STAT_RX_64, + OCELOT_STAT_RX_65_127, + OCELOT_STAT_RX_128_255, + OCELOT_STAT_RX_256_511, + OCELOT_STAT_RX_512_1023, + OCELOT_STAT_RX_1024_1526, + OCELOT_STAT_RX_1527_MAX, + OCELOT_STAT_RX_PAUSE, + OCELOT_STAT_RX_CONTROL, + OCELOT_STAT_RX_LONGS, + OCELOT_STAT_RX_CLASSIFIED_DROPS, + OCELOT_STAT_RX_RED_PRIO_0, + OCELOT_STAT_RX_RED_PRIO_1, + OCELOT_STAT_RX_RED_PRIO_2, + OCELOT_STAT_RX_RED_PRIO_3, + OCELOT_STAT_RX_RED_PRIO_4, + OCELOT_STAT_RX_RED_PRIO_5, + OCELOT_STAT_RX_RED_PRIO_6, + OCELOT_STAT_RX_RED_PRIO_7, + OCELOT_STAT_RX_YELLOW_PRIO_0, + OCELOT_STAT_RX_YELLOW_PRIO_1, + OCELOT_STAT_RX_YELLOW_PRIO_2, + OCELOT_STAT_RX_YELLOW_PRIO_3, + OCELOT_STAT_RX_YELLOW_PRIO_4, + OCELOT_STAT_RX_YELLOW_PRIO_5, + OCELOT_STAT_RX_YELLOW_PRIO_6, + OCELOT_STAT_RX_YELLOW_PRIO_7, + OCELOT_STAT_RX_GREEN_PRIO_0, + OCELOT_STAT_RX_GREEN_PRIO_1, + OCELOT_STAT_RX_GREEN_PRIO_2, + OCELOT_STAT_RX_GREEN_PRIO_3, + OCELOT_STAT_RX_GREEN_PRIO_4, + OCELOT_STAT_RX_GREEN_PRIO_5, + OCELOT_STAT_RX_GREEN_PRIO_6, + OCELOT_STAT_RX_GREEN_PRIO_7, + OCELOT_STAT_TX_OCTETS, + OCELOT_STAT_TX_UNICAST, + OCELOT_STAT_TX_MULTICAST, + OCELOT_STAT_TX_BROADCAST, + OCELOT_STAT_TX_COLLISION, + OCELOT_STAT_TX_DROPS, + OCELOT_STAT_TX_PAUSE, + OCELOT_STAT_TX_64, + OCELOT_STAT_TX_65_127, + OCELOT_STAT_TX_128_255, + OCELOT_STAT_TX_256_511, + OCELOT_STAT_TX_512_1023, + OCELOT_STAT_TX_1024_1526, + OCELOT_STAT_TX_1527_MAX, + OCELOT_STAT_TX_YELLOW_PRIO_0, + OCELOT_STAT_TX_YELLOW_PRIO_1, + OCELOT_STAT_TX_YELLOW_PRIO_2, + OCELOT_STAT_TX_YELLOW_PRIO_3, + OCELOT_STAT_TX_YELLOW_PRIO_4, + OCELOT_STAT_TX_YELLOW_PRIO_5, + OCELOT_STAT_TX_YELLOW_PRIO_6, + OCELOT_STAT_TX_YELLOW_PRIO_7, + OCELOT_STAT_TX_GREEN_PRIO_0, + OCELOT_STAT_TX_GREEN_PRIO_1, + OCELOT_STAT_TX_GREEN_PRIO_2, + OCELOT_STAT_TX_GREEN_PRIO_3, + OCELOT_STAT_TX_GREEN_PRIO_4, + OCELOT_STAT_TX_GREEN_PRIO_5, + OCELOT_STAT_TX_GREEN_PRIO_6, + OCELOT_STAT_TX_GREEN_PRIO_7, + OCELOT_STAT_TX_AGED, + OCELOT_STAT_DROP_LOCAL, + OCELOT_STAT_DROP_TAIL, + OCELOT_STAT_DROP_YELLOW_PRIO_0, + OCELOT_STAT_DROP_YELLOW_PRIO_1, + OCELOT_STAT_DROP_YELLOW_PRIO_2, + OCELOT_STAT_DROP_YELLOW_PRIO_3, + OCELOT_STAT_DROP_YELLOW_PRIO_4, + OCELOT_STAT_DROP_YELLOW_PRIO_5, + OCELOT_STAT_DROP_YELLOW_PRIO_6, + OCELOT_STAT_DROP_YELLOW_PRIO_7, + OCELOT_STAT_DROP_GREEN_PRIO_0, + OCELOT_STAT_DROP_GREEN_PRIO_1, + OCELOT_STAT_DROP_GREEN_PRIO_2, + OCELOT_STAT_DROP_GREEN_PRIO_3, + OCELOT_STAT_DROP_GREEN_PRIO_4, + OCELOT_STAT_DROP_GREEN_PRIO_5, + OCELOT_STAT_DROP_GREEN_PRIO_6, + OCELOT_STAT_DROP_GREEN_PRIO_7, + OCELOT_NUM_STATS, +}; + struct ocelot_stat_layout { u32 offset; char name[ETH_GSTRING_LEN]; }; -#define OCELOT_STAT_END { .name = "" } - struct ocelot_stats_region { struct list_head node; u32 offset; @@ -709,7 +799,6 @@ struct ocelot { const u32 *const *map; const struct ocelot_stat_layout *stats_layout; struct list_head stats_regions; - unsigned int num_stats; u32 pool_size[OCELOT_SB_NUM][OCELOT_SB_POOL_NUM]; int packet_buffer_size; -- cgit v1.3-15-g9ac1 From d4c367650704de091d4c1f6bb379c0a5c389c73a Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 16 Aug 2022 16:53:51 +0300 Subject: net: mscc: ocelot: keep ocelot_stat_layout by reg address, not offset With so many counter addresses recently discovered as being wrong, it is desirable to at least have a central database of information, rather than two: one through the SYS_COUNT_* registers (used for ndo_get_stats64), and the other through the offset field of struct ocelot_stat_layout elements (used for ethtool -S). The strategy will be to keep the SYS_COUNT_* definitions as the single source of truth, but for that we need to expand our current definitions to cover all registers. Then we need to convert the ocelot region creation logic, and stats worker, to the read semantics imposed by going through SYS_COUNT_* absolute register addresses, rather than offsets of 32-bit words relative to SYS_COUNT_RX_OCTETS (which should have been SYS_CNT, by the way). Signed-off-by: Vladimir Oltean Signed-off-by: Jakub Kicinski --- drivers/net/dsa/ocelot/felix_vsc9959.c | 253 +++++++++++++++++----------- drivers/net/dsa/ocelot/seville_vsc9953.c | 255 ++++++++++++++++++----------- drivers/net/ethernet/mscc/ocelot.c | 11 +- drivers/net/ethernet/mscc/ocelot_vsc7514.c | 186 ++++++++++----------- drivers/net/ethernet/mscc/vsc7514_regs.c | 58 +++++++ include/soc/mscc/ocelot.h | 66 +++++++- 6 files changed, 540 insertions(+), 289 deletions(-) (limited to 'include') diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index c9f270f24b1c..1cdce8a98d1d 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -274,10 +274,14 @@ static const u32 vsc9959_rew_regmap[] = { static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_RX_OCTETS, 0x000000), + REG(SYS_COUNT_RX_UNICAST, 0x000004), REG(SYS_COUNT_RX_MULTICAST, 0x000008), + REG(SYS_COUNT_RX_BROADCAST, 0x00000c), REG(SYS_COUNT_RX_SHORTS, 0x000010), REG(SYS_COUNT_RX_FRAGMENTS, 0x000014), REG(SYS_COUNT_RX_JABBERS, 0x000018), + REG(SYS_COUNT_RX_CRC_ALIGN_ERRS, 0x00001c), + REG(SYS_COUNT_RX_SYM_ERRS, 0x000020), REG(SYS_COUNT_RX_64, 0x000024), REG(SYS_COUNT_RX_65_127, 0x000028), REG(SYS_COUNT_RX_128_255, 0x00002c), @@ -288,9 +292,38 @@ static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_RX_PAUSE, 0x000040), REG(SYS_COUNT_RX_CONTROL, 0x000044), REG(SYS_COUNT_RX_LONGS, 0x000048), + REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x00004c), + REG(SYS_COUNT_RX_RED_PRIO_0, 0x000050), + REG(SYS_COUNT_RX_RED_PRIO_1, 0x000054), + REG(SYS_COUNT_RX_RED_PRIO_2, 0x000058), + REG(SYS_COUNT_RX_RED_PRIO_3, 0x00005c), + REG(SYS_COUNT_RX_RED_PRIO_4, 0x000060), + REG(SYS_COUNT_RX_RED_PRIO_5, 0x000064), + REG(SYS_COUNT_RX_RED_PRIO_6, 0x000068), + REG(SYS_COUNT_RX_RED_PRIO_7, 0x00006c), + REG(SYS_COUNT_RX_YELLOW_PRIO_0, 0x000070), + REG(SYS_COUNT_RX_YELLOW_PRIO_1, 0x000074), + REG(SYS_COUNT_RX_YELLOW_PRIO_2, 0x000078), + REG(SYS_COUNT_RX_YELLOW_PRIO_3, 0x00007c), + REG(SYS_COUNT_RX_YELLOW_PRIO_4, 0x000080), + REG(SYS_COUNT_RX_YELLOW_PRIO_5, 0x000084), + REG(SYS_COUNT_RX_YELLOW_PRIO_6, 0x000088), + REG(SYS_COUNT_RX_YELLOW_PRIO_7, 0x00008c), + REG(SYS_COUNT_RX_GREEN_PRIO_0, 0x000090), + REG(SYS_COUNT_RX_GREEN_PRIO_1, 0x000094), + REG(SYS_COUNT_RX_GREEN_PRIO_2, 0x000098), + REG(SYS_COUNT_RX_GREEN_PRIO_3, 0x00009c), + REG(SYS_COUNT_RX_GREEN_PRIO_4, 0x0000a0), + REG(SYS_COUNT_RX_GREEN_PRIO_5, 0x0000a4), + REG(SYS_COUNT_RX_GREEN_PRIO_6, 0x0000a8), + REG(SYS_COUNT_RX_GREEN_PRIO_7, 0x0000ac), REG(SYS_COUNT_TX_OCTETS, 0x000200), + REG(SYS_COUNT_TX_UNICAST, 0x000204), + REG(SYS_COUNT_TX_MULTICAST, 0x000208), + REG(SYS_COUNT_TX_BROADCAST, 0x00020c), REG(SYS_COUNT_TX_COLLISION, 0x000210), REG(SYS_COUNT_TX_DROPS, 0x000214), + REG(SYS_COUNT_TX_PAUSE, 0x000218), REG(SYS_COUNT_TX_64, 0x00021c), REG(SYS_COUNT_TX_65_127, 0x000220), REG(SYS_COUNT_TX_128_255, 0x000224), @@ -298,7 +331,41 @@ static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_TX_512_1023, 0x00022c), REG(SYS_COUNT_TX_1024_1526, 0x000230), REG(SYS_COUNT_TX_1527_MAX, 0x000234), + REG(SYS_COUNT_TX_YELLOW_PRIO_0, 0x000238), + REG(SYS_COUNT_TX_YELLOW_PRIO_1, 0x00023c), + REG(SYS_COUNT_TX_YELLOW_PRIO_2, 0x000240), + REG(SYS_COUNT_TX_YELLOW_PRIO_3, 0x000244), + REG(SYS_COUNT_TX_YELLOW_PRIO_4, 0x000248), + REG(SYS_COUNT_TX_YELLOW_PRIO_5, 0x00024c), + REG(SYS_COUNT_TX_YELLOW_PRIO_6, 0x000250), + REG(SYS_COUNT_TX_YELLOW_PRIO_7, 0x000254), + REG(SYS_COUNT_TX_GREEN_PRIO_0, 0x000258), + REG(SYS_COUNT_TX_GREEN_PRIO_1, 0x00025c), + REG(SYS_COUNT_TX_GREEN_PRIO_2, 0x000260), + REG(SYS_COUNT_TX_GREEN_PRIO_3, 0x000264), + REG(SYS_COUNT_TX_GREEN_PRIO_4, 0x000268), + REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00026c), + REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000270), + REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000274), REG(SYS_COUNT_TX_AGING, 0x000278), + REG(SYS_COUNT_DROP_LOCAL, 0x000400), + REG(SYS_COUNT_DROP_TAIL, 0x000404), + REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000408), + REG(SYS_COUNT_DROP_YELLOW_PRIO_1, 0x00040c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_2, 0x000410), + REG(SYS_COUNT_DROP_YELLOW_PRIO_3, 0x000414), + REG(SYS_COUNT_DROP_YELLOW_PRIO_4, 0x000418), + REG(SYS_COUNT_DROP_YELLOW_PRIO_5, 0x00041c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_6, 0x000420), + REG(SYS_COUNT_DROP_YELLOW_PRIO_7, 0x000424), + REG(SYS_COUNT_DROP_GREEN_PRIO_0, 0x000428), + REG(SYS_COUNT_DROP_GREEN_PRIO_1, 0x00042c), + REG(SYS_COUNT_DROP_GREEN_PRIO_2, 0x000430), + REG(SYS_COUNT_DROP_GREEN_PRIO_3, 0x000434), + REG(SYS_COUNT_DROP_GREEN_PRIO_4, 0x000438), + REG(SYS_COUNT_DROP_GREEN_PRIO_5, 0x00043c), + REG(SYS_COUNT_DROP_GREEN_PRIO_6, 0x000440), + REG(SYS_COUNT_DROP_GREEN_PRIO_7, 0x000444), REG(SYS_RESET_CFG, 0x000e00), REG(SYS_SR_ETYPE_CFG, 0x000e04), REG(SYS_VLAN_ETYPE_CFG, 0x000e08), @@ -554,375 +621,375 @@ static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = { static const struct ocelot_stat_layout vsc9959_stats_layout[OCELOT_NUM_STATS] = { [OCELOT_STAT_RX_OCTETS] = { .name = "rx_octets", - .offset = 0x00, + .reg = SYS_COUNT_RX_OCTETS, }, [OCELOT_STAT_RX_UNICAST] = { .name = "rx_unicast", - .offset = 0x01, + .reg = SYS_COUNT_RX_UNICAST, }, [OCELOT_STAT_RX_MULTICAST] = { .name = "rx_multicast", - .offset = 0x02, + .reg = SYS_COUNT_RX_MULTICAST, }, [OCELOT_STAT_RX_BROADCAST] = { .name = "rx_broadcast", - .offset = 0x03, + .reg = SYS_COUNT_RX_BROADCAST, }, [OCELOT_STAT_RX_SHORTS] = { .name = "rx_shorts", - .offset = 0x04, + .reg = SYS_COUNT_RX_SHORTS, }, [OCELOT_STAT_RX_FRAGMENTS] = { .name = "rx_fragments", - .offset = 0x05, + .reg = SYS_COUNT_RX_FRAGMENTS, }, [OCELOT_STAT_RX_JABBERS] = { .name = "rx_jabbers", - .offset = 0x06, + .reg = SYS_COUNT_RX_JABBERS, }, [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { .name = "rx_crc_align_errs", - .offset = 0x07, + .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, }, [OCELOT_STAT_RX_SYM_ERRS] = { .name = "rx_sym_errs", - .offset = 0x08, + .reg = SYS_COUNT_RX_SYM_ERRS, }, [OCELOT_STAT_RX_64] = { .name = "rx_frames_below_65_octets", - .offset = 0x09, + .reg = SYS_COUNT_RX_64, }, [OCELOT_STAT_RX_65_127] = { .name = "rx_frames_65_to_127_octets", - .offset = 0x0A, + .reg = SYS_COUNT_RX_65_127, }, [OCELOT_STAT_RX_128_255] = { .name = "rx_frames_128_to_255_octets", - .offset = 0x0B, + .reg = SYS_COUNT_RX_128_255, }, [OCELOT_STAT_RX_256_511] = { .name = "rx_frames_256_to_511_octets", - .offset = 0x0C, + .reg = SYS_COUNT_RX_256_511, }, [OCELOT_STAT_RX_512_1023] = { .name = "rx_frames_512_to_1023_octets", - .offset = 0x0D, + .reg = SYS_COUNT_RX_512_1023, }, [OCELOT_STAT_RX_1024_1526] = { .name = "rx_frames_1024_to_1526_octets", - .offset = 0x0E, + .reg = SYS_COUNT_RX_1024_1526, }, [OCELOT_STAT_RX_1527_MAX] = { .name = "rx_frames_over_1526_octets", - .offset = 0x0F, + .reg = SYS_COUNT_RX_1527_MAX, }, [OCELOT_STAT_RX_PAUSE] = { .name = "rx_pause", - .offset = 0x10, + .reg = SYS_COUNT_RX_PAUSE, }, [OCELOT_STAT_RX_CONTROL] = { .name = "rx_control", - .offset = 0x11, + .reg = SYS_COUNT_RX_CONTROL, }, [OCELOT_STAT_RX_LONGS] = { .name = "rx_longs", - .offset = 0x12, + .reg = SYS_COUNT_RX_LONGS, }, [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { .name = "rx_classified_drops", - .offset = 0x13, + .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, }, [OCELOT_STAT_RX_RED_PRIO_0] = { .name = "rx_red_prio_0", - .offset = 0x14, + .reg = SYS_COUNT_RX_RED_PRIO_0, }, [OCELOT_STAT_RX_RED_PRIO_1] = { .name = "rx_red_prio_1", - .offset = 0x15, + .reg = SYS_COUNT_RX_RED_PRIO_1, }, [OCELOT_STAT_RX_RED_PRIO_2] = { .name = "rx_red_prio_2", - .offset = 0x16, + .reg = SYS_COUNT_RX_RED_PRIO_2, }, [OCELOT_STAT_RX_RED_PRIO_3] = { .name = "rx_red_prio_3", - .offset = 0x17, + .reg = SYS_COUNT_RX_RED_PRIO_3, }, [OCELOT_STAT_RX_RED_PRIO_4] = { .name = "rx_red_prio_4", - .offset = 0x18, + .reg = SYS_COUNT_RX_RED_PRIO_4, }, [OCELOT_STAT_RX_RED_PRIO_5] = { .name = "rx_red_prio_5", - .offset = 0x19, + .reg = SYS_COUNT_RX_RED_PRIO_5, }, [OCELOT_STAT_RX_RED_PRIO_6] = { .name = "rx_red_prio_6", - .offset = 0x1A, + .reg = SYS_COUNT_RX_RED_PRIO_6, }, [OCELOT_STAT_RX_RED_PRIO_7] = { .name = "rx_red_prio_7", - .offset = 0x1B, + .reg = SYS_COUNT_RX_RED_PRIO_7, }, [OCELOT_STAT_RX_YELLOW_PRIO_0] = { .name = "rx_yellow_prio_0", - .offset = 0x1C, + .reg = SYS_COUNT_RX_YELLOW_PRIO_0, }, [OCELOT_STAT_RX_YELLOW_PRIO_1] = { .name = "rx_yellow_prio_1", - .offset = 0x1D, + .reg = SYS_COUNT_RX_YELLOW_PRIO_1, }, [OCELOT_STAT_RX_YELLOW_PRIO_2] = { .name = "rx_yellow_prio_2", - .offset = 0x1E, + .reg = SYS_COUNT_RX_YELLOW_PRIO_2, }, [OCELOT_STAT_RX_YELLOW_PRIO_3] = { .name = "rx_yellow_prio_3", - .offset = 0x1F, + .reg = SYS_COUNT_RX_YELLOW_PRIO_3, }, [OCELOT_STAT_RX_YELLOW_PRIO_4] = { .name = "rx_yellow_prio_4", - .offset = 0x20, + .reg = SYS_COUNT_RX_YELLOW_PRIO_4, }, [OCELOT_STAT_RX_YELLOW_PRIO_5] = { .name = "rx_yellow_prio_5", - .offset = 0x21, + .reg = SYS_COUNT_RX_YELLOW_PRIO_5, }, [OCELOT_STAT_RX_YELLOW_PRIO_6] = { .name = "rx_yellow_prio_6", - .offset = 0x22, + .reg = SYS_COUNT_RX_YELLOW_PRIO_6, }, [OCELOT_STAT_RX_YELLOW_PRIO_7] = { .name = "rx_yellow_prio_7", - .offset = 0x23, + .reg = SYS_COUNT_RX_YELLOW_PRIO_7, }, [OCELOT_STAT_RX_GREEN_PRIO_0] = { .name = "rx_green_prio_0", - .offset = 0x24, + .reg = SYS_COUNT_RX_GREEN_PRIO_0, }, [OCELOT_STAT_RX_GREEN_PRIO_1] = { .name = "rx_green_prio_1", - .offset = 0x25, + .reg = SYS_COUNT_RX_GREEN_PRIO_1, }, [OCELOT_STAT_RX_GREEN_PRIO_2] = { .name = "rx_green_prio_2", - .offset = 0x26, + .reg = SYS_COUNT_RX_GREEN_PRIO_2, }, [OCELOT_STAT_RX_GREEN_PRIO_3] = { .name = "rx_green_prio_3", - .offset = 0x27, + .reg = SYS_COUNT_RX_GREEN_PRIO_3, }, [OCELOT_STAT_RX_GREEN_PRIO_4] = { .name = "rx_green_prio_4", - .offset = 0x28, + .reg = SYS_COUNT_RX_GREEN_PRIO_4, }, [OCELOT_STAT_RX_GREEN_PRIO_5] = { .name = "rx_green_prio_5", - .offset = 0x29, + .reg = SYS_COUNT_RX_GREEN_PRIO_5, }, [OCELOT_STAT_RX_GREEN_PRIO_6] = { .name = "rx_green_prio_6", - .offset = 0x2A, + .reg = SYS_COUNT_RX_GREEN_PRIO_6, }, [OCELOT_STAT_RX_GREEN_PRIO_7] = { .name = "rx_green_prio_7", - .offset = 0x2B, + .reg = SYS_COUNT_RX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_OCTETS] = { .name = "tx_octets", - .offset = 0x80, + .reg = SYS_COUNT_TX_OCTETS, }, [OCELOT_STAT_TX_UNICAST] = { .name = "tx_unicast", - .offset = 0x81, + .reg = SYS_COUNT_TX_UNICAST, }, [OCELOT_STAT_TX_MULTICAST] = { .name = "tx_multicast", - .offset = 0x82, + .reg = SYS_COUNT_TX_MULTICAST, }, [OCELOT_STAT_TX_BROADCAST] = { .name = "tx_broadcast", - .offset = 0x83, + .reg = SYS_COUNT_TX_BROADCAST, }, [OCELOT_STAT_TX_COLLISION] = { .name = "tx_collision", - .offset = 0x84, + .reg = SYS_COUNT_TX_COLLISION, }, [OCELOT_STAT_TX_DROPS] = { .name = "tx_drops", - .offset = 0x85, + .reg = SYS_COUNT_TX_DROPS, }, [OCELOT_STAT_TX_PAUSE] = { .name = "tx_pause", - .offset = 0x86, + .reg = SYS_COUNT_TX_PAUSE, }, [OCELOT_STAT_TX_64] = { .name = "tx_frames_below_65_octets", - .offset = 0x87, + .reg = SYS_COUNT_TX_64, }, [OCELOT_STAT_TX_65_127] = { .name = "tx_frames_65_to_127_octets", - .offset = 0x88, + .reg = SYS_COUNT_TX_65_127, }, [OCELOT_STAT_TX_128_255] = { .name = "tx_frames_128_255_octets", - .offset = 0x89, + .reg = SYS_COUNT_TX_128_255, }, [OCELOT_STAT_TX_256_511] = { .name = "tx_frames_256_511_octets", - .offset = 0x8A, + .reg = SYS_COUNT_TX_256_511, }, [OCELOT_STAT_TX_512_1023] = { .name = "tx_frames_512_1023_octets", - .offset = 0x8B, + .reg = SYS_COUNT_TX_512_1023, }, [OCELOT_STAT_TX_1024_1526] = { .name = "tx_frames_1024_1526_octets", - .offset = 0x8C, + .reg = SYS_COUNT_TX_1024_1526, }, [OCELOT_STAT_TX_1527_MAX] = { .name = "tx_frames_over_1526_octets", - .offset = 0x8D, + .reg = SYS_COUNT_TX_1527_MAX, }, [OCELOT_STAT_TX_YELLOW_PRIO_0] = { .name = "tx_yellow_prio_0", - .offset = 0x8E, + .reg = SYS_COUNT_TX_YELLOW_PRIO_0, }, [OCELOT_STAT_TX_YELLOW_PRIO_1] = { .name = "tx_yellow_prio_1", - .offset = 0x8F, + .reg = SYS_COUNT_TX_YELLOW_PRIO_1, }, [OCELOT_STAT_TX_YELLOW_PRIO_2] = { .name = "tx_yellow_prio_2", - .offset = 0x90, + .reg = SYS_COUNT_TX_YELLOW_PRIO_2, }, [OCELOT_STAT_TX_YELLOW_PRIO_3] = { .name = "tx_yellow_prio_3", - .offset = 0x91, + .reg = SYS_COUNT_TX_YELLOW_PRIO_3, }, [OCELOT_STAT_TX_YELLOW_PRIO_4] = { .name = "tx_yellow_prio_4", - .offset = 0x92, + .reg = SYS_COUNT_TX_YELLOW_PRIO_4, }, [OCELOT_STAT_TX_YELLOW_PRIO_5] = { .name = "tx_yellow_prio_5", - .offset = 0x93, + .reg = SYS_COUNT_TX_YELLOW_PRIO_5, }, [OCELOT_STAT_TX_YELLOW_PRIO_6] = { .name = "tx_yellow_prio_6", - .offset = 0x94, + .reg = SYS_COUNT_TX_YELLOW_PRIO_6, }, [OCELOT_STAT_TX_YELLOW_PRIO_7] = { .name = "tx_yellow_prio_7", - .offset = 0x95, + .reg = SYS_COUNT_TX_YELLOW_PRIO_7, }, [OCELOT_STAT_TX_GREEN_PRIO_0] = { .name = "tx_green_prio_0", - .offset = 0x96, + .reg = SYS_COUNT_TX_GREEN_PRIO_0, }, [OCELOT_STAT_TX_GREEN_PRIO_1] = { .name = "tx_green_prio_1", - .offset = 0x97, + .reg = SYS_COUNT_TX_GREEN_PRIO_1, }, [OCELOT_STAT_TX_GREEN_PRIO_2] = { .name = "tx_green_prio_2", - .offset = 0x98, + .reg = SYS_COUNT_TX_GREEN_PRIO_2, }, [OCELOT_STAT_TX_GREEN_PRIO_3] = { .name = "tx_green_prio_3", - .offset = 0x99, + .reg = SYS_COUNT_TX_GREEN_PRIO_3, }, [OCELOT_STAT_TX_GREEN_PRIO_4] = { .name = "tx_green_prio_4", - .offset = 0x9A, + .reg = SYS_COUNT_TX_GREEN_PRIO_4, }, [OCELOT_STAT_TX_GREEN_PRIO_5] = { .name = "tx_green_prio_5", - .offset = 0x9B, + .reg = SYS_COUNT_TX_GREEN_PRIO_5, }, [OCELOT_STAT_TX_GREEN_PRIO_6] = { .name = "tx_green_prio_6", - .offset = 0x9C, + .reg = SYS_COUNT_TX_GREEN_PRIO_6, }, [OCELOT_STAT_TX_GREEN_PRIO_7] = { .name = "tx_green_prio_7", - .offset = 0x9D, + .reg = SYS_COUNT_TX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_AGED] = { .name = "tx_aged", - .offset = 0x9E, + .reg = SYS_COUNT_TX_AGING, }, [OCELOT_STAT_DROP_LOCAL] = { .name = "drop_local", - .offset = 0x100, + .reg = SYS_COUNT_DROP_LOCAL, }, [OCELOT_STAT_DROP_TAIL] = { .name = "drop_tail", - .offset = 0x101, + .reg = SYS_COUNT_DROP_TAIL, }, [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { .name = "drop_yellow_prio_0", - .offset = 0x102, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, }, [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { .name = "drop_yellow_prio_1", - .offset = 0x103, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, }, [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { .name = "drop_yellow_prio_2", - .offset = 0x104, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, }, [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { .name = "drop_yellow_prio_3", - .offset = 0x105, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, }, [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { .name = "drop_yellow_prio_4", - .offset = 0x106, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, }, [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { .name = "drop_yellow_prio_5", - .offset = 0x107, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, }, [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { .name = "drop_yellow_prio_6", - .offset = 0x108, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, }, [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { .name = "drop_yellow_prio_7", - .offset = 0x109, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, }, [OCELOT_STAT_DROP_GREEN_PRIO_0] = { .name = "drop_green_prio_0", - .offset = 0x10A, + .reg = SYS_COUNT_DROP_GREEN_PRIO_0, }, [OCELOT_STAT_DROP_GREEN_PRIO_1] = { .name = "drop_green_prio_1", - .offset = 0x10B, + .reg = SYS_COUNT_DROP_GREEN_PRIO_1, }, [OCELOT_STAT_DROP_GREEN_PRIO_2] = { .name = "drop_green_prio_2", - .offset = 0x10C, + .reg = SYS_COUNT_DROP_GREEN_PRIO_2, }, [OCELOT_STAT_DROP_GREEN_PRIO_3] = { .name = "drop_green_prio_3", - .offset = 0x10D, + .reg = SYS_COUNT_DROP_GREEN_PRIO_3, }, [OCELOT_STAT_DROP_GREEN_PRIO_4] = { .name = "drop_green_prio_4", - .offset = 0x10E, + .reg = SYS_COUNT_DROP_GREEN_PRIO_4, }, [OCELOT_STAT_DROP_GREEN_PRIO_5] = { .name = "drop_green_prio_5", - .offset = 0x10F, + .reg = SYS_COUNT_DROP_GREEN_PRIO_5, }, [OCELOT_STAT_DROP_GREEN_PRIO_6] = { .name = "drop_green_prio_6", - .offset = 0x110, + .reg = SYS_COUNT_DROP_GREEN_PRIO_6, }, [OCELOT_STAT_DROP_GREEN_PRIO_7] = { .name = "drop_green_prio_7", - .offset = 0x111, + .reg = SYS_COUNT_DROP_GREEN_PRIO_7, }, }; diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index fe5d4642d0bc..b34f4cdfe814 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -270,10 +270,14 @@ static const u32 vsc9953_rew_regmap[] = { static const u32 vsc9953_sys_regmap[] = { REG(SYS_COUNT_RX_OCTETS, 0x000000), + REG(SYS_COUNT_RX_UNICAST, 0x000004), REG(SYS_COUNT_RX_MULTICAST, 0x000008), + REG(SYS_COUNT_RX_BROADCAST, 0x00000c), REG(SYS_COUNT_RX_SHORTS, 0x000010), REG(SYS_COUNT_RX_FRAGMENTS, 0x000014), REG(SYS_COUNT_RX_JABBERS, 0x000018), + REG(SYS_COUNT_RX_CRC_ALIGN_ERRS, 0x00001c), + REG(SYS_COUNT_RX_SYM_ERRS, 0x000020), REG(SYS_COUNT_RX_64, 0x000024), REG(SYS_COUNT_RX_65_127, 0x000028), REG(SYS_COUNT_RX_128_255, 0x00002c), @@ -281,10 +285,41 @@ static const u32 vsc9953_sys_regmap[] = { REG(SYS_COUNT_RX_512_1023, 0x000034), REG(SYS_COUNT_RX_1024_1526, 0x000038), REG(SYS_COUNT_RX_1527_MAX, 0x00003c), + REG(SYS_COUNT_RX_PAUSE, 0x000040), + REG(SYS_COUNT_RX_CONTROL, 0x000044), REG(SYS_COUNT_RX_LONGS, 0x000048), + REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x00004c), + REG(SYS_COUNT_RX_RED_PRIO_0, 0x000050), + REG(SYS_COUNT_RX_RED_PRIO_1, 0x000054), + REG(SYS_COUNT_RX_RED_PRIO_2, 0x000058), + REG(SYS_COUNT_RX_RED_PRIO_3, 0x00005c), + REG(SYS_COUNT_RX_RED_PRIO_4, 0x000060), + REG(SYS_COUNT_RX_RED_PRIO_5, 0x000064), + REG(SYS_COUNT_RX_RED_PRIO_6, 0x000068), + REG(SYS_COUNT_RX_RED_PRIO_7, 0x00006c), + REG(SYS_COUNT_RX_YELLOW_PRIO_0, 0x000070), + REG(SYS_COUNT_RX_YELLOW_PRIO_1, 0x000074), + REG(SYS_COUNT_RX_YELLOW_PRIO_2, 0x000078), + REG(SYS_COUNT_RX_YELLOW_PRIO_3, 0x00007c), + REG(SYS_COUNT_RX_YELLOW_PRIO_4, 0x000080), + REG(SYS_COUNT_RX_YELLOW_PRIO_5, 0x000084), + REG(SYS_COUNT_RX_YELLOW_PRIO_6, 0x000088), + REG(SYS_COUNT_RX_YELLOW_PRIO_7, 0x00008c), + REG(SYS_COUNT_RX_GREEN_PRIO_0, 0x000090), + REG(SYS_COUNT_RX_GREEN_PRIO_1, 0x000094), + REG(SYS_COUNT_RX_GREEN_PRIO_2, 0x000098), + REG(SYS_COUNT_RX_GREEN_PRIO_3, 0x00009c), + REG(SYS_COUNT_RX_GREEN_PRIO_4, 0x0000a0), + REG(SYS_COUNT_RX_GREEN_PRIO_5, 0x0000a4), + REG(SYS_COUNT_RX_GREEN_PRIO_6, 0x0000a8), + REG(SYS_COUNT_RX_GREEN_PRIO_7, 0x0000ac), REG(SYS_COUNT_TX_OCTETS, 0x000100), + REG(SYS_COUNT_TX_UNICAST, 0x000104), + REG(SYS_COUNT_TX_MULTICAST, 0x000108), + REG(SYS_COUNT_TX_BROADCAST, 0x00010c), REG(SYS_COUNT_TX_COLLISION, 0x000110), REG(SYS_COUNT_TX_DROPS, 0x000114), + REG(SYS_COUNT_TX_PAUSE, 0x000118), REG(SYS_COUNT_TX_64, 0x00011c), REG(SYS_COUNT_TX_65_127, 0x000120), REG(SYS_COUNT_TX_128_255, 0x000124), @@ -292,7 +327,41 @@ static const u32 vsc9953_sys_regmap[] = { REG(SYS_COUNT_TX_512_1023, 0x00012c), REG(SYS_COUNT_TX_1024_1526, 0x000130), REG(SYS_COUNT_TX_1527_MAX, 0x000134), + REG(SYS_COUNT_TX_YELLOW_PRIO_0, 0x000138), + REG(SYS_COUNT_TX_YELLOW_PRIO_1, 0x00013c), + REG(SYS_COUNT_TX_YELLOW_PRIO_2, 0x000140), + REG(SYS_COUNT_TX_YELLOW_PRIO_3, 0x000144), + REG(SYS_COUNT_TX_YELLOW_PRIO_4, 0x000148), + REG(SYS_COUNT_TX_YELLOW_PRIO_5, 0x00014c), + REG(SYS_COUNT_TX_YELLOW_PRIO_6, 0x000150), + REG(SYS_COUNT_TX_YELLOW_PRIO_7, 0x000154), + REG(SYS_COUNT_TX_GREEN_PRIO_0, 0x000158), + REG(SYS_COUNT_TX_GREEN_PRIO_1, 0x00015c), + REG(SYS_COUNT_TX_GREEN_PRIO_2, 0x000160), + REG(SYS_COUNT_TX_GREEN_PRIO_3, 0x000164), + REG(SYS_COUNT_TX_GREEN_PRIO_4, 0x000168), + REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00016c), + REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000170), + REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000174), REG(SYS_COUNT_TX_AGING, 0x000178), + REG(SYS_COUNT_DROP_LOCAL, 0x000200), + REG(SYS_COUNT_DROP_TAIL, 0x000204), + REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000208), + REG(SYS_COUNT_DROP_YELLOW_PRIO_1, 0x00020c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_2, 0x000210), + REG(SYS_COUNT_DROP_YELLOW_PRIO_3, 0x000214), + REG(SYS_COUNT_DROP_YELLOW_PRIO_4, 0x000218), + REG(SYS_COUNT_DROP_YELLOW_PRIO_5, 0x00021c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_6, 0x000220), + REG(SYS_COUNT_DROP_YELLOW_PRIO_7, 0x000224), + REG(SYS_COUNT_DROP_GREEN_PRIO_0, 0x000228), + REG(SYS_COUNT_DROP_GREEN_PRIO_1, 0x00022c), + REG(SYS_COUNT_DROP_GREEN_PRIO_2, 0x000230), + REG(SYS_COUNT_DROP_GREEN_PRIO_3, 0x000234), + REG(SYS_COUNT_DROP_GREEN_PRIO_4, 0x000238), + REG(SYS_COUNT_DROP_GREEN_PRIO_5, 0x00023c), + REG(SYS_COUNT_DROP_GREEN_PRIO_6, 0x000240), + REG(SYS_COUNT_DROP_GREEN_PRIO_7, 0x000244), REG(SYS_RESET_CFG, 0x000318), REG_RESERVED(SYS_SR_ETYPE_CFG), REG(SYS_VLAN_ETYPE_CFG, 0x000320), @@ -548,375 +617,375 @@ static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = { static const struct ocelot_stat_layout vsc9953_stats_layout[OCELOT_NUM_STATS] = { [OCELOT_STAT_RX_OCTETS] = { .name = "rx_octets", - .offset = 0x00, + .reg = SYS_COUNT_RX_OCTETS, }, [OCELOT_STAT_RX_UNICAST] = { .name = "rx_unicast", - .offset = 0x01, + .reg = SYS_COUNT_RX_UNICAST, }, [OCELOT_STAT_RX_MULTICAST] = { .name = "rx_multicast", - .offset = 0x02, + .reg = SYS_COUNT_RX_MULTICAST, }, [OCELOT_STAT_RX_BROADCAST] = { .name = "rx_broadcast", - .offset = 0x03, + .reg = SYS_COUNT_RX_BROADCAST, }, [OCELOT_STAT_RX_SHORTS] = { .name = "rx_shorts", - .offset = 0x04, + .reg = SYS_COUNT_RX_SHORTS, }, [OCELOT_STAT_RX_FRAGMENTS] = { .name = "rx_fragments", - .offset = 0x05, + .reg = SYS_COUNT_RX_FRAGMENTS, }, [OCELOT_STAT_RX_JABBERS] = { .name = "rx_jabbers", - .offset = 0x06, + .reg = SYS_COUNT_RX_JABBERS, }, [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { .name = "rx_crc_align_errs", - .offset = 0x07, + .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, }, [OCELOT_STAT_RX_SYM_ERRS] = { .name = "rx_sym_errs", - .offset = 0x08, + .reg = SYS_COUNT_RX_SYM_ERRS, }, [OCELOT_STAT_RX_64] = { .name = "rx_frames_below_65_octets", - .offset = 0x09, + .reg = SYS_COUNT_RX_64, }, [OCELOT_STAT_RX_65_127] = { .name = "rx_frames_65_to_127_octets", - .offset = 0x0A, + .reg = SYS_COUNT_RX_65_127, }, [OCELOT_STAT_RX_128_255] = { .name = "rx_frames_128_to_255_octets", - .offset = 0x0B, + .reg = SYS_COUNT_RX_128_255, }, [OCELOT_STAT_RX_256_511] = { .name = "rx_frames_256_to_511_octets", - .offset = 0x0C, + .reg = SYS_COUNT_RX_256_511, }, [OCELOT_STAT_RX_512_1023] = { .name = "rx_frames_512_to_1023_octets", - .offset = 0x0D, + .reg = SYS_COUNT_RX_512_1023, }, [OCELOT_STAT_RX_1024_1526] = { .name = "rx_frames_1024_to_1526_octets", - .offset = 0x0E, + .reg = SYS_COUNT_RX_1024_1526, }, [OCELOT_STAT_RX_1527_MAX] = { .name = "rx_frames_over_1526_octets", - .offset = 0x0F, + .reg = SYS_COUNT_RX_1527_MAX, }, [OCELOT_STAT_RX_PAUSE] = { .name = "rx_pause", - .offset = 0x10, + .reg = SYS_COUNT_RX_PAUSE, }, [OCELOT_STAT_RX_CONTROL] = { .name = "rx_control", - .offset = 0x11, + .reg = SYS_COUNT_RX_CONTROL, }, [OCELOT_STAT_RX_LONGS] = { .name = "rx_longs", - .offset = 0x12, + .reg = SYS_COUNT_RX_LONGS, }, [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { .name = "rx_classified_drops", - .offset = 0x13, + .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, }, [OCELOT_STAT_RX_RED_PRIO_0] = { .name = "rx_red_prio_0", - .offset = 0x14, + .reg = SYS_COUNT_RX_RED_PRIO_0, }, [OCELOT_STAT_RX_RED_PRIO_1] = { .name = "rx_red_prio_1", - .offset = 0x15, + .reg = SYS_COUNT_RX_RED_PRIO_1, }, [OCELOT_STAT_RX_RED_PRIO_2] = { .name = "rx_red_prio_2", - .offset = 0x16, + .reg = SYS_COUNT_RX_RED_PRIO_2, }, [OCELOT_STAT_RX_RED_PRIO_3] = { .name = "rx_red_prio_3", - .offset = 0x17, + .reg = SYS_COUNT_RX_RED_PRIO_3, }, [OCELOT_STAT_RX_RED_PRIO_4] = { .name = "rx_red_prio_4", - .offset = 0x18, + .reg = SYS_COUNT_RX_RED_PRIO_4, }, [OCELOT_STAT_RX_RED_PRIO_5] = { .name = "rx_red_prio_5", - .offset = 0x19, + .reg = SYS_COUNT_RX_RED_PRIO_5, }, [OCELOT_STAT_RX_RED_PRIO_6] = { .name = "rx_red_prio_6", - .offset = 0x1A, + .reg = SYS_COUNT_RX_RED_PRIO_6, }, [OCELOT_STAT_RX_RED_PRIO_7] = { .name = "rx_red_prio_7", - .offset = 0x1B, + .reg = SYS_COUNT_RX_RED_PRIO_7, }, [OCELOT_STAT_RX_YELLOW_PRIO_0] = { .name = "rx_yellow_prio_0", - .offset = 0x1C, + .reg = SYS_COUNT_RX_YELLOW_PRIO_0, }, [OCELOT_STAT_RX_YELLOW_PRIO_1] = { .name = "rx_yellow_prio_1", - .offset = 0x1D, + .reg = SYS_COUNT_RX_YELLOW_PRIO_1, }, [OCELOT_STAT_RX_YELLOW_PRIO_2] = { .name = "rx_yellow_prio_2", - .offset = 0x1E, + .reg = SYS_COUNT_RX_YELLOW_PRIO_2, }, [OCELOT_STAT_RX_YELLOW_PRIO_3] = { .name = "rx_yellow_prio_3", - .offset = 0x1F, + .reg = SYS_COUNT_RX_YELLOW_PRIO_3, }, [OCELOT_STAT_RX_YELLOW_PRIO_4] = { .name = "rx_yellow_prio_4", - .offset = 0x20, + .reg = SYS_COUNT_RX_YELLOW_PRIO_4, }, [OCELOT_STAT_RX_YELLOW_PRIO_5] = { .name = "rx_yellow_prio_5", - .offset = 0x21, + .reg = SYS_COUNT_RX_YELLOW_PRIO_5, }, [OCELOT_STAT_RX_YELLOW_PRIO_6] = { .name = "rx_yellow_prio_6", - .offset = 0x22, + .reg = SYS_COUNT_RX_YELLOW_PRIO_6, }, [OCELOT_STAT_RX_YELLOW_PRIO_7] = { .name = "rx_yellow_prio_7", - .offset = 0x23, + .reg = SYS_COUNT_RX_YELLOW_PRIO_7, }, [OCELOT_STAT_RX_GREEN_PRIO_0] = { .name = "rx_green_prio_0", - .offset = 0x24, + .reg = SYS_COUNT_RX_GREEN_PRIO_0, }, [OCELOT_STAT_RX_GREEN_PRIO_1] = { .name = "rx_green_prio_1", - .offset = 0x25, + .reg = SYS_COUNT_RX_GREEN_PRIO_1, }, [OCELOT_STAT_RX_GREEN_PRIO_2] = { .name = "rx_green_prio_2", - .offset = 0x26, + .reg = SYS_COUNT_RX_GREEN_PRIO_2, }, [OCELOT_STAT_RX_GREEN_PRIO_3] = { .name = "rx_green_prio_3", - .offset = 0x27, + .reg = SYS_COUNT_RX_GREEN_PRIO_3, }, [OCELOT_STAT_RX_GREEN_PRIO_4] = { .name = "rx_green_prio_4", - .offset = 0x28, + .reg = SYS_COUNT_RX_GREEN_PRIO_4, }, [OCELOT_STAT_RX_GREEN_PRIO_5] = { .name = "rx_green_prio_5", - .offset = 0x29, + .reg = SYS_COUNT_RX_GREEN_PRIO_5, }, [OCELOT_STAT_RX_GREEN_PRIO_6] = { .name = "rx_green_prio_6", - .offset = 0x2A, + .reg = SYS_COUNT_RX_GREEN_PRIO_6, }, [OCELOT_STAT_RX_GREEN_PRIO_7] = { .name = "rx_green_prio_7", - .offset = 0x2B, + .reg = SYS_COUNT_RX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_OCTETS] = { .name = "tx_octets", - .offset = 0x40, + .reg = SYS_COUNT_TX_OCTETS, }, [OCELOT_STAT_TX_UNICAST] = { .name = "tx_unicast", - .offset = 0x41, + .reg = SYS_COUNT_TX_UNICAST, }, [OCELOT_STAT_TX_MULTICAST] = { .name = "tx_multicast", - .offset = 0x42, + .reg = SYS_COUNT_TX_MULTICAST, }, [OCELOT_STAT_TX_BROADCAST] = { .name = "tx_broadcast", - .offset = 0x43, + .reg = SYS_COUNT_TX_BROADCAST, }, [OCELOT_STAT_TX_COLLISION] = { .name = "tx_collision", - .offset = 0x44, + .reg = SYS_COUNT_TX_COLLISION, }, [OCELOT_STAT_TX_DROPS] = { .name = "tx_drops", - .offset = 0x45, + .reg = SYS_COUNT_TX_DROPS, }, [OCELOT_STAT_TX_PAUSE] = { .name = "tx_pause", - .offset = 0x46, + .reg = SYS_COUNT_TX_PAUSE, }, [OCELOT_STAT_TX_64] = { .name = "tx_frames_below_65_octets", - .offset = 0x47, + .reg = SYS_COUNT_TX_64, }, [OCELOT_STAT_TX_65_127] = { .name = "tx_frames_65_to_127_octets", - .offset = 0x48, + .reg = SYS_COUNT_TX_65_127, }, [OCELOT_STAT_TX_128_255] = { .name = "tx_frames_128_255_octets", - .offset = 0x49, + .reg = SYS_COUNT_TX_128_255, }, [OCELOT_STAT_TX_256_511] = { .name = "tx_frames_256_511_octets", - .offset = 0x4A, + .reg = SYS_COUNT_TX_256_511, }, [OCELOT_STAT_TX_512_1023] = { .name = "tx_frames_512_1023_octets", - .offset = 0x4B, + .reg = SYS_COUNT_TX_512_1023, }, [OCELOT_STAT_TX_1024_1526] = { .name = "tx_frames_1024_1526_octets", - .offset = 0x4C, + .reg = SYS_COUNT_TX_1024_1526, }, [OCELOT_STAT_TX_1527_MAX] = { .name = "tx_frames_over_1526_octets", - .offset = 0x4D, + .reg = SYS_COUNT_TX_1527_MAX, }, [OCELOT_STAT_TX_YELLOW_PRIO_0] = { .name = "tx_yellow_prio_0", - .offset = 0x4E, + .reg = SYS_COUNT_TX_YELLOW_PRIO_0, }, [OCELOT_STAT_TX_YELLOW_PRIO_1] = { .name = "tx_yellow_prio_1", - .offset = 0x4F, + .reg = SYS_COUNT_TX_YELLOW_PRIO_1, }, [OCELOT_STAT_TX_YELLOW_PRIO_2] = { .name = "tx_yellow_prio_2", - .offset = 0x50, + .reg = SYS_COUNT_TX_YELLOW_PRIO_2, }, [OCELOT_STAT_TX_YELLOW_PRIO_3] = { .name = "tx_yellow_prio_3", - .offset = 0x51, + .reg = SYS_COUNT_TX_YELLOW_PRIO_3, }, [OCELOT_STAT_TX_YELLOW_PRIO_4] = { .name = "tx_yellow_prio_4", - .offset = 0x52, + .reg = SYS_COUNT_TX_YELLOW_PRIO_4, }, [OCELOT_STAT_TX_YELLOW_PRIO_5] = { .name = "tx_yellow_prio_5", - .offset = 0x53, + .reg = SYS_COUNT_TX_YELLOW_PRIO_5, }, [OCELOT_STAT_TX_YELLOW_PRIO_6] = { .name = "tx_yellow_prio_6", - .offset = 0x54, + .reg = SYS_COUNT_TX_YELLOW_PRIO_6, }, [OCELOT_STAT_TX_YELLOW_PRIO_7] = { .name = "tx_yellow_prio_7", - .offset = 0x55, + .reg = SYS_COUNT_TX_YELLOW_PRIO_7, }, [OCELOT_STAT_TX_GREEN_PRIO_0] = { .name = "tx_green_prio_0", - .offset = 0x56, + .reg = SYS_COUNT_TX_GREEN_PRIO_0, }, [OCELOT_STAT_TX_GREEN_PRIO_1] = { .name = "tx_green_prio_1", - .offset = 0x57, + .reg = SYS_COUNT_TX_GREEN_PRIO_1, }, [OCELOT_STAT_TX_GREEN_PRIO_2] = { .name = "tx_green_prio_2", - .offset = 0x58, + .reg = SYS_COUNT_TX_GREEN_PRIO_2, }, [OCELOT_STAT_TX_GREEN_PRIO_3] = { .name = "tx_green_prio_3", - .offset = 0x59, + .reg = SYS_COUNT_TX_GREEN_PRIO_3, }, [OCELOT_STAT_TX_GREEN_PRIO_4] = { .name = "tx_green_prio_4", - .offset = 0x5A, + .reg = SYS_COUNT_TX_GREEN_PRIO_4, }, [OCELOT_STAT_TX_GREEN_PRIO_5] = { .name = "tx_green_prio_5", - .offset = 0x5B, + .reg = SYS_COUNT_TX_GREEN_PRIO_5, }, [OCELOT_STAT_TX_GREEN_PRIO_6] = { .name = "tx_green_prio_6", - .offset = 0x5C, + .reg = SYS_COUNT_TX_GREEN_PRIO_6, }, [OCELOT_STAT_TX_GREEN_PRIO_7] = { .name = "tx_green_prio_7", - .offset = 0x5D, + .reg = SYS_COUNT_TX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_AGED] = { .name = "tx_aged", - .offset = 0x5E, + .reg = SYS_COUNT_TX_AGING, }, [OCELOT_STAT_DROP_LOCAL] = { .name = "drop_local", - .offset = 0x80, + .reg = SYS_COUNT_DROP_LOCAL, }, [OCELOT_STAT_DROP_TAIL] = { .name = "drop_tail", - .offset = 0x81, + .reg = SYS_COUNT_DROP_TAIL, }, [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { .name = "drop_yellow_prio_0", - .offset = 0x82, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, }, [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { .name = "drop_yellow_prio_1", - .offset = 0x83, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, }, [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { .name = "drop_yellow_prio_2", - .offset = 0x84, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, }, [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { .name = "drop_yellow_prio_3", - .offset = 0x85, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, }, [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { .name = "drop_yellow_prio_4", - .offset = 0x86, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, }, [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { .name = "drop_yellow_prio_5", - .offset = 0x87, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, }, [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { .name = "drop_yellow_prio_6", - .offset = 0x88, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, }, [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { .name = "drop_yellow_prio_7", - .offset = 0x89, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, }, [OCELOT_STAT_DROP_GREEN_PRIO_0] = { .name = "drop_green_prio_0", - .offset = 0x8A, + .reg = SYS_COUNT_DROP_GREEN_PRIO_0, }, [OCELOT_STAT_DROP_GREEN_PRIO_1] = { .name = "drop_green_prio_1", - .offset = 0x8B, + .reg = SYS_COUNT_DROP_GREEN_PRIO_1, }, [OCELOT_STAT_DROP_GREEN_PRIO_2] = { .name = "drop_green_prio_2", - .offset = 0x8C, + .reg = SYS_COUNT_DROP_GREEN_PRIO_2, }, [OCELOT_STAT_DROP_GREEN_PRIO_3] = { .name = "drop_green_prio_3", - .offset = 0x8D, + .reg = SYS_COUNT_DROP_GREEN_PRIO_3, }, [OCELOT_STAT_DROP_GREEN_PRIO_4] = { .name = "drop_green_prio_4", - .offset = 0x8E, + .reg = SYS_COUNT_DROP_GREEN_PRIO_4, }, [OCELOT_STAT_DROP_GREEN_PRIO_5] = { .name = "drop_green_prio_5", - .offset = 0x8F, + .reg = SYS_COUNT_DROP_GREEN_PRIO_5, }, [OCELOT_STAT_DROP_GREEN_PRIO_6] = { .name = "drop_green_prio_6", - .offset = 0x90, + .reg = SYS_COUNT_DROP_GREEN_PRIO_6, }, [OCELOT_STAT_DROP_GREEN_PRIO_7] = { .name = "drop_green_prio_7", - .offset = 0x91, + .reg = SYS_COUNT_DROP_GREEN_PRIO_7, }, }; diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 68991b021c56..306026e6aa11 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -1881,9 +1881,8 @@ static int ocelot_port_update_stats(struct ocelot *ocelot, int port) ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port), SYS_STAT_CFG); list_for_each_entry(region, &ocelot->stats_regions, node) { - err = ocelot_bulk_read_rix(ocelot, SYS_COUNT_RX_OCTETS, - region->offset, region->buf, - region->count); + err = ocelot_bulk_read(ocelot, region->base, region->buf, + region->count); if (err) return err; @@ -1978,7 +1977,7 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot) if (ocelot->stats_layout[i].name[0] == '\0') continue; - if (region && ocelot->stats_layout[i].offset == last + 1) { + if (region && ocelot->stats_layout[i].reg == last + 4) { region->count++; } else { region = devm_kzalloc(ocelot->dev, sizeof(*region), @@ -1986,12 +1985,12 @@ static int ocelot_prepare_stats_regions(struct ocelot *ocelot) if (!region) return -ENOMEM; - region->offset = ocelot->stats_layout[i].offset; + region->base = ocelot->stats_layout[i].reg; region->count = 1; list_add_tail(®ion->node, &ocelot->stats_regions); } - last = ocelot->stats_layout[i].offset; + last = ocelot->stats_layout[i].reg; } list_for_each_entry(region, &ocelot->stats_regions, node) { diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index 9ff910560043..9c488953f541 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -99,375 +99,375 @@ static const struct reg_field ocelot_regfields[REGFIELD_MAX] = { static const struct ocelot_stat_layout ocelot_stats_layout[OCELOT_NUM_STATS] = { [OCELOT_STAT_RX_OCTETS] = { .name = "rx_octets", - .offset = 0x00, + .reg = SYS_COUNT_RX_OCTETS, }, [OCELOT_STAT_RX_UNICAST] = { .name = "rx_unicast", - .offset = 0x01, + .reg = SYS_COUNT_RX_UNICAST, }, [OCELOT_STAT_RX_MULTICAST] = { .name = "rx_multicast", - .offset = 0x02, + .reg = SYS_COUNT_RX_MULTICAST, }, [OCELOT_STAT_RX_BROADCAST] = { .name = "rx_broadcast", - .offset = 0x03, + .reg = SYS_COUNT_RX_BROADCAST, }, [OCELOT_STAT_RX_SHORTS] = { .name = "rx_shorts", - .offset = 0x04, + .reg = SYS_COUNT_RX_SHORTS, }, [OCELOT_STAT_RX_FRAGMENTS] = { .name = "rx_fragments", - .offset = 0x05, + .reg = SYS_COUNT_RX_FRAGMENTS, }, [OCELOT_STAT_RX_JABBERS] = { .name = "rx_jabbers", - .offset = 0x06, + .reg = SYS_COUNT_RX_JABBERS, }, [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { .name = "rx_crc_align_errs", - .offset = 0x07, + .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, }, [OCELOT_STAT_RX_SYM_ERRS] = { .name = "rx_sym_errs", - .offset = 0x08, + .reg = SYS_COUNT_RX_SYM_ERRS, }, [OCELOT_STAT_RX_64] = { .name = "rx_frames_below_65_octets", - .offset = 0x09, + .reg = SYS_COUNT_RX_64, }, [OCELOT_STAT_RX_65_127] = { .name = "rx_frames_65_to_127_octets", - .offset = 0x0A, + .reg = SYS_COUNT_RX_65_127, }, [OCELOT_STAT_RX_128_255] = { .name = "rx_frames_128_to_255_octets", - .offset = 0x0B, + .reg = SYS_COUNT_RX_128_255, }, [OCELOT_STAT_RX_256_511] = { .name = "rx_frames_256_to_511_octets", - .offset = 0x0C, + .reg = SYS_COUNT_RX_256_511, }, [OCELOT_STAT_RX_512_1023] = { .name = "rx_frames_512_to_1023_octets", - .offset = 0x0D, + .reg = SYS_COUNT_RX_512_1023, }, [OCELOT_STAT_RX_1024_1526] = { .name = "rx_frames_1024_to_1526_octets", - .offset = 0x0E, + .reg = SYS_COUNT_RX_1024_1526, }, [OCELOT_STAT_RX_1527_MAX] = { .name = "rx_frames_over_1526_octets", - .offset = 0x0F, + .reg = SYS_COUNT_RX_1527_MAX, }, [OCELOT_STAT_RX_PAUSE] = { .name = "rx_pause", - .offset = 0x10, + .reg = SYS_COUNT_RX_PAUSE, }, [OCELOT_STAT_RX_CONTROL] = { .name = "rx_control", - .offset = 0x11, + .reg = SYS_COUNT_RX_CONTROL, }, [OCELOT_STAT_RX_LONGS] = { .name = "rx_longs", - .offset = 0x12, + .reg = SYS_COUNT_RX_LONGS, }, [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { .name = "rx_classified_drops", - .offset = 0x13, + .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, }, [OCELOT_STAT_RX_RED_PRIO_0] = { .name = "rx_red_prio_0", - .offset = 0x14, + .reg = SYS_COUNT_RX_RED_PRIO_0, }, [OCELOT_STAT_RX_RED_PRIO_1] = { .name = "rx_red_prio_1", - .offset = 0x15, + .reg = SYS_COUNT_RX_RED_PRIO_1, }, [OCELOT_STAT_RX_RED_PRIO_2] = { .name = "rx_red_prio_2", - .offset = 0x16, + .reg = SYS_COUNT_RX_RED_PRIO_2, }, [OCELOT_STAT_RX_RED_PRIO_3] = { .name = "rx_red_prio_3", - .offset = 0x17, + .reg = SYS_COUNT_RX_RED_PRIO_3, }, [OCELOT_STAT_RX_RED_PRIO_4] = { .name = "rx_red_prio_4", - .offset = 0x18, + .reg = SYS_COUNT_RX_RED_PRIO_4, }, [OCELOT_STAT_RX_RED_PRIO_5] = { .name = "rx_red_prio_5", - .offset = 0x19, + .reg = SYS_COUNT_RX_RED_PRIO_5, }, [OCELOT_STAT_RX_RED_PRIO_6] = { .name = "rx_red_prio_6", - .offset = 0x1A, + .reg = SYS_COUNT_RX_RED_PRIO_6, }, [OCELOT_STAT_RX_RED_PRIO_7] = { .name = "rx_red_prio_7", - .offset = 0x1B, + .reg = SYS_COUNT_RX_RED_PRIO_7, }, [OCELOT_STAT_RX_YELLOW_PRIO_0] = { .name = "rx_yellow_prio_0", - .offset = 0x1C, + .reg = SYS_COUNT_RX_YELLOW_PRIO_0, }, [OCELOT_STAT_RX_YELLOW_PRIO_1] = { .name = "rx_yellow_prio_1", - .offset = 0x1D, + .reg = SYS_COUNT_RX_YELLOW_PRIO_1, }, [OCELOT_STAT_RX_YELLOW_PRIO_2] = { .name = "rx_yellow_prio_2", - .offset = 0x1E, + .reg = SYS_COUNT_RX_YELLOW_PRIO_2, }, [OCELOT_STAT_RX_YELLOW_PRIO_3] = { .name = "rx_yellow_prio_3", - .offset = 0x1F, + .reg = SYS_COUNT_RX_YELLOW_PRIO_3, }, [OCELOT_STAT_RX_YELLOW_PRIO_4] = { .name = "rx_yellow_prio_4", - .offset = 0x20, + .reg = SYS_COUNT_RX_YELLOW_PRIO_4, }, [OCELOT_STAT_RX_YELLOW_PRIO_5] = { .name = "rx_yellow_prio_5", - .offset = 0x21, + .reg = SYS_COUNT_RX_YELLOW_PRIO_5, }, [OCELOT_STAT_RX_YELLOW_PRIO_6] = { .name = "rx_yellow_prio_6", - .offset = 0x22, + .reg = SYS_COUNT_RX_YELLOW_PRIO_6, }, [OCELOT_STAT_RX_YELLOW_PRIO_7] = { .name = "rx_yellow_prio_7", - .offset = 0x23, + .reg = SYS_COUNT_RX_YELLOW_PRIO_7, }, [OCELOT_STAT_RX_GREEN_PRIO_0] = { .name = "rx_green_prio_0", - .offset = 0x24, + .reg = SYS_COUNT_RX_GREEN_PRIO_0, }, [OCELOT_STAT_RX_GREEN_PRIO_1] = { .name = "rx_green_prio_1", - .offset = 0x25, + .reg = SYS_COUNT_RX_GREEN_PRIO_1, }, [OCELOT_STAT_RX_GREEN_PRIO_2] = { .name = "rx_green_prio_2", - .offset = 0x26, + .reg = SYS_COUNT_RX_GREEN_PRIO_2, }, [OCELOT_STAT_RX_GREEN_PRIO_3] = { .name = "rx_green_prio_3", - .offset = 0x27, + .reg = SYS_COUNT_RX_GREEN_PRIO_3, }, [OCELOT_STAT_RX_GREEN_PRIO_4] = { .name = "rx_green_prio_4", - .offset = 0x28, + .reg = SYS_COUNT_RX_GREEN_PRIO_4, }, [OCELOT_STAT_RX_GREEN_PRIO_5] = { .name = "rx_green_prio_5", - .offset = 0x29, + .reg = SYS_COUNT_RX_GREEN_PRIO_5, }, [OCELOT_STAT_RX_GREEN_PRIO_6] = { .name = "rx_green_prio_6", - .offset = 0x2A, + .reg = SYS_COUNT_RX_GREEN_PRIO_6, }, [OCELOT_STAT_RX_GREEN_PRIO_7] = { .name = "rx_green_prio_7", - .offset = 0x2B, + .reg = SYS_COUNT_RX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_OCTETS] = { .name = "tx_octets", - .offset = 0x40, + .reg = SYS_COUNT_TX_OCTETS, }, [OCELOT_STAT_TX_UNICAST] = { .name = "tx_unicast", - .offset = 0x41, + .reg = SYS_COUNT_TX_UNICAST, }, [OCELOT_STAT_TX_MULTICAST] = { .name = "tx_multicast", - .offset = 0x42, + .reg = SYS_COUNT_TX_MULTICAST, }, [OCELOT_STAT_TX_BROADCAST] = { .name = "tx_broadcast", - .offset = 0x43, + .reg = SYS_COUNT_TX_BROADCAST, }, [OCELOT_STAT_TX_COLLISION] = { .name = "tx_collision", - .offset = 0x44, + .reg = SYS_COUNT_TX_COLLISION, }, [OCELOT_STAT_TX_DROPS] = { .name = "tx_drops", - .offset = 0x45, + .reg = SYS_COUNT_TX_DROPS, }, [OCELOT_STAT_TX_PAUSE] = { .name = "tx_pause", - .offset = 0x46, + .reg = SYS_COUNT_TX_PAUSE, }, [OCELOT_STAT_TX_64] = { .name = "tx_frames_below_65_octets", - .offset = 0x47, + .reg = SYS_COUNT_TX_64, }, [OCELOT_STAT_TX_65_127] = { .name = "tx_frames_65_to_127_octets", - .offset = 0x48, + .reg = SYS_COUNT_TX_65_127, }, [OCELOT_STAT_TX_128_255] = { .name = "tx_frames_128_255_octets", - .offset = 0x49, + .reg = SYS_COUNT_TX_128_255, }, [OCELOT_STAT_TX_256_511] = { .name = "tx_frames_256_511_octets", - .offset = 0x4A, + .reg = SYS_COUNT_TX_256_511, }, [OCELOT_STAT_TX_512_1023] = { .name = "tx_frames_512_1023_octets", - .offset = 0x4B, + .reg = SYS_COUNT_TX_512_1023, }, [OCELOT_STAT_TX_1024_1526] = { .name = "tx_frames_1024_1526_octets", - .offset = 0x4C, + .reg = SYS_COUNT_TX_1024_1526, }, [OCELOT_STAT_TX_1527_MAX] = { .name = "tx_frames_over_1526_octets", - .offset = 0x4D, + .reg = SYS_COUNT_TX_1527_MAX, }, [OCELOT_STAT_TX_YELLOW_PRIO_0] = { .name = "tx_yellow_prio_0", - .offset = 0x4E, + .reg = SYS_COUNT_TX_YELLOW_PRIO_0, }, [OCELOT_STAT_TX_YELLOW_PRIO_1] = { .name = "tx_yellow_prio_1", - .offset = 0x4F, + .reg = SYS_COUNT_TX_YELLOW_PRIO_1, }, [OCELOT_STAT_TX_YELLOW_PRIO_2] = { .name = "tx_yellow_prio_2", - .offset = 0x50, + .reg = SYS_COUNT_TX_YELLOW_PRIO_2, }, [OCELOT_STAT_TX_YELLOW_PRIO_3] = { .name = "tx_yellow_prio_3", - .offset = 0x51, + .reg = SYS_COUNT_TX_YELLOW_PRIO_3, }, [OCELOT_STAT_TX_YELLOW_PRIO_4] = { .name = "tx_yellow_prio_4", - .offset = 0x52, + .reg = SYS_COUNT_TX_YELLOW_PRIO_4, }, [OCELOT_STAT_TX_YELLOW_PRIO_5] = { .name = "tx_yellow_prio_5", - .offset = 0x53, + .reg = SYS_COUNT_TX_YELLOW_PRIO_5, }, [OCELOT_STAT_TX_YELLOW_PRIO_6] = { .name = "tx_yellow_prio_6", - .offset = 0x54, + .reg = SYS_COUNT_TX_YELLOW_PRIO_6, }, [OCELOT_STAT_TX_YELLOW_PRIO_7] = { .name = "tx_yellow_prio_7", - .offset = 0x55, + .reg = SYS_COUNT_TX_YELLOW_PRIO_7, }, [OCELOT_STAT_TX_GREEN_PRIO_0] = { .name = "tx_green_prio_0", - .offset = 0x56, + .reg = SYS_COUNT_TX_GREEN_PRIO_0, }, [OCELOT_STAT_TX_GREEN_PRIO_1] = { .name = "tx_green_prio_1", - .offset = 0x57, + .reg = SYS_COUNT_TX_GREEN_PRIO_1, }, [OCELOT_STAT_TX_GREEN_PRIO_2] = { .name = "tx_green_prio_2", - .offset = 0x58, + .reg = SYS_COUNT_TX_GREEN_PRIO_2, }, [OCELOT_STAT_TX_GREEN_PRIO_3] = { .name = "tx_green_prio_3", - .offset = 0x59, + .reg = SYS_COUNT_TX_GREEN_PRIO_3, }, [OCELOT_STAT_TX_GREEN_PRIO_4] = { .name = "tx_green_prio_4", - .offset = 0x5A, + .reg = SYS_COUNT_TX_GREEN_PRIO_4, }, [OCELOT_STAT_TX_GREEN_PRIO_5] = { .name = "tx_green_prio_5", - .offset = 0x5B, + .reg = SYS_COUNT_TX_GREEN_PRIO_5, }, [OCELOT_STAT_TX_GREEN_PRIO_6] = { .name = "tx_green_prio_6", - .offset = 0x5C, + .reg = SYS_COUNT_TX_GREEN_PRIO_6, }, [OCELOT_STAT_TX_GREEN_PRIO_7] = { .name = "tx_green_prio_7", - .offset = 0x5D, + .reg = SYS_COUNT_TX_GREEN_PRIO_7, }, [OCELOT_STAT_TX_AGED] = { .name = "tx_aged", - .offset = 0x5E, + .reg = SYS_COUNT_TX_AGING, }, [OCELOT_STAT_DROP_LOCAL] = { .name = "drop_local", - .offset = 0x80, + .reg = SYS_COUNT_DROP_LOCAL, }, [OCELOT_STAT_DROP_TAIL] = { .name = "drop_tail", - .offset = 0x81, + .reg = SYS_COUNT_DROP_TAIL, }, [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { .name = "drop_yellow_prio_0", - .offset = 0x82, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, }, [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { .name = "drop_yellow_prio_1", - .offset = 0x83, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, }, [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { .name = "drop_yellow_prio_2", - .offset = 0x84, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, }, [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { .name = "drop_yellow_prio_3", - .offset = 0x85, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, }, [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { .name = "drop_yellow_prio_4", - .offset = 0x86, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, }, [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { .name = "drop_yellow_prio_5", - .offset = 0x87, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, }, [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { .name = "drop_yellow_prio_6", - .offset = 0x88, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, }, [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { .name = "drop_yellow_prio_7", - .offset = 0x89, + .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, }, [OCELOT_STAT_DROP_GREEN_PRIO_0] = { .name = "drop_green_prio_0", - .offset = 0x8A, + .reg = SYS_COUNT_DROP_GREEN_PRIO_0, }, [OCELOT_STAT_DROP_GREEN_PRIO_1] = { .name = "drop_green_prio_1", - .offset = 0x8B, + .reg = SYS_COUNT_DROP_GREEN_PRIO_1, }, [OCELOT_STAT_DROP_GREEN_PRIO_2] = { .name = "drop_green_prio_2", - .offset = 0x8C, + .reg = SYS_COUNT_DROP_GREEN_PRIO_2, }, [OCELOT_STAT_DROP_GREEN_PRIO_3] = { .name = "drop_green_prio_3", - .offset = 0x8D, + .reg = SYS_COUNT_DROP_GREEN_PRIO_3, }, [OCELOT_STAT_DROP_GREEN_PRIO_4] = { .name = "drop_green_prio_4", - .offset = 0x8E, + .reg = SYS_COUNT_DROP_GREEN_PRIO_4, }, [OCELOT_STAT_DROP_GREEN_PRIO_5] = { .name = "drop_green_prio_5", - .offset = 0x8F, + .reg = SYS_COUNT_DROP_GREEN_PRIO_5, }, [OCELOT_STAT_DROP_GREEN_PRIO_6] = { .name = "drop_green_prio_6", - .offset = 0x90, + .reg = SYS_COUNT_DROP_GREEN_PRIO_6, }, [OCELOT_STAT_DROP_GREEN_PRIO_7] = { .name = "drop_green_prio_7", - .offset = 0x91, + .reg = SYS_COUNT_DROP_GREEN_PRIO_7, }, }; diff --git a/drivers/net/ethernet/mscc/vsc7514_regs.c b/drivers/net/ethernet/mscc/vsc7514_regs.c index 8ff935f7f150..9cf82ecf191c 100644 --- a/drivers/net/ethernet/mscc/vsc7514_regs.c +++ b/drivers/net/ethernet/mscc/vsc7514_regs.c @@ -188,6 +188,30 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_COUNT_RX_CONTROL, 0x000044), REG(SYS_COUNT_RX_LONGS, 0x000048), REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x00004c), + REG(SYS_COUNT_RX_RED_PRIO_0, 0x000050), + REG(SYS_COUNT_RX_RED_PRIO_1, 0x000054), + REG(SYS_COUNT_RX_RED_PRIO_2, 0x000058), + REG(SYS_COUNT_RX_RED_PRIO_3, 0x00005c), + REG(SYS_COUNT_RX_RED_PRIO_4, 0x000060), + REG(SYS_COUNT_RX_RED_PRIO_5, 0x000064), + REG(SYS_COUNT_RX_RED_PRIO_6, 0x000068), + REG(SYS_COUNT_RX_RED_PRIO_7, 0x00006c), + REG(SYS_COUNT_RX_YELLOW_PRIO_0, 0x000070), + REG(SYS_COUNT_RX_YELLOW_PRIO_1, 0x000074), + REG(SYS_COUNT_RX_YELLOW_PRIO_2, 0x000078), + REG(SYS_COUNT_RX_YELLOW_PRIO_3, 0x00007c), + REG(SYS_COUNT_RX_YELLOW_PRIO_4, 0x000080), + REG(SYS_COUNT_RX_YELLOW_PRIO_5, 0x000084), + REG(SYS_COUNT_RX_YELLOW_PRIO_6, 0x000088), + REG(SYS_COUNT_RX_YELLOW_PRIO_7, 0x00008c), + REG(SYS_COUNT_RX_GREEN_PRIO_0, 0x000090), + REG(SYS_COUNT_RX_GREEN_PRIO_1, 0x000094), + REG(SYS_COUNT_RX_GREEN_PRIO_2, 0x000098), + REG(SYS_COUNT_RX_GREEN_PRIO_3, 0x00009c), + REG(SYS_COUNT_RX_GREEN_PRIO_4, 0x0000a0), + REG(SYS_COUNT_RX_GREEN_PRIO_5, 0x0000a4), + REG(SYS_COUNT_RX_GREEN_PRIO_6, 0x0000a8), + REG(SYS_COUNT_RX_GREEN_PRIO_7, 0x0000ac), REG(SYS_COUNT_TX_OCTETS, 0x000100), REG(SYS_COUNT_TX_UNICAST, 0x000104), REG(SYS_COUNT_TX_MULTICAST, 0x000108), @@ -202,7 +226,41 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_COUNT_TX_512_1023, 0x00012c), REG(SYS_COUNT_TX_1024_1526, 0x000130), REG(SYS_COUNT_TX_1527_MAX, 0x000134), + REG(SYS_COUNT_TX_YELLOW_PRIO_0, 0x000138), + REG(SYS_COUNT_TX_YELLOW_PRIO_1, 0x00013c), + REG(SYS_COUNT_TX_YELLOW_PRIO_2, 0x000140), + REG(SYS_COUNT_TX_YELLOW_PRIO_3, 0x000144), + REG(SYS_COUNT_TX_YELLOW_PRIO_4, 0x000148), + REG(SYS_COUNT_TX_YELLOW_PRIO_5, 0x00014c), + REG(SYS_COUNT_TX_YELLOW_PRIO_6, 0x000150), + REG(SYS_COUNT_TX_YELLOW_PRIO_7, 0x000154), + REG(SYS_COUNT_TX_GREEN_PRIO_0, 0x000158), + REG(SYS_COUNT_TX_GREEN_PRIO_1, 0x00015c), + REG(SYS_COUNT_TX_GREEN_PRIO_2, 0x000160), + REG(SYS_COUNT_TX_GREEN_PRIO_3, 0x000164), + REG(SYS_COUNT_TX_GREEN_PRIO_4, 0x000168), + REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00016c), + REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000170), + REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000174), REG(SYS_COUNT_TX_AGING, 0x000178), + REG(SYS_COUNT_DROP_LOCAL, 0x000200), + REG(SYS_COUNT_DROP_TAIL, 0x000204), + REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000208), + REG(SYS_COUNT_DROP_YELLOW_PRIO_1, 0x00020c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_2, 0x000210), + REG(SYS_COUNT_DROP_YELLOW_PRIO_3, 0x000214), + REG(SYS_COUNT_DROP_YELLOW_PRIO_4, 0x000218), + REG(SYS_COUNT_DROP_YELLOW_PRIO_5, 0x00021c), + REG(SYS_COUNT_DROP_YELLOW_PRIO_6, 0x000220), + REG(SYS_COUNT_DROP_YELLOW_PRIO_7, 0x000214), + REG(SYS_COUNT_DROP_GREEN_PRIO_0, 0x000218), + REG(SYS_COUNT_DROP_GREEN_PRIO_1, 0x00021c), + REG(SYS_COUNT_DROP_GREEN_PRIO_2, 0x000220), + REG(SYS_COUNT_DROP_GREEN_PRIO_3, 0x000224), + REG(SYS_COUNT_DROP_GREEN_PRIO_4, 0x000228), + REG(SYS_COUNT_DROP_GREEN_PRIO_5, 0x00022c), + REG(SYS_COUNT_DROP_GREEN_PRIO_6, 0x000230), + REG(SYS_COUNT_DROP_GREEN_PRIO_7, 0x000234), REG(SYS_RESET_CFG, 0x000508), REG(SYS_CMID, 0x00050c), REG(SYS_VLAN_ETYPE_CFG, 0x000510), diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index 2428bc64cb1d..2edea901bbd5 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -338,6 +338,30 @@ enum ocelot_reg { SYS_COUNT_RX_CONTROL, SYS_COUNT_RX_LONGS, SYS_COUNT_RX_CLASSIFIED_DROPS, + SYS_COUNT_RX_RED_PRIO_0, + SYS_COUNT_RX_RED_PRIO_1, + SYS_COUNT_RX_RED_PRIO_2, + SYS_COUNT_RX_RED_PRIO_3, + SYS_COUNT_RX_RED_PRIO_4, + SYS_COUNT_RX_RED_PRIO_5, + SYS_COUNT_RX_RED_PRIO_6, + SYS_COUNT_RX_RED_PRIO_7, + SYS_COUNT_RX_YELLOW_PRIO_0, + SYS_COUNT_RX_YELLOW_PRIO_1, + SYS_COUNT_RX_YELLOW_PRIO_2, + SYS_COUNT_RX_YELLOW_PRIO_3, + SYS_COUNT_RX_YELLOW_PRIO_4, + SYS_COUNT_RX_YELLOW_PRIO_5, + SYS_COUNT_RX_YELLOW_PRIO_6, + SYS_COUNT_RX_YELLOW_PRIO_7, + SYS_COUNT_RX_GREEN_PRIO_0, + SYS_COUNT_RX_GREEN_PRIO_1, + SYS_COUNT_RX_GREEN_PRIO_2, + SYS_COUNT_RX_GREEN_PRIO_3, + SYS_COUNT_RX_GREEN_PRIO_4, + SYS_COUNT_RX_GREEN_PRIO_5, + SYS_COUNT_RX_GREEN_PRIO_6, + SYS_COUNT_RX_GREEN_PRIO_7, SYS_COUNT_TX_OCTETS, SYS_COUNT_TX_UNICAST, SYS_COUNT_TX_MULTICAST, @@ -352,7 +376,41 @@ enum ocelot_reg { SYS_COUNT_TX_512_1023, SYS_COUNT_TX_1024_1526, SYS_COUNT_TX_1527_MAX, + SYS_COUNT_TX_YELLOW_PRIO_0, + SYS_COUNT_TX_YELLOW_PRIO_1, + SYS_COUNT_TX_YELLOW_PRIO_2, + SYS_COUNT_TX_YELLOW_PRIO_3, + SYS_COUNT_TX_YELLOW_PRIO_4, + SYS_COUNT_TX_YELLOW_PRIO_5, + SYS_COUNT_TX_YELLOW_PRIO_6, + SYS_COUNT_TX_YELLOW_PRIO_7, + SYS_COUNT_TX_GREEN_PRIO_0, + SYS_COUNT_TX_GREEN_PRIO_1, + SYS_COUNT_TX_GREEN_PRIO_2, + SYS_COUNT_TX_GREEN_PRIO_3, + SYS_COUNT_TX_GREEN_PRIO_4, + SYS_COUNT_TX_GREEN_PRIO_5, + SYS_COUNT_TX_GREEN_PRIO_6, + SYS_COUNT_TX_GREEN_PRIO_7, SYS_COUNT_TX_AGING, + SYS_COUNT_DROP_LOCAL, + SYS_COUNT_DROP_TAIL, + SYS_COUNT_DROP_YELLOW_PRIO_0, + SYS_COUNT_DROP_YELLOW_PRIO_1, + SYS_COUNT_DROP_YELLOW_PRIO_2, + SYS_COUNT_DROP_YELLOW_PRIO_3, + SYS_COUNT_DROP_YELLOW_PRIO_4, + SYS_COUNT_DROP_YELLOW_PRIO_5, + SYS_COUNT_DROP_YELLOW_PRIO_6, + SYS_COUNT_DROP_YELLOW_PRIO_7, + SYS_COUNT_DROP_GREEN_PRIO_0, + SYS_COUNT_DROP_GREEN_PRIO_1, + SYS_COUNT_DROP_GREEN_PRIO_2, + SYS_COUNT_DROP_GREEN_PRIO_3, + SYS_COUNT_DROP_GREEN_PRIO_4, + SYS_COUNT_DROP_GREEN_PRIO_5, + SYS_COUNT_DROP_GREEN_PRIO_6, + SYS_COUNT_DROP_GREEN_PRIO_7, SYS_RESET_CFG, SYS_SR_ETYPE_CFG, SYS_VLAN_ETYPE_CFG, @@ -633,13 +691,13 @@ enum ocelot_stat { }; struct ocelot_stat_layout { - u32 offset; + u32 reg; char name[ETH_GSTRING_LEN]; }; struct ocelot_stats_region { struct list_head node; - u32 offset; + u32 base; int count; u32 *buf; }; @@ -877,8 +935,8 @@ struct ocelot_policer { u32 burst; /* bytes */ }; -#define ocelot_bulk_read_rix(ocelot, reg, ri, buf, count) \ - __ocelot_bulk_read_ix(ocelot, reg, reg##_RSZ * (ri), buf, count) +#define ocelot_bulk_read(ocelot, reg, buf, count) \ + __ocelot_bulk_read_ix(ocelot, reg, 0, buf, count) #define ocelot_read_ix(ocelot, reg, gi, ri) \ __ocelot_read_ix(ocelot, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri)) -- cgit v1.3-15-g9ac1