From 2d0e30c30f84d08dc16f0f2af41f1b8a85f0755e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 21 Oct 2016 12:46:33 +0200 Subject: bpf: add helper for retrieving current numa node id Use case is mainly for soreuseport to select sockets for the local numa node, but since generic, lets also add this for other networking and tracing program types. Suggested-by: Eric Dumazet Signed-off-by: Daniel Borkmann Acked-by: Alexei Starovoitov Acked-by: Eric Dumazet Signed-off-by: David S. Miller --- kernel/trace/bpf_trace.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 5dcb99281259..fa77311dadb2 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -422,6 +422,8 @@ static const struct bpf_func_proto *tracing_func_proto(enum bpf_func_id func_id) return bpf_get_trace_printk_proto(); case BPF_FUNC_get_smp_processor_id: return &bpf_get_smp_processor_id_proto; + case BPF_FUNC_get_numa_node_id: + return &bpf_get_numa_node_id_proto; case BPF_FUNC_perf_event_read: return &bpf_perf_event_read_proto; case BPF_FUNC_probe_write_user: -- cgit v1.3-7-g2ca7 From ef295ecf090d3e86e5b742fc6ab34f1122a43773 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 28 Oct 2016 08:48:16 -0600 Subject: block: better op and flags encoding Now that we don't need the common flags to overflow outside the range of a 32-bit type we can encode them the same way for both the bio and request fields. This in addition allows us to place the operation first (and make some room for more ops while we're at it) and to stop having to shift around the operation values. In addition this allows passing around only one value in the block layer instead of two (and eventuall also in the file systems, but we can do that later) and thus clean up a lot of code. Last but not least this allows decreasing the size of the cmd_flags field in struct request to 32-bits. Various functions passing this value could also be updated, but I'd like to avoid the churn for now. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- Documentation/block/biodoc.txt | 4 +- block/blk-core.c | 60 ++++++++++-------------------- block/blk-flush.c | 2 +- block/blk-lib.c | 2 +- block/blk-map.c | 2 + block/blk-mq.c | 28 ++++++-------- block/cfq-iosched.c | 66 ++++++++++++++++----------------- block/elevator.c | 4 +- drivers/md/dm-crypt.c | 2 +- drivers/scsi/sd.c | 3 +- fs/btrfs/inode.c | 5 +-- fs/buffer.c | 2 +- fs/f2fs/f2fs.h | 2 +- fs/gfs2/lops.c | 2 +- include/linux/blk-cgroup.h | 11 +++--- include/linux/blk_types.h | 83 +++++++++++++++++++----------------------- include/linux/blkdev.h | 26 +------------ include/linux/blktrace_api.h | 2 +- include/linux/dm-io.h | 2 +- include/linux/elevator.h | 4 +- include/trace/events/bcache.h | 12 ++---- include/trace/events/block.h | 31 ++++++---------- kernel/trace/blktrace.c | 14 +++---- 23 files changed, 148 insertions(+), 221 deletions(-) (limited to 'kernel/trace') diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt index 6acea160298c..01ddeaf64b0f 100644 --- a/Documentation/block/biodoc.txt +++ b/Documentation/block/biodoc.txt @@ -553,8 +553,8 @@ struct request { struct request_list *rl; } -See the rq_flag_bits definitions for an explanation of the various flags -available. Some bits are used by the block layer or i/o scheduler. +See the req_ops and req_flag_bits definitions for an explanation of the various +flags available. Some bits are used by the block layer or i/o scheduler. The behaviour of the various sector counts are almost the same as before, except that since we have multi-segment bios, current_nr_sectors refers diff --git a/block/blk-core.c b/block/blk-core.c index fd416651a676..0bfaa54d3e9f 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1056,8 +1056,7 @@ static struct io_context *rq_ioc(struct bio *bio) /** * __get_request - get a free request * @rl: request list to allocate from - * @op: REQ_OP_READ/REQ_OP_WRITE - * @op_flags: rq_flag_bits + * @op: operation and flags * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1068,23 +1067,22 @@ static struct io_context *rq_ioc(struct bio *bio) * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *__get_request(struct request_list *rl, int op, - int op_flags, struct bio *bio, - gfp_t gfp_mask) +static struct request *__get_request(struct request_list *rl, unsigned int op, + struct bio *bio, gfp_t gfp_mask) { struct request_queue *q = rl->q; struct request *rq; struct elevator_type *et = q->elevator->type; struct io_context *ioc = rq_ioc(bio); struct io_cq *icq = NULL; - const bool is_sync = rw_is_sync(op, op_flags) != 0; + const bool is_sync = op_is_sync(op); int may_queue; req_flags_t rq_flags = RQF_ALLOCED; if (unlikely(blk_queue_dying(q))) return ERR_PTR(-ENODEV); - may_queue = elv_may_queue(q, op, op_flags); + may_queue = elv_may_queue(q, op); if (may_queue == ELV_MQUEUE_NO) goto rq_starved; @@ -1154,7 +1152,7 @@ static struct request *__get_request(struct request_list *rl, int op, blk_rq_init(q, rq); blk_rq_set_rl(rq, rl); - req_set_op_attrs(rq, op, op_flags); + rq->cmd_flags = op; rq->rq_flags = rq_flags; /* init elvpriv */ @@ -1232,8 +1230,7 @@ rq_starved: /** * get_request - get a free request * @q: request_queue to allocate request from - * @op: REQ_OP_READ/REQ_OP_WRITE - * @op_flags: rq_flag_bits + * @op: operation and flags * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * @@ -1244,18 +1241,17 @@ rq_starved: * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ -static struct request *get_request(struct request_queue *q, int op, - int op_flags, struct bio *bio, - gfp_t gfp_mask) +static struct request *get_request(struct request_queue *q, unsigned int op, + struct bio *bio, gfp_t gfp_mask) { - const bool is_sync = rw_is_sync(op, op_flags) != 0; + const bool is_sync = op_is_sync(op); DEFINE_WAIT(wait); struct request_list *rl; struct request *rq; rl = blk_get_rl(q, bio); /* transferred to @rq on success */ retry: - rq = __get_request(rl, op, op_flags, bio, gfp_mask); + rq = __get_request(rl, op, bio, gfp_mask); if (!IS_ERR(rq)) return rq; @@ -1297,7 +1293,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, create_io_context(gfp_mask, q->node); spin_lock_irq(q->queue_lock); - rq = get_request(q, rw, 0, NULL, gfp_mask); + rq = get_request(q, rw, NULL, gfp_mask); if (IS_ERR(rq)) { spin_unlock_irq(q->queue_lock); return rq; @@ -1446,7 +1442,7 @@ void __blk_put_request(struct request_queue *q, struct request *req) */ if (rq_flags & RQF_ALLOCED) { struct request_list *rl = blk_rq_rl(req); - bool sync = rw_is_sync(req_op(req), req->cmd_flags); + bool sync = op_is_sync(req->cmd_flags); BUG_ON(!list_empty(&req->queuelist)); BUG_ON(ELV_ON_HASH(req)); @@ -1652,8 +1648,6 @@ out: void init_request_from_bio(struct request *req, struct bio *bio) { req->cmd_type = REQ_TYPE_FS; - - req->cmd_flags |= bio->bi_opf & REQ_COMMON_MASK; if (bio->bi_opf & REQ_RAHEAD) req->cmd_flags |= REQ_FAILFAST_MASK; @@ -1665,9 +1659,8 @@ void init_request_from_bio(struct request *req, struct bio *bio) static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) { - const bool sync = !!(bio->bi_opf & REQ_SYNC); struct blk_plug *plug; - int el_ret, rw_flags = 0, where = ELEVATOR_INSERT_SORT; + int el_ret, where = ELEVATOR_INSERT_SORT; struct request *req; unsigned int request_count = 0; @@ -1722,24 +1715,11 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) } get_rq: - /* - * This sync check and mask will be re-done in init_request_from_bio(), - * but we need to set it earlier to expose the sync flag to the - * rq allocator and io schedulers. - */ - if (sync) - rw_flags |= REQ_SYNC; - - /* - * Add in META/PRIO flags, if set, before we get to the IO scheduler - */ - rw_flags |= (bio->bi_opf & (REQ_META | REQ_PRIO)); - /* * Grab a free request. This is might sleep but can not fail. * Returns with the queue unlocked. */ - req = get_request(q, bio_data_dir(bio), rw_flags, bio, GFP_NOIO); + req = get_request(q, bio->bi_opf, bio, GFP_NOIO); if (IS_ERR(req)) { bio->bi_error = PTR_ERR(req); bio_endio(bio); @@ -2946,8 +2926,6 @@ EXPORT_SYMBOL_GPL(__blk_end_request_err); void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio) { - req_set_op(rq, bio_op(bio)); - if (bio_has_data(bio)) rq->nr_phys_segments = bio_phys_segments(q, bio); @@ -3031,8 +3009,7 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); static void __blk_rq_prep_clone(struct request *dst, struct request *src) { dst->cpu = src->cpu; - req_set_op_attrs(dst, req_op(src), - (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE); + dst->cmd_flags = src->cmd_flags | REQ_NOMERGE; dst->cmd_type = src->cmd_type; dst->__sector = blk_rq_pos(src); dst->__data_len = blk_rq_bytes(src); @@ -3537,8 +3514,11 @@ EXPORT_SYMBOL(blk_set_runtime_active); int __init blk_dev_init(void) { - BUILD_BUG_ON(__REQ_NR_BITS > 8 * + BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); + BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * FIELD_SIZEOF(struct request, cmd_flags)); + BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * + FIELD_SIZEOF(struct bio, bi_opf)); /* used for unplugging and affects IO latency/throughput - HIGHPRI */ kblockd_workqueue = alloc_workqueue("kblockd", diff --git a/block/blk-flush.c b/block/blk-flush.c index 3990b9cfbda5..95f1d4d357df 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -330,7 +330,7 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) } flush_rq->cmd_type = REQ_TYPE_FS; - req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH); + flush_rq->cmd_flags = REQ_OP_FLUSH | WRITE_FLUSH; flush_rq->rq_flags |= RQF_FLUSH_SEQ; flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io; diff --git a/block/blk-lib.c b/block/blk-lib.c index 46fe9248410d..18abda862915 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -29,7 +29,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = *biop; unsigned int granularity; - enum req_op op; + unsigned int op; int alignment; sector_t bs_mask; diff --git a/block/blk-map.c b/block/blk-map.c index 2c5ae5fef473..0173a72a8aa9 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -16,6 +16,8 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) { if (!rq->bio) { + rq->cmd_flags &= REQ_OP_MASK; + rq->cmd_flags |= (bio->bi_opf & REQ_OP_MASK); blk_rq_bio_prep(rq->q, rq, bio); } else { if (!ll_back_merge_fn(rq->q, rq, bio)) diff --git a/block/blk-mq.c b/block/blk-mq.c index b49c6658eb05..2da1a0ee3318 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -139,14 +139,13 @@ bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) EXPORT_SYMBOL(blk_mq_can_queue); static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, - struct request *rq, int op, - unsigned int op_flags) + struct request *rq, unsigned int op) { INIT_LIST_HEAD(&rq->queuelist); /* csd/requeue_work/fifo_time is initialized before use */ rq->q = q; rq->mq_ctx = ctx; - req_set_op_attrs(rq, op, op_flags); + rq->cmd_flags = op; if (blk_queue_io_stat(q)) rq->rq_flags |= RQF_IO_STAT; /* do not touch atomic flags, it needs atomic ops against the timer */ @@ -183,11 +182,11 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, rq->end_io_data = NULL; rq->next_rq = NULL; - ctx->rq_dispatched[rw_is_sync(op, op_flags)]++; + ctx->rq_dispatched[op_is_sync(op)]++; } static struct request * -__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags) +__blk_mq_alloc_request(struct blk_mq_alloc_data *data, unsigned int op) { struct request *rq; unsigned int tag; @@ -202,7 +201,7 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags) } rq->tag = tag; - blk_mq_rq_ctx_init(data->q, data->ctx, rq, op, op_flags); + blk_mq_rq_ctx_init(data->q, data->ctx, rq, op); return rq; } @@ -225,7 +224,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, ctx = blk_mq_get_ctx(q); hctx = blk_mq_map_queue(q, ctx->cpu); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw, 0); + rq = __blk_mq_alloc_request(&alloc_data, rw); blk_mq_put_ctx(ctx); if (!rq) { @@ -277,7 +276,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw, ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask)); blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); - rq = __blk_mq_alloc_request(&alloc_data, rw, 0); + rq = __blk_mq_alloc_request(&alloc_data, rw); if (!rq) { ret = -EWOULDBLOCK; goto out_queue_exit; @@ -1196,19 +1195,14 @@ static struct request *blk_mq_map_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx; struct blk_mq_ctx *ctx; struct request *rq; - int op = bio_data_dir(bio); - int op_flags = 0; blk_queue_enter_live(q); ctx = blk_mq_get_ctx(q); hctx = blk_mq_map_queue(q, ctx->cpu); - if (rw_is_sync(bio_op(bio), bio->bi_opf)) - op_flags |= REQ_SYNC; - - trace_block_getrq(q, bio, op); + trace_block_getrq(q, bio, bio->bi_opf); blk_mq_set_alloc_data(data, q, 0, ctx, hctx); - rq = __blk_mq_alloc_request(data, op, op_flags); + rq = __blk_mq_alloc_request(data, bio->bi_opf); data->hctx->queued++; return rq; @@ -1256,7 +1250,7 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie) */ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio_op(bio), bio->bi_opf); + const int is_sync = op_is_sync(bio->bi_opf); const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA); struct blk_mq_alloc_data data; struct request *rq; @@ -1350,7 +1344,7 @@ done: */ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio) { - const int is_sync = rw_is_sync(bio_op(bio), bio->bi_opf); + const int is_sync = op_is_sync(bio->bi_opf); const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA); struct blk_plug *plug; unsigned int request_count = 0; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 5e24d880306c..c96186adaa66 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -667,10 +667,10 @@ static inline void cfqg_put(struct cfq_group *cfqg) } while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int op, - int op_flags) + struct cfq_group *curr_cfqg, + unsigned int op) { - blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, 1); + blkg_rwstat_add(&cfqg->stats.queued, op, 1); cfqg_stats_end_empty_time(&cfqg->stats); cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg); } @@ -684,30 +684,29 @@ static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, #endif } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, - int op_flags) +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, + unsigned int op) { - blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, -1); + blkg_rwstat_add(&cfqg->stats.queued, op, -1); } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, - int op_flags) +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, + unsigned int op) { - blkg_rwstat_add(&cfqg->stats.merged, op, op_flags, 1); + blkg_rwstat_add(&cfqg->stats.merged, op, 1); } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int op, - int op_flags) + uint64_t start_time, uint64_t io_start_time, + unsigned int op) { struct cfqg_stats *stats = &cfqg->stats; unsigned long long now = sched_clock(); if (time_after64(now, io_start_time)) - blkg_rwstat_add(&stats->service_time, op, op_flags, - now - io_start_time); + blkg_rwstat_add(&stats->service_time, op, now - io_start_time); if (time_after64(io_start_time, start_time)) - blkg_rwstat_add(&stats->wait_time, op, op_flags, + blkg_rwstat_add(&stats->wait_time, op, io_start_time - start_time); } @@ -786,16 +785,16 @@ static inline void cfqg_put(struct cfq_group *cfqg) { } #define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0) static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg, - struct cfq_group *curr_cfqg, int op, int op_flags) { } + struct cfq_group *curr_cfqg, unsigned int op) { } static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, uint64_t time, unsigned long unaccounted_time) { } -static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op, - int op_flags) { } -static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op, - int op_flags) { } +static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, + unsigned int op) { } +static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, + unsigned int op) { } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, - uint64_t start_time, uint64_t io_start_time, int op, - int op_flags) { } + uint64_t start_time, uint64_t io_start_time, + unsigned int op) { } #endif /* CONFIG_CFQ_GROUP_IOSCHED */ @@ -2474,10 +2473,10 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) { elv_rb_del(&cfqq->sort_list, rq); cfqq->queued[rq_is_sync(rq)]--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); cfq_add_rq_rb(rq); cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group, - req_op(rq), rq->cmd_flags); + rq->cmd_flags); } static struct request * @@ -2530,7 +2529,7 @@ static void cfq_remove_request(struct request *rq) cfq_del_rq_rb(rq); cfqq->cfqd->rq_queued--; - cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags); + cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags); if (rq->cmd_flags & REQ_PRIO) { WARN_ON(!cfqq->prio_pending); cfqq->prio_pending--; @@ -2565,7 +2564,7 @@ static void cfq_merged_request(struct request_queue *q, struct request *req, static void cfq_bio_merged(struct request_queue *q, struct request *req, struct bio *bio) { - cfqg_stats_update_io_merged(RQ_CFQG(req), bio_op(bio), bio->bi_opf); + cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_opf); } static void @@ -2588,7 +2587,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, if (cfqq->next_rq == next) cfqq->next_rq = rq; cfq_remove_request(next); - cfqg_stats_update_io_merged(RQ_CFQG(rq), req_op(next), next->cmd_flags); + cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags); cfqq = RQ_CFQQ(next); /* @@ -4142,7 +4141,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)]; list_add_tail(&rq->queuelist, &cfqq->fifo); cfq_add_rq_rb(rq); - cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq), + cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, rq->cmd_flags); cfq_rq_enqueued(cfqd, cfqq, rq); } @@ -4240,8 +4239,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfqq->dispatched--; (RQ_CFQG(rq))->dispatched--; cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq), - rq_io_start_time_ns(rq), req_op(rq), - rq->cmd_flags); + rq_io_start_time_ns(rq), rq->cmd_flags); cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; @@ -4319,14 +4317,14 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfq_schedule_dispatch(cfqd); } -static void cfqq_boost_on_prio(struct cfq_queue *cfqq, int op_flags) +static void cfqq_boost_on_prio(struct cfq_queue *cfqq, unsigned int op) { /* * If REQ_PRIO is set, boost class and prio level, if it's below * BE/NORM. If prio is not set, restore the potentially boosted * class/prio level. */ - if (!(op_flags & REQ_PRIO)) { + if (!(op & REQ_PRIO)) { cfqq->ioprio_class = cfqq->org_ioprio_class; cfqq->ioprio = cfqq->org_ioprio; } else { @@ -4347,7 +4345,7 @@ static inline int __cfq_may_queue(struct cfq_queue *cfqq) return ELV_MQUEUE_MAY; } -static int cfq_may_queue(struct request_queue *q, int op, int op_flags) +static int cfq_may_queue(struct request_queue *q, unsigned int op) { struct cfq_data *cfqd = q->elevator->elevator_data; struct task_struct *tsk = current; @@ -4364,10 +4362,10 @@ static int cfq_may_queue(struct request_queue *q, int op, int op_flags) if (!cic) return ELV_MQUEUE_MAY; - cfqq = cic_to_cfqq(cic, rw_is_sync(op, op_flags)); + cfqq = cic_to_cfqq(cic, op_is_sync(op)); if (cfqq) { cfq_init_prio_data(cfqq, cic); - cfqq_boost_on_prio(cfqq, op_flags); + cfqq_boost_on_prio(cfqq, op); return __cfq_may_queue(cfqq); } diff --git a/block/elevator.c b/block/elevator.c index ac80f89a0842..a18a5db274e4 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -714,12 +714,12 @@ void elv_put_request(struct request_queue *q, struct request *rq) e->type->ops.elevator_put_req_fn(rq); } -int elv_may_queue(struct request_queue *q, int op, int op_flags) +int elv_may_queue(struct request_queue *q, unsigned int op) { struct elevator_queue *e = q->elevator; if (e->type->ops.elevator_may_queue_fn) - return e->type->ops.elevator_may_queue_fn(q, op, op_flags); + return e->type->ops.elevator_may_queue_fn(q, op); return ELV_MQUEUE_MAY; } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index a2768835d394..68a9eb4f3f36 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1135,7 +1135,7 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone) clone->bi_private = io; clone->bi_end_io = crypt_endio; clone->bi_bdev = cc->dev->bdev; - bio_set_op_attrs(clone, bio_op(io->base_bio), bio_flags(io->base_bio)); + clone->bi_opf = io->base_bio->bi_opf; } static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index cef1f78031d4..65738b0aad36 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -1031,8 +1031,7 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt) } else if (rq_data_dir(rq) == READ) { SCpnt->cmnd[0] = READ_6; } else { - scmd_printk(KERN_ERR, SCpnt, "Unknown command %llu,%llx\n", - req_op(rq), (unsigned long long) rq->cmd_flags); + scmd_printk(KERN_ERR, SCpnt, "Unknown command %d\n", req_op(rq)); goto out; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 2b790bda7998..9a377079af26 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8427,7 +8427,7 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip, if (!bio) return -ENOMEM; - bio_set_op_attrs(bio, bio_op(orig_bio), bio_flags(orig_bio)); + bio->bi_opf = orig_bio->bi_opf; bio->bi_private = dip; bio->bi_end_io = btrfs_end_dio_bio; btrfs_io_bio(bio)->logical = file_offset; @@ -8465,8 +8465,7 @@ next_block: start_sector, GFP_NOFS); if (!bio) goto out_err; - bio_set_op_attrs(bio, bio_op(orig_bio), - bio_flags(orig_bio)); + bio->bi_opf = orig_bio->bi_opf; bio->bi_private = dip; bio->bi_end_io = btrfs_end_dio_bio; btrfs_io_bio(bio)->logical = file_offset; diff --git a/fs/buffer.c b/fs/buffer.c index b205a629001d..a29335867e30 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3118,7 +3118,7 @@ EXPORT_SYMBOL(submit_bh); /** * ll_rw_block: low-level access to block devices (DEPRECATED) * @op: whether to %READ or %WRITE - * @op_flags: rq_flag_bits + * @op_flags: req_flag_bits * @nr: number of &struct buffer_heads in the array * @bhs: array of pointers to &struct buffer_head * diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 9e8de18a168a..2cf4f7f09e32 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -688,7 +688,7 @@ struct f2fs_io_info { struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */ enum page_type type; /* contains DATA/NODE/META/META_FLUSH */ int op; /* contains REQ_OP_ */ - int op_flags; /* rq_flag_bits */ + int op_flags; /* req_flag_bits */ block_t new_blkaddr; /* new block address to be written */ block_t old_blkaddr; /* old block address before Cow */ struct page *page; /* page to be written */ diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 49d5a1b61b06..b1f9144b42c7 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c @@ -231,7 +231,7 @@ static void gfs2_end_log_write(struct bio *bio) * gfs2_log_flush_bio - Submit any pending log bio * @sdp: The superblock * @op: REQ_OP - * @op_flags: rq_flag_bits + * @op_flags: req_flag_bits * * Submit any pending part-built or full bio to the block device. If * there is no pending bio, then this is a no-op. diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index 3bf5d33800ab..ddaf28d0988f 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -581,15 +581,14 @@ static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat) /** * blkg_rwstat_add - add a value to a blkg_rwstat * @rwstat: target blkg_rwstat - * @op: REQ_OP - * @op_flags: rq_flag_bits + * @op: REQ_OP and flags * @val: value to add * * Add @val to @rwstat. The counters are chosen according to @rw. The * caller is responsible for synchronizing calls to this function. */ static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, - int op, int op_flags, uint64_t val) + unsigned int op, uint64_t val) { struct percpu_counter *cnt; @@ -600,7 +599,7 @@ static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, __percpu_counter_add(cnt, val, BLKG_STAT_CPU_BATCH); - if (op_flags & REQ_SYNC) + if (op & REQ_SYNC) cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC]; else cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC]; @@ -705,9 +704,9 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, if (!throtl) { blkg = blkg ?: q->root_blkg; - blkg_rwstat_add(&blkg->stat_bytes, bio_op(bio), bio->bi_opf, + blkg_rwstat_add(&blkg->stat_bytes, bio->bi_opf, bio->bi_iter.bi_size); - blkg_rwstat_add(&blkg->stat_ios, bio_op(bio), bio->bi_opf, 1); + blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1); } rcu_read_unlock(); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index ec69a8fe3b29..dca972d67548 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -88,24 +88,6 @@ struct bio { struct bio_vec bi_inline_vecs[0]; }; -#define BIO_OP_SHIFT (8 * FIELD_SIZEOF(struct bio, bi_opf) - REQ_OP_BITS) -#define bio_flags(bio) ((bio)->bi_opf & ((1 << BIO_OP_SHIFT) - 1)) -#define bio_op(bio) ((bio)->bi_opf >> BIO_OP_SHIFT) - -#define bio_set_op_attrs(bio, op, op_flags) do { \ - if (__builtin_constant_p(op)) \ - BUILD_BUG_ON((op) + 0U >= (1U << REQ_OP_BITS)); \ - else \ - WARN_ON_ONCE((op) + 0U >= (1U << REQ_OP_BITS)); \ - if (__builtin_constant_p(op_flags)) \ - BUILD_BUG_ON((op_flags) + 0U >= (1U << BIO_OP_SHIFT)); \ - else \ - WARN_ON_ONCE((op_flags) + 0U >= (1U << BIO_OP_SHIFT)); \ - (bio)->bi_opf = bio_flags(bio); \ - (bio)->bi_opf |= (((op) + 0U) << BIO_OP_SHIFT); \ - (bio)->bi_opf |= (op_flags); \ -} while (0) - #define BIO_RESET_BYTES offsetof(struct bio, bi_max_vecs) /* @@ -147,26 +129,40 @@ struct bio { #endif /* CONFIG_BLOCK */ /* - * Request flags. For use in the cmd_flags field of struct request, and in - * bi_opf of struct bio. Note that some flags are only valid in either one. + * Operations and flags common to the bio and request structures. + * We use 8 bits for encoding the operation, and the remaining 24 for flags. */ -enum rq_flag_bits { - /* common flags */ - __REQ_FAILFAST_DEV, /* no driver retries of device errors */ +#define REQ_OP_BITS 8 +#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1) +#define REQ_FLAG_BITS 24 + +enum req_opf { + REQ_OP_READ, + REQ_OP_WRITE, + REQ_OP_DISCARD, /* request to discard sectors */ + REQ_OP_SECURE_ERASE, /* request to securely erase sectors */ + REQ_OP_WRITE_SAME, /* write same block many times */ + REQ_OP_FLUSH, /* request for cache flush */ + REQ_OP_ZONE_REPORT, /* Get zone information */ + REQ_OP_ZONE_RESET, /* Reset a zone write pointer */ + + REQ_OP_LAST, +}; + +enum req_flag_bits { + __REQ_FAILFAST_DEV = /* no driver retries of device errors */ + REQ_OP_BITS, __REQ_FAILFAST_TRANSPORT, /* no driver retries of transport errors */ __REQ_FAILFAST_DRIVER, /* no driver retries of driver errors */ - __REQ_SYNC, /* request is sync (sync write or read) */ __REQ_META, /* metadata io request */ __REQ_PRIO, /* boost priority in cfq */ - __REQ_NOMERGE, /* don't touch this for merging */ __REQ_NOIDLE, /* don't anticipate more IO after this one */ __REQ_INTEGRITY, /* I/O includes block integrity payload */ __REQ_FUA, /* forced unit access */ __REQ_PREFLUSH, /* request for cache flush */ __REQ_RAHEAD, /* read ahead, can fail anytime */ - __REQ_NR_BITS, /* stops here */ }; @@ -176,37 +172,32 @@ enum rq_flag_bits { #define REQ_SYNC (1ULL << __REQ_SYNC) #define REQ_META (1ULL << __REQ_META) #define REQ_PRIO (1ULL << __REQ_PRIO) +#define REQ_NOMERGE (1ULL << __REQ_NOMERGE) #define REQ_NOIDLE (1ULL << __REQ_NOIDLE) #define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY) +#define REQ_FUA (1ULL << __REQ_FUA) +#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH) +#define REQ_RAHEAD (1ULL << __REQ_RAHEAD) #define REQ_FAILFAST_MASK \ (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER) -#define REQ_COMMON_MASK \ - (REQ_FAILFAST_MASK | REQ_SYNC | REQ_META | REQ_PRIO | REQ_NOIDLE | \ - REQ_PREFLUSH | REQ_FUA | REQ_INTEGRITY | REQ_NOMERGE | REQ_RAHEAD) -#define REQ_CLONE_MASK REQ_COMMON_MASK -/* This mask is used for both bio and request merge checking */ #define REQ_NOMERGE_FLAGS \ (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA) -#define REQ_RAHEAD (1ULL << __REQ_RAHEAD) -#define REQ_FUA (1ULL << __REQ_FUA) -#define REQ_NOMERGE (1ULL << __REQ_NOMERGE) -#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH) +#define bio_op(bio) \ + ((bio)->bi_opf & REQ_OP_MASK) +#define req_op(req) \ + ((req)->cmd_flags & REQ_OP_MASK) -enum req_op { - REQ_OP_READ, - REQ_OP_WRITE, - REQ_OP_DISCARD, /* request to discard sectors */ - REQ_OP_SECURE_ERASE, /* request to securely erase sectors */ - REQ_OP_WRITE_SAME, /* write same block many times */ - REQ_OP_FLUSH, /* request for cache flush */ - REQ_OP_ZONE_REPORT, /* Get zone information */ - REQ_OP_ZONE_RESET, /* Reset a zone write pointer */ -}; +/* obsolete, don't use in new code */ +#define bio_set_op_attrs(bio, op, op_flags) \ + ((bio)->bi_opf |= (op | op_flags)) -#define REQ_OP_BITS 3 +static inline bool op_is_sync(unsigned int op) +{ + return (op & REQ_OP_MASK) == REQ_OP_READ || (op & REQ_SYNC); +} typedef unsigned int blk_qc_t; #define BLK_QC_T_NONE -1U diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b4415feac679..8396da2bb698 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -142,7 +142,7 @@ struct request { int cpu; unsigned cmd_type; - u64 cmd_flags; + unsigned int cmd_flags; /* op and common flags */ req_flags_t rq_flags; unsigned long atomic_flags; @@ -244,20 +244,6 @@ struct request { struct request *next_rq; }; -#define REQ_OP_SHIFT (8 * sizeof(u64) - REQ_OP_BITS) -#define req_op(req) ((req)->cmd_flags >> REQ_OP_SHIFT) - -#define req_set_op(req, op) do { \ - WARN_ON(op >= (1 << REQ_OP_BITS)); \ - (req)->cmd_flags &= ((1ULL << REQ_OP_SHIFT) - 1); \ - (req)->cmd_flags |= ((u64) (op) << REQ_OP_SHIFT); \ -} while (0) - -#define req_set_op_attrs(req, op, flags) do { \ - req_set_op(req, op); \ - (req)->cmd_flags |= flags; \ -} while (0) - static inline unsigned short req_get_ioprio(struct request *req) { return req->ioprio; @@ -741,17 +727,9 @@ static inline unsigned int blk_queue_zone_size(struct request_queue *q) return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0; } -/* - * We regard a request as sync, if either a read or a sync write - */ -static inline bool rw_is_sync(int op, unsigned int rw_flags) -{ - return op == REQ_OP_READ || (rw_flags & REQ_SYNC); -} - static inline bool rq_is_sync(struct request *rq) { - return rw_is_sync(req_op(rq), rq->cmd_flags); + return op_is_sync(rq->cmd_flags); } static inline bool blk_rl_full(struct request_list *rl, bool sync) diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h index cceb72f9e29f..e417f080219a 100644 --- a/include/linux/blktrace_api.h +++ b/include/linux/blktrace_api.h @@ -118,7 +118,7 @@ static inline int blk_cmd_buf_len(struct request *rq) } extern void blk_dump_cmd(char *buf, struct request *rq); -extern void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes); +extern void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes); #endif /* CONFIG_EVENT_TRACING && CONFIG_BLOCK */ diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index b91b023deffb..a52c6580cc9a 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -58,7 +58,7 @@ struct dm_io_notify { struct dm_io_client; struct dm_io_request { int bi_op; /* REQ_OP */ - int bi_op_flags; /* rq_flag_bits */ + int bi_op_flags; /* req_flag_bits */ struct dm_io_memory mem; /* Memory to use for io */ struct dm_io_notify notify; /* Synchronous if notify.fn is NULL */ struct dm_io_client *client; /* Client memory handler */ diff --git a/include/linux/elevator.h b/include/linux/elevator.h index e7f358d2e5fc..f219c9aed360 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -30,7 +30,7 @@ typedef int (elevator_dispatch_fn) (struct request_queue *, int); typedef void (elevator_add_req_fn) (struct request_queue *, struct request *); typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *); typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *); -typedef int (elevator_may_queue_fn) (struct request_queue *, int, int); +typedef int (elevator_may_queue_fn) (struct request_queue *, unsigned int); typedef void (elevator_init_icq_fn) (struct io_cq *); typedef void (elevator_exit_icq_fn) (struct io_cq *); @@ -139,7 +139,7 @@ extern struct request *elv_former_request(struct request_queue *, struct request extern struct request *elv_latter_request(struct request_queue *, struct request *); extern int elv_register_queue(struct request_queue *q); extern void elv_unregister_queue(struct request_queue *q); -extern int elv_may_queue(struct request_queue *, int, int); +extern int elv_may_queue(struct request_queue *, unsigned int); extern void elv_completed_request(struct request_queue *, struct request *); extern int elv_set_request(struct request_queue *q, struct request *rq, struct bio *bio, gfp_t gfp_mask); diff --git a/include/trace/events/bcache.h b/include/trace/events/bcache.h index d336b890e31f..df3e9ae5ad8d 100644 --- a/include/trace/events/bcache.h +++ b/include/trace/events/bcache.h @@ -27,8 +27,7 @@ DECLARE_EVENT_CLASS(bcache_request, __entry->sector = bio->bi_iter.bi_sector; __entry->orig_sector = bio->bi_iter.bi_sector - 16; __entry->nr_sector = bio->bi_iter.bi_size >> 9; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), TP_printk("%d,%d %s %llu + %u (from %d,%d @ %llu)", @@ -102,8 +101,7 @@ DECLARE_EVENT_CLASS(bcache_bio, __entry->dev = bio->bi_bdev->bd_dev; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio->bi_iter.bi_size >> 9; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), TP_printk("%d,%d %s %llu + %u", @@ -138,8 +136,7 @@ TRACE_EVENT(bcache_read, __entry->dev = bio->bi_bdev->bd_dev; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio->bi_iter.bi_size >> 9; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); __entry->cache_hit = hit; __entry->bypass = bypass; ), @@ -170,8 +167,7 @@ TRACE_EVENT(bcache_write, __entry->inode = inode; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio->bi_iter.bi_size >> 9; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); __entry->writeback = writeback; __entry->bypass = bypass; ), diff --git a/include/trace/events/block.h b/include/trace/events/block.h index 8f3a163b8166..3e02e3a25413 100644 --- a/include/trace/events/block.h +++ b/include/trace/events/block.h @@ -84,8 +84,7 @@ DECLARE_EVENT_CLASS(block_rq_with_error, 0 : blk_rq_sectors(rq); __entry->errors = rq->errors; - blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags, - blk_rq_bytes(rq)); + blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq)); blk_dump_cmd(__get_str(cmd), rq); ), @@ -163,7 +162,7 @@ TRACE_EVENT(block_rq_complete, __entry->nr_sector = nr_bytes >> 9; __entry->errors = rq->errors; - blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags, nr_bytes); + blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, nr_bytes); blk_dump_cmd(__get_str(cmd), rq); ), @@ -199,8 +198,7 @@ DECLARE_EVENT_CLASS(block_rq, __entry->bytes = (rq->cmd_type == REQ_TYPE_BLOCK_PC) ? blk_rq_bytes(rq) : 0; - blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags, - blk_rq_bytes(rq)); + blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq)); blk_dump_cmd(__get_str(cmd), rq); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -274,8 +272,7 @@ TRACE_EVENT(block_bio_bounce, bio->bi_bdev->bd_dev : 0; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio_sectors(bio); - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -313,8 +310,7 @@ TRACE_EVENT(block_bio_complete, __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio_sectors(bio); __entry->error = error; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), TP_printk("%d,%d %s %llu + %u [%d]", @@ -341,8 +337,7 @@ DECLARE_EVENT_CLASS(block_bio_merge, __entry->dev = bio->bi_bdev->bd_dev; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio_sectors(bio); - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -409,8 +404,7 @@ TRACE_EVENT(block_bio_queue, __entry->dev = bio->bi_bdev->bd_dev; __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio_sectors(bio); - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -438,7 +432,7 @@ DECLARE_EVENT_CLASS(block_get_rq, __entry->dev = bio ? bio->bi_bdev->bd_dev : 0; __entry->sector = bio ? bio->bi_iter.bi_sector : 0; __entry->nr_sector = bio ? bio_sectors(bio) : 0; - blk_fill_rwbs(__entry->rwbs, bio ? bio_op(bio) : 0, + blk_fill_rwbs(__entry->rwbs, bio ? bio->bi_opf : 0, __entry->nr_sector); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -573,8 +567,7 @@ TRACE_EVENT(block_split, __entry->dev = bio->bi_bdev->bd_dev; __entry->sector = bio->bi_iter.bi_sector; __entry->new_sector = new_sector; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); memcpy(__entry->comm, current->comm, TASK_COMM_LEN); ), @@ -617,8 +610,7 @@ TRACE_EVENT(block_bio_remap, __entry->nr_sector = bio_sectors(bio); __entry->old_dev = dev; __entry->old_sector = from; - blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf, - bio->bi_iter.bi_size); + blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), TP_printk("%d,%d %s %llu + %u <- (%d,%d) %llu", @@ -664,8 +656,7 @@ TRACE_EVENT(block_rq_remap, __entry->old_dev = dev; __entry->old_sector = from; __entry->nr_bios = blk_rq_count_bios(rq); - blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags, - blk_rq_bytes(rq)); + blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq)); ), TP_printk("%d,%d %s %llu + %u <- (%d,%d) %llu %u", diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index dbafc5df03f3..95cecbf67f5c 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -1777,14 +1777,14 @@ void blk_dump_cmd(char *buf, struct request *rq) } } -void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes) +void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes) { int i = 0; - if (rw & REQ_PREFLUSH) + if (op & REQ_PREFLUSH) rwbs[i++] = 'F'; - switch (op) { + switch (op & REQ_OP_MASK) { case REQ_OP_WRITE: case REQ_OP_WRITE_SAME: rwbs[i++] = 'W'; @@ -1806,13 +1806,13 @@ void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes) rwbs[i++] = 'N'; } - if (rw & REQ_FUA) + if (op & REQ_FUA) rwbs[i++] = 'F'; - if (rw & REQ_RAHEAD) + if (op & REQ_RAHEAD) rwbs[i++] = 'A'; - if (rw & REQ_SYNC) + if (op & REQ_SYNC) rwbs[i++] = 'S'; - if (rw & REQ_META) + if (op & REQ_META) rwbs[i++] = 'M'; rwbs[i] = '\0'; -- cgit v1.3-7-g2ca7 From 60f1d5e3bac44b598f67d36062da96c095d2b700 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 5 Oct 2016 20:58:15 +0900 Subject: ftrace: Support full glob matching Use glob_match() to support flexible glob wildcards (*,?) and character classes ([) for ftrace. Since the full glob matching is slower than the current partial matching routines(*pat, pat*, *pat*), this leaves those routines and just add MATCH_GLOB for complex glob expression. e.g. ---- [root@localhost tracing]# echo 'sched*group' > set_ftrace_filter [root@localhost tracing]# cat set_ftrace_filter sched_free_group sched_change_group sched_create_group sched_online_group sched_destroy_group sched_offline_group [root@localhost tracing]# echo '[Ss]y[Ss]_*' > set_ftrace_filter [root@localhost tracing]# head set_ftrace_filter sys_arch_prctl sys_rt_sigreturn sys_ioperm SyS_iopl sys_modify_ldt SyS_mmap SyS_set_thread_area SyS_get_thread_area SyS_set_tid_address sys_fork ---- Link: http://lkml.kernel.org/r/147566869501.29136.6462645009894738056.stgit@devbox Acked-by: Namhyung Kim Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt --- Documentation/trace/events.txt | 9 +++------ Documentation/trace/ftrace.txt | 9 +++------ kernel/trace/Kconfig | 2 ++ kernel/trace/ftrace.c | 4 ++++ kernel/trace/trace.c | 2 +- kernel/trace/trace.h | 2 ++ kernel/trace/trace_events_filter.c | 17 ++++++++++++++++- 7 files changed, 31 insertions(+), 14 deletions(-) (limited to 'kernel/trace') diff --git a/Documentation/trace/events.txt b/Documentation/trace/events.txt index 08d74d75150d..2cc08d4a326e 100644 --- a/Documentation/trace/events.txt +++ b/Documentation/trace/events.txt @@ -189,16 +189,13 @@ And for string fields they are: ==, !=, ~ -The glob (~) only accepts a wild card character (*) at the start and or -end of the string. For example: +The glob (~) accepts a wild card character (*,?) and character classes +([). For example: prev_comm ~ "*sh" prev_comm ~ "sh*" prev_comm ~ "*sh*" - -But does not allow for it to be within the string: - - prev_comm ~ "ba*sh" <-- is invalid + prev_comm ~ "ba*sh" 5.2 Setting filters ------------------- diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt index 185c39fea2a0..1bc66c1db0cb 100644 --- a/Documentation/trace/ftrace.txt +++ b/Documentation/trace/ftrace.txt @@ -2218,16 +2218,13 @@ hrtimer_interrupt sys_nanosleep -Perhaps this is not enough. The filters also allow simple wild -cards. Only the following are currently available +Perhaps this is not enough. The filters also allow glob(7) matching. * - will match functions that begin with * - will match functions that end with ** - will match functions that have in it - -These are the only wild cards which are supported. - - * will not work. + * - will match functions that begin with + and end with Note: It is better to use quotes to enclose the wild cards, otherwise the shell may expand the parameters into names diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 2a96b063d659..d5038005eb5d 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -70,6 +70,7 @@ config FTRACE_NMI_ENTER config EVENT_TRACING select CONTEXT_SWITCH_TRACER + select GLOB bool config CONTEXT_SWITCH_TRACER @@ -133,6 +134,7 @@ config FUNCTION_TRACER select KALLSYMS select GENERIC_TRACER select CONTEXT_SWITCH_TRACER + select GLOB help Enable the kernel to trace every kernel function. This is done by using a compiler feature to insert a small, 5-byte No-Operation diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index da87b3cba5b3..356bb70d071e 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3511,6 +3511,10 @@ static int ftrace_match(char *str, struct ftrace_glob *g) memcmp(str + slen - g->len, g->search, g->len) == 0) matched = 1; break; + case MATCH_GLOB: + if (glob_match(g->search, str)) + matched = 1; + break; } return matched; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8696ce6bf2f6..d904516dfdab 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4065,7 +4065,7 @@ static const char readme_msg[] = "\n available_filter_functions - list of functions that can be filtered on\n" " set_ftrace_filter\t- echo function name in here to only trace these\n" "\t\t\t functions\n" - "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" + "\t accepts: func_full_name or glob-matching-pattern\n" "\t modules: Can select a group via module\n" "\t Format: :mod:\n" "\t example: echo :mod:ext3 > set_ftrace_filter\n" diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index fd24b1f9ac43..4b7918902ab8 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -15,6 +15,7 @@ #include #include #include +#include #ifdef CONFIG_FTRACE_SYSCALLS #include /* For NR_SYSCALLS */ @@ -1257,6 +1258,7 @@ enum regex_type { MATCH_FRONT_ONLY, MATCH_MIDDLE_ONLY, MATCH_END_ONLY, + MATCH_GLOB, }; struct regex { diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 9daa9b3bc6d9..e1c7e2cdc240 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -344,6 +344,12 @@ static int regex_match_end(char *str, struct regex *r, int len) return 0; } +static int regex_match_glob(char *str, struct regex *r, int len __maybe_unused) +{ + if (glob_match(r->pattern, str)) + return 1; + return 0; +} /** * filter_parse_regex - parse a basic regex * @buff: the raw regex @@ -380,14 +386,20 @@ enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not) if (!i) { *search = buff + 1; type = MATCH_END_ONLY; - } else { + } else if (i == len - 1) { if (type == MATCH_END_ONLY) type = MATCH_MIDDLE_ONLY; else type = MATCH_FRONT_ONLY; buff[i] = 0; break; + } else { /* pattern continues, use full glob */ + type = MATCH_GLOB; + break; } + } else if (strchr("[?\\", buff[i])) { + type = MATCH_GLOB; + break; } } @@ -420,6 +432,9 @@ static void filter_build_regex(struct filter_pred *pred) case MATCH_END_ONLY: r->match = regex_match_end; break; + case MATCH_GLOB: + r->match = regex_match_glob; + break; } pred->not ^= not; -- cgit v1.3-7-g2ca7 From fdf5b679864b8a1165712164b8f17debeb6b10b6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Tue, 25 Oct 2016 16:14:28 -0400 Subject: tracing: Optimise comparison filters and fix binary and for 64 bit Currently the filter logic for comparisons (like greater-than and less-than) are used, they share the same function and a switch statement is used to jump to the comparison type to perform. This is done in the extreme hot path of the tracing code, and it does not take much more space to create a unique comparison function to perform each type of comparison and remove the switch statement. Also, a bug was found where the binary and operation for 64 bits could fail if the resulting bits were greater than 32 bits, because the result was passed into a 32 bit variable. This was fixed when adding the separate binary and function. Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_filter.c | 80 +++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 32 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index e1c7e2cdc240..1ba7a6b86f55 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -145,34 +145,50 @@ struct pred_stack { /* If not of not match is equal to not of not, then it is a match */ #define DEFINE_COMPARISON_PRED(type) \ -static int filter_pred_##type(struct filter_pred *pred, void *event) \ +static int filter_pred_LT_##type(struct filter_pred *pred, void *event) \ { \ type *addr = (type *)(event + pred->offset); \ type val = (type)pred->val; \ - int match = 0; \ - \ - switch (pred->op) { \ - case OP_LT: \ - match = (*addr < val); \ - break; \ - case OP_LE: \ - match = (*addr <= val); \ - break; \ - case OP_GT: \ - match = (*addr > val); \ - break; \ - case OP_GE: \ - match = (*addr >= val); \ - break; \ - case OP_BAND: \ - match = (*addr & val); \ - break; \ - default: \ - break; \ - } \ - \ + int match = (*addr < val); \ return !!match == !pred->not; \ -} +} \ +static int filter_pred_LE_##type(struct filter_pred *pred, void *event) \ +{ \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + int match = (*addr <= val); \ + return !!match == !pred->not; \ +} \ +static int filter_pred_GT_##type(struct filter_pred *pred, void *event) \ +{ \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + int match = (*addr > val); \ + return !!match == !pred->not; \ +} \ +static int filter_pred_GE_##type(struct filter_pred *pred, void *event) \ +{ \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + int match = (*addr >= val); \ + return !!match == !pred->not; \ +} \ +static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \ +{ \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + int match = !!(*addr & val); \ + return match == !pred->not; \ +} \ +static const filter_pred_fn_t pred_funcs_##type[] = { \ + filter_pred_LT_##type, \ + filter_pred_LE_##type, \ + filter_pred_GT_##type, \ + filter_pred_GE_##type, \ + filter_pred_BAND_##type, \ +}; + +#define PRED_FUNC_START OP_LT #define DEFINE_EQUALITY_PRED(size) \ static int filter_pred_##size(struct filter_pred *pred, void *event) \ @@ -982,33 +998,33 @@ static filter_pred_fn_t select_comparison_fn(int op, int field_size, if (op == OP_EQ || op == OP_NE) fn = filter_pred_64; else if (field_is_signed) - fn = filter_pred_s64; + fn = pred_funcs_s64[op - PRED_FUNC_START]; else - fn = filter_pred_u64; + fn = pred_funcs_u64[op - PRED_FUNC_START]; break; case 4: if (op == OP_EQ || op == OP_NE) fn = filter_pred_32; else if (field_is_signed) - fn = filter_pred_s32; + fn = pred_funcs_s32[op - PRED_FUNC_START]; else - fn = filter_pred_u32; + fn = pred_funcs_u32[op - PRED_FUNC_START]; break; case 2: if (op == OP_EQ || op == OP_NE) fn = filter_pred_16; else if (field_is_signed) - fn = filter_pred_s16; + fn = pred_funcs_s16[op - PRED_FUNC_START]; else - fn = filter_pred_u16; + fn = pred_funcs_u16[op - PRED_FUNC_START]; break; case 1: if (op == OP_EQ || op == OP_NE) fn = filter_pred_8; else if (field_is_signed) - fn = filter_pred_s8; + fn = pred_funcs_s8[op - PRED_FUNC_START]; else - fn = filter_pred_u8; + fn = pred_funcs_u8[op - PRED_FUNC_START]; break; } -- cgit v1.3-7-g2ca7 From 3f303fbccfc423a28765df6e1be0428fdf1aac59 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 26 Oct 2016 15:58:03 -0400 Subject: tracing/filter: Define op as the enum that it is The trace_events_file.c filter logic can be a bit complex. I copy this into a userspace program where I can debug it a bit easier. One issue is the op is defined in most places as an int instead of as an enum, and gdb just gives the value when debugging. Having the actual op name shown in gdb is more useful. This has no functionality change, but helps in debugging when the file is debugged in user space. Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_filter.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 1ba7a6b86f55..59a411ff60c7 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -108,12 +108,12 @@ static char *err_text[] = { }; struct opstack_op { - int op; + enum filter_op_ids op; struct list_head list; }; struct postfix_elt { - int op; + enum filter_op_ids op; char *operand; struct list_head list; }; @@ -977,7 +977,7 @@ int filter_assign_type(const char *type) return FILTER_OTHER; } -static bool is_legal_op(struct ftrace_event_field *field, int op) +static bool is_legal_op(struct ftrace_event_field *field, enum filter_op_ids op) { if (is_string_field(field) && (op != OP_EQ && op != OP_NE && op != OP_GLOB)) @@ -988,8 +988,8 @@ static bool is_legal_op(struct ftrace_event_field *field, int op) return true; } -static filter_pred_fn_t select_comparison_fn(int op, int field_size, - int field_is_signed) +static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, + int field_size, int field_is_signed) { filter_pred_fn_t fn = NULL; @@ -1197,7 +1197,8 @@ static inline int append_operand_char(struct filter_parse_state *ps, char c) return 0; } -static int filter_opstack_push(struct filter_parse_state *ps, int op) +static int filter_opstack_push(struct filter_parse_state *ps, + enum filter_op_ids op) { struct opstack_op *opstack_op; @@ -1231,7 +1232,7 @@ static int filter_opstack_top(struct filter_parse_state *ps) static int filter_opstack_pop(struct filter_parse_state *ps) { struct opstack_op *opstack_op; - int op; + enum filter_op_ids op; if (filter_opstack_empty(ps)) return OP_NONE; @@ -1276,7 +1277,7 @@ static int postfix_append_operand(struct filter_parse_state *ps, char *operand) return 0; } -static int postfix_append_op(struct filter_parse_state *ps, int op) +static int postfix_append_op(struct filter_parse_state *ps, enum filter_op_ids op) { struct postfix_elt *elt; @@ -1306,8 +1307,8 @@ static void postfix_clear(struct filter_parse_state *ps) static int filter_parse(struct filter_parse_state *ps) { + enum filter_op_ids op, top_op; int in_string = 0; - int op, top_op; char ch; while ((ch = infix_next(ps))) { @@ -1398,7 +1399,8 @@ parse_operand: static struct filter_pred *create_pred(struct filter_parse_state *ps, struct trace_event_call *call, - int op, char *operand1, char *operand2) + enum filter_op_ids op, + char *operand1, char *operand2) { struct ftrace_event_field *field; static struct filter_pred pred; -- cgit v1.3-7-g2ca7 From 8d414bd2f77ce858f6b9d119c63b9ce29cf0b75d Mon Sep 17 00:00:00 2001 From: Zhou Chengming Date: Mon, 14 Nov 2016 11:19:13 +0800 Subject: tracing: Allow wakeup_dl tracer to be used by instances Allow wakeup_dl tracer to be used by instances, like wakeup tracer and wakeup_rt tracer. Link: http://lkml.kernel.org/r/1479093553-31264-1-git-send-email-zhouchengming1@huawei.com Signed-off-by: Zhou Chengming Signed-off-by: Steven Rostedt --- kernel/trace/trace_sched_wakeup.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 9d4399b553a3..1bf2324dc682 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -790,6 +790,7 @@ static struct tracer wakeup_dl_tracer __read_mostly = #endif .open = wakeup_trace_open, .close = wakeup_trace_close, + .allow_instances = true, .use_max_tr = true, }; -- cgit v1.3-7-g2ca7 From fa32e8557b470f5ff90babc6cbacc61535a81a0f Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 6 Jul 2016 15:25:08 -0400 Subject: tracing: Add new trace_marker_raw A new file is created: /sys/kernel/debug/tracing/trace_marker_raw This allows for appications to create data structures and write the binary data directly into it, and then read the trace data out from trace_pipe_raw into the same type of data structure. This saves on converting numbers into ASCII that would be required by trace_marker. Suggested-by: Olof Johansson Signed-off-by: Steven Rostedt --- Documentation/trace/ftrace.txt | 6 ++ kernel/trace/trace.c | 165 +++++++++++++++++++++++++++++++++-------- kernel/trace/trace.h | 2 + kernel/trace/trace_entries.h | 15 ++++ kernel/trace/trace_output.c | 30 ++++++++ 5 files changed, 187 insertions(+), 31 deletions(-) (limited to 'kernel/trace') diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt index 1bc66c1db0cb..6c374c5fe400 100644 --- a/Documentation/trace/ftrace.txt +++ b/Documentation/trace/ftrace.txt @@ -396,6 +396,12 @@ of ftrace. Here is a list of some of the key files: trace_fd = open("trace_marker", WR_ONLY); + trace_marker_raw: + + This is similar to trace_marker above, but is meant for for binary data + to be written to it, where a tool can be used to parse the data + from trace_pipe_raw. + uprobe_events: Add dynamic tracepoints in programs. diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index d904516dfdab..57069e7f369c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4054,6 +4054,7 @@ static const char readme_msg[] = " x86-tsc: TSC cycle counter\n" #endif "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" + "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" " tracing_cpumask\t- Limit which CPUs to trace\n" " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" "\t\t\t Remove sub-buffer with rmdir\n" @@ -5514,35 +5515,15 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp) return 0; } -static ssize_t -tracing_mark_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *fpos) +static inline int lock_user_pages(const char __user *ubuf, size_t cnt, + struct page **pages, void **map_page, + int *offset) { unsigned long addr = (unsigned long)ubuf; - struct trace_array *tr = filp->private_data; - struct ring_buffer_event *event; - struct ring_buffer *buffer; - struct print_entry *entry; - unsigned long irq_flags; - struct page *pages[2]; - void *map_page[2]; int nr_pages = 1; - ssize_t written; - int offset; - int size; - int len; int ret; int i; - if (tracing_disabled) - return -EINVAL; - - if (!(tr->trace_flags & TRACE_ITER_MARKERS)) - return -EINVAL; - - if (cnt > TRACE_BUF_SIZE) - cnt = TRACE_BUF_SIZE; - /* * Userspace is injecting traces into the kernel trace buffer. * We want to be as non intrusive as possible. @@ -5557,26 +5538,70 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, * pages directly. We then write the data directly into the * ring buffer. */ - BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); /* check if we cross pages */ if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK)) nr_pages = 2; - offset = addr & (PAGE_SIZE - 1); + *offset = addr & (PAGE_SIZE - 1); addr &= PAGE_MASK; ret = get_user_pages_fast(addr, nr_pages, 0, pages); if (ret < nr_pages) { while (--ret >= 0) put_page(pages[ret]); - written = -EFAULT; - goto out; + return -EFAULT; } for (i = 0; i < nr_pages; i++) map_page[i] = kmap_atomic(pages[i]); + return nr_pages; +} + +static inline void unlock_user_pages(struct page **pages, + void **map_page, int nr_pages) +{ + int i; + + for (i = nr_pages - 1; i >= 0; i--) { + kunmap_atomic(map_page[i]); + put_page(pages[i]); + } +} + +static ssize_t +tracing_mark_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *fpos) +{ + struct trace_array *tr = filp->private_data; + struct ring_buffer_event *event; + struct ring_buffer *buffer; + struct print_entry *entry; + unsigned long irq_flags; + struct page *pages[2]; + void *map_page[2]; + int nr_pages = 1; + ssize_t written; + int offset; + int size; + int len; + + if (tracing_disabled) + return -EINVAL; + + if (!(tr->trace_flags & TRACE_ITER_MARKERS)) + return -EINVAL; + + if (cnt > TRACE_BUF_SIZE) + cnt = TRACE_BUF_SIZE; + + BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); + + nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset); + if (nr_pages < 0) + return nr_pages; + local_save_flags(irq_flags); size = sizeof(*entry) + cnt + 2; /* possible \n added */ buffer = tr->trace_buffer.buffer; @@ -5611,11 +5636,79 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, *fpos += written; out_unlock: - for (i = nr_pages - 1; i >= 0; i--) { - kunmap_atomic(map_page[i]); - put_page(pages[i]); + unlock_user_pages(pages, map_page, nr_pages); + + return written; +} + +/* Limit it for now to 3K (including tag) */ +#define RAW_DATA_MAX_SIZE (1024*3) + +static ssize_t +tracing_mark_raw_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *fpos) +{ + struct trace_array *tr = filp->private_data; + struct ring_buffer_event *event; + struct ring_buffer *buffer; + struct raw_data_entry *entry; + unsigned long irq_flags; + struct page *pages[2]; + void *map_page[2]; + int nr_pages = 1; + ssize_t written; + int offset; + int size; + int len; + + if (tracing_disabled) + return -EINVAL; + + if (!(tr->trace_flags & TRACE_ITER_MARKERS)) + return -EINVAL; + + /* The marker must at least have a tag id */ + if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE) + return -EINVAL; + + if (cnt > TRACE_BUF_SIZE) + cnt = TRACE_BUF_SIZE; + + BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); + + nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset); + if (nr_pages < 0) + return nr_pages; + + local_save_flags(irq_flags); + size = sizeof(*entry) + cnt; + buffer = tr->trace_buffer.buffer; + event = trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, + irq_flags, preempt_count()); + if (!event) { + /* Ring buffer disabled, return as if not open for write */ + written = -EBADF; + goto out_unlock; } - out: + + entry = ring_buffer_event_data(event); + + if (nr_pages == 2) { + len = PAGE_SIZE - offset; + memcpy(&entry->id, map_page[0] + offset, len); + memcpy(((char *)&entry->id) + len, map_page[1], cnt - len); + } else + memcpy(&entry->id, map_page[0] + offset, cnt); + + __buffer_unlock_commit(buffer, event); + + written = cnt; + + *fpos += written; + + out_unlock: + unlock_user_pages(pages, map_page, nr_pages); + return written; } @@ -5945,6 +6038,13 @@ static const struct file_operations tracing_mark_fops = { .release = tracing_release_generic_tr, }; +static const struct file_operations tracing_mark_raw_fops = { + .open = tracing_open_generic_tr, + .write = tracing_mark_raw_write, + .llseek = generic_file_llseek, + .release = tracing_release_generic_tr, +}; + static const struct file_operations trace_clock_fops = { .open = tracing_clock_open, .read = seq_read, @@ -7214,6 +7314,9 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) trace_create_file("trace_marker", 0220, d_tracer, tr, &tracing_mark_fops); + trace_create_file("trace_marker_raw", 0220, d_tracer, + tr, &tracing_mark_raw_fops); + trace_create_file("trace_clock", 0644, d_tracer, tr, &trace_clock_fops); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 4b7918902ab8..9294f8606ade 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -40,6 +40,7 @@ enum trace_type { TRACE_BLK, TRACE_BPUTS, TRACE_HWLAT, + TRACE_RAW_DATA, __TRACE_LAST_TYPE, }; @@ -331,6 +332,7 @@ extern void __ftrace_bad_type(void); IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \ IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \ IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \ + IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\ IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \ TRACE_MMIO_RW); \ IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \ diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h index d1cc37e78f99..eb7396b7e7c3 100644 --- a/kernel/trace/trace_entries.h +++ b/kernel/trace/trace_entries.h @@ -244,6 +244,21 @@ FTRACE_ENTRY(print, print_entry, FILTER_OTHER ); +FTRACE_ENTRY(raw_data, raw_data_entry, + + TRACE_RAW_DATA, + + F_STRUCT( + __field( unsigned int, id ) + __dynamic_array( char, buf ) + ), + + F_printk("id:%04x %08x", + __entry->id, (int)__entry->buf[0]), + + FILTER_OTHER +); + FTRACE_ENTRY(bputs, bputs_entry, TRACE_BPUTS, diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 3fc20422c166..5d33a7352919 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -1288,6 +1288,35 @@ static struct trace_event trace_print_event = { .funcs = &trace_print_funcs, }; +static enum print_line_t trace_raw_data(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct raw_data_entry *field; + int i; + + trace_assign_type(field, iter->ent); + + trace_seq_printf(&iter->seq, "# %x buf:", field->id); + + for (i = 0; i < iter->ent_size - offsetof(struct raw_data_entry, buf); i++) + trace_seq_printf(&iter->seq, " %02x", + (unsigned char)field->buf[i]); + + trace_seq_putc(&iter->seq, '\n'); + + return trace_handle_return(&iter->seq); +} + +static struct trace_event_functions trace_raw_data_funcs = { + .trace = trace_raw_data, + .raw = trace_raw_data, +}; + +static struct trace_event trace_raw_data_event = { + .type = TRACE_RAW_DATA, + .funcs = &trace_raw_data_funcs, +}; + static struct trace_event *events[] __initdata = { &trace_fn_event, @@ -1299,6 +1328,7 @@ static struct trace_event *events[] __initdata = { &trace_bprint_event, &trace_print_event, &trace_hwlat_event, + &trace_raw_data_event, NULL }; -- cgit v1.3-7-g2ca7 From d032ae8921ea792c1e6b2abb44022b2403f651f6 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 15 Nov 2016 12:31:20 -0800 Subject: ftrace: Provide API to use global filtering for ftrace ops Currently the global_ops filtering hash is not available to outside users registering for function tracing. Provide an API for those users to be able to choose global filtering. This is in preparation for pstore's ftrace feature to be able to use the global filters. Suggested-by: Steven Rostedt Cc: Anton Vorontsov Cc: Colin Cross Cc: Kees Cook Cc: Tony Luck Signed-off-by: Joel Fernandes Acked-by: Steven Rostedt Signed-off-by: Kees Cook --- include/linux/ftrace.h | 2 ++ kernel/trace/ftrace.c | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) (limited to 'kernel/trace') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index b3d34d3e0e7e..d4a884db16a3 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -398,6 +398,7 @@ int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, void ftrace_set_global_filter(unsigned char *buf, int len, int reset); void ftrace_set_global_notrace(unsigned char *buf, int len, int reset); void ftrace_free_filter(struct ftrace_ops *ops); +void ftrace_ops_set_global_filter(struct ftrace_ops *ops); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); @@ -645,6 +646,7 @@ static inline unsigned long ftrace_location(unsigned long ip) #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; }) #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; }) #define ftrace_free_filter(ops) do { } while (0) +#define ftrace_ops_set_global_filter(ops) do { } while (0) static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { return -ENODEV; } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 2050a7652a86..89d46e1c9302 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -4239,6 +4239,23 @@ int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, } EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); +/** + * ftrace_ops_set_global_filter - setup ops to use global filters + * @ops - the ops which will use the global filters + * + * ftrace users who need global function trace filtering should call this. + * It can set the global filter only if ops were not initialized before. + */ +void ftrace_ops_set_global_filter(struct ftrace_ops *ops) +{ + if (ops->flags & FTRACE_OPS_FL_INITIALIZED) + return; + + ftrace_ops_init(ops); + ops->func_hash = &global_ops.local_hash; +} +EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter); + static int ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, int reset, int enable) -- cgit v1.3-7-g2ca7 From 478409dd683db76cbcfe7bf8332a37f01deb0a2d Mon Sep 17 00:00:00 2001 From: Chunyan Zhang Date: Mon, 21 Nov 2016 15:57:18 +0800 Subject: tracing: Add hook to function tracing for other subsystems to use Currently Function traces can be only exported to the ring buffer. This adds a trace_export concept which can process traces and export them to a registered destination as an addition to the current one that outputs to Ftrace - i.e. ring buffer. In this way, if we want function traces to be sent to other destinations rather than only to the ring buffer, we just need to register a new trace_export and implement its own .write() function for writing traces to storage. With this patch, only function tracing (trace type is TRACE_FN) is supported. Link: http://lkml.kernel.org/r/1479715043-6534-2-git-send-email-zhang.chunyan@linaro.org Signed-off-by: Chunyan Zhang Signed-off-by: Steven Rostedt --- include/linux/trace.h | 28 +++++++++++ kernel/trace/trace.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 include/linux/trace.h (limited to 'kernel/trace') diff --git a/include/linux/trace.h b/include/linux/trace.h new file mode 100644 index 000000000000..9330a58e2651 --- /dev/null +++ b/include/linux/trace.h @@ -0,0 +1,28 @@ +#ifndef _LINUX_TRACE_H +#define _LINUX_TRACE_H + +#ifdef CONFIG_TRACING +/* + * The trace export - an export of Ftrace output. The trace_export + * can process traces and export them to a registered destination as + * an addition to the current only output of Ftrace - i.e. ring buffer. + * + * If you want traces to be sent to some other place rather than ring + * buffer only, just need to register a new trace_export and implement + * its own .write() function for writing traces to the storage. + * + * next - pointer to the next trace_export + * write - copy traces which have been delt with ->commit() to + * the destination + */ +struct trace_export { + struct trace_export __rcu *next; + void (*write)(const void *, unsigned int); +}; + +int register_ftrace_export(struct trace_export *export); +int unregister_ftrace_export(struct trace_export *export); + +#endif /* CONFIG_TRACING */ + +#endif /* _LINUX_TRACE_H */ diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 57069e7f369c..edccdff8a36d 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include "trace.h" @@ -2128,6 +2129,129 @@ void trace_buffer_unlock_commit_regs(struct trace_array *tr, ftrace_trace_userstack(buffer, flags, pc); } +static void +trace_process_export(struct trace_export *export, + struct ring_buffer_event *event) +{ + struct trace_entry *entry; + unsigned int size = 0; + + entry = ring_buffer_event_data(event); + size = ring_buffer_event_length(event); + export->write(entry, size); +} + +static DEFINE_MUTEX(ftrace_export_lock); + +static struct trace_export __rcu *ftrace_exports_list __read_mostly; + +static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled); + +static inline void ftrace_exports_enable(void) +{ + static_branch_enable(&ftrace_exports_enabled); +} + +static inline void ftrace_exports_disable(void) +{ + static_branch_disable(&ftrace_exports_enabled); +} + +void ftrace_exports(struct ring_buffer_event *event) +{ + struct trace_export *export; + + preempt_disable_notrace(); + + export = rcu_dereference_raw_notrace(ftrace_exports_list); + while (export) { + trace_process_export(export, event); + export = rcu_dereference_raw_notrace(export->next); + } + + preempt_enable_notrace(); +} + +static inline void +add_trace_export(struct trace_export **list, struct trace_export *export) +{ + rcu_assign_pointer(export->next, *list); + /* + * We are entering export into the list but another + * CPU might be walking that list. We need to make sure + * the export->next pointer is valid before another CPU sees + * the export pointer included into the list. + */ + rcu_assign_pointer(*list, export); +} + +static inline int +rm_trace_export(struct trace_export **list, struct trace_export *export) +{ + struct trace_export **p; + + for (p = list; *p != NULL; p = &(*p)->next) + if (*p == export) + break; + + if (*p != export) + return -1; + + rcu_assign_pointer(*p, (*p)->next); + + return 0; +} + +static inline void +add_ftrace_export(struct trace_export **list, struct trace_export *export) +{ + if (*list == NULL) + ftrace_exports_enable(); + + add_trace_export(list, export); +} + +static inline int +rm_ftrace_export(struct trace_export **list, struct trace_export *export) +{ + int ret; + + ret = rm_trace_export(list, export); + if (*list == NULL) + ftrace_exports_disable(); + + return ret; +} + +int register_ftrace_export(struct trace_export *export) +{ + if (WARN_ON_ONCE(!export->write)) + return -1; + + mutex_lock(&ftrace_export_lock); + + add_ftrace_export(&ftrace_exports_list, export); + + mutex_unlock(&ftrace_export_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(register_ftrace_export); + +int unregister_ftrace_export(struct trace_export *export) +{ + int ret; + + mutex_lock(&ftrace_export_lock); + + ret = rm_ftrace_export(&ftrace_exports_list, export); + + mutex_unlock(&ftrace_export_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(unregister_ftrace_export); + void trace_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, unsigned long flags, @@ -2146,8 +2270,11 @@ trace_function(struct trace_array *tr, entry->ip = ip; entry->parent_ip = parent_ip; - if (!call_filter_check_discard(call, entry, buffer, event)) + if (!call_filter_check_discard(call, entry, buffer, event)) { + if (static_branch_unlikely(&ftrace_exports_enabled)) + ftrace_exports(event); __buffer_unlock_commit(buffer, event); + } } #ifdef CONFIG_STACKTRACE -- cgit v1.3-7-g2ca7 From 7d436400223bb46e9f88e6bba6f8d867acf0d82c Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Tue, 22 Nov 2016 18:32:03 -0500 Subject: tracing: Add error checks to creation of event files The creation of the set_event_pid file was assigned to a variable "entry" but that variable was never used. Ideally, it should be used to check if the file was created and warn if it was not. The files header_page, header_event should also be checked and a warning if they fail to be created. The "enable" file was moved up, as it is a more crucial file to have and a hard failure (return -ENOMEM) should be returned if it is not created. Reported-by: David Binderman Signed-off-by: Steven Rostedt --- kernel/trace/trace_events.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 03c0a48c3ac4..ba67ede48822 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2843,20 +2843,32 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) return -ENOMEM; } + entry = trace_create_file("enable", 0644, d_events, + tr, &ftrace_tr_enable_fops); + if (!entry) { + pr_warn("Could not create tracefs 'enable' entry\n"); + return -ENOMEM; + } + + /* There are not as crucial, just warn if they are not created */ + entry = tracefs_create_file("set_event_pid", 0644, parent, tr, &ftrace_set_event_pid_fops); + if (!entry) + pr_warn("Could not create tracefs 'set_event_pid' entry\n"); /* ring buffer internal formats */ - trace_create_file("header_page", 0444, d_events, - ring_buffer_print_page_header, - &ftrace_show_header_fops); - - trace_create_file("header_event", 0444, d_events, - ring_buffer_print_entry_header, - &ftrace_show_header_fops); + entry = trace_create_file("header_page", 0444, d_events, + ring_buffer_print_page_header, + &ftrace_show_header_fops); + if (!entry) + pr_warn("Could not create tracefs 'header_page' entry\n"); - trace_create_file("enable", 0644, d_events, - tr, &ftrace_tr_enable_fops); + entry = trace_create_file("header_event", 0444, d_events, + ring_buffer_print_entry_header, + &ftrace_show_header_fops); + if (!entry) + pr_warn("Could not create tracefs 'header_event' entry\n"); tr->event_dir = d_events; -- cgit v1.3-7-g2ca7 From 3e9a8aadca4807b4eadd33a50014c9b2767a4f1f Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 11:29:58 -0500 Subject: tracing: Create a always_inlined __trace_buffer_lock_reserve() As Andi Kleen pointed out in the Link below, the trace events has quite a bit of code execution. A lot of that happens to be calling functions, where some of them should simply be inlined. One of these functions happens to be trace_buffer_lock_reserve() which is also a global, but it is used throughout the file it is defined in. Create a __trace_buffer_lock_reserve() that is always inlined that the file can benefit from. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 87 +++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 39 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index edccdff8a36d..490533726b54 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -739,6 +739,31 @@ static inline void ftrace_trace_stack(struct trace_array *tr, #endif +static __always_inline void +trace_event_setup(struct ring_buffer_event *event, + int type, unsigned long flags, int pc) +{ + struct trace_entry *ent = ring_buffer_event_data(event); + + tracing_generic_entry_update(ent, flags, pc); + ent->type = type; +} + +static __always_inline struct ring_buffer_event * +__trace_buffer_lock_reserve(struct ring_buffer *buffer, + int type, + unsigned long len, + unsigned long flags, int pc) +{ + struct ring_buffer_event *event; + + event = ring_buffer_lock_reserve(buffer, len); + if (event != NULL) + trace_event_setup(event, type, flags, pc); + + return event; +} + static void tracer_tracing_on(struct trace_array *tr) { if (tr->trace_buffer.buffer) @@ -795,8 +820,8 @@ int __trace_puts(unsigned long ip, const char *str, int size) local_save_flags(irq_flags); buffer = global_trace.trace_buffer.buffer; - event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, - irq_flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, + irq_flags, pc); if (!event) return 0; @@ -843,8 +868,8 @@ int __trace_bputs(unsigned long ip, const char *str) local_save_flags(irq_flags); buffer = global_trace.trace_buffer.buffer; - event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, - irq_flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, + irq_flags, pc); if (!event) return 0; @@ -1913,29 +1938,13 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, } EXPORT_SYMBOL_GPL(tracing_generic_entry_update); -static __always_inline void -trace_event_setup(struct ring_buffer_event *event, - int type, unsigned long flags, int pc) -{ - struct trace_entry *ent = ring_buffer_event_data(event); - - tracing_generic_entry_update(ent, flags, pc); - ent->type = type; -} - struct ring_buffer_event * trace_buffer_lock_reserve(struct ring_buffer *buffer, int type, unsigned long len, unsigned long flags, int pc) { - struct ring_buffer_event *event; - - event = ring_buffer_lock_reserve(buffer, len); - if (event != NULL) - trace_event_setup(event, type, flags, pc); - - return event; + return __trace_buffer_lock_reserve(buffer, type, len, flags, pc); } DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); @@ -2090,8 +2099,8 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, this_cpu_dec(trace_buffered_event_cnt); } - entry = trace_buffer_lock_reserve(*current_rb, - type, len, flags, pc); + entry = __trace_buffer_lock_reserve(*current_rb, + type, len, flags, pc); /* * If tracing is off, but we have triggers enabled * we still need to look at the event data. Use the temp_buffer @@ -2100,8 +2109,8 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, */ if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { *current_rb = temp_buffer; - entry = trace_buffer_lock_reserve(*current_rb, - type, len, flags, pc); + entry = __trace_buffer_lock_reserve(*current_rb, + type, len, flags, pc); } return entry; } @@ -2262,8 +2271,8 @@ trace_function(struct trace_array *tr, struct ring_buffer_event *event; struct ftrace_entry *entry; - event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), - flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), + flags, pc); if (!event) return; entry = ring_buffer_event_data(event); @@ -2342,8 +2351,8 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer, size *= sizeof(unsigned long); - event = trace_buffer_lock_reserve(buffer, TRACE_STACK, - sizeof(*entry) + size, flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, + sizeof(*entry) + size, flags, pc); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -2444,8 +2453,8 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) __this_cpu_inc(user_stack_count); - event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, - sizeof(*entry), flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, + sizeof(*entry), flags, pc); if (!event) goto out_drop_count; entry = ring_buffer_event_data(event); @@ -2615,8 +2624,8 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) local_save_flags(flags); size = sizeof(*entry) + sizeof(u32) * len; buffer = tr->trace_buffer.buffer; - event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, - flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, + flags, pc); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -2671,8 +2680,8 @@ __trace_array_vprintk(struct ring_buffer *buffer, local_save_flags(flags); size = sizeof(*entry) + len + 1; - event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, - flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, + flags, pc); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -5732,8 +5741,8 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, local_save_flags(irq_flags); size = sizeof(*entry) + cnt + 2; /* possible \n added */ buffer = tr->trace_buffer.buffer; - event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, - irq_flags, preempt_count()); + event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, + irq_flags, preempt_count()); if (!event) { /* Ring buffer disabled, return as if not open for write */ written = -EBADF; @@ -5810,8 +5819,8 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, local_save_flags(irq_flags); size = sizeof(*entry) + cnt; buffer = tr->trace_buffer.buffer; - event = trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, - irq_flags, preempt_count()); + event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, + irq_flags, preempt_count()); if (!event) { /* Ring buffer disabled, return as if not open for write */ written = -EBADF; -- cgit v1.3-7-g2ca7 From fa7ffb39efccd574163ebc5dbfe4ff066186f261 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 11:36:30 -0500 Subject: ring-buffer: Make rb_reserve_next_event() always inlined The function rb_reserved_next_event() is called by two functions: ring_buffer_lock_reserve() and ring_buffer_write(). This is in a very hot path of the tracing code, and it is best that they are not functions. The two callers are basically wrapers for rb_reserver_next_event(). Removing the function calls can save execution time in the hotpath of tracing. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c143739b8d7..1f3580cee6cc 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2733,7 +2733,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, return event; } -static struct ring_buffer_event * +static __always_inline struct ring_buffer_event * rb_reserve_next_event(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer, unsigned long length) -- cgit v1.3-7-g2ca7 From 929ddbf3ef4e07fef67e93e998020d49d2533724 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 11:40:34 -0500 Subject: ring-buffer: Always inline rb_event_data() The rb_event_data() is the fast path of getting the ring buffer data from an event. Externally, ring_buffer_event_data() is used to access this function. But unfortunately, rb_event_data() is not inlined, and calling ring_buffer_event_data() causes that function to be called again. Force rb_event_data() to be inlined to lower the number of operations needed when calling ring_buffer_event_data(). Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 1f3580cee6cc..2760aaca6d1b 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -245,7 +245,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event) EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */ -static void * +static __always_inline void * rb_event_data(struct ring_buffer_event *event) { if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) -- cgit v1.3-7-g2ca7 From 4239174570da080f3623724d97062bf55de7e36b Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 15:52:45 -0500 Subject: tracing: Make tracepoint_printk a static_key Currently, when tracepoint_printk is set (enabled by the "tp_printk" kernel command line), it causes trace events to print via printk(). This is a very dangerous operation, but is useful for debugging. The issue is, it's seldom used, but it is always checked even if it's not enabled by the kernel command line. Instead of having this feature called by a branch against a variable, turn that variable into a static key, and this will remove the test and jump. To simplify things, the functions output_printk() and trace_event_buffer_commit() were moved from trace_events.c to trace.c. Signed-off-by: Steven Rostedt --- include/linux/ftrace.h | 4 +++ kernel/sysctl.c | 2 +- kernel/trace/trace.c | 78 +++++++++++++++++++++++++++++++++++++++++++++ kernel/trace/trace_events.c | 40 ----------------------- 4 files changed, 83 insertions(+), 41 deletions(-) (limited to 'kernel/trace') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index b3d34d3e0e7e..8700049fd0e5 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -945,6 +945,10 @@ extern int __disable_trace_on_warning; #define INIT_TRACE_RECURSION .trace_recursion = 0, #endif +int tracepoint_printk_sysctl(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + #else /* CONFIG_TRACING */ static inline void disable_trace_on_warning(void) { } #endif /* CONFIG_TRACING */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 706309f9ed84..6ccc60dfbc7a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -634,7 +634,7 @@ static struct ctl_table kern_table[] = { .data = &tracepoint_printk, .maxlen = sizeof(tracepoint_printk), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = tracepoint_printk_sysctl, }, #endif #ifdef CONFIG_KEXEC_CORE diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 490533726b54..725e8b2c453f 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -69,6 +69,7 @@ bool __read_mostly tracing_selftest_disabled; /* Pipe tracepoints to printk */ struct trace_iterator *tracepoint_print_iter; int tracepoint_printk; +static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); /* For tracers that don't implement custom flags */ static struct tracer_opt dummy_tracer_opt[] = { @@ -2116,6 +2117,81 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb, } EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); +static DEFINE_SPINLOCK(tracepoint_iter_lock); +static DEFINE_MUTEX(tracepoint_printk_mutex); + +static void output_printk(struct trace_event_buffer *fbuffer) +{ + struct trace_event_call *event_call; + struct trace_event *event; + unsigned long flags; + struct trace_iterator *iter = tracepoint_print_iter; + + /* We should never get here if iter is NULL */ + if (WARN_ON_ONCE(!iter)) + return; + + event_call = fbuffer->trace_file->event_call; + if (!event_call || !event_call->event.funcs || + !event_call->event.funcs->trace) + return; + + event = &fbuffer->trace_file->event_call->event; + + spin_lock_irqsave(&tracepoint_iter_lock, flags); + trace_seq_init(&iter->seq); + iter->ent = fbuffer->entry; + event_call->event.funcs->trace(iter, 0, event); + trace_seq_putc(&iter->seq, 0); + printk("%s", iter->seq.buffer); + + spin_unlock_irqrestore(&tracepoint_iter_lock, flags); +} + +int tracepoint_printk_sysctl(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int save_tracepoint_printk; + int ret; + + mutex_lock(&tracepoint_printk_mutex); + save_tracepoint_printk = tracepoint_printk; + + ret = proc_dointvec(table, write, buffer, lenp, ppos); + + /* + * This will force exiting early, as tracepoint_printk + * is always zero when tracepoint_printk_iter is not allocated + */ + if (!tracepoint_print_iter) + tracepoint_printk = 0; + + if (save_tracepoint_printk == tracepoint_printk) + goto out; + + if (tracepoint_printk) + static_key_enable(&tracepoint_printk_key.key); + else + static_key_disable(&tracepoint_printk_key.key); + + out: + mutex_unlock(&tracepoint_printk_mutex); + + return ret; +} + +void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) +{ + if (static_key_false(&tracepoint_printk_key.key)) + output_printk(fbuffer); + + event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer, + fbuffer->event, fbuffer->entry, + fbuffer->flags, fbuffer->pc); +} +EXPORT_SYMBOL_GPL(trace_event_buffer_commit); + void trace_buffer_unlock_commit_regs(struct trace_array *tr, struct ring_buffer *buffer, struct ring_buffer_event *event, @@ -7977,6 +8053,8 @@ void __init trace_init(void) kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); if (WARN_ON(!tracepoint_print_iter)) tracepoint_printk = 0; + else + static_key_enable(&tracepoint_printk_key.key); } tracer_alloc_buffers(); trace_event_init(); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index ba67ede48822..d35fc2b0d304 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -283,46 +283,6 @@ void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer, } EXPORT_SYMBOL_GPL(trace_event_buffer_reserve); -static DEFINE_SPINLOCK(tracepoint_iter_lock); - -static void output_printk(struct trace_event_buffer *fbuffer) -{ - struct trace_event_call *event_call; - struct trace_event *event; - unsigned long flags; - struct trace_iterator *iter = tracepoint_print_iter; - - if (!iter) - return; - - event_call = fbuffer->trace_file->event_call; - if (!event_call || !event_call->event.funcs || - !event_call->event.funcs->trace) - return; - - event = &fbuffer->trace_file->event_call->event; - - spin_lock_irqsave(&tracepoint_iter_lock, flags); - trace_seq_init(&iter->seq); - iter->ent = fbuffer->entry; - event_call->event.funcs->trace(iter, 0, event); - trace_seq_putc(&iter->seq, 0); - printk("%s", iter->seq.buffer); - - spin_unlock_irqrestore(&tracepoint_iter_lock, flags); -} - -void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) -{ - if (tracepoint_printk) - output_printk(fbuffer); - - event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer, - fbuffer->event, fbuffer->entry, - fbuffer->flags, fbuffer->pc); -} -EXPORT_SYMBOL_GPL(trace_event_buffer_commit); - int trace_event_reg(struct trace_event_call *call, enum trace_reg type, void *data) { -- cgit v1.3-7-g2ca7 From 52ffabe3848a1ebd944cdf7801a77247b1cb46d5 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 20:28:38 -0500 Subject: tracing: Make __buffer_unlock_commit() always_inline The function __buffer_unlock_commit() is called in a few places outside of trace.c. But for the most part, it should really be inlined, as it is in the hot path of the trace_events. For the callers outside of trace.c, create a new function trace_buffer_unlock_commit_nostack(), as the reason it was used was to avoid the stack tracing that trace_buffer_unlock_commit() could do. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 41 +++++++++++++++++++++++------------- kernel/trace/trace.h | 4 ++-- kernel/trace/trace_branch.c | 2 +- kernel/trace/trace_functions_graph.c | 4 ++-- kernel/trace/trace_hwlat.c | 2 +- 5 files changed, 32 insertions(+), 21 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 725e8b2c453f..60416bf7c591 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -794,6 +794,22 @@ void tracing_on(void) } EXPORT_SYMBOL_GPL(tracing_on); + +static __always_inline void +__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) +{ + __this_cpu_write(trace_cmdline_save, true); + + /* If this is the temp buffer, we need to commit fully */ + if (this_cpu_read(trace_buffered_event) == event) { + /* Length is in event->array[0] */ + ring_buffer_write(buffer, event->array[0], &event->array[1]); + /* Release the temp buffer */ + this_cpu_dec(trace_buffered_event_cnt); + } else + ring_buffer_unlock_commit(buffer, event); +} + /** * __trace_puts - write a constant string into the trace buffer. * @ip: The address of the caller @@ -2059,21 +2075,6 @@ void trace_buffered_event_disable(void) preempt_enable(); } -void -__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event) -{ - __this_cpu_write(trace_cmdline_save, true); - - /* If this is the temp buffer, we need to commit fully */ - if (this_cpu_read(trace_buffered_event) == event) { - /* Length is in event->array[0] */ - ring_buffer_write(buffer, event->array[0], &event->array[1]); - /* Release the temp buffer */ - this_cpu_dec(trace_buffered_event_cnt); - } else - ring_buffer_unlock_commit(buffer, event); -} - static struct ring_buffer *temp_buffer; struct ring_buffer_event * @@ -2214,6 +2215,16 @@ void trace_buffer_unlock_commit_regs(struct trace_array *tr, ftrace_trace_userstack(buffer, flags, pc); } +/* + * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. + */ +void +trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer, + struct ring_buffer_event *event) +{ + __buffer_unlock_commit(buffer, event); +} + static void trace_process_export(struct trace_export *export, struct ring_buffer_event *event) diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 9294f8606ade..37602e722336 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -602,8 +602,8 @@ struct trace_entry *tracing_get_trace_entry(struct trace_array *tr, struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts); -void __buffer_unlock_commit(struct ring_buffer *buffer, - struct ring_buffer_event *event); +void trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer, + struct ring_buffer_event *event); int trace_empty(struct trace_iterator *iter); diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index 3a2a73716a5b..75489de546b6 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c @@ -81,7 +81,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) entry->correct = val == expect; if (!call_filter_check_discard(call, entry, buffer, event)) - __buffer_unlock_commit(buffer, event); + trace_buffer_unlock_commit_nostack(buffer, event); out: current->trace_recursion &= ~TRACE_BRANCH_BIT; diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 4e480e870474..8e1a115439fa 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -358,7 +358,7 @@ int __trace_graph_entry(struct trace_array *tr, entry = ring_buffer_event_data(event); entry->graph_ent = *trace; if (!call_filter_check_discard(call, entry, buffer, event)) - __buffer_unlock_commit(buffer, event); + trace_buffer_unlock_commit_nostack(buffer, event); return 1; } @@ -469,7 +469,7 @@ void __trace_graph_return(struct trace_array *tr, entry = ring_buffer_event_data(event); entry->ret = *trace; if (!call_filter_check_discard(call, entry, buffer, event)) - __buffer_unlock_commit(buffer, event); + trace_buffer_unlock_commit_nostack(buffer, event); } void trace_graph_return(struct ftrace_graph_ret *trace) diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index b97286c48735..775569ec50d0 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -127,7 +127,7 @@ static void trace_hwlat_sample(struct hwlat_sample *sample) entry->nmi_count = sample->nmi_count; if (!call_filter_check_discard(call, entry, buffer, event)) - __buffer_unlock_commit(buffer, event); + trace_buffer_unlock_commit_nostack(buffer, event); } /* Macros to encapsulate the time capturing infrastructure */ -- cgit v1.3-7-g2ca7 From 2289d5672f99d36764cbf66e13b5401f700e7043 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 20:35:32 -0500 Subject: ring-buffer: Force inline of hotpath helper functions There's several small helper functions in ring_buffer.c that are used in the hot path. For some reason, even though they are marked inline, gcc tends not to enforce it. Make sure these functions are always inlined. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 2760aaca6d1b..04068c0ed927 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1829,48 +1829,48 @@ void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val) } EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite); -static inline void * +static __always_inline void * __rb_data_page_index(struct buffer_data_page *bpage, unsigned index) { return bpage->data + index; } -static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index) +static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index) { return bpage->page->data + index; } -static inline struct ring_buffer_event * +static __always_inline struct ring_buffer_event * rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer) { return __rb_page_index(cpu_buffer->reader_page, cpu_buffer->reader_page->read); } -static inline struct ring_buffer_event * +static __always_inline struct ring_buffer_event * rb_iter_head_event(struct ring_buffer_iter *iter) { return __rb_page_index(iter->head_page, iter->head); } -static inline unsigned rb_page_commit(struct buffer_page *bpage) +static __always_inline unsigned rb_page_commit(struct buffer_page *bpage) { return local_read(&bpage->page->commit); } /* Size is determined by what has been committed */ -static inline unsigned rb_page_size(struct buffer_page *bpage) +static __always_inline unsigned rb_page_size(struct buffer_page *bpage) { return rb_page_commit(bpage); } -static inline unsigned +static __always_inline unsigned rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) { return rb_page_commit(cpu_buffer->commit_page); } -static inline unsigned +static __always_inline unsigned rb_event_index(struct ring_buffer_event *event) { unsigned long addr = (unsigned long)event; -- cgit v1.3-7-g2ca7 From babe3fce95e6490b88a2a21d90eb4ba9884edb82 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 20:38:39 -0500 Subject: ring-buffer: Froce rb_update_write_stamp() to be inlined The function rb_update_write_stamp() is in the hotpath of the ring buffer recording. Make sure that it is inlined as well. There's not many places that call it. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 04068c0ed927..04579a0c7711 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2486,7 +2486,7 @@ static inline void rb_event_discard(struct ring_buffer_event *event) event->time_delta = 1; } -static inline bool +static __always_inline bool rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event) { @@ -2500,7 +2500,7 @@ rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer, rb_commit_index(cpu_buffer) == index; } -static void +static __always_inline void rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event *event) { -- cgit v1.3-7-g2ca7 From 38e11df134297ea3860c7aad8263ece27db01308 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 23 Nov 2016 20:42:31 -0500 Subject: ring-buffer: Force rb_end_commit() and rb_set_commit_to_write() inline Both rb_end_commit() and rb_set_commit_to_write() are in the fast path of the ring buffer recording. Make sure they are always inlined. Link: http://lkml.kernel.org/r/20161121183700.GW26852@two.firstfloor.org Reported-by: Andi Kleen Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 04579a0c7711..f17476f9d7f4 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2386,7 +2386,7 @@ static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer) local_inc(&cpu_buffer->commits); } -static void +static __always_inline void rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) { unsigned long max_count; @@ -2441,7 +2441,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) goto again; } -static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer) +static __always_inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer) { unsigned long commits; -- cgit v1.3-7-g2ca7 From 80ec3552107ac16a836dbff4cf3c23fdd3256ee3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Mon, 28 Nov 2016 14:35:23 -0800 Subject: trace: Add an option for boot clock as trace clock Unlike monotonic clock, boot clock as a trace clock will account for time spent in suspend useful for tracing suspend/resume. This uses earlier introduced infrastructure for using the fast boot clock. Signed-off-by: Joel Fernandes Signed-off-by: John Stultz Reviewed-by: Thomas Gleixner Acked-by: Steven Rostedt Cc: Prarit Bhargava Cc: Richard Cochran Link: http://lkml.kernel.org/r/1480372524-15181-7-git-send-email-john.stultz@linaro.org Signed-off-by: Thomas Gleixner --- kernel/trace/trace.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8696ce6bf2f6..f7b64db285df 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1125,6 +1125,7 @@ static struct { { trace_clock, "perf", 1 }, { ktime_get_mono_fast_ns, "mono", 1 }, { ktime_get_raw_fast_ns, "mono_raw", 1 }, + { ktime_get_boot_fast_ns, "boot", 1 }, ARCH_TRACE_CLOCKS }; -- cgit v1.3-7-g2ca7 From b32614c03413f8a6025d8677c2b7c0ee976e63d4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:34 +0100 Subject: tracing/rb: Convert to hotplug state machine Install the callbacks via the state machine. The notifier in struct ring_buffer is replaced by the multi instance interface. Upon __ring_buffer_alloc() invocation, cpuhp_state_add_instance() will invoke the trace_rb_cpu_prepare() on each CPU. This callback may now fail. This means __ring_buffer_alloc() will fail and cleanup (like previously) and during a CPU up event this failure will not allow the CPU to come up. Signed-off-by: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161126231350.10321-7-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + include/linux/ring_buffer.h | 6 ++ kernel/trace/ring_buffer.c | 135 +++++++++++++++----------------------------- kernel/trace/trace.c | 15 ++++- 4 files changed, 66 insertions(+), 91 deletions(-) (limited to 'kernel/trace') diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index e3771fb959c0..18bcfeb2463e 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -62,6 +62,7 @@ enum cpuhp_state { CPUHP_TOPOLOGY_PREPARE, CPUHP_NET_IUCV_PREPARE, CPUHP_ARM_BL_PREPARE, + CPUHP_TRACE_RB_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 4acc552e9279..b6d4568795a7 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -198,4 +198,10 @@ enum ring_buffer_flags { RB_FL_OVERWRITE = 1 << 0, }; +#ifdef CONFIG_RING_BUFFER +int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); +#else +#define trace_rb_cpu_prepare NULL +#endif + #endif /* _LINUX_RING_BUFFER_H */ diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c143739b8d7..a7a055f167c7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -479,9 +479,7 @@ struct ring_buffer { struct ring_buffer_per_cpu **buffers; -#ifdef CONFIG_HOTPLUG_CPU - struct notifier_block cpu_notify; -#endif + struct hlist_node node; u64 (*clock)(void); struct rb_irq_work irq_work; @@ -1274,11 +1272,6 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } -#ifdef CONFIG_HOTPLUG_CPU -static int rb_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu); -#endif - /** * __ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. @@ -1296,6 +1289,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, long nr_pages; int bsize; int cpu; + int ret; /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), @@ -1318,17 +1312,6 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (nr_pages < 2) nr_pages = 2; - /* - * In case of non-hotplug cpu, if the ring-buffer is allocated - * in early initcall, it will not be notified of secondary cpus. - * In that off case, we need to allocate for all possible cpus. - */ -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_begin(); - cpumask_copy(buffer->cpumask, cpu_online_mask); -#else - cpumask_copy(buffer->cpumask, cpu_possible_mask); -#endif buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; @@ -1337,19 +1320,15 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (!buffer->buffers) goto fail_free_cpumask; - for_each_buffer_cpu(buffer, cpu) { - buffer->buffers[cpu] = - rb_allocate_cpu_buffer(buffer, nr_pages, cpu); - if (!buffer->buffers[cpu]) - goto fail_free_buffers; - } + cpu = raw_smp_processor_id(); + cpumask_set_cpu(cpu, buffer->cpumask); + buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); + if (!buffer->buffers[cpu]) + goto fail_free_buffers; -#ifdef CONFIG_HOTPLUG_CPU - buffer->cpu_notify.notifier_call = rb_cpu_notify; - buffer->cpu_notify.priority = 0; - __register_cpu_notifier(&buffer->cpu_notify); - cpu_notifier_register_done(); -#endif + ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); + if (ret < 0) + goto fail_free_buffers; mutex_init(&buffer->mutex); @@ -1364,9 +1343,6 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, fail_free_cpumask: free_cpumask_var(buffer->cpumask); -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_done(); -#endif fail_free_buffer: kfree(buffer); @@ -1383,18 +1359,11 @@ ring_buffer_free(struct ring_buffer *buffer) { int cpu; -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_begin(); - __unregister_cpu_notifier(&buffer->cpu_notify); -#endif + cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); for_each_buffer_cpu(buffer, cpu) rb_free_cpu_buffer(buffer->buffers[cpu]); -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_done(); -#endif - kfree(buffer->buffers); free_cpumask_var(buffer->cpumask); @@ -4633,62 +4602,48 @@ int ring_buffer_read_page(struct ring_buffer *buffer, } EXPORT_SYMBOL_GPL(ring_buffer_read_page); -#ifdef CONFIG_HOTPLUG_CPU -static int rb_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +/* + * We only allocate new buffers, never free them if the CPU goes down. + * If we were to free the buffer, then the user would lose any trace that was in + * the buffer. + */ +int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) { - struct ring_buffer *buffer = - container_of(self, struct ring_buffer, cpu_notify); - long cpu = (long)hcpu; + struct ring_buffer *buffer; long nr_pages_same; int cpu_i; unsigned long nr_pages; - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - if (cpumask_test_cpu(cpu, buffer->cpumask)) - return NOTIFY_OK; - - nr_pages = 0; - nr_pages_same = 1; - /* check if all cpu sizes are same */ - for_each_buffer_cpu(buffer, cpu_i) { - /* fill in the size from first enabled cpu */ - if (nr_pages == 0) - nr_pages = buffer->buffers[cpu_i]->nr_pages; - if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { - nr_pages_same = 0; - break; - } - } - /* allocate minimum pages, user can later expand it */ - if (!nr_pages_same) - nr_pages = 2; - buffer->buffers[cpu] = - rb_allocate_cpu_buffer(buffer, nr_pages, cpu); - if (!buffer->buffers[cpu]) { - WARN(1, "failed to allocate ring buffer on CPU %ld\n", - cpu); - return NOTIFY_OK; + buffer = container_of(node, struct ring_buffer, node); + if (cpumask_test_cpu(cpu, buffer->cpumask)) + return 0; + + nr_pages = 0; + nr_pages_same = 1; + /* check if all cpu sizes are same */ + for_each_buffer_cpu(buffer, cpu_i) { + /* fill in the size from first enabled cpu */ + if (nr_pages == 0) + nr_pages = buffer->buffers[cpu_i]->nr_pages; + if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { + nr_pages_same = 0; + break; } - smp_wmb(); - cpumask_set_cpu(cpu, buffer->cpumask); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - /* - * Do nothing. - * If we were to free the buffer, then the user would - * lose any trace that was in the buffer. - */ - break; - default: - break; } - return NOTIFY_OK; + /* allocate minimum pages, user can later expand it */ + if (!nr_pages_same) + nr_pages = 2; + buffer->buffers[cpu] = + rb_allocate_cpu_buffer(buffer, nr_pages, cpu); + if (!buffer->buffers[cpu]) { + WARN(1, "failed to allocate ring buffer on CPU %u\n", + cpu); + return -ENOMEM; + } + smp_wmb(); + cpumask_set_cpu(cpu, buffer->cpumask); + return 0; } -#endif #ifdef CONFIG_RING_BUFFER_STARTUP_TEST /* diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8696ce6bf2f6..465d56febc5b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7659,10 +7659,21 @@ __init static int tracer_alloc_buffers(void) raw_spin_lock_init(&global_trace.start_lock); + /* + * The prepare callbacks allocates some memory for the ring buffer. We + * don't free the buffer if the if the CPU goes down. If we were to free + * the buffer, then the user would lose any trace that was in the + * buffer. The memory will be removed once the "instance" is removed. + */ + ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, + "trace/RB:preapre", trace_rb_cpu_prepare, + NULL); + if (ret < 0) + goto out_free_cpumask; /* Used for event triggers */ temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); if (!temp_buffer) - goto out_free_cpumask; + goto out_rm_hp_state; if (trace_create_savedcmd() < 0) goto out_free_temp_buffer; @@ -7723,6 +7734,8 @@ out_free_savedcmd: free_saved_cmdlines_buffer(savedcmd); out_free_temp_buffer: ring_buffer_free(temp_buffer); +out_rm_hp_state: + cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); out_free_cpumask: free_cpumask_var(global_trace.tracing_cpumask); out_free_buffer_mask: -- cgit v1.3-7-g2ca7 From b18cc3de00ec3442cf40ac60787dbe0703b99e24 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 7 Dec 2016 14:31:33 +0100 Subject: tracing/rb: Init the CPU mask on allocation Before commit b32614c03413 ("tracing/rb: Convert to hotplug state machine") the allocated cpumask was initialized to the mask of online or possible CPUs. After the CPU hotplug changes the buffer initialization moved to trace_rb_cpu_prepare() but the cpumask is allocated with alloc_cpumask() and therefor has random content. As a consequence the cpu buffers are not initialized and a later access dereferences a NULL pointer. Use zalloc_cpumask() instead so trace_rb_cpu_prepare() initializes the buffers properly. Fixes: b32614c03413 ("tracing/rb: Convert to hotplug state machine") Reported-by: Tetsuo Handa Signed-off-by: Sebastian Andrzej Siewior Cc: rostedt@goodmis.org Link: http://lkml.kernel.org/r/20161207133133.hzkcqfllxcdi3joz@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index a7a055f167c7..89a2611a1635 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1297,7 +1297,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (!buffer) return NULL; - if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) + if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) goto fail_free_buffer; nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); -- cgit v1.3-7-g2ca7 From 8cf868affdc459beee1a941df0cfaba1673740e3 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 28 Nov 2016 13:03:21 -0500 Subject: tracing: Have the reg function allow to fail Some tracepoints have a registration function that gets enabled when the tracepoint is enabled. There may be cases that the registraction function must fail (for example, can't allocate enough memory). In this case, the tracepoint should also fail to register, otherwise the user would not know why the tracepoint is not working. Cc: David Howells Cc: Seiji Aguchi Cc: Anton Blanchard Cc: Mathieu Desnoyers Signed-off-by: Steven Rostedt --- arch/powerpc/include/asm/trace.h | 4 ++-- arch/powerpc/platforms/powernv/opal-tracepoints.c | 6 ++++-- arch/powerpc/platforms/pseries/lpar.c | 6 ++++-- arch/x86/include/asm/trace/exceptions.h | 2 +- arch/x86/include/asm/trace/irq_vectors.h | 2 +- arch/x86/kernel/tracepoint.c | 3 ++- drivers/i2c/i2c-core.c | 3 ++- include/linux/tracepoint-defs.h | 2 +- include/linux/tracepoint.h | 2 +- include/trace/events/i2c.h | 2 +- kernel/trace/trace_benchmark.c | 3 ++- kernel/trace/trace_benchmark.h | 2 +- kernel/tracepoint.c | 12 +++++++++--- samples/trace_events/trace-events-sample.c | 3 ++- samples/trace_events/trace-events-sample.h | 2 +- 15 files changed, 34 insertions(+), 20 deletions(-) (limited to 'kernel/trace') diff --git a/arch/powerpc/include/asm/trace.h b/arch/powerpc/include/asm/trace.h index 32e36b16773f..c05cef6ee06c 100644 --- a/arch/powerpc/include/asm/trace.h +++ b/arch/powerpc/include/asm/trace.h @@ -54,7 +54,7 @@ DEFINE_EVENT(ppc64_interrupt_class, timer_interrupt_exit, ); #ifdef CONFIG_PPC_PSERIES -extern void hcall_tracepoint_regfunc(void); +extern int hcall_tracepoint_regfunc(void); extern void hcall_tracepoint_unregfunc(void); TRACE_EVENT_FN_COND(hcall_entry, @@ -104,7 +104,7 @@ TRACE_EVENT_FN_COND(hcall_exit, #endif #ifdef CONFIG_PPC_POWERNV -extern void opal_tracepoint_regfunc(void); +extern int opal_tracepoint_regfunc(void); extern void opal_tracepoint_unregfunc(void); TRACE_EVENT_FN(opal_entry, diff --git a/arch/powerpc/platforms/powernv/opal-tracepoints.c b/arch/powerpc/platforms/powernv/opal-tracepoints.c index 1e496b780efd..3c447002edff 100644 --- a/arch/powerpc/platforms/powernv/opal-tracepoints.c +++ b/arch/powerpc/platforms/powernv/opal-tracepoints.c @@ -6,9 +6,10 @@ #ifdef HAVE_JUMP_LABEL struct static_key opal_tracepoint_key = STATIC_KEY_INIT; -void opal_tracepoint_regfunc(void) +int opal_tracepoint_regfunc(void) { static_key_slow_inc(&opal_tracepoint_key); + return 0; } void opal_tracepoint_unregfunc(void) @@ -25,9 +26,10 @@ void opal_tracepoint_unregfunc(void) /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ extern long opal_tracepoint_refcount; -void opal_tracepoint_regfunc(void) +int opal_tracepoint_regfunc(void) { opal_tracepoint_refcount++; + return 0; } void opal_tracepoint_unregfunc(void) diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index aa35245d8d6d..c0423ce3955c 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -661,9 +661,10 @@ EXPORT_SYMBOL(arch_free_page); #ifdef HAVE_JUMP_LABEL struct static_key hcall_tracepoint_key = STATIC_KEY_INIT; -void hcall_tracepoint_regfunc(void) +int hcall_tracepoint_regfunc(void) { static_key_slow_inc(&hcall_tracepoint_key); + return 0; } void hcall_tracepoint_unregfunc(void) @@ -680,9 +681,10 @@ void hcall_tracepoint_unregfunc(void) /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ extern long hcall_tracepoint_refcount; -void hcall_tracepoint_regfunc(void) +int hcall_tracepoint_regfunc(void) { hcall_tracepoint_refcount++; + return 0; } void hcall_tracepoint_unregfunc(void) diff --git a/arch/x86/include/asm/trace/exceptions.h b/arch/x86/include/asm/trace/exceptions.h index 2fbc66c7885b..2422b14c50a7 100644 --- a/arch/x86/include/asm/trace/exceptions.h +++ b/arch/x86/include/asm/trace/exceptions.h @@ -6,7 +6,7 @@ #include -extern void trace_irq_vector_regfunc(void); +extern int trace_irq_vector_regfunc(void); extern void trace_irq_vector_unregfunc(void); DECLARE_EVENT_CLASS(x86_exceptions, diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h index 38a09a13a9bc..32dd6a9e343c 100644 --- a/arch/x86/include/asm/trace/irq_vectors.h +++ b/arch/x86/include/asm/trace/irq_vectors.h @@ -6,7 +6,7 @@ #include -extern void trace_irq_vector_regfunc(void); +extern int trace_irq_vector_regfunc(void); extern void trace_irq_vector_unregfunc(void); DECLARE_EVENT_CLASS(x86_irq_vector, diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c index 1c113db9ed57..15515132bf0d 100644 --- a/arch/x86/kernel/tracepoint.c +++ b/arch/x86/kernel/tracepoint.c @@ -34,7 +34,7 @@ static void switch_idt(void *arg) local_irq_restore(flags); } -void trace_irq_vector_regfunc(void) +int trace_irq_vector_regfunc(void) { mutex_lock(&irq_vector_mutex); if (!trace_irq_vector_refcount) { @@ -44,6 +44,7 @@ void trace_irq_vector_regfunc(void) } trace_irq_vector_refcount++; mutex_unlock(&irq_vector_mutex); + return 0; } void trace_irq_vector_unregfunc(void) diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c index b432b64e307a..6a2b995d7fc4 100644 --- a/drivers/i2c/i2c-core.c +++ b/drivers/i2c/i2c-core.c @@ -77,9 +77,10 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver); static struct static_key i2c_trace_msg = STATIC_KEY_INIT_FALSE; static bool is_registered; -void i2c_transfer_trace_reg(void) +int i2c_transfer_trace_reg(void) { static_key_slow_inc(&i2c_trace_msg); + return 0; } void i2c_transfer_trace_unreg(void) diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h index 4ac89acb6136..a03192052066 100644 --- a/include/linux/tracepoint-defs.h +++ b/include/linux/tracepoint-defs.h @@ -29,7 +29,7 @@ struct tracepoint_func { struct tracepoint { const char *name; /* Tracepoint name */ struct static_key key; - void (*regfunc)(void); + int (*regfunc)(void); void (*unregfunc)(void); struct tracepoint_func __rcu *funcs; }; diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index be586c632a0c..f72fcfe0e66a 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -81,7 +81,7 @@ static inline void tracepoint_synchronize_unregister(void) } #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS -extern void syscall_regfunc(void); +extern int syscall_regfunc(void); extern void syscall_unregfunc(void); #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */ diff --git a/include/trace/events/i2c.h b/include/trace/events/i2c.h index fe17187df65d..4abb8eab34d3 100644 --- a/include/trace/events/i2c.h +++ b/include/trace/events/i2c.h @@ -20,7 +20,7 @@ /* * drivers/i2c/i2c-core.c */ -extern void i2c_transfer_trace_reg(void); +extern int i2c_transfer_trace_reg(void); extern void i2c_transfer_trace_unreg(void); /* diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index 0f109c4130d3..f76d0416dd83 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c @@ -164,11 +164,12 @@ static int benchmark_event_kthread(void *arg) * When the benchmark tracepoint is enabled, it calls this * function and the thread that calls the tracepoint is created. */ -void trace_benchmark_reg(void) +int trace_benchmark_reg(void) { bm_event_thread = kthread_run(benchmark_event_kthread, NULL, "event_benchmark"); WARN_ON(!bm_event_thread); + return 0; } /* diff --git a/kernel/trace/trace_benchmark.h b/kernel/trace/trace_benchmark.h index 3c1df1df4e29..ebdbfc2f2a64 100644 --- a/kernel/trace/trace_benchmark.h +++ b/kernel/trace/trace_benchmark.h @@ -6,7 +6,7 @@ #include -extern void trace_benchmark_reg(void); +extern int trace_benchmark_reg(void); extern void trace_benchmark_unreg(void); #define BENCHMARK_EVENT_STRLEN 128 diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index d0639d917899..1f9a31f934a4 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -194,9 +194,13 @@ static int tracepoint_add_func(struct tracepoint *tp, struct tracepoint_func *func, int prio) { struct tracepoint_func *old, *tp_funcs; + int ret; - if (tp->regfunc && !static_key_enabled(&tp->key)) - tp->regfunc(); + if (tp->regfunc && !static_key_enabled(&tp->key)) { + ret = tp->regfunc(); + if (ret < 0) + return ret; + } tp_funcs = rcu_dereference_protected(tp->funcs, lockdep_is_held(&tracepoints_mutex)); @@ -529,7 +533,7 @@ EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint); /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ static int sys_tracepoint_refcount; -void syscall_regfunc(void) +int syscall_regfunc(void) { struct task_struct *p, *t; @@ -541,6 +545,8 @@ void syscall_regfunc(void) read_unlock(&tasklist_lock); } sys_tracepoint_refcount++; + + return 0; } void syscall_unregfunc(void) diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c index 880a7d1d27d2..30e282d33d4d 100644 --- a/samples/trace_events/trace-events-sample.c +++ b/samples/trace_events/trace-events-sample.c @@ -79,7 +79,7 @@ static int simple_thread_fn(void *arg) static DEFINE_MUTEX(thread_mutex); -void foo_bar_reg(void) +int foo_bar_reg(void) { pr_info("Starting thread for foo_bar_fn\n"); /* @@ -90,6 +90,7 @@ void foo_bar_reg(void) mutex_lock(&thread_mutex); simple_tsk_fn = kthread_run(simple_thread_fn, NULL, "event-sample-fn"); mutex_unlock(&thread_mutex); + return 0; } void foo_bar_unreg(void) diff --git a/samples/trace_events/trace-events-sample.h b/samples/trace_events/trace-events-sample.h index d6b75bb495b3..76a75ab7a608 100644 --- a/samples/trace_events/trace-events-sample.h +++ b/samples/trace_events/trace-events-sample.h @@ -354,7 +354,7 @@ TRACE_EVENT_CONDITION(foo_bar_with_cond, TP_printk("foo %s %d", __get_str(foo), __entry->bar) ); -void foo_bar_reg(void); +int foo_bar_reg(void); void foo_bar_unreg(void); /* -- cgit v1.3-7-g2ca7 From 1dd349ab74d278cee50fc24dc26c023f3b149642 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 28 Nov 2016 13:17:25 -0500 Subject: tracing: Do not start benchmark on boot up Trace events are enabled very early on boot up via the boot command line parameter. The benchmark tool creates a new thread to perform the trace event benchmarking. But at start up, it is called before scheduling is set up and because it creates a new thread before the init thread is created, this crashes the kernel. Have the benchmark fail to register when started via the kernel command line. Also, since the registering of a tracepoint now can handle failure cases, return -ENOMEM instead of warning if the thread cannot be created. Signed-off-by: Steven Rostedt --- kernel/trace/trace_benchmark.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index f76d0416dd83..2bc7dc3e8ff8 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c @@ -166,9 +166,18 @@ static int benchmark_event_kthread(void *arg) */ int trace_benchmark_reg(void) { + if (system_state != SYSTEM_RUNNING) { + pr_warning("trace benchmark cannot be started via kernel command line\n"); + return -EBUSY; + } + bm_event_thread = kthread_run(benchmark_event_kthread, NULL, "event_benchmark"); - WARN_ON(!bm_event_thread); + if (!bm_event_thread) { + pr_warning("trace benchmark failed to create kernel thread\n"); + return -ENOMEM; + } + return 0; } @@ -183,6 +192,7 @@ void trace_benchmark_unreg(void) return; kthread_stop(bm_event_thread); + bm_event_thread = NULL; strcpy(bm_str, "START"); bm_total = 0; -- cgit v1.3-7-g2ca7 From 989a0a3d248192b6f5d16cc2aea95faed89bb7ce Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 28 Nov 2016 13:54:57 -0500 Subject: tracing: Have system enable return error if one of the events fail If one of the events within a system fails to enable when "1" is written to the system "enable" file, it should return an error. Note, some events may still be enabled, but the user should know that something did go wrong. Signed-off-by: Steven Rostedt --- kernel/trace/trace_events.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index d35fc2b0d304..93116549a284 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -702,6 +702,7 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match, struct trace_event_call *call; const char *name; int ret = -EINVAL; + int eret = 0; list_for_each_entry(file, &tr->events, list) { @@ -725,9 +726,17 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match, if (event && strcmp(event, name) != 0) continue; - ftrace_event_enable_disable(file, set); + ret = ftrace_event_enable_disable(file, set); - ret = 0; + /* + * Save the first error and return that. Some events + * may still have been enabled, but let the user + * know that something went wrong. + */ + if (ret && !eret) + eret = ret; + + ret = eret; } return ret; -- cgit v1.3-7-g2ca7 From 9c1f6bb8c88ab6c2779bdaf12d4f3407d336f085 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 28 Nov 2016 17:48:07 -0500 Subject: tracing: Allow benchmark to be enabled at early_initcall() The trace event start up selftests fails when the trace benchmark is enabled, because it is disabled during boot. It really only needs to be disabled before scheduling is set up, as it creates a thread. Signed-off-by: Steven Rostedt --- kernel/trace/trace_benchmark.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index 2bc7dc3e8ff8..e3b488825ae3 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c @@ -21,6 +21,8 @@ static u64 bm_stddev; static unsigned int bm_avg; static unsigned int bm_std; +static bool ok_to_run; + /* * This gets called in a loop recording the time it took to write * the tracepoint. What it writes is the time statistics of the last @@ -166,7 +168,7 @@ static int benchmark_event_kthread(void *arg) */ int trace_benchmark_reg(void) { - if (system_state != SYSTEM_RUNNING) { + if (!ok_to_run) { pr_warning("trace benchmark cannot be started via kernel command line\n"); return -EBUSY; } @@ -207,3 +209,12 @@ void trace_benchmark_unreg(void) bm_avg = 0; bm_stddev = 0; } + +static __init int ok_to_run_trace_benchmark(void) +{ + ok_to_run = true; + + return 0; +} + +early_initcall(ok_to_run_trace_benchmark); -- cgit v1.3-7-g2ca7 From 656c7f0d2d2b3237a31b105d5ed217a65350104f Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Thu, 8 Dec 2016 12:40:18 -0500 Subject: tracing: Replace kmap with copy_from_user() in trace_marker writing Instead of using get_user_pages_fast() and kmap_atomic() when writing to the trace_marker file, just allocate enough space on the ring buffer directly, and write into it via copy_from_user(). Writing into the trace_marker file use to allocate a temporary buffer to perform the copy_from_user(), as we didn't want to write into the ring buffer if the copy failed. But as a trace_marker write is suppose to be extremely fast, and allocating memory causes other tracepoints to trigger, Peter Zijlstra suggested using get_user_pages_fast() and kmap_atomic() to keep the user space pages in memory and reading it directly. But Henrik Austad had issues with this because it required taking the mm->mmap_sem and causing long delays with the write. Instead, just allocate the space in the ring buffer and use copy_from_user() directly. If it faults, return -EFAULT and write "" into the ring buffer. Link: http://lkml.kernel.org/r/20161208124018.72dd0f86@gandalf.local.home Cc: Ingo Molnar Cc: Henrik Austad Cc: Peter Zijlstra Updates: d696b58ca2c3ca "tracing: Do not allocate buffer for trace_marker" Suggested-by: Thomas Gleixner Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 139 ++++++++++++++------------------------------------- 1 file changed, 37 insertions(+), 102 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 60416bf7c591..6f420d7b703b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -5738,61 +5738,6 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp) return 0; } -static inline int lock_user_pages(const char __user *ubuf, size_t cnt, - struct page **pages, void **map_page, - int *offset) -{ - unsigned long addr = (unsigned long)ubuf; - int nr_pages = 1; - int ret; - int i; - - /* - * Userspace is injecting traces into the kernel trace buffer. - * We want to be as non intrusive as possible. - * To do so, we do not want to allocate any special buffers - * or take any locks, but instead write the userspace data - * straight into the ring buffer. - * - * First we need to pin the userspace buffer into memory, - * which, most likely it is, because it just referenced it. - * But there's no guarantee that it is. By using get_user_pages_fast() - * and kmap_atomic/kunmap_atomic() we can get access to the - * pages directly. We then write the data directly into the - * ring buffer. - */ - - /* check if we cross pages */ - if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK)) - nr_pages = 2; - - *offset = addr & (PAGE_SIZE - 1); - addr &= PAGE_MASK; - - ret = get_user_pages_fast(addr, nr_pages, 0, pages); - if (ret < nr_pages) { - while (--ret >= 0) - put_page(pages[ret]); - return -EFAULT; - } - - for (i = 0; i < nr_pages; i++) - map_page[i] = kmap_atomic(pages[i]); - - return nr_pages; -} - -static inline void unlock_user_pages(struct page **pages, - void **map_page, int nr_pages) -{ - int i; - - for (i = nr_pages - 1; i >= 0; i--) { - kunmap_atomic(map_page[i]); - put_page(pages[i]); - } -} - static ssize_t tracing_mark_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *fpos) @@ -5802,14 +5747,14 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, struct ring_buffer *buffer; struct print_entry *entry; unsigned long irq_flags; - struct page *pages[2]; - void *map_page[2]; - int nr_pages = 1; + const char faulted[] = ""; ssize_t written; - int offset; int size; int len; +/* Used in tracing_mark_raw_write() as well */ +#define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */ + if (tracing_disabled) return -EINVAL; @@ -5821,30 +5766,31 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); - nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset); - if (nr_pages < 0) - return nr_pages; - local_save_flags(irq_flags); - size = sizeof(*entry) + cnt + 2; /* possible \n added */ + size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */ + + /* If less than "", then make sure we can still add that */ + if (cnt < FAULTED_SIZE) + size += FAULTED_SIZE - cnt; + buffer = tr->trace_buffer.buffer; event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, irq_flags, preempt_count()); - if (!event) { + if (unlikely(!event)) /* Ring buffer disabled, return as if not open for write */ - written = -EBADF; - goto out_unlock; - } + return -EBADF; entry = ring_buffer_event_data(event); entry->ip = _THIS_IP_; - if (nr_pages == 2) { - len = PAGE_SIZE - offset; - memcpy(&entry->buf, map_page[0] + offset, len); - memcpy(&entry->buf[len], map_page[1], cnt - len); + len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); + if (len) { + memcpy(&entry->buf, faulted, FAULTED_SIZE); + cnt = FAULTED_SIZE; + written = -EFAULT; } else - memcpy(&entry->buf, map_page[0] + offset, cnt); + written = cnt; + len = cnt; if (entry->buf[cnt - 1] != '\n') { entry->buf[cnt] = '\n'; @@ -5854,12 +5800,8 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, __buffer_unlock_commit(buffer, event); - written = cnt; - - *fpos += written; - - out_unlock: - unlock_user_pages(pages, map_page, nr_pages); + if (written > 0) + *fpos += written; return written; } @@ -5875,15 +5817,14 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, struct ring_buffer_event *event; struct ring_buffer *buffer; struct raw_data_entry *entry; + const char faulted[] = ""; unsigned long irq_flags; - struct page *pages[2]; - void *map_page[2]; - int nr_pages = 1; ssize_t written; - int offset; int size; int len; +#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int)) + if (tracing_disabled) return -EINVAL; @@ -5899,38 +5840,32 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE); - nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset); - if (nr_pages < 0) - return nr_pages; - local_save_flags(irq_flags); size = sizeof(*entry) + cnt; + if (cnt < FAULT_SIZE_ID) + size += FAULT_SIZE_ID - cnt; + buffer = tr->trace_buffer.buffer; event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, irq_flags, preempt_count()); - if (!event) { + if (!event) /* Ring buffer disabled, return as if not open for write */ - written = -EBADF; - goto out_unlock; - } + return -EBADF; entry = ring_buffer_event_data(event); - if (nr_pages == 2) { - len = PAGE_SIZE - offset; - memcpy(&entry->id, map_page[0] + offset, len); - memcpy(((char *)&entry->id) + len, map_page[1], cnt - len); + len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); + if (len) { + entry->id = -1; + memcpy(&entry->buf, faulted, FAULTED_SIZE); + written = -EFAULT; } else - memcpy(&entry->id, map_page[0] + offset, cnt); + written = cnt; __buffer_unlock_commit(buffer, event); - written = cnt; - - *fpos += written; - - out_unlock: - unlock_user_pages(pages, map_page, nr_pages); + if (written > 0) + *fpos += written; return written; } -- cgit v1.3-7-g2ca7 From 794de08a16cf1fc1bf785dc48f66d36218cf6d88 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Thu, 8 Dec 2016 20:54:49 -0500 Subject: fgraph: Handle a case where a tracer ignores set_graph_notrace Both the wakeup and irqsoff tracers can use the function graph tracer when the display-graph option is set. The problem is that they ignore the notrace file, and record the entry of functions that would be ignored by the function_graph tracer. This causes the trace->depth to be recorded into the ring buffer. The set_graph_notrace uses a trick by adding a large negative number to the trace->depth when a graph function is to be ignored. On trace output, the graph function uses the depth to record a stack of functions. But since the depth is negative, it accesses the array with a negative number and causes an out of bounds access that can cause a kernel oops or corrupt data. Have the print functions handle cases where a tracer still records functions even when they are in set_graph_notrace. Also add warnings if the depth is below zero before accessing the array. Note, the function graph logic will still prevent the return of these functions from being recorded, which means that they will be left hanging without a return. For example: # echo '*spin*' > set_graph_notrace # echo 1 > options/display-graph # echo wakeup > current_tracer # cat trace [...] _raw_spin_lock() { preempt_count_add() { do_raw_spin_lock() { update_rq_clock(); Where it should look like: _raw_spin_lock() { preempt_count_add(); do_raw_spin_lock(); } update_rq_clock(); Cc: stable@vger.kernel.org Cc: Namhyung Kim Fixes: 29ad23b00474 ("ftrace: Add set_graph_notrace filter") Signed-off-by: Steven Rostedt --- kernel/trace/trace_functions_graph.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 8e1a115439fa..566f7327c3aa 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -842,6 +842,10 @@ print_graph_entry_leaf(struct trace_iterator *iter, cpu_data = per_cpu_ptr(data->cpu_data, cpu); + /* If a graph tracer ignored set_graph_notrace */ + if (call->depth < -1) + call->depth += FTRACE_NOTRACE_DEPTH; + /* * Comments display at + 1 to depth. Since * this is a leaf function, keep the comments @@ -850,7 +854,8 @@ print_graph_entry_leaf(struct trace_iterator *iter, cpu_data->depth = call->depth - 1; /* No need to keep this function around for this depth */ - if (call->depth < FTRACE_RETFUNC_DEPTH) + if (call->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(call->depth < 0)) cpu_data->enter_funcs[call->depth] = 0; } @@ -880,11 +885,16 @@ print_graph_entry_nested(struct trace_iterator *iter, struct fgraph_cpu_data *cpu_data; int cpu = iter->cpu; + /* If a graph tracer ignored set_graph_notrace */ + if (call->depth < -1) + call->depth += FTRACE_NOTRACE_DEPTH; + cpu_data = per_cpu_ptr(data->cpu_data, cpu); cpu_data->depth = call->depth; /* Save this function pointer to see if the exit matches */ - if (call->depth < FTRACE_RETFUNC_DEPTH) + if (call->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(call->depth < 0)) cpu_data->enter_funcs[call->depth] = call->func; } @@ -1114,7 +1124,8 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, */ cpu_data->depth = trace->depth - 1; - if (trace->depth < FTRACE_RETFUNC_DEPTH) { + if (trace->depth < FTRACE_RETFUNC_DEPTH && + !WARN_ON_ONCE(trace->depth < 0)) { if (cpu_data->enter_funcs[trace->depth] != trace->func) func_match = 0; cpu_data->enter_funcs[trace->depth] = 0; -- cgit v1.3-7-g2ca7 From 1a41442864e35bff859582fe9c5d051d0b1040ba Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Thu, 8 Dec 2016 19:28:28 -0500 Subject: tracing/fgraph: Have wakeup and irqsoff tracers ignore graph functions too Currently both the wakeup and irqsoff traces do not handle set_graph_notrace well. The ftrace infrastructure will ignore the return paths of all functions leaving them hanging without an end: # echo '*spin*' > set_graph_notrace # cat trace [...] _raw_spin_lock() { preempt_count_add() { do_raw_spin_lock() { update_rq_clock(); Where the '*spin*' functions should have looked like this: _raw_spin_lock() { preempt_count_add(); do_raw_spin_lock(); } update_rq_clock(); Instead, have the wakeup and irqsoff tracers ignore the functions that are set by the set_graph_notrace like the function_graph tracer does. Move the logic in the function_graph tracer into a header to allow wakeup and irqsoff tracers to use it as well. Cc: Namhyung Kim Signed-off-by: Steven Rostedt --- kernel/trace/trace.h | 11 +++++++++++ kernel/trace/trace_functions_graph.c | 14 +++++++------- kernel/trace/trace_irqsoff.c | 12 ++++++++++++ kernel/trace/trace_sched_wakeup.c | 12 ++++++++++++ 4 files changed, 42 insertions(+), 7 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 37602e722336..c2234494f40c 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -846,6 +846,17 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr) return 0; } #endif /* CONFIG_DYNAMIC_FTRACE */ + +extern unsigned int fgraph_max_depth; + +static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace) +{ + /* trace it when it is-nested-in or is a function enabled. */ + return !(trace->depth || ftrace_graph_addr(trace->func)) || + (trace->depth < 0) || + (fgraph_max_depth && trace->depth >= fgraph_max_depth); +} + #else /* CONFIG_FUNCTION_GRAPH_TRACER */ static inline enum print_line_t print_graph_function_flags(struct trace_iterator *iter, u32 flags) diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 566f7327c3aa..d56123cdcc89 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -65,7 +65,7 @@ struct fgraph_data { #define TRACE_GRAPH_INDENT 2 -static unsigned int max_depth; +unsigned int fgraph_max_depth; static struct tracer_opt trace_opts[] = { /* Display overruns? (for self-debug purpose) */ @@ -384,10 +384,10 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) if (!ftrace_trace_task(tr)) return 0; - /* trace it when it is-nested-in or is a function enabled. */ - if ((!(trace->depth || ftrace_graph_addr(trace->func)) || - ftrace_graph_ignore_irqs()) || (trace->depth < 0) || - (max_depth && trace->depth >= max_depth)) + if (ftrace_graph_ignore_func(trace)) + return 0; + + if (ftrace_graph_ignore_irqs()) return 0; /* @@ -1500,7 +1500,7 @@ graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt, if (ret) return ret; - max_depth = val; + fgraph_max_depth = val; *ppos += cnt; @@ -1514,7 +1514,7 @@ graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt, char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/ int n; - n = sprintf(buf, "%d\n", max_depth); + n = sprintf(buf, "%d\n", fgraph_max_depth); return simple_read_from_buffer(ubuf, cnt, ppos, buf, n); } diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index 03cdff84d026..86654d7e1afe 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c @@ -175,6 +175,18 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) int ret; int pc; + if (ftrace_graph_ignore_func(trace)) + return 0; + /* + * Do not trace a function if it's filtered by set_graph_notrace. + * Make the index of ret stack negative to indicate that it should + * ignore further functions. But it needs its own ret stack entry + * to recover the original index in order to continue tracing after + * returning from the function. + */ + if (ftrace_graph_notrace_addr(trace->func)) + return 1; + if (!func_prolog_dec(tr, &data, &flags)) return 0; diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 1bf2324dc682..5d0bb025bb21 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -239,6 +239,18 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace) unsigned long flags; int pc, ret = 0; + if (ftrace_graph_ignore_func(trace)) + return 0; + /* + * Do not trace a function if it's filtered by set_graph_notrace. + * Make the index of ret stack negative to indicate that it should + * ignore further functions. But it needs its own ret stack entry + * to recover the original index in order to continue tracing after + * returning from the function. + */ + if (ftrace_graph_notrace_addr(trace->func)) + return 1; + if (!func_prolog_preempt_disable(tr, &data, &pc)) return 0; -- cgit v1.3-7-g2ca7 From c59f29cb144a6a0dfac16ede9dc8eafc02dc56ca Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 9 Dec 2016 21:50:17 +0530 Subject: tracing: Use SOFTIRQ_OFFSET for softirq dectection for more accurate results The 's' flag is supposed to indicate that a softirq is running. This can be detected by testing the preempt_count with SOFTIRQ_OFFSET. The current code tests the preempt_count with SOFTIRQ_MASK, which would be true even when softirqs are disabled but not serving a softirq. Link: http://lkml.kernel.org/r/1481300417-3564-1-git-send-email-pkondeti@codeaurora.org Signed-off-by: Pavankumar Kondeti Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6f420d7b703b..970aafe80b49 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1949,7 +1949,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, #endif ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) | ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | - ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | + ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) | (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); } -- cgit v1.3-7-g2ca7 From 99e6f6e8134b0c9d5d82eb2af1068a57199125e4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 7 Dec 2016 14:31:33 +0100 Subject: tracing/rb: Init the CPU mask on allocation Before commit b32614c03413 ("tracing/rb: Convert to hotplug state machine") the allocated cpumask was initialized to the mask of ONLINE or POSSIBLE CPUs. After the CPU hotplug changes the buffer initialisation moved to trace_rb_cpu_prepare() but I forgot to initially set the cpumask to zero. This is done now. Link: http://lkml.kernel.org/r/20161207133133.hzkcqfllxcdi3joz@linutronix.de Fixes: b32614c03413 ("tracing/rb: Convert to hotplug state machine") Reported-by: Tetsuo Handa Tested-by: Tetsuo Handa Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index f17476f9d7f4..7edfd41d506c 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1303,7 +1303,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (!buffer) return NULL; - if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) + if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) goto fail_free_buffer; nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); -- cgit v1.3-7-g2ca7 From f18f97ac43d72ae84fe012dd4f19fe3f8f901469 Mon Sep 17 00:00:00 2001 From: Marcin Nowakowski Date: Fri, 9 Dec 2016 15:19:37 +0100 Subject: tracing/kprobes: Add a helper method to return number of probe hits The number of probe hits is stored in a percpu variable and therefore can't be read directly. Add a helper method trace_kprobe_nhit() that performs the required calculation. It will be used in a follow-up commit that changes kprobe selftests to verify the number of probe hits. Link: http://lkml.kernel.org/r/1481293178-3128-1-git-send-email-marcin.nowakowski@imgtec.com Acked-by: Masami Hiramatsu Signed-off-by: Marcin Nowakowski Signed-off-by: Steven Rostedt --- kernel/trace/trace_kprobe.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index eb6c9f1d3a93..a2af1bceca29 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -73,6 +73,17 @@ static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk) return !!strchr(trace_kprobe_symbol(tk), ':'); } +static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) +{ + unsigned long nhit = 0; + int cpu; + + for_each_possible_cpu(cpu) + nhit += *per_cpu_ptr(tk->nhit, cpu); + + return nhit; +} + static int register_kprobe_event(struct trace_kprobe *tk); static int unregister_kprobe_event(struct trace_kprobe *tk); @@ -882,14 +893,10 @@ static const struct file_operations kprobe_events_ops = { static int probes_profile_seq_show(struct seq_file *m, void *v) { struct trace_kprobe *tk = v; - unsigned long nhit = 0; - int cpu; - - for_each_possible_cpu(cpu) - nhit += *per_cpu_ptr(tk->nhit, cpu); seq_printf(m, " %-44s %15lu %15lu\n", - trace_event_name(&tk->tp.call), nhit, + trace_event_name(&tk->tp.call), + trace_kprobe_nhit(tk), tk->rp.kp.nmissed); return 0; -- cgit v1.3-7-g2ca7 From d4d7ccc834fe2401084e01fb043ad70ac410b19d Mon Sep 17 00:00:00 2001 From: Marcin Nowakowski Date: Fri, 9 Dec 2016 15:19:38 +0100 Subject: kprobes/trace: Fix kprobe selftest for newer gcc Commit 265a5b7ee3eb ("kprobes/trace: Fix kprobe selftest for gcc 4.6") has added __used attribute to kprobe_trace_selftest_target to ensure that the method is listed in kallsyms table. However, even though the method remains in the kernel image, the actual call is optimized away as there are no side effects and the return value is never checked. Add a return value check and a 'noinline' attribute to ensure that an inlined copy of the method is not used by the caller. Also add checks that verify that the kprobe was really hit, as at the moment the tests show positive results despite the test method being optimized away. Finally, add __init annotations to find_trace_probe_file() and kprobe_trace_selftest_target() as they are only called from within an __init method. Link: http://lkml.kernel.org/r/1481293178-3128-2-git-send-email-marcin.nowakowski@imgtec.com Acked-by: Masami Hiramatsu Signed-off-by: Marcin Nowakowski Signed-off-by: Steven Rostedt --- kernel/trace/trace_kprobe.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index a2af1bceca29..a133ecd741e4 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1361,18 +1361,18 @@ fs_initcall(init_kprobe_trace); #ifdef CONFIG_FTRACE_STARTUP_TEST - /* * The "__used" keeps gcc from removing the function symbol - * from the kallsyms table. + * from the kallsyms table. 'noinline' makes sure that there + * isn't an inlined version used by the test method below */ -static __used int kprobe_trace_selftest_target(int a1, int a2, int a3, - int a4, int a5, int a6) +static __used __init noinline int +kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6) { return a1 + a2 + a3 + a4 + a5 + a6; } -static struct trace_event_file * +static struct __init trace_event_file * find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) { struct trace_event_file *file; @@ -1450,12 +1450,25 @@ static __init int kprobe_trace_self_tests_init(void) ret = target(1, 2, 3, 4, 5, 6); + /* + * Not expecting an error here, the check is only to prevent the + * optimizer from removing the call to target() as otherwise there + * are no side-effects and the call is never performed. + */ + if (ret != 21) + warn++; + /* Disable trace points before removing it */ tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); if (WARN_ON_ONCE(tk == NULL)) { pr_warn("error on getting test probe.\n"); warn++; } else { + if (trace_kprobe_nhit(tk) != 1) { + pr_warn("incorrect number of testprobe hits\n"); + warn++; + } + file = find_trace_probe_file(tk, top_trace_array()); if (WARN_ON_ONCE(file == NULL)) { pr_warn("error on getting probe file.\n"); @@ -1469,6 +1482,11 @@ static __init int kprobe_trace_self_tests_init(void) pr_warn("error on getting 2nd test probe.\n"); warn++; } else { + if (trace_kprobe_nhit(tk) != 1) { + pr_warn("incorrect number of testprobe2 hits\n"); + warn++; + } + file = find_trace_probe_file(tk, top_trace_array()); if (WARN_ON_ONCE(file == NULL)) { pr_warn("error on getting probe file.\n"); -- cgit v1.3-7-g2ca7 From a5a1d1c2914b5316924c7893eb683a5420ebd3be Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Dec 2016 20:32:01 +0100 Subject: clocksource: Use a plain u64 instead of cycle_t There is no point in having an extra type for extra confusion. u64 is unambiguous. Conversion was done with the following coccinelle script: @rem@ @@ -typedef u64 cycle_t; @fix@ typedef cycle_t; @@ -cycle_t +u64 Signed-off-by: Thomas Gleixner Cc: Peter Zijlstra Cc: John Stultz --- arch/alpha/kernel/time.c | 4 +- arch/arm/mach-davinci/time.c | 2 +- arch/arm/mach-ep93xx/timer-ep93xx.c | 4 +- arch/arm/mach-footbridge/dc21285-timer.c | 2 +- arch/arm/mach-ixp4xx/common.c | 2 +- arch/arm/mach-mmp/time.c | 2 +- arch/arm/mach-omap2/timer.c | 4 +- arch/arm/plat-iop/time.c | 2 +- arch/avr32/kernel/time.c | 4 +- arch/blackfin/kernel/time-ts.c | 4 +- arch/c6x/kernel/time.c | 2 +- arch/hexagon/kernel/time.c | 4 +- arch/ia64/kernel/cyclone.c | 4 +- arch/ia64/kernel/fsyscall_gtod_data.h | 6 +-- arch/ia64/kernel/time.c | 6 +-- arch/ia64/sn/kernel/sn2/timer.c | 4 +- arch/m68k/68000/timers.c | 2 +- arch/m68k/coldfire/dma_timer.c | 2 +- arch/m68k/coldfire/pit.c | 2 +- arch/m68k/coldfire/sltimers.c | 2 +- arch/m68k/coldfire/timers.c | 2 +- arch/microblaze/kernel/timer.c | 6 +-- arch/mips/alchemy/common/time.c | 2 +- arch/mips/cavium-octeon/csrc-octeon.c | 2 +- arch/mips/jz4740/time.c | 2 +- arch/mips/kernel/cevt-txx9.c | 2 +- arch/mips/kernel/csrc-bcm1480.c | 4 +- arch/mips/kernel/csrc-ioasic.c | 2 +- arch/mips/kernel/csrc-r4k.c | 2 +- arch/mips/kernel/csrc-sb1250.c | 4 +- arch/mips/loongson32/common/time.c | 4 +- arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c | 4 +- arch/mips/loongson64/loongson-3/hpet.c | 4 +- arch/mips/mti-malta/malta-time.c | 2 +- arch/mips/netlogic/common/time.c | 4 +- arch/mips/sgi-ip27/ip27-timer.c | 2 +- arch/mn10300/kernel/csrc-mn10300.c | 2 +- arch/nios2/kernel/time.c | 2 +- arch/openrisc/kernel/time.c | 4 +- arch/parisc/kernel/time.c | 2 +- arch/powerpc/kernel/time.c | 14 +++--- arch/s390/kernel/time.c | 2 +- arch/sparc/kernel/time_32.c | 2 +- arch/sparc/kernel/time_64.c | 2 +- arch/um/kernel/time.c | 2 +- arch/unicore32/kernel/time.c | 2 +- arch/x86/entry/vdso/vclock_gettime.c | 8 +-- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/include/asm/pvclock.h | 7 ++- arch/x86/include/asm/tsc.h | 2 +- arch/x86/include/asm/vgtod.h | 4 +- arch/x86/kernel/apb_timer.c | 4 +- arch/x86/kernel/cpu/mshyperv.c | 4 +- arch/x86/kernel/hpet.c | 14 +++--- arch/x86/kernel/kvmclock.c | 10 ++-- arch/x86/kernel/pvclock.c | 4 +- arch/x86/kernel/tsc.c | 6 +-- arch/x86/kvm/x86.c | 14 +++--- arch/x86/lguest/boot.c | 2 +- arch/x86/platform/uv/uv_time.c | 8 +-- arch/x86/xen/time.c | 6 +-- arch/x86/xen/xen-ops.h | 2 +- arch/xtensa/kernel/time.c | 4 +- drivers/char/hpet.c | 4 +- drivers/clocksource/acpi_pm.c | 14 +++--- drivers/clocksource/arc_timer.c | 12 ++--- drivers/clocksource/arm_arch_timer.c | 4 +- drivers/clocksource/arm_global_timer.c | 2 +- drivers/clocksource/cadence_ttc_timer.c | 4 +- drivers/clocksource/clksrc-dbx500-prcmu.c | 2 +- drivers/clocksource/dw_apb_timer.c | 8 +-- drivers/clocksource/em_sti.c | 12 ++--- drivers/clocksource/exynos_mct.c | 6 +-- drivers/clocksource/h8300_timer16.c | 2 +- drivers/clocksource/h8300_tpu.c | 2 +- drivers/clocksource/i8253.c | 4 +- drivers/clocksource/jcore-pit.c | 2 +- drivers/clocksource/metag_generic.c | 2 +- drivers/clocksource/mips-gic-timer.c | 2 +- drivers/clocksource/mmio.c | 18 +++---- drivers/clocksource/mxs_timer.c | 2 +- drivers/clocksource/qcom-timer.c | 2 +- drivers/clocksource/samsung_pwm_timer.c | 2 +- drivers/clocksource/scx200_hrt.c | 4 +- drivers/clocksource/sh_cmt.c | 2 +- drivers/clocksource/sh_tmu.c | 2 +- drivers/clocksource/tcb_clksrc.c | 4 +- drivers/clocksource/time-pistachio.c | 4 +- drivers/clocksource/timer-atlas7.c | 2 +- drivers/clocksource/timer-atmel-pit.c | 2 +- drivers/clocksource/timer-atmel-st.c | 2 +- drivers/clocksource/timer-nps.c | 4 +- drivers/clocksource/timer-prima2.c | 2 +- drivers/clocksource/timer-sun5i.c | 2 +- drivers/clocksource/timer-ti-32k.c | 4 +- drivers/clocksource/vt8500_timer.c | 4 +- drivers/hv/hv.c | 8 +-- drivers/irqchip/irq-mips-gic.c | 16 +++--- drivers/net/ethernet/amd/xgbe/xgbe-ptp.c | 2 +- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 2 +- drivers/net/ethernet/freescale/fec_ptp.c | 2 +- drivers/net/ethernet/intel/e1000e/netdev.c | 18 +++---- drivers/net/ethernet/intel/e1000e/ptp.c | 4 +- drivers/net/ethernet/intel/igb/igb_ptp.c | 4 +- drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c | 4 +- drivers/net/ethernet/mellanox/mlx4/en_clock.c | 2 +- drivers/net/ethernet/mellanox/mlx4/main.c | 4 +- drivers/net/ethernet/mellanox/mlx5/core/en_clock.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/main.c | 4 +- .../net/ethernet/mellanox/mlx5/core/mlx5_core.h | 2 +- drivers/net/ethernet/ti/cpts.c | 2 +- include/kvm/arm_arch_timer.h | 4 +- include/linux/clocksource.h | 22 ++++----- include/linux/dw_apb_timer.h | 2 +- include/linux/irqchip/mips-gic.h | 8 +-- include/linux/mlx4/device.h | 2 +- include/linux/timecounter.h | 12 ++--- include/linux/timekeeper_internal.h | 10 ++-- include/linux/timekeeping.h | 4 +- include/linux/types.h | 3 -- kernel/time/clocksource.c | 2 +- kernel/time/jiffies.c | 4 +- kernel/time/timecounter.c | 6 +-- kernel/time/timekeeping.c | 57 ++++++++++------------ kernel/time/timekeeping_internal.h | 6 +-- kernel/trace/ftrace.c | 4 +- kernel/trace/trace.c | 6 +-- kernel/trace/trace.h | 8 +-- kernel/trace/trace_irqsoff.c | 4 +- kernel/trace/trace_sched_wakeup.c | 4 +- sound/hda/hdac_stream.c | 6 +-- virt/kvm/arm/arch_timer.c | 6 +-- 132 files changed, 320 insertions(+), 327 deletions(-) (limited to 'kernel/trace') diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 992000e3d9e4..3bfe058d75d9 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -133,7 +133,7 @@ init_rtc_clockevent(void) * The QEMU clock as a clocksource primitive. */ -static cycle_t +static u64 qemu_cs_read(struct clocksource *cs) { return qemu_get_vmtime(); @@ -260,7 +260,7 @@ common_init_rtc(void) * use this method when WTINT is in use. */ -static cycle_t read_rpcc(struct clocksource *cs) +static u64 read_rpcc(struct clocksource *cs) { return rpcc(); } diff --git a/arch/arm/mach-davinci/time.c b/arch/arm/mach-davinci/time.c index 6c18445a4639..034f865fe78e 100644 --- a/arch/arm/mach-davinci/time.c +++ b/arch/arm/mach-davinci/time.c @@ -268,7 +268,7 @@ static void __init timer_init(void) /* * clocksource */ -static cycle_t read_cycles(struct clocksource *cs) +static u64 read_cycles(struct clocksource *cs) { struct timer_s *t = &timers[TID_CLOCKSOURCE]; diff --git a/arch/arm/mach-ep93xx/timer-ep93xx.c b/arch/arm/mach-ep93xx/timer-ep93xx.c index e5f791145bd0..874cbc91b669 100644 --- a/arch/arm/mach-ep93xx/timer-ep93xx.c +++ b/arch/arm/mach-ep93xx/timer-ep93xx.c @@ -59,13 +59,13 @@ static u64 notrace ep93xx_read_sched_clock(void) return ret; } -cycle_t ep93xx_clocksource_read(struct clocksource *c) +u64 ep93xx_clocksource_read(struct clocksource *c) { u64 ret; ret = readl(EP93XX_TIMER4_VALUE_LOW); ret |= ((u64) (readl(EP93XX_TIMER4_VALUE_HIGH) & 0xff) << 32); - return (cycle_t) ret; + return (u64) ret; } static int ep93xx_clkevt_set_next_event(unsigned long next, diff --git a/arch/arm/mach-footbridge/dc21285-timer.c b/arch/arm/mach-footbridge/dc21285-timer.c index 810edc78c817..75395a720e63 100644 --- a/arch/arm/mach-footbridge/dc21285-timer.c +++ b/arch/arm/mach-footbridge/dc21285-timer.c @@ -19,7 +19,7 @@ #include "common.h" -static cycle_t cksrc_dc21285_read(struct clocksource *cs) +static u64 cksrc_dc21285_read(struct clocksource *cs) { return cs->mask - *CSR_TIMER2_VALUE; } diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 0f08f611c1a6..846e033c56fa 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -493,7 +493,7 @@ static u64 notrace ixp4xx_read_sched_clock(void) * clocksource */ -static cycle_t ixp4xx_clocksource_read(struct clocksource *c) +static u64 ixp4xx_clocksource_read(struct clocksource *c) { return *IXP4XX_OSTS; } diff --git a/arch/arm/mach-mmp/time.c b/arch/arm/mach-mmp/time.c index 3c2c92aaa0ae..96ad1db0b04b 100644 --- a/arch/arm/mach-mmp/time.c +++ b/arch/arm/mach-mmp/time.c @@ -144,7 +144,7 @@ static struct clock_event_device ckevt = { .set_state_oneshot = timer_set_shutdown, }; -static cycle_t clksrc_read(struct clocksource *cs) +static u64 clksrc_read(struct clocksource *cs) { return timer_read(); } diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c index 5e2e2218a402..56128da23c3a 100644 --- a/arch/arm/mach-omap2/timer.c +++ b/arch/arm/mach-omap2/timer.c @@ -369,9 +369,9 @@ static bool use_gptimer_clksrc __initdata; /* * clocksource */ -static cycle_t clocksource_read_cycles(struct clocksource *cs) +static u64 clocksource_read_cycles(struct clocksource *cs) { - return (cycle_t)__omap_dm_timer_read_counter(&clksrc, + return (u64)__omap_dm_timer_read_counter(&clksrc, OMAP_TIMER_NONPOSTED); } diff --git a/arch/arm/plat-iop/time.c b/arch/arm/plat-iop/time.c index ed8d129d4bea..2cff0010f677 100644 --- a/arch/arm/plat-iop/time.c +++ b/arch/arm/plat-iop/time.c @@ -38,7 +38,7 @@ /* * IOP clocksource (free-running timer 1). */ -static cycle_t notrace iop_clocksource_read(struct clocksource *unused) +static u64 notrace iop_clocksource_read(struct clocksource *unused) { return 0xffffffffu - read_tcr1(); } diff --git a/arch/avr32/kernel/time.c b/arch/avr32/kernel/time.c index a124c55733db..4d9b69615979 100644 --- a/arch/avr32/kernel/time.c +++ b/arch/avr32/kernel/time.c @@ -20,9 +20,9 @@ static bool disable_cpu_idle_poll; -static cycle_t read_cycle_count(struct clocksource *cs) +static u64 read_cycle_count(struct clocksource *cs) { - return (cycle_t)sysreg_read(COUNT); + return (u64)sysreg_read(COUNT); } /* diff --git a/arch/blackfin/kernel/time-ts.c b/arch/blackfin/kernel/time-ts.c index fb9e95f1b719..0e9fcf841d67 100644 --- a/arch/blackfin/kernel/time-ts.c +++ b/arch/blackfin/kernel/time-ts.c @@ -26,7 +26,7 @@ #if defined(CONFIG_CYCLES_CLOCKSOURCE) -static notrace cycle_t bfin_read_cycles(struct clocksource *cs) +static notrace u64 bfin_read_cycles(struct clocksource *cs) { #ifdef CONFIG_CPU_FREQ return __bfin_cycles_off + (get_cycles() << __bfin_cycles_mod); @@ -80,7 +80,7 @@ void __init setup_gptimer0(void) enable_gptimers(TIMER0bit); } -static cycle_t bfin_read_gptimer0(struct clocksource *cs) +static u64 bfin_read_gptimer0(struct clocksource *cs) { return bfin_read_TIMER0_COUNTER(); } diff --git a/arch/c6x/kernel/time.c b/arch/c6x/kernel/time.c index 04845aaf5985..6a8e00a1f6d5 100644 --- a/arch/c6x/kernel/time.c +++ b/arch/c6x/kernel/time.c @@ -26,7 +26,7 @@ static u32 sched_clock_multiplier; #define SCHED_CLOCK_SHIFT 16 -static cycle_t tsc_read(struct clocksource *cs) +static u64 tsc_read(struct clocksource *cs) { return get_cycles(); } diff --git a/arch/hexagon/kernel/time.c b/arch/hexagon/kernel/time.c index a6a1d1f8309a..ff4e9bf995e9 100644 --- a/arch/hexagon/kernel/time.c +++ b/arch/hexagon/kernel/time.c @@ -72,9 +72,9 @@ struct adsp_hw_timer_struct { /* Look for "TCX0" for related constants. */ static __iomem struct adsp_hw_timer_struct *rtos_timer; -static cycle_t timer_get_cycles(struct clocksource *cs) +static u64 timer_get_cycles(struct clocksource *cs) { - return (cycle_t) __vmgettime(); + return (u64) __vmgettime(); } static struct clocksource hexagon_clocksource = { diff --git a/arch/ia64/kernel/cyclone.c b/arch/ia64/kernel/cyclone.c index 5fa3848ba224..ee1a4afbf9da 100644 --- a/arch/ia64/kernel/cyclone.c +++ b/arch/ia64/kernel/cyclone.c @@ -21,9 +21,9 @@ void __init cyclone_setup(void) static void __iomem *cyclone_mc; -static cycle_t read_cyclone(struct clocksource *cs) +static u64 read_cyclone(struct clocksource *cs) { - return (cycle_t)readq((void __iomem *)cyclone_mc); + return (u64)readq((void __iomem *)cyclone_mc); } static struct clocksource clocksource_cyclone = { diff --git a/arch/ia64/kernel/fsyscall_gtod_data.h b/arch/ia64/kernel/fsyscall_gtod_data.h index 146b15b5fec3..dcc514917731 100644 --- a/arch/ia64/kernel/fsyscall_gtod_data.h +++ b/arch/ia64/kernel/fsyscall_gtod_data.h @@ -9,15 +9,15 @@ struct fsyscall_gtod_data_t { seqcount_t seq; struct timespec wall_time; struct timespec monotonic_time; - cycle_t clk_mask; + u64 clk_mask; u32 clk_mult; u32 clk_shift; void *clk_fsys_mmio; - cycle_t clk_cycle_last; + u64 clk_cycle_last; } ____cacheline_aligned; struct itc_jitter_data_t { int itc_jitter; - cycle_t itc_lastcycle; + u64 itc_lastcycle; } ____cacheline_aligned; diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 021f44ab4bfb..71775b95d6cc 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -31,7 +31,7 @@ #include "fsyscall_gtod_data.h" -static cycle_t itc_get_cycles(struct clocksource *cs); +static u64 itc_get_cycles(struct clocksource *cs); struct fsyscall_gtod_data_t fsyscall_gtod_data; @@ -323,7 +323,7 @@ void ia64_init_itm(void) } } -static cycle_t itc_get_cycles(struct clocksource *cs) +static u64 itc_get_cycles(struct clocksource *cs) { unsigned long lcycle, now, ret; @@ -397,7 +397,7 @@ void update_vsyscall_tz(void) } void update_vsyscall_old(struct timespec *wall, struct timespec *wtm, - struct clocksource *c, u32 mult, cycle_t cycle_last) + struct clocksource *c, u32 mult, u64 cycle_last) { write_seqcount_begin(&fsyscall_gtod_data.seq); diff --git a/arch/ia64/sn/kernel/sn2/timer.c b/arch/ia64/sn/kernel/sn2/timer.c index abab8f99e913..66edc36426ed 100644 --- a/arch/ia64/sn/kernel/sn2/timer.c +++ b/arch/ia64/sn/kernel/sn2/timer.c @@ -22,9 +22,9 @@ extern unsigned long sn_rtc_cycles_per_second; -static cycle_t read_sn2(struct clocksource *cs) +static u64 read_sn2(struct clocksource *cs) { - return (cycle_t)readq(RTC_COUNTER_ADDR); + return (u64)readq(RTC_COUNTER_ADDR); } static struct clocksource clocksource_sn2 = { diff --git a/arch/m68k/68000/timers.c b/arch/m68k/68000/timers.c index 99a98698bc95..252455bce144 100644 --- a/arch/m68k/68000/timers.c +++ b/arch/m68k/68000/timers.c @@ -76,7 +76,7 @@ static struct irqaction m68328_timer_irq = { /***************************************************************************/ -static cycle_t m68328_read_clk(struct clocksource *cs) +static u64 m68328_read_clk(struct clocksource *cs) { unsigned long flags; u32 cycles; diff --git a/arch/m68k/coldfire/dma_timer.c b/arch/m68k/coldfire/dma_timer.c index 235ad57c4707..8273eea57874 100644 --- a/arch/m68k/coldfire/dma_timer.c +++ b/arch/m68k/coldfire/dma_timer.c @@ -34,7 +34,7 @@ #define DMA_DTMR_CLK_DIV_16 (2 << 1) #define DMA_DTMR_ENABLE (1 << 0) -static cycle_t cf_dt_get_cycles(struct clocksource *cs) +static u64 cf_dt_get_cycles(struct clocksource *cs) { return __raw_readl(DTCN0); } diff --git a/arch/m68k/coldfire/pit.c b/arch/m68k/coldfire/pit.c index d86a9ffb3f13..175553d5b8ed 100644 --- a/arch/m68k/coldfire/pit.c +++ b/arch/m68k/coldfire/pit.c @@ -118,7 +118,7 @@ static struct irqaction pit_irq = { /***************************************************************************/ -static cycle_t pit_read_clk(struct clocksource *cs) +static u64 pit_read_clk(struct clocksource *cs) { unsigned long flags; u32 cycles; diff --git a/arch/m68k/coldfire/sltimers.c b/arch/m68k/coldfire/sltimers.c index 831a08cf6f40..3292c0d68b18 100644 --- a/arch/m68k/coldfire/sltimers.c +++ b/arch/m68k/coldfire/sltimers.c @@ -97,7 +97,7 @@ static struct irqaction mcfslt_timer_irq = { .handler = mcfslt_tick, }; -static cycle_t mcfslt_read_clk(struct clocksource *cs) +static u64 mcfslt_read_clk(struct clocksource *cs) { unsigned long flags; u32 cycles, scnt; diff --git a/arch/m68k/coldfire/timers.c b/arch/m68k/coldfire/timers.c index cd496a20fcc7..2dc7a58204f6 100644 --- a/arch/m68k/coldfire/timers.c +++ b/arch/m68k/coldfire/timers.c @@ -89,7 +89,7 @@ static struct irqaction mcftmr_timer_irq = { /***************************************************************************/ -static cycle_t mcftmr_read_clk(struct clocksource *cs) +static u64 mcftmr_read_clk(struct clocksource *cs) { unsigned long flags; u32 cycles; diff --git a/arch/microblaze/kernel/timer.c b/arch/microblaze/kernel/timer.c index 9e954959f605..1d6fad50fa76 100644 --- a/arch/microblaze/kernel/timer.c +++ b/arch/microblaze/kernel/timer.c @@ -190,17 +190,17 @@ static u64 xilinx_clock_read(void) return read_fn(timer_baseaddr + TCR1); } -static cycle_t xilinx_read(struct clocksource *cs) +static u64 xilinx_read(struct clocksource *cs) { /* reading actual value of timer 1 */ - return (cycle_t)xilinx_clock_read(); + return (u64)xilinx_clock_read(); } static struct timecounter xilinx_tc = { .cc = NULL, }; -static cycle_t xilinx_cc_read(const struct cyclecounter *cc) +static u64 xilinx_cc_read(const struct cyclecounter *cc) { return xilinx_read(NULL); } diff --git a/arch/mips/alchemy/common/time.c b/arch/mips/alchemy/common/time.c index f99d3ec17a45..e1bec5a77c39 100644 --- a/arch/mips/alchemy/common/time.c +++ b/arch/mips/alchemy/common/time.c @@ -44,7 +44,7 @@ /* 32kHz clock enabled and detected */ #define CNTR_OK (SYS_CNTRL_E0 | SYS_CNTRL_32S) -static cycle_t au1x_counter1_read(struct clocksource *cs) +static u64 au1x_counter1_read(struct clocksource *cs) { return alchemy_rdsys(AU1000_SYS_RTCREAD); } diff --git a/arch/mips/cavium-octeon/csrc-octeon.c b/arch/mips/cavium-octeon/csrc-octeon.c index 23c2344a3552..39f153fe0022 100644 --- a/arch/mips/cavium-octeon/csrc-octeon.c +++ b/arch/mips/cavium-octeon/csrc-octeon.c @@ -98,7 +98,7 @@ void octeon_init_cvmcount(void) local_irq_restore(flags); } -static cycle_t octeon_cvmcount_read(struct clocksource *cs) +static u64 octeon_cvmcount_read(struct clocksource *cs) { return read_c0_cvmcount(); } diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c index 1f7ca2c9f262..bcf8f8c62737 100644 --- a/arch/mips/jz4740/time.c +++ b/arch/mips/jz4740/time.c @@ -34,7 +34,7 @@ static uint16_t jz4740_jiffies_per_tick; -static cycle_t jz4740_clocksource_read(struct clocksource *cs) +static u64 jz4740_clocksource_read(struct clocksource *cs) { return jz4740_timer_get_count(TIMER_CLOCKSOURCE); } diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index 537eefdf838f..aaca60d6ffc3 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -27,7 +27,7 @@ struct txx9_clocksource { struct txx9_tmr_reg __iomem *tmrptr; }; -static cycle_t txx9_cs_read(struct clocksource *cs) +static u64 txx9_cs_read(struct clocksource *cs) { struct txx9_clocksource *txx9_cs = container_of(cs, struct txx9_clocksource, cs); diff --git a/arch/mips/kernel/csrc-bcm1480.c b/arch/mips/kernel/csrc-bcm1480.c index 7f65b53d1b24..f011261e9506 100644 --- a/arch/mips/kernel/csrc-bcm1480.c +++ b/arch/mips/kernel/csrc-bcm1480.c @@ -25,9 +25,9 @@ #include -static cycle_t bcm1480_hpt_read(struct clocksource *cs) +static u64 bcm1480_hpt_read(struct clocksource *cs) { - return (cycle_t) __raw_readq(IOADDR(A_SCD_ZBBUS_CYCLE_COUNT)); + return (u64) __raw_readq(IOADDR(A_SCD_ZBBUS_CYCLE_COUNT)); } struct clocksource bcm1480_clocksource = { diff --git a/arch/mips/kernel/csrc-ioasic.c b/arch/mips/kernel/csrc-ioasic.c index 722f5589cd1d..f6acd1e58c26 100644 --- a/arch/mips/kernel/csrc-ioasic.c +++ b/arch/mips/kernel/csrc-ioasic.c @@ -22,7 +22,7 @@ #include #include -static cycle_t dec_ioasic_hpt_read(struct clocksource *cs) +static u64 dec_ioasic_hpt_read(struct clocksource *cs) { return ioasic_read(IO_REG_FCTR); } diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index d76275da54cb..eed099f35bf1 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -11,7 +11,7 @@ #include -static cycle_t c0_hpt_read(struct clocksource *cs) +static u64 c0_hpt_read(struct clocksource *cs) { return read_c0_count(); } diff --git a/arch/mips/kernel/csrc-sb1250.c b/arch/mips/kernel/csrc-sb1250.c index d915652b4d56..b07b7310d3f4 100644 --- a/arch/mips/kernel/csrc-sb1250.c +++ b/arch/mips/kernel/csrc-sb1250.c @@ -30,7 +30,7 @@ * The HPT is free running from SB1250_HPT_VALUE down to 0 then starts over * again. */ -static inline cycle_t sb1250_hpt_get_cycles(void) +static inline u64 sb1250_hpt_get_cycles(void) { unsigned int count; void __iomem *addr; @@ -41,7 +41,7 @@ static inline cycle_t sb1250_hpt_get_cycles(void) return SB1250_HPT_VALUE - count; } -static cycle_t sb1250_hpt_read(struct clocksource *cs) +static u64 sb1250_hpt_read(struct clocksource *cs) { return sb1250_hpt_get_cycles(); } diff --git a/arch/mips/loongson32/common/time.c b/arch/mips/loongson32/common/time.c index ff224f0020e5..e6f972d35252 100644 --- a/arch/mips/loongson32/common/time.c +++ b/arch/mips/loongson32/common/time.c @@ -63,7 +63,7 @@ void __init ls1x_pwmtimer_init(void) ls1x_pwmtimer_restart(); } -static cycle_t ls1x_clocksource_read(struct clocksource *cs) +static u64 ls1x_clocksource_read(struct clocksource *cs) { unsigned long flags; int count; @@ -107,7 +107,7 @@ static cycle_t ls1x_clocksource_read(struct clocksource *cs) raw_spin_unlock_irqrestore(&ls1x_timer_lock, flags); - return (cycle_t) (jifs * ls1x_jiffies_per_tick) + count; + return (u64) (jifs * ls1x_jiffies_per_tick) + count; } static struct clocksource ls1x_clocksource = { diff --git a/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c b/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c index da77d412514c..9edfa55a0e78 100644 --- a/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c +++ b/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c @@ -144,7 +144,7 @@ void __init setup_mfgpt0_timer(void) * to just read by itself. So use jiffies to emulate a free * running counter: */ -static cycle_t mfgpt_read(struct clocksource *cs) +static u64 mfgpt_read(struct clocksource *cs) { unsigned long flags; int count; @@ -188,7 +188,7 @@ static cycle_t mfgpt_read(struct clocksource *cs) raw_spin_unlock_irqrestore(&mfgpt_lock, flags); - return (cycle_t) (jifs * COMPARE) + count; + return (u64) (jifs * COMPARE) + count; } static struct clocksource clocksource_mfgpt = { diff --git a/arch/mips/loongson64/loongson-3/hpet.c b/arch/mips/loongson64/loongson-3/hpet.c index 4788bea62a6a..24afe364637b 100644 --- a/arch/mips/loongson64/loongson-3/hpet.c +++ b/arch/mips/loongson64/loongson-3/hpet.c @@ -248,9 +248,9 @@ void __init setup_hpet_timer(void) pr_info("hpet clock event device register\n"); } -static cycle_t hpet_read_counter(struct clocksource *cs) +static u64 hpet_read_counter(struct clocksource *cs) { - return (cycle_t)hpet_read(HPET_COUNTER); + return (u64)hpet_read(HPET_COUNTER); } static void hpet_suspend(struct clocksource *cs) diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c index 7407da04f8d6..1829a9031eec 100644 --- a/arch/mips/mti-malta/malta-time.c +++ b/arch/mips/mti-malta/malta-time.c @@ -75,7 +75,7 @@ static void __init estimate_frequencies(void) unsigned int count, start; unsigned char secs1, secs2, ctrl; int secs; - cycle_t giccount = 0, gicstart = 0; + u64 giccount = 0, gicstart = 0; #if defined(CONFIG_KVM_GUEST) && CONFIG_KVM_GUEST_TIMER_FREQ mips_hpt_frequency = CONFIG_KVM_GUEST_TIMER_FREQ * 1000000; diff --git a/arch/mips/netlogic/common/time.c b/arch/mips/netlogic/common/time.c index 5873c83e65be..cbbf0d48216b 100644 --- a/arch/mips/netlogic/common/time.c +++ b/arch/mips/netlogic/common/time.c @@ -59,14 +59,14 @@ unsigned int get_c0_compare_int(void) return IRQ_TIMER; } -static cycle_t nlm_get_pic_timer(struct clocksource *cs) +static u64 nlm_get_pic_timer(struct clocksource *cs) { uint64_t picbase = nlm_get_node(0)->picbase; return ~nlm_pic_read_timer(picbase, PIC_CLOCK_TIMER); } -static cycle_t nlm_get_pic_timer32(struct clocksource *cs) +static u64 nlm_get_pic_timer32(struct clocksource *cs) { uint64_t picbase = nlm_get_node(0)->picbase; diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 42d6cb9f956e..695c51bdd7dc 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -140,7 +140,7 @@ static void __init hub_rt_clock_event_global_init(void) setup_irq(irq, &hub_rt_irqaction); } -static cycle_t hub_rt_read(struct clocksource *cs) +static u64 hub_rt_read(struct clocksource *cs) { return REMOTE_HUB_L(cputonasid(0), PI_RT_COUNT); } diff --git a/arch/mn10300/kernel/csrc-mn10300.c b/arch/mn10300/kernel/csrc-mn10300.c index 45644cf18c41..6b74df3661f2 100644 --- a/arch/mn10300/kernel/csrc-mn10300.c +++ b/arch/mn10300/kernel/csrc-mn10300.c @@ -13,7 +13,7 @@ #include #include "internal.h" -static cycle_t mn10300_read(struct clocksource *cs) +static u64 mn10300_read(struct clocksource *cs) { return read_timestamp_counter(); } diff --git a/arch/nios2/kernel/time.c b/arch/nios2/kernel/time.c index 746bf5caaffc..6e2bdc9b8530 100644 --- a/arch/nios2/kernel/time.c +++ b/arch/nios2/kernel/time.c @@ -81,7 +81,7 @@ static inline unsigned long read_timersnapshot(struct nios2_timer *timer) return count; } -static cycle_t nios2_timer_read(struct clocksource *cs) +static u64 nios2_timer_read(struct clocksource *cs) { struct nios2_clocksource *nios2_cs = to_nios2_clksource(cs); unsigned long flags; diff --git a/arch/openrisc/kernel/time.c b/arch/openrisc/kernel/time.c index 50e970183dcd..687c11d048d7 100644 --- a/arch/openrisc/kernel/time.c +++ b/arch/openrisc/kernel/time.c @@ -117,9 +117,9 @@ static __init void openrisc_clockevent_init(void) * is 32 bits wide and runs at the CPU clock frequency. */ -static cycle_t openrisc_timer_read(struct clocksource *cs) +static u64 openrisc_timer_read(struct clocksource *cs) { - return (cycle_t) mfspr(SPR_TTCR); + return (u64) mfspr(SPR_TTCR); } static struct clocksource openrisc_timer = { diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c index 037d81f00520..da0d9cb63403 100644 --- a/arch/parisc/kernel/time.c +++ b/arch/parisc/kernel/time.c @@ -137,7 +137,7 @@ EXPORT_SYMBOL(profile_pc); /* clock source code */ -static cycle_t notrace read_cr16(struct clocksource *cs) +static u64 notrace read_cr16(struct clocksource *cs) { return get_cycles(); } diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 19397e2a8bf5..bc2e08d415fa 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -80,7 +80,7 @@ #include #include -static cycle_t rtc_read(struct clocksource *); +static u64 rtc_read(struct clocksource *); static struct clocksource clocksource_rtc = { .name = "rtc", .rating = 400, @@ -89,7 +89,7 @@ static struct clocksource clocksource_rtc = { .read = rtc_read, }; -static cycle_t timebase_read(struct clocksource *); +static u64 timebase_read(struct clocksource *); static struct clocksource clocksource_timebase = { .name = "timebase", .rating = 400, @@ -802,18 +802,18 @@ void read_persistent_clock(struct timespec *ts) } /* clocksource code */ -static cycle_t rtc_read(struct clocksource *cs) +static u64 rtc_read(struct clocksource *cs) { - return (cycle_t)get_rtc(); + return (u64)get_rtc(); } -static cycle_t timebase_read(struct clocksource *cs) +static u64 timebase_read(struct clocksource *cs) { - return (cycle_t)get_tb(); + return (u64)get_tb(); } void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm, - struct clocksource *clock, u32 mult, cycle_t cycle_last) + struct clocksource *clock, u32 mult, u64 cycle_last) { u64 new_tb_to_xs, new_stamp_xsec; u32 frac_sec; diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index ec76315c9ee5..52949df88529 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -209,7 +209,7 @@ void read_boot_clock64(struct timespec64 *ts) tod_to_timeval(clock - TOD_UNIX_EPOCH, ts); } -static cycle_t read_tod_clock(struct clocksource *cs) +static u64 read_tod_clock(struct clocksource *cs) { unsigned long long now, adj; diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index 1affabc96b08..244062bdaa56 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -148,7 +148,7 @@ static unsigned int sbus_cycles_offset(void) return offset; } -static cycle_t timer_cs_read(struct clocksource *cs) +static u64 timer_cs_read(struct clocksource *cs) { unsigned int seq, offset; u64 cycles; diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 807f7e2ce014..12a6d3555cb8 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -770,7 +770,7 @@ void udelay(unsigned long usecs) } EXPORT_SYMBOL(udelay); -static cycle_t clocksource_tick_read(struct clocksource *cs) +static u64 clocksource_tick_read(struct clocksource *cs) { return tick_ops->get_tick(); } diff --git a/arch/um/kernel/time.c b/arch/um/kernel/time.c index 25c23666d592..ba87a27d6715 100644 --- a/arch/um/kernel/time.c +++ b/arch/um/kernel/time.c @@ -83,7 +83,7 @@ static irqreturn_t um_timer(int irq, void *dev) return IRQ_HANDLED; } -static cycle_t timer_read(struct clocksource *cs) +static u64 timer_read(struct clocksource *cs) { return os_nsecs() / TIMER_MULTIPLIER; } diff --git a/arch/unicore32/kernel/time.c b/arch/unicore32/kernel/time.c index ac4c5449bb88..fceaa673f861 100644 --- a/arch/unicore32/kernel/time.c +++ b/arch/unicore32/kernel/time.c @@ -62,7 +62,7 @@ static struct clock_event_device ckevt_puv3_osmr0 = { .set_state_oneshot = puv3_osmr0_shutdown, }; -static cycle_t puv3_read_oscr(struct clocksource *cs) +static u64 puv3_read_oscr(struct clocksource *cs) { return readl(OST_OSCR); } diff --git a/arch/x86/entry/vdso/vclock_gettime.c b/arch/x86/entry/vdso/vclock_gettime.c index 02223cb4bcfd..9d4d6e138311 100644 --- a/arch/x86/entry/vdso/vclock_gettime.c +++ b/arch/x86/entry/vdso/vclock_gettime.c @@ -92,10 +92,10 @@ static notrace const struct pvclock_vsyscall_time_info *get_pvti0(void) return (const struct pvclock_vsyscall_time_info *)&pvclock_page; } -static notrace cycle_t vread_pvclock(int *mode) +static notrace u64 vread_pvclock(int *mode) { const struct pvclock_vcpu_time_info *pvti = &get_pvti0()->pvti; - cycle_t ret; + u64 ret; u64 last; u32 version; @@ -142,9 +142,9 @@ static notrace cycle_t vread_pvclock(int *mode) } #endif -notrace static cycle_t vread_tsc(void) +notrace static u64 vread_tsc(void) { - cycle_t ret = (cycle_t)rdtsc_ordered(); + u64 ret = (u64)rdtsc_ordered(); u64 last = gtod->cycle_last; if (likely(ret >= last)) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 2e25038dbd93..a7066dc1a7e9 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -768,7 +768,7 @@ struct kvm_arch { spinlock_t pvclock_gtod_sync_lock; bool use_master_clock; u64 master_kernel_ns; - cycle_t master_cycle_now; + u64 master_cycle_now; struct delayed_work kvmclock_update_work; struct delayed_work kvmclock_sync_work; diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h index 3ad741b84072..448cfe1b48cf 100644 --- a/arch/x86/include/asm/pvclock.h +++ b/arch/x86/include/asm/pvclock.h @@ -14,7 +14,7 @@ static inline struct pvclock_vsyscall_time_info *pvclock_pvti_cpu0_va(void) #endif /* some helper functions for xen and kvm pv clock sources */ -cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src); +u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src); u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src); void pvclock_set_flags(u8 flags); unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src); @@ -87,11 +87,10 @@ static inline u64 pvclock_scale_delta(u64 delta, u32 mul_frac, int shift) } static __always_inline -cycle_t __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src, - u64 tsc) +u64 __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src, u64 tsc) { u64 delta = tsc - src->tsc_timestamp; - cycle_t offset = pvclock_scale_delta(delta, src->tsc_to_system_mul, + u64 offset = pvclock_scale_delta(delta, src->tsc_to_system_mul, src->tsc_shift); return src->system_time + offset; } diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index abb1fdcc545a..f5e6f1c417df 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -29,7 +29,7 @@ static inline cycles_t get_cycles(void) return rdtsc(); } -extern struct system_counterval_t convert_art_to_tsc(cycle_t art); +extern struct system_counterval_t convert_art_to_tsc(u64 art); extern void tsc_init(void); extern void mark_tsc_unstable(char *reason); diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h index 3a01996db58f..022e59714562 100644 --- a/arch/x86/include/asm/vgtod.h +++ b/arch/x86/include/asm/vgtod.h @@ -17,8 +17,8 @@ struct vsyscall_gtod_data { unsigned seq; int vclock_mode; - cycle_t cycle_last; - cycle_t mask; + u64 cycle_last; + u64 mask; u32 mult; u32 shift; diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c index 456316f6c868..092ea664d2c6 100644 --- a/arch/x86/kernel/apb_timer.c +++ b/arch/x86/kernel/apb_timer.c @@ -247,7 +247,7 @@ void apbt_setup_secondary_clock(void) {} static int apbt_clocksource_register(void) { u64 start, now; - cycle_t t1; + u64 t1; /* Start the counter, use timer 2 as source, timer 0/1 for event */ dw_apb_clocksource_start(clocksource_apbt); @@ -355,7 +355,7 @@ unsigned long apbt_quick_calibrate(void) { int i, scale; u64 old, new; - cycle_t t1, t2; + u64 t1, t2; unsigned long khz = 0; u32 loop, shift; diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index f37e02e41a77..65e20c97e04b 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -133,9 +133,9 @@ static uint32_t __init ms_hyperv_platform(void) return 0; } -static cycle_t read_hv_clock(struct clocksource *arg) +static u64 read_hv_clock(struct clocksource *arg) { - cycle_t current_tick; + u64 current_tick; /* * Read the partition counter to get the current tick count. This count * is set to 0 when the partition is created and is incremented in diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 274fab99169d..367756d55980 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -791,7 +791,7 @@ static union hpet_lock hpet __cacheline_aligned = { { .lock = __ARCH_SPIN_LOCK_UNLOCKED, }, }; -static cycle_t read_hpet(struct clocksource *cs) +static u64 read_hpet(struct clocksource *cs) { unsigned long flags; union hpet_lock old, new; @@ -802,7 +802,7 @@ static cycle_t read_hpet(struct clocksource *cs) * Read HPET directly if in NMI. */ if (in_nmi()) - return (cycle_t)hpet_readl(HPET_COUNTER); + return (u64)hpet_readl(HPET_COUNTER); /* * Read the current state of the lock and HPET value atomically. @@ -821,7 +821,7 @@ static cycle_t read_hpet(struct clocksource *cs) WRITE_ONCE(hpet.value, new.value); arch_spin_unlock(&hpet.lock); local_irq_restore(flags); - return (cycle_t)new.value; + return (u64)new.value; } local_irq_restore(flags); @@ -843,15 +843,15 @@ contended: new.lockval = READ_ONCE(hpet.lockval); } while ((new.value == old.value) && arch_spin_is_locked(&new.lock)); - return (cycle_t)new.value; + return (u64)new.value; } #else /* * For UP or 32-bit. */ -static cycle_t read_hpet(struct clocksource *cs) +static u64 read_hpet(struct clocksource *cs) { - return (cycle_t)hpet_readl(HPET_COUNTER); + return (u64)hpet_readl(HPET_COUNTER); } #endif @@ -867,7 +867,7 @@ static struct clocksource clocksource_hpet = { static int hpet_clocksource_register(void) { u64 start, now; - cycle_t t1; + u64 t1; /* Start the counter */ hpet_restart_counter(); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 60b9949f1e65..2a5cafdf8808 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -32,7 +32,7 @@ static int kvmclock __ro_after_init = 1; static int msr_kvm_system_time = MSR_KVM_SYSTEM_TIME; static int msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK; -static cycle_t kvm_sched_clock_offset; +static u64 kvm_sched_clock_offset; static int parse_no_kvmclock(char *arg) { @@ -79,10 +79,10 @@ static int kvm_set_wallclock(const struct timespec *now) return -1; } -static cycle_t kvm_clock_read(void) +static u64 kvm_clock_read(void) { struct pvclock_vcpu_time_info *src; - cycle_t ret; + u64 ret; int cpu; preempt_disable_notrace(); @@ -93,12 +93,12 @@ static cycle_t kvm_clock_read(void) return ret; } -static cycle_t kvm_clock_get_cycles(struct clocksource *cs) +static u64 kvm_clock_get_cycles(struct clocksource *cs) { return kvm_clock_read(); } -static cycle_t kvm_sched_clock_read(void) +static u64 kvm_sched_clock_read(void) { return kvm_clock_read() - kvm_sched_clock_offset; } diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c index 5b2cc889ce34..9e93fe5803b4 100644 --- a/arch/x86/kernel/pvclock.c +++ b/arch/x86/kernel/pvclock.c @@ -71,10 +71,10 @@ u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src) return flags & valid_flags; } -cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) +u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) { unsigned version; - cycle_t ret; + u64 ret; u64 last; u8 flags; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 0aed75a1e31b..be3a49ee0356 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1101,9 +1101,9 @@ static void tsc_resume(struct clocksource *cs) * checking the result of read_tsc() - cycle_last for being negative. * That works because CLOCKSOURCE_MASK(64) does not mask out any bit. */ -static cycle_t read_tsc(struct clocksource *cs) +static u64 read_tsc(struct clocksource *cs) { - return (cycle_t)rdtsc_ordered(); + return (u64)rdtsc_ordered(); } /* @@ -1192,7 +1192,7 @@ int unsynchronized_tsc(void) /* * Convert ART to TSC given numerator/denominator found in detect_art() */ -struct system_counterval_t convert_art_to_tsc(cycle_t art) +struct system_counterval_t convert_art_to_tsc(u64 art) { u64 tmp, res, rem; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 445c51b6cf6d..ed04398f52c1 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1131,8 +1131,8 @@ struct pvclock_gtod_data { struct { /* extract of a clocksource struct */ int vclock_mode; - cycle_t cycle_last; - cycle_t mask; + u64 cycle_last; + u64 mask; u32 mult; u32 shift; } clock; @@ -1572,9 +1572,9 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) #ifdef CONFIG_X86_64 -static cycle_t read_tsc(void) +static u64 read_tsc(void) { - cycle_t ret = (cycle_t)rdtsc_ordered(); + u64 ret = (u64)rdtsc_ordered(); u64 last = pvclock_gtod_data.clock.cycle_last; if (likely(ret >= last)) @@ -1592,7 +1592,7 @@ static cycle_t read_tsc(void) return last; } -static inline u64 vgettsc(cycle_t *cycle_now) +static inline u64 vgettsc(u64 *cycle_now) { long v; struct pvclock_gtod_data *gtod = &pvclock_gtod_data; @@ -1603,7 +1603,7 @@ static inline u64 vgettsc(cycle_t *cycle_now) return v * gtod->clock.mult; } -static int do_monotonic_boot(s64 *t, cycle_t *cycle_now) +static int do_monotonic_boot(s64 *t, u64 *cycle_now) { struct pvclock_gtod_data *gtod = &pvclock_gtod_data; unsigned long seq; @@ -1624,7 +1624,7 @@ static int do_monotonic_boot(s64 *t, cycle_t *cycle_now) } /* returns true if host is using tsc clocksource */ -static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now) +static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *cycle_now) { /* checked again under seqlock below */ if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC) diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index 4ca0d78adcf0..d3289d7e78fa 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c @@ -916,7 +916,7 @@ static unsigned long lguest_tsc_khz(void) * If we can't use the TSC, the kernel falls back to our lower-priority * "lguest_clock", where we read the time value given to us by the Host. */ -static cycle_t lguest_clock_read(struct clocksource *cs) +static u64 lguest_clock_read(struct clocksource *cs) { unsigned long sec, nsec; diff --git a/arch/x86/platform/uv/uv_time.c b/arch/x86/platform/uv/uv_time.c index b333fc45f9ec..2ee7632d4916 100644 --- a/arch/x86/platform/uv/uv_time.c +++ b/arch/x86/platform/uv/uv_time.c @@ -30,7 +30,7 @@ #define RTC_NAME "sgi_rtc" -static cycle_t uv_read_rtc(struct clocksource *cs); +static u64 uv_read_rtc(struct clocksource *cs); static int uv_rtc_next_event(unsigned long, struct clock_event_device *); static int uv_rtc_shutdown(struct clock_event_device *evt); @@ -38,7 +38,7 @@ static struct clocksource clocksource_uv = { .name = RTC_NAME, .rating = 299, .read = uv_read_rtc, - .mask = (cycle_t)UVH_RTC_REAL_TIME_CLOCK_MASK, + .mask = (u64)UVH_RTC_REAL_TIME_CLOCK_MASK, .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; @@ -296,7 +296,7 @@ static int uv_rtc_unset_timer(int cpu, int force) * cachelines of it's own page. This allows faster simultaneous reads * from a given socket. */ -static cycle_t uv_read_rtc(struct clocksource *cs) +static u64 uv_read_rtc(struct clocksource *cs) { unsigned long offset; @@ -305,7 +305,7 @@ static cycle_t uv_read_rtc(struct clocksource *cs) else offset = (uv_blade_processor_id() * L1_CACHE_BYTES) % PAGE_SIZE; - return (cycle_t)uv_read_local_mmr(UVH_RTC | offset); + return (u64)uv_read_local_mmr(UVH_RTC | offset); } /* diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 33d8f6a7829d..1e69956d7852 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -39,10 +39,10 @@ static unsigned long xen_tsc_khz(void) return pvclock_tsc_khz(info); } -cycle_t xen_clocksource_read(void) +u64 xen_clocksource_read(void) { struct pvclock_vcpu_time_info *src; - cycle_t ret; + u64 ret; preempt_disable_notrace(); src = &__this_cpu_read(xen_vcpu)->time; @@ -51,7 +51,7 @@ cycle_t xen_clocksource_read(void) return ret; } -static cycle_t xen_clocksource_get_cycles(struct clocksource *cs) +static u64 xen_clocksource_get_cycles(struct clocksource *cs) { return xen_clocksource_read(); } diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 3cbce3b085e7..ac0a2b0f9e62 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -67,7 +67,7 @@ void xen_init_irq_ops(void); void xen_setup_timer(int cpu); void xen_setup_runstate_info(int cpu); void xen_teardown_timer(int cpu); -cycle_t xen_clocksource_read(void); +u64 xen_clocksource_read(void); void xen_setup_cpu_clockevents(void); void __init xen_init_time_ops(void); void __init xen_hvm_init_time_ops(void); diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c index be81e69b25bc..668c1056f9e4 100644 --- a/arch/xtensa/kernel/time.c +++ b/arch/xtensa/kernel/time.c @@ -34,9 +34,9 @@ unsigned long ccount_freq; /* ccount Hz */ EXPORT_SYMBOL(ccount_freq); -static cycle_t ccount_read(struct clocksource *cs) +static u64 ccount_read(struct clocksource *cs) { - return (cycle_t)get_ccount(); + return (u64)get_ccount(); } static u64 notrace ccount_sched_clock_read(void) diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c index be54e5331a45..20b32bb8c2af 100644 --- a/drivers/char/hpet.c +++ b/drivers/char/hpet.c @@ -69,9 +69,9 @@ static u32 hpet_nhpet, hpet_max_freq = HPET_USER_FREQ; #ifdef CONFIG_IA64 static void __iomem *hpet_mctr; -static cycle_t read_hpet(struct clocksource *cs) +static u64 read_hpet(struct clocksource *cs) { - return (cycle_t)read_counter((void __iomem *)hpet_mctr); + return (u64)read_counter((void __iomem *)hpet_mctr); } static struct clocksource clocksource_hpet = { diff --git a/drivers/clocksource/acpi_pm.c b/drivers/clocksource/acpi_pm.c index 28037d0b8dcd..1961e3539b57 100644 --- a/drivers/clocksource/acpi_pm.c +++ b/drivers/clocksource/acpi_pm.c @@ -58,16 +58,16 @@ u32 acpi_pm_read_verified(void) return v2; } -static cycle_t acpi_pm_read(struct clocksource *cs) +static u64 acpi_pm_read(struct clocksource *cs) { - return (cycle_t)read_pmtmr(); + return (u64)read_pmtmr(); } static struct clocksource clocksource_acpi_pm = { .name = "acpi_pm", .rating = 200, .read = acpi_pm_read, - .mask = (cycle_t)ACPI_PM_MASK, + .mask = (u64)ACPI_PM_MASK, .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; @@ -81,9 +81,9 @@ static int __init acpi_pm_good_setup(char *__str) } __setup("acpi_pm_good", acpi_pm_good_setup); -static cycle_t acpi_pm_read_slow(struct clocksource *cs) +static u64 acpi_pm_read_slow(struct clocksource *cs) { - return (cycle_t)acpi_pm_read_verified(); + return (u64)acpi_pm_read_verified(); } static inline void acpi_pm_need_workaround(void) @@ -145,7 +145,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, PCI_DEVICE_ID_SERVERWORKS_LE, */ static int verify_pmtmr_rate(void) { - cycle_t value1, value2; + u64 value1, value2; unsigned long count, delta; mach_prepare_counter(); @@ -175,7 +175,7 @@ static int verify_pmtmr_rate(void) static int __init init_acpi_pm_clocksource(void) { - cycle_t value1, value2; + u64 value1, value2; unsigned int i, j = 0; if (!pmtmr_ioport) diff --git a/drivers/clocksource/arc_timer.c b/drivers/clocksource/arc_timer.c index a49748d826c0..3ea46343024f 100644 --- a/drivers/clocksource/arc_timer.c +++ b/drivers/clocksource/arc_timer.c @@ -56,7 +56,7 @@ static int noinline arc_get_timer_clk(struct device_node *node) #ifdef CONFIG_ARC_TIMERS_64BIT -static cycle_t arc_read_gfrc(struct clocksource *cs) +static u64 arc_read_gfrc(struct clocksource *cs) { unsigned long flags; u32 l, h; @@ -71,7 +71,7 @@ static cycle_t arc_read_gfrc(struct clocksource *cs) local_irq_restore(flags); - return (((cycle_t)h) << 32) | l; + return (((u64)h) << 32) | l; } static struct clocksource arc_counter_gfrc = { @@ -105,7 +105,7 @@ CLOCKSOURCE_OF_DECLARE(arc_gfrc, "snps,archs-timer-gfrc", arc_cs_setup_gfrc); #define AUX_RTC_LOW 0x104 #define AUX_RTC_HIGH 0x105 -static cycle_t arc_read_rtc(struct clocksource *cs) +static u64 arc_read_rtc(struct clocksource *cs) { unsigned long status; u32 l, h; @@ -122,7 +122,7 @@ static cycle_t arc_read_rtc(struct clocksource *cs) status = read_aux_reg(AUX_RTC_CTRL); } while (!(status & _BITUL(31))); - return (((cycle_t)h) << 32) | l; + return (((u64)h) << 32) | l; } static struct clocksource arc_counter_rtc = { @@ -166,9 +166,9 @@ CLOCKSOURCE_OF_DECLARE(arc_rtc, "snps,archs-timer-rtc", arc_cs_setup_rtc); * 32bit TIMER1 to keep counting monotonically and wraparound */ -static cycle_t arc_read_timer1(struct clocksource *cs) +static u64 arc_read_timer1(struct clocksource *cs) { - return (cycle_t) read_aux_reg(ARC_REG_TIMER1_CNT); + return (u64) read_aux_reg(ARC_REG_TIMER1_CNT); } static struct clocksource arc_counter_timer1 = { diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 02fef6830e72..394e417414d3 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -562,12 +562,12 @@ static u64 arch_counter_get_cntvct_mem(void) */ u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; -static cycle_t arch_counter_read(struct clocksource *cs) +static u64 arch_counter_read(struct clocksource *cs) { return arch_timer_read_counter(); } -static cycle_t arch_counter_read_cc(const struct cyclecounter *cc) +static u64 arch_counter_read_cc(const struct cyclecounter *cc) { return arch_timer_read_counter(); } diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c index 8da03298f844..570cc58baec4 100644 --- a/drivers/clocksource/arm_global_timer.c +++ b/drivers/clocksource/arm_global_timer.c @@ -195,7 +195,7 @@ static int gt_dying_cpu(unsigned int cpu) return 0; } -static cycle_t gt_clocksource_read(struct clocksource *cs) +static u64 gt_clocksource_read(struct clocksource *cs) { return gt_counter_read(); } diff --git a/drivers/clocksource/cadence_ttc_timer.c b/drivers/clocksource/cadence_ttc_timer.c index fbfbdec13b08..44e5e951583b 100644 --- a/drivers/clocksource/cadence_ttc_timer.c +++ b/drivers/clocksource/cadence_ttc_timer.c @@ -158,11 +158,11 @@ static irqreturn_t ttc_clock_event_interrupt(int irq, void *dev_id) * * returns: Current timer counter register value **/ -static cycle_t __ttc_clocksource_read(struct clocksource *cs) +static u64 __ttc_clocksource_read(struct clocksource *cs) { struct ttc_timer *timer = &to_ttc_timer_clksrc(cs)->ttc; - return (cycle_t)readl_relaxed(timer->base_addr + + return (u64)readl_relaxed(timer->base_addr + TTC_COUNT_VAL_OFFSET); } diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c index 77a365f573d7..c69e2772658d 100644 --- a/drivers/clocksource/clksrc-dbx500-prcmu.c +++ b/drivers/clocksource/clksrc-dbx500-prcmu.c @@ -30,7 +30,7 @@ static void __iomem *clksrc_dbx500_timer_base; -static cycle_t notrace clksrc_dbx500_prcmu_read(struct clocksource *cs) +static u64 notrace clksrc_dbx500_prcmu_read(struct clocksource *cs) { void __iomem *base = clksrc_dbx500_timer_base; u32 count, count2; diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c index 797505aa2ba4..63e4f5519577 100644 --- a/drivers/clocksource/dw_apb_timer.c +++ b/drivers/clocksource/dw_apb_timer.c @@ -348,7 +348,7 @@ void dw_apb_clocksource_start(struct dw_apb_clocksource *dw_cs) dw_apb_clocksource_read(dw_cs); } -static cycle_t __apbt_read_clocksource(struct clocksource *cs) +static u64 __apbt_read_clocksource(struct clocksource *cs) { u32 current_count; struct dw_apb_clocksource *dw_cs = @@ -357,7 +357,7 @@ static cycle_t __apbt_read_clocksource(struct clocksource *cs) current_count = apbt_readl_relaxed(&dw_cs->timer, APBTMR_N_CURRENT_VALUE); - return (cycle_t)~current_count; + return (u64)~current_count; } static void apbt_restart_clocksource(struct clocksource *cs) @@ -416,7 +416,7 @@ void dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs) * * @dw_cs: The clocksource to read. */ -cycle_t dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs) +u64 dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs) { - return (cycle_t)~apbt_readl(&dw_cs->timer, APBTMR_N_CURRENT_VALUE); + return (u64)~apbt_readl(&dw_cs->timer, APBTMR_N_CURRENT_VALUE); } diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c index 19bb1792d647..aff87df07449 100644 --- a/drivers/clocksource/em_sti.c +++ b/drivers/clocksource/em_sti.c @@ -110,9 +110,9 @@ static void em_sti_disable(struct em_sti_priv *p) clk_disable_unprepare(p->clk); } -static cycle_t em_sti_count(struct em_sti_priv *p) +static u64 em_sti_count(struct em_sti_priv *p) { - cycle_t ticks; + u64 ticks; unsigned long flags; /* the STI hardware buffers the 48-bit count, but to @@ -121,14 +121,14 @@ static cycle_t em_sti_count(struct em_sti_priv *p) * Always read STI_COUNT_H before STI_COUNT_L. */ raw_spin_lock_irqsave(&p->lock, flags); - ticks = (cycle_t)(em_sti_read(p, STI_COUNT_H) & 0xffff) << 32; + ticks = (u64)(em_sti_read(p, STI_COUNT_H) & 0xffff) << 32; ticks |= em_sti_read(p, STI_COUNT_L); raw_spin_unlock_irqrestore(&p->lock, flags); return ticks; } -static cycle_t em_sti_set_next(struct em_sti_priv *p, cycle_t next) +static u64 em_sti_set_next(struct em_sti_priv *p, u64 next) { unsigned long flags; @@ -198,7 +198,7 @@ static struct em_sti_priv *cs_to_em_sti(struct clocksource *cs) return container_of(cs, struct em_sti_priv, cs); } -static cycle_t em_sti_clocksource_read(struct clocksource *cs) +static u64 em_sti_clocksource_read(struct clocksource *cs) { return em_sti_count(cs_to_em_sti(cs)); } @@ -271,7 +271,7 @@ static int em_sti_clock_event_next(unsigned long delta, struct clock_event_device *ced) { struct em_sti_priv *p = ced_to_em_sti(ced); - cycle_t next; + u64 next; int safe; next = em_sti_set_next(p, em_sti_count(p) + delta); diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 8f3488b80896..c8b9f834f4de 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -183,7 +183,7 @@ static u64 exynos4_read_count_64(void) hi2 = readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_U); } while (hi != hi2); - return ((cycle_t)hi << 32) | lo; + return ((u64)hi << 32) | lo; } /** @@ -199,7 +199,7 @@ static u32 notrace exynos4_read_count_32(void) return readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_L); } -static cycle_t exynos4_frc_read(struct clocksource *cs) +static u64 exynos4_frc_read(struct clocksource *cs) { return exynos4_read_count_32(); } @@ -266,7 +266,7 @@ static void exynos4_mct_comp0_stop(void) static void exynos4_mct_comp0_start(bool periodic, unsigned long cycles) { unsigned int tcon; - cycle_t comp_cycle; + u64 comp_cycle; tcon = readl_relaxed(reg_base + EXYNOS4_MCT_G_TCON); diff --git a/drivers/clocksource/h8300_timer16.c b/drivers/clocksource/h8300_timer16.c index 07d9d5be9054..5b27fb9997c2 100644 --- a/drivers/clocksource/h8300_timer16.c +++ b/drivers/clocksource/h8300_timer16.c @@ -72,7 +72,7 @@ static inline struct timer16_priv *cs_to_priv(struct clocksource *cs) return container_of(cs, struct timer16_priv, cs); } -static cycle_t timer16_clocksource_read(struct clocksource *cs) +static u64 timer16_clocksource_read(struct clocksource *cs) { struct timer16_priv *p = cs_to_priv(cs); unsigned long raw, value; diff --git a/drivers/clocksource/h8300_tpu.c b/drivers/clocksource/h8300_tpu.c index 7bdf1991c847..72e1cf2b3096 100644 --- a/drivers/clocksource/h8300_tpu.c +++ b/drivers/clocksource/h8300_tpu.c @@ -64,7 +64,7 @@ static inline struct tpu_priv *cs_to_priv(struct clocksource *cs) return container_of(cs, struct tpu_priv, cs); } -static cycle_t tpu_clocksource_read(struct clocksource *cs) +static u64 tpu_clocksource_read(struct clocksource *cs) { struct tpu_priv *p = cs_to_priv(cs); unsigned long flags; diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index 0efd36e483ab..64f6490740d7 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c @@ -25,7 +25,7 @@ EXPORT_SYMBOL(i8253_lock); * to just read by itself. So use jiffies to emulate a free * running counter: */ -static cycle_t i8253_read(struct clocksource *cs) +static u64 i8253_read(struct clocksource *cs) { static int old_count; static u32 old_jifs; @@ -83,7 +83,7 @@ static cycle_t i8253_read(struct clocksource *cs) count = (PIT_LATCH - 1) - count; - return (cycle_t)(jifs * PIT_LATCH) + count; + return (u64)(jifs * PIT_LATCH) + count; } static struct clocksource i8253_cs = { diff --git a/drivers/clocksource/jcore-pit.c b/drivers/clocksource/jcore-pit.c index 54e1665aa03c..e90a6cfcb061 100644 --- a/drivers/clocksource/jcore-pit.c +++ b/drivers/clocksource/jcore-pit.c @@ -57,7 +57,7 @@ static notrace u64 jcore_sched_clock_read(void) return seclo * NSEC_PER_SEC + nsec; } -static cycle_t jcore_clocksource_read(struct clocksource *cs) +static u64 jcore_clocksource_read(struct clocksource *cs) { return jcore_sched_clock_read(); } diff --git a/drivers/clocksource/metag_generic.c b/drivers/clocksource/metag_generic.c index a80ab3e446b7..8d06a0f7ff26 100644 --- a/drivers/clocksource/metag_generic.c +++ b/drivers/clocksource/metag_generic.c @@ -56,7 +56,7 @@ static int metag_timer_set_next_event(unsigned long delta, return 0; } -static cycle_t metag_clocksource_read(struct clocksource *cs) +static u64 metag_clocksource_read(struct clocksource *cs) { return __core_reg_get(TXTIMER); } diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index 7a960cd01104..7b86d07c99b4 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -125,7 +125,7 @@ static int gic_clockevent_init(void) return 0; } -static cycle_t gic_hpt_read(struct clocksource *cs) +static u64 gic_hpt_read(struct clocksource *cs) { return gic_read_count(); } diff --git a/drivers/clocksource/mmio.c b/drivers/clocksource/mmio.c index c4f7d7a9b689..4c4df981d8cc 100644 --- a/drivers/clocksource/mmio.c +++ b/drivers/clocksource/mmio.c @@ -20,24 +20,24 @@ static inline struct clocksource_mmio *to_mmio_clksrc(struct clocksource *c) return container_of(c, struct clocksource_mmio, clksrc); } -cycle_t clocksource_mmio_readl_up(struct clocksource *c) +u64 clocksource_mmio_readl_up(struct clocksource *c) { - return (cycle_t)readl_relaxed(to_mmio_clksrc(c)->reg); + return (u64)readl_relaxed(to_mmio_clksrc(c)->reg); } -cycle_t clocksource_mmio_readl_down(struct clocksource *c) +u64 clocksource_mmio_readl_down(struct clocksource *c) { - return ~(cycle_t)readl_relaxed(to_mmio_clksrc(c)->reg) & c->mask; + return ~(u64)readl_relaxed(to_mmio_clksrc(c)->reg) & c->mask; } -cycle_t clocksource_mmio_readw_up(struct clocksource *c) +u64 clocksource_mmio_readw_up(struct clocksource *c) { - return (cycle_t)readw_relaxed(to_mmio_clksrc(c)->reg); + return (u64)readw_relaxed(to_mmio_clksrc(c)->reg); } -cycle_t clocksource_mmio_readw_down(struct clocksource *c) +u64 clocksource_mmio_readw_down(struct clocksource *c) { - return ~(cycle_t)readw_relaxed(to_mmio_clksrc(c)->reg) & c->mask; + return ~(u64)readw_relaxed(to_mmio_clksrc(c)->reg) & c->mask; } /** @@ -51,7 +51,7 @@ cycle_t clocksource_mmio_readw_down(struct clocksource *c) */ int __init clocksource_mmio_init(void __iomem *base, const char *name, unsigned long hz, int rating, unsigned bits, - cycle_t (*read)(struct clocksource *)) + u64 (*read)(struct clocksource *)) { struct clocksource_mmio *cs; diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index 0ba0a913b41d..99b77aff0839 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -97,7 +97,7 @@ static void timrot_irq_acknowledge(void) HW_TIMROT_TIMCTRLn(0) + STMP_OFFSET_REG_CLR); } -static cycle_t timrotv1_get_cycles(struct clocksource *cs) +static u64 timrotv1_get_cycles(struct clocksource *cs) { return ~((__raw_readl(mxs_timrot_base + HW_TIMROT_TIMCOUNTn(1)) & 0xffff0000) >> 16); diff --git a/drivers/clocksource/qcom-timer.c b/drivers/clocksource/qcom-timer.c index 3283cfa2aa52..d5d048d890d4 100644 --- a/drivers/clocksource/qcom-timer.c +++ b/drivers/clocksource/qcom-timer.c @@ -89,7 +89,7 @@ static struct clock_event_device __percpu *msm_evt; static void __iomem *source_base; -static notrace cycle_t msm_read_timer_count(struct clocksource *cs) +static notrace u64 msm_read_timer_count(struct clocksource *cs) { return readl_relaxed(source_base + TIMER_COUNT_VAL); } diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index 54565bd0093b..0093ece661fe 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -307,7 +307,7 @@ static void samsung_clocksource_resume(struct clocksource *cs) samsung_time_start(pwm.source_id, true); } -static cycle_t notrace samsung_clocksource_read(struct clocksource *c) +static u64 notrace samsung_clocksource_read(struct clocksource *c) { return ~readl_relaxed(pwm.source_reg); } diff --git a/drivers/clocksource/scx200_hrt.c b/drivers/clocksource/scx200_hrt.c index 64f9e8294434..a46660bf6588 100644 --- a/drivers/clocksource/scx200_hrt.c +++ b/drivers/clocksource/scx200_hrt.c @@ -43,10 +43,10 @@ MODULE_PARM_DESC(ppm, "+-adjust to actual XO freq (ppm)"); /* The base timer frequency, * 27 if selected */ #define HRT_FREQ 1000000 -static cycle_t read_hrt(struct clocksource *cs) +static u64 read_hrt(struct clocksource *cs) { /* Read the timer value */ - return (cycle_t) inl(scx200_cb_base + SCx200_TIMER_OFFSET); + return (u64) inl(scx200_cb_base + SCx200_TIMER_OFFSET); } static struct clocksource cs_hrt = { diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index 103c49362c68..28757edf6aca 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -612,7 +612,7 @@ static struct sh_cmt_channel *cs_to_sh_cmt(struct clocksource *cs) return container_of(cs, struct sh_cmt_channel, cs); } -static cycle_t sh_cmt_clocksource_read(struct clocksource *cs) +static u64 sh_cmt_clocksource_read(struct clocksource *cs) { struct sh_cmt_channel *ch = cs_to_sh_cmt(cs); unsigned long flags, raw; diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index 469e776ec17a..1fbf2aadcfd4 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -255,7 +255,7 @@ static struct sh_tmu_channel *cs_to_sh_tmu(struct clocksource *cs) return container_of(cs, struct sh_tmu_channel, cs); } -static cycle_t sh_tmu_clocksource_read(struct clocksource *cs) +static u64 sh_tmu_clocksource_read(struct clocksource *cs) { struct sh_tmu_channel *ch = cs_to_sh_tmu(cs); diff --git a/drivers/clocksource/tcb_clksrc.c b/drivers/clocksource/tcb_clksrc.c index 4da2af9694a2..d4ca9962a759 100644 --- a/drivers/clocksource/tcb_clksrc.c +++ b/drivers/clocksource/tcb_clksrc.c @@ -41,7 +41,7 @@ static void __iomem *tcaddr; -static cycle_t tc_get_cycles(struct clocksource *cs) +static u64 tc_get_cycles(struct clocksource *cs) { unsigned long flags; u32 lower, upper; @@ -56,7 +56,7 @@ static cycle_t tc_get_cycles(struct clocksource *cs) return (upper << 16) | lower; } -static cycle_t tc_get_cycles32(struct clocksource *cs) +static u64 tc_get_cycles32(struct clocksource *cs) { return __raw_readl(tcaddr + ATMEL_TC_REG(0, CV)); } diff --git a/drivers/clocksource/time-pistachio.c b/drivers/clocksource/time-pistachio.c index a8e6c7df853d..3710e4d9dcba 100644 --- a/drivers/clocksource/time-pistachio.c +++ b/drivers/clocksource/time-pistachio.c @@ -67,7 +67,7 @@ static inline void gpt_writel(void __iomem *base, u32 value, u32 offset, writel(value, base + 0x20 * gpt_id + offset); } -static cycle_t notrace +static u64 notrace pistachio_clocksource_read_cycles(struct clocksource *cs) { struct pistachio_clocksource *pcs = to_pistachio_clocksource(cs); @@ -84,7 +84,7 @@ pistachio_clocksource_read_cycles(struct clocksource *cs) counter = gpt_readl(pcs->base, TIMER_CURRENT_VALUE, 0); raw_spin_unlock_irqrestore(&pcs->lock, flags); - return (cycle_t)~counter; + return (u64)~counter; } static u64 notrace pistachio_read_sched_clock(void) diff --git a/drivers/clocksource/timer-atlas7.c b/drivers/clocksource/timer-atlas7.c index 4334e0330ada..db0f21e7d7d2 100644 --- a/drivers/clocksource/timer-atlas7.c +++ b/drivers/clocksource/timer-atlas7.c @@ -85,7 +85,7 @@ static irqreturn_t sirfsoc_timer_interrupt(int irq, void *dev_id) } /* read 64-bit timer counter */ -static cycle_t sirfsoc_timer_read(struct clocksource *cs) +static u64 sirfsoc_timer_read(struct clocksource *cs) { u64 cycles; diff --git a/drivers/clocksource/timer-atmel-pit.c b/drivers/clocksource/timer-atmel-pit.c index 6555821bbdae..c0b5df3167a0 100644 --- a/drivers/clocksource/timer-atmel-pit.c +++ b/drivers/clocksource/timer-atmel-pit.c @@ -73,7 +73,7 @@ static inline void pit_write(void __iomem *base, unsigned int reg_offset, unsign * Clocksource: just a monotonic counter of MCK/16 cycles. * We don't care whether or not PIT irqs are enabled. */ -static cycle_t read_pit_clk(struct clocksource *cs) +static u64 read_pit_clk(struct clocksource *cs) { struct pit_data *data = clksrc_to_pit_data(cs); unsigned long flags; diff --git a/drivers/clocksource/timer-atmel-st.c b/drivers/clocksource/timer-atmel-st.c index e90ab5b63a90..be4ac7604136 100644 --- a/drivers/clocksource/timer-atmel-st.c +++ b/drivers/clocksource/timer-atmel-st.c @@ -92,7 +92,7 @@ static irqreturn_t at91rm9200_timer_interrupt(int irq, void *dev_id) return IRQ_NONE; } -static cycle_t read_clk32k(struct clocksource *cs) +static u64 read_clk32k(struct clocksource *cs) { return read_CRTR(); } diff --git a/drivers/clocksource/timer-nps.c b/drivers/clocksource/timer-nps.c index 8da5e93b6810..da1f7986e477 100644 --- a/drivers/clocksource/timer-nps.c +++ b/drivers/clocksource/timer-nps.c @@ -77,11 +77,11 @@ static int __init nps_get_timer_clk(struct device_node *node, return 0; } -static cycle_t nps_clksrc_read(struct clocksource *clksrc) +static u64 nps_clksrc_read(struct clocksource *clksrc) { int cluster = raw_smp_processor_id() >> NPS_CLUSTER_OFFSET; - return (cycle_t)ioread32be(nps_msu_reg_low_addr[cluster]); + return (u64)ioread32be(nps_msu_reg_low_addr[cluster]); } static int __init nps_setup_clocksource(struct device_node *node) diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index c32148ec7a38..bfa981ac1eaf 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -72,7 +72,7 @@ static irqreturn_t sirfsoc_timer_interrupt(int irq, void *dev_id) } /* read 64-bit timer counter */ -static cycle_t notrace sirfsoc_timer_read(struct clocksource *cs) +static u64 notrace sirfsoc_timer_read(struct clocksource *cs) { u64 cycles; diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c index 4f87f3e76d83..a3e662b15964 100644 --- a/drivers/clocksource/timer-sun5i.c +++ b/drivers/clocksource/timer-sun5i.c @@ -152,7 +152,7 @@ static irqreturn_t sun5i_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static cycle_t sun5i_clksrc_read(struct clocksource *clksrc) +static u64 sun5i_clksrc_read(struct clocksource *clksrc) { struct sun5i_timer_clksrc *cs = to_sun5i_timer_clksrc(clksrc); diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c index cf5b14e442e4..624067712ef0 100644 --- a/drivers/clocksource/timer-ti-32k.c +++ b/drivers/clocksource/timer-ti-32k.c @@ -65,11 +65,11 @@ static inline struct ti_32k *to_ti_32k(struct clocksource *cs) return container_of(cs, struct ti_32k, cs); } -static cycle_t notrace ti_32k_read_cycles(struct clocksource *cs) +static u64 notrace ti_32k_read_cycles(struct clocksource *cs) { struct ti_32k *ti = to_ti_32k(cs); - return (cycle_t)readl_relaxed(ti->counter); + return (u64)readl_relaxed(ti->counter); } static struct ti_32k ti_32k_timer = { diff --git a/drivers/clocksource/vt8500_timer.c b/drivers/clocksource/vt8500_timer.c index b15069483fbd..d02b51075ad1 100644 --- a/drivers/clocksource/vt8500_timer.c +++ b/drivers/clocksource/vt8500_timer.c @@ -53,7 +53,7 @@ static void __iomem *regbase; -static cycle_t vt8500_timer_read(struct clocksource *cs) +static u64 vt8500_timer_read(struct clocksource *cs) { int loops = msecs_to_loops(10); writel(3, regbase + TIMER_CTRL_VAL); @@ -75,7 +75,7 @@ static int vt8500_timer_set_next_event(unsigned long cycles, struct clock_event_device *evt) { int loops = msecs_to_loops(10); - cycle_t alarm = clocksource.read(&clocksource) + cycles; + u64 alarm = clocksource.read(&clocksource) + cycles; while ((readl(regbase + TIMER_AS_VAL) & TIMER_MATCH_W_ACTIVE) && --loops) cpu_relax(); diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index 446802ae8f1b..b44b32f21e61 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -135,9 +135,9 @@ u64 hv_do_hypercall(u64 control, void *input, void *output) EXPORT_SYMBOL_GPL(hv_do_hypercall); #ifdef CONFIG_X86_64 -static cycle_t read_hv_clock_tsc(struct clocksource *arg) +static u64 read_hv_clock_tsc(struct clocksource *arg) { - cycle_t current_tick; + u64 current_tick; struct ms_hyperv_tsc_page *tsc_pg = hv_context.tsc_page; if (tsc_pg->tsc_sequence != 0) { @@ -146,7 +146,7 @@ static cycle_t read_hv_clock_tsc(struct clocksource *arg) */ while (1) { - cycle_t tmp; + u64 tmp; u32 sequence = tsc_pg->tsc_sequence; u64 cur_tsc; u64 scale = tsc_pg->tsc_scale; @@ -350,7 +350,7 @@ int hv_post_message(union hv_connection_id connection_id, static int hv_ce_set_next_event(unsigned long delta, struct clock_event_device *evt) { - cycle_t current_tick; + u64 current_tick; WARN_ON(!clockevent_state_oneshot(evt)); diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index c0178a122940..c01c09e9916d 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -152,12 +152,12 @@ static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe) } #ifdef CONFIG_CLKSRC_MIPS_GIC -cycle_t gic_read_count(void) +u64 gic_read_count(void) { unsigned int hi, hi2, lo; if (mips_cm_is64) - return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER)); + return (u64)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER)); do { hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32)); @@ -165,7 +165,7 @@ cycle_t gic_read_count(void) hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32)); } while (hi2 != hi); - return (((cycle_t) hi) << 32) + lo; + return (((u64) hi) << 32) + lo; } unsigned int gic_get_count_width(void) @@ -179,7 +179,7 @@ unsigned int gic_get_count_width(void) return bits; } -void gic_write_compare(cycle_t cnt) +void gic_write_compare(u64 cnt) { if (mips_cm_is64) { gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt); @@ -191,7 +191,7 @@ void gic_write_compare(cycle_t cnt) } } -void gic_write_cpu_compare(cycle_t cnt, int cpu) +void gic_write_cpu_compare(u64 cnt, int cpu) { unsigned long flags; @@ -211,17 +211,17 @@ void gic_write_cpu_compare(cycle_t cnt, int cpu) local_irq_restore(flags); } -cycle_t gic_read_compare(void) +u64 gic_read_compare(void) { unsigned int hi, lo; if (mips_cm_is64) - return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE)); + return (u64)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE)); hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI)); lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO)); - return (((cycle_t) hi) << 32) + lo; + return (((u64) hi) << 32) + lo; } void gic_start_count(void) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ptp.c b/drivers/net/ethernet/amd/xgbe/xgbe-ptp.c index b03e4f58d02e..a533a6cc2d53 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ptp.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ptp.c @@ -122,7 +122,7 @@ #include "xgbe.h" #include "xgbe-common.h" -static cycle_t xgbe_cc_read(const struct cyclecounter *cc) +static u64 xgbe_cc_read(const struct cyclecounter *cc) { struct xgbe_prv_data *pdata = container_of(cc, struct xgbe_prv_data, diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 688617ac8c29..d8d06fdfc42b 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -15223,7 +15223,7 @@ void bnx2x_set_rx_ts(struct bnx2x *bp, struct sk_buff *skb) } /* Read the PHC */ -static cycle_t bnx2x_cyclecounter_read(const struct cyclecounter *cc) +static u64 bnx2x_cyclecounter_read(const struct cyclecounter *cc) { struct bnx2x *bp = container_of(cc, struct bnx2x, cyclecounter); int port = BP_PORT(bp); diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index f9e74461bdc0..6ebad3fac81d 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -230,7 +230,7 @@ static int fec_ptp_enable_pps(struct fec_enet_private *fep, uint enable) * cyclecounter structure used to construct a ns counter from the * arbitrary fixed point registers */ -static cycle_t fec_ptp_read(const struct cyclecounter *cc) +static u64 fec_ptp_read(const struct cyclecounter *cc) { struct fec_enet_private *fep = container_of(cc, struct fec_enet_private, cc); diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index ffcf35af4881..eccf1da9356b 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -4305,24 +4305,24 @@ void e1000e_reinit_locked(struct e1000_adapter *adapter) /** * e1000e_sanitize_systim - sanitize raw cycle counter reads * @hw: pointer to the HW structure - * @systim: cycle_t value read, sanitized and returned + * @systim: time value read, sanitized and returned * * Errata for 82574/82583 possible bad bits read from SYSTIMH/L: * check to see that the time is incrementing at a reasonable * rate and is a multiple of incvalue. **/ -static cycle_t e1000e_sanitize_systim(struct e1000_hw *hw, cycle_t systim) +static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim) { u64 time_delta, rem, temp; - cycle_t systim_next; + u64 systim_next; u32 incvalue; int i; incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK; for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) { /* latch SYSTIMH on read of SYSTIML */ - systim_next = (cycle_t)er32(SYSTIML); - systim_next |= (cycle_t)er32(SYSTIMH) << 32; + systim_next = (u64)er32(SYSTIML); + systim_next |= (u64)er32(SYSTIMH) << 32; time_delta = systim_next - systim; temp = time_delta; @@ -4342,13 +4342,13 @@ static cycle_t e1000e_sanitize_systim(struct e1000_hw *hw, cycle_t systim) * e1000e_cyclecounter_read - read raw cycle counter (used by time counter) * @cc: cyclecounter structure **/ -static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc) +static u64 e1000e_cyclecounter_read(const struct cyclecounter *cc) { struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter, cc); struct e1000_hw *hw = &adapter->hw; u32 systimel, systimeh; - cycle_t systim; + u64 systim; /* SYSTIMH latching upon SYSTIML read does not work well. * This means that if SYSTIML overflows after we read it but before * we read SYSTIMH, the value of SYSTIMH has been incremented and we @@ -4368,8 +4368,8 @@ static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc) systimel = systimel_2; } } - systim = (cycle_t)systimel; - systim |= (cycle_t)systimeh << 32; + systim = (u64)systimel; + systim |= (u64)systimeh << 32; if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW) systim = e1000e_sanitize_systim(hw, systim); diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c index ad03763e009a..34cc3be0df8e 100644 --- a/drivers/net/ethernet/intel/e1000e/ptp.c +++ b/drivers/net/ethernet/intel/e1000e/ptp.c @@ -127,8 +127,8 @@ static int e1000e_phc_get_syncdevicetime(ktime_t *device, unsigned long flags; int i; u32 tsync_ctrl; - cycle_t dev_cycles; - cycle_t sys_cycles; + u64 dev_cycles; + u64 sys_cycles; tsync_ctrl = er32(TSYNCTXCTL); tsync_ctrl |= E1000_TSYNCTXCTL_START_SYNC | diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c index c30eea8399a7..c4477552ce9e 100644 --- a/drivers/net/ethernet/intel/igb/igb_ptp.c +++ b/drivers/net/ethernet/intel/igb/igb_ptp.c @@ -77,7 +77,7 @@ static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter); /* SYSTIM read access for the 82576 */ -static cycle_t igb_ptp_read_82576(const struct cyclecounter *cc) +static u64 igb_ptp_read_82576(const struct cyclecounter *cc) { struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc); struct e1000_hw *hw = &igb->hw; @@ -94,7 +94,7 @@ static cycle_t igb_ptp_read_82576(const struct cyclecounter *cc) } /* SYSTIM read access for the 82580 */ -static cycle_t igb_ptp_read_82580(const struct cyclecounter *cc) +static u64 igb_ptp_read_82580(const struct cyclecounter *cc) { struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc); struct e1000_hw *hw = &igb->hw; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c index a92277683a64..1efb404431e9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c @@ -245,7 +245,7 @@ static void ixgbe_ptp_setup_sdp_x540(struct ixgbe_adapter *adapter) * result of SYSTIME is 32bits of "billions of cycles" and 32 bits of * "cycles", rather than seconds and nanoseconds. */ -static cycle_t ixgbe_ptp_read_X550(const struct cyclecounter *hw_cc) +static u64 ixgbe_ptp_read_X550(const struct cyclecounter *hw_cc) { struct ixgbe_adapter *adapter = container_of(hw_cc, struct ixgbe_adapter, hw_cc); @@ -282,7 +282,7 @@ static cycle_t ixgbe_ptp_read_X550(const struct cyclecounter *hw_cc) * cyclecounter structure used to construct a ns counter from the * arbitrary fixed point registers */ -static cycle_t ixgbe_ptp_read_82599(const struct cyclecounter *cc) +static u64 ixgbe_ptp_read_82599(const struct cyclecounter *cc) { struct ixgbe_adapter *adapter = container_of(cc, struct ixgbe_adapter, hw_cc); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c index a5fc46bbcbe2..015198c14fa8 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c @@ -38,7 +38,7 @@ /* mlx4_en_read_clock - read raw cycle counter (to be used by time counter) */ -static cycle_t mlx4_en_read_clock(const struct cyclecounter *tc) +static u64 mlx4_en_read_clock(const struct cyclecounter *tc) { struct mlx4_en_dev *mdev = container_of(tc, struct mlx4_en_dev, cycles); diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index b2ca8a635b2e..5e7840a7a33b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -1823,10 +1823,10 @@ static void unmap_bf_area(struct mlx4_dev *dev) io_mapping_free(mlx4_priv(dev)->bf_mapping); } -cycle_t mlx4_read_clock(struct mlx4_dev *dev) +u64 mlx4_read_clock(struct mlx4_dev *dev) { u32 clockhi, clocklo, clockhi1; - cycle_t cycles; + u64 cycles; int i; struct mlx4_priv *priv = mlx4_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c b/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c index 2cd8e56a573b..746a92c13644 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c @@ -49,7 +49,7 @@ void mlx5e_fill_hwstamp(struct mlx5e_tstamp *tstamp, u64 timestamp, hwts->hwtstamp = ns_to_ktime(nsec); } -static cycle_t mlx5e_read_internal_timer(const struct cyclecounter *cc) +static u64 mlx5e_read_internal_timer(const struct cyclecounter *cc) { struct mlx5e_tstamp *tstamp = container_of(cc, struct mlx5e_tstamp, cycles); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 7b4c339a8a9a..54e5a786f191 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -557,7 +557,7 @@ int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } -cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev) +u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev) { u32 timer_h, timer_h1, timer_l; @@ -567,7 +567,7 @@ cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev) if (timer_h != timer_h1) /* wrap around */ timer_l = ioread32be(&dev->iseg->internal_timer_l); - return (cycle_t)timer_l | (cycle_t)timer_h1 << 32; + return (u64)timer_l | (u64)timer_h1 << 32; } static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index e0a8fbdd1446..d4a99c9757cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -106,7 +106,7 @@ int mlx5_modify_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy, int mlx5_destroy_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy, u32 element_id); int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev); -cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev); +u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev); u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx); struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn); void mlx5_cq_tasklet_cb(unsigned long data); diff --git a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c index 0c0d48e5bea4..32279d21c836 100644 --- a/drivers/net/ethernet/ti/cpts.c +++ b/drivers/net/ethernet/ti/cpts.c @@ -121,7 +121,7 @@ static int cpts_fifo_read(struct cpts *cpts, int match) return type == match ? 0 : -1; } -static cycle_t cpts_systim_read(const struct cyclecounter *cc) +static u64 cpts_systim_read(const struct cyclecounter *cc) { u64 val = 0; struct cpts_event *event; diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index dda39d8fa189..b717ed9d2b75 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -25,13 +25,13 @@ struct arch_timer_kvm { /* Virtual offset */ - cycle_t cntvoff; + u64 cntvoff; }; struct arch_timer_cpu { /* Registers: control register, timer value */ u32 cntv_ctl; /* Saved/restored */ - cycle_t cntv_cval; /* Saved/restored */ + u64 cntv_cval; /* Saved/restored */ /* * Anything that is not used directly from assembly code goes diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 65602d395a52..e315d04a2fd9 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -75,8 +75,8 @@ struct module; * structure. */ struct clocksource { - cycle_t (*read)(struct clocksource *cs); - cycle_t mask; + u64 (*read)(struct clocksource *cs); + u64 mask; u32 mult; u32 shift; u64 max_idle_ns; @@ -98,8 +98,8 @@ struct clocksource { #ifdef CONFIG_CLOCKSOURCE_WATCHDOG /* Watchdog related data, used by the framework */ struct list_head wd_list; - cycle_t cs_last; - cycle_t wd_last; + u64 cs_last; + u64 wd_last; #endif struct module *owner; }; @@ -117,7 +117,7 @@ struct clocksource { #define CLOCK_SOURCE_RESELECT 0x100 /* simplify initialization of mask field */ -#define CLOCKSOURCE_MASK(bits) (cycle_t)((bits) < 64 ? ((1ULL<<(bits))-1) : -1) +#define CLOCKSOURCE_MASK(bits) (u64)((bits) < 64 ? ((1ULL<<(bits))-1) : -1) static inline u32 clocksource_freq2mult(u32 freq, u32 shift_constant, u64 from) { @@ -176,7 +176,7 @@ static inline u32 clocksource_hz2mult(u32 hz, u32 shift_constant) * * XXX - This could use some mult_lxl_ll() asm optimization */ -static inline s64 clocksource_cyc2ns(cycle_t cycles, u32 mult, u32 shift) +static inline s64 clocksource_cyc2ns(u64 cycles, u32 mult, u32 shift) { return ((u64) cycles * mult) >> shift; } @@ -236,13 +236,13 @@ static inline void __clocksource_update_freq_khz(struct clocksource *cs, u32 khz extern int timekeeping_notify(struct clocksource *clock); -extern cycle_t clocksource_mmio_readl_up(struct clocksource *); -extern cycle_t clocksource_mmio_readl_down(struct clocksource *); -extern cycle_t clocksource_mmio_readw_up(struct clocksource *); -extern cycle_t clocksource_mmio_readw_down(struct clocksource *); +extern u64 clocksource_mmio_readl_up(struct clocksource *); +extern u64 clocksource_mmio_readl_down(struct clocksource *); +extern u64 clocksource_mmio_readw_up(struct clocksource *); +extern u64 clocksource_mmio_readw_down(struct clocksource *); extern int clocksource_mmio_init(void __iomem *, const char *, - unsigned long, int, unsigned, cycle_t (*)(struct clocksource *)); + unsigned long, int, unsigned, u64 (*)(struct clocksource *)); extern int clocksource_i8253_init(void); diff --git a/include/linux/dw_apb_timer.h b/include/linux/dw_apb_timer.h index 1f79b20918b1..4334106f44c3 100644 --- a/include/linux/dw_apb_timer.h +++ b/include/linux/dw_apb_timer.h @@ -50,6 +50,6 @@ dw_apb_clocksource_init(unsigned rating, const char *name, void __iomem *base, unsigned long freq); void dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs); void dw_apb_clocksource_start(struct dw_apb_clocksource *dw_cs); -cycle_t dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs); +u64 dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs); #endif /* __DW_APB_TIMER_H__ */ diff --git a/include/linux/irqchip/mips-gic.h b/include/linux/irqchip/mips-gic.h index 81f930b0bca9..7b49c71c968b 100644 --- a/include/linux/irqchip/mips-gic.h +++ b/include/linux/irqchip/mips-gic.h @@ -259,11 +259,11 @@ extern void gic_init(unsigned long gic_base_addr, unsigned long gic_addrspace_size, unsigned int cpu_vec, unsigned int irqbase); extern void gic_clocksource_init(unsigned int); -extern cycle_t gic_read_count(void); +extern u64 gic_read_count(void); extern unsigned int gic_get_count_width(void); -extern cycle_t gic_read_compare(void); -extern void gic_write_compare(cycle_t cnt); -extern void gic_write_cpu_compare(cycle_t cnt, int cpu); +extern u64 gic_read_compare(void); +extern void gic_write_compare(u64 cnt); +extern void gic_write_cpu_compare(u64 cnt, int cpu); extern void gic_start_count(void); extern void gic_stop_count(void); extern int gic_get_c0_compare_int(void); diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index c9f379689dd0..93bdb3485192 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -1460,7 +1460,7 @@ int mlx4_get_roce_gid_from_slave(struct mlx4_dev *dev, int port, int slave_id, int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, u32 max_range_qpn); -cycle_t mlx4_read_clock(struct mlx4_dev *dev); +u64 mlx4_read_clock(struct mlx4_dev *dev); struct mlx4_active_ports { DECLARE_BITMAP(ports, MLX4_MAX_PORTS); diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h index 4382035a75bb..2496ad4cfc99 100644 --- a/include/linux/timecounter.h +++ b/include/linux/timecounter.h @@ -20,7 +20,7 @@ #include /* simplify initialization of mask field */ -#define CYCLECOUNTER_MASK(bits) (cycle_t)((bits) < 64 ? ((1ULL<<(bits))-1) : -1) +#define CYCLECOUNTER_MASK(bits) (u64)((bits) < 64 ? ((1ULL<<(bits))-1) : -1) /** * struct cyclecounter - hardware abstraction for a free running counter @@ -37,8 +37,8 @@ * @shift: cycle to nanosecond divisor (power of two) */ struct cyclecounter { - cycle_t (*read)(const struct cyclecounter *cc); - cycle_t mask; + u64 (*read)(const struct cyclecounter *cc); + u64 mask; u32 mult; u32 shift; }; @@ -63,7 +63,7 @@ struct cyclecounter { */ struct timecounter { const struct cyclecounter *cc; - cycle_t cycle_last; + u64 cycle_last; u64 nsec; u64 mask; u64 frac; @@ -77,7 +77,7 @@ struct timecounter { * @frac: pointer to storage for the fractional nanoseconds. */ static inline u64 cyclecounter_cyc2ns(const struct cyclecounter *cc, - cycle_t cycles, u64 mask, u64 *frac) + u64 cycles, u64 mask, u64 *frac) { u64 ns = (u64) cycles; @@ -134,6 +134,6 @@ extern u64 timecounter_read(struct timecounter *tc); * in the past. */ extern u64 timecounter_cyc2time(struct timecounter *tc, - cycle_t cycle_tstamp); + u64 cycle_tstamp); #endif diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index e88005459035..110f4532188c 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -29,9 +29,9 @@ */ struct tk_read_base { struct clocksource *clock; - cycle_t (*read)(struct clocksource *cs); - cycle_t mask; - cycle_t cycle_last; + u64 (*read)(struct clocksource *cs); + u64 mask; + u64 cycle_last; u32 mult; u32 shift; u64 xtime_nsec; @@ -97,7 +97,7 @@ struct timekeeper { struct timespec64 raw_time; /* The following members are for timekeeping internal use */ - cycle_t cycle_interval; + u64 cycle_interval; u64 xtime_interval; s64 xtime_remainder; u32 raw_interval; @@ -136,7 +136,7 @@ extern void update_vsyscall_tz(void); extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, struct clocksource *c, u32 mult, - cycle_t cycle_last); + u64 cycle_last); extern void update_vsyscall_tz(void); #else diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 361f8bf1429d..d2e804e15c3e 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -293,7 +293,7 @@ extern void ktime_get_raw_and_real_ts64(struct timespec64 *ts_raw, * @cs_was_changed_seq: The sequence number of clocksource change events */ struct system_time_snapshot { - cycle_t cycles; + u64 cycles; ktime_t real; ktime_t raw; unsigned int clock_was_set_seq; @@ -321,7 +321,7 @@ struct system_device_crosststamp { * timekeeping code to verify comparibility of two cycle values */ struct system_counterval_t { - cycle_t cycles; + u64 cycles; struct clocksource *cs; }; diff --git a/include/linux/types.h b/include/linux/types.h index d501ad3ba247..1e7bd24848fc 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -228,8 +228,5 @@ struct callback_head { typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); -/* clocksource cycle base type */ -typedef u64 cycle_t; - #endif /* __ASSEMBLY__ */ #endif /* _LINUX_TYPES_H */ diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 150242ccfcd2..665985b0a89a 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -170,7 +170,7 @@ void clocksource_mark_unstable(struct clocksource *cs) static void clocksource_watchdog(unsigned long data) { struct clocksource *cs; - cycle_t csnow, wdnow, cslast, wdlast, delta; + u64 csnow, wdnow, cslast, wdlast, delta; int64_t wd_nsec, cs_nsec; int next_cpu, reset_pending; diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 555e21f7b966..a4a0e478e44d 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -59,9 +59,9 @@ #define JIFFIES_SHIFT 8 #endif -static cycle_t jiffies_read(struct clocksource *cs) +static u64 jiffies_read(struct clocksource *cs) { - return (cycle_t) jiffies; + return (u64) jiffies; } static struct clocksource clocksource_jiffies = { diff --git a/kernel/time/timecounter.c b/kernel/time/timecounter.c index 4687b3104bae..8afd78932bdf 100644 --- a/kernel/time/timecounter.c +++ b/kernel/time/timecounter.c @@ -43,7 +43,7 @@ EXPORT_SYMBOL_GPL(timecounter_init); */ static u64 timecounter_read_delta(struct timecounter *tc) { - cycle_t cycle_now, cycle_delta; + u64 cycle_now, cycle_delta; u64 ns_offset; /* read cycle counter: */ @@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(timecounter_read); * time previous to the time stored in the cycle counter. */ static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, - cycle_t cycles, u64 mask, u64 frac) + u64 cycles, u64 mask, u64 frac) { u64 ns = (u64) cycles; @@ -90,7 +90,7 @@ static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, } u64 timecounter_cyc2time(struct timecounter *tc, - cycle_t cycle_tstamp) + u64 cycle_tstamp) { u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask; u64 nsec = tc->nsec, frac = tc->frac; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index da233cdf89b0..f4152a69277f 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -119,10 +119,10 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) #ifdef CONFIG_DEBUG_TIMEKEEPING #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */ -static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset) +static void timekeeping_check_update(struct timekeeper *tk, u64 offset) { - cycle_t max_cycles = tk->tkr_mono.clock->max_cycles; + u64 max_cycles = tk->tkr_mono.clock->max_cycles; const char *name = tk->tkr_mono.clock->name; if (offset > max_cycles) { @@ -158,10 +158,10 @@ static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset) } } -static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr) +static inline u64 timekeeping_get_delta(struct tk_read_base *tkr) { struct timekeeper *tk = &tk_core.timekeeper; - cycle_t now, last, mask, max, delta; + u64 now, last, mask, max, delta; unsigned int seq; /* @@ -199,12 +199,12 @@ static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr) return delta; } #else -static inline void timekeeping_check_update(struct timekeeper *tk, cycle_t offset) +static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset) { } -static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr) +static inline u64 timekeeping_get_delta(struct tk_read_base *tkr) { - cycle_t cycle_now, delta; + u64 cycle_now, delta; /* read clocksource */ cycle_now = tkr->read(tkr->clock); @@ -229,7 +229,7 @@ static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr) */ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) { - cycle_t interval; + u64 interval; u64 tmp, ntpinterval; struct clocksource *old_clock; @@ -254,7 +254,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) if (tmp == 0) tmp = 1; - interval = (cycle_t) tmp; + interval = (u64) tmp; tk->cycle_interval = interval; /* Go back from cycles -> shifted ns */ @@ -298,8 +298,7 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; static inline u32 arch_gettimeoffset(void) { return 0; } #endif -static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, - cycle_t delta) +static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, u64 delta) { u64 nsec; @@ -312,16 +311,15 @@ static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, static inline u64 timekeeping_get_ns(struct tk_read_base *tkr) { - cycle_t delta; + u64 delta; delta = timekeeping_get_delta(tkr); return timekeeping_delta_to_ns(tkr, delta); } -static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr, - cycle_t cycles) +static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr, u64 cycles) { - cycle_t delta; + u64 delta; /* calculate the delta since the last update_wall_time */ delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask); @@ -454,9 +452,9 @@ u64 notrace ktime_get_boot_fast_ns(void) EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); /* Suspend-time cycles value for halted fast timekeeper. */ -static cycle_t cycles_at_suspend; +static u64 cycles_at_suspend; -static cycle_t dummy_clock_read(struct clocksource *cs) +static u64 dummy_clock_read(struct clocksource *cs) { return cycles_at_suspend; } @@ -650,7 +648,7 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) static void timekeeping_forward_now(struct timekeeper *tk) { struct clocksource *clock = tk->tkr_mono.clock; - cycle_t cycle_now, delta; + u64 cycle_now, delta; u64 nsec; cycle_now = tk->tkr_mono.read(clock); @@ -923,7 +921,7 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot) ktime_t base_real; u64 nsec_raw; u64 nsec_real; - cycle_t now; + u64 now; WARN_ON_ONCE(timekeeping_suspended); @@ -982,8 +980,8 @@ static int scale64_check_overflow(u64 mult, u64 div, u64 *base) * interval is partial_history_cycles. */ static int adjust_historical_crosststamp(struct system_time_snapshot *history, - cycle_t partial_history_cycles, - cycle_t total_history_cycles, + u64 partial_history_cycles, + u64 total_history_cycles, bool discontinuity, struct system_device_crosststamp *ts) { @@ -1047,7 +1045,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, /* * cycle_between - true if test occurs chronologically between before and after */ -static bool cycle_between(cycle_t before, cycle_t test, cycle_t after) +static bool cycle_between(u64 before, u64 test, u64 after) { if (test > before && test < after) return true; @@ -1077,7 +1075,7 @@ int get_device_system_crosststamp(int (*get_time_fn) { struct system_counterval_t system_counterval; struct timekeeper *tk = &tk_core.timekeeper; - cycle_t cycles, now, interval_start; + u64 cycles, now, interval_start; unsigned int clock_was_set_seq = 0; ktime_t base_real, base_raw; u64 nsec_real, nsec_raw; @@ -1138,7 +1136,7 @@ int get_device_system_crosststamp(int (*get_time_fn) * current interval */ if (do_interp) { - cycle_t partial_history_cycles, total_history_cycles; + u64 partial_history_cycles, total_history_cycles; bool discontinuity; /* @@ -1644,7 +1642,7 @@ void timekeeping_resume(void) struct clocksource *clock = tk->tkr_mono.clock; unsigned long flags; struct timespec64 ts_new, ts_delta; - cycle_t cycle_now; + u64 cycle_now; sleeptime_injected = false; read_persistent_clock64(&ts_new); @@ -2010,11 +2008,10 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) * * Returns the unconsumed cycles. */ -static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, - u32 shift, - unsigned int *clock_set) +static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, + u32 shift, unsigned int *clock_set) { - cycle_t interval = tk->cycle_interval << shift; + u64 interval = tk->cycle_interval << shift; u64 raw_nsecs; /* If the offset is smaller than a shifted interval, do nothing */ @@ -2055,7 +2052,7 @@ void update_wall_time(void) { struct timekeeper *real_tk = &tk_core.timekeeper; struct timekeeper *tk = &shadow_timekeeper; - cycle_t offset; + u64 offset; int shift = 0, maxshift; unsigned int clock_set = 0; unsigned long flags; diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h index 5be76270ec4a..9a18f121f399 100644 --- a/kernel/time/timekeeping_internal.h +++ b/kernel/time/timekeeping_internal.h @@ -13,9 +13,9 @@ extern void tk_debug_account_sleep_time(struct timespec64 *t); #endif #ifdef CONFIG_CLOCKSOURCE_VALIDATE_LAST_CYCLE -static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) +static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) { - cycle_t ret = (now - last) & mask; + u64 ret = (now - last) & mask; /* * Prevent time going backwards by checking the MSB of mask in @@ -24,7 +24,7 @@ static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) return ret & ~(mask >> 1) ? 0 : ret; } #else -static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) +static inline u64 clocksource_delta(u64 now, u64 last, u64 mask) { return (now - last) & mask; } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 1f0f547c54da..eb230f06ba41 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -2847,7 +2847,7 @@ static void ftrace_shutdown_sysctl(void) } } -static cycle_t ftrace_update_time; +static u64 ftrace_update_time; unsigned long ftrace_update_tot_cnt; static inline int ops_traces_mod(struct ftrace_ops *ops) @@ -2894,7 +2894,7 @@ static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) { struct ftrace_page *pg; struct dyn_ftrace *p; - cycle_t start, stop; + u64 start, stop; unsigned long update_cnt = 0; unsigned long rec_flags = 0; int i; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 66f829c47bec..d7449783987a 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -236,7 +236,7 @@ static int __init set_tracepoint_printk(char *str) } __setup("tp_printk", set_tracepoint_printk); -unsigned long long ns2usecs(cycle_t nsec) +unsigned long long ns2usecs(u64 nsec) { nsec += 500; do_div(nsec, 1000); @@ -573,7 +573,7 @@ int trace_pid_write(struct trace_pid_list *filtered_pids, return read; } -static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu) +static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu) { u64 ts; @@ -587,7 +587,7 @@ static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu) return ts; } -cycle_t ftrace_now(int cpu) +u64 ftrace_now(int cpu) { return buffer_ftrace_now(&global_trace.trace_buffer, cpu); } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index c2234494f40c..1ea51ab53edf 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -159,7 +159,7 @@ struct trace_array_cpu { unsigned long policy; unsigned long rt_priority; unsigned long skipped_entries; - cycle_t preempt_timestamp; + u64 preempt_timestamp; pid_t pid; kuid_t uid; char comm[TASK_COMM_LEN]; @@ -177,7 +177,7 @@ struct trace_buffer { struct trace_array *tr; struct ring_buffer *buffer; struct trace_array_cpu __percpu *data; - cycle_t time_start; + u64 time_start; int cpu; }; @@ -689,7 +689,7 @@ static inline void __trace_stack(struct trace_array *tr, unsigned long flags, } #endif /* CONFIG_STACKTRACE */ -extern cycle_t ftrace_now(int cpu); +extern u64 ftrace_now(int cpu); extern void trace_find_cmdline(int pid, char comm[]); extern void trace_event_follow_fork(struct trace_array *tr, bool enable); @@ -736,7 +736,7 @@ extern int trace_selftest_startup_branch(struct tracer *trace, #endif /* CONFIG_FTRACE_STARTUP_TEST */ extern void *head_page(struct trace_array_cpu *data); -extern unsigned long long ns2usecs(cycle_t nsec); +extern unsigned long long ns2usecs(u64 nsec); extern int trace_vbprintk(unsigned long ip, const char *fmt, va_list args); extern int diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index 86654d7e1afe..7758bc0617cb 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c @@ -298,7 +298,7 @@ static void irqsoff_print_header(struct seq_file *s) /* * Should this new latency be reported/recorded? */ -static bool report_latency(struct trace_array *tr, cycle_t delta) +static bool report_latency(struct trace_array *tr, u64 delta) { if (tracing_thresh) { if (delta < tracing_thresh) @@ -316,7 +316,7 @@ check_critical_timing(struct trace_array *tr, unsigned long parent_ip, int cpu) { - cycle_t T0, T1, delta; + u64 T0, T1, delta; unsigned long flags; int pc; diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 5d0bb025bb21..ddec53b67646 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -358,7 +358,7 @@ static void wakeup_print_header(struct seq_file *s) /* * Should this new latency be reported/recorded? */ -static bool report_latency(struct trace_array *tr, cycle_t delta) +static bool report_latency(struct trace_array *tr, u64 delta) { if (tracing_thresh) { if (delta < tracing_thresh) @@ -440,7 +440,7 @@ probe_wakeup_sched_switch(void *ignore, bool preempt, struct task_struct *prev, struct task_struct *next) { struct trace_array_cpu *data; - cycle_t T0, T1, delta; + u64 T0, T1, delta; unsigned long flags; long disabled; int cpu; diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c index 38990a77d7b7..c6994ebb4567 100644 --- a/sound/hda/hdac_stream.c +++ b/sound/hda/hdac_stream.c @@ -465,7 +465,7 @@ int snd_hdac_stream_set_params(struct hdac_stream *azx_dev, } EXPORT_SYMBOL_GPL(snd_hdac_stream_set_params); -static cycle_t azx_cc_read(const struct cyclecounter *cc) +static u64 azx_cc_read(const struct cyclecounter *cc) { struct hdac_stream *azx_dev = container_of(cc, struct hdac_stream, cc); @@ -473,7 +473,7 @@ static cycle_t azx_cc_read(const struct cyclecounter *cc) } static void azx_timecounter_init(struct hdac_stream *azx_dev, - bool force, cycle_t last) + bool force, u64 last) { struct timecounter *tc = &azx_dev->tc; struct cyclecounter *cc = &azx_dev->cc; @@ -523,7 +523,7 @@ void snd_hdac_stream_timecounter_init(struct hdac_stream *azx_dev, struct snd_pcm_runtime *runtime = azx_dev->substream->runtime; struct hdac_stream *s; bool inited = false; - cycle_t cycle_last = 0; + u64 cycle_last = 0; int i = 0; list_for_each_entry(s, &bus->stream_list, list) { diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index ae95fc0e3214..97b657adb3bd 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -39,7 +39,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) vcpu->arch.timer_cpu.active_cleared_last = false; } -static cycle_t kvm_phys_timer_read(void) +static u64 kvm_phys_timer_read(void) { return timecounter->cc->read(timecounter->cc); } @@ -102,7 +102,7 @@ static void kvm_timer_inject_irq_work(struct work_struct *work) static u64 kvm_timer_compute_delta(struct kvm_vcpu *vcpu) { - cycle_t cval, now; + u64 cval, now; cval = vcpu->arch.timer_cpu.cntv_cval; now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff; @@ -155,7 +155,7 @@ static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu) bool kvm_timer_should_fire(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; - cycle_t cval, now; + u64 cval, now; if (!kvm_timer_irq_can_fire(vcpu)) return false; -- cgit v1.3-7-g2ca7