diff options
Diffstat (limited to 'drivers/tty/tty_buffer.c')
| -rw-r--r-- | drivers/tty/tty_buffer.c | 409 |
1 files changed, 244 insertions, 165 deletions
diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index ec145a59f199..5e287dedce01 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -5,6 +5,7 @@ #include <linux/types.h> #include <linux/errno.h> +#include <linux/minmax.h> #include <linux/tty.h> #include <linux/tty_driver.h> #include <linux/tty_flip.h> @@ -17,7 +18,7 @@ #include <linux/delay.h> #include <linux/module.h> #include <linux/ratelimit.h> - +#include "tty.h" #define MIN_TTYB_SIZE 256 #define TTYB_ALIGN_MASK 255 @@ -32,27 +33,22 @@ * We default to dicing tty buffer allocations to this many characters * in order to avoid multiple page allocations. We know the size of * tty_buffer itself but it must also be taken into account that the - * the buffer is 256 byte aligned. See tty_buffer_find for the allocation - * logic this must match + * buffer is 256 byte aligned. See tty_buffer_find for the allocation + * logic this must match. */ #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF) /** - * tty_buffer_lock_exclusive - gain exclusive access to buffer - * tty_buffer_unlock_exclusive - release exclusive access - * - * @port - tty_port owning the flip buffer + * tty_buffer_lock_exclusive - gain exclusive access to buffer + * @port: tty port owning the flip buffer * - * Guarantees safe use of the line discipline's receive_buf() method by - * excluding the buffer work and any pending flush from using the flip - * buffer. Data can continue to be added concurrently to the flip buffer - * from the driver side. + * Guarantees safe use of the &tty_ldisc_ops.receive_buf() method by excluding + * the buffer work and any pending flush from using the flip buffer. Data can + * continue to be added concurrently to the flip buffer from the driver side. * - * On release, the buffer work is restarted if there is data in the - * flip buffer + * See also tty_buffer_unlock_exclusive(). */ - void tty_buffer_lock_exclusive(struct tty_port *port) { struct tty_bufhead *buf = &port->buf; @@ -62,6 +58,14 @@ void tty_buffer_lock_exclusive(struct tty_port *port) } EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive); +/** + * tty_buffer_unlock_exclusive - release exclusive access + * @port: tty port owning the flip buffer + * + * The buffer work is restarted if there is data in the flip buffer. + * + * See also tty_buffer_lock_exclusive(). + */ void tty_buffer_unlock_exclusive(struct tty_port *port) { struct tty_bufhead *buf = &port->buf; @@ -77,20 +81,20 @@ void tty_buffer_unlock_exclusive(struct tty_port *port) EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive); /** - * tty_buffer_space_avail - return unused buffer space - * @port - tty_port owning the flip buffer + * tty_buffer_space_avail - return unused buffer space + * @port: tty port owning the flip buffer * - * Returns the # of bytes which can be written by the driver without - * reaching the buffer limit. + * Returns: the # of bytes which can be written by the driver without reaching + * the buffer limit. * - * Note: this does not guarantee that memory is available to write - * the returned # of bytes (use tty_prepare_flip_string_xxx() to - * pre-allocate if memory guarantee is required). + * Note: this does not guarantee that memory is available to write the returned + * # of bytes (use tty_prepare_flip_string() to pre-allocate if memory + * guarantee is required). */ - -int tty_buffer_space_avail(struct tty_port *port) +unsigned int tty_buffer_space_avail(struct tty_port *port) { int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used); + return max(space, 0); } EXPORT_SYMBOL_GPL(tty_buffer_space_avail); @@ -101,18 +105,18 @@ static void tty_buffer_reset(struct tty_buffer *p, size_t size) p->size = size; p->next = NULL; p->commit = 0; + p->lookahead = 0; p->read = 0; p->flags = 0; } /** - * tty_buffer_free_all - free buffers used by a tty - * @tty: tty to free from + * tty_buffer_free_all - free buffers used by a tty + * @port: tty port to free from * - * Remove all the buffers pending on a tty whether queued with data - * or in the free ring. Must be called when the tty is no longer in use + * Remove all the buffers pending on a tty whether queued with data or in the + * free ring. Must be called when the tty is no longer in use. */ - void tty_buffer_free_all(struct tty_port *port) { struct tty_bufhead *buf = &port->buf; @@ -141,17 +145,17 @@ void tty_buffer_free_all(struct tty_port *port) } /** - * tty_buffer_alloc - allocate a tty buffer - * @tty: tty device - * @size: desired size (characters) - * - * Allocate a new tty buffer to hold the desired number of characters. - * We round our buffers off in 256 character chunks to get better - * allocation behaviour. - * Return NULL if out of memory or the allocation would exceed the - * per device queue + * tty_buffer_alloc - allocate a tty buffer + * @port: tty port + * @size: desired size (characters) + * + * Allocate a new tty buffer to hold the desired number of characters. We + * round our buffers off in 256 character chunks to get better allocation + * behaviour. + * + * Returns: %NULL if out of memory or the allocation would exceed the per + * device queue. */ - static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) { struct llist_node *free; @@ -169,10 +173,12 @@ static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) } /* Should possibly check if this fails for the largest buffer we - have queued and recycle that ? */ + * have queued and recycle that ? + */ if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit) return NULL; - p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC); + p = kmalloc(sizeof(struct tty_buffer) + 2 * size, + GFP_ATOMIC | __GFP_NOWARN); if (p == NULL) return NULL; @@ -183,14 +189,13 @@ found: } /** - * tty_buffer_free - free a tty buffer - * @tty: tty owning the buffer - * @b: the buffer to free + * tty_buffer_free - free a tty buffer + * @port: tty port owning the buffer + * @b: the buffer to free * - * Free a tty buffer, or add it to the free list according to our - * internal strategy + * Free a tty buffer, or add it to the free list according to our internal + * strategy. */ - static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) { struct tty_bufhead *buf = &port->buf; @@ -205,17 +210,15 @@ static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) } /** - * tty_buffer_flush - flush full tty buffers - * @tty: tty to flush - * @ld: optional ldisc ptr (must be referenced) + * tty_buffer_flush - flush full tty buffers + * @tty: tty to flush + * @ld: optional ldisc ptr (must be referenced) * - * flush all the buffers containing receive data. If ld != NULL, - * flush the ldisc input buffer. + * Flush all the buffers containing receive data. If @ld != %NULL, flush the + * ldisc input buffer. * - * Locking: takes buffer lock to ensure single-threaded flip buffer - * 'consumer' + * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. */ - void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) { struct tty_port *port = tty->port; @@ -233,6 +236,7 @@ void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) buf->head = next; } buf->head->read = buf->head->commit; + buf->head->lookahead = buf->head->read; if (ld && ld->ops->flush_buffer) ld->ops->flush_buffer(tty); @@ -242,17 +246,18 @@ void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) } /** - * tty_buffer_request_room - grow tty buffer if needed - * @tty: tty structure - * @size: size desired - * @flags: buffer flags if new buffer allocated (default = 0) + * __tty_buffer_request_room - grow tty buffer if needed + * @port: tty port + * @size: size desired + * @flags: buffer flags if new buffer allocated (default = 0) + * + * Make at least @size bytes of linear space available for the tty buffer. * - * Make at least size bytes of linear space available for the tty - * buffer. If we fail return the size we managed to find. + * Will change over to a new buffer if the current buffer is encoded as + * %TTY_NORMAL (so has no flags buffer) and the new buffer requires a flags + * buffer. * - * Will change over to a new buffer if the current buffer is encoded as - * TTY_NORMAL (so has no flags buffer) and the new buffer requires - * a flags buffer. + * Returns: the size we managed to find. */ static int __tty_buffer_request_room(struct tty_port *port, size_t size, int flags) @@ -274,13 +279,15 @@ static int __tty_buffer_request_room(struct tty_port *port, size_t size, if (n != NULL) { n->flags = flags; buf->tail = n; - /* paired w/ acquire in flush_to_ldisc(); ensures - * flush_to_ldisc() sees buffer data. + /* + * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() + * ensures they see all buffer data. */ smp_store_release(&b->commit, b->used); - /* paired w/ acquire in flush_to_ldisc(); ensures the - * latest commit value can be read before the head is - * advanced to the next buffer + /* + * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() + * ensures the latest commit value can be read before the head + * is advanced to the next buffer. */ smp_store_release(&b->next, n); } else if (change) @@ -298,25 +305,28 @@ int tty_buffer_request_room(struct tty_port *port, size_t size) EXPORT_SYMBOL_GPL(tty_buffer_request_room); /** - * tty_insert_flip_string_fixed_flag - Add characters to the tty buffer - * @port: tty port - * @chars: characters - * @flag: flag value for each character - * @size: size - * - * Queue a series of bytes to the tty buffering. All the characters - * passed are marked with the supplied flag. Returns the number added. + * tty_insert_flip_string_fixed_flag - add characters to the tty buffer + * @port: tty port + * @chars: characters + * @flag: flag value for each character + * @size: size + * + * Queue a series of bytes to the tty buffering. All the characters passed are + * marked with the supplied flag. + * + * Returns: the number added. */ - int tty_insert_flip_string_fixed_flag(struct tty_port *port, const unsigned char *chars, char flag, size_t size) { int copied = 0; + do { int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0; int space = __tty_buffer_request_room(port, goal, flags); struct tty_buffer *tb = port->buf.tail; + if (unlikely(space == 0)) break; memcpy(char_buf_ptr(tb, tb->used), chars, space); @@ -326,32 +336,35 @@ int tty_insert_flip_string_fixed_flag(struct tty_port *port, copied += space; chars += space; /* There is a small chance that we need to split the data over - several buffers. If this is the case we must loop */ + * several buffers. If this is the case we must loop. + */ } while (unlikely(size > copied)); return copied; } EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag); /** - * tty_insert_flip_string_flags - Add characters to the tty buffer - * @port: tty port - * @chars: characters - * @flags: flag bytes - * @size: size - * - * Queue a series of bytes to the tty buffering. For each character - * the flags array indicates the status of the character. Returns the - * number added. + * tty_insert_flip_string_flags - add characters to the tty buffer + * @port: tty port + * @chars: characters + * @flags: flag bytes + * @size: size + * + * Queue a series of bytes to the tty buffering. For each character the flags + * array indicates the status of the character. + * + * Returns: the number added. */ - int tty_insert_flip_string_flags(struct tty_port *port, const unsigned char *chars, const char *flags, size_t size) { int copied = 0; + do { int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); int space = tty_buffer_request_room(port, goal); struct tty_buffer *tb = port->buf.tail; + if (unlikely(space == 0)) break; memcpy(char_buf_ptr(tb, tb->used), chars, space); @@ -361,20 +374,21 @@ int tty_insert_flip_string_flags(struct tty_port *port, chars += space; flags += space; /* There is a small chance that we need to split the data over - several buffers. If this is the case we must loop */ + * several buffers. If this is the case we must loop. + */ } while (unlikely(size > copied)); return copied; } EXPORT_SYMBOL(tty_insert_flip_string_flags); /** - * __tty_insert_flip_char - Add one character to the tty buffer - * @port: tty port - * @ch: character - * @flag: flag byte + * __tty_insert_flip_char - add one character to the tty buffer + * @port: tty port + * @ch: character + * @flag: flag byte * - * Queue a single byte to the tty buffering, with an optional flag. - * This is the slow path of tty_insert_flip_char. + * Queue a single byte @ch to the tty buffering, with an optional flag. This is + * the slow path of tty_insert_flip_char(). */ int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag) { @@ -394,45 +408,27 @@ int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag) EXPORT_SYMBOL(__tty_insert_flip_char); /** - * tty_schedule_flip - push characters to ldisc - * @port: tty port to push from + * tty_prepare_flip_string - make room for characters + * @port: tty port + * @chars: return pointer for character write area + * @size: desired size * - * Takes any pending buffers and transfers their ownership to the - * ldisc side of the queue. It then schedules those characters for - * processing by the line discipline. - */ - -void tty_schedule_flip(struct tty_port *port) -{ - struct tty_bufhead *buf = &port->buf; - - /* paired w/ acquire in flush_to_ldisc(); ensures - * flush_to_ldisc() sees buffer data. - */ - smp_store_release(&buf->tail->commit, buf->tail->used); - queue_work(system_unbound_wq, &buf->work); -} -EXPORT_SYMBOL(tty_schedule_flip); - -/** - * tty_prepare_flip_string - make room for characters - * @port: tty port - * @chars: return pointer for character write area - * @size: desired size - * - * Prepare a block of space in the buffer for data. Returns the length - * available and buffer pointer to the space which is now allocated and - * accounted for as ready for normal characters. This is used for drivers - * that need their own block copy routines into the buffer. There is no - * guarantee the buffer is a DMA target! + * Prepare a block of space in the buffer for data. + * + * This is used for drivers that need their own block copy routines into the + * buffer. There is no guarantee the buffer is a DMA target! + * + * Returns: the length available and buffer pointer (@chars) to the space which + * is now allocated and accounted for as ready for normal characters. */ - int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars, size_t size) { int space = __tty_buffer_request_room(port, size, TTYB_NORMAL); + if (likely(space)) { struct tty_buffer *tb = port->buf.tail; + *chars = char_buf_ptr(tb, tb->used); if (~tb->flags & TTYB_NORMAL) memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space); @@ -443,19 +439,19 @@ int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars, EXPORT_SYMBOL_GPL(tty_prepare_flip_string); /** - * tty_ldisc_receive_buf - forward data to line discipline - * @ld: line discipline to process input - * @p: char buffer - * @f: TTY_* flags buffer - * @count: number of bytes to process + * tty_ldisc_receive_buf - forward data to line discipline + * @ld: line discipline to process input + * @p: char buffer + * @f: %TTY_NORMAL, %TTY_BREAK, etc. flags buffer + * @count: number of bytes to process * - * Callers other than flush_to_ldisc() need to exclude the kworker - * from concurrent use of the line discipline, see paste_selection(). + * Callers other than flush_to_ldisc() need to exclude the kworker from + * concurrent use of the line discipline, see paste_selection(). * - * Returns the number of bytes processed + * Returns: the number of bytes processed. */ int tty_ldisc_receive_buf(struct tty_ldisc *ld, const unsigned char *p, - char *f, int count) + const char *f, int count) { if (ld->ops->receive_buf2) count = ld->ops->receive_buf2(ld->tty, p, f, count); @@ -468,11 +464,49 @@ int tty_ldisc_receive_buf(struct tty_ldisc *ld, const unsigned char *p, } EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); +static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) +{ + head->lookahead = max(head->lookahead, head->read); + + while (head) { + struct tty_buffer *next; + unsigned int count; + + /* + * Paired w/ release in __tty_buffer_request_room(); + * ensures commit value read is not stale if the head + * is advancing to the next buffer. + */ + next = smp_load_acquire(&head->next); + /* + * Paired w/ release in __tty_buffer_request_room() or in + * tty_buffer_flush(); ensures we see the committed buffer data. + */ + count = smp_load_acquire(&head->commit) - head->lookahead; + if (!count) { + head = next; + continue; + } + + if (port->client_ops->lookahead_buf) { + unsigned char *p, *f = NULL; + + p = char_buf_ptr(head, head->lookahead); + if (~head->flags & TTYB_NORMAL) + f = flag_buf_ptr(head, head->lookahead); + + port->client_ops->lookahead_buf(port, p, f, count); + } + + head->lookahead += count; + } +} + static int receive_buf(struct tty_port *port, struct tty_buffer *head, int count) { unsigned char *p = char_buf_ptr(head, head->read); - char *f = NULL; + const char *f = NULL; int n; if (~head->flags & TTYB_NORMAL) @@ -485,18 +519,16 @@ receive_buf(struct tty_port *port, struct tty_buffer *head, int count) } /** - * flush_to_ldisc - * @work: tty structure passed from work queue. + * flush_to_ldisc - flush data from buffer to ldisc + * @work: tty structure passed from work queue. * - * This routine is called out of the software interrupt to flush data - * from the buffer chain to the line discipline. + * This routine is called out of the software interrupt to flush data from the + * buffer chain to the line discipline. * - * The receive_buf method is single threaded for each tty instance. + * The receive_buf() method is single threaded for each tty instance. * - * Locking: takes buffer lock to ensure single-threaded flip buffer - * 'consumer' + * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. */ - static void flush_to_ldisc(struct work_struct *work) { struct tty_port *port = container_of(work, struct tty_port, buf.work); @@ -507,7 +539,7 @@ static void flush_to_ldisc(struct work_struct *work) while (1) { struct tty_buffer *head = buf->head; struct tty_buffer *next; - int count; + int count, rcvd; /* Ldisc or user is trying to gain exclusive access */ if (atomic_read(&buf->priority)) @@ -530,41 +562,87 @@ static void flush_to_ldisc(struct work_struct *work) continue; } - count = receive_buf(port, head, count); - if (!count) + rcvd = receive_buf(port, head, count); + head->read += rcvd; + if (rcvd < count) + lookahead_bufs(port, head); + if (!rcvd) break; - head->read += count; + + if (need_resched()) + cond_resched(); } mutex_unlock(&buf->lock); } +static inline void tty_flip_buffer_commit(struct tty_buffer *tail) +{ + /* + * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees + * buffer data. + */ + smp_store_release(&tail->commit, tail->used); +} + /** - * tty_flip_buffer_push - terminal - * @port: tty port to push + * tty_flip_buffer_push - push terminal buffers + * @port: tty port to push * - * Queue a push of the terminal flip buffers to the line discipline. - * Can be called from IRQ/atomic context. + * Queue a push of the terminal flip buffers to the line discipline. Can be + * called from IRQ/atomic context. * - * In the event of the queue being busy for flipping the work will be - * held off and retried later. + * In the event of the queue being busy for flipping the work will be held off + * and retried later. */ - void tty_flip_buffer_push(struct tty_port *port) { - tty_schedule_flip(port); + struct tty_bufhead *buf = &port->buf; + + tty_flip_buffer_commit(buf->tail); + queue_work(system_unbound_wq, &buf->work); } EXPORT_SYMBOL(tty_flip_buffer_push); /** - * tty_buffer_init - prepare a tty buffer structure - * @tty: tty to initialise + * tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and + * push + * @port: tty port + * @chars: characters + * @size: size + * + * The function combines tty_insert_flip_string() and tty_flip_buffer_push() + * with the exception of properly holding the @port->lock. + * + * To be used only internally (by pty currently). * - * Set up the initial state of the buffer management for a tty device. - * Must be called before the other tty buffer functions are used. + * Returns: the number added. */ +int tty_insert_flip_string_and_push_buffer(struct tty_port *port, + const unsigned char *chars, size_t size) +{ + struct tty_bufhead *buf = &port->buf; + unsigned long flags; + spin_lock_irqsave(&port->lock, flags); + size = tty_insert_flip_string(port, chars, size); + if (size) + tty_flip_buffer_commit(buf->tail); + spin_unlock_irqrestore(&port->lock, flags); + + queue_work(system_unbound_wq, &buf->work); + + return size; +} + +/** + * tty_buffer_init - prepare a tty buffer structure + * @port: tty port to initialise + * + * Set up the initial state of the buffer management for a tty device. Must be + * called before the other tty buffer functions are used. + */ void tty_buffer_init(struct tty_port *port) { struct tty_bufhead *buf = &port->buf; @@ -581,13 +659,14 @@ void tty_buffer_init(struct tty_port *port) } /** - * tty_buffer_set_limit - change the tty buffer memory limit - * @port: tty port to change + * tty_buffer_set_limit - change the tty buffer memory limit + * @port: tty port to change + * @limit: memory limit to set * - * Change the tty buffer memory limit. - * Must be called before the other tty buffer functions are used. + * Change the tty buffer memory limit. + * + * Must be called before the other tty buffer functions are used. */ - int tty_buffer_set_limit(struct tty_port *port, int limit) { if (limit < MIN_TTYB_SIZE) |