diff options
Diffstat (limited to 'drivers/staging/rdma/hfi1/pio_copy.c')
| -rw-r--r-- | drivers/staging/rdma/hfi1/pio_copy.c | 867 |
1 files changed, 0 insertions, 867 deletions
diff --git a/drivers/staging/rdma/hfi1/pio_copy.c b/drivers/staging/rdma/hfi1/pio_copy.c deleted file mode 100644 index 8c25e1b58849..000000000000 --- a/drivers/staging/rdma/hfi1/pio_copy.c +++ /dev/null @@ -1,867 +0,0 @@ -/* - * Copyright(c) 2015, 2016 Intel Corporation. - * - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * BSD LICENSE - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "hfi.h" - -/* additive distance between non-SOP and SOP space */ -#define SOP_DISTANCE (TXE_PIO_SIZE / 2) -#define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1) -/* number of QUADWORDs in a block */ -#define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64)) - -/** - * pio_copy - copy data block to MMIO space - * @pbuf: a number of blocks allocated within a PIO send context - * @pbc: PBC to send - * @from: source, must be 8 byte aligned - * @count: number of DWORD (32-bit) quantities to copy from source - * - * Copy data from source to PIO Send Buffer memory, 8 bytes at a time. - * Must always write full BLOCK_SIZE bytes blocks. The first block must - * be written to the corresponding SOP=1 address. - * - * Known: - * o pbuf->start always starts on a block boundary - * o pbuf can wrap only at a block boundary - */ -void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc, - const void *from, size_t count) -{ - void __iomem *dest = pbuf->start + SOP_DISTANCE; - void __iomem *send = dest + PIO_BLOCK_SIZE; - void __iomem *dend; /* 8-byte data end */ - - /* write the PBC */ - writeq(pbc, dest); - dest += sizeof(u64); - - /* calculate where the QWORD data ends - in SOP=1 space */ - dend = dest + ((count >> 1) * sizeof(u64)); - - if (dend < send) { - /* - * all QWORD data is within the SOP block, does *not* - * reach the end of the SOP block - */ - - while (dest < dend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - /* - * No boundary checks are needed here: - * 0. We're not on the SOP block boundary - * 1. The possible DWORD dangle will still be within - * the SOP block - * 2. We cannot wrap except on a block boundary. - */ - } else { - /* QWORD data extends _to_ or beyond the SOP block */ - - /* write 8-byte SOP chunk data */ - while (dest < send) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - /* drop out of the SOP range */ - dest -= SOP_DISTANCE; - dend -= SOP_DISTANCE; - - /* - * If the wrap comes before or matches the data end, - * copy until until the wrap, then wrap. - * - * If the data ends at the end of the SOP above and - * the buffer wraps, then pbuf->end == dend == dest - * and nothing will get written, but we will wrap in - * case there is a dangling DWORD. - */ - if (pbuf->end <= dend) { - while (dest < pbuf->end) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - - dest -= pbuf->size; - dend -= pbuf->size; - } - - /* write 8-byte non-SOP, non-wrap chunk data */ - while (dest < dend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - } - /* at this point we have wrapped if we are going to wrap */ - - /* write dangling u32, if any */ - if (count & 1) { - union mix val; - - val.val64 = 0; - val.val32[0] = *(u32 *)from; - writeq(val.val64, dest); - dest += sizeof(u64); - } - /* - * fill in rest of block, no need to check pbuf->end - * as we only wrap on a block boundary - */ - while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) { - writeq(0, dest); - dest += sizeof(u64); - } - - /* finished with this buffer */ - this_cpu_dec(*pbuf->sc->buffers_allocated); - preempt_enable(); -} - -/* USE_SHIFTS is faster in user-space tests on a Xeon X5570 @ 2.93GHz */ -#define USE_SHIFTS 1 -#ifdef USE_SHIFTS -/* - * Handle carry bytes using shifts and masks. - * - * NOTE: the value the unused portion of carry is expected to always be zero. - */ - -/* - * "zero" shift - bit shift used to zero out upper bytes. Input is - * the count of LSB bytes to preserve. - */ -#define zshift(x) (8 * (8 - (x))) - -/* - * "merge" shift - bit shift used to merge with carry bytes. Input is - * the LSB byte count to move beyond. - */ -#define mshift(x) (8 * (x)) - -/* - * Read nbytes bytes from "from" and return them in the LSB bytes - * of pbuf->carry. Other bytes are zeroed. Any previous value - * pbuf->carry is lost. - * - * NOTES: - * o do not read from from if nbytes is zero - * o from may _not_ be u64 aligned - * o nbytes must not span a QW boundary - */ -static inline void read_low_bytes(struct pio_buf *pbuf, const void *from, - unsigned int nbytes) -{ - unsigned long off; - - if (nbytes == 0) { - pbuf->carry.val64 = 0; - } else { - /* align our pointer */ - off = (unsigned long)from & 0x7; - from = (void *)((unsigned long)from & ~0x7l); - pbuf->carry.val64 = ((*(u64 *)from) - << zshift(nbytes + off))/* zero upper bytes */ - >> zshift(nbytes); /* place at bottom */ - } - pbuf->carry_bytes = nbytes; -} - -/* - * Read nbytes bytes from "from" and put them at the next significant bytes - * of pbuf->carry. Unused bytes are zeroed. It is expected that the extra - * read does not overfill carry. - * - * NOTES: - * o from may _not_ be u64 aligned - * o nbytes may span a QW boundary - */ -static inline void read_extra_bytes(struct pio_buf *pbuf, - const void *from, unsigned int nbytes) -{ - unsigned long off = (unsigned long)from & 0x7; - unsigned int room, xbytes; - - /* align our pointer */ - from = (void *)((unsigned long)from & ~0x7l); - - /* check count first - don't read anything if count is zero */ - while (nbytes) { - /* find the number of bytes in this u64 */ - room = 8 - off; /* this u64 has room for this many bytes */ - xbytes = min(room, nbytes); - - /* - * shift down to zero lower bytes, shift up to zero upper - * bytes, shift back down to move into place - */ - pbuf->carry.val64 |= (((*(u64 *)from) - >> mshift(off)) - << zshift(xbytes)) - >> zshift(xbytes + pbuf->carry_bytes); - off = 0; - pbuf->carry_bytes += xbytes; - nbytes -= xbytes; - from += sizeof(u64); - } -} - -/* - * Zero extra bytes from the end of pbuf->carry. - * - * NOTES: - * o zbytes <= old_bytes - */ -static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes) -{ - unsigned int remaining; - - if (zbytes == 0) /* nothing to do */ - return; - - remaining = pbuf->carry_bytes - zbytes; /* remaining bytes */ - - /* NOTE: zshift only guaranteed to work if remaining != 0 */ - if (remaining) - pbuf->carry.val64 = (pbuf->carry.val64 << zshift(remaining)) - >> zshift(remaining); - else - pbuf->carry.val64 = 0; - pbuf->carry_bytes = remaining; -} - -/* - * Write a quad word using parts of pbuf->carry and the next 8 bytes of src. - * Put the unused part of the next 8 bytes of src into the LSB bytes of - * pbuf->carry with the upper bytes zeroed.. - * - * NOTES: - * o result must keep unused bytes zeroed - * o src must be u64 aligned - */ -static inline void merge_write8( - struct pio_buf *pbuf, - void __iomem *dest, - const void *src) -{ - u64 new, temp; - - new = *(u64 *)src; - temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes)); - writeq(temp, dest); - pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes); -} - -/* - * Write a quad word using all bytes of carry. - */ -static inline void carry8_write8(union mix carry, void __iomem *dest) -{ - writeq(carry.val64, dest); -} - -/* - * Write a quad word using all the valid bytes of carry. If carry - * has zero valid bytes, nothing is written. - * Returns 0 on nothing written, non-zero on quad word written. - */ -static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest) -{ - if (pbuf->carry_bytes) { - /* unused bytes are always kept zeroed, so just write */ - writeq(pbuf->carry.val64, dest); - return 1; - } - - return 0; -} - -#else /* USE_SHIFTS */ -/* - * Handle carry bytes using byte copies. - * - * NOTE: the value the unused portion of carry is left uninitialized. - */ - -/* - * Jump copy - no-loop copy for < 8 bytes. - */ -static inline void jcopy(u8 *dest, const u8 *src, u32 n) -{ - switch (n) { - case 7: - *dest++ = *src++; - case 6: - *dest++ = *src++; - case 5: - *dest++ = *src++; - case 4: - *dest++ = *src++; - case 3: - *dest++ = *src++; - case 2: - *dest++ = *src++; - case 1: - *dest++ = *src++; - } -} - -/* - * Read nbytes from "from" and and place them in the low bytes - * of pbuf->carry. Other bytes are left as-is. Any previous - * value in pbuf->carry is lost. - * - * NOTES: - * o do not read from from if nbytes is zero - * o from may _not_ be u64 aligned. - */ -static inline void read_low_bytes(struct pio_buf *pbuf, const void *from, - unsigned int nbytes) -{ - jcopy(&pbuf->carry.val8[0], from, nbytes); - pbuf->carry_bytes = nbytes; -} - -/* - * Read nbytes bytes from "from" and put them at the end of pbuf->carry. - * It is expected that the extra read does not overfill carry. - * - * NOTES: - * o from may _not_ be u64 aligned - * o nbytes may span a QW boundary - */ -static inline void read_extra_bytes(struct pio_buf *pbuf, - const void *from, unsigned int nbytes) -{ - jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes); - pbuf->carry_bytes += nbytes; -} - -/* - * Zero extra bytes from the end of pbuf->carry. - * - * We do not care about the value of unused bytes in carry, so just - * reduce the byte count. - * - * NOTES: - * o zbytes <= old_bytes - */ -static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes) -{ - pbuf->carry_bytes -= zbytes; -} - -/* - * Write a quad word using parts of pbuf->carry and the next 8 bytes of src. - * Put the unused part of the next 8 bytes of src into the low bytes of - * pbuf->carry. - */ -static inline void merge_write8( - struct pio_buf *pbuf, - void *dest, - const void *src) -{ - u32 remainder = 8 - pbuf->carry_bytes; - - jcopy(&pbuf->carry.val8[pbuf->carry_bytes], src, remainder); - writeq(pbuf->carry.val64, dest); - jcopy(&pbuf->carry.val8[0], src + remainder, pbuf->carry_bytes); -} - -/* - * Write a quad word using all bytes of carry. - */ -static inline void carry8_write8(union mix carry, void *dest) -{ - writeq(carry.val64, dest); -} - -/* - * Write a quad word using all the valid bytes of carry. If carry - * has zero valid bytes, nothing is written. - * Returns 0 on nothing written, non-zero on quad word written. - */ -static inline int carry_write8(struct pio_buf *pbuf, void *dest) -{ - if (pbuf->carry_bytes) { - u64 zero = 0; - - jcopy(&pbuf->carry.val8[pbuf->carry_bytes], (u8 *)&zero, - 8 - pbuf->carry_bytes); - writeq(pbuf->carry.val64, dest); - return 1; - } - - return 0; -} -#endif /* USE_SHIFTS */ - -/* - * Segmented PIO Copy - start - * - * Start a PIO copy. - * - * @pbuf: destination buffer - * @pbc: the PBC for the PIO buffer - * @from: data source, QWORD aligned - * @nbytes: bytes to copy - */ -void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc, - const void *from, size_t nbytes) -{ - void __iomem *dest = pbuf->start + SOP_DISTANCE; - void __iomem *send = dest + PIO_BLOCK_SIZE; - void __iomem *dend; /* 8-byte data end */ - - writeq(pbc, dest); - dest += sizeof(u64); - - /* calculate where the QWORD data ends - in SOP=1 space */ - dend = dest + ((nbytes >> 3) * sizeof(u64)); - - if (dend < send) { - /* - * all QWORD data is within the SOP block, does *not* - * reach the end of the SOP block - */ - - while (dest < dend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - /* - * No boundary checks are needed here: - * 0. We're not on the SOP block boundary - * 1. The possible DWORD dangle will still be within - * the SOP block - * 2. We cannot wrap except on a block boundary. - */ - } else { - /* QWORD data extends _to_ or beyond the SOP block */ - - /* write 8-byte SOP chunk data */ - while (dest < send) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - /* drop out of the SOP range */ - dest -= SOP_DISTANCE; - dend -= SOP_DISTANCE; - - /* - * If the wrap comes before or matches the data end, - * copy until until the wrap, then wrap. - * - * If the data ends at the end of the SOP above and - * the buffer wraps, then pbuf->end == dend == dest - * and nothing will get written, but we will wrap in - * case there is a dangling DWORD. - */ - if (pbuf->end <= dend) { - while (dest < pbuf->end) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - - dest -= pbuf->size; - dend -= pbuf->size; - } - - /* write 8-byte non-SOP, non-wrap chunk data */ - while (dest < dend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - } - /* at this point we have wrapped if we are going to wrap */ - - /* ...but it doesn't matter as we're done writing */ - - /* save dangling bytes, if any */ - read_low_bytes(pbuf, from, nbytes & 0x7); - - pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3); -} - -/* - * Mid copy helper, "mixed case" - source is 64-bit aligned but carry - * bytes are non-zero. - * - * Whole u64s must be written to the chip, so bytes must be manually merged. - * - * @pbuf: destination buffer - * @from: data source, is QWORD aligned. - * @nbytes: bytes to copy - * - * Must handle nbytes < 8. - */ -static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes) -{ - void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); - void __iomem *dend; /* 8-byte data end */ - unsigned long qw_to_write = (pbuf->carry_bytes + nbytes) >> 3; - unsigned long bytes_left = (pbuf->carry_bytes + nbytes) & 0x7; - - /* calculate 8-byte data end */ - dend = dest + (qw_to_write * sizeof(u64)); - - if (pbuf->qw_written < PIO_BLOCK_QWS) { - /* - * Still within SOP block. We don't need to check for - * wrap because we are still in the first block and - * can only wrap on block boundaries. - */ - void __iomem *send; /* SOP end */ - void __iomem *xend; - - /* - * calculate the end of data or end of block, whichever - * comes first - */ - send = pbuf->start + PIO_BLOCK_SIZE; - xend = min(send, dend); - - /* shift up to SOP=1 space */ - dest += SOP_DISTANCE; - xend += SOP_DISTANCE; - - /* write 8-byte chunk data */ - while (dest < xend) { - merge_write8(pbuf, dest, from); - from += sizeof(u64); - dest += sizeof(u64); - } - - /* shift down to SOP=0 space */ - dest -= SOP_DISTANCE; - } - /* - * At this point dest could be (either, both, or neither): - * - at dend - * - at the wrap - */ - - /* - * If the wrap comes before or matches the data end, - * copy until until the wrap, then wrap. - * - * If dest is at the wrap, we will fall into the if, - * not do the loop, when wrap. - * - * If the data ends at the end of the SOP above and - * the buffer wraps, then pbuf->end == dend == dest - * and nothing will get written. - */ - if (pbuf->end <= dend) { - while (dest < pbuf->end) { - merge_write8(pbuf, dest, from); - from += sizeof(u64); - dest += sizeof(u64); - } - - dest -= pbuf->size; - dend -= pbuf->size; - } - - /* write 8-byte non-SOP, non-wrap chunk data */ - while (dest < dend) { - merge_write8(pbuf, dest, from); - from += sizeof(u64); - dest += sizeof(u64); - } - - /* adjust carry */ - if (pbuf->carry_bytes < bytes_left) { - /* need to read more */ - read_extra_bytes(pbuf, from, bytes_left - pbuf->carry_bytes); - } else { - /* remove invalid bytes */ - zero_extra_bytes(pbuf, pbuf->carry_bytes - bytes_left); - } - - pbuf->qw_written += qw_to_write; -} - -/* - * Mid copy helper, "straight case" - source pointer is 64-bit aligned - * with no carry bytes. - * - * @pbuf: destination buffer - * @from: data source, is QWORD aligned - * @nbytes: bytes to copy - * - * Must handle nbytes < 8. - */ -static void mid_copy_straight(struct pio_buf *pbuf, - const void *from, size_t nbytes) -{ - void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); - void __iomem *dend; /* 8-byte data end */ - - /* calculate 8-byte data end */ - dend = dest + ((nbytes >> 3) * sizeof(u64)); - - if (pbuf->qw_written < PIO_BLOCK_QWS) { - /* - * Still within SOP block. We don't need to check for - * wrap because we are still in the first block and - * can only wrap on block boundaries. - */ - void __iomem *send; /* SOP end */ - void __iomem *xend; - - /* - * calculate the end of data or end of block, whichever - * comes first - */ - send = pbuf->start + PIO_BLOCK_SIZE; - xend = min(send, dend); - - /* shift up to SOP=1 space */ - dest += SOP_DISTANCE; - xend += SOP_DISTANCE; - - /* write 8-byte chunk data */ - while (dest < xend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - - /* shift down to SOP=0 space */ - dest -= SOP_DISTANCE; - } - /* - * At this point dest could be (either, both, or neither): - * - at dend - * - at the wrap - */ - - /* - * If the wrap comes before or matches the data end, - * copy until until the wrap, then wrap. - * - * If dest is at the wrap, we will fall into the if, - * not do the loop, when wrap. - * - * If the data ends at the end of the SOP above and - * the buffer wraps, then pbuf->end == dend == dest - * and nothing will get written. - */ - if (pbuf->end <= dend) { - while (dest < pbuf->end) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - - dest -= pbuf->size; - dend -= pbuf->size; - } - - /* write 8-byte non-SOP, non-wrap chunk data */ - while (dest < dend) { - writeq(*(u64 *)from, dest); - from += sizeof(u64); - dest += sizeof(u64); - } - - /* we know carry_bytes was zero on entry to this routine */ - read_low_bytes(pbuf, from, nbytes & 0x7); - - pbuf->qw_written += nbytes >> 3; -} - -/* - * Segmented PIO Copy - middle - * - * Must handle any aligned tail and any aligned source with any byte count. - * - * @pbuf: a number of blocks allocated within a PIO send context - * @from: data source - * @nbytes: number of bytes to copy - */ -void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes) -{ - unsigned long from_align = (unsigned long)from & 0x7; - - if (pbuf->carry_bytes + nbytes < 8) { - /* not enough bytes to fill a QW */ - read_extra_bytes(pbuf, from, nbytes); - return; - } - - if (from_align) { - /* misaligned source pointer - align it */ - unsigned long to_align; - - /* bytes to read to align "from" */ - to_align = 8 - from_align; - - /* - * In the advance-to-alignment logic below, we do not need - * to check if we are using more than nbytes. This is because - * if we are here, we already know that carry+nbytes will - * fill at least one QW. - */ - if (pbuf->carry_bytes + to_align < 8) { - /* not enough align bytes to fill a QW */ - read_extra_bytes(pbuf, from, to_align); - from += to_align; - nbytes -= to_align; - } else { - /* bytes to fill carry */ - unsigned long to_fill = 8 - pbuf->carry_bytes; - /* bytes left over to be read */ - unsigned long extra = to_align - to_fill; - void __iomem *dest; - - /* fill carry... */ - read_extra_bytes(pbuf, from, to_fill); - from += to_fill; - nbytes -= to_fill; - - /* ...now write carry */ - dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); - - /* - * The two checks immediately below cannot both be - * true, hence the else. If we have wrapped, we - * cannot still be within the first block. - * Conversely, if we are still in the first block, we - * cannot have wrapped. We do the wrap check first - * as that is more likely. - */ - /* adjust if we've wrapped */ - if (dest >= pbuf->end) - dest -= pbuf->size; - /* jump to SOP range if within the first block */ - else if (pbuf->qw_written < PIO_BLOCK_QWS) - dest += SOP_DISTANCE; - - carry8_write8(pbuf->carry, dest); - pbuf->qw_written++; - - /* read any extra bytes to do final alignment */ - /* this will overwrite anything in pbuf->carry */ - read_low_bytes(pbuf, from, extra); - from += extra; - nbytes -= extra; - } - - /* at this point, from is QW aligned */ - } - - if (pbuf->carry_bytes) - mid_copy_mix(pbuf, from, nbytes); - else - mid_copy_straight(pbuf, from, nbytes); -} - -/* - * Segmented PIO Copy - end - * - * Write any remainder (in pbuf->carry) and finish writing the whole block. - * - * @pbuf: a number of blocks allocated within a PIO send context - */ -void seg_pio_copy_end(struct pio_buf *pbuf) -{ - void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); - - /* - * The two checks immediately below cannot both be true, hence the - * else. If we have wrapped, we cannot still be within the first - * block. Conversely, if we are still in the first block, we - * cannot have wrapped. We do the wrap check first as that is - * more likely. - */ - /* adjust if we have wrapped */ - if (dest >= pbuf->end) - dest -= pbuf->size; - /* jump to the SOP range if within the first block */ - else if (pbuf->qw_written < PIO_BLOCK_QWS) - dest += SOP_DISTANCE; - - /* write final bytes, if any */ - if (carry_write8(pbuf, dest)) { - dest += sizeof(u64); - /* - * NOTE: We do not need to recalculate whether dest needs - * SOP_DISTANCE or not. - * - * If we are in the first block and the dangle write - * keeps us in the same block, dest will need - * to retain SOP_DISTANCE in the loop below. - * - * If we are in the first block and the dangle write pushes - * us to the next block, then loop below will not run - * and dest is not used. Hence we do not need to update - * it. - * - * If we are past the first block, then SOP_DISTANCE - * was never added, so there is nothing to do. - */ - } - - /* fill in rest of block */ - while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) { - writeq(0, dest); - dest += sizeof(u64); - } - - /* finished with this buffer */ - this_cpu_dec(*pbuf->sc->buffers_allocated); - preempt_enable(); -} |