diff options
Diffstat (limited to 'drivers/staging/rdma/hfi1/pio_copy.c')
| -rw-r--r-- | drivers/staging/rdma/hfi1/pio_copy.c | 858 |
1 files changed, 858 insertions, 0 deletions
diff --git a/drivers/staging/rdma/hfi1/pio_copy.c b/drivers/staging/rdma/hfi1/pio_copy.c new file mode 100644 index 000000000000..8972bbc02038 --- /dev/null +++ b/drivers/staging/rdma/hfi1/pio_copy.c @@ -0,0 +1,858 @@ +/* + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2015 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * BSD LICENSE + * + * Copyright(c) 2015 Intel Corporation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * - Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "hfi.h" + +/* additive distance between non-SOP and SOP space */ +#define SOP_DISTANCE (TXE_PIO_SIZE / 2) +#define PIO_BLOCK_MASK (PIO_BLOCK_SIZE-1) +/* number of QUADWORDs in a block */ +#define PIO_BLOCK_QWS (PIO_BLOCK_SIZE/sizeof(u64)) + +/** + * pio_copy - copy data block to MMIO space + * @pbuf: a number of blocks allocated within a PIO send context + * @pbc: PBC to send + * @from: source, must be 8 byte aligned + * @count: number of DWORD (32-bit) quantities to copy from source + * + * Copy data from source to PIO Send Buffer memory, 8 bytes at a time. + * Must always write full BLOCK_SIZE bytes blocks. The first block must + * be written to the corresponding SOP=1 address. + * + * Known: + * o pbuf->start always starts on a block boundary + * o pbuf can wrap only at a block boundary + */ +void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc, + const void *from, size_t count) +{ + void __iomem *dest = pbuf->start + SOP_DISTANCE; + void __iomem *send = dest + PIO_BLOCK_SIZE; + void __iomem *dend; /* 8-byte data end */ + + /* write the PBC */ + writeq(pbc, dest); + dest += sizeof(u64); + + /* calculate where the QWORD data ends - in SOP=1 space */ + dend = dest + ((count>>1) * sizeof(u64)); + + if (dend < send) { + /* all QWORD data is within the SOP block, does *not* + reach the end of the SOP block */ + + while (dest < dend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + /* + * No boundary checks are needed here: + * 0. We're not on the SOP block boundary + * 1. The possible DWORD dangle will still be within + * the SOP block + * 2. We cannot wrap except on a block boundary. + */ + } else { + /* QWORD data extends _to_ or beyond the SOP block */ + + /* write 8-byte SOP chunk data */ + while (dest < send) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + /* drop out of the SOP range */ + dest -= SOP_DISTANCE; + dend -= SOP_DISTANCE; + + /* + * If the wrap comes before or matches the data end, + * copy until until the wrap, then wrap. + * + * If the data ends at the end of the SOP above and + * the buffer wraps, then pbuf->end == dend == dest + * and nothing will get written, but we will wrap in + * case there is a dangling DWORD. + */ + if (pbuf->end <= dend) { + while (dest < pbuf->end) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + + dest -= pbuf->size; + dend -= pbuf->size; + } + + /* write 8-byte non-SOP, non-wrap chunk data */ + while (dest < dend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + } + /* at this point we have wrapped if we are going to wrap */ + + /* write dangling u32, if any */ + if (count & 1) { + union mix val; + + val.val64 = 0; + val.val32[0] = *(u32 *)from; + writeq(val.val64, dest); + dest += sizeof(u64); + } + /* fill in rest of block, no need to check pbuf->end + as we only wrap on a block boundary */ + while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) { + writeq(0, dest); + dest += sizeof(u64); + } + + /* finished with this buffer */ + atomic_dec(&pbuf->sc->buffers_allocated); +} + +/* USE_SHIFTS is faster in user-space tests on a Xeon X5570 @ 2.93GHz */ +#define USE_SHIFTS 1 +#ifdef USE_SHIFTS +/* + * Handle carry bytes using shifts and masks. + * + * NOTE: the value the unused portion of carry is expected to always be zero. + */ + +/* + * "zero" shift - bit shift used to zero out upper bytes. Input is + * the count of LSB bytes to preserve. + */ +#define zshift(x) (8 * (8-(x))) + +/* + * "merge" shift - bit shift used to merge with carry bytes. Input is + * the LSB byte count to move beyond. + */ +#define mshift(x) (8 * (x)) + +/* + * Read nbytes bytes from "from" and return them in the LSB bytes + * of pbuf->carry. Other bytes are zeroed. Any previous value + * pbuf->carry is lost. + * + * NOTES: + * o do not read from from if nbytes is zero + * o from may _not_ be u64 aligned + * o nbytes must not span a QW boundary + */ +static inline void read_low_bytes(struct pio_buf *pbuf, const void *from, + unsigned int nbytes) +{ + unsigned long off; + + if (nbytes == 0) { + pbuf->carry.val64 = 0; + } else { + /* align our pointer */ + off = (unsigned long)from & 0x7; + from = (void *)((unsigned long)from & ~0x7l); + pbuf->carry.val64 = ((*(u64 *)from) + << zshift(nbytes + off))/* zero upper bytes */ + >> zshift(nbytes); /* place at bottom */ + } + pbuf->carry_bytes = nbytes; +} + +/* + * Read nbytes bytes from "from" and put them at the next significant bytes + * of pbuf->carry. Unused bytes are zeroed. It is expected that the extra + * read does not overfill carry. + * + * NOTES: + * o from may _not_ be u64 aligned + * o nbytes may span a QW boundary + */ +static inline void read_extra_bytes(struct pio_buf *pbuf, + const void *from, unsigned int nbytes) +{ + unsigned long off = (unsigned long)from & 0x7; + unsigned int room, xbytes; + + /* align our pointer */ + from = (void *)((unsigned long)from & ~0x7l); + + /* check count first - don't read anything if count is zero */ + while (nbytes) { + /* find the number of bytes in this u64 */ + room = 8 - off; /* this u64 has room for this many bytes */ + xbytes = nbytes > room ? room : nbytes; + + /* + * shift down to zero lower bytes, shift up to zero upper + * bytes, shift back down to move into place + */ + pbuf->carry.val64 |= (((*(u64 *)from) + >> mshift(off)) + << zshift(xbytes)) + >> zshift(xbytes+pbuf->carry_bytes); + off = 0; + pbuf->carry_bytes += xbytes; + nbytes -= xbytes; + from += sizeof(u64); + } +} + +/* + * Zero extra bytes from the end of pbuf->carry. + * + * NOTES: + * o zbytes <= old_bytes + */ +static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes) +{ + unsigned int remaining; + + if (zbytes == 0) /* nothing to do */ + return; + + remaining = pbuf->carry_bytes - zbytes; /* remaining bytes */ + + /* NOTE: zshift only guaranteed to work if remaining != 0 */ + if (remaining) + pbuf->carry.val64 = (pbuf->carry.val64 << zshift(remaining)) + >> zshift(remaining); + else + pbuf->carry.val64 = 0; + pbuf->carry_bytes = remaining; +} + +/* + * Write a quad word using parts of pbuf->carry and the next 8 bytes of src. + * Put the unused part of the next 8 bytes of src into the LSB bytes of + * pbuf->carry with the upper bytes zeroed.. + * + * NOTES: + * o result must keep unused bytes zeroed + * o src must be u64 aligned + */ +static inline void merge_write8( + struct pio_buf *pbuf, + void __iomem *dest, + const void *src) +{ + u64 new, temp; + + new = *(u64 *)src; + temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes)); + writeq(temp, dest); + pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes); +} + +/* + * Write a quad word using all bytes of carry. + */ +static inline void carry8_write8(union mix carry, void __iomem *dest) +{ + writeq(carry.val64, dest); +} + +/* + * Write a quad word using all the valid bytes of carry. If carry + * has zero valid bytes, nothing is written. + * Returns 0 on nothing written, non-zero on quad word written. + */ +static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest) +{ + if (pbuf->carry_bytes) { + /* unused bytes are always kept zeroed, so just write */ + writeq(pbuf->carry.val64, dest); + return 1; + } + + return 0; +} + +#else /* USE_SHIFTS */ +/* + * Handle carry bytes using byte copies. + * + * NOTE: the value the unused portion of carry is left uninitialized. + */ + +/* + * Jump copy - no-loop copy for < 8 bytes. + */ +static inline void jcopy(u8 *dest, const u8 *src, u32 n) +{ + switch (n) { + case 7: + *dest++ = *src++; + case 6: + *dest++ = *src++; + case 5: + *dest++ = *src++; + case 4: + *dest++ = *src++; + case 3: + *dest++ = *src++; + case 2: + *dest++ = *src++; + case 1: + *dest++ = *src++; + } +} + +/* + * Read nbytes from "from" and and place them in the low bytes + * of pbuf->carry. Other bytes are left as-is. Any previous + * value in pbuf->carry is lost. + * + * NOTES: + * o do not read from from if nbytes is zero + * o from may _not_ be u64 aligned. + */ +static inline void read_low_bytes(struct pio_buf *pbuf, const void *from, + unsigned int nbytes) +{ + jcopy(&pbuf->carry.val8[0], from, nbytes); + pbuf->carry_bytes = nbytes; +} + +/* + * Read nbytes bytes from "from" and put them at the end of pbuf->carry. + * It is expected that the extra read does not overfill carry. + * + * NOTES: + * o from may _not_ be u64 aligned + * o nbytes may span a QW boundary + */ +static inline void read_extra_bytes(struct pio_buf *pbuf, + const void *from, unsigned int nbytes) +{ + jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes); + pbuf->carry_bytes += nbytes; +} + +/* + * Zero extra bytes from the end of pbuf->carry. + * + * We do not care about the value of unused bytes in carry, so just + * reduce the byte count. + * + * NOTES: + * o zbytes <= old_bytes + */ +static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes) +{ + pbuf->carry_bytes -= zbytes; +} + +/* + * Write a quad word using parts of pbuf->carry and the next 8 bytes of src. + * Put the unused part of the next 8 bytes of src into the low bytes of + * pbuf->carry. + */ +static inline void merge_write8( + struct pio_buf *pbuf, + void *dest, + const void *src) +{ + u32 remainder = 8 - pbuf->carry_bytes; + + jcopy(&pbuf->carry.val8[pbuf->carry_bytes], src, remainder); + writeq(pbuf->carry.val64, dest); + jcopy(&pbuf->carry.val8[0], src+remainder, pbuf->carry_bytes); +} + +/* + * Write a quad word using all bytes of carry. + */ +static inline void carry8_write8(union mix carry, void *dest) +{ + writeq(carry.val64, dest); +} + +/* + * Write a quad word using all the valid bytes of carry. If carry + * has zero valid bytes, nothing is written. + * Returns 0 on nothing written, non-zero on quad word written. + */ +static inline int carry_write8(struct pio_buf *pbuf, void *dest) +{ + if (pbuf->carry_bytes) { + u64 zero = 0; + + jcopy(&pbuf->carry.val8[pbuf->carry_bytes], (u8 *)&zero, + 8 - pbuf->carry_bytes); + writeq(pbuf->carry.val64, dest); + return 1; + } + + return 0; +} +#endif /* USE_SHIFTS */ + +/* + * Segmented PIO Copy - start + * + * Start a PIO copy. + * + * @pbuf: destination buffer + * @pbc: the PBC for the PIO buffer + * @from: data source, QWORD aligned + * @nbytes: bytes to copy + */ +void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc, + const void *from, size_t nbytes) +{ + void __iomem *dest = pbuf->start + SOP_DISTANCE; + void __iomem *send = dest + PIO_BLOCK_SIZE; + void __iomem *dend; /* 8-byte data end */ + + writeq(pbc, dest); + dest += sizeof(u64); + + /* calculate where the QWORD data ends - in SOP=1 space */ + dend = dest + ((nbytes>>3) * sizeof(u64)); + + if (dend < send) { + /* all QWORD data is within the SOP block, does *not* + reach the end of the SOP block */ + + while (dest < dend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + /* + * No boundary checks are needed here: + * 0. We're not on the SOP block boundary + * 1. The possible DWORD dangle will still be within + * the SOP block + * 2. We cannot wrap except on a block boundary. + */ + } else { + /* QWORD data extends _to_ or beyond the SOP block */ + + /* write 8-byte SOP chunk data */ + while (dest < send) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + /* drop out of the SOP range */ + dest -= SOP_DISTANCE; + dend -= SOP_DISTANCE; + + /* + * If the wrap comes before or matches the data end, + * copy until until the wrap, then wrap. + * + * If the data ends at the end of the SOP above and + * the buffer wraps, then pbuf->end == dend == dest + * and nothing will get written, but we will wrap in + * case there is a dangling DWORD. + */ + if (pbuf->end <= dend) { + while (dest < pbuf->end) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + + dest -= pbuf->size; + dend -= pbuf->size; + } + + /* write 8-byte non-SOP, non-wrap chunk data */ + while (dest < dend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + } + /* at this point we have wrapped if we are going to wrap */ + + /* ...but it doesn't matter as we're done writing */ + + /* save dangling bytes, if any */ + read_low_bytes(pbuf, from, nbytes & 0x7); + + pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3); +} + +/* + * Mid copy helper, "mixed case" - source is 64-bit aligned but carry + * bytes are non-zero. + * + * Whole u64s must be written to the chip, so bytes must be manually merged. + * + * @pbuf: destination buffer + * @from: data source, is QWORD aligned. + * @nbytes: bytes to copy + * + * Must handle nbytes < 8. + */ +static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes) +{ + void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); + void __iomem *dend; /* 8-byte data end */ + unsigned long qw_to_write = (pbuf->carry_bytes + nbytes) >> 3; + unsigned long bytes_left = (pbuf->carry_bytes + nbytes) & 0x7; + + /* calculate 8-byte data end */ + dend = dest + (qw_to_write * sizeof(u64)); + + if (pbuf->qw_written < PIO_BLOCK_QWS) { + /* + * Still within SOP block. We don't need to check for + * wrap because we are still in the first block and + * can only wrap on block boundaries. + */ + void __iomem *send; /* SOP end */ + void __iomem *xend; + + /* calculate the end of data or end of block, whichever + comes first */ + send = pbuf->start + PIO_BLOCK_SIZE; + xend = send < dend ? send : dend; + + /* shift up to SOP=1 space */ + dest += SOP_DISTANCE; + xend += SOP_DISTANCE; + + /* write 8-byte chunk data */ + while (dest < xend) { + merge_write8(pbuf, dest, from); + from += sizeof(u64); + dest += sizeof(u64); + } + + /* shift down to SOP=0 space */ + dest -= SOP_DISTANCE; + } + /* + * At this point dest could be (either, both, or neither): + * - at dend + * - at the wrap + */ + + /* + * If the wrap comes before or matches the data end, + * copy until until the wrap, then wrap. + * + * If dest is at the wrap, we will fall into the if, + * not do the loop, when wrap. + * + * If the data ends at the end of the SOP above and + * the buffer wraps, then pbuf->end == dend == dest + * and nothing will get written. + */ + if (pbuf->end <= dend) { + while (dest < pbuf->end) { + merge_write8(pbuf, dest, from); + from += sizeof(u64); + dest += sizeof(u64); + } + + dest -= pbuf->size; + dend -= pbuf->size; + } + + /* write 8-byte non-SOP, non-wrap chunk data */ + while (dest < dend) { + merge_write8(pbuf, dest, from); + from += sizeof(u64); + dest += sizeof(u64); + } + + /* adjust carry */ + if (pbuf->carry_bytes < bytes_left) { + /* need to read more */ + read_extra_bytes(pbuf, from, bytes_left - pbuf->carry_bytes); + } else { + /* remove invalid bytes */ + zero_extra_bytes(pbuf, pbuf->carry_bytes - bytes_left); + } + + pbuf->qw_written += qw_to_write; +} + +/* + * Mid copy helper, "straight case" - source pointer is 64-bit aligned + * with no carry bytes. + * + * @pbuf: destination buffer + * @from: data source, is QWORD aligned + * @nbytes: bytes to copy + * + * Must handle nbytes < 8. + */ +static void mid_copy_straight(struct pio_buf *pbuf, + const void *from, size_t nbytes) +{ + void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); + void __iomem *dend; /* 8-byte data end */ + + /* calculate 8-byte data end */ + dend = dest + ((nbytes>>3) * sizeof(u64)); + + if (pbuf->qw_written < PIO_BLOCK_QWS) { + /* + * Still within SOP block. We don't need to check for + * wrap because we are still in the first block and + * can only wrap on block boundaries. + */ + void __iomem *send; /* SOP end */ + void __iomem *xend; + + /* calculate the end of data or end of block, whichever + comes first */ + send = pbuf->start + PIO_BLOCK_SIZE; + xend = send < dend ? send : dend; + + /* shift up to SOP=1 space */ + dest += SOP_DISTANCE; + xend += SOP_DISTANCE; + + /* write 8-byte chunk data */ + while (dest < xend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + + /* shift down to SOP=0 space */ + dest -= SOP_DISTANCE; + } + /* + * At this point dest could be (either, both, or neither): + * - at dend + * - at the wrap + */ + + /* + * If the wrap comes before or matches the data end, + * copy until until the wrap, then wrap. + * + * If dest is at the wrap, we will fall into the if, + * not do the loop, when wrap. + * + * If the data ends at the end of the SOP above and + * the buffer wraps, then pbuf->end == dend == dest + * and nothing will get written. + */ + if (pbuf->end <= dend) { + while (dest < pbuf->end) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + + dest -= pbuf->size; + dend -= pbuf->size; + } + + /* write 8-byte non-SOP, non-wrap chunk data */ + while (dest < dend) { + writeq(*(u64 *)from, dest); + from += sizeof(u64); + dest += sizeof(u64); + } + + /* we know carry_bytes was zero on entry to this routine */ + read_low_bytes(pbuf, from, nbytes & 0x7); + + pbuf->qw_written += nbytes>>3; +} + +/* + * Segmented PIO Copy - middle + * + * Must handle any aligned tail and any aligned source with any byte count. + * + * @pbuf: a number of blocks allocated within a PIO send context + * @from: data source + * @nbytes: number of bytes to copy + */ +void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes) +{ + unsigned long from_align = (unsigned long)from & 0x7; + + if (pbuf->carry_bytes + nbytes < 8) { + /* not enough bytes to fill a QW */ + read_extra_bytes(pbuf, from, nbytes); + return; + } + + if (from_align) { + /* misaligned source pointer - align it */ + unsigned long to_align; + + /* bytes to read to align "from" */ + to_align = 8 - from_align; + + /* + * In the advance-to-alignment logic below, we do not need + * to check if we are using more than nbytes. This is because + * if we are here, we already know that carry+nbytes will + * fill at least one QW. + */ + if (pbuf->carry_bytes + to_align < 8) { + /* not enough align bytes to fill a QW */ + read_extra_bytes(pbuf, from, to_align); + from += to_align; + nbytes -= to_align; + } else { + /* bytes to fill carry */ + unsigned long to_fill = 8 - pbuf->carry_bytes; + /* bytes left over to be read */ + unsigned long extra = to_align - to_fill; + void __iomem *dest; + + /* fill carry... */ + read_extra_bytes(pbuf, from, to_fill); + from += to_fill; + nbytes -= to_fill; + + /* ...now write carry */ + dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); + + /* + * The two checks immediately below cannot both be + * true, hence the else. If we have wrapped, we + * cannot still be within the first block. + * Conversely, if we are still in the first block, we + * cannot have wrapped. We do the wrap check first + * as that is more likely. + */ + /* adjust if we've wrapped */ + if (dest >= pbuf->end) + dest -= pbuf->size; + /* jump to SOP range if within the first block */ + else if (pbuf->qw_written < PIO_BLOCK_QWS) + dest += SOP_DISTANCE; + + carry8_write8(pbuf->carry, dest); + pbuf->qw_written++; + + /* read any extra bytes to do final alignment */ + /* this will overwrite anything in pbuf->carry */ + read_low_bytes(pbuf, from, extra); + from += extra; + nbytes -= extra; + } + + /* at this point, from is QW aligned */ + } + + if (pbuf->carry_bytes) + mid_copy_mix(pbuf, from, nbytes); + else + mid_copy_straight(pbuf, from, nbytes); +} + +/* + * Segmented PIO Copy - end + * + * Write any remainder (in pbuf->carry) and finish writing the whole block. + * + * @pbuf: a number of blocks allocated within a PIO send context + */ +void seg_pio_copy_end(struct pio_buf *pbuf) +{ + void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64)); + + /* + * The two checks immediately below cannot both be true, hence the + * else. If we have wrapped, we cannot still be within the first + * block. Conversely, if we are still in the first block, we + * cannot have wrapped. We do the wrap check first as that is + * more likely. + */ + /* adjust if we have wrapped */ + if (dest >= pbuf->end) + dest -= pbuf->size; + /* jump to the SOP range if within the first block */ + else if (pbuf->qw_written < PIO_BLOCK_QWS) + dest += SOP_DISTANCE; + + /* write final bytes, if any */ + if (carry_write8(pbuf, dest)) { + dest += sizeof(u64); + /* + * NOTE: We do not need to recalculate whether dest needs + * SOP_DISTANCE or not. + * + * If we are in the first block and the dangle write + * keeps us in the same block, dest will need + * to retain SOP_DISTANCE in the loop below. + * + * If we are in the first block and the dangle write pushes + * us to the next block, then loop below will not run + * and dest is not used. Hence we do not need to update + * it. + * + * If we are past the first block, then SOP_DISTANCE + * was never added, so there is nothing to do. + */ + } + + /* fill in rest of block */ + while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) { + writeq(0, dest); + dest += sizeof(u64); + } + + /* finished with this buffer */ + atomic_dec(&pbuf->sc->buffers_allocated); +} |