// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ /* * nfp6000_pcie.c * Authors: Jakub Kicinski * Jason McMullan * Rolf Neugebauer * * Multiplexes the NFP BARs between NFP internal resources and * implements the PCIe specific interface for generic CPP bus access. * * The BARs are managed with refcounts and are allocated/acquired * using target, token and offset/size matching. The generic CPP bus * abstraction builds upon this BAR interface. */ #include #include #include #include #include #include #include #include #include #include #include #include "nfp_cpp.h" #include "nfp6000/nfp6000.h" #include "nfp6000_pcie.h" #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0) #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \ (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30) #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3) #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28) #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3) #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0) #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff) #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \ (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24) #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f) #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14) #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff) #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0) #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff) #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \ (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18) #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f) #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10) #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff) #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0) #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff) #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16) #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f) #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0) #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff) #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27) #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3) #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3 #define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29) #define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7) #define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0 #define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1 #define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2 #define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7 #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23) #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf) #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21) #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3) #define NFP_PCIE_EM 0x020000 #define NFP_PCIE_SRAM 0x000000 /* Minimal size of the PCIe cfg memory we depend on being mapped, * queue controller and DMA controller don't have to be covered. */ #define NFP_PCI_MIN_MAP_SIZE 0x080000 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize) #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize) #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2)) #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) #define NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \ (0x400 + ((bar) * 8 + (slot)) * 4) #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \ (((bar) * 8 + (slot)) * 4) /* The number of explicit BARs to reserve. * Minimum is 0, maximum is 4 on the NFP6000. * The NFP3800 can have only one per PF. */ #define NFP_PCIE_EXPLICIT_BARS 2 struct nfp6000_pcie; struct nfp6000_area_priv; /** * struct nfp_bar - describes BAR configuration and usage * @nfp: backlink to owner * @barcfg: cached contents of BAR config CSR * @base: the BAR's base CPP offset * @mask: mask for the BAR aperture (read only) * @bitsize: bitsize of BAR aperture (read only) * @index: index of the BAR * @refcnt: number of current users * @iomem: mapped IO memory * @resource: iomem resource window */ struct nfp_bar { struct nfp6000_pcie *nfp; u32 barcfg; u64 base; /* CPP address base */ u64 mask; /* Bit mask of the bar */ u32 bitsize; /* Bit size of the bar */ int index; atomic_t refcnt; void __iomem *iomem; struct resource *resource; }; #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8) struct nfp6000_pcie { struct pci_dev *pdev; struct device *dev; /* PCI BAR management */ spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */ int bars; struct nfp_bar bar[NFP_PCI_BAR_MAX]; wait_queue_head_t bar_waiters; /* Reserved BAR access */ struct { void __iomem *csr; void __iomem *em; void __iomem *expl[4]; } iomem; /* Explicit IO access */ struct { struct mutex mutex; /* Lock access to this explicit group */ u8 master_id; u8 signal_ref; void __iomem *data; struct { void __iomem *addr; int bitsize; int free[4]; } group[4]; } expl; }; static u32 nfp_bar_maptype(struct nfp_bar *bar) { return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); } static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar) { return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8; } static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar) { return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2) + nfp_bar_resource_len(bar) * (bar->index & 7); } #define TARGET_WIDTH_32 4 #define TARGET_WIDTH_64 8 static int compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar, u32 *bar_config, u64 *bar_base, int tgt, int act, int tok, u64 offset, size_t size, int width) { int bitsize; u32 newcfg; if (tgt >= NFP_CPP_NUM_TARGETS) return -EINVAL; switch (width) { case 8: newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT); break; case 4: newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); break; case 0: newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE); break; default: return -EINVAL; } if (act != NFP_CPP_ACTION_RW && act != 0) { /* Fixed CPP mapping with specific action */ u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1); newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED); newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act); newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); if ((offset & mask) != ((offset + size - 1) & mask)) return -EINVAL; offset &= mask; bitsize = 40 - 16; } else { u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1); /* Bulk mapping */ newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_BULK); newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); if ((offset & mask) != ((offset + size - 1) & mask)) return -EINVAL; offset &= mask; bitsize = 40 - 21; } if (bar->bitsize < bitsize) return -EINVAL; newcfg |= offset >> bitsize; if (bar_base) *bar_base = offset; if (bar_config) *bar_config = newcfg; return 0; } static int nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg) { int base, slot; int xbar; base = bar->index >> 3; slot = bar->index & 7; if (nfp->iomem.csr) { xbar = NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(base, slot); writel(newcfg, nfp->iomem.csr + xbar); /* Readback to ensure BAR is flushed */ readl(nfp->iomem.csr + xbar); } else { xbar = NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(base, slot); pci_write_config_dword(nfp->pdev, xbar, newcfg); } bar->barcfg = newcfg; return 0; } static int reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar, int tgt, int act, int tok, u64 offset, size_t size, int width) { u64 newbase; u32 newcfg; int err; err = compute_bar(nfp, bar, &newcfg, &newbase, tgt, act, tok, offset, size, width); if (err) return err; bar->base = newbase; return nfp6000_bar_write(nfp, bar, newcfg); } /* Check if BAR can be used with the given parameters. */ static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) { int bartgt, baract, bartok; int barwidth; u32 maptype; maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg); bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg); baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg); barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg); switch (barwidth) { case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT: barwidth = 4; break; case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT: barwidth = 8; break; case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE: barwidth = 0; break; default: barwidth = -1; break; } switch (maptype) { case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET: bartok = -1; /* FALLTHROUGH */ case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK: baract = NFP_CPP_ACTION_RW; if (act == 0) act = NFP_CPP_ACTION_RW; /* FALLTHROUGH */ case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED: break; default: /* We don't match explicit bars through the area interface */ return 0; } /* Make sure to match up the width */ if (barwidth != width) return 0; if ((bartgt < 0 || bartgt == tgt) && (bartok < 0 || bartok == tok) && (baract == act) && bar->base <= offset && (bar->base + (1 << bar->bitsize)) >= (offset + size)) return 1; /* No match */ return 0; } static int find_matching_bar(struct nfp6000_pcie *nfp, u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) { int n; for (n = 0; n < nfp->bars; n++) { struct nfp_bar *bar = &nfp->bar[n]; if (matching_bar(bar, tgt, act, tok, offset, size, width)) return n; } return -1; } /* Return EAGAIN if no resource is available */ static int find_unused_bar_noblock(const struct nfp6000_pcie *nfp, int tgt, int act, int tok, u64 offset, size_t size, int width) { int n, busy = 0; for (n = 0; n < nfp->bars; n++) { const struct nfp_bar *bar = &nfp->bar[n]; int err; if (!bar->bitsize) continue; /* Just check to see if we can make it fit... */ err = compute_bar(nfp, bar, NULL, NULL, tgt, act, tok, offset, size, width); if (err) continue; if (!atomic_read(&bar->refcnt)) return n; busy++; } if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n", tgt, act, tok, offset, size, width)) return -EINVAL; return -EAGAIN; } static int find_unused_bar_and_lock(struct nfp6000_pcie *nfp, int tgt, int act, int tok, u64 offset, size_t size, int width) { unsigned long flags; int n; spin_lock_irqsave(&nfp->bar_lock, flags); n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width); if (n < 0) spin_unlock_irqrestore(&nfp->bar_lock, flags); else __release(&nfp->bar_lock); return n; } static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar) { atomic_inc(&bar->refcnt); } static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar) { if (atomic_dec_and_test(&bar->refcnt)) wake_up_interruptible(&nfp->bar_waiters); } static int nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum, u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) { return wait_event_interruptible(nfp->bar_waiters, (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok, offset, size, width)) != -EAGAIN); } static int nfp_alloc_bar(struct nfp6000_pcie *nfp, u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width, int nonblocking) { unsigned long irqflags; int barnum, retval; if (size > (1 << 24)) return -EINVAL; spin_lock_irqsave(&nfp->bar_lock, irqflags); barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width); if (barnum >= 0) { /* Found a perfect match. */ nfp_bar_get(nfp, &nfp->bar[barnum]); spin_unlock_irqrestore(&nfp->bar_lock, irqflags); return barnum; } barnum = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width); if (barnum < 0) { if (nonblocking) goto err_nobar; /* Wait until a BAR becomes available. The * find_unused_bar function will reclaim the bar_lock * if a free BAR is found. */ spin_unlock_irqrestore(&nfp->bar_lock, irqflags); retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok, offset, size, width); if (retval) return retval; __acquire(&nfp->bar_lock); } nfp_bar_get(nfp, &nfp->bar[barnum]); retval = reconfigure_bar(nfp, &nfp->bar[barnum], tgt, act, tok, offset, size, width); if (retval < 0) { nfp_bar_put(nfp, &nfp->bar[barnum]); barnum = retval; } err_nobar: spin_unlock_irqrestore(&nfp->bar_lock, irqflags); return barnum; } static void disable_bars(struct nfp6000_pcie *nfp); static int bar_cmp(const void *aptr, const void *bptr) { const struct nfp_bar *a = aptr, *b = bptr; if (a->bitsize == b->bitsize) return a->index - b->index; else return a->bitsize - b->bitsize; } /* Map all PCI bars and fetch the actual BAR configurations from the * board. We assume that the BAR with the PCIe config block is * already mapped. * * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM) * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA) * BAR0.2: -- * BAR0.3: -- * BAR0.4: Reserved for Explicit 0.0-0.3 access * BAR0.5: Reserved for Explicit 1.0-1.3 access * BAR0.6: Reserved for Explicit 2.0-2.3 access * BAR0.7: Reserved for Explicit 3.0-3.3 access * * BAR1.0-BAR1.7: -- * BAR2.0-BAR2.7: -- */ static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) { const u32 barcfg_msix_general = NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT; const u32 barcfg_msix_xpb = NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT | NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress( NFP_CPP_TARGET_ISLAND_XPB); const u32 barcfg_explicit[4] = { NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0), NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1), NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2), NFP_PCIE_BAR_PCIE2CPP_MapType( NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3), }; char status_msg[196] = {}; int i, err, bars_free; struct nfp_bar *bar; int expl_groups; char *msg, *end; msg = status_msg + snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: "); end = status_msg + sizeof(status_msg) - 1; bar = &nfp->bar[0]; for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) { struct resource *res; res = &nfp->pdev->resource[(i >> 3) * 2]; /* Skip over BARs that are not IORESOURCE_MEM */ if (!(resource_type(res) & IORESOURCE_MEM)) { bar--; continue; } bar->resource = res; bar->barcfg = 0; bar->nfp = nfp; bar->index = i; bar->mask = nfp_bar_resource_len(bar) - 1; bar->bitsize = fls(bar->mask); bar->base = 0; bar->iomem = NULL; } nfp->bars = bar - &nfp->bar[0]; if (nfp->bars < 8) { dev_err(nfp->dev, "No usable BARs found!\n"); return -EINVAL; } bars_free = nfp->bars; /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70) */ mutex_init(&nfp->expl.mutex); nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4) << 4; nfp->expl.signal_ref = 0x10; /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ bar = &nfp->bar[0]; if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE) bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar), nfp_bar_resource_len(bar)); if (bar->iomem) { int pf; msg += snprintf(msg, end - msg, "0.0: General/MSI-X SRAM, "); atomic_inc(&bar->refcnt); bars_free--; nfp6000_bar_write(nfp, bar, barcfg_msix_general); nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 0x1000; switch (nfp->pdev->device) { case PCI_DEVICE_ID_NETRONOME_NFP3800: pf = nfp->pdev->devfn & 7; nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf); break; case PCI_DEVICE_ID_NETRONOME_NFP4000: case PCI_DEVICE_ID_NETRONOME_NFP5000: case PCI_DEVICE_ID_NETRONOME_NFP6000: nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0); break; default: dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", nfp->pdev->device); err = -EINVAL; goto err_unmap_bar0; } nfp->iomem.em = bar->iomem + NFP_PCIE_EM; } switch (nfp->pdev->device) { case PCI_DEVICE_ID_NETRONOME_NFP3800: expl_groups = 1; break; case PCI_DEVICE_ID_NETRONOME_NFP4000: case PCI_DEVICE_ID_NETRONOME_NFP5000: case PCI_DEVICE_ID_NETRONOME_NFP6000: expl_groups = 4; break; default: dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", nfp->pdev->device); err = -EINVAL; goto err_unmap_bar0; } /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */ bar = &nfp->bar[1]; msg += snprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, "); atomic_inc(&bar->refcnt); bars_free--; nfp6000_bar_write(nfp, bar, barcfg_msix_xpb); /* Use BAR0.4..BAR0.7 for EXPL IO */ for (i = 0; i < 4; i++) { int j; if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) { nfp->expl.group[i].bitsize = 0; continue; } bar = &nfp->bar[4 + i]; bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar), nfp_bar_resource_len(bar)); if (bar->iomem) { msg += snprintf(msg, end - msg, "0.%d: Explicit%d, ", 4 + i, i); atomic_inc(&bar->refcnt); bars_free--; nfp->expl.group[i].bitsize = bar->bitsize; nfp->expl.group[i].addr = bar->iomem; nfp6000_bar_write(nfp, bar, barcfg_explicit[i]); for (j = 0; j < 4; j++) nfp->expl.group[i].free[j] = true; } nfp->iomem.expl[i] = bar->iomem; } /* Sort bars by bit size - use the smallest possible first. */ sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]), bar_cmp, NULL); dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars); return 0; err_unmap_bar0: if (nfp->bar[0].iomem) iounmap(nfp->bar[0].iomem); return err; } static void disable_bars(struct nfp6000_pcie *nfp) { struct nfp_bar *bar = &nfp->bar[0]; int n; for (n = 0; n < nfp->bars; n++, bar++) { if (bar->iomem) { iounmap(bar->iomem); bar->iomem = NULL; } } } /* * Generic CPP bus access interface. */ struct nfp6000_area_priv { atomic_t refcnt; struct nfp_bar *bar; u32 bar_offset; u32 target; u32 action; u32 token; u64 offset; struct { int read; int write; int bar; } width; size_t size; void __iomem *iomem; phys_addr_t phys; struct resource resource; }; static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest, unsigned long long address, unsigned long size) { struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); u32 target = NFP_CPP_ID_TARGET_of(dest); u32 action = NFP_CPP_ID_ACTION_of(dest); u32 token = NFP_CPP_ID_TOKEN_of(dest); int pp; pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address); if (pp < 0) return pp; priv->width.read = PUSH_WIDTH(pp); priv->width.write = PULL_WIDTH(pp); if (priv->width.read > 0 && priv->width.write > 0 && priv->width.read != priv->width.write) { return -EINVAL; } if (priv->width.read > 0) priv->width.bar = priv->width.read; else priv->width.bar = priv->width.write; atomic_set(&priv->refcnt, 0); priv->bar = NULL; priv->target = target; priv->action = action; priv->token = token; priv->offset = address; priv->size = size; memset(&priv->resource, 0, sizeof(priv->resource)); return 0; } static void nfp6000_area_cleanup(struct nfp_cpp_area *area) { } static void priv_area_get(struct nfp_cpp_area *area) { struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); atomic_inc(&priv->refcnt); } static int priv_area_put(struct nfp_cpp_area *area) { struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); if (WARN_ON(!atomic_read(&priv->refcnt))) return 0; return atomic_dec_and_test(&priv->refcnt); } static int nfp6000_area_acquire(struct nfp_cpp_area *area) { struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); int barnum, err; if (priv->bar) { /* Already allocated. */ priv_area_get(area); return 0; } barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token, priv->offset, priv->size, priv->width.bar, 1); if (barnum < 0) { err = barnum; goto err_alloc_bar; } priv->bar = &nfp->bar[barnum]; /* Calculate offset into BAR. */ if (nfp_bar_maptype(priv->bar) == NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) { priv->bar_offset = priv->offset & (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1); priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET( priv->bar, priv->target); priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET( priv->bar, priv->token); } else { priv->bar_offset = priv->offset & priv->bar->mask; } /* We don't actually try to acquire the resource area using * request_resource. This would prevent sharing the mapped * BAR between multiple CPP areas and prevent us from * effectively utilizing the limited amount of BAR resources. */ priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset; priv->resource.name = nfp_cpp_area_name(area); priv->resource.start = priv->phys; priv->resource.end = priv->resource.start + priv->size - 1; priv->resource.flags = IORESOURCE_MEM; /* If the bar is already mapped in, use its mapping */ if (priv->bar->iomem) priv->iomem = priv->bar->iomem + priv->bar_offset; else /* Must have been too big. Sub-allocate. */ priv->iomem = ioremap_nocache(priv->phys, priv->size); if (IS_ERR_OR_NULL(priv->iomem)) { dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n", (int)priv->size, priv->bar->index); err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem); priv->iomem = NULL; goto err_iomem_remap; } priv_area_get(area); return 0; err_iomem_remap: nfp_bar_put(nfp, priv->bar); priv->bar = NULL; err_alloc_bar: return err; } static void nfp6000_area_release(struct nfp_cpp_area *area) { struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); if (!priv_area_put(area)) return; if (!priv->bar->iomem) iounmap(priv->iomem); nfp_bar_put(nfp, priv->bar); priv->bar = NULL; priv->iomem = NULL; } static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area) { struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); return priv->phys; } static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area) { struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); return priv->iomem; } static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area) { /* Use the BAR resource as the resource for the CPP area. * This enables us to share the BAR among multiple CPP areas * without resource conflicts. */ struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); return priv->bar->resource; } static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, unsigned long offset, unsigned int length) { u64 __maybe_unused *wrptr64 = kernel_vaddr; const u64 __iomem __maybe_unused *rdptr64; struct nfp6000_area_priv *priv; u32 *wrptr32 = kernel_vaddr; const u32 __iomem *rdptr32; int n, width; priv = nfp_cpp_area_priv(area); rdptr64 = priv->iomem + offset; rdptr32 = priv->iomem + offset; if (offset + length > priv->size) return -EFAULT; width = priv->width.read; if (width <= 0) return -EINVAL; /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && priv->action == NFP_CPP_ACTION_RW && (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) width = TARGET_WIDTH_32; /* Unaligned? Translate to an explicit access */ if ((priv->offset + offset) & (width - 1)) return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area), NFP_CPP_ID(priv->target, priv->action, priv->token), priv->offset + offset, kernel_vaddr, length, width); if (WARN_ON(!priv->bar)) return -EFAULT; switch (width) { case TARGET_WIDTH_32: if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) return -EINVAL; for (n = 0; n < length; n += sizeof(u32)) *wrptr32++ = __raw_readl(rdptr32++); return n; #ifdef __raw_readq case TARGET_WIDTH_64: if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) return -EINVAL; for (n = 0; n < length; n += sizeof(u64)) *wrptr64++ = __raw_readq(rdptr64++); return n; #endif default: return -EINVAL; } } static int nfp6000_area_write(struct nfp_cpp_area *area, const void *kernel_vaddr, unsigned long offset, unsigned int length) { const u64 __maybe_unused *rdptr64 = kernel_vaddr; u64 __iomem __maybe_unused *wrptr64; const u32 *rdptr32 = kernel_vaddr; struct nfp6000_area_priv *priv; u32 __iomem *wrptr32; int n, width; priv = nfp_cpp_area_priv(area); wrptr64 = priv->iomem + offset; wrptr32 = priv->iomem + offset; if (offset + length > priv->size) return -EFAULT; width = priv->width.write; if (width <= 0) return -EINVAL; /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */ if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && priv->action == NFP_CPP_ACTION_RW && (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) width = TARGET_WIDTH_32; /* Unaligned? Translate to an explicit access */ if ((priv->offset + offset) & (width - 1)) return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area), NFP_CPP_ID(priv->target, priv->action, priv->token), priv->offset + offset, kernel_vaddr, length, width); if (WARN_ON(!priv->bar)) return -EFAULT; switch (width) { case TARGET_WIDTH_32: if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) return -EINVAL; for (n = 0; n < length; n += sizeof(u32)) { __raw_writel(*rdptr32++, wrptr32++); wmb(); } return n; #ifdef __raw_writeq case TARGET_WIDTH_64: if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) return -EINVAL; for (n = 0; n < length; n += sizeof(u64)) { __raw_writeq(*rdptr64++, wrptr64++); wmb(); } return n; #endif default: return -EINVAL; } } struct nfp6000_explicit_priv { struct nfp6000_pcie *nfp; struct { int group; int area; } bar; int bitsize; void __iomem *data; void __iomem *addr; }; static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl) { struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl)); struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); int i, j; mutex_lock(&nfp->expl.mutex); for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) { if (!nfp->expl.group[i].bitsize) continue; for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) { u16 data_offset; if (!nfp->expl.group[i].free[j]) continue; priv->nfp = nfp; priv->bar.group = i; priv->bar.area = j; priv->bitsize = nfp->expl.group[i].bitsize - 2; data_offset = (priv->bar.group << 9) + (priv->bar.area << 7); priv->data = nfp->expl.data + data_offset; priv->addr = nfp->expl.group[i].addr + (priv->bar.area << priv->bitsize); nfp->expl.group[i].free[j] = false; mutex_unlock(&nfp->expl.mutex); return 0; } } mutex_unlock(&nfp->expl.mutex); return -EAGAIN; } static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl) { struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); struct nfp6000_pcie *nfp = priv->nfp; mutex_lock(&nfp->expl.mutex); nfp->expl.group[priv->bar.group].free[priv->bar.area] = true; mutex_unlock(&nfp->expl.mutex); } static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl, const void *buff, size_t len) { struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); const u32 *src = buff; size_t i; for (i = 0; i < len; i += sizeof(u32)) writel(*(src++), priv->data + i); return i; } static int nfp6000_explicit_do(struct nfp_cpp_explicit *expl, const struct nfp_cpp_explicit_command *cmd, u64 address) { struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); u8 signal_master, signal_ref, data_master; struct nfp6000_pcie *nfp = priv->nfp; int sigmask = 0; u16 data_ref; u32 csr[3]; if (cmd->siga_mode) sigmask |= 1 << cmd->siga; if (cmd->sigb_mode) sigmask |= 1 << cmd->sigb; signal_master = cmd->signal_master; if (!signal_master) signal_master = nfp->expl.master_id; signal_ref = cmd->signal_ref; if (signal_master == nfp->expl.master_id) signal_ref = nfp->expl.signal_ref + ((priv->bar.group * 4 + priv->bar.area) << 1); data_master = cmd->data_master; if (!data_master) data_master = nfp->expl.master_id; data_ref = cmd->data_ref; if (data_master == nfp->expl.master_id) data_ref = 0x1000 + (priv->bar.group << 9) + (priv->bar.area << 7); csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) | NFP_PCIE_BAR_EXPLICIT_BAR0_Token( NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) | NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16); csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) | NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) | NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref); csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target( NFP_CPP_ID_TARGET_of(cmd->cpp_id)) | NFP_PCIE_BAR_EXPLICIT_BAR2_Action( NFP_CPP_ID_ACTION_of(cmd->cpp_id)) | NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) | NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) | NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master); if (nfp->iomem.csr) { writel(csr[0], nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, priv->bar.area)); writel(csr[1], nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, priv->bar.area)); writel(csr[2], nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, priv->bar.area)); /* Readback to ensure BAR is flushed */ readl(nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, priv->bar.area)); readl(nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, priv->bar.area)); readl(nfp->iomem.csr + NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, priv->bar.area)); } else { pci_write_config_dword(nfp->pdev, 0x400 + NFP_PCIE_BAR_EXPLICIT_BAR0( priv->bar.group, priv->bar.area), csr[0]); pci_write_config_dword(nfp->pdev, 0x400 + NFP_PCIE_BAR_EXPLICIT_BAR1( priv->bar.group, priv->bar.area), csr[1]); pci_write_config_dword(nfp->pdev, 0x400 + NFP_PCIE_BAR_EXPLICIT_BAR2( priv->bar.group, priv->bar.area), csr[2]); } /* Issue the 'kickoff' transaction */ readb(priv->addr + (address & ((1 << priv->bitsize) - 1))); return sigmask; } static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl, void *buff, size_t len) { struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); u32 *dst = buff; size_t i; for (i = 0; i < len; i += sizeof(u32)) *(dst++) = readl(priv->data + i); return i; } static int nfp6000_init(struct nfp_cpp *cpp) { nfp_cpp_area_cache_add(cpp, SZ_64K); nfp_cpp_area_cache_add(cpp, SZ_64K); nfp_cpp_area_cache_add(cpp, SZ_256K); return 0; } static void nfp6000_free(struct nfp_cpp *cpp) { struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp); disable_bars(nfp); kfree(nfp); } static int nfp6000_read_serial(struct device *dev, u8 *serial) { struct pci_dev *pdev = to_pci_dev(dev); int pos; u32 reg; pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN); if (!pos) { dev_err(dev, "can't find PCIe Serial Number Capability\n"); return -EINVAL; } pci_read_config_dword(pdev, pos + 4, ®); put_unaligned_be16(reg >> 16, serial + 4); pci_read_config_dword(pdev, pos + 8, ®); put_unaligned_be32(reg, serial); return 0; } static int nfp6000_get_interface(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); int pos; u32 reg; pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN); if (!pos) { dev_err(dev, "can't find PCIe Serial Number Capability\n"); return -EINVAL; } pci_read_config_dword(pdev, pos + 4, ®); return reg & 0xffff; } static const struct nfp_cpp_operations nfp6000_pcie_ops = { .owner = THIS_MODULE, .init = nfp6000_init, .free = nfp6000_free, .read_serial = nfp6000_read_serial, .get_interface = nfp6000_get_interface, .area_priv_size = sizeof(struct nfp6000_area_priv), .area_init = nfp6000_area_init, .area_cleanup = nfp6000_area_cleanup, .area_acquire = nfp6000_area_acquire, .area_release = nfp6000_area_release, .area_phys = nfp6000_area_phys, .area_iomem = nfp6000_area_iomem, .area_resource = nfp6000_area_resource, .area_read = nfp6000_area_read, .area_write = nfp6000_area_write, .explicit_priv_size = sizeof(struct nfp6000_explicit_priv), .explicit_acquire = nfp6000_explicit_acquire, .explicit_release = nfp6000_explicit_release, .explicit_put = nfp6000_explicit_put, .explicit_do = nfp6000_explicit_do, .explicit_get = nfp6000_explicit_get, }; /** * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device * @pdev: NFP6000 PCI device * * Return: NFP CPP handle */ struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev) { struct nfp6000_pcie *nfp; u16 interface; int err; /* Finished with card initialization. */ dev_info(&pdev->dev, "Netronome Flow Processor NFP4000/NFP5000/NFP6000 PCIe Card Probe\n"); pcie_print_link_status(pdev); nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); if (!nfp) { err = -ENOMEM; goto err_ret; } nfp->dev = &pdev->dev; nfp->pdev = pdev; init_waitqueue_head(&nfp->bar_waiters); spin_lock_init(&nfp->bar_lock); interface = nfp6000_get_interface(&pdev->dev); if (NFP_CPP_INTERFACE_TYPE_of(interface) != NFP_CPP_INTERFACE_TYPE_PCI) { dev_err(&pdev->dev, "Interface type %d is not the expected %d\n", NFP_CPP_INTERFACE_TYPE_of(interface), NFP_CPP_INTERFACE_TYPE_PCI); err = -ENODEV; goto err_free_nfp; } if (NFP_CPP_INTERFACE_CHANNEL_of(interface) != NFP_CPP_INTERFACE_CHANNEL_PEROPENER) { dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n", NFP_CPP_INTERFACE_CHANNEL_of(interface), NFP_CPP_INTERFACE_CHANNEL_PEROPENER); err = -ENODEV; goto err_free_nfp; } err = enable_bars(nfp, interface); if (err) goto err_free_nfp; /* Probe for all the common NFP devices */ return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp); err_free_nfp: kfree(nfp); err_ret: dev_err(&pdev->dev, "NFP6000 PCI setup failed\n"); return ERR_PTR(err); }