diff options
| author | 2020-08-03 15:06:44 +0000 | |
|---|---|---|
| committer | 2020-08-03 15:06:44 +0000 | |
| commit | b64793999546ed8adebaeebd9d8345d18db8927d (patch) | |
| tree | 4357c27b561d73b0e089727c6ed659f2ceff5f47 /gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp | |
| parent | Add support for UTF-8 DISPLAY-HINTs with octet length. For now only (diff) | |
| download | wireguard-openbsd-b64793999546ed8adebaeebd9d8345d18db8927d.tar.xz wireguard-openbsd-b64793999546ed8adebaeebd9d8345d18db8927d.zip | |
Remove LLVM 8.0.1 files.
Diffstat (limited to 'gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp')
| -rw-r--r-- | gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp | 1875 |
1 files changed, 0 insertions, 1875 deletions
diff --git a/gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp deleted file mode 100644 index 722699907ca..00000000000 --- a/gnu/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp +++ /dev/null @@ -1,1875 +0,0 @@ -//===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This implements a simple VLIW packetizer using DFA. The packetizer works on -// machine basic blocks. For each instruction I in BB, the packetizer consults -// the DFA to see if machine resources are available to execute I. If so, the -// packetizer checks if I depends on any instruction J in the current packet. -// If no dependency is found, I is added to current packet and machine resource -// is marked as taken. If any dependency is found, a target API call is made to -// prune the dependence. -// -//===----------------------------------------------------------------------===// - -#include "HexagonVLIWPacketizer.h" -#include "Hexagon.h" -#include "HexagonInstrInfo.h" -#include "HexagonRegisterInfo.h" -#include "HexagonSubtarget.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineInstrBundle.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineOperand.h" -#include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/IR/DebugLoc.h" -#include "llvm/MC/MCInstrDesc.h" -#include "llvm/Pass.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" -#include <cassert> -#include <cstdint> -#include <iterator> - -using namespace llvm; - -#define DEBUG_TYPE "packets" - -static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden, - cl::ZeroOrMore, cl::init(false), - cl::desc("Disable Hexagon packetizer pass")); - -cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden, - cl::ZeroOrMore, cl::init(true), - cl::desc("Allow slot1 store and slot0 load")); - -static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles", - cl::ZeroOrMore, cl::Hidden, cl::init(true), - cl::desc("Allow non-solo packetization of volatile memory references")); - -static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false), - cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC")); - -static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores", - cl::init(false), cl::Hidden, cl::ZeroOrMore, - cl::desc("Disable vector double new-value-stores")); - -extern cl::opt<bool> ScheduleInlineAsm; - -namespace llvm { - -FunctionPass *createHexagonPacketizer(bool Minimal); -void initializeHexagonPacketizerPass(PassRegistry&); - -} // end namespace llvm - -namespace { - - class HexagonPacketizer : public MachineFunctionPass { - public: - static char ID; - - HexagonPacketizer(bool Min = false) - : MachineFunctionPass(ID), Minimal(Min) {} - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - AU.addRequired<AAResultsWrapperPass>(); - AU.addRequired<MachineBranchProbabilityInfo>(); - AU.addRequired<MachineDominatorTree>(); - AU.addRequired<MachineLoopInfo>(); - AU.addPreserved<MachineDominatorTree>(); - AU.addPreserved<MachineLoopInfo>(); - MachineFunctionPass::getAnalysisUsage(AU); - } - - StringRef getPassName() const override { return "Hexagon Packetizer"; } - bool runOnMachineFunction(MachineFunction &Fn) override; - - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::NoVRegs); - } - - private: - const HexagonInstrInfo *HII; - const HexagonRegisterInfo *HRI; - const bool Minimal; - }; - -} // end anonymous namespace - -char HexagonPacketizer::ID = 0; - -INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer", - "Hexagon Packetizer", false, false) -INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) -INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) -INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer", - "Hexagon Packetizer", false, false) - -HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF, - MachineLoopInfo &MLI, AliasAnalysis *AA, - const MachineBranchProbabilityInfo *MBPI, bool Minimal) - : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI), - Minimal(Minimal) { - HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo(); - HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); - - addMutation(llvm::make_unique<HexagonSubtarget::UsrOverflowMutation>()); - addMutation(llvm::make_unique<HexagonSubtarget::HVXMemLatencyMutation>()); - addMutation(llvm::make_unique<HexagonSubtarget::BankConflictMutation>()); -} - -// Check if FirstI modifies a register that SecondI reads. -static bool hasWriteToReadDep(const MachineInstr &FirstI, - const MachineInstr &SecondI, - const TargetRegisterInfo *TRI) { - for (auto &MO : FirstI.operands()) { - if (!MO.isReg() || !MO.isDef()) - continue; - unsigned R = MO.getReg(); - if (SecondI.readsRegister(R, TRI)) - return true; - } - return false; -} - - -static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI, - MachineBasicBlock::iterator BundleIt, bool Before) { - MachineBasicBlock::instr_iterator InsertPt; - if (Before) - InsertPt = BundleIt.getInstrIterator(); - else - InsertPt = std::next(BundleIt).getInstrIterator(); - - MachineBasicBlock &B = *MI.getParent(); - // The instruction should at least be bundled with the preceding instruction - // (there will always be one, i.e. BUNDLE, if nothing else). - assert(MI.isBundledWithPred()); - if (MI.isBundledWithSucc()) { - MI.clearFlag(MachineInstr::BundledSucc); - MI.clearFlag(MachineInstr::BundledPred); - } else { - // If it's not bundled with the successor (i.e. it is the last one - // in the bundle), then we can simply unbundle it from the predecessor, - // which will take care of updating the predecessor's flag. - MI.unbundleFromPred(); - } - B.splice(InsertPt, &B, MI.getIterator()); - - // Get the size of the bundle without asserting. - MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator(); - MachineBasicBlock::const_instr_iterator E = B.instr_end(); - unsigned Size = 0; - for (++I; I != E && I->isBundledWithPred(); ++I) - ++Size; - - // If there are still two or more instructions, then there is nothing - // else to be done. - if (Size > 1) - return BundleIt; - - // Otherwise, extract the single instruction out and delete the bundle. - MachineBasicBlock::iterator NextIt = std::next(BundleIt); - MachineInstr &SingleI = *BundleIt->getNextNode(); - SingleI.unbundleFromPred(); - assert(!SingleI.isBundledWithSucc()); - BundleIt->eraseFromParent(); - return NextIt; -} - -bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) { - auto &HST = MF.getSubtarget<HexagonSubtarget>(); - HII = HST.getInstrInfo(); - HRI = HST.getRegisterInfo(); - auto &MLI = getAnalysis<MachineLoopInfo>(); - auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); - auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); - - if (EnableGenAllInsnClass) - HII->genAllInsnTimingClasses(MF); - - // Instantiate the packetizer. - bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() || - skipFunction(MF.getFunction()); - HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI, MinOnly); - - // DFA state table should not be empty. - assert(Packetizer.getResourceTracker() && "Empty DFA table!"); - - // Loop over all basic blocks and remove KILL pseudo-instructions - // These instructions confuse the dependence analysis. Consider: - // D0 = ... (Insn 0) - // R0 = KILL R0, D0 (Insn 1) - // R0 = ... (Insn 2) - // Here, Insn 1 will result in the dependence graph not emitting an output - // dependence between Insn 0 and Insn 2. This can lead to incorrect - // packetization - for (MachineBasicBlock &MB : MF) { - auto End = MB.end(); - auto MI = MB.begin(); - while (MI != End) { - auto NextI = std::next(MI); - if (MI->isKill()) { - MB.erase(MI); - End = MB.end(); - } - MI = NextI; - } - } - - // Loop over all of the basic blocks. - for (auto &MB : MF) { - auto Begin = MB.begin(), End = MB.end(); - while (Begin != End) { - // Find the first non-boundary starting from the end of the last - // scheduling region. - MachineBasicBlock::iterator RB = Begin; - while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF)) - ++RB; - // Find the first boundary starting from the beginning of the new - // region. - MachineBasicBlock::iterator RE = RB; - while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF)) - ++RE; - // Add the scheduling boundary if it's not block end. - if (RE != End) - ++RE; - // If RB == End, then RE == End. - if (RB != End) - Packetizer.PacketizeMIs(&MB, RB, RE); - - Begin = RE; - } - } - - Packetizer.unpacketizeSoloInstrs(MF); - return true; -} - -// Reserve resources for a constant extender. Trigger an assertion if the -// reservation fails. -void HexagonPacketizerList::reserveResourcesForConstExt() { - if (!tryAllocateResourcesForConstExt(true)) - llvm_unreachable("Resources not available"); -} - -bool HexagonPacketizerList::canReserveResourcesForConstExt() { - return tryAllocateResourcesForConstExt(false); -} - -// Allocate resources (i.e. 4 bytes) for constant extender. If succeeded, -// return true, otherwise, return false. -bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) { - auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc()); - bool Avail = ResourceTracker->canReserveResources(*ExtMI); - if (Reserve && Avail) - ResourceTracker->reserveResources(*ExtMI); - MF.DeleteMachineInstr(ExtMI); - return Avail; -} - -bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI, - SDep::Kind DepType, unsigned DepReg) { - // Check for LR dependence. - if (DepReg == HRI->getRARegister()) - return true; - - if (HII->isDeallocRet(MI)) - if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister()) - return true; - - // Call-like instructions can be packetized with preceding instructions - // that define registers implicitly used or modified by the call. Explicit - // uses are still prohibited, as in the case of indirect calls: - // r0 = ... - // J2_jumpr r0 - if (DepType == SDep::Data) { - for (const MachineOperand MO : MI.operands()) - if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit()) - return true; - } - - return false; -} - -static bool isRegDependence(const SDep::Kind DepType) { - return DepType == SDep::Data || DepType == SDep::Anti || - DepType == SDep::Output; -} - -static bool isDirectJump(const MachineInstr &MI) { - return MI.getOpcode() == Hexagon::J2_jump; -} - -static bool isSchedBarrier(const MachineInstr &MI) { - switch (MI.getOpcode()) { - case Hexagon::Y2_barrier: - return true; - } - return false; -} - -static bool isControlFlow(const MachineInstr &MI) { - return MI.getDesc().isTerminator() || MI.getDesc().isCall(); -} - -/// Returns true if the instruction modifies a callee-saved register. -static bool doesModifyCalleeSavedReg(const MachineInstr &MI, - const TargetRegisterInfo *TRI) { - const MachineFunction &MF = *MI.getParent()->getParent(); - for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR) - if (MI.modifiesRegister(*CSR, TRI)) - return true; - return false; -} - -// Returns true if an instruction can be promoted to .new predicate or -// new-value store. -bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI, - const TargetRegisterClass *NewRC) { - // Vector stores can be predicated, and can be new-value stores, but - // they cannot be predicated on a .new predicate value. - if (NewRC == &Hexagon::PredRegsRegClass) { - if (HII->isHVXVec(MI) && MI.mayStore()) - return false; - return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0; - } - // If the class is not PredRegs, it could only apply to new-value stores. - return HII->mayBeNewStore(MI); -} - -// Promote an instructiont to its .cur form. -// At this time, we have already made a call to canPromoteToDotCur and made -// sure that it can *indeed* be promoted. -bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI, - SDep::Kind DepType, MachineBasicBlock::iterator &MII, - const TargetRegisterClass* RC) { - assert(DepType == SDep::Data); - int CurOpcode = HII->getDotCurOp(MI); - MI.setDesc(HII->get(CurOpcode)); - return true; -} - -void HexagonPacketizerList::cleanUpDotCur() { - MachineInstr *MI = nullptr; - for (auto BI : CurrentPacketMIs) { - LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump();); - if (HII->isDotCurInst(*BI)) { - MI = BI; - continue; - } - if (MI) { - for (auto &MO : BI->operands()) - if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg()) - return; - } - } - if (!MI) - return; - // We did not find a use of the CUR, so de-cur it. - MI->setDesc(HII->get(HII->getNonDotCurOp(*MI))); - LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump();); -} - -// Check to see if an instruction can be dot cur. -bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI, - const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, - const TargetRegisterClass *RC) { - if (!HII->isHVXVec(MI)) - return false; - if (!HII->isHVXVec(*MII)) - return false; - - // Already a dot new instruction. - if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI)) - return false; - - if (!HII->mayBeCurLoad(MI)) - return false; - - // The "cur value" cannot come from inline asm. - if (PacketSU->getInstr()->isInlineAsm()) - return false; - - // Make sure candidate instruction uses cur. - LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump(); - dbgs() << "in packet\n";); - MachineInstr &MJ = *MII; - LLVM_DEBUG({ - dbgs() << "Checking CUR against "; - MJ.dump(); - }); - unsigned DestReg = MI.getOperand(0).getReg(); - bool FoundMatch = false; - for (auto &MO : MJ.operands()) - if (MO.isReg() && MO.getReg() == DestReg) - FoundMatch = true; - if (!FoundMatch) - return false; - - // Check for existing uses of a vector register within the packet which - // would be affected by converting a vector load into .cur formt. - for (auto BI : CurrentPacketMIs) { - LLVM_DEBUG(dbgs() << "packet has "; BI->dump();); - if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo())) - return false; - } - - LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump();); - // We can convert the opcode into a .cur. - return true; -} - -// Promote an instruction to its .new form. At this time, we have already -// made a call to canPromoteToDotNew and made sure that it can *indeed* be -// promoted. -bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI, - SDep::Kind DepType, MachineBasicBlock::iterator &MII, - const TargetRegisterClass* RC) { - assert(DepType == SDep::Data); - int NewOpcode; - if (RC == &Hexagon::PredRegsRegClass) - NewOpcode = HII->getDotNewPredOp(MI, MBPI); - else - NewOpcode = HII->getDotNewOp(MI); - MI.setDesc(HII->get(NewOpcode)); - return true; -} - -bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) { - int NewOpcode = HII->getDotOldOp(MI); - MI.setDesc(HII->get(NewOpcode)); - return true; -} - -bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) { - unsigned Opc = MI.getOpcode(); - switch (Opc) { - case Hexagon::S2_storerd_io: - case Hexagon::S2_storeri_io: - case Hexagon::S2_storerh_io: - case Hexagon::S2_storerb_io: - break; - default: - llvm_unreachable("Unexpected instruction"); - } - unsigned FrameSize = MF.getFrameInfo().getStackSize(); - MachineOperand &Off = MI.getOperand(1); - int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE); - if (HII->isValidOffset(Opc, NewOff, HRI)) { - Off.setImm(NewOff); - return true; - } - return false; -} - -void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) { - unsigned Opc = MI.getOpcode(); - switch (Opc) { - case Hexagon::S2_storerd_io: - case Hexagon::S2_storeri_io: - case Hexagon::S2_storerh_io: - case Hexagon::S2_storerb_io: - break; - default: - llvm_unreachable("Unexpected instruction"); - } - unsigned FrameSize = MF.getFrameInfo().getStackSize(); - MachineOperand &Off = MI.getOperand(1); - Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE); -} - -/// Return true if we can update the offset in MI so that MI and MJ -/// can be packetized together. -bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) { - assert(SUI->getInstr() && SUJ->getInstr()); - MachineInstr &MI = *SUI->getInstr(); - MachineInstr &MJ = *SUJ->getInstr(); - - unsigned BPI, OPI; - if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI)) - return false; - unsigned BPJ, OPJ; - if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ)) - return false; - unsigned Reg = MI.getOperand(BPI).getReg(); - if (Reg != MJ.getOperand(BPJ).getReg()) - return false; - // Make sure that the dependences do not restrict adding MI to the packet. - // That is, ignore anti dependences, and make sure the only data dependence - // involves the specific register. - for (const auto &PI : SUI->Preds) - if (PI.getKind() != SDep::Anti && - (PI.getKind() != SDep::Data || PI.getReg() != Reg)) - return false; - int Incr; - if (!HII->getIncrementValue(MJ, Incr)) - return false; - - int64_t Offset = MI.getOperand(OPI).getImm(); - if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI)) - return false; - - MI.getOperand(OPI).setImm(Offset + Incr); - ChangedOffset = Offset; - return true; -} - -/// Undo the changed offset. This is needed if the instruction cannot be -/// added to the current packet due to a different instruction. -void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) { - unsigned BP, OP; - if (!HII->getBaseAndOffsetPosition(MI, BP, OP)) - llvm_unreachable("Unable to find base and offset operands."); - MI.getOperand(OP).setImm(ChangedOffset); -} - -enum PredicateKind { - PK_False, - PK_True, - PK_Unknown -}; - -/// Returns true if an instruction is predicated on p0 and false if it's -/// predicated on !p0. -static PredicateKind getPredicateSense(const MachineInstr &MI, - const HexagonInstrInfo *HII) { - if (!HII->isPredicated(MI)) - return PK_Unknown; - if (HII->isPredicatedTrue(MI)) - return PK_True; - return PK_False; -} - -static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI, - const HexagonInstrInfo *HII) { - assert(HII->isPostIncrement(MI) && "Not a post increment operation."); -#ifndef NDEBUG - // Post Increment means duplicates. Use dense map to find duplicates in the - // list. Caution: Densemap initializes with the minimum of 64 buckets, - // whereas there are at most 5 operands in the post increment. - DenseSet<unsigned> DefRegsSet; - for (auto &MO : MI.operands()) - if (MO.isReg() && MO.isDef()) - DefRegsSet.insert(MO.getReg()); - - for (auto &MO : MI.operands()) - if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg())) - return MO; -#else - if (MI.mayLoad()) { - const MachineOperand &Op1 = MI.getOperand(1); - // The 2nd operand is always the post increment operand in load. - assert(Op1.isReg() && "Post increment operand has be to a register."); - return Op1; - } - if (MI.getDesc().mayStore()) { - const MachineOperand &Op0 = MI.getOperand(0); - // The 1st operand is always the post increment operand in store. - assert(Op0.isReg() && "Post increment operand has be to a register."); - return Op0; - } -#endif - // we should never come here. - llvm_unreachable("mayLoad or mayStore not set for Post Increment operation"); -} - -// Get the value being stored. -static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) { - // value being stored is always the last operand. - return MI.getOperand(MI.getNumOperands()-1); -} - -static bool isLoadAbsSet(const MachineInstr &MI) { - unsigned Opc = MI.getOpcode(); - switch (Opc) { - case Hexagon::L4_loadrd_ap: - case Hexagon::L4_loadrb_ap: - case Hexagon::L4_loadrh_ap: - case Hexagon::L4_loadrub_ap: - case Hexagon::L4_loadruh_ap: - case Hexagon::L4_loadri_ap: - return true; - } - return false; -} - -static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) { - assert(isLoadAbsSet(MI)); - return MI.getOperand(1); -} - -// Can be new value store? -// Following restrictions are to be respected in convert a store into -// a new value store. -// 1. If an instruction uses auto-increment, its address register cannot -// be a new-value register. Arch Spec 5.4.2.1 -// 2. If an instruction uses absolute-set addressing mode, its address -// register cannot be a new-value register. Arch Spec 5.4.2.1. -// 3. If an instruction produces a 64-bit result, its registers cannot be used -// as new-value registers. Arch Spec 5.4.2.2. -// 4. If the instruction that sets the new-value register is conditional, then -// the instruction that uses the new-value register must also be conditional, -// and both must always have their predicates evaluate identically. -// Arch Spec 5.4.2.3. -// 5. There is an implied restriction that a packet cannot have another store, -// if there is a new value store in the packet. Corollary: if there is -// already a store in a packet, there can not be a new value store. -// Arch Spec: 3.4.4.2 -bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI, - const MachineInstr &PacketMI, unsigned DepReg) { - // Make sure we are looking at the store, that can be promoted. - if (!HII->mayBeNewStore(MI)) - return false; - - // Make sure there is dependency and can be new value'd. - const MachineOperand &Val = getStoreValueOperand(MI); - if (Val.isReg() && Val.getReg() != DepReg) - return false; - - const MCInstrDesc& MCID = PacketMI.getDesc(); - - // First operand is always the result. - const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF); - // Double regs can not feed into new value store: PRM section: 5.4.2.2. - if (PacketRC == &Hexagon::DoubleRegsRegClass) - return false; - - // New-value stores are of class NV (slot 0), dual stores require class ST - // in slot 0 (PRM 5.5). - for (auto I : CurrentPacketMIs) { - SUnit *PacketSU = MIToSUnit.find(I)->second; - if (PacketSU->getInstr()->mayStore()) - return false; - } - - // Make sure it's NOT the post increment register that we are going to - // new value. - if (HII->isPostIncrement(MI) && - getPostIncrementOperand(MI, HII).getReg() == DepReg) { - return false; - } - - if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() && - getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) { - // If source is post_inc, or absolute-set addressing, it can not feed - // into new value store - // r3 = memw(r2++#4) - // memw(r30 + #-1404) = r2.new -> can not be new value store - // arch spec section: 5.4.2.1. - return false; - } - - if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg) - return false; - - // If the source that feeds the store is predicated, new value store must - // also be predicated. - if (HII->isPredicated(PacketMI)) { - if (!HII->isPredicated(MI)) - return false; - - // Check to make sure that they both will have their predicates - // evaluate identically. - unsigned predRegNumSrc = 0; - unsigned predRegNumDst = 0; - const TargetRegisterClass* predRegClass = nullptr; - - // Get predicate register used in the source instruction. - for (auto &MO : PacketMI.operands()) { - if (!MO.isReg()) - continue; - predRegNumSrc = MO.getReg(); - predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc); - if (predRegClass == &Hexagon::PredRegsRegClass) - break; - } - assert((predRegClass == &Hexagon::PredRegsRegClass) && - "predicate register not found in a predicated PacketMI instruction"); - - // Get predicate register used in new-value store instruction. - for (auto &MO : MI.operands()) { - if (!MO.isReg()) - continue; - predRegNumDst = MO.getReg(); - predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst); - if (predRegClass == &Hexagon::PredRegsRegClass) - break; - } - assert((predRegClass == &Hexagon::PredRegsRegClass) && - "predicate register not found in a predicated MI instruction"); - - // New-value register producer and user (store) need to satisfy these - // constraints: - // 1) Both instructions should be predicated on the same register. - // 2) If producer of the new-value register is .new predicated then store - // should also be .new predicated and if producer is not .new predicated - // then store should not be .new predicated. - // 3) Both new-value register producer and user should have same predicate - // sense, i.e, either both should be negated or both should be non-negated. - if (predRegNumDst != predRegNumSrc || - HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) || - getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII)) - return false; - } - - // Make sure that other than the new-value register no other store instruction - // register has been modified in the same packet. Predicate registers can be - // modified by they should not be modified between the producer and the store - // instruction as it will make them both conditional on different values. - // We already know this to be true for all the instructions before and - // including PacketMI. Howerver, we need to perform the check for the - // remaining instructions in the packet. - - unsigned StartCheck = 0; - - for (auto I : CurrentPacketMIs) { - SUnit *TempSU = MIToSUnit.find(I)->second; - MachineInstr &TempMI = *TempSU->getInstr(); - - // Following condition is true for all the instructions until PacketMI is - // reached (StartCheck is set to 0 before the for loop). - // StartCheck flag is 1 for all the instructions after PacketMI. - if (&TempMI != &PacketMI && !StartCheck) // Start processing only after - continue; // encountering PacketMI. - - StartCheck = 1; - if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence. - continue; - - for (auto &MO : MI.operands()) - if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI)) - return false; - } - - // Make sure that for non-POST_INC stores: - // 1. The only use of reg is DepReg and no other registers. - // This handles base+index registers. - // The following store can not be dot new. - // Eg. r0 = add(r0, #3) - // memw(r1+r0<<#2) = r0 - if (!HII->isPostIncrement(MI)) { - for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) { - const MachineOperand &MO = MI.getOperand(opNum); - if (MO.isReg() && MO.getReg() == DepReg) - return false; - } - } - - // If data definition is because of implicit definition of the register, - // do not newify the store. Eg. - // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12 - // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343] - for (auto &MO : PacketMI.operands()) { - if (MO.isRegMask() && MO.clobbersPhysReg(DepReg)) - return false; - if (!MO.isReg() || !MO.isDef() || !MO.isImplicit()) - continue; - unsigned R = MO.getReg(); - if (R == DepReg || HRI->isSuperRegister(DepReg, R)) - return false; - } - - // Handle imp-use of super reg case. There is a target independent side - // change that should prevent this situation but I am handling it for - // just-in-case. For example, we cannot newify R2 in the following case: - // %r3 = A2_tfrsi 0; - // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1; - for (auto &MO : MI.operands()) { - if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg) - return false; - } - - // Can be dot new store. - return true; -} - -// Can this MI to promoted to either new value store or new value jump. -bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI, - const SUnit *PacketSU, unsigned DepReg, - MachineBasicBlock::iterator &MII) { - if (!HII->mayBeNewStore(MI)) - return false; - - // Check to see the store can be new value'ed. - MachineInstr &PacketMI = *PacketSU->getInstr(); - if (canPromoteToNewValueStore(MI, PacketMI, DepReg)) - return true; - - // Check to see the compare/jump can be new value'ed. - // This is done as a pass on its own. Don't need to check it here. - return false; -} - -static bool isImplicitDependency(const MachineInstr &I, bool CheckDef, - unsigned DepReg) { - for (auto &MO : I.operands()) { - if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg)) - return true; - if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit()) - continue; - if (CheckDef == MO.isDef()) - return true; - } - return false; -} - -// Check to see if an instruction can be dot new. -bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI, - const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, - const TargetRegisterClass* RC) { - // Already a dot new instruction. - if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI)) - return false; - - if (!isNewifiable(MI, RC)) - return false; - - const MachineInstr &PI = *PacketSU->getInstr(); - - // The "new value" cannot come from inline asm. - if (PI.isInlineAsm()) - return false; - - // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no - // sense. - if (PI.isImplicitDef()) - return false; - - // If dependency is trough an implicitly defined register, we should not - // newify the use. - if (isImplicitDependency(PI, true, DepReg) || - isImplicitDependency(MI, false, DepReg)) - return false; - - const MCInstrDesc& MCID = PI.getDesc(); - const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF); - if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass) - return false; - - // predicate .new - if (RC == &Hexagon::PredRegsRegClass) - return HII->predCanBeUsedAsDotNew(PI, DepReg); - - if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI)) - return false; - - // Create a dot new machine instruction to see if resources can be - // allocated. If not, bail out now. - int NewOpcode = HII->getDotNewOp(MI); - const MCInstrDesc &D = HII->get(NewOpcode); - MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc()); - bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI); - MF.DeleteMachineInstr(NewMI); - if (!ResourcesAvailable) - return false; - - // New Value Store only. New Value Jump generated as a separate pass. - if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII)) - return false; - - return true; -} - -// Go through the packet instructions and search for an anti dependency between -// them and DepReg from MI. Consider this case: -// Trying to add -// a) %r1 = TFRI_cdNotPt %p3, 2 -// to this packet: -// { -// b) %p0 = C2_or killed %p3, killed %p0 -// c) %p3 = C2_tfrrp %r23 -// d) %r1 = C2_cmovenewit %p3, 4 -// } -// The P3 from a) and d) will be complements after -// a)'s P3 is converted to .new form -// Anti-dep between c) and b) is irrelevant for this case -bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI, - unsigned DepReg) { - SUnit *PacketSUDep = MIToSUnit.find(&MI)->second; - - for (auto I : CurrentPacketMIs) { - // We only care for dependencies to predicated instructions - if (!HII->isPredicated(*I)) - continue; - - // Scheduling Unit for current insn in the packet - SUnit *PacketSU = MIToSUnit.find(I)->second; - - // Look at dependencies between current members of the packet and - // predicate defining instruction MI. Make sure that dependency is - // on the exact register we care about. - if (PacketSU->isSucc(PacketSUDep)) { - for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { - auto &Dep = PacketSU->Succs[i]; - if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti && - Dep.getReg() == DepReg) - return true; - } - } - } - - return false; -} - -/// Gets the predicate register of a predicated instruction. -static unsigned getPredicatedRegister(MachineInstr &MI, - const HexagonInstrInfo *QII) { - /// We use the following rule: The first predicate register that is a use is - /// the predicate register of a predicated instruction. - assert(QII->isPredicated(MI) && "Must be predicated instruction"); - - for (auto &Op : MI.operands()) { - if (Op.isReg() && Op.getReg() && Op.isUse() && - Hexagon::PredRegsRegClass.contains(Op.getReg())) - return Op.getReg(); - } - - llvm_unreachable("Unknown instruction operand layout"); - return 0; -} - -// Given two predicated instructions, this function detects whether -// the predicates are complements. -bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1, - MachineInstr &MI2) { - // If we don't know the predicate sense of the instructions bail out early, we - // need it later. - if (getPredicateSense(MI1, HII) == PK_Unknown || - getPredicateSense(MI2, HII) == PK_Unknown) - return false; - - // Scheduling unit for candidate. - SUnit *SU = MIToSUnit[&MI1]; - - // One corner case deals with the following scenario: - // Trying to add - // a) %r24 = A2_tfrt %p0, %r25 - // to this packet: - // { - // b) %r25 = A2_tfrf %p0, %r24 - // c) %p0 = C2_cmpeqi %r26, 1 - // } - // - // On general check a) and b) are complements, but presence of c) will - // convert a) to .new form, and then it is not a complement. - // We attempt to detect it by analyzing existing dependencies in the packet. - - // Analyze relationships between all existing members of the packet. - // Look for Anti dependecy on the same predicate reg as used in the - // candidate. - for (auto I : CurrentPacketMIs) { - // Scheduling Unit for current insn in the packet. - SUnit *PacketSU = MIToSUnit.find(I)->second; - - // If this instruction in the packet is succeeded by the candidate... - if (PacketSU->isSucc(SU)) { - for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { - auto Dep = PacketSU->Succs[i]; - // The corner case exist when there is true data dependency between - // candidate and one of current packet members, this dep is on - // predicate reg, and there already exist anti dep on the same pred in - // the packet. - if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data && - Hexagon::PredRegsRegClass.contains(Dep.getReg())) { - // Here I know that I is predicate setting instruction with true - // data dep to candidate on the register we care about - c) in the - // above example. Now I need to see if there is an anti dependency - // from c) to any other instruction in the same packet on the pred - // reg of interest. - if (restrictingDepExistInPacket(*I, Dep.getReg())) - return false; - } - } - } - } - - // If the above case does not apply, check regular complement condition. - // Check that the predicate register is the same and that the predicate - // sense is different We also need to differentiate .old vs. .new: !p0 - // is not complementary to p0.new. - unsigned PReg1 = getPredicatedRegister(MI1, HII); - unsigned PReg2 = getPredicatedRegister(MI2, HII); - return PReg1 == PReg2 && - Hexagon::PredRegsRegClass.contains(PReg1) && - Hexagon::PredRegsRegClass.contains(PReg2) && - getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) && - HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2); -} - -// Initialize packetizer flags. -void HexagonPacketizerList::initPacketizerState() { - Dependence = false; - PromotedToDotNew = false; - GlueToNewValueJump = false; - GlueAllocframeStore = false; - FoundSequentialDependence = false; - ChangedOffset = INT64_MAX; -} - -// Ignore bundling of pseudo instructions. -bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI, - const MachineBasicBlock *) { - if (MI.isDebugInstr()) - return true; - - if (MI.isCFIInstruction()) - return false; - - // We must print out inline assembly. - if (MI.isInlineAsm()) - return false; - - if (MI.isImplicitDef()) - return false; - - // We check if MI has any functional units mapped to it. If it doesn't, - // we ignore the instruction. - const MCInstrDesc& TID = MI.getDesc(); - auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass()); - unsigned FuncUnits = IS->getUnits(); - return !FuncUnits; -} - -bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) { - // Ensure any bundles created by gather packetize remain seperate. - if (MI.isBundle()) - return true; - - if (MI.isEHLabel() || MI.isCFIInstruction()) - return true; - - // Consider inline asm to not be a solo instruction by default. - // Inline asm will be put in a packet temporarily, but then it will be - // removed, and placed outside of the packet (before or after, depending - // on dependencies). This is to reduce the impact of inline asm as a - // "packet splitting" instruction. - if (MI.isInlineAsm() && !ScheduleInlineAsm) - return true; - - if (isSchedBarrier(MI)) - return true; - - if (HII->isSolo(MI)) - return true; - - if (MI.getOpcode() == Hexagon::A2_nop) - return true; - - return false; -} - -// Quick check if instructions MI and MJ cannot coexist in the same packet. -// Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm", -// but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm". -// For full test call this function twice: -// cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI) -// Doing the test only one way saves the amount of code in this function, -// since every test would need to be repeated with the MI and MJ reversed. -static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ, - const HexagonInstrInfo &HII) { - const MachineFunction *MF = MI.getParent()->getParent(); - if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() && - HII.isHVXMemWithAIndirect(MI, MJ)) - return true; - - // An inline asm cannot be together with a branch, because we may not be - // able to remove the asm out after packetizing (i.e. if the asm must be - // moved past the bundle). Similarly, two asms cannot be together to avoid - // complications when determining their relative order outside of a bundle. - if (MI.isInlineAsm()) - return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() || - MJ.isCall() || MJ.isTerminator(); - - // New-value stores cannot coexist with any other stores. - if (HII.isNewValueStore(MI) && MJ.mayStore()) - return true; - - switch (MI.getOpcode()) { - case Hexagon::S2_storew_locked: - case Hexagon::S4_stored_locked: - case Hexagon::L2_loadw_locked: - case Hexagon::L4_loadd_locked: - case Hexagon::Y2_dccleana: - case Hexagon::Y2_dccleaninva: - case Hexagon::Y2_dcinva: - case Hexagon::Y2_dczeroa: - case Hexagon::Y4_l2fetch: - case Hexagon::Y5_l2fetch: { - // These instructions can only be grouped with ALU32 or non-floating-point - // XTYPE instructions. Since there is no convenient way of identifying fp - // XTYPE instructions, only allow grouping with ALU32 for now. - unsigned TJ = HII.getType(MJ); - if (TJ != HexagonII::TypeALU32_2op && - TJ != HexagonII::TypeALU32_3op && - TJ != HexagonII::TypeALU32_ADDI) - return true; - break; - } - default: - break; - } - - // "False" really means that the quick check failed to determine if - // I and J cannot coexist. - return false; -} - -// Full, symmetric check. -bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI, - const MachineInstr &MJ) { - return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII); -} - -void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) { - for (auto &B : MF) { - MachineBasicBlock::iterator BundleIt; - MachineBasicBlock::instr_iterator NextI; - for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) { - NextI = std::next(I); - MachineInstr &MI = *I; - if (MI.isBundle()) - BundleIt = I; - if (!MI.isInsideBundle()) - continue; - - // Decide on where to insert the instruction that we are pulling out. - // Debug instructions always go before the bundle, but the placement of - // INLINE_ASM depends on potential dependencies. By default, try to - // put it before the bundle, but if the asm writes to a register that - // other instructions in the bundle read, then we need to place it - // after the bundle (to preserve the bundle semantics). - bool InsertBeforeBundle; - if (MI.isInlineAsm()) - InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI); - else if (MI.isDebugValue()) - InsertBeforeBundle = true; - else - continue; - - BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle); - } - } -} - -// Check if a given instruction is of class "system". -static bool isSystemInstr(const MachineInstr &MI) { - unsigned Opc = MI.getOpcode(); - switch (Opc) { - case Hexagon::Y2_barrier: - case Hexagon::Y2_dcfetchbo: - case Hexagon::Y4_l2fetch: - case Hexagon::Y5_l2fetch: - return true; - } - return false; -} - -bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I, - const MachineInstr &J) { - // The dependence graph may not include edges between dead definitions, - // so without extra checks, we could end up packetizing two instruction - // defining the same (dead) register. - if (I.isCall() || J.isCall()) - return false; - if (HII->isPredicated(I) || HII->isPredicated(J)) - return false; - - BitVector DeadDefs(Hexagon::NUM_TARGET_REGS); - for (auto &MO : I.operands()) { - if (!MO.isReg() || !MO.isDef() || !MO.isDead()) - continue; - DeadDefs[MO.getReg()] = true; - } - - for (auto &MO : J.operands()) { - if (!MO.isReg() || !MO.isDef() || !MO.isDead()) - continue; - unsigned R = MO.getReg(); - if (R != Hexagon::USR_OVF && DeadDefs[R]) - return true; - } - return false; -} - -bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I, - const MachineInstr &J) { - // A save callee-save register function call can only be in a packet - // with instructions that don't write to the callee-save registers. - if ((HII->isSaveCalleeSavedRegsCall(I) && - doesModifyCalleeSavedReg(J, HRI)) || - (HII->isSaveCalleeSavedRegsCall(J) && - doesModifyCalleeSavedReg(I, HRI))) - return true; - - // Two control flow instructions cannot go in the same packet. - if (isControlFlow(I) && isControlFlow(J)) - return true; - - // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot - // contain a speculative indirect jump, - // a new-value compare jump or a dealloc_return. - auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool { - if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI)) - return true; - if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI)) - return true; - return false; - }; - - if (HII->isLoopN(I) && isBadForLoopN(J)) - return true; - if (HII->isLoopN(J) && isBadForLoopN(I)) - return true; - - // dealloc_return cannot appear in the same packet as a conditional or - // unconditional jump. - return HII->isDeallocRet(I) && - (J.isBranch() || J.isCall() || J.isBarrier()); -} - -bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I, - const MachineInstr &J) { - // Adding I to a packet that has J. - - // Regmasks are not reflected in the scheduling dependency graph, so - // we need to check them manually. This code assumes that regmasks only - // occur on calls, and the problematic case is when we add an instruction - // defining a register R to a packet that has a call that clobbers R via - // a regmask. Those cannot be packetized together, because the call will - // be executed last. That's also a reson why it is ok to add a call - // clobbering R to a packet that defines R. - - // Look for regmasks in J. - for (const MachineOperand &OpJ : J.operands()) { - if (!OpJ.isRegMask()) - continue; - assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call"); - for (const MachineOperand &OpI : I.operands()) { - if (OpI.isReg()) { - if (OpJ.clobbersPhysReg(OpI.getReg())) - return true; - } else if (OpI.isRegMask()) { - // Both are regmasks. Assume that they intersect. - return true; - } - } - } - return false; -} - -bool HexagonPacketizerList::hasDualStoreDependence(const MachineInstr &I, - const MachineInstr &J) { - bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J); - bool StoreI = I.mayStore(), StoreJ = J.mayStore(); - if ((SysI && StoreJ) || (SysJ && StoreI)) - return true; - - if (StoreI && StoreJ) { - if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I)) - return true; - } else { - // A memop cannot be in the same packet with another memop or a store. - // Two stores can be together, but here I and J cannot both be stores. - bool MopStI = HII->isMemOp(I) || StoreI; - bool MopStJ = HII->isMemOp(J) || StoreJ; - if (MopStI && MopStJ) - return true; - } - - return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J)); -} - -// SUI is the current instruction that is out side of the current packet. -// SUJ is the current instruction inside the current packet against which that -// SUI will be packetized. -bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) { - assert(SUI->getInstr() && SUJ->getInstr()); - MachineInstr &I = *SUI->getInstr(); - MachineInstr &J = *SUJ->getInstr(); - - // Clear IgnoreDepMIs when Packet starts. - if (CurrentPacketMIs.size() == 1) - IgnoreDepMIs.clear(); - - MachineBasicBlock::iterator II = I.getIterator(); - - // Solo instructions cannot go in the packet. - assert(!isSoloInstruction(I) && "Unexpected solo instr!"); - - if (cannotCoexist(I, J)) - return false; - - Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J); - if (Dependence) - return false; - - // Regmasks are not accounted for in the scheduling graph, so we need - // to explicitly check for dependencies caused by them. They should only - // appear on calls, so it's not too pessimistic to reject all regmask - // dependencies. - Dependence = hasRegMaskDependence(I, J); - if (Dependence) - return false; - - // Dual-store does not allow second store, if the first store is not - // in SLOT0. New value store, new value jump, dealloc_return and memop - // always take SLOT0. Arch spec 3.4.4.2. - Dependence = hasDualStoreDependence(I, J); - if (Dependence) - return false; - - // If an instruction feeds new value jump, glue it. - MachineBasicBlock::iterator NextMII = I.getIterator(); - ++NextMII; - if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) { - MachineInstr &NextMI = *NextMII; - - bool secondRegMatch = false; - const MachineOperand &NOp0 = NextMI.getOperand(0); - const MachineOperand &NOp1 = NextMI.getOperand(1); - - if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg()) - secondRegMatch = true; - - for (MachineInstr *PI : CurrentPacketMIs) { - // NVJ can not be part of the dual jump - Arch Spec: section 7.8. - if (PI->isCall()) { - Dependence = true; - break; - } - // Validate: - // 1. Packet does not have a store in it. - // 2. If the first operand of the nvj is newified, and the second - // operand is also a reg, it (second reg) is not defined in - // the same packet. - // 3. If the second operand of the nvj is newified, (which means - // first operand is also a reg), first reg is not defined in - // the same packet. - if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() || - HII->isLoopN(*PI)) { - Dependence = true; - break; - } - // Check #2/#3. - const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1; - if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) { - Dependence = true; - break; - } - } - - GlueToNewValueJump = true; - if (Dependence) - return false; - } - - // There no dependency between a prolog instruction and its successor. - if (!SUJ->isSucc(SUI)) - return true; - - for (unsigned i = 0; i < SUJ->Succs.size(); ++i) { - if (FoundSequentialDependence) - break; - - if (SUJ->Succs[i].getSUnit() != SUI) - continue; - - SDep::Kind DepType = SUJ->Succs[i].getKind(); - // For direct calls: - // Ignore register dependences for call instructions for packetization - // purposes except for those due to r31 and predicate registers. - // - // For indirect calls: - // Same as direct calls + check for true dependences to the register - // used in the indirect call. - // - // We completely ignore Order dependences for call instructions. - // - // For returns: - // Ignore register dependences for return instructions like jumpr, - // dealloc return unless we have dependencies on the explicit uses - // of the registers used by jumpr (like r31) or dealloc return - // (like r29 or r30). - unsigned DepReg = 0; - const TargetRegisterClass *RC = nullptr; - if (DepType == SDep::Data) { - DepReg = SUJ->Succs[i].getReg(); - RC = HRI->getMinimalPhysRegClass(DepReg); - } - - if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) { - if (!isRegDependence(DepType)) - continue; - if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg())) - continue; - } - - if (DepType == SDep::Data) { - if (canPromoteToDotCur(J, SUJ, DepReg, II, RC)) - if (promoteToDotCur(J, DepType, II, RC)) - continue; - } - - // Data dpendence ok if we have load.cur. - if (DepType == SDep::Data && HII->isDotCurInst(J)) { - if (HII->isHVXVec(I)) - continue; - } - - // For instructions that can be promoted to dot-new, try to promote. - if (DepType == SDep::Data) { - if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) { - if (promoteToDotNew(I, DepType, II, RC)) { - PromotedToDotNew = true; - if (cannotCoexist(I, J)) - FoundSequentialDependence = true; - continue; - } - } - if (HII->isNewValueJump(I)) - continue; - } - - // For predicated instructions, if the predicates are complements then - // there can be no dependence. - if (HII->isPredicated(I) && HII->isPredicated(J) && - arePredicatesComplements(I, J)) { - // Not always safe to do this translation. - // DAG Builder attempts to reduce dependence edges using transitive - // nature of dependencies. Here is an example: - // - // r0 = tfr_pt ... (1) - // r0 = tfr_pf ... (2) - // r0 = tfr_pt ... (3) - // - // There will be an output dependence between (1)->(2) and (2)->(3). - // However, there is no dependence edge between (1)->(3). This results - // in all 3 instructions going in the same packet. We ignore dependce - // only once to avoid this situation. - auto Itr = find(IgnoreDepMIs, &J); - if (Itr != IgnoreDepMIs.end()) { - Dependence = true; - return false; - } - IgnoreDepMIs.push_back(&I); - continue; - } - - // Ignore Order dependences between unconditional direct branches - // and non-control-flow instructions. - if (isDirectJump(I) && !J.isBranch() && !J.isCall() && - DepType == SDep::Order) - continue; - - // Ignore all dependences for jumps except for true and output - // dependences. - if (I.isConditionalBranch() && DepType != SDep::Data && - DepType != SDep::Output) - continue; - - if (DepType == SDep::Output) { - FoundSequentialDependence = true; - break; - } - - // For Order dependences: - // 1. Volatile loads/stores can be packetized together, unless other - // rules prevent is. - // 2. Store followed by a load is not allowed. - // 3. Store followed by a store is valid. - // 4. Load followed by any memory operation is allowed. - if (DepType == SDep::Order) { - if (!PacketizeVolatiles) { - bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef(); - if (OrdRefs) { - FoundSequentialDependence = true; - break; - } - } - // J is first, I is second. - bool LoadJ = J.mayLoad(), StoreJ = J.mayStore(); - bool LoadI = I.mayLoad(), StoreI = I.mayStore(); - bool NVStoreJ = HII->isNewValueStore(J); - bool NVStoreI = HII->isNewValueStore(I); - bool IsVecJ = HII->isHVXVec(J); - bool IsVecI = HII->isHVXVec(I); - - if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() && - ((LoadJ && StoreI && !NVStoreI) || - (StoreJ && LoadI && !NVStoreJ)) && - (J.getOpcode() != Hexagon::S2_allocframe && - I.getOpcode() != Hexagon::S2_allocframe) && - (J.getOpcode() != Hexagon::L2_deallocframe && - I.getOpcode() != Hexagon::L2_deallocframe) && - (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI)) - setmemShufDisabled(true); - else - if (StoreJ && LoadI && alias(J, I)) { - FoundSequentialDependence = true; - break; - } - - if (!StoreJ) - if (!LoadJ || (!LoadI && !StoreI)) { - // If J is neither load nor store, assume a dependency. - // If J is a load, but I is neither, also assume a dependency. - FoundSequentialDependence = true; - break; - } - // Store followed by store: not OK on V2. - // Store followed by load: not OK on all. - // Load followed by store: OK on all. - // Load followed by load: OK on all. - continue; - } - - // Special case for ALLOCFRAME: even though there is dependency - // between ALLOCFRAME and subsequent store, allow it to be packetized - // in a same packet. This implies that the store is using the caller's - // SP. Hence, offset needs to be updated accordingly. - if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) { - unsigned Opc = I.getOpcode(); - switch (Opc) { - case Hexagon::S2_storerd_io: - case Hexagon::S2_storeri_io: - case Hexagon::S2_storerh_io: - case Hexagon::S2_storerb_io: - if (I.getOperand(0).getReg() == HRI->getStackRegister()) { - // Since this store is to be glued with allocframe in the same - // packet, it will use SP of the previous stack frame, i.e. - // caller's SP. Therefore, we need to recalculate offset - // according to this change. - GlueAllocframeStore = useCallersSP(I); - if (GlueAllocframeStore) - continue; - } - break; - default: - break; - } - } - - // There are certain anti-dependencies that cannot be ignored. - // Specifically: - // J2_call ... implicit-def %r0 ; SUJ - // R0 = ... ; SUI - // Those cannot be packetized together, since the call will observe - // the effect of the assignment to R0. - if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) { - // Check if I defines any volatile register. We should also check - // registers that the call may read, but these happen to be a - // subset of the volatile register set. - for (const MachineOperand &Op : I.operands()) { - if (Op.isReg() && Op.isDef()) { - unsigned R = Op.getReg(); - if (!J.readsRegister(R, HRI) && !J.modifiesRegister(R, HRI)) - continue; - } else if (!Op.isRegMask()) { - // If I has a regmask assume dependency. - continue; - } - FoundSequentialDependence = true; - break; - } - } - - // Skip over remaining anti-dependences. Two instructions that are - // anti-dependent can share a packet, since in most such cases all - // operands are read before any modifications take place. - // The exceptions are branch and call instructions, since they are - // executed after all other instructions have completed (at least - // conceptually). - if (DepType != SDep::Anti) { - FoundSequentialDependence = true; - break; - } - } - - if (FoundSequentialDependence) { - Dependence = true; - return false; - } - - return true; -} - -bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) { - assert(SUI->getInstr() && SUJ->getInstr()); - MachineInstr &I = *SUI->getInstr(); - MachineInstr &J = *SUJ->getInstr(); - - bool Coexist = !cannotCoexist(I, J); - - if (Coexist && !Dependence) - return true; - - // Check if the instruction was promoted to a dot-new. If so, demote it - // back into a dot-old. - if (PromotedToDotNew) - demoteToDotOld(I); - - cleanUpDotCur(); - // Check if the instruction (must be a store) was glued with an allocframe - // instruction. If so, restore its offset to its original value, i.e. use - // current SP instead of caller's SP. - if (GlueAllocframeStore) { - useCalleesSP(I); - GlueAllocframeStore = false; - } - - if (ChangedOffset != INT64_MAX) - undoChangedOffset(I); - - if (GlueToNewValueJump) { - // Putting I and J together would prevent the new-value jump from being - // packetized with the producer. In that case I and J must be separated. - GlueToNewValueJump = false; - return false; - } - - if (!Coexist) - return false; - - if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) { - FoundSequentialDependence = false; - Dependence = false; - return true; - } - - return false; -} - - -bool HexagonPacketizerList::foundLSInPacket() { - bool FoundLoad = false; - bool FoundStore = false; - - for (auto MJ : CurrentPacketMIs) { - unsigned Opc = MJ->getOpcode(); - if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe) - continue; - if (HII->isMemOp(*MJ)) - continue; - if (MJ->mayLoad()) - FoundLoad = true; - if (MJ->mayStore() && !HII->isNewValueStore(*MJ)) - FoundStore = true; - } - return FoundLoad && FoundStore; -} - - -MachineBasicBlock::iterator -HexagonPacketizerList::addToPacket(MachineInstr &MI) { - MachineBasicBlock::iterator MII = MI.getIterator(); - MachineBasicBlock *MBB = MI.getParent(); - - if (CurrentPacketMIs.empty()) - PacketStalls = false; - PacketStalls |= producesStall(MI); - - if (MI.isImplicitDef()) { - // Add to the packet to allow subsequent instructions to be checked - // properly. - CurrentPacketMIs.push_back(&MI); - return MII; - } - assert(ResourceTracker->canReserveResources(MI)); - - bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI); - bool Good = true; - - if (GlueToNewValueJump) { - MachineInstr &NvjMI = *++MII; - // We need to put both instructions in the same packet: MI and NvjMI. - // Either of them can require a constant extender. Try to add both to - // the current packet, and if that fails, end the packet and start a - // new one. - ResourceTracker->reserveResources(MI); - if (ExtMI) - Good = tryAllocateResourcesForConstExt(true); - - bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI); - if (Good) { - if (ResourceTracker->canReserveResources(NvjMI)) - ResourceTracker->reserveResources(NvjMI); - else - Good = false; - } - if (Good && ExtNvjMI) - Good = tryAllocateResourcesForConstExt(true); - - if (!Good) { - endPacket(MBB, MI); - assert(ResourceTracker->canReserveResources(MI)); - ResourceTracker->reserveResources(MI); - if (ExtMI) { - assert(canReserveResourcesForConstExt()); - tryAllocateResourcesForConstExt(true); - } - assert(ResourceTracker->canReserveResources(NvjMI)); - ResourceTracker->reserveResources(NvjMI); - if (ExtNvjMI) { - assert(canReserveResourcesForConstExt()); - reserveResourcesForConstExt(); - } - } - CurrentPacketMIs.push_back(&MI); - CurrentPacketMIs.push_back(&NvjMI); - return MII; - } - - ResourceTracker->reserveResources(MI); - if (ExtMI && !tryAllocateResourcesForConstExt(true)) { - endPacket(MBB, MI); - if (PromotedToDotNew) - demoteToDotOld(MI); - if (GlueAllocframeStore) { - useCalleesSP(MI); - GlueAllocframeStore = false; - } - ResourceTracker->reserveResources(MI); - reserveResourcesForConstExt(); - } - - CurrentPacketMIs.push_back(&MI); - return MII; -} - -void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB, - MachineBasicBlock::iterator EndMI) { - // Replace VLIWPacketizerList::endPacket(MBB, EndMI). - - bool memShufDisabled = getmemShufDisabled(); - if (memShufDisabled && !foundLSInPacket()) { - setmemShufDisabled(false); - LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n"); - } - memShufDisabled = getmemShufDisabled(); - - OldPacketMIs.clear(); - for (MachineInstr *MI : CurrentPacketMIs) { - MachineBasicBlock::instr_iterator NextMI = std::next(MI->getIterator()); - for (auto &I : make_range(HII->expandVGatherPseudo(*MI), NextMI)) - OldPacketMIs.push_back(&I); - } - CurrentPacketMIs.clear(); - - if (OldPacketMIs.size() > 1) { - MachineBasicBlock::instr_iterator FirstMI(OldPacketMIs.front()); - MachineBasicBlock::instr_iterator LastMI(EndMI.getInstrIterator()); - finalizeBundle(*MBB, FirstMI, LastMI); - auto BundleMII = std::prev(FirstMI); - if (memShufDisabled) - HII->setBundleNoShuf(BundleMII); - - setmemShufDisabled(false); - } - - ResourceTracker->clearResources(); - LLVM_DEBUG(dbgs() << "End packet\n"); -} - -bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) { - if (Minimal) - return false; - return !producesStall(MI); -} - -// V60 forward scheduling. -bool HexagonPacketizerList::producesStall(const MachineInstr &I) { - // If the packet already stalls, then ignore the stall from a subsequent - // instruction in the same packet. - if (PacketStalls) - return false; - - // Check whether the previous packet is in a different loop. If this is the - // case, there is little point in trying to avoid a stall because that would - // favor the rare case (loop entry) over the common case (loop iteration). - // - // TODO: We should really be able to check all the incoming edges if this is - // the first packet in a basic block, so we can avoid stalls from the loop - // backedge. - if (!OldPacketMIs.empty()) { - auto *OldBB = OldPacketMIs.front()->getParent(); - auto *ThisBB = I.getParent(); - if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB)) - return false; - } - - SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)]; - - // If the latency is 0 and there is a data dependence between this - // instruction and any instruction in the current packet, we disregard any - // potential stalls due to the instructions in the previous packet. Most of - // the instruction pairs that can go together in the same packet have 0 - // latency between them. The exceptions are - // 1. NewValueJumps as they're generated much later and the latencies can't - // be changed at that point. - // 2. .cur instructions, if its consumer has a 0 latency successor (such as - // .new). In this case, the latency between .cur and the consumer stays - // non-zero even though we can have both .cur and .new in the same packet. - // Changing the latency to 0 is not an option as it causes software pipeliner - // to not pipeline in some cases. - - // For Example: - // { - // I1: v6.cur = vmem(r0++#1) - // I2: v7 = valign(v6,v4,r2) - // I3: vmem(r5++#1) = v7.new - // } - // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2. - - for (auto J : CurrentPacketMIs) { - SUnit *SUJ = MIToSUnit[J]; - for (auto &Pred : SUI->Preds) - if (Pred.getSUnit() == SUJ) - if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) || - HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I)) - return false; - } - - // Check if the latency is greater than one between this instruction and any - // instruction in the previous packet. - for (auto J : OldPacketMIs) { - SUnit *SUJ = MIToSUnit[J]; - for (auto &Pred : SUI->Preds) - if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1) - return true; - } - - return false; -} - -//===----------------------------------------------------------------------===// -// Public Constructor Functions -//===----------------------------------------------------------------------===// - -FunctionPass *llvm::createHexagonPacketizer(bool Minimal) { - return new HexagonPacketizer(Minimal); -} |
