summaryrefslogtreecommitdiffstats
path: root/gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
diff options
context:
space:
mode:
authorpatrick <patrick@openbsd.org>2020-08-03 15:06:44 +0000
committerpatrick <patrick@openbsd.org>2020-08-03 15:06:44 +0000
commitb64793999546ed8adebaeebd9d8345d18db8927d (patch)
tree4357c27b561d73b0e089727c6ed659f2ceff5f47 /gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
parentAdd support for UTF-8 DISPLAY-HINTs with octet length. For now only (diff)
downloadwireguard-openbsd-b64793999546ed8adebaeebd9d8345d18db8927d.tar.xz
wireguard-openbsd-b64793999546ed8adebaeebd9d8345d18db8927d.zip
Remove LLVM 8.0.1 files.
Diffstat (limited to 'gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp')
-rw-r--r--gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp2007
1 files changed, 0 insertions, 2007 deletions
diff --git a/gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
deleted file mode 100644
index 239cf49ca8a..00000000000
--- a/gnu/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ /dev/null
@@ -1,2007 +0,0 @@
-//===- HexagonHardwareLoops.cpp - Identify and generate hardware loops ----===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass identifies loops where we can generate the Hexagon hardware
-// loop instruction. The hardware loop can perform loop branches with a
-// zero-cycle overhead.
-//
-// The pattern that defines the induction variable can changed depending on
-// prior optimizations. For example, the IndVarSimplify phase run by 'opt'
-// normalizes induction variables, and the Loop Strength Reduction pass
-// run by 'llc' may also make changes to the induction variable.
-// The pattern detected by this phase is due to running Strength Reduction.
-//
-// Criteria for hardware loops:
-// - Countable loops (w/ ind. var for a trip count)
-// - Assumes loops are normalized by IndVarSimplify
-// - Try inner-most loops first
-// - No function calls in loops.
-//
-//===----------------------------------------------------------------------===//
-
-#include "HexagonInstrInfo.h"
-#include "HexagonSubtarget.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineDominators.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
-#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/TargetRegisterInfo.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DebugLoc.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/raw_ostream.h"
-#include <cassert>
-#include <cstdint>
-#include <cstdlib>
-#include <iterator>
-#include <map>
-#include <set>
-#include <string>
-#include <utility>
-#include <vector>
-
-using namespace llvm;
-
-#define DEBUG_TYPE "hwloops"
-
-#ifndef NDEBUG
-static cl::opt<int> HWLoopLimit("hexagon-max-hwloop", cl::Hidden, cl::init(-1));
-
-// Option to create preheader only for a specific function.
-static cl::opt<std::string> PHFn("hexagon-hwloop-phfn", cl::Hidden,
- cl::init(""));
-#endif
-
-// Option to create a preheader if one doesn't exist.
-static cl::opt<bool> HWCreatePreheader("hexagon-hwloop-preheader",
- cl::Hidden, cl::init(true),
- cl::desc("Add a preheader to a hardware loop if one doesn't exist"));
-
-// Turn it off by default. If a preheader block is not created here, the
-// software pipeliner may be unable to find a block suitable to serve as
-// a preheader. In that case SWP will not run.
-static cl::opt<bool> SpecPreheader("hwloop-spec-preheader", cl::init(false),
- cl::Hidden, cl::ZeroOrMore, cl::desc("Allow speculation of preheader "
- "instructions"));
-
-STATISTIC(NumHWLoops, "Number of loops converted to hardware loops");
-
-namespace llvm {
-
- FunctionPass *createHexagonHardwareLoops();
- void initializeHexagonHardwareLoopsPass(PassRegistry&);
-
-} // end namespace llvm
-
-namespace {
-
- class CountValue;
-
- struct HexagonHardwareLoops : public MachineFunctionPass {
- MachineLoopInfo *MLI;
- MachineRegisterInfo *MRI;
- MachineDominatorTree *MDT;
- const HexagonInstrInfo *TII;
- const HexagonRegisterInfo *TRI;
-#ifndef NDEBUG
- static int Counter;
-#endif
-
- public:
- static char ID;
-
- HexagonHardwareLoops() : MachineFunctionPass(ID) {}
-
- bool runOnMachineFunction(MachineFunction &MF) override;
-
- StringRef getPassName() const override { return "Hexagon Hardware Loops"; }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MachineDominatorTree>();
- AU.addRequired<MachineLoopInfo>();
- MachineFunctionPass::getAnalysisUsage(AU);
- }
-
- private:
- using LoopFeederMap = std::map<unsigned, MachineInstr *>;
-
- /// Kinds of comparisons in the compare instructions.
- struct Comparison {
- enum Kind {
- EQ = 0x01,
- NE = 0x02,
- L = 0x04,
- G = 0x08,
- U = 0x40,
- LTs = L,
- LEs = L | EQ,
- GTs = G,
- GEs = G | EQ,
- LTu = L | U,
- LEu = L | EQ | U,
- GTu = G | U,
- GEu = G | EQ | U
- };
-
- static Kind getSwappedComparison(Kind Cmp) {
- assert ((!((Cmp & L) && (Cmp & G))) && "Malformed comparison operator");
- if ((Cmp & L) || (Cmp & G))
- return (Kind)(Cmp ^ (L|G));
- return Cmp;
- }
-
- static Kind getNegatedComparison(Kind Cmp) {
- if ((Cmp & L) || (Cmp & G))
- return (Kind)((Cmp ^ (L | G)) ^ EQ);
- if ((Cmp & NE) || (Cmp & EQ))
- return (Kind)(Cmp ^ (EQ | NE));
- return (Kind)0;
- }
-
- static bool isSigned(Kind Cmp) {
- return (Cmp & (L | G) && !(Cmp & U));
- }
-
- static bool isUnsigned(Kind Cmp) {
- return (Cmp & U);
- }
- };
-
- /// Find the register that contains the loop controlling
- /// induction variable.
- /// If successful, it will return true and set the \p Reg, \p IVBump
- /// and \p IVOp arguments. Otherwise it will return false.
- /// The returned induction register is the register R that follows the
- /// following induction pattern:
- /// loop:
- /// R = phi ..., [ R.next, LatchBlock ]
- /// R.next = R + #bump
- /// if (R.next < #N) goto loop
- /// IVBump is the immediate value added to R, and IVOp is the instruction
- /// "R.next = R + #bump".
- bool findInductionRegister(MachineLoop *L, unsigned &Reg,
- int64_t &IVBump, MachineInstr *&IVOp) const;
-
- /// Return the comparison kind for the specified opcode.
- Comparison::Kind getComparisonKind(unsigned CondOpc,
- MachineOperand *InitialValue,
- const MachineOperand *Endvalue,
- int64_t IVBump) const;
-
- /// Analyze the statements in a loop to determine if the loop
- /// has a computable trip count and, if so, return a value that represents
- /// the trip count expression.
- CountValue *getLoopTripCount(MachineLoop *L,
- SmallVectorImpl<MachineInstr *> &OldInsts);
-
- /// Return the expression that represents the number of times
- /// a loop iterates. The function takes the operands that represent the
- /// loop start value, loop end value, and induction value. Based upon
- /// these operands, the function attempts to compute the trip count.
- /// If the trip count is not directly available (as an immediate value,
- /// or a register), the function will attempt to insert computation of it
- /// to the loop's preheader.
- CountValue *computeCount(MachineLoop *Loop, const MachineOperand *Start,
- const MachineOperand *End, unsigned IVReg,
- int64_t IVBump, Comparison::Kind Cmp) const;
-
- /// Return true if the instruction is not valid within a hardware
- /// loop.
- bool isInvalidLoopOperation(const MachineInstr *MI,
- bool IsInnerHWLoop) const;
-
- /// Return true if the loop contains an instruction that inhibits
- /// using the hardware loop.
- bool containsInvalidInstruction(MachineLoop *L, bool IsInnerHWLoop) const;
-
- /// Given a loop, check if we can convert it to a hardware loop.
- /// If so, then perform the conversion and return true.
- bool convertToHardwareLoop(MachineLoop *L, bool &L0used, bool &L1used);
-
- /// Return true if the instruction is now dead.
- bool isDead(const MachineInstr *MI,
- SmallVectorImpl<MachineInstr *> &DeadPhis) const;
-
- /// Remove the instruction if it is now dead.
- void removeIfDead(MachineInstr *MI);
-
- /// Make sure that the "bump" instruction executes before the
- /// compare. We need that for the IV fixup, so that the compare
- /// instruction would not use a bumped value that has not yet been
- /// defined. If the instructions are out of order, try to reorder them.
- bool orderBumpCompare(MachineInstr *BumpI, MachineInstr *CmpI);
-
- /// Return true if MO and MI pair is visited only once. If visited
- /// more than once, this indicates there is recursion. In such a case,
- /// return false.
- bool isLoopFeeder(MachineLoop *L, MachineBasicBlock *A, MachineInstr *MI,
- const MachineOperand *MO,
- LoopFeederMap &LoopFeederPhi) const;
-
- /// Return true if the Phi may generate a value that may underflow,
- /// or may wrap.
- bool phiMayWrapOrUnderflow(MachineInstr *Phi, const MachineOperand *EndVal,
- MachineBasicBlock *MBB, MachineLoop *L,
- LoopFeederMap &LoopFeederPhi) const;
-
- /// Return true if the induction variable may underflow an unsigned
- /// value in the first iteration.
- bool loopCountMayWrapOrUnderFlow(const MachineOperand *InitVal,
- const MachineOperand *EndVal,
- MachineBasicBlock *MBB, MachineLoop *L,
- LoopFeederMap &LoopFeederPhi) const;
-
- /// Check if the given operand has a compile-time known constant
- /// value. Return true if yes, and false otherwise. When returning true, set
- /// Val to the corresponding constant value.
- bool checkForImmediate(const MachineOperand &MO, int64_t &Val) const;
-
- /// Check if the operand has a compile-time known constant value.
- bool isImmediate(const MachineOperand &MO) const {
- int64_t V;
- return checkForImmediate(MO, V);
- }
-
- /// Return the immediate for the specified operand.
- int64_t getImmediate(const MachineOperand &MO) const {
- int64_t V;
- if (!checkForImmediate(MO, V))
- llvm_unreachable("Invalid operand");
- return V;
- }
-
- /// Reset the given machine operand to now refer to a new immediate
- /// value. Assumes that the operand was already referencing an immediate
- /// value, either directly, or via a register.
- void setImmediate(MachineOperand &MO, int64_t Val);
-
- /// Fix the data flow of the induction variable.
- /// The desired flow is: phi ---> bump -+-> comparison-in-latch.
- /// |
- /// +-> back to phi
- /// where "bump" is the increment of the induction variable:
- /// iv = iv + #const.
- /// Due to some prior code transformations, the actual flow may look
- /// like this:
- /// phi -+-> bump ---> back to phi
- /// |
- /// +-> comparison-in-latch (against upper_bound-bump),
- /// i.e. the comparison that controls the loop execution may be using
- /// the value of the induction variable from before the increment.
- ///
- /// Return true if the loop's flow is the desired one (i.e. it's
- /// either been fixed, or no fixing was necessary).
- /// Otherwise, return false. This can happen if the induction variable
- /// couldn't be identified, or if the value in the latch's comparison
- /// cannot be adjusted to reflect the post-bump value.
- bool fixupInductionVariable(MachineLoop *L);
-
- /// Given a loop, if it does not have a preheader, create one.
- /// Return the block that is the preheader.
- MachineBasicBlock *createPreheaderForLoop(MachineLoop *L);
- };
-
- char HexagonHardwareLoops::ID = 0;
-#ifndef NDEBUG
- int HexagonHardwareLoops::Counter = 0;
-#endif
-
- /// Abstraction for a trip count of a loop. A smaller version
- /// of the MachineOperand class without the concerns of changing the
- /// operand representation.
- class CountValue {
- public:
- enum CountValueType {
- CV_Register,
- CV_Immediate
- };
-
- private:
- CountValueType Kind;
- union Values {
- struct {
- unsigned Reg;
- unsigned Sub;
- } R;
- unsigned ImmVal;
- } Contents;
-
- public:
- explicit CountValue(CountValueType t, unsigned v, unsigned u = 0) {
- Kind = t;
- if (Kind == CV_Register) {
- Contents.R.Reg = v;
- Contents.R.Sub = u;
- } else {
- Contents.ImmVal = v;
- }
- }
-
- bool isReg() const { return Kind == CV_Register; }
- bool isImm() const { return Kind == CV_Immediate; }
-
- unsigned getReg() const {
- assert(isReg() && "Wrong CountValue accessor");
- return Contents.R.Reg;
- }
-
- unsigned getSubReg() const {
- assert(isReg() && "Wrong CountValue accessor");
- return Contents.R.Sub;
- }
-
- unsigned getImm() const {
- assert(isImm() && "Wrong CountValue accessor");
- return Contents.ImmVal;
- }
-
- void print(raw_ostream &OS, const TargetRegisterInfo *TRI = nullptr) const {
- if (isReg()) { OS << printReg(Contents.R.Reg, TRI, Contents.R.Sub); }
- if (isImm()) { OS << Contents.ImmVal; }
- }
- };
-
-} // end anonymous namespace
-
-INITIALIZE_PASS_BEGIN(HexagonHardwareLoops, "hwloops",
- "Hexagon Hardware Loops", false, false)
-INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
-INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
-INITIALIZE_PASS_END(HexagonHardwareLoops, "hwloops",
- "Hexagon Hardware Loops", false, false)
-
-FunctionPass *llvm::createHexagonHardwareLoops() {
- return new HexagonHardwareLoops();
-}
-
-bool HexagonHardwareLoops::runOnMachineFunction(MachineFunction &MF) {
- LLVM_DEBUG(dbgs() << "********* Hexagon Hardware Loops *********\n");
- if (skipFunction(MF.getFunction()))
- return false;
-
- bool Changed = false;
-
- MLI = &getAnalysis<MachineLoopInfo>();
- MRI = &MF.getRegInfo();
- MDT = &getAnalysis<MachineDominatorTree>();
- const HexagonSubtarget &HST = MF.getSubtarget<HexagonSubtarget>();
- TII = HST.getInstrInfo();
- TRI = HST.getRegisterInfo();
-
- for (auto &L : *MLI)
- if (!L->getParentLoop()) {
- bool L0Used = false;
- bool L1Used = false;
- Changed |= convertToHardwareLoop(L, L0Used, L1Used);
- }
-
- return Changed;
-}
-
-bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
- unsigned &Reg,
- int64_t &IVBump,
- MachineInstr *&IVOp
- ) const {
- MachineBasicBlock *Header = L->getHeader();
- MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader);
- MachineBasicBlock *Latch = L->getLoopLatch();
- MachineBasicBlock *ExitingBlock = L->findLoopControlBlock();
- if (!Header || !Preheader || !Latch || !ExitingBlock)
- return false;
-
- // This pair represents an induction register together with an immediate
- // value that will be added to it in each loop iteration.
- using RegisterBump = std::pair<unsigned, int64_t>;
-
- // Mapping: R.next -> (R, bump), where R, R.next and bump are derived
- // from an induction operation
- // R.next = R + bump
- // where bump is an immediate value.
- using InductionMap = std::map<unsigned, RegisterBump>;
-
- InductionMap IndMap;
-
- using instr_iterator = MachineBasicBlock::instr_iterator;
-
- for (instr_iterator I = Header->instr_begin(), E = Header->instr_end();
- I != E && I->isPHI(); ++I) {
- MachineInstr *Phi = &*I;
-
- // Have a PHI instruction. Get the operand that corresponds to the
- // latch block, and see if is a result of an addition of form "reg+imm",
- // where the "reg" is defined by the PHI node we are looking at.
- for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) {
- if (Phi->getOperand(i+1).getMBB() != Latch)
- continue;
-
- unsigned PhiOpReg = Phi->getOperand(i).getReg();
- MachineInstr *DI = MRI->getVRegDef(PhiOpReg);
-
- if (DI->getDesc().isAdd()) {
- // If the register operand to the add is the PHI we're looking at, this
- // meets the induction pattern.
- unsigned IndReg = DI->getOperand(1).getReg();
- MachineOperand &Opnd2 = DI->getOperand(2);
- int64_t V;
- if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) {
- unsigned UpdReg = DI->getOperand(0).getReg();
- IndMap.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V)));
- }
- }
- } // for (i)
- } // for (instr)
-
- SmallVector<MachineOperand,2> Cond;
- MachineBasicBlock *TB = nullptr, *FB = nullptr;
- bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false);
- if (NotAnalyzed)
- return false;
-
- unsigned PredR, PredPos, PredRegFlags;
- if (!TII->getPredReg(Cond, PredR, PredPos, PredRegFlags))
- return false;
-
- MachineInstr *PredI = MRI->getVRegDef(PredR);
- if (!PredI->isCompare())
- return false;
-
- unsigned CmpReg1 = 0, CmpReg2 = 0;
- int CmpImm = 0, CmpMask = 0;
- bool CmpAnalyzed =
- TII->analyzeCompare(*PredI, CmpReg1, CmpReg2, CmpMask, CmpImm);
- // Fail if the compare was not analyzed, or it's not comparing a register
- // with an immediate value. Not checking the mask here, since we handle
- // the individual compare opcodes (including A4_cmpb*) later on.
- if (!CmpAnalyzed)
- return false;
-
- // Exactly one of the input registers to the comparison should be among
- // the induction registers.
- InductionMap::iterator IndMapEnd = IndMap.end();
- InductionMap::iterator F = IndMapEnd;
- if (CmpReg1 != 0) {
- InductionMap::iterator F1 = IndMap.find(CmpReg1);
- if (F1 != IndMapEnd)
- F = F1;
- }
- if (CmpReg2 != 0) {
- InductionMap::iterator F2 = IndMap.find(CmpReg2);
- if (F2 != IndMapEnd) {
- if (F != IndMapEnd)
- return false;
- F = F2;
- }
- }
- if (F == IndMapEnd)
- return false;
-
- Reg = F->second.first;
- IVBump = F->second.second;
- IVOp = MRI->getVRegDef(F->first);
- return true;
-}
-
-// Return the comparison kind for the specified opcode.
-HexagonHardwareLoops::Comparison::Kind
-HexagonHardwareLoops::getComparisonKind(unsigned CondOpc,
- MachineOperand *InitialValue,
- const MachineOperand *EndValue,
- int64_t IVBump) const {
- Comparison::Kind Cmp = (Comparison::Kind)0;
- switch (CondOpc) {
- case Hexagon::C2_cmpeq:
- case Hexagon::C2_cmpeqi:
- case Hexagon::C2_cmpeqp:
- Cmp = Comparison::EQ;
- break;
- case Hexagon::C4_cmpneq:
- case Hexagon::C4_cmpneqi:
- Cmp = Comparison::NE;
- break;
- case Hexagon::C2_cmplt:
- Cmp = Comparison::LTs;
- break;
- case Hexagon::C2_cmpltu:
- Cmp = Comparison::LTu;
- break;
- case Hexagon::C4_cmplte:
- case Hexagon::C4_cmpltei:
- Cmp = Comparison::LEs;
- break;
- case Hexagon::C4_cmplteu:
- case Hexagon::C4_cmplteui:
- Cmp = Comparison::LEu;
- break;
- case Hexagon::C2_cmpgt:
- case Hexagon::C2_cmpgti:
- case Hexagon::C2_cmpgtp:
- Cmp = Comparison::GTs;
- break;
- case Hexagon::C2_cmpgtu:
- case Hexagon::C2_cmpgtui:
- case Hexagon::C2_cmpgtup:
- Cmp = Comparison::GTu;
- break;
- case Hexagon::C2_cmpgei:
- Cmp = Comparison::GEs;
- break;
- case Hexagon::C2_cmpgeui:
- Cmp = Comparison::GEs;
- break;
- default:
- return (Comparison::Kind)0;
- }
- return Cmp;
-}
-
-/// Analyze the statements in a loop to determine if the loop has
-/// a computable trip count and, if so, return a value that represents
-/// the trip count expression.
-///
-/// This function iterates over the phi nodes in the loop to check for
-/// induction variable patterns that are used in the calculation for
-/// the number of time the loop is executed.
-CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
- SmallVectorImpl<MachineInstr *> &OldInsts) {
- MachineBasicBlock *TopMBB = L->getTopBlock();
- MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin();
- assert(PI != TopMBB->pred_end() &&
- "Loop must have more than one incoming edge!");
- MachineBasicBlock *Backedge = *PI++;
- if (PI == TopMBB->pred_end()) // dead loop?
- return nullptr;
- MachineBasicBlock *Incoming = *PI++;
- if (PI != TopMBB->pred_end()) // multiple backedges?
- return nullptr;
-
- // Make sure there is one incoming and one backedge and determine which
- // is which.
- if (L->contains(Incoming)) {
- if (L->contains(Backedge))
- return nullptr;
- std::swap(Incoming, Backedge);
- } else if (!L->contains(Backedge))
- return nullptr;
-
- // Look for the cmp instruction to determine if we can get a useful trip
- // count. The trip count can be either a register or an immediate. The
- // location of the value depends upon the type (reg or imm).
- MachineBasicBlock *ExitingBlock = L->findLoopControlBlock();
- if (!ExitingBlock)
- return nullptr;
-
- unsigned IVReg = 0;
- int64_t IVBump = 0;
- MachineInstr *IVOp;
- bool FoundIV = findInductionRegister(L, IVReg, IVBump, IVOp);
- if (!FoundIV)
- return nullptr;
-
- MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader);
-
- MachineOperand *InitialValue = nullptr;
- MachineInstr *IV_Phi = MRI->getVRegDef(IVReg);
- MachineBasicBlock *Latch = L->getLoopLatch();
- for (unsigned i = 1, n = IV_Phi->getNumOperands(); i < n; i += 2) {
- MachineBasicBlock *MBB = IV_Phi->getOperand(i+1).getMBB();
- if (MBB == Preheader)
- InitialValue = &IV_Phi->getOperand(i);
- else if (MBB == Latch)
- IVReg = IV_Phi->getOperand(i).getReg(); // Want IV reg after bump.
- }
- if (!InitialValue)
- return nullptr;
-
- SmallVector<MachineOperand,2> Cond;
- MachineBasicBlock *TB = nullptr, *FB = nullptr;
- bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false);
- if (NotAnalyzed)
- return nullptr;
-
- MachineBasicBlock *Header = L->getHeader();
- // TB must be non-null. If FB is also non-null, one of them must be
- // the header. Otherwise, branch to TB could be exiting the loop, and
- // the fall through can go to the header.
- assert (TB && "Exit block without a branch?");
- if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) {
- MachineBasicBlock *LTB = nullptr, *LFB = nullptr;
- SmallVector<MachineOperand,2> LCond;
- bool NotAnalyzed = TII->analyzeBranch(*Latch, LTB, LFB, LCond, false);
- if (NotAnalyzed)
- return nullptr;
- if (TB == Latch)
- TB = (LTB == Header) ? LTB : LFB;
- else
- FB = (LTB == Header) ? LTB: LFB;
- }
- assert ((!FB || TB == Header || FB == Header) && "Branches not to header?");
- if (!TB || (FB && TB != Header && FB != Header))
- return nullptr;
-
- // Branches of form "if (!P) ..." cause HexagonInstrInfo::AnalyzeBranch
- // to put imm(0), followed by P in the vector Cond.
- // If TB is not the header, it means that the "not-taken" path must lead
- // to the header.
- bool Negated = TII->predOpcodeHasNot(Cond) ^ (TB != Header);
- unsigned PredReg, PredPos, PredRegFlags;
- if (!TII->getPredReg(Cond, PredReg, PredPos, PredRegFlags))
- return nullptr;
- MachineInstr *CondI = MRI->getVRegDef(PredReg);
- unsigned CondOpc = CondI->getOpcode();
-
- unsigned CmpReg1 = 0, CmpReg2 = 0;
- int Mask = 0, ImmValue = 0;
- bool AnalyzedCmp =
- TII->analyzeCompare(*CondI, CmpReg1, CmpReg2, Mask, ImmValue);
- if (!AnalyzedCmp)
- return nullptr;
-
- // The comparison operator type determines how we compute the loop
- // trip count.
- OldInsts.push_back(CondI);
- OldInsts.push_back(IVOp);
-
- // Sadly, the following code gets information based on the position
- // of the operands in the compare instruction. This has to be done
- // this way, because the comparisons check for a specific relationship
- // between the operands (e.g. is-less-than), rather than to find out
- // what relationship the operands are in (as on PPC).
- Comparison::Kind Cmp;
- bool isSwapped = false;
- const MachineOperand &Op1 = CondI->getOperand(1);
- const MachineOperand &Op2 = CondI->getOperand(2);
- const MachineOperand *EndValue = nullptr;
-
- if (Op1.isReg()) {
- if (Op2.isImm() || Op1.getReg() == IVReg)
- EndValue = &Op2;
- else {
- EndValue = &Op1;
- isSwapped = true;
- }
- }
-
- if (!EndValue)
- return nullptr;
-
- Cmp = getComparisonKind(CondOpc, InitialValue, EndValue, IVBump);
- if (!Cmp)
- return nullptr;
- if (Negated)
- Cmp = Comparison::getNegatedComparison(Cmp);
- if (isSwapped)
- Cmp = Comparison::getSwappedComparison(Cmp);
-
- if (InitialValue->isReg()) {
- unsigned R = InitialValue->getReg();
- MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent();
- if (!MDT->properlyDominates(DefBB, Header)) {
- int64_t V;
- if (!checkForImmediate(*InitialValue, V))
- return nullptr;
- }
- OldInsts.push_back(MRI->getVRegDef(R));
- }
- if (EndValue->isReg()) {
- unsigned R = EndValue->getReg();
- MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent();
- if (!MDT->properlyDominates(DefBB, Header)) {
- int64_t V;
- if (!checkForImmediate(*EndValue, V))
- return nullptr;
- }
- OldInsts.push_back(MRI->getVRegDef(R));
- }
-
- return computeCount(L, InitialValue, EndValue, IVReg, IVBump, Cmp);
-}
-
-/// Helper function that returns the expression that represents the
-/// number of times a loop iterates. The function takes the operands that
-/// represent the loop start value, loop end value, and induction value.
-/// Based upon these operands, the function attempts to compute the trip count.
-CountValue *HexagonHardwareLoops::computeCount(MachineLoop *Loop,
- const MachineOperand *Start,
- const MachineOperand *End,
- unsigned IVReg,
- int64_t IVBump,
- Comparison::Kind Cmp) const {
- // Cannot handle comparison EQ, i.e. while (A == B).
- if (Cmp == Comparison::EQ)
- return nullptr;
-
- // Check if either the start or end values are an assignment of an immediate.
- // If so, use the immediate value rather than the register.
- if (Start->isReg()) {
- const MachineInstr *StartValInstr = MRI->getVRegDef(Start->getReg());
- if (StartValInstr && (StartValInstr->getOpcode() == Hexagon::A2_tfrsi ||
- StartValInstr->getOpcode() == Hexagon::A2_tfrpi))
- Start = &StartValInstr->getOperand(1);
- }
- if (End->isReg()) {
- const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg());
- if (EndValInstr && (EndValInstr->getOpcode() == Hexagon::A2_tfrsi ||
- EndValInstr->getOpcode() == Hexagon::A2_tfrpi))
- End = &EndValInstr->getOperand(1);
- }
-
- if (!Start->isReg() && !Start->isImm())
- return nullptr;
- if (!End->isReg() && !End->isImm())
- return nullptr;
-
- bool CmpLess = Cmp & Comparison::L;
- bool CmpGreater = Cmp & Comparison::G;
- bool CmpHasEqual = Cmp & Comparison::EQ;
-
- // Avoid certain wrap-arounds. This doesn't detect all wrap-arounds.
- if (CmpLess && IVBump < 0)
- // Loop going while iv is "less" with the iv value going down. Must wrap.
- return nullptr;
-
- if (CmpGreater && IVBump > 0)
- // Loop going while iv is "greater" with the iv value going up. Must wrap.
- return nullptr;
-
- // Phis that may feed into the loop.
- LoopFeederMap LoopFeederPhi;
-
- // Check if the initial value may be zero and can be decremented in the first
- // iteration. If the value is zero, the endloop instruction will not decrement
- // the loop counter, so we shouldn't generate a hardware loop in this case.
- if (loopCountMayWrapOrUnderFlow(Start, End, Loop->getLoopPreheader(), Loop,
- LoopFeederPhi))
- return nullptr;
-
- if (Start->isImm() && End->isImm()) {
- // Both, start and end are immediates.
- int64_t StartV = Start->getImm();
- int64_t EndV = End->getImm();
- int64_t Dist = EndV - StartV;
- if (Dist == 0)
- return nullptr;
-
- bool Exact = (Dist % IVBump) == 0;
-
- if (Cmp == Comparison::NE) {
- if (!Exact)
- return nullptr;
- if ((Dist < 0) ^ (IVBump < 0))
- return nullptr;
- }
-
- // For comparisons that include the final value (i.e. include equality
- // with the final value), we need to increase the distance by 1.
- if (CmpHasEqual)
- Dist = Dist > 0 ? Dist+1 : Dist-1;
-
- // For the loop to iterate, CmpLess should imply Dist > 0. Similarly,
- // CmpGreater should imply Dist < 0. These conditions could actually
- // fail, for example, in unreachable code (which may still appear to be
- // reachable in the CFG).
- if ((CmpLess && Dist < 0) || (CmpGreater && Dist > 0))
- return nullptr;
-
- // "Normalized" distance, i.e. with the bump set to +-1.
- int64_t Dist1 = (IVBump > 0) ? (Dist + (IVBump - 1)) / IVBump
- : (-Dist + (-IVBump - 1)) / (-IVBump);
- assert (Dist1 > 0 && "Fishy thing. Both operands have the same sign.");
-
- uint64_t Count = Dist1;
-
- if (Count > 0xFFFFFFFFULL)
- return nullptr;
-
- return new CountValue(CountValue::CV_Immediate, Count);
- }
-
- // A general case: Start and End are some values, but the actual
- // iteration count may not be available. If it is not, insert
- // a computation of it into the preheader.
-
- // If the induction variable bump is not a power of 2, quit.
- // Othwerise we'd need a general integer division.
- if (!isPowerOf2_64(std::abs(IVBump)))
- return nullptr;
-
- MachineBasicBlock *PH = MLI->findLoopPreheader(Loop, SpecPreheader);
- assert (PH && "Should have a preheader by now");
- MachineBasicBlock::iterator InsertPos = PH->getFirstTerminator();
- DebugLoc DL;
- if (InsertPos != PH->end())
- DL = InsertPos->getDebugLoc();
-
- // If Start is an immediate and End is a register, the trip count
- // will be "reg - imm". Hexagon's "subtract immediate" instruction
- // is actually "reg + -imm".
-
- // If the loop IV is going downwards, i.e. if the bump is negative,
- // then the iteration count (computed as End-Start) will need to be
- // negated. To avoid the negation, just swap Start and End.
- if (IVBump < 0) {
- std::swap(Start, End);
- IVBump = -IVBump;
- }
- // Cmp may now have a wrong direction, e.g. LEs may now be GEs.
- // Signedness, and "including equality" are preserved.
-
- bool RegToImm = Start->isReg() && End->isImm(); // for (reg..imm)
- bool RegToReg = Start->isReg() && End->isReg(); // for (reg..reg)
-
- int64_t StartV = 0, EndV = 0;
- if (Start->isImm())
- StartV = Start->getImm();
- if (End->isImm())
- EndV = End->getImm();
-
- int64_t AdjV = 0;
- // To compute the iteration count, we would need this computation:
- // Count = (End - Start + (IVBump-1)) / IVBump
- // or, when CmpHasEqual:
- // Count = (End - Start + (IVBump-1)+1) / IVBump
- // The "IVBump-1" part is the adjustment (AdjV). We can avoid
- // generating an instruction specifically to add it if we can adjust
- // the immediate values for Start or End.
-
- if (CmpHasEqual) {
- // Need to add 1 to the total iteration count.
- if (Start->isImm())
- StartV--;
- else if (End->isImm())
- EndV++;
- else
- AdjV += 1;
- }
-
- if (Cmp != Comparison::NE) {
- if (Start->isImm())
- StartV -= (IVBump-1);
- else if (End->isImm())
- EndV += (IVBump-1);
- else
- AdjV += (IVBump-1);
- }
-
- unsigned R = 0, SR = 0;
- if (Start->isReg()) {
- R = Start->getReg();
- SR = Start->getSubReg();
- } else {
- R = End->getReg();
- SR = End->getSubReg();
- }
- const TargetRegisterClass *RC = MRI->getRegClass(R);
- // Hardware loops cannot handle 64-bit registers. If it's a double
- // register, it has to have a subregister.
- if (!SR && RC == &Hexagon::DoubleRegsRegClass)
- return nullptr;
- const TargetRegisterClass *IntRC = &Hexagon::IntRegsRegClass;
-
- // Compute DistR (register with the distance between Start and End).
- unsigned DistR, DistSR;
-
- // Avoid special case, where the start value is an imm(0).
- if (Start->isImm() && StartV == 0) {
- DistR = End->getReg();
- DistSR = End->getSubReg();
- } else {
- const MCInstrDesc &SubD = RegToReg ? TII->get(Hexagon::A2_sub) :
- (RegToImm ? TII->get(Hexagon::A2_subri) :
- TII->get(Hexagon::A2_addi));
- if (RegToReg || RegToImm) {
- unsigned SubR = MRI->createVirtualRegister(IntRC);
- MachineInstrBuilder SubIB =
- BuildMI(*PH, InsertPos, DL, SubD, SubR);
-
- if (RegToReg)
- SubIB.addReg(End->getReg(), 0, End->getSubReg())
- .addReg(Start->getReg(), 0, Start->getSubReg());
- else
- SubIB.addImm(EndV)
- .addReg(Start->getReg(), 0, Start->getSubReg());
- DistR = SubR;
- } else {
- // If the loop has been unrolled, we should use the original loop count
- // instead of recalculating the value. This will avoid additional
- // 'Add' instruction.
- const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg());
- if (EndValInstr->getOpcode() == Hexagon::A2_addi &&
- EndValInstr->getOperand(1).getSubReg() == 0 &&
- EndValInstr->getOperand(2).getImm() == StartV) {
- DistR = EndValInstr->getOperand(1).getReg();
- } else {
- unsigned SubR = MRI->createVirtualRegister(IntRC);
- MachineInstrBuilder SubIB =
- BuildMI(*PH, InsertPos, DL, SubD, SubR);
- SubIB.addReg(End->getReg(), 0, End->getSubReg())
- .addImm(-StartV);
- DistR = SubR;
- }
- }
- DistSR = 0;
- }
-
- // From DistR, compute AdjR (register with the adjusted distance).
- unsigned AdjR, AdjSR;
-
- if (AdjV == 0) {
- AdjR = DistR;
- AdjSR = DistSR;
- } else {
- // Generate CountR = ADD DistR, AdjVal
- unsigned AddR = MRI->createVirtualRegister(IntRC);
- MCInstrDesc const &AddD = TII->get(Hexagon::A2_addi);
- BuildMI(*PH, InsertPos, DL, AddD, AddR)
- .addReg(DistR, 0, DistSR)
- .addImm(AdjV);
-
- AdjR = AddR;
- AdjSR = 0;
- }
-
- // From AdjR, compute CountR (register with the final count).
- unsigned CountR, CountSR;
-
- if (IVBump == 1) {
- CountR = AdjR;
- CountSR = AdjSR;
- } else {
- // The IV bump is a power of two. Log_2(IV bump) is the shift amount.
- unsigned Shift = Log2_32(IVBump);
-
- // Generate NormR = LSR DistR, Shift.
- unsigned LsrR = MRI->createVirtualRegister(IntRC);
- const MCInstrDesc &LsrD = TII->get(Hexagon::S2_lsr_i_r);
- BuildMI(*PH, InsertPos, DL, LsrD, LsrR)
- .addReg(AdjR, 0, AdjSR)
- .addImm(Shift);
-
- CountR = LsrR;
- CountSR = 0;
- }
-
- return new CountValue(CountValue::CV_Register, CountR, CountSR);
-}
-
-/// Return true if the operation is invalid within hardware loop.
-bool HexagonHardwareLoops::isInvalidLoopOperation(const MachineInstr *MI,
- bool IsInnerHWLoop) const {
- // Call is not allowed because the callee may use a hardware loop except for
- // the case when the call never returns.
- if (MI->getDesc().isCall())
- return !TII->doesNotReturn(*MI);
-
- // Check if the instruction defines a hardware loop register.
- using namespace Hexagon;
-
- static const unsigned Regs01[] = { LC0, SA0, LC1, SA1 };
- static const unsigned Regs1[] = { LC1, SA1 };
- auto CheckRegs = IsInnerHWLoop ? makeArrayRef(Regs01, array_lengthof(Regs01))
- : makeArrayRef(Regs1, array_lengthof(Regs1));
- for (unsigned R : CheckRegs)
- if (MI->modifiesRegister(R, TRI))
- return true;
-
- return false;
-}
-
-/// Return true if the loop contains an instruction that inhibits
-/// the use of the hardware loop instruction.
-bool HexagonHardwareLoops::containsInvalidInstruction(MachineLoop *L,
- bool IsInnerHWLoop) const {
- LLVM_DEBUG(dbgs() << "\nhw_loop head, "
- << printMBBReference(**L->block_begin()));
- for (MachineBasicBlock *MBB : L->getBlocks()) {
- for (MachineBasicBlock::iterator
- MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) {
- const MachineInstr *MI = &*MII;
- if (isInvalidLoopOperation(MI, IsInnerHWLoop)) {
- LLVM_DEBUG(dbgs() << "\nCannot convert to hw_loop due to:";
- MI->dump(););
- return true;
- }
- }
- }
- return false;
-}
-
-/// Returns true if the instruction is dead. This was essentially
-/// copied from DeadMachineInstructionElim::isDead, but with special cases
-/// for inline asm, physical registers and instructions with side effects
-/// removed.
-bool HexagonHardwareLoops::isDead(const MachineInstr *MI,
- SmallVectorImpl<MachineInstr *> &DeadPhis) const {
- // Examine each operand.
- for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
- const MachineOperand &MO = MI->getOperand(i);
- if (!MO.isReg() || !MO.isDef())
- continue;
-
- unsigned Reg = MO.getReg();
- if (MRI->use_nodbg_empty(Reg))
- continue;
-
- using use_nodbg_iterator = MachineRegisterInfo::use_nodbg_iterator;
-
- // This instruction has users, but if the only user is the phi node for the
- // parent block, and the only use of that phi node is this instruction, then
- // this instruction is dead: both it (and the phi node) can be removed.
- use_nodbg_iterator I = MRI->use_nodbg_begin(Reg);
- use_nodbg_iterator End = MRI->use_nodbg_end();
- if (std::next(I) != End || !I->getParent()->isPHI())
- return false;
-
- MachineInstr *OnePhi = I->getParent();
- for (unsigned j = 0, f = OnePhi->getNumOperands(); j != f; ++j) {
- const MachineOperand &OPO = OnePhi->getOperand(j);
- if (!OPO.isReg() || !OPO.isDef())
- continue;
-
- unsigned OPReg = OPO.getReg();
- use_nodbg_iterator nextJ;
- for (use_nodbg_iterator J = MRI->use_nodbg_begin(OPReg);
- J != End; J = nextJ) {
- nextJ = std::next(J);
- MachineOperand &Use = *J;
- MachineInstr *UseMI = Use.getParent();
-
- // If the phi node has a user that is not MI, bail.
- if (MI != UseMI)
- return false;
- }
- }
- DeadPhis.push_back(OnePhi);
- }
-
- // If there are no defs with uses, the instruction is dead.
- return true;
-}
-
-void HexagonHardwareLoops::removeIfDead(MachineInstr *MI) {
- // This procedure was essentially copied from DeadMachineInstructionElim.
-
- SmallVector<MachineInstr*, 1> DeadPhis;
- if (isDead(MI, DeadPhis)) {
- LLVM_DEBUG(dbgs() << "HW looping will remove: " << *MI);
-
- // It is possible that some DBG_VALUE instructions refer to this
- // instruction. Examine each def operand for such references;
- // if found, mark the DBG_VALUE as undef (but don't delete it).
- for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
- const MachineOperand &MO = MI->getOperand(i);
- if (!MO.isReg() || !MO.isDef())
- continue;
- unsigned Reg = MO.getReg();
- MachineRegisterInfo::use_iterator nextI;
- for (MachineRegisterInfo::use_iterator I = MRI->use_begin(Reg),
- E = MRI->use_end(); I != E; I = nextI) {
- nextI = std::next(I); // I is invalidated by the setReg
- MachineOperand &Use = *I;
- MachineInstr *UseMI = I->getParent();
- if (UseMI == MI)
- continue;
- if (Use.isDebug())
- UseMI->getOperand(0).setReg(0U);
- }
- }
-
- MI->eraseFromParent();
- for (unsigned i = 0; i < DeadPhis.size(); ++i)
- DeadPhis[i]->eraseFromParent();
- }
-}
-
-/// Check if the loop is a candidate for converting to a hardware
-/// loop. If so, then perform the transformation.
-///
-/// This function works on innermost loops first. A loop can be converted
-/// if it is a counting loop; either a register value or an immediate.
-///
-/// The code makes several assumptions about the representation of the loop
-/// in llvm.
-bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L,
- bool &RecL0used,
- bool &RecL1used) {
- // This is just for sanity.
- assert(L->getHeader() && "Loop without a header?");
-
- bool Changed = false;
- bool L0Used = false;
- bool L1Used = false;
-
- // Process nested loops first.
- for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) {
- Changed |= convertToHardwareLoop(*I, RecL0used, RecL1used);
- L0Used |= RecL0used;
- L1Used |= RecL1used;
- }
-
- // If a nested loop has been converted, then we can't convert this loop.
- if (Changed && L0Used && L1Used)
- return Changed;
-
- unsigned LOOP_i;
- unsigned LOOP_r;
- unsigned ENDLOOP;
-
- // Flag used to track loopN instruction:
- // 1 - Hardware loop is being generated for the inner most loop.
- // 0 - Hardware loop is being generated for the outer loop.
- unsigned IsInnerHWLoop = 1;
-
- if (L0Used) {
- LOOP_i = Hexagon::J2_loop1i;
- LOOP_r = Hexagon::J2_loop1r;
- ENDLOOP = Hexagon::ENDLOOP1;
- IsInnerHWLoop = 0;
- } else {
- LOOP_i = Hexagon::J2_loop0i;
- LOOP_r = Hexagon::J2_loop0r;
- ENDLOOP = Hexagon::ENDLOOP0;
- }
-
-#ifndef NDEBUG
- // Stop trying after reaching the limit (if any).
- int Limit = HWLoopLimit;
- if (Limit >= 0) {
- if (Counter >= HWLoopLimit)
- return false;
- Counter++;
- }
-#endif
-
- // Does the loop contain any invalid instructions?
- if (containsInvalidInstruction(L, IsInnerHWLoop))
- return false;
-
- MachineBasicBlock *LastMBB = L->findLoopControlBlock();
- // Don't generate hw loop if the loop has more than one exit.
- if (!LastMBB)
- return false;
-
- MachineBasicBlock::iterator LastI = LastMBB->getFirstTerminator();
- if (LastI == LastMBB->end())
- return false;
-
- // Is the induction variable bump feeding the latch condition?
- if (!fixupInductionVariable(L))
- return false;
-
- // Ensure the loop has a preheader: the loop instruction will be
- // placed there.
- MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader);
- if (!Preheader) {
- Preheader = createPreheaderForLoop(L);
- if (!Preheader)
- return false;
- }
-
- MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator();
-
- SmallVector<MachineInstr*, 2> OldInsts;
- // Are we able to determine the trip count for the loop?
- CountValue *TripCount = getLoopTripCount(L, OldInsts);
- if (!TripCount)
- return false;
-
- // Is the trip count available in the preheader?
- if (TripCount->isReg()) {
- // There will be a use of the register inserted into the preheader,
- // so make sure that the register is actually defined at that point.
- MachineInstr *TCDef = MRI->getVRegDef(TripCount->getReg());
- MachineBasicBlock *BBDef = TCDef->getParent();
- if (!MDT->dominates(BBDef, Preheader))
- return false;
- }
-
- // Determine the loop start.
- MachineBasicBlock *TopBlock = L->getTopBlock();
- MachineBasicBlock *ExitingBlock = L->findLoopControlBlock();
- MachineBasicBlock *LoopStart = nullptr;
- if (ExitingBlock != L->getLoopLatch()) {
- MachineBasicBlock *TB = nullptr, *FB = nullptr;
- SmallVector<MachineOperand, 2> Cond;
-
- if (TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false))
- return false;
-
- if (L->contains(TB))
- LoopStart = TB;
- else if (L->contains(FB))
- LoopStart = FB;
- else
- return false;
- }
- else
- LoopStart = TopBlock;
-
- // Convert the loop to a hardware loop.
- LLVM_DEBUG(dbgs() << "Change to hardware loop at "; L->dump());
- DebugLoc DL;
- if (InsertPos != Preheader->end())
- DL = InsertPos->getDebugLoc();
-
- if (TripCount->isReg()) {
- // Create a copy of the loop count register.
- unsigned CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass);
- BuildMI(*Preheader, InsertPos, DL, TII->get(TargetOpcode::COPY), CountReg)
- .addReg(TripCount->getReg(), 0, TripCount->getSubReg());
- // Add the Loop instruction to the beginning of the loop.
- BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r)).addMBB(LoopStart)
- .addReg(CountReg);
- } else {
- assert(TripCount->isImm() && "Expecting immediate value for trip count");
- // Add the Loop immediate instruction to the beginning of the loop,
- // if the immediate fits in the instructions. Otherwise, we need to
- // create a new virtual register.
- int64_t CountImm = TripCount->getImm();
- if (!TII->isValidOffset(LOOP_i, CountImm, TRI)) {
- unsigned CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass);
- BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::A2_tfrsi), CountReg)
- .addImm(CountImm);
- BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r))
- .addMBB(LoopStart).addReg(CountReg);
- } else
- BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_i))
- .addMBB(LoopStart).addImm(CountImm);
- }
-
- // Make sure the loop start always has a reference in the CFG. We need
- // to create a BlockAddress operand to get this mechanism to work both the
- // MachineBasicBlock and BasicBlock objects need the flag set.
- LoopStart->setHasAddressTaken();
- // This line is needed to set the hasAddressTaken flag on the BasicBlock
- // object.
- BlockAddress::get(const_cast<BasicBlock *>(LoopStart->getBasicBlock()));
-
- // Replace the loop branch with an endloop instruction.
- DebugLoc LastIDL = LastI->getDebugLoc();
- BuildMI(*LastMBB, LastI, LastIDL, TII->get(ENDLOOP)).addMBB(LoopStart);
-
- // The loop ends with either:
- // - a conditional branch followed by an unconditional branch, or
- // - a conditional branch to the loop start.
- if (LastI->getOpcode() == Hexagon::J2_jumpt ||
- LastI->getOpcode() == Hexagon::J2_jumpf) {
- // Delete one and change/add an uncond. branch to out of the loop.
- MachineBasicBlock *BranchTarget = LastI->getOperand(1).getMBB();
- LastI = LastMBB->erase(LastI);
- if (!L->contains(BranchTarget)) {
- if (LastI != LastMBB->end())
- LastI = LastMBB->erase(LastI);
- SmallVector<MachineOperand, 0> Cond;
- TII->insertBranch(*LastMBB, BranchTarget, nullptr, Cond, LastIDL);
- }
- } else {
- // Conditional branch to loop start; just delete it.
- LastMBB->erase(LastI);
- }
- delete TripCount;
-
- // The induction operation and the comparison may now be
- // unneeded. If these are unneeded, then remove them.
- for (unsigned i = 0; i < OldInsts.size(); ++i)
- removeIfDead(OldInsts[i]);
-
- ++NumHWLoops;
-
- // Set RecL1used and RecL0used only after hardware loop has been
- // successfully generated. Doing it earlier can cause wrong loop instruction
- // to be used.
- if (L0Used) // Loop0 was already used. So, the correct loop must be loop1.
- RecL1used = true;
- else
- RecL0used = true;
-
- return true;
-}
-
-bool HexagonHardwareLoops::orderBumpCompare(MachineInstr *BumpI,
- MachineInstr *CmpI) {
- assert (BumpI != CmpI && "Bump and compare in the same instruction?");
-
- MachineBasicBlock *BB = BumpI->getParent();
- if (CmpI->getParent() != BB)
- return false;
-
- using instr_iterator = MachineBasicBlock::instr_iterator;
-
- // Check if things are in order to begin with.
- for (instr_iterator I(BumpI), E = BB->instr_end(); I != E; ++I)
- if (&*I == CmpI)
- return true;
-
- // Out of order.
- unsigned PredR = CmpI->getOperand(0).getReg();
- bool FoundBump = false;
- instr_iterator CmpIt = CmpI->getIterator(), NextIt = std::next(CmpIt);
- for (instr_iterator I = NextIt, E = BB->instr_end(); I != E; ++I) {
- MachineInstr *In = &*I;
- for (unsigned i = 0, n = In->getNumOperands(); i < n; ++i) {
- MachineOperand &MO = In->getOperand(i);
- if (MO.isReg() && MO.isUse()) {
- if (MO.getReg() == PredR) // Found an intervening use of PredR.
- return false;
- }
- }
-
- if (In == BumpI) {
- BB->splice(++BumpI->getIterator(), BB, CmpI->getIterator());
- FoundBump = true;
- break;
- }
- }
- assert (FoundBump && "Cannot determine instruction order");
- return FoundBump;
-}
-
-/// This function is required to break recursion. Visiting phis in a loop may
-/// result in recursion during compilation. We break the recursion by making
-/// sure that we visit a MachineOperand and its definition in a
-/// MachineInstruction only once. If we attempt to visit more than once, then
-/// there is recursion, and will return false.
-bool HexagonHardwareLoops::isLoopFeeder(MachineLoop *L, MachineBasicBlock *A,
- MachineInstr *MI,
- const MachineOperand *MO,
- LoopFeederMap &LoopFeederPhi) const {
- if (LoopFeederPhi.find(MO->getReg()) == LoopFeederPhi.end()) {
- LLVM_DEBUG(dbgs() << "\nhw_loop head, "
- << printMBBReference(**L->block_begin()));
- // Ignore all BBs that form Loop.
- for (MachineBasicBlock *MBB : L->getBlocks()) {
- if (A == MBB)
- return false;
- }
- MachineInstr *Def = MRI->getVRegDef(MO->getReg());
- LoopFeederPhi.insert(std::make_pair(MO->getReg(), Def));
- return true;
- } else
- // Already visited node.
- return false;
-}
-
-/// Return true if a Phi may generate a value that can underflow.
-/// This function calls loopCountMayWrapOrUnderFlow for each Phi operand.
-bool HexagonHardwareLoops::phiMayWrapOrUnderflow(
- MachineInstr *Phi, const MachineOperand *EndVal, MachineBasicBlock *MBB,
- MachineLoop *L, LoopFeederMap &LoopFeederPhi) const {
- assert(Phi->isPHI() && "Expecting a Phi.");
- // Walk through each Phi, and its used operands. Make sure that
- // if there is recursion in Phi, we won't generate hardware loops.
- for (int i = 1, n = Phi->getNumOperands(); i < n; i += 2)
- if (isLoopFeeder(L, MBB, Phi, &(Phi->getOperand(i)), LoopFeederPhi))
- if (loopCountMayWrapOrUnderFlow(&(Phi->getOperand(i)), EndVal,
- Phi->getParent(), L, LoopFeederPhi))
- return true;
- return false;
-}
-
-/// Return true if the induction variable can underflow in the first iteration.
-/// An example, is an initial unsigned value that is 0 and is decrement in the
-/// first itertion of a do-while loop. In this case, we cannot generate a
-/// hardware loop because the endloop instruction does not decrement the loop
-/// counter if it is <= 1. We only need to perform this analysis if the
-/// initial value is a register.
-///
-/// This function assumes the initial value may underfow unless proven
-/// otherwise. If the type is signed, then we don't care because signed
-/// underflow is undefined. We attempt to prove the initial value is not
-/// zero by perfoming a crude analysis of the loop counter. This function
-/// checks if the initial value is used in any comparison prior to the loop
-/// and, if so, assumes the comparison is a range check. This is inexact,
-/// but will catch the simple cases.
-bool HexagonHardwareLoops::loopCountMayWrapOrUnderFlow(
- const MachineOperand *InitVal, const MachineOperand *EndVal,
- MachineBasicBlock *MBB, MachineLoop *L,
- LoopFeederMap &LoopFeederPhi) const {
- // Only check register values since they are unknown.
- if (!InitVal->isReg())
- return false;
-
- if (!EndVal->isImm())
- return false;
-
- // A register value that is assigned an immediate is a known value, and it
- // won't underflow in the first iteration.
- int64_t Imm;
- if (checkForImmediate(*InitVal, Imm))
- return (EndVal->getImm() == Imm);
-
- unsigned Reg = InitVal->getReg();
-
- // We don't know the value of a physical register.
- if (!TargetRegisterInfo::isVirtualRegister(Reg))
- return true;
-
- MachineInstr *Def = MRI->getVRegDef(Reg);
- if (!Def)
- return true;
-
- // If the initial value is a Phi or copy and the operands may not underflow,
- // then the definition cannot be underflow either.
- if (Def->isPHI() && !phiMayWrapOrUnderflow(Def, EndVal, Def->getParent(),
- L, LoopFeederPhi))
- return false;
- if (Def->isCopy() && !loopCountMayWrapOrUnderFlow(&(Def->getOperand(1)),
- EndVal, Def->getParent(),
- L, LoopFeederPhi))
- return false;
-
- // Iterate over the uses of the initial value. If the initial value is used
- // in a compare, then we assume this is a range check that ensures the loop
- // doesn't underflow. This is not an exact test and should be improved.
- for (MachineRegisterInfo::use_instr_nodbg_iterator I = MRI->use_instr_nodbg_begin(Reg),
- E = MRI->use_instr_nodbg_end(); I != E; ++I) {
- MachineInstr *MI = &*I;
- unsigned CmpReg1 = 0, CmpReg2 = 0;
- int CmpMask = 0, CmpValue = 0;
-
- if (!TII->analyzeCompare(*MI, CmpReg1, CmpReg2, CmpMask, CmpValue))
- continue;
-
- MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
- SmallVector<MachineOperand, 2> Cond;
- if (TII->analyzeBranch(*MI->getParent(), TBB, FBB, Cond, false))
- continue;
-
- Comparison::Kind Cmp =
- getComparisonKind(MI->getOpcode(), nullptr, nullptr, 0);
- if (Cmp == 0)
- continue;
- if (TII->predOpcodeHasNot(Cond) ^ (TBB != MBB))
- Cmp = Comparison::getNegatedComparison(Cmp);
- if (CmpReg2 != 0 && CmpReg2 == Reg)
- Cmp = Comparison::getSwappedComparison(Cmp);
-
- // Signed underflow is undefined.
- if (Comparison::isSigned(Cmp))
- return false;
-
- // Check if there is a comparison of the initial value. If the initial value
- // is greater than or not equal to another value, then assume this is a
- // range check.
- if ((Cmp & Comparison::G) || Cmp == Comparison::NE)
- return false;
- }
-
- // OK - this is a hack that needs to be improved. We really need to analyze
- // the instructions performed on the initial value. This works on the simplest
- // cases only.
- if (!Def->isCopy() && !Def->isPHI())
- return false;
-
- return true;
-}
-
-bool HexagonHardwareLoops::checkForImmediate(const MachineOperand &MO,
- int64_t &Val) const {
- if (MO.isImm()) {
- Val = MO.getImm();
- return true;
- }
- if (!MO.isReg())
- return false;
-
- // MO is a register. Check whether it is defined as an immediate value,
- // and if so, get the value of it in TV. That value will then need to be
- // processed to handle potential subregisters in MO.
- int64_t TV;
-
- unsigned R = MO.getReg();
- if (!TargetRegisterInfo::isVirtualRegister(R))
- return false;
- MachineInstr *DI = MRI->getVRegDef(R);
- unsigned DOpc = DI->getOpcode();
- switch (DOpc) {
- case TargetOpcode::COPY:
- case Hexagon::A2_tfrsi:
- case Hexagon::A2_tfrpi:
- case Hexagon::CONST32:
- case Hexagon::CONST64:
- // Call recursively to avoid an extra check whether operand(1) is
- // indeed an immediate (it could be a global address, for example),
- // plus we can handle COPY at the same time.
- if (!checkForImmediate(DI->getOperand(1), TV))
- return false;
- break;
- case Hexagon::A2_combineii:
- case Hexagon::A4_combineir:
- case Hexagon::A4_combineii:
- case Hexagon::A4_combineri:
- case Hexagon::A2_combinew: {
- const MachineOperand &S1 = DI->getOperand(1);
- const MachineOperand &S2 = DI->getOperand(2);
- int64_t V1, V2;
- if (!checkForImmediate(S1, V1) || !checkForImmediate(S2, V2))
- return false;
- TV = V2 | (static_cast<uint64_t>(V1) << 32);
- break;
- }
- case TargetOpcode::REG_SEQUENCE: {
- const MachineOperand &S1 = DI->getOperand(1);
- const MachineOperand &S3 = DI->getOperand(3);
- int64_t V1, V3;
- if (!checkForImmediate(S1, V1) || !checkForImmediate(S3, V3))
- return false;
- unsigned Sub2 = DI->getOperand(2).getImm();
- unsigned Sub4 = DI->getOperand(4).getImm();
- if (Sub2 == Hexagon::isub_lo && Sub4 == Hexagon::isub_hi)
- TV = V1 | (V3 << 32);
- else if (Sub2 == Hexagon::isub_hi && Sub4 == Hexagon::isub_lo)
- TV = V3 | (V1 << 32);
- else
- llvm_unreachable("Unexpected form of REG_SEQUENCE");
- break;
- }
-
- default:
- return false;
- }
-
- // By now, we should have successfully obtained the immediate value defining
- // the register referenced in MO. Handle a potential use of a subregister.
- switch (MO.getSubReg()) {
- case Hexagon::isub_lo:
- Val = TV & 0xFFFFFFFFULL;
- break;
- case Hexagon::isub_hi:
- Val = (TV >> 32) & 0xFFFFFFFFULL;
- break;
- default:
- Val = TV;
- break;
- }
- return true;
-}
-
-void HexagonHardwareLoops::setImmediate(MachineOperand &MO, int64_t Val) {
- if (MO.isImm()) {
- MO.setImm(Val);
- return;
- }
-
- assert(MO.isReg());
- unsigned R = MO.getReg();
- MachineInstr *DI = MRI->getVRegDef(R);
-
- const TargetRegisterClass *RC = MRI->getRegClass(R);
- unsigned NewR = MRI->createVirtualRegister(RC);
- MachineBasicBlock &B = *DI->getParent();
- DebugLoc DL = DI->getDebugLoc();
- BuildMI(B, DI, DL, TII->get(DI->getOpcode()), NewR).addImm(Val);
- MO.setReg(NewR);
-}
-
-static bool isImmValidForOpcode(unsigned CmpOpc, int64_t Imm) {
- // These two instructions are not extendable.
- if (CmpOpc == Hexagon::A4_cmpbeqi)
- return isUInt<8>(Imm);
- if (CmpOpc == Hexagon::A4_cmpbgti)
- return isInt<8>(Imm);
- // The rest of the comparison-with-immediate instructions are extendable.
- return true;
-}
-
-bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
- MachineBasicBlock *Header = L->getHeader();
- MachineBasicBlock *Latch = L->getLoopLatch();
- MachineBasicBlock *ExitingBlock = L->findLoopControlBlock();
-
- if (!(Header && Latch && ExitingBlock))
- return false;
-
- // These data structures follow the same concept as the corresponding
- // ones in findInductionRegister (where some comments are).
- using RegisterBump = std::pair<unsigned, int64_t>;
- using RegisterInduction = std::pair<unsigned, RegisterBump>;
- using RegisterInductionSet = std::set<RegisterInduction>;
-
- // Register candidates for induction variables, with their associated bumps.
- RegisterInductionSet IndRegs;
-
- // Look for induction patterns:
- // %1 = PHI ..., [ latch, %2 ]
- // %2 = ADD %1, imm
- using instr_iterator = MachineBasicBlock::instr_iterator;
-
- for (instr_iterator I = Header->instr_begin(), E = Header->instr_end();
- I != E && I->isPHI(); ++I) {
- MachineInstr *Phi = &*I;
-
- // Have a PHI instruction.
- for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) {
- if (Phi->getOperand(i+1).getMBB() != Latch)
- continue;
-
- unsigned PhiReg = Phi->getOperand(i).getReg();
- MachineInstr *DI = MRI->getVRegDef(PhiReg);
-
- if (DI->getDesc().isAdd()) {
- // If the register operand to the add/sub is the PHI we are looking
- // at, this meets the induction pattern.
- unsigned IndReg = DI->getOperand(1).getReg();
- MachineOperand &Opnd2 = DI->getOperand(2);
- int64_t V;
- if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) {
- unsigned UpdReg = DI->getOperand(0).getReg();
- IndRegs.insert(std::make_pair(UpdReg, std::make_pair(IndReg, V)));
- }
- }
- } // for (i)
- } // for (instr)
-
- if (IndRegs.empty())
- return false;
-
- MachineBasicBlock *TB = nullptr, *FB = nullptr;
- SmallVector<MachineOperand,2> Cond;
- // AnalyzeBranch returns true if it fails to analyze branch.
- bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false);
- if (NotAnalyzed || Cond.empty())
- return false;
-
- if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) {
- MachineBasicBlock *LTB = nullptr, *LFB = nullptr;
- SmallVector<MachineOperand,2> LCond;
- bool NotAnalyzed = TII->analyzeBranch(*Latch, LTB, LFB, LCond, false);
- if (NotAnalyzed)
- return false;
-
- // Since latch is not the exiting block, the latch branch should be an
- // unconditional branch to the loop header.
- if (TB == Latch)
- TB = (LTB == Header) ? LTB : LFB;
- else
- FB = (LTB == Header) ? LTB : LFB;
- }
- if (TB != Header) {
- if (FB != Header) {
- // The latch/exit block does not go back to the header.
- return false;
- }
- // FB is the header (i.e., uncond. jump to branch header)
- // In this case, the LoopBody -> TB should not be a back edge otherwise
- // it could result in an infinite loop after conversion to hw_loop.
- // This case can happen when the Latch has two jumps like this:
- // Jmp_c OuterLoopHeader <-- TB
- // Jmp InnerLoopHeader <-- FB
- if (MDT->dominates(TB, FB))
- return false;
- }
-
- // Expecting a predicate register as a condition. It won't be a hardware
- // predicate register at this point yet, just a vreg.
- // HexagonInstrInfo::AnalyzeBranch for negated branches inserts imm(0)
- // into Cond, followed by the predicate register. For non-negated branches
- // it's just the register.
- unsigned CSz = Cond.size();
- if (CSz != 1 && CSz != 2)
- return false;
-
- if (!Cond[CSz-1].isReg())
- return false;
-
- unsigned P = Cond[CSz-1].getReg();
- MachineInstr *PredDef = MRI->getVRegDef(P);
-
- if (!PredDef->isCompare())
- return false;
-
- SmallSet<unsigned,2> CmpRegs;
- MachineOperand *CmpImmOp = nullptr;
-
- // Go over all operands to the compare and look for immediate and register
- // operands. Assume that if the compare has a single register use and a
- // single immediate operand, then the register is being compared with the
- // immediate value.
- for (unsigned i = 0, n = PredDef->getNumOperands(); i < n; ++i) {
- MachineOperand &MO = PredDef->getOperand(i);
- if (MO.isReg()) {
- // Skip all implicit references. In one case there was:
- // %140 = FCMPUGT32_rr %138, %139, implicit %usr
- if (MO.isImplicit())
- continue;
- if (MO.isUse()) {
- if (!isImmediate(MO)) {
- CmpRegs.insert(MO.getReg());
- continue;
- }
- // Consider the register to be the "immediate" operand.
- if (CmpImmOp)
- return false;
- CmpImmOp = &MO;
- }
- } else if (MO.isImm()) {
- if (CmpImmOp) // A second immediate argument? Confusing. Bail out.
- return false;
- CmpImmOp = &MO;
- }
- }
-
- if (CmpRegs.empty())
- return false;
-
- // Check if the compared register follows the order we want. Fix if needed.
- for (RegisterInductionSet::iterator I = IndRegs.begin(), E = IndRegs.end();
- I != E; ++I) {
- // This is a success. If the register used in the comparison is one that
- // we have identified as a bumped (updated) induction register, there is
- // nothing to do.
- if (CmpRegs.count(I->first))
- return true;
-
- // Otherwise, if the register being compared comes out of a PHI node,
- // and has been recognized as following the induction pattern, and is
- // compared against an immediate, we can fix it.
- const RegisterBump &RB = I->second;
- if (CmpRegs.count(RB.first)) {
- if (!CmpImmOp) {
- // If both operands to the compare instruction are registers, see if
- // it can be changed to use induction register as one of the operands.
- MachineInstr *IndI = nullptr;
- MachineInstr *nonIndI = nullptr;
- MachineOperand *IndMO = nullptr;
- MachineOperand *nonIndMO = nullptr;
-
- for (unsigned i = 1, n = PredDef->getNumOperands(); i < n; ++i) {
- MachineOperand &MO = PredDef->getOperand(i);
- if (MO.isReg() && MO.getReg() == RB.first) {
- LLVM_DEBUG(dbgs() << "\n DefMI(" << i
- << ") = " << *(MRI->getVRegDef(I->first)));
- if (IndI)
- return false;
-
- IndI = MRI->getVRegDef(I->first);
- IndMO = &MO;
- } else if (MO.isReg()) {
- LLVM_DEBUG(dbgs() << "\n DefMI(" << i
- << ") = " << *(MRI->getVRegDef(MO.getReg())));
- if (nonIndI)
- return false;
-
- nonIndI = MRI->getVRegDef(MO.getReg());
- nonIndMO = &MO;
- }
- }
- if (IndI && nonIndI &&
- nonIndI->getOpcode() == Hexagon::A2_addi &&
- nonIndI->getOperand(2).isImm() &&
- nonIndI->getOperand(2).getImm() == - RB.second) {
- bool Order = orderBumpCompare(IndI, PredDef);
- if (Order) {
- IndMO->setReg(I->first);
- nonIndMO->setReg(nonIndI->getOperand(1).getReg());
- return true;
- }
- }
- return false;
- }
-
- // It is not valid to do this transformation on an unsigned comparison
- // because it may underflow.
- Comparison::Kind Cmp =
- getComparisonKind(PredDef->getOpcode(), nullptr, nullptr, 0);
- if (!Cmp || Comparison::isUnsigned(Cmp))
- return false;
-
- // If the register is being compared against an immediate, try changing
- // the compare instruction to use induction register and adjust the
- // immediate operand.
- int64_t CmpImm = getImmediate(*CmpImmOp);
- int64_t V = RB.second;
- // Handle Overflow (64-bit).
- if (((V > 0) && (CmpImm > INT64_MAX - V)) ||
- ((V < 0) && (CmpImm < INT64_MIN - V)))
- return false;
- CmpImm += V;
- // Most comparisons of register against an immediate value allow
- // the immediate to be constant-extended. There are some exceptions
- // though. Make sure the new combination will work.
- if (CmpImmOp->isImm())
- if (!isImmValidForOpcode(PredDef->getOpcode(), CmpImm))
- return false;
-
- // Make sure that the compare happens after the bump. Otherwise,
- // after the fixup, the compare would use a yet-undefined register.
- MachineInstr *BumpI = MRI->getVRegDef(I->first);
- bool Order = orderBumpCompare(BumpI, PredDef);
- if (!Order)
- return false;
-
- // Finally, fix the compare instruction.
- setImmediate(*CmpImmOp, CmpImm);
- for (unsigned i = 0, n = PredDef->getNumOperands(); i < n; ++i) {
- MachineOperand &MO = PredDef->getOperand(i);
- if (MO.isReg() && MO.getReg() == RB.first) {
- MO.setReg(I->first);
- return true;
- }
- }
- }
- }
-
- return false;
-}
-
-/// createPreheaderForLoop - Create a preheader for a given loop.
-MachineBasicBlock *HexagonHardwareLoops::createPreheaderForLoop(
- MachineLoop *L) {
- if (MachineBasicBlock *TmpPH = MLI->findLoopPreheader(L, SpecPreheader))
- return TmpPH;
- if (!HWCreatePreheader)
- return nullptr;
-
- MachineBasicBlock *Header = L->getHeader();
- MachineBasicBlock *Latch = L->getLoopLatch();
- MachineBasicBlock *ExitingBlock = L->findLoopControlBlock();
- MachineFunction *MF = Header->getParent();
- DebugLoc DL;
-
-#ifndef NDEBUG
- if ((!PHFn.empty()) && (PHFn != MF->getName()))
- return nullptr;
-#endif
-
- if (!Latch || !ExitingBlock || Header->hasAddressTaken())
- return nullptr;
-
- using instr_iterator = MachineBasicBlock::instr_iterator;
-
- // Verify that all existing predecessors have analyzable branches
- // (or no branches at all).
- using MBBVector = std::vector<MachineBasicBlock *>;
-
- MBBVector Preds(Header->pred_begin(), Header->pred_end());
- SmallVector<MachineOperand,2> Tmp1;
- MachineBasicBlock *TB = nullptr, *FB = nullptr;
-
- if (TII->analyzeBranch(*ExitingBlock, TB, FB, Tmp1, false))
- return nullptr;
-
- for (MBBVector::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) {
- MachineBasicBlock *PB = *I;
- bool NotAnalyzed = TII->analyzeBranch(*PB, TB, FB, Tmp1, false);
- if (NotAnalyzed)
- return nullptr;
- }
-
- MachineBasicBlock *NewPH = MF->CreateMachineBasicBlock();
- MF->insert(Header->getIterator(), NewPH);
-
- if (Header->pred_size() > 2) {
- // Ensure that the header has only two predecessors: the preheader and
- // the loop latch. Any additional predecessors of the header should
- // join at the newly created preheader. Inspect all PHI nodes from the
- // header and create appropriate corresponding PHI nodes in the preheader.
-
- for (instr_iterator I = Header->instr_begin(), E = Header->instr_end();
- I != E && I->isPHI(); ++I) {
- MachineInstr *PN = &*I;
-
- const MCInstrDesc &PD = TII->get(TargetOpcode::PHI);
- MachineInstr *NewPN = MF->CreateMachineInstr(PD, DL);
- NewPH->insert(NewPH->end(), NewPN);
-
- unsigned PR = PN->getOperand(0).getReg();
- const TargetRegisterClass *RC = MRI->getRegClass(PR);
- unsigned NewPR = MRI->createVirtualRegister(RC);
- NewPN->addOperand(MachineOperand::CreateReg(NewPR, true));
-
- // Copy all non-latch operands of a header's PHI node to the newly
- // created PHI node in the preheader.
- for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) {
- unsigned PredR = PN->getOperand(i).getReg();
- unsigned PredRSub = PN->getOperand(i).getSubReg();
- MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB();
- if (PredB == Latch)
- continue;
-
- MachineOperand MO = MachineOperand::CreateReg(PredR, false);
- MO.setSubReg(PredRSub);
- NewPN->addOperand(MO);
- NewPN->addOperand(MachineOperand::CreateMBB(PredB));
- }
-
- // Remove copied operands from the old PHI node and add the value
- // coming from the preheader's PHI.
- for (int i = PN->getNumOperands()-2; i > 0; i -= 2) {
- MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB();
- if (PredB != Latch) {
- PN->RemoveOperand(i+1);
- PN->RemoveOperand(i);
- }
- }
- PN->addOperand(MachineOperand::CreateReg(NewPR, false));
- PN->addOperand(MachineOperand::CreateMBB(NewPH));
- }
- } else {
- assert(Header->pred_size() == 2);
-
- // The header has only two predecessors, but the non-latch predecessor
- // is not a preheader (e.g. it has other successors, etc.)
- // In such a case we don't need any extra PHI nodes in the new preheader,
- // all we need is to adjust existing PHIs in the header to now refer to
- // the new preheader.
- for (instr_iterator I = Header->instr_begin(), E = Header->instr_end();
- I != E && I->isPHI(); ++I) {
- MachineInstr *PN = &*I;
- for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) {
- MachineOperand &MO = PN->getOperand(i+1);
- if (MO.getMBB() != Latch)
- MO.setMBB(NewPH);
- }
- }
- }
-
- // "Reroute" the CFG edges to link in the new preheader.
- // If any of the predecessors falls through to the header, insert a branch
- // to the new preheader in that place.
- SmallVector<MachineOperand,1> Tmp2;
- SmallVector<MachineOperand,1> EmptyCond;
-
- TB = FB = nullptr;
-
- for (MBBVector::iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) {
- MachineBasicBlock *PB = *I;
- if (PB != Latch) {
- Tmp2.clear();
- bool NotAnalyzed = TII->analyzeBranch(*PB, TB, FB, Tmp2, false);
- (void)NotAnalyzed; // suppress compiler warning
- assert (!NotAnalyzed && "Should be analyzable!");
- if (TB != Header && (Tmp2.empty() || FB != Header))
- TII->insertBranch(*PB, NewPH, nullptr, EmptyCond, DL);
- PB->ReplaceUsesOfBlockWith(Header, NewPH);
- }
- }
-
- // It can happen that the latch block will fall through into the header.
- // Insert an unconditional branch to the header.
- TB = FB = nullptr;
- bool LatchNotAnalyzed = TII->analyzeBranch(*Latch, TB, FB, Tmp2, false);
- (void)LatchNotAnalyzed; // suppress compiler warning
- assert (!LatchNotAnalyzed && "Should be analyzable!");
- if (!TB && !FB)
- TII->insertBranch(*Latch, Header, nullptr, EmptyCond, DL);
-
- // Finally, the branch from the preheader to the header.
- TII->insertBranch(*NewPH, Header, nullptr, EmptyCond, DL);
- NewPH->addSuccessor(Header);
-
- MachineLoop *ParentLoop = L->getParentLoop();
- if (ParentLoop)
- ParentLoop->addBasicBlockToLoop(NewPH, MLI->getBase());
-
- // Update the dominator information with the new preheader.
- if (MDT) {
- if (MachineDomTreeNode *HN = MDT->getNode(Header)) {
- if (MachineDomTreeNode *DHN = HN->getIDom()) {
- MDT->addNewBlock(NewPH, DHN->getBlock());
- MDT->changeImmediateDominator(Header, NewPH);
- }
- }
- }
-
- return NewPH;
-}