diff options
Diffstat (limited to 'gnu/llvm/lib/CodeGen/RegisterPressure.cpp')
| -rw-r--r-- | gnu/llvm/lib/CodeGen/RegisterPressure.cpp | 695 |
1 files changed, 518 insertions, 177 deletions
diff --git a/gnu/llvm/lib/CodeGen/RegisterPressure.cpp b/gnu/llvm/lib/CodeGen/RegisterPressure.cpp index f33dc3e1049..a21d6c1d4d6 100644 --- a/gnu/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/gnu/llvm/lib/CodeGen/RegisterPressure.cpp @@ -24,7 +24,13 @@ using namespace llvm; /// Increase pressure for each pressure set provided by TargetRegisterInfo. static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure, - PSetIterator PSetI) { + const MachineRegisterInfo &MRI, unsigned Reg, + LaneBitmask PrevMask, LaneBitmask NewMask) { + assert((PrevMask & ~NewMask) == 0 && "Must not remove bits"); + if (PrevMask != 0 || NewMask == 0) + return; + + PSetIterator PSetI = MRI.getPressureSets(Reg); unsigned Weight = PSetI.getWeight(); for (; PSetI.isValid(); ++PSetI) CurrSetPressure[*PSetI] += Weight; @@ -32,7 +38,13 @@ static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure, /// Decrease pressure for each pressure set provided by TargetRegisterInfo. static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure, - PSetIterator PSetI) { + const MachineRegisterInfo &MRI, unsigned Reg, + LaneBitmask PrevMask, LaneBitmask NewMask) { + assert((NewMask & !PrevMask) == 0 && "Must not add bits"); + if (NewMask != 0 || PrevMask == 0) + return; + + PSetIterator PSetI = MRI.getPressureSets(Reg); unsigned Weight = PSetI.getWeight(); for (; PSetI.isValid(); ++PSetI) { assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow"); @@ -59,12 +71,20 @@ void RegisterPressure::dump(const TargetRegisterInfo *TRI) const { dbgs() << "Max Pressure: "; dumpRegSetPressure(MaxSetPressure, TRI); dbgs() << "Live In: "; - for (unsigned Reg : LiveInRegs) - dbgs() << PrintVRegOrUnit(Reg, TRI) << " "; + for (const RegisterMaskPair &P : LiveInRegs) { + dbgs() << PrintVRegOrUnit(P.RegUnit, TRI); + if (P.LaneMask != ~0u) + dbgs() << ':' << PrintLaneMask(P.LaneMask); + dbgs() << ' '; + } dbgs() << '\n'; dbgs() << "Live Out: "; - for (unsigned Reg : LiveOutRegs) - dbgs() << PrintVRegOrUnit(Reg, TRI) << " "; + for (const RegisterMaskPair &P : LiveOutRegs) { + dbgs() << PrintVRegOrUnit(P.RegUnit, TRI); + if (P.LaneMask != ~0u) + dbgs() << ':' << PrintLaneMask(P.LaneMask); + dbgs() << ' '; + } dbgs() << '\n'; } @@ -89,24 +109,25 @@ void PressureDiff::dump(const TargetRegisterInfo &TRI) const { dbgs() << '\n'; } -/// Increase the current pressure as impacted by these registers and bump -/// the high water mark if needed. -void RegPressureTracker::increaseRegPressure(ArrayRef<unsigned> RegUnits) { - for (unsigned RegUnit : RegUnits) { - PSetIterator PSetI = MRI->getPressureSets(RegUnit); - unsigned Weight = PSetI.getWeight(); - for (; PSetI.isValid(); ++PSetI) { - CurrSetPressure[*PSetI] += Weight; - P.MaxSetPressure[*PSetI] = - std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]); - } +void RegPressureTracker::increaseRegPressure(unsigned RegUnit, + LaneBitmask PreviousMask, + LaneBitmask NewMask) { + if (PreviousMask != 0 || NewMask == 0) + return; + + PSetIterator PSetI = MRI->getPressureSets(RegUnit); + unsigned Weight = PSetI.getWeight(); + for (; PSetI.isValid(); ++PSetI) { + CurrSetPressure[*PSetI] += Weight; + P.MaxSetPressure[*PSetI] = + std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]); } } -/// Simply decrease the current pressure as impacted by these registers. -void RegPressureTracker::decreaseRegPressure(ArrayRef<unsigned> RegUnits) { - for (unsigned RegUnit : RegUnits) - decreaseSetPressure(CurrSetPressure, MRI->getPressureSets(RegUnit)); +void RegPressureTracker::decreaseRegPressure(unsigned RegUnit, + LaneBitmask PreviousMask, + LaneBitmask NewMask) { + decreaseSetPressure(CurrSetPressure, *MRI, RegUnit, PreviousMask, NewMask); } /// Clear the result so it can be used for another round of pressure tracking. @@ -201,8 +222,7 @@ void RegPressureTracker::init(const MachineFunction *mf, const LiveIntervals *lis, const MachineBasicBlock *mbb, MachineBasicBlock::const_iterator pos, - bool ShouldTrackUntiedDefs) -{ + bool TrackLaneMasks, bool TrackUntiedDefs) { reset(); MF = mf; @@ -210,7 +230,8 @@ void RegPressureTracker::init(const MachineFunction *mf, RCI = rci; MRI = &MF->getRegInfo(); MBB = mbb; - TrackUntiedDefs = ShouldTrackUntiedDefs; + this->TrackUntiedDefs = TrackUntiedDefs; + this->TrackLaneMasks = TrackLaneMasks; if (RequireIntervals) { assert(lis && "IntervalPressure requires LiveIntervals"); @@ -250,7 +271,7 @@ SlotIndex RegPressureTracker::getCurrSlot() const { ++IdxPos; if (IdxPos == MBB->end()) return LIS->getMBBEndIdx(MBB); - return LIS->getInstructionIndex(IdxPos).getRegSlot(); + return LIS->getInstructionIndex(*IdxPos).getRegSlot(); } /// Set the boundary for the top of the region and summarize live ins. @@ -297,20 +318,106 @@ void RegPressureTracker::closeRegion() { void RegPressureTracker::initLiveThru(const RegPressureTracker &RPTracker) { LiveThruPressure.assign(TRI->getNumRegPressureSets(), 0); assert(isBottomClosed() && "need bottom-up tracking to intialize."); - for (unsigned Reg : P.LiveOutRegs) { - if (TargetRegisterInfo::isVirtualRegister(Reg) - && !RPTracker.hasUntiedDef(Reg)) { - increaseSetPressure(LiveThruPressure, MRI->getPressureSets(Reg)); + for (const RegisterMaskPair &Pair : P.LiveOutRegs) { + unsigned RegUnit = Pair.RegUnit; + if (TargetRegisterInfo::isVirtualRegister(RegUnit) + && !RPTracker.hasUntiedDef(RegUnit)) + increaseSetPressure(LiveThruPressure, *MRI, RegUnit, 0, Pair.LaneMask); + } +} + +static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits, + unsigned RegUnit) { + auto I = std::find_if(RegUnits.begin(), RegUnits.end(), + [RegUnit](const RegisterMaskPair Other) { + return Other.RegUnit == RegUnit; + }); + if (I == RegUnits.end()) + return 0; + return I->LaneMask; +} + +static void addRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits, + RegisterMaskPair Pair) { + unsigned RegUnit = Pair.RegUnit; + assert(Pair.LaneMask != 0); + auto I = std::find_if(RegUnits.begin(), RegUnits.end(), + [RegUnit](const RegisterMaskPair Other) { + return Other.RegUnit == RegUnit; + }); + if (I == RegUnits.end()) { + RegUnits.push_back(Pair); + } else { + I->LaneMask |= Pair.LaneMask; + } +} + +static void setRegZero(SmallVectorImpl<RegisterMaskPair> &RegUnits, + unsigned RegUnit) { + auto I = std::find_if(RegUnits.begin(), RegUnits.end(), + [RegUnit](const RegisterMaskPair Other) { + return Other.RegUnit == RegUnit; + }); + if (I == RegUnits.end()) { + RegUnits.push_back(RegisterMaskPair(RegUnit, 0)); + } else { + I->LaneMask = 0; + } +} + +static void removeRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits, + RegisterMaskPair Pair) { + unsigned RegUnit = Pair.RegUnit; + assert(Pair.LaneMask != 0); + auto I = std::find_if(RegUnits.begin(), RegUnits.end(), + [RegUnit](const RegisterMaskPair Other) { + return Other.RegUnit == RegUnit; + }); + if (I != RegUnits.end()) { + I->LaneMask &= ~Pair.LaneMask; + if (I->LaneMask == 0) + RegUnits.erase(I); + } +} + +static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS, + const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit, + SlotIndex Pos, LaneBitmask SafeDefault, + bool(*Property)(const LiveRange &LR, SlotIndex Pos)) { + if (TargetRegisterInfo::isVirtualRegister(RegUnit)) { + const LiveInterval &LI = LIS.getInterval(RegUnit); + LaneBitmask Result = 0; + if (TrackLaneMasks && LI.hasSubRanges()) { + for (const LiveInterval::SubRange &SR : LI.subranges()) { + if (Property(SR, Pos)) + Result |= SR.LaneMask; + } + } else if (Property(LI, Pos)) { + Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit) : ~0u; } + + return Result; + } else { + const LiveRange *LR = LIS.getCachedRegUnit(RegUnit); + // Be prepared for missing liveranges: We usually do not compute liveranges + // for physical registers on targets with many registers (GPUs). + if (LR == nullptr) + return SafeDefault; + return Property(*LR, Pos) ? ~0u : 0; } } -/// \brief Convenient wrapper for checking membership in RegisterOperands. -/// (std::count() doesn't have an early exit). -static bool containsReg(ArrayRef<unsigned> RegUnits, unsigned RegUnit) { - return std::find(RegUnits.begin(), RegUnits.end(), RegUnit) != RegUnits.end(); +static LaneBitmask getLiveLanesAt(const LiveIntervals &LIS, + const MachineRegisterInfo &MRI, + bool TrackLaneMasks, unsigned RegUnit, + SlotIndex Pos) { + return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos, ~0u, + [](const LiveRange &LR, SlotIndex Pos) { + return LR.liveAt(Pos); + }); } + namespace { /// Collect this instruction's unique uses and defs into SmallVectors for @@ -325,19 +432,25 @@ class RegisterOperandsCollector { RegisterOperandsCollector(RegisterOperands &RegOpers, const TargetRegisterInfo &TRI, - const MachineRegisterInfo &MRI, - bool IgnoreDead) + const MachineRegisterInfo &MRI, bool IgnoreDead) : RegOpers(RegOpers), TRI(TRI), MRI(MRI), IgnoreDead(IgnoreDead) {} void collectInstr(const MachineInstr &MI) const { - for (ConstMIBundleOperands OperI(&MI); OperI.isValid(); ++OperI) + for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) collectOperand(*OperI); // Remove redundant physreg dead defs. - SmallVectorImpl<unsigned>::iterator I = - std::remove_if(RegOpers.DeadDefs.begin(), RegOpers.DeadDefs.end(), - std::bind1st(std::ptr_fun(containsReg), RegOpers.Defs)); - RegOpers.DeadDefs.erase(I, RegOpers.DeadDefs.end()); + for (const RegisterMaskPair &P : RegOpers.Defs) + removeRegLanes(RegOpers.DeadDefs, P); + } + + void collectInstrLanes(const MachineInstr &MI) const { + for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) + collectOperandLanes(*OperI); + + // Remove redundant physreg dead defs. + for (const RegisterMaskPair &P : RegOpers.Defs) + removeRegLanes(RegOpers.DeadDefs, P); } /// Push this operand's register onto the correct vectors. @@ -345,28 +458,65 @@ class RegisterOperandsCollector { if (!MO.isReg() || !MO.getReg()) return; unsigned Reg = MO.getReg(); - if (MO.readsReg()) - pushRegUnits(Reg, RegOpers.Uses); - if (MO.isDef()) { + if (MO.isUse()) { + if (!MO.isUndef() && !MO.isInternalRead()) + pushReg(Reg, RegOpers.Uses); + } else { + assert(MO.isDef()); + // Subregister definitions may imply a register read. + if (MO.readsReg()) + pushReg(Reg, RegOpers.Uses); + if (MO.isDead()) { if (!IgnoreDead) - pushRegUnits(Reg, RegOpers.DeadDefs); + pushReg(Reg, RegOpers.DeadDefs); } else - pushRegUnits(Reg, RegOpers.Defs); + pushReg(Reg, RegOpers.Defs); } } - void pushRegUnits(unsigned Reg, SmallVectorImpl<unsigned> &RegUnits) const { + void pushReg(unsigned Reg, + SmallVectorImpl<RegisterMaskPair> &RegUnits) const { if (TargetRegisterInfo::isVirtualRegister(Reg)) { - if (containsReg(RegUnits, Reg)) - return; - RegUnits.push_back(Reg); + addRegLanes(RegUnits, RegisterMaskPair(Reg, ~0u)); } else if (MRI.isAllocatable(Reg)) { - for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units) { - if (containsReg(RegUnits, *Units)) - continue; - RegUnits.push_back(*Units); - } + for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units) + addRegLanes(RegUnits, RegisterMaskPair(*Units, ~0u)); + } + } + + void collectOperandLanes(const MachineOperand &MO) const { + if (!MO.isReg() || !MO.getReg()) + return; + unsigned Reg = MO.getReg(); + unsigned SubRegIdx = MO.getSubReg(); + if (MO.isUse()) { + if (!MO.isUndef() && !MO.isInternalRead()) + pushRegLanes(Reg, SubRegIdx, RegOpers.Uses); + } else { + assert(MO.isDef()); + // Treat read-undef subreg defs as definitions of the whole register. + if (MO.isUndef()) + SubRegIdx = 0; + + if (MO.isDead()) { + if (!IgnoreDead) + pushRegLanes(Reg, SubRegIdx, RegOpers.DeadDefs); + } else + pushRegLanes(Reg, SubRegIdx, RegOpers.Defs); + } + } + + void pushRegLanes(unsigned Reg, unsigned SubRegIdx, + SmallVectorImpl<RegisterMaskPair> &RegUnits) const { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + LaneBitmask LaneMask = SubRegIdx != 0 + ? TRI.getSubRegIndexLaneMask(SubRegIdx) + : MRI.getMaxLaneMaskForVReg(Reg); + addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneMask)); + } else if (MRI.isAllocatable(Reg)) { + for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units) + addRegLanes(RegUnits, RegisterMaskPair(*Units, ~0u)); } } @@ -378,24 +528,26 @@ class RegisterOperandsCollector { void RegisterOperands::collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, - bool IgnoreDead) { + bool TrackLaneMasks, bool IgnoreDead) { RegisterOperandsCollector Collector(*this, TRI, MRI, IgnoreDead); - Collector.collectInstr(MI); + if (TrackLaneMasks) + Collector.collectInstrLanes(MI); + else + Collector.collectInstr(MI); } void RegisterOperands::detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS) { - SlotIndex SlotIdx = LIS.getInstructionIndex(&MI); - for (SmallVectorImpl<unsigned>::iterator RI = Defs.begin(); - RI != Defs.end(); /*empty*/) { - unsigned Reg = *RI; + SlotIndex SlotIdx = LIS.getInstructionIndex(MI); + for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) { + unsigned Reg = RI->RegUnit; const LiveRange *LR = getLiveRange(LIS, Reg); if (LR != nullptr) { LiveQueryResult LRQ = LR->Query(SlotIdx); if (LRQ.isDeadDef()) { // LiveIntervals knows this is a dead even though it's MachineOperand is // not flagged as such. - DeadDefs.push_back(Reg); + DeadDefs.push_back(*RI); RI = Defs.erase(RI); continue; } @@ -404,6 +556,52 @@ void RegisterOperands::detectDeadDefs(const MachineInstr &MI, } } +void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS, + const MachineRegisterInfo &MRI, + SlotIndex Pos, + MachineInstr *AddFlagsMI) { + for (auto I = Defs.begin(); I != Defs.end(); ) { + LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit, + Pos.getDeadSlot()); + // If the the def is all that is live after the instruction, then in case + // of a subregister def we need a read-undef flag. + unsigned RegUnit = I->RegUnit; + if (TargetRegisterInfo::isVirtualRegister(RegUnit) && + AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask) == 0) + AddFlagsMI->setRegisterDefReadUndef(RegUnit); + + LaneBitmask ActualDef = I->LaneMask & LiveAfter; + if (ActualDef == 0) { + I = Defs.erase(I); + } else { + I->LaneMask = ActualDef; + ++I; + } + } + for (auto I = Uses.begin(); I != Uses.end(); ) { + LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit, + Pos.getBaseIndex()); + LaneBitmask LaneMask = I->LaneMask & LiveBefore; + if (LaneMask == 0) { + I = Uses.erase(I); + } else { + I->LaneMask = LaneMask; + ++I; + } + } + if (AddFlagsMI != nullptr) { + for (const RegisterMaskPair &P : DeadDefs) { + unsigned RegUnit = P.RegUnit; + if (!TargetRegisterInfo::isVirtualRegister(RegUnit)) + continue; + LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit, + Pos.getDeadSlot()); + if (LiveAfter == 0) + AddFlagsMI->setRegisterDefReadUndef(RegUnit); + } + } +} + /// Initialize an array of N PressureDiffs. void PressureDiffs::init(unsigned N) { Size = N; @@ -421,11 +619,11 @@ void PressureDiffs::addInstruction(unsigned Idx, const MachineRegisterInfo &MRI) { PressureDiff &PDiff = (*this)[Idx]; assert(!PDiff.begin()->isValid() && "stale PDiff"); - for (unsigned Reg : RegOpers.Defs) - PDiff.addPressureChange(Reg, true, &MRI); + for (const RegisterMaskPair &P : RegOpers.Defs) + PDiff.addPressureChange(P.RegUnit, true, &MRI); - for (unsigned Reg : RegOpers.Uses) - PDiff.addPressureChange(Reg, false, &MRI); + for (const RegisterMaskPair &P : RegOpers.Uses) + PDiff.addPressureChange(P.RegUnit, false, &MRI); } /// Add a change in pressure to the pressure diff of a given instruction. @@ -465,33 +663,58 @@ void PressureDiff::addPressureChange(unsigned RegUnit, bool IsDec, } /// Force liveness of registers. -void RegPressureTracker::addLiveRegs(ArrayRef<unsigned> Regs) { - for (unsigned Reg : Regs) { - if (LiveRegs.insert(Reg)) - increaseRegPressure(Reg); +void RegPressureTracker::addLiveRegs(ArrayRef<RegisterMaskPair> Regs) { + for (const RegisterMaskPair &P : Regs) { + LaneBitmask PrevMask = LiveRegs.insert(P); + LaneBitmask NewMask = PrevMask | P.LaneMask; + increaseRegPressure(P.RegUnit, PrevMask, NewMask); } } -/// Add Reg to the live in set and increase max pressure. -void RegPressureTracker::discoverLiveIn(unsigned Reg) { - assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); - if (containsReg(P.LiveInRegs, Reg)) - return; +void RegPressureTracker::discoverLiveInOrOut(RegisterMaskPair Pair, + SmallVectorImpl<RegisterMaskPair> &LiveInOrOut) { + assert(Pair.LaneMask != 0); + + unsigned RegUnit = Pair.RegUnit; + auto I = std::find_if(LiveInOrOut.begin(), LiveInOrOut.end(), + [RegUnit](const RegisterMaskPair &Other) { + return Other.RegUnit == RegUnit; + }); + LaneBitmask PrevMask; + LaneBitmask NewMask; + if (I == LiveInOrOut.end()) { + PrevMask = 0; + NewMask = Pair.LaneMask; + LiveInOrOut.push_back(Pair); + } else { + PrevMask = I->LaneMask; + NewMask = PrevMask | Pair.LaneMask; + I->LaneMask = NewMask; + } + increaseSetPressure(P.MaxSetPressure, *MRI, RegUnit, PrevMask, NewMask); +} - // At live in discovery, unconditionally increase the high water mark. - P.LiveInRegs.push_back(Reg); - increaseSetPressure(P.MaxSetPressure, MRI->getPressureSets(Reg)); +void RegPressureTracker::discoverLiveIn(RegisterMaskPair Pair) { + discoverLiveInOrOut(Pair, P.LiveInRegs); } -/// Add Reg to the live out set and increase max pressure. -void RegPressureTracker::discoverLiveOut(unsigned Reg) { - assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); - if (containsReg(P.LiveOutRegs, Reg)) - return; +void RegPressureTracker::discoverLiveOut(RegisterMaskPair Pair) { + discoverLiveInOrOut(Pair, P.LiveOutRegs); +} - // At live out discovery, unconditionally increase the high water mark. - P.LiveOutRegs.push_back(Reg); - increaseSetPressure(P.MaxSetPressure, MRI->getPressureSets(Reg)); +void RegPressureTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) { + for (const RegisterMaskPair &P : DeadDefs) { + unsigned Reg = P.RegUnit; + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask BumpedMask = LiveMask | P.LaneMask; + increaseRegPressure(Reg, LiveMask, BumpedMask); + } + for (const RegisterMaskPair &P : DeadDefs) { + unsigned Reg = P.RegUnit; + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask BumpedMask = LiveMask | P.LaneMask; + decreaseRegPressure(Reg, BumpedMask, LiveMask); + } } /// Recede across the previous instruction. If LiveUses is provided, record any @@ -500,48 +723,88 @@ void RegPressureTracker::discoverLiveOut(unsigned Reg) { /// difference pointer is provided record the changes is pressure caused by this /// instruction independent of liveness. void RegPressureTracker::recede(const RegisterOperands &RegOpers, - SmallVectorImpl<unsigned> *LiveUses) { + SmallVectorImpl<RegisterMaskPair> *LiveUses) { assert(!CurrPos->isDebugValue()); // Boost pressure for all dead defs together. - increaseRegPressure(RegOpers.DeadDefs); - decreaseRegPressure(RegOpers.DeadDefs); + bumpDeadDefs(RegOpers.DeadDefs); // Kill liveness at live defs. // TODO: consider earlyclobbers? - for (unsigned Reg : RegOpers.Defs) { - if (LiveRegs.erase(Reg)) - decreaseRegPressure(Reg); - else - discoverLiveOut(Reg); + for (const RegisterMaskPair &Def : RegOpers.Defs) { + unsigned Reg = Def.RegUnit; + + LaneBitmask PreviousMask = LiveRegs.erase(Def); + LaneBitmask NewMask = PreviousMask & ~Def.LaneMask; + + LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask; + if (LiveOut != 0) { + discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); + // Retroactively model effects on pressure of the live out lanes. + increaseSetPressure(CurrSetPressure, *MRI, Reg, 0, LiveOut); + PreviousMask = LiveOut; + } + + if (NewMask == 0) { + // Add a 0 entry to LiveUses as a marker that the complete vreg has become + // dead. + if (TrackLaneMasks && LiveUses != nullptr) + setRegZero(*LiveUses, Reg); + } + + decreaseRegPressure(Reg, PreviousMask, NewMask); } SlotIndex SlotIdx; if (RequireIntervals) - SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); + SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot(); // Generate liveness for uses. - for (unsigned Reg : RegOpers.Uses) { - if (!LiveRegs.contains(Reg)) { - // Adjust liveouts if LiveIntervals are available. - if (RequireIntervals) { - const LiveRange *LR = getLiveRange(*LIS, Reg); - if (LR) { - LiveQueryResult LRQ = LR->Query(SlotIdx); - if (!LRQ.isKill() && !LRQ.valueDefined()) - discoverLiveOut(Reg); + for (const RegisterMaskPair &Use : RegOpers.Uses) { + unsigned Reg = Use.RegUnit; + assert(Use.LaneMask != 0); + LaneBitmask PreviousMask = LiveRegs.insert(Use); + LaneBitmask NewMask = PreviousMask | Use.LaneMask; + if (NewMask == PreviousMask) + continue; + + // Did the register just become live? + if (PreviousMask == 0) { + if (LiveUses != nullptr) { + if (!TrackLaneMasks) { + addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); + } else { + auto I = std::find_if(LiveUses->begin(), LiveUses->end(), + [Reg](const RegisterMaskPair Other) { + return Other.RegUnit == Reg; + }); + bool IsRedef = I != LiveUses->end(); + if (IsRedef) { + // ignore re-defs here... + assert(I->LaneMask == 0); + removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); + } else { + addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); + } } } - increaseRegPressure(Reg); - LiveRegs.insert(Reg); - if (LiveUses && !containsReg(*LiveUses, Reg)) - LiveUses->push_back(Reg); + + // Discover live outs if this may be the first occurance of this register. + if (RequireIntervals) { + LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx); + if (LiveOut != 0) + discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); + } } + + increaseRegPressure(Reg, PreviousMask, NewMask); } if (TrackUntiedDefs) { - for (unsigned Reg : RegOpers.Defs) { - if (TargetRegisterInfo::isVirtualRegister(Reg) && !LiveRegs.contains(Reg)) - UntiedDefs.insert(Reg); + for (const RegisterMaskPair &Def : RegOpers.Defs) { + unsigned RegUnit = Def.RegUnit; + if (TargetRegisterInfo::isVirtualRegister(RegUnit) && + (LiveRegs.contains(RegUnit) & Def.LaneMask) == 0) + UntiedDefs.insert(RegUnit); } } } @@ -562,29 +825,32 @@ void RegPressureTracker::recedeSkipDebugValues() { SlotIndex SlotIdx; if (RequireIntervals) - SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); + SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot(); // Open the top of the region using slot indexes. if (RequireIntervals && isTopClosed()) static_cast<IntervalPressure&>(P).openTop(SlotIdx); } -void RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses) { +void RegPressureTracker::recede(SmallVectorImpl<RegisterMaskPair> *LiveUses) { recedeSkipDebugValues(); const MachineInstr &MI = *CurrPos; RegisterOperands RegOpers; - RegOpers.collect(MI, *TRI, *MRI); - if (RequireIntervals) + RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false); + if (TrackLaneMasks) { + SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot(); + RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx); + } else if (RequireIntervals) { RegOpers.detectDeadDefs(MI, *LIS); + } recede(RegOpers, LiveUses); } /// Advance across the current instruction. -void RegPressureTracker::advance() { +void RegPressureTracker::advance(const RegisterOperands &RegOpers) { assert(!TrackUntiedDefs && "unsupported mode"); - assert(CurrPos != MBB->end()); if (!isTopClosed()) closeTop(); @@ -601,39 +867,34 @@ void RegPressureTracker::advance() { static_cast<RegionPressure&>(P).openBottom(CurrPos); } - RegisterOperands RegOpers; - RegOpers.collect(*CurrPos, *TRI, *MRI); - - for (unsigned Reg : RegOpers.Uses) { - // Discover live-ins. - bool isLive = LiveRegs.contains(Reg); - if (!isLive) - discoverLiveIn(Reg); + for (const RegisterMaskPair &Use : RegOpers.Uses) { + unsigned Reg = Use.RegUnit; + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask LiveIn = Use.LaneMask & ~LiveMask; + if (LiveIn != 0) { + discoverLiveIn(RegisterMaskPair(Reg, LiveIn)); + increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn); + LiveRegs.insert(RegisterMaskPair(Reg, LiveIn)); + } // Kill liveness at last uses. - bool lastUse = false; if (RequireIntervals) { - const LiveRange *LR = getLiveRange(*LIS, Reg); - lastUse = LR && LR->Query(SlotIdx).isKill(); - } else { - // Allocatable physregs are always single-use before register rewriting. - lastUse = !TargetRegisterInfo::isVirtualRegister(Reg); + LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); + if (LastUseMask != 0) { + LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask)); + decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask); + } } - if (lastUse && isLive) { - LiveRegs.erase(Reg); - decreaseRegPressure(Reg); - } else if (!lastUse && !isLive) - increaseRegPressure(Reg); } // Generate liveness for defs. - for (unsigned Reg : RegOpers.Defs) { - if (LiveRegs.insert(Reg)) - increaseRegPressure(Reg); + for (const RegisterMaskPair &Def : RegOpers.Defs) { + LaneBitmask PreviousMask = LiveRegs.insert(Def); + LaneBitmask NewMask = PreviousMask | Def.LaneMask; + increaseRegPressure(Def.RegUnit, PreviousMask, NewMask); } // Boost pressure for all dead defs together. - increaseRegPressure(RegOpers.DeadDefs); - decreaseRegPressure(RegOpers.DeadDefs); + bumpDeadDefs(RegOpers.DeadDefs); // Find the next instruction. do @@ -641,6 +902,17 @@ void RegPressureTracker::advance() { while (CurrPos != MBB->end() && CurrPos->isDebugValue()); } +void RegPressureTracker::advance() { + const MachineInstr &MI = *CurrPos; + RegisterOperands RegOpers; + RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false); + if (TrackLaneMasks) { + SlotIndex SlotIdx = getCurrSlot(); + RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx); + } + advance(RegOpers); +} + /// Find the max change in excess pressure across all sets. static void computeExcessPressureDelta(ArrayRef<unsigned> OldPressureVec, ArrayRef<unsigned> NewPressureVec, @@ -728,22 +1000,38 @@ static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec, void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) { assert(!MI->isDebugValue() && "Expect a nondebug instruction."); + SlotIndex SlotIdx; + if (RequireIntervals) + SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot(); + // Account for register pressure similar to RegPressureTracker::recede(). RegisterOperands RegOpers; - RegOpers.collect(*MI, *TRI, *MRI, /*IgnoreDead=*/true); + RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true); assert(RegOpers.DeadDefs.size() == 0); - if (RequireIntervals) + if (TrackLaneMasks) + RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx); + else if (RequireIntervals) RegOpers.detectDeadDefs(*MI, *LIS); + // Boost max pressure for all dead defs together. + // Since CurrSetPressure and MaxSetPressure + bumpDeadDefs(RegOpers.DeadDefs); + // Kill liveness at live defs. - for (unsigned Reg : RegOpers.Defs) { - if (!containsReg(RegOpers.Uses, Reg)) - decreaseRegPressure(Reg); + for (const RegisterMaskPair &P : RegOpers.Defs) { + unsigned Reg = P.RegUnit; + LaneBitmask LiveLanes = LiveRegs.contains(Reg); + LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg); + LaneBitmask DefLanes = P.LaneMask; + LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes; + decreaseRegPressure(Reg, LiveLanes, LiveAfter); } // Generate liveness for uses. - for (unsigned Reg : RegOpers.Uses) { - if (!LiveRegs.contains(Reg)) - increaseRegPressure(Reg); + for (const RegisterMaskPair &P : RegOpers.Uses) { + unsigned Reg = P.RegUnit; + LaneBitmask LiveLanes = LiveRegs.contains(Reg); + LaneBitmask LiveAfter = LiveLanes | P.LaneMask; + increaseRegPressure(Reg, LiveLanes, LiveAfter); } } @@ -888,15 +1176,58 @@ getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff, } /// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx). -static bool findUseBetween(unsigned Reg, SlotIndex PriorUseIdx, - SlotIndex NextUseIdx, const MachineRegisterInfo &MRI, - const LiveIntervals *LIS) { - for (const MachineInstr &MI : MRI.use_nodbg_instructions(Reg)) { - SlotIndex InstSlot = LIS->getInstructionIndex(&MI).getRegSlot(); - if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) - return true; +/// The query starts with a lane bitmask which gets lanes/bits removed for every +/// use we find. +static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask, + SlotIndex PriorUseIdx, SlotIndex NextUseIdx, + const MachineRegisterInfo &MRI, + const LiveIntervals *LIS) { + const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); + for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) { + if (MO.isUndef()) + continue; + const MachineInstr *MI = MO.getParent(); + SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot(); + if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) { + unsigned SubRegIdx = MO.getSubReg(); + LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx); + LastUseMask &= ~UseMask; + if (LastUseMask == 0) + return 0; + } } - return false; + return LastUseMask; +} + +LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit, + SlotIndex Pos) const { + assert(RequireIntervals); + return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, ~0u, + [](const LiveRange &LR, SlotIndex Pos) { + return LR.liveAt(Pos); + }); +} + +LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit, + SlotIndex Pos) const { + assert(RequireIntervals); + return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, + Pos.getBaseIndex(), 0, + [](const LiveRange &LR, SlotIndex Pos) { + const LiveRange::Segment *S = LR.getSegmentContaining(Pos); + return S != nullptr && S->end == Pos.getRegSlot(); + }); +} + +LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit, + SlotIndex Pos) const { + assert(RequireIntervals); + return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, 0u, + [](const LiveRange &LR, SlotIndex Pos) { + const LiveRange::Segment *S = LR.getSegmentContaining(Pos); + return S != nullptr && S->start < Pos.getRegSlot(true) && + S->end != Pos.getDeadSlot(); + }); } /// Record the downward impact of a single instruction on current register @@ -908,39 +1239,49 @@ static bool findUseBetween(unsigned Reg, SlotIndex PriorUseIdx, void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) { assert(!MI->isDebugValue() && "Expect a nondebug instruction."); - // Account for register pressure similar to RegPressureTracker::recede(). - RegisterOperands RegOpers; - RegOpers.collect(*MI, *TRI, *MRI); - - // Kill liveness at last uses. Assume allocatable physregs are single-use - // rather than checking LiveIntervals. SlotIndex SlotIdx; if (RequireIntervals) - SlotIdx = LIS->getInstructionIndex(MI).getRegSlot(); + SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot(); - for (unsigned Reg : RegOpers.Uses) { - if (RequireIntervals) { + // Account for register pressure similar to RegPressureTracker::recede(). + RegisterOperands RegOpers; + RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false); + if (TrackLaneMasks) + RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx); + + if (RequireIntervals) { + for (const RegisterMaskPair &Use : RegOpers.Uses) { + unsigned Reg = Use.RegUnit; + LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); + if (LastUseMask == 0) + continue; + // The LastUseMask is queried from the liveness information of instruction + // which may be further down the schedule. Some lanes may actually not be + // last uses for the current position. // FIXME: allow the caller to pass in the list of vreg uses that remain // to be bottom-scheduled to avoid searching uses at each query. SlotIndex CurrIdx = getCurrSlot(); - const LiveRange *LR = getLiveRange(*LIS, Reg); - if (LR) { - LiveQueryResult LRQ = LR->Query(SlotIdx); - if (LRQ.isKill() && !findUseBetween(Reg, CurrIdx, SlotIdx, *MRI, LIS)) - decreaseRegPressure(Reg); - } - } else if (!TargetRegisterInfo::isVirtualRegister(Reg)) { - // Allocatable physregs are always single-use before register rewriting. - decreaseRegPressure(Reg); + LastUseMask + = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS); + if (LastUseMask == 0) + continue; + + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask NewMask = LiveMask & ~LastUseMask; + decreaseRegPressure(Reg, LiveMask, NewMask); } } // Generate liveness for defs. - increaseRegPressure(RegOpers.Defs); + for (const RegisterMaskPair &Def : RegOpers.Defs) { + unsigned Reg = Def.RegUnit; + LaneBitmask LiveMask = LiveRegs.contains(Reg); + LaneBitmask NewMask = LiveMask | Def.LaneMask; + increaseRegPressure(Reg, LiveMask, NewMask); + } // Boost pressure for all dead defs together. - increaseRegPressure(RegOpers.DeadDefs); - decreaseRegPressure(RegOpers.DeadDefs); + bumpDeadDefs(RegOpers.DeadDefs); } /// Consider the pressure increase caused by traversing this instruction |