summaryrefslogtreecommitdiffstats
path: root/gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
diff options
context:
space:
mode:
authorpatrick <patrick@openbsd.org>2017-01-24 08:32:59 +0000
committerpatrick <patrick@openbsd.org>2017-01-24 08:32:59 +0000
commit53d771aafdbe5b919f264f53cba3788e2c4cffd2 (patch)
tree7eca39498be0ff1e3a6daf583cd9ca5886bb2636 /gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
parentIn preparation of compiling our kernels with -ffreestanding, explicitly map (diff)
downloadwireguard-openbsd-53d771aafdbe5b919f264f53cba3788e2c4cffd2.tar.xz
wireguard-openbsd-53d771aafdbe5b919f264f53cba3788e2c4cffd2.zip
Import LLVM 4.0.0 rc1 including clang and lld to help the current
development effort on OpenBSD/arm64.
Diffstat (limited to 'gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
-rw-r--r--gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp1013
1 files changed, 634 insertions, 379 deletions
diff --git a/gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 29d11c79ac2..e225ba8703b 100644
--- a/gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/gnu/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -134,7 +134,7 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) {
// we care if the resultant vector is all ones, not whether the individual
// constants are.
SDValue NotZero = N->getOperand(i);
- unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
+ unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
if (CN->getAPIntValue().countTrailingOnes() < EltSize)
return false;
@@ -173,7 +173,7 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) {
// We only want to check enough bits to cover the vector elements, because
// we care if the resultant vector is all zeros, not whether the individual
// constants are.
- unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
+ unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
if (CN->getAPIntValue().countTrailingZeros() < EltSize)
return false;
@@ -403,7 +403,6 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
ID.AddPointer(GA->getGlobal());
ID.AddInteger(GA->getOffset());
ID.AddInteger(GA->getTargetFlags());
- ID.AddInteger(GA->getAddressSpace());
break;
}
case ISD::BasicBlock:
@@ -521,24 +520,6 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
AddNodeIDCustom(ID, N);
}
-/// encodeMemSDNodeFlags - Generic routine for computing a value for use in
-/// the CSE map that carries volatility, temporalness, indexing mode, and
-/// extension/truncation information.
-///
-static inline unsigned
-encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
- bool isNonTemporal, bool isInvariant) {
- assert((ConvType & 3) == ConvType &&
- "ConvType may not require more than 2 bits!");
- assert((AM & 7) == AM &&
- "AM may not require more than 3 bits!");
- return ConvType |
- (AM << 2) |
- (isVolatile << 5) |
- (isNonTemporal << 6) |
- (isInvariant << 7);
-}
-
//===----------------------------------------------------------------------===//
// SelectionDAG Class
//===----------------------------------------------------------------------===//
@@ -1030,7 +1011,7 @@ SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
"getZeroExtendInReg should use the vector element type instead of "
"the vector type!");
if (Op.getValueType() == VT) return Op;
- unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
+ unsigned BitWidth = Op.getScalarValueSizeInBits();
APInt Imm = APInt::getLowBitsSet(BitWidth,
VT.getSizeInBits());
return getNode(ISD::AND, DL, Op.getValueType(), Op,
@@ -1040,7 +1021,7 @@ SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
EVT VT) {
assert(VT.isVector() && "This DAG node is restricted to vector types.");
- assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
+ assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
"The sizes of the input and result must match in order to perform the "
"extend in-register.");
assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
@@ -1051,7 +1032,7 @@ SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
EVT VT) {
assert(VT.isVector() && "This DAG node is restricted to vector types.");
- assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
+ assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
"The sizes of the input and result must match in order to perform the "
"extend in-register.");
assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
@@ -1062,7 +1043,7 @@ SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL,
EVT VT) {
assert(VT.isVector() && "This DAG node is restricted to vector types.");
- assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
+ assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
"The sizes of the input and result must match in order to perform the "
"extend in-register.");
assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
@@ -1123,7 +1104,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
TargetLowering::TypePromoteInteger) {
EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
- APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
+ APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
Elt = ConstantInt::get(*getContext(), NewVal);
}
// In other cases the element type is illegal and needs to be expanded, for
@@ -1149,7 +1130,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
SmallVector<SDValue, 2> EltParts;
for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
- .trunc(ViaEltSizeInBits), DL,
+ .zextOrTrunc(ViaEltSizeInBits), DL,
ViaEltVT, isT, isO));
}
@@ -1166,12 +1147,9 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
// This situation occurs in MIPS MSA.
SmallVector<SDValue, 8> Ops;
- for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
+ for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
-
- SDValue Result = getNode(ISD::BITCAST, DL, VT,
- getNode(ISD::BUILD_VECTOR, DL, ViaVecVT, Ops));
- return Result;
+ return getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
}
assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
@@ -1280,7 +1258,6 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
ID.AddPointer(GV);
ID.AddInteger(Offset);
ID.AddInteger(TargetFlags);
- ID.AddInteger(GV->getType()->getAddressSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
return SDValue(E, 0);
@@ -1333,7 +1310,9 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
assert((TargetFlags == 0 || isTarget) &&
"Cannot set target flags on target-independent globals");
if (Alignment == 0)
- Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
+ Alignment = MF->getFunction()->optForSize()
+ ? getDataLayout().getABITypeAlignment(C->getType())
+ : getDataLayout().getPrefTypeAlignment(C->getType());
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opc, getVTList(VT), None);
@@ -1650,31 +1629,6 @@ SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
}
-SDValue SelectionDAG::getConvertRndSat(EVT VT, const SDLoc &dl, SDValue Val,
- SDValue DTy, SDValue STy, SDValue Rnd,
- SDValue Sat, ISD::CvtCode Code) {
- // If the src and dest types are the same and the conversion is between
- // integer types of the same sign or two floats, no conversion is necessary.
- if (DTy == STy &&
- (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
- return Val;
-
- FoldingSetNodeID ID;
- SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
- AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
- void* IP = nullptr;
- if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
- return SDValue(E, 0);
-
- auto *N =
- newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), Code);
- createOperands(N, Ops);
-
- CSEMap.InsertNode(N, IP);
- InsertNode(N);
- return SDValue(N, 0);
-}
-
SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
@@ -1863,13 +1817,13 @@ SDValue SelectionDAG::expandVACopy(SDNode *Node) {
}
SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
- MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
+ MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
unsigned ByteSize = VT.getStoreSize();
Type *Ty = VT.getTypeForEVT(*getContext());
unsigned StackAlign =
std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
- int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
+ int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
}
@@ -1881,8 +1835,8 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
unsigned Align =
std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
- MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
- int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
+ MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
+ int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
}
@@ -1943,29 +1897,29 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
default: break;
case ISD::SETEQ: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
case ISD::SETNE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
R==APFloat::cmpLessThan, dl, VT);
case ISD::SETLT: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
case ISD::SETGT: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
case ISD::SETLE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
R==APFloat::cmpEqual, dl, VT);
case ISD::SETGE: if (R==APFloat::cmpUnordered)
return getUNDEF(VT);
- // fall through
+ LLVM_FALLTHROUGH;
case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
R==APFloat::cmpEqual, dl, VT);
case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, dl, VT);
@@ -1998,11 +1952,7 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
/// use this predicate to simplify operations downstream.
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
- // This predicate is not safe for vector operations.
- if (Op.getValueType().isVector())
- return false;
-
- unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
+ unsigned BitWidth = Op.getScalarValueSizeInBits();
return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
}
@@ -2016,28 +1966,244 @@ bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
return (KnownZero & Mask) == Mask;
}
+/// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
+/// is less than the element bit-width of the shift node, return it.
+static const APInt *getValidShiftAmountConstant(SDValue V) {
+ if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
+ // Shifting more than the bitwidth is not valid.
+ const APInt &ShAmt = SA->getAPIntValue();
+ if (ShAmt.ult(V.getScalarValueSizeInBits()))
+ return &ShAmt;
+ }
+ return nullptr;
+}
+
/// Determine which bits of Op are known to be either zero or one and return
-/// them in the KnownZero/KnownOne bitsets.
+/// them in the KnownZero/KnownOne bitsets. For vectors, the known bits are
+/// those that are shared by every vector element.
void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
APInt &KnownOne, unsigned Depth) const {
- unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
+ EVT VT = Op.getValueType();
+ APInt DemandedElts = VT.isVector()
+ ? APInt::getAllOnesValue(VT.getVectorNumElements())
+ : APInt(1, 1);
+ computeKnownBits(Op, KnownZero, KnownOne, DemandedElts, Depth);
+}
+
+/// Determine which bits of Op are known to be either zero or one and return
+/// them in the KnownZero/KnownOne bitsets. The DemandedElts argument allows
+/// us to only collect the known bits that are shared by the requested vector
+/// elements.
+/// TODO: We only support DemandedElts on a few opcodes so far, the remainder
+/// should be added when they become necessary.
+void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
+ APInt &KnownOne, const APInt &DemandedElts,
+ unsigned Depth) const {
+ unsigned BitWidth = Op.getScalarValueSizeInBits();
KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
if (Depth == 6)
return; // Limit search depth.
APInt KnownZero2, KnownOne2;
+ unsigned NumElts = DemandedElts.getBitWidth();
- switch (Op.getOpcode()) {
+ if (!DemandedElts)
+ return; // No demanded elts, better to assume we don't know anything.
+
+ unsigned Opcode = Op.getOpcode();
+ switch (Opcode) {
case ISD::Constant:
// We know all of the bits for a constant!
KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
KnownZero = ~KnownOne;
break;
+ case ISD::BUILD_VECTOR:
+ // Collect the known bits that are shared by every demanded vector element.
+ assert(NumElts == Op.getValueType().getVectorNumElements() &&
+ "Unexpected vector size");
+ KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
+ for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
+ if (!DemandedElts[i])
+ continue;
+
+ SDValue SrcOp = Op.getOperand(i);
+ computeKnownBits(SrcOp, KnownZero2, KnownOne2, Depth + 1);
+
+ // BUILD_VECTOR can implicitly truncate sources, we must handle this.
+ if (SrcOp.getValueSizeInBits() != BitWidth) {
+ assert(SrcOp.getValueSizeInBits() > BitWidth &&
+ "Expected BUILD_VECTOR implicit truncation");
+ KnownOne2 = KnownOne2.trunc(BitWidth);
+ KnownZero2 = KnownZero2.trunc(BitWidth);
+ }
+
+ // Known bits are the values that are shared by every demanded element.
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
+ }
+ break;
+ case ISD::VECTOR_SHUFFLE: {
+ // Collect the known bits that are shared by every vector element referenced
+ // by the shuffle.
+ APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
+ KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
+ const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
+ assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
+ for (unsigned i = 0; i != NumElts; ++i) {
+ if (!DemandedElts[i])
+ continue;
+
+ int M = SVN->getMaskElt(i);
+ if (M < 0) {
+ // For UNDEF elements, we don't know anything about the common state of
+ // the shuffle result.
+ KnownOne.clearAllBits();
+ KnownZero.clearAllBits();
+ DemandedLHS.clearAllBits();
+ DemandedRHS.clearAllBits();
+ break;
+ }
+
+ if ((unsigned)M < NumElts)
+ DemandedLHS.setBit((unsigned)M % NumElts);
+ else
+ DemandedRHS.setBit((unsigned)M % NumElts);
+ }
+ // Known bits are the values that are shared by every demanded element.
+ if (!!DemandedLHS) {
+ SDValue LHS = Op.getOperand(0);
+ computeKnownBits(LHS, KnownZero2, KnownOne2, DemandedLHS, Depth + 1);
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ }
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
+ if (!!DemandedRHS) {
+ SDValue RHS = Op.getOperand(1);
+ computeKnownBits(RHS, KnownZero2, KnownOne2, DemandedRHS, Depth + 1);
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ }
+ break;
+ }
+ case ISD::CONCAT_VECTORS: {
+ // Split DemandedElts and test each of the demanded subvectors.
+ KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
+ EVT SubVectorVT = Op.getOperand(0).getValueType();
+ unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
+ unsigned NumSubVectors = Op.getNumOperands();
+ for (unsigned i = 0; i != NumSubVectors; ++i) {
+ APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
+ DemandedSub = DemandedSub.trunc(NumSubVectorElts);
+ if (!!DemandedSub) {
+ SDValue Sub = Op.getOperand(i);
+ computeKnownBits(Sub, KnownZero2, KnownOne2, DemandedSub, Depth + 1);
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ }
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
+ }
+ break;
+ }
+ case ISD::EXTRACT_SUBVECTOR: {
+ // If we know the element index, just demand that subvector elements,
+ // otherwise demand them all.
+ SDValue Src = Op.getOperand(0);
+ ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+ unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+ if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
+ // Offset the demanded elts by the subvector index.
+ uint64_t Idx = SubIdx->getZExtValue();
+ APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
+ computeKnownBits(Src, KnownZero, KnownOne, DemandedSrc, Depth + 1);
+ } else {
+ computeKnownBits(Src, KnownZero, KnownOne, Depth + 1);
+ }
+ break;
+ }
+ case ISD::BITCAST: {
+ SDValue N0 = Op.getOperand(0);
+ unsigned SubBitWidth = N0.getScalarValueSizeInBits();
+
+ // Ignore bitcasts from floating point.
+ if (!N0.getValueType().isInteger())
+ break;
+
+ // Fast handling of 'identity' bitcasts.
+ if (BitWidth == SubBitWidth) {
+ computeKnownBits(N0, KnownZero, KnownOne, DemandedElts, Depth + 1);
+ break;
+ }
+
+ // Support big-endian targets when it becomes useful.
+ bool IsLE = getDataLayout().isLittleEndian();
+ if (!IsLE)
+ break;
+
+ // Bitcast 'small element' vector to 'large element' scalar/vector.
+ if ((BitWidth % SubBitWidth) == 0) {
+ assert(N0.getValueType().isVector() && "Expected bitcast from vector");
+
+ // Collect known bits for the (larger) output by collecting the known
+ // bits from each set of sub elements and shift these into place.
+ // We need to separately call computeKnownBits for each set of
+ // sub elements as the knownbits for each is likely to be different.
+ unsigned SubScale = BitWidth / SubBitWidth;
+ APInt SubDemandedElts(NumElts * SubScale, 0);
+ for (unsigned i = 0; i != NumElts; ++i)
+ if (DemandedElts[i])
+ SubDemandedElts.setBit(i * SubScale);
+
+ for (unsigned i = 0; i != SubScale; ++i) {
+ computeKnownBits(N0, KnownZero2, KnownOne2, SubDemandedElts.shl(i),
+ Depth + 1);
+ KnownOne |= KnownOne2.zext(BitWidth).shl(SubBitWidth * i);
+ KnownZero |= KnownZero2.zext(BitWidth).shl(SubBitWidth * i);
+ }
+ }
+
+ // Bitcast 'large element' scalar/vector to 'small element' vector.
+ if ((SubBitWidth % BitWidth) == 0) {
+ assert(Op.getValueType().isVector() && "Expected bitcast to vector");
+
+ // Collect known bits for the (smaller) output by collecting the known
+ // bits from the overlapping larger input elements and extracting the
+ // sub sections we actually care about.
+ unsigned SubScale = SubBitWidth / BitWidth;
+ APInt SubDemandedElts(NumElts / SubScale, 0);
+ for (unsigned i = 0; i != NumElts; ++i)
+ if (DemandedElts[i])
+ SubDemandedElts.setBit(i / SubScale);
+
+ computeKnownBits(N0, KnownZero2, KnownOne2, SubDemandedElts, Depth + 1);
+
+ KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
+ for (unsigned i = 0; i != NumElts; ++i)
+ if (DemandedElts[i]) {
+ unsigned Offset = (i % SubScale) * BitWidth;
+ KnownOne &= KnownOne2.lshr(Offset).trunc(BitWidth);
+ KnownZero &= KnownZero2.lshr(Offset).trunc(BitWidth);
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
+ }
+ }
+ break;
+ }
case ISD::AND:
// If either the LHS or the RHS are Zero, the result is zero.
- computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// Output known-1 bits are only known if set in both the LHS & RHS.
KnownOne &= KnownOne2;
@@ -2045,8 +2211,10 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
KnownZero |= KnownZero2;
break;
case ISD::OR:
- computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// Output known-0 bits are only known if clear in both the LHS & RHS.
KnownZero &= KnownZero2;
@@ -2054,8 +2222,10 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
KnownOne |= KnownOne2;
break;
case ISD::XOR: {
- computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// Output known-0 bits are known if clear or set in both the LHS & RHS.
APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
@@ -2065,11 +2235,13 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
}
case ISD::MUL: {
- computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// If low bits are zero in either operand, output low known-0 bits.
- // Also compute a conserative estimate for high known-0 bits.
+ // Also compute a conservative estimate for high known-0 bits.
// More trickiness is possible, but this is sufficient for the
// interesting case of alignment computation.
KnownOne.clearAllBits();
@@ -2089,12 +2261,12 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
// For the purposes of computing leading zeros we can conservatively
// treat a udiv as a logical right shift by the power of 2 known to
// be less than the denominator.
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
unsigned LeadZ = KnownZero2.countLeadingOnes();
- KnownOne2.clearAllBits();
- KnownZero2.clearAllBits();
- computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
if (RHSUnknownLeadingOnes != BitWidth)
LeadZ = std::min(BitWidth,
@@ -2105,6 +2277,9 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
case ISD::SELECT:
computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
// Only known if known in both the LHS and RHS.
@@ -2113,6 +2288,9 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
case ISD::SELECT_CC:
computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
// Only known if known in both the LHS and RHS.
@@ -2144,58 +2322,37 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
break;
case ISD::SHL:
- // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
- if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
- unsigned ShAmt = SA->getZExtValue();
-
- // If the shift count is an invalid immediate, don't do anything.
- if (ShAmt >= BitWidth)
- break;
-
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
- KnownZero <<= ShAmt;
- KnownOne <<= ShAmt;
- // low bits known zero.
- KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
+ if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ KnownZero = KnownZero << *ShAmt;
+ KnownOne = KnownOne << *ShAmt;
+ // Low bits are known zero.
+ KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt->getZExtValue());
}
break;
case ISD::SRL:
- // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
- if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
- unsigned ShAmt = SA->getZExtValue();
-
- // If the shift count is an invalid immediate, don't do anything.
- if (ShAmt >= BitWidth)
- break;
-
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
- KnownZero = KnownZero.lshr(ShAmt);
- KnownOne = KnownOne.lshr(ShAmt);
-
- APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
- KnownZero |= HighBits; // High bits known zero.
+ if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ KnownZero = KnownZero.lshr(*ShAmt);
+ KnownOne = KnownOne.lshr(*ShAmt);
+ // High bits are known zero.
+ APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt->getZExtValue());
+ KnownZero |= HighBits;
}
break;
case ISD::SRA:
- if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
- unsigned ShAmt = SA->getZExtValue();
-
- // If the shift count is an invalid immediate, don't do anything.
- if (ShAmt >= BitWidth)
- break;
-
- // If any of the demanded bits are produced by the sign extension, we also
- // demand the input sign bit.
- APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
-
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
- KnownZero = KnownZero.lshr(ShAmt);
- KnownOne = KnownOne.lshr(ShAmt);
-
- // Handle the sign bits.
+ if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ KnownZero = KnownZero.lshr(*ShAmt);
+ KnownOne = KnownOne.lshr(*ShAmt);
+ // If we know the value of the sign bit, then we know it is copied across
+ // the high bits by the shift amount.
+ APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt->getZExtValue());
APInt SignBit = APInt::getSignBit(BitWidth);
- SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
-
+ SignBit = SignBit.lshr(*ShAmt); // Adjust to where it is now in the mask.
if (KnownZero.intersects(SignBit)) {
KnownZero |= HighBits; // New bits are known zero.
} else if (KnownOne.intersects(SignBit)) {
@@ -2205,7 +2362,7 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
case ISD::SIGN_EXTEND_INREG: {
EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
- unsigned EBits = EVT.getScalarType().getSizeInBits();
+ unsigned EBits = EVT.getScalarSizeInBits();
// Sign extension. Compute the demanded bits in the result that are not
// present in the input.
@@ -2220,7 +2377,8 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
if (NewBits.getBoolValue())
InputDemandedBits |= InSignBit;
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
KnownOne &= InputDemandedBits;
KnownZero &= InputDemandedBits;
@@ -2253,7 +2411,7 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
// If this is a ZEXTLoad and we are looking at the loaded value.
if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
EVT VT = LD->getMemoryVT();
- unsigned MemBits = VT.getScalarType().getSizeInBits();
+ unsigned MemBits = VT.getScalarSizeInBits();
KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
} else if (const MDNode *Ranges = LD->getRanges()) {
if (LD->getExtensionType() == ISD::NON_EXTLOAD)
@@ -2263,11 +2421,12 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
case ISD::ZERO_EXTEND: {
EVT InVT = Op.getOperand(0).getValueType();
- unsigned InBits = InVT.getScalarType().getSizeInBits();
+ unsigned InBits = InVT.getScalarSizeInBits();
APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
KnownZero = KnownZero.trunc(InBits);
KnownOne = KnownOne.trunc(InBits);
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
KnownZero = KnownZero.zext(BitWidth);
KnownOne = KnownOne.zext(BitWidth);
KnownZero |= NewBits;
@@ -2275,30 +2434,22 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
case ISD::SIGN_EXTEND: {
EVT InVT = Op.getOperand(0).getValueType();
- unsigned InBits = InVT.getScalarType().getSizeInBits();
- APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
+ unsigned InBits = InVT.getScalarSizeInBits();
KnownZero = KnownZero.trunc(InBits);
KnownOne = KnownOne.trunc(InBits);
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
- // Note if the sign bit is known to be zero or one.
- bool SignBitKnownZero = KnownZero.isNegative();
- bool SignBitKnownOne = KnownOne.isNegative();
-
- KnownZero = KnownZero.zext(BitWidth);
- KnownOne = KnownOne.zext(BitWidth);
-
- // If the sign bit is known zero or one, the top bits match.
- if (SignBitKnownZero)
- KnownZero |= NewBits;
- else if (SignBitKnownOne)
- KnownOne |= NewBits;
+ // If the sign bit is known to be zero or one, then sext will extend
+ // it to the top bits, else it will just zext.
+ KnownZero = KnownZero.sext(BitWidth);
+ KnownOne = KnownOne.sext(BitWidth);
break;
}
case ISD::ANY_EXTEND: {
EVT InVT = Op.getOperand(0).getValueType();
- unsigned InBits = InVT.getScalarType().getSizeInBits();
+ unsigned InBits = InVT.getScalarSizeInBits();
KnownZero = KnownZero.trunc(InBits);
KnownOne = KnownOne.trunc(InBits);
computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
@@ -2308,10 +2459,11 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
case ISD::TRUNCATE: {
EVT InVT = Op.getOperand(0).getValueType();
- unsigned InBits = InVT.getScalarType().getSizeInBits();
+ unsigned InBits = InVT.getScalarSizeInBits();
KnownZero = KnownZero.zext(InBits);
KnownOne = KnownOne.zext(InBits);
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
KnownZero = KnownZero.trunc(BitWidth);
KnownOne = KnownOne.trunc(BitWidth);
break;
@@ -2330,7 +2482,7 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
case ISD::SUB: {
- if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
+ if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) {
// We know that the top bits of C-X are clear if X contains less bits
// than C (i.e. no wrap-around can happen). For example, 20-X is
// positive if we can prove that X is >= 0 and < 16.
@@ -2338,7 +2490,8 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
// NLZ can't be BitWidth with no sign bit
APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
- computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// If all of the MaskV bits are known to be zero, then we know the
// output top bits are zero, because we now know that the output is
@@ -2350,8 +2503,8 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
}
}
+ LLVM_FALLTHROUGH;
}
- // fall through
case ISD::ADD:
case ISD::ADDE: {
// Output known-0 bits are known if clear or set in both the low clear bits
@@ -2361,17 +2514,19 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
// known to be clear. For example, if one input has the top 10 bits clear
// and the other has the top 8 bits clear, we know the top 7 bits of the
// output must be clear.
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
unsigned KnownZeroHigh = KnownZero2.countLeadingOnes();
unsigned KnownZeroLow = KnownZero2.countTrailingOnes();
- computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
KnownZeroHigh = std::min(KnownZeroHigh,
KnownZero2.countLeadingOnes());
KnownZeroLow = std::min(KnownZeroLow,
KnownZero2.countTrailingOnes());
- if (Op.getOpcode() == ISD::ADD) {
+ if (Opcode == ISD::ADD) {
KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroLow);
if (KnownZeroHigh > 1)
KnownZero |= APInt::getHighBitsSet(BitWidth, KnownZeroHigh - 1);
@@ -2387,11 +2542,12 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
}
case ISD::SREM:
- if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
const APInt &RA = Rem->getAPIntValue().abs();
if (RA.isPowerOf2()) {
APInt LowBits = RA - 1;
- computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// The low bits of the first operand are unchanged by the srem.
KnownZero = KnownZero2 & LowBits;
@@ -2411,11 +2567,12 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
break;
case ISD::UREM: {
- if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
const APInt &RA = Rem->getAPIntValue();
if (RA.isPowerOf2()) {
APInt LowBits = (RA - 1);
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
// The upper bits are all zero, the lower ones are unchanged.
KnownZero = KnownZero2 | ~LowBits;
@@ -2426,8 +2583,10 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
// Since the result is less than or equal to either operand, any leading
// zero bits in either operand must also exist in the result.
- computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
- computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
KnownZero2.countLeadingOnes());
@@ -2437,9 +2596,8 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
case ISD::EXTRACT_ELEMENT: {
computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
- const unsigned Index =
- cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
- const unsigned BitWidth = Op.getValueType().getSizeInBits();
+ const unsigned Index = Op.getConstantOperandVal(1);
+ const unsigned BitWidth = Op.getValueSizeInBits();
// Remove low part of known bits mask
KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
@@ -2450,8 +2608,74 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
KnownOne = KnownOne.trunc(BitWidth);
break;
}
+ case ISD::EXTRACT_VECTOR_ELT: {
+ SDValue InVec = Op.getOperand(0);
+ SDValue EltNo = Op.getOperand(1);
+ EVT VecVT = InVec.getValueType();
+ const unsigned BitWidth = Op.getValueSizeInBits();
+ const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
+ const unsigned NumSrcElts = VecVT.getVectorNumElements();
+ // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
+ // anything about the extended bits.
+ if (BitWidth > EltBitWidth) {
+ KnownZero = KnownZero.trunc(EltBitWidth);
+ KnownOne = KnownOne.trunc(EltBitWidth);
+ }
+ ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
+ if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
+ // If we know the element index, just demand that vector element.
+ unsigned Idx = ConstEltNo->getZExtValue();
+ APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
+ computeKnownBits(InVec, KnownZero, KnownOne, DemandedElt, Depth + 1);
+ } else {
+ // Unknown element index, so ignore DemandedElts and demand them all.
+ computeKnownBits(InVec, KnownZero, KnownOne, Depth + 1);
+ }
+ if (BitWidth > EltBitWidth) {
+ KnownZero = KnownZero.zext(BitWidth);
+ KnownOne = KnownOne.zext(BitWidth);
+ }
+ break;
+ }
+ case ISD::INSERT_VECTOR_ELT: {
+ SDValue InVec = Op.getOperand(0);
+ SDValue InVal = Op.getOperand(1);
+ SDValue EltNo = Op.getOperand(2);
+
+ ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
+ if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
+ // If we know the element index, split the demand between the
+ // source vector and the inserted element.
+ KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
+ unsigned EltIdx = CEltNo->getZExtValue();
+
+ // If we demand the inserted element then add its common known bits.
+ if (DemandedElts[EltIdx]) {
+ computeKnownBits(InVal, KnownZero2, KnownOne2, Depth + 1);
+ KnownOne &= KnownOne2.zextOrTrunc(KnownOne.getBitWidth());
+ KnownZero &= KnownZero2.zextOrTrunc(KnownZero.getBitWidth());;
+ }
+
+ // If we demand the source vector then add its common known bits, ensuring
+ // that we don't demand the inserted element.
+ APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
+ if (!!VectorElts) {
+ computeKnownBits(InVec, KnownZero2, KnownOne2, VectorElts, Depth + 1);
+ KnownOne &= KnownOne2;
+ KnownZero &= KnownZero2;
+ }
+ } else {
+ // Unknown element index, so ignore DemandedElts and demand them all.
+ computeKnownBits(InVec, KnownZero, KnownOne, Depth + 1);
+ computeKnownBits(InVal, KnownZero2, KnownOne2, Depth + 1);
+ KnownOne &= KnownOne2.zextOrTrunc(KnownOne.getBitWidth());
+ KnownZero &= KnownZero2.zextOrTrunc(KnownZero.getBitWidth());;
+ }
+ break;
+ }
case ISD::BSWAP: {
- computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
+ computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
KnownZero = KnownZero2.byteSwap();
KnownOne = KnownOne2.byteSwap();
break;
@@ -2460,13 +2684,15 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX: {
- APInt Op0Zero, Op0One;
- APInt Op1Zero, Op1One;
- computeKnownBits(Op.getOperand(0), Op0Zero, Op0One, Depth);
- computeKnownBits(Op.getOperand(1), Op1Zero, Op1One, Depth);
-
- KnownZero = Op0Zero & Op1Zero;
- KnownOne = Op0One & Op1One;
+ computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, DemandedElts,
+ Depth + 1);
+ // If we don't know any bits, early out.
+ if (!KnownOne && !KnownZero)
+ break;
+ computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, DemandedElts,
+ Depth + 1);
+ KnownZero &= KnownZero2;
+ KnownOne &= KnownOne2;
break;
}
case ISD::FrameIndex:
@@ -2479,9 +2705,9 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
break;
default:
- if (Op.getOpcode() < ISD::BUILTIN_OP_END)
+ if (Opcode < ISD::BUILTIN_OP_END)
break;
- // Fallthrough
+ LLVM_FALLTHROUGH;
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_W_CHAIN:
case ISD::INTRINSIC_VOID:
@@ -2494,6 +2720,13 @@ void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
}
bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
+ EVT OpVT = Val.getValueType();
+ unsigned BitWidth = OpVT.getScalarSizeInBits();
+
+ // Is the constant a known power of 2?
+ if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
+ return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
+
// A left-shift of a constant one will have exactly one bit set because
// shifting the bit off the end is undefined.
if (Val.getOpcode() == ISD::SHL) {
@@ -2510,12 +2743,19 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
return true;
}
+ // Are all operands of a build vector constant powers of two?
+ if (Val.getOpcode() == ISD::BUILD_VECTOR)
+ if (llvm::all_of(Val->ops(), [this, BitWidth](SDValue E) {
+ if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
+ return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
+ return false;
+ }))
+ return true;
+
// More could be done here, though the above checks are enough
// to handle some common cases.
// Fall back to computeKnownBits to catch other known cases.
- EVT OpVT = Val.getValueType();
- unsigned BitWidth = OpVT.getScalarType().getSizeInBits();
APInt KnownZero, KnownOne;
computeKnownBits(Val, KnownZero, KnownOne);
return (KnownZero.countPopulation() == BitWidth - 1) &&
@@ -2525,7 +2765,7 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
EVT VT = Op.getValueType();
assert(VT.isInteger() && "Invalid VT!");
- unsigned VTBits = VT.getScalarType().getSizeInBits();
+ unsigned VTBits = VT.getScalarSizeInBits();
unsigned Tmp, Tmp2;
unsigned FirstAnswer = 1;
@@ -2547,14 +2787,12 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
}
case ISD::SIGN_EXTEND:
- Tmp =
- VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
+ Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
case ISD::SIGN_EXTEND_INREG:
// Max of the input and what this extends.
- Tmp =
- cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
+ Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
Tmp = VTBits-Tmp+1;
Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
@@ -2563,17 +2801,18 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
case ISD::SRA:
Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
// SRA X, C -> adds C sign bits.
- if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
- Tmp += C->getZExtValue();
- if (Tmp > VTBits) Tmp = VTBits;
+ if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1))) {
+ APInt ShiftVal = C->getAPIntValue();
+ ShiftVal += Tmp;
+ Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
}
return Tmp;
case ISD::SHL:
- if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+ if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1))) {
// shl destroys sign bits.
Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
- if (C->getZExtValue() >= VTBits || // Bad shift.
- C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
+ if (C->getAPIntValue().uge(VTBits) || // Bad shift.
+ C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out.
return Tmp - C->getZExtValue();
}
break;
@@ -2679,7 +2918,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
if (Tmp2 == 1) return 1;
// Handle NEG.
- if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
+ if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
if (CLHS->isNullValue()) {
APInt KnownZero, KnownOne;
computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
@@ -2701,25 +2940,50 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
if (Tmp == 1) return 1; // Early out.
return std::min(Tmp, Tmp2)-1;
- case ISD::TRUNCATE:
- // FIXME: it's tricky to do anything useful for this, but it is an important
- // case for targets like X86.
+ case ISD::TRUNCATE: {
+ // Check if the sign bits of source go down as far as the truncated value.
+ unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
+ unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+ if (NumSrcSignBits > (NumSrcBits - VTBits))
+ return NumSrcSignBits - (NumSrcBits - VTBits);
break;
+ }
case ISD::EXTRACT_ELEMENT: {
const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
- const int BitWidth = Op.getValueType().getSizeInBits();
- const int Items =
- Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
+ const int BitWidth = Op.getValueSizeInBits();
+ const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
// Get reverse index (starting from 1), Op1 value indexes elements from
// little end. Sign starts at big end.
- const int rIndex = Items - 1 -
- cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+ const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
// If the sign portion ends in our element the subtraction gives correct
// result. Otherwise it gives either negative or > bitwidth result
return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
}
+ case ISD::EXTRACT_VECTOR_ELT: {
+ // At the moment we keep this simple and skip tracking the specific
+ // element. This way we get the lowest common denominator for all elements
+ // of the vector.
+ // TODO: get information for given vector element
+ const unsigned BitWidth = Op.getValueSizeInBits();
+ const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
+ // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
+ // anything about sign bits. But if the sizes match we can derive knowledge
+ // about sign bits from the vector operand.
+ if (BitWidth == EltBitWidth)
+ return ComputeNumSignBits(Op.getOperand(0), Depth+1);
+ break;
+ }
+ case ISD::EXTRACT_SUBVECTOR:
+ return ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+ case ISD::CONCAT_VECTORS:
+ // Determine the minimum number of sign bits across all input vectors.
+ // Early out if the result is already 1.
+ Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
+ for (unsigned i = 1, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i)
+ Tmp = std::min(Tmp, ComputeNumSignBits(Op.getOperand(i), Depth + 1));
+ return Tmp;
}
// If we are looking at the loaded value of the SDNode.
@@ -2730,10 +2994,10 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
switch (ExtType) {
default: break;
case ISD::SEXTLOAD: // '17' bits known
- Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
+ Tmp = LD->getMemoryVT().getScalarSizeInBits();
return VTBits-Tmp+1;
case ISD::ZEXTLOAD: // '16' bits known
- Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
+ Tmp = LD->getMemoryVT().getScalarSizeInBits();
return VTBits-Tmp;
}
}
@@ -2842,6 +3106,16 @@ bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
ArrayRef<SDValue> Ops,
llvm::SelectionDAG &DAG) {
+ assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
+ assert(llvm::all_of(Ops,
+ [Ops](SDValue Op) {
+ return Ops[0].getValueType() == Op.getValueType();
+ }) &&
+ "Concatenation of vectors with inconsistent value types!");
+ assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
+ VT.getVectorNumElements() &&
+ "Incorrect element count in vector concatenation!");
+
if (Ops.size() == 1)
return Ops[0];
@@ -2875,7 +3149,7 @@ static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
? DAG.getZExtOrTrunc(Op, DL, SVT)
: DAG.getSExtOrTrunc(Op, DL, SVT);
- return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
+ return DAG.getBuildVector(VT, DL, Elts);
}
/// Gets or creates the specified node.
@@ -2924,13 +3198,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
}
case ISD::BITCAST:
if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
- return getConstantFP(APFloat(APFloat::IEEEhalf, Val), DL, VT);
+ return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
- return getConstantFP(APFloat(APFloat::IEEEsingle, Val), DL, VT);
+ return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
- return getConstantFP(APFloat(APFloat::IEEEdouble, Val), DL, VT);
+ return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
- return getConstantFP(APFloat(APFloat::IEEEquad, Val), DL, VT);
+ return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
break;
case ISD::BSWAP:
return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
@@ -3162,8 +3436,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
break;
case ISD::BITCAST:
// Basic sanity checking.
- assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
- && "Cannot BITCAST between types of different sizes!");
+ assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
+ "Cannot BITCAST between types of different sizes!");
if (VT == Operand.getValueType()) return Operand; // noop conversion.
if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
@@ -3333,25 +3607,22 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
EVT SVT = VT.getScalarType();
SmallVector<SDValue, 4> Outputs;
for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
- ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
- ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
- if (!V1 || !V2) // Not a constant, bail.
- return SDValue();
-
- if (V1->isOpaque() || V2->isOpaque())
- return SDValue();
+ SDValue V1 = BV1->getOperand(I);
+ SDValue V2 = BV2->getOperand(I);
// Avoid BUILD_VECTOR nodes that perform implicit truncation.
- // FIXME: This is valid and could be handled by truncating the APInts.
+ // FIXME: This is valid and could be handled by truncation.
if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
return SDValue();
// Fold one vector element.
- std::pair<APInt, bool> Folded = FoldValue(Opcode, V1->getAPIntValue(),
- V2->getAPIntValue());
- if (!Folded.second)
+ SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
+
+ // Scalar folding only succeeded if the result is a constant or UNDEF.
+ if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
+ ScalarResult.getOpcode() != ISD::ConstantFP)
return SDValue();
- Outputs.push_back(getConstant(Folded.first, DL, SVT));
+ Outputs.push_back(ScalarResult);
}
assert(VT.getVectorNumElements() == Outputs.size() &&
@@ -3394,8 +3665,8 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
// All operands must be vector types with the same number of elements as
// the result type and must be either UNDEF or a build vector of constant
// or UNDEF scalars.
- if (!std::all_of(Ops.begin(), Ops.end(), IsConstantBuildVectorOrUndef) ||
- !std::all_of(Ops.begin(), Ops.end(), IsScalarOrSameVectorSize))
+ if (!all_of(Ops, IsConstantBuildVectorOrUndef) ||
+ !all_of(Ops, IsScalarOrSameVectorSize))
return SDValue();
// If we are comparing vectors, then the result needs to be a i1 boolean
@@ -3577,8 +3848,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
// amounts. This catches things like trying to shift an i1024 value by an
// i8, which is easy to fall into in generic code that uses
// TLI.getShiftAmount().
- assert(N2.getValueType().getSizeInBits() >=
- Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
+ assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
"Invalid use of small shift amount with oversized value!");
// Always fold shifts of i1 values so the code generator doesn't need to
@@ -3609,7 +3879,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
assert(VT.isFloatingPoint() &&
N1.getValueType().isFloatingPoint() &&
VT.bitsLE(N1.getValueType()) &&
- N2C && "Invalid FP_ROUND!");
+ N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
+ "Invalid FP_ROUND!");
if (N1.getValueType() == VT) return N1; // noop conversion.
break;
case ISD::AssertSext:
@@ -3640,7 +3911,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
if (EVT == VT) return N1; // Not actually extending
auto SignExtendInReg = [&](APInt Val) {
- unsigned FromBits = EVT.getScalarType().getSizeInBits();
+ unsigned FromBits = EVT.getScalarSizeInBits();
Val <<= Val.getBitWidth() - FromBits;
Val = Val.ashr(Val.getBitWidth() - FromBits);
return getConstant(Val, DL, VT.getScalarType());
@@ -3768,6 +4039,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
// Trivial extraction.
if (VT.getSimpleVT() == N1.getSimpleValueType())
return N1;
+
+ // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
+ // during shuffle legalization.
+ if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
+ VT == N1.getOperand(1).getValueType())
+ return N1.getOperand(1);
}
break;
}
@@ -3868,7 +4145,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
// Handle undef ^ undef -> 0 special case. This is a common
// idiom (misuse).
return getConstant(0, DL, VT);
- // fallthrough
+ LLVM_FALLTHROUGH;
case ISD::ADD:
case ISD::ADDC:
case ISD::ADDE:
@@ -3977,6 +4254,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
break;
case ISD::VECTOR_SHUFFLE:
llvm_unreachable("should use getVectorShuffle constructor!");
+ case ISD::INSERT_VECTOR_ELT: {
+ ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
+ // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
+ if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
+ return getUNDEF(VT);
+ break;
+ }
case ISD::INSERT_SUBVECTOR: {
SDValue Index = N3;
if (VT.isSimple() && N1.getValueType().isSimple()
@@ -4072,7 +4356,7 @@ static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
const SDLoc &dl) {
assert(!Value.isUndef());
- unsigned NumBits = VT.getScalarType().getSizeInBits();
+ unsigned NumBits = VT.getScalarSizeInBits();
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
assert(C->getAPIntValue().getBitWidth() == 8);
APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
@@ -4306,10 +4590,10 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
std::vector<EVT> MemOps;
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
- MachineFrameInfo *MFI = MF.getFrameInfo();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
- if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
unsigned SrcAlign = DAG.InferPtrAlignment(Src);
if (Align > SrcAlign)
@@ -4342,8 +4626,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
if (NewAlign > Align) {
// Give the stack frame object a larger alignment if needed.
- if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
- MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI.setObjectAlignment(FI->getIndex(), NewAlign);
Align = NewAlign;
}
}
@@ -4422,10 +4706,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
std::vector<EVT> MemOps;
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
- MachineFrameInfo *MFI = MF.getFrameInfo();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
- if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
unsigned SrcAlign = DAG.InferPtrAlignment(Src);
if (Align > SrcAlign)
@@ -4445,8 +4729,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
if (NewAlign > Align) {
// Give the stack frame object a larger alignment if needed.
- if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
- MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI.setObjectAlignment(FI->getIndex(), NewAlign);
Align = NewAlign;
}
}
@@ -4519,10 +4803,10 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
std::vector<EVT> MemOps;
bool DstAlignCanChange = false;
MachineFunction &MF = DAG.getMachineFunction();
- MachineFrameInfo *MFI = MF.getFrameInfo();
+ MachineFrameInfo &MFI = MF.getFrameInfo();
bool OptSize = shouldLowerMemFuncForSize(MF);
FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
- if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
+ if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
DstAlignCanChange = true;
bool IsZeroVal =
isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
@@ -4538,8 +4822,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
if (NewAlign > Align) {
// Give the stack frame object a larger alignment if needed.
- if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
- MFI->setObjectAlignment(FI->getIndex(), NewAlign);
+ if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
+ MFI.setObjectAlignment(FI->getIndex(), NewAlign);
Align = NewAlign;
}
}
@@ -4796,10 +5080,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTList, ArrayRef<SDValue> Ops,
- MachineMemOperand *MMO,
- AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
FoldingSetNodeID ID;
ID.AddInteger(MemVT.getRawBits());
AddNodeIDNode(ID, Opcode, VTList, Ops);
@@ -4811,8 +5092,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
}
auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
- VTList, MemVT, MMO, SuccessOrdering,
- FailureOrdering, SynchScope);
+ VTList, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
@@ -4820,14 +5100,6 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
return SDValue(N, 0);
}
-SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
- SDVTList VTList, ArrayRef<SDValue> Ops,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
- return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
- Ordering, SynchScope);
-}
-
SDValue SelectionDAG::getAtomicCmpSwap(
unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
@@ -4847,26 +5119,23 @@ SDValue SelectionDAG::getAtomicCmpSwap(
auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
MachineMemOperand::MOStore;
MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
+ MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
+ AAMDNodes(), nullptr, SynchScope, SuccessOrdering,
+ FailureOrdering);
- return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
- SuccessOrdering, FailureOrdering, SynchScope);
+ return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
}
SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
EVT MemVT, SDVTList VTs, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp,
- MachineMemOperand *MMO,
- AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
- SuccessOrdering, FailureOrdering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
@@ -4892,16 +5161,15 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
MachineMemOperand *MMO =
MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
- MemVT.getStoreSize(), Alignment);
+ MemVT.getStoreSize(), Alignment, AAMDNodes(),
+ nullptr, SynchScope, Ordering);
- return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
- Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDValue Chain, SDValue Ptr, SDValue Val,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB ||
Opcode == ISD::ATOMIC_LOAD_AND ||
@@ -4921,18 +5189,17 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr, Val};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
EVT VT, SDValue Chain, SDValue Ptr,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
SDVTList VTs = getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
/// getMergeValues - Create a MERGE_VALUES node from the given operands.
@@ -5056,7 +5323,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
if (Alignment == 0) // Ensure that codegen never sees alignment 0
- Alignment = getEVTAlignment(VT);
+ Alignment = getEVTAlignment(MemVT);
MMOFlags |= MachineMemOperand::MOLoad;
assert((MMOFlags & MachineMemOperand::MOStore) == 0);
@@ -5101,9 +5368,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
ID.AddInteger(MemVT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
- MMO->isNonTemporal(),
- MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
+ dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5160,12 +5426,14 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
ISD::MemIndexedMode AM) {
LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
- // Don't propagate the invariant flag.
+ // Don't propagate the invariant or dereferenceable flags.
auto MMOFlags =
- LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant;
+ LD->getMemOperand()->getFlags() &
+ ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
LD->getChain(), Base, Offset, LD->getPointerInfo(),
- LD->getMemoryVT(), LD->getAlignment(), MMOFlags);
+ LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
+ LD->getAAInfo());
}
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
@@ -5200,8 +5468,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
- MMO->isNonTemporal(), MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
+ dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5265,8 +5533,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
ID.AddInteger(SVT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
- MMO->isNonTemporal(), MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
+ dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5311,17 +5579,15 @@ SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
SDValue Ptr, SDValue Mask, SDValue Src0,
EVT MemVT, MachineMemOperand *MMO,
- ISD::LoadExtType ExtTy) {
+ ISD::LoadExtType ExtTy, bool isExpanding) {
SDVTList VTs = getVTList(VT, MVT::Other);
SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
- MMO->isVolatile(),
- MMO->isNonTemporal(),
- MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
+ dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5329,7 +5595,7 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
- ExtTy, MemVT, MMO);
+ ExtTy, isExpanding, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
@@ -5340,7 +5606,7 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
SDValue Val, SDValue Ptr, SDValue Mask,
EVT MemVT, MachineMemOperand *MMO,
- bool isTrunc) {
+ bool IsTruncating, bool IsCompressing) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
EVT VT = Val.getValueType();
@@ -5349,8 +5615,8 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
- MMO->isNonTemporal(), MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
+ dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5358,7 +5624,7 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
- isTrunc, MemVT, MMO);
+ IsTruncating, IsCompressing, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
@@ -5374,10 +5640,8 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED,
- MMO->isVolatile(),
- MMO->isNonTemporal(),
- MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
+ dl.getIROrder(), VTs, VT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5411,9 +5675,8 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
- MMO->isNonTemporal(),
- MMO->isInvariant()));
+ ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
+ dl.getIROrder(), VTs, VT, MMO));
ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -5545,7 +5808,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
// If the and is only masking out bits that cannot effect the shift,
// eliminate the and.
- unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
+ unsigned NumBits = VT.getScalarSizeInBits()*2;
if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
}
@@ -5870,21 +6133,6 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
}
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
- EVT VT1, EVT VT2, EVT VT3, EVT VT4,
- ArrayRef<SDValue> Ops) {
- SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
- return SelectNodeTo(N, MachineOpc, VTs, Ops);
-}
-
-SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
- EVT VT1, EVT VT2,
- SDValue Op1) {
- SDVTList VTs = getVTList(VT1, VT2);
- SDValue Ops[] = { Op1 };
- return SelectNodeTo(N, MachineOpc, VTs, Ops);
-}
-
-SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
EVT VT1, EVT VT2,
SDValue Op1, SDValue Op2) {
SDVTList VTs = getVTList(VT1, VT2);
@@ -5893,24 +6141,6 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
}
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
- EVT VT1, EVT VT2,
- SDValue Op1, SDValue Op2,
- SDValue Op3) {
- SDVTList VTs = getVTList(VT1, VT2);
- SDValue Ops[] = { Op1, Op2, Op3 };
- return SelectNodeTo(N, MachineOpc, VTs, Ops);
-}
-
-SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
- EVT VT1, EVT VT2, EVT VT3,
- SDValue Op1, SDValue Op2,
- SDValue Op3) {
- SDVTList VTs = getVTList(VT1, VT2, VT3);
- SDValue Ops[] = { Op1, Op2, Op3 };
- return SelectNodeTo(N, MachineOpc, VTs, Ops);
-}
-
-SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
SDVTList VTs,ArrayRef<SDValue> Ops) {
SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
// Reset the NodeID to -1.
@@ -5922,14 +6152,14 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
return New;
}
-/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
+/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
/// the line number information on the merged node since it is not possible to
/// preserve the information that operation is associated with multiple lines.
/// This will make the debugger working better at -O0, were there is a higher
/// probability having other instructions associated with that line.
///
/// For IROrder, we keep the smaller of the two
-SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, const SDLoc &OLoc) {
+SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
DebugLoc NLoc = N->getDebugLoc();
if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
N->setDebugLoc(DebugLoc());
@@ -5963,7 +6193,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opc, VTs, Ops);
if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
- return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
+ return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
}
if (!RemoveNodeFromCSEMaps(N))
@@ -6050,19 +6280,6 @@ MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
}
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
- EVT VT1, EVT VT2) {
- SDVTList VTs = getVTList(VT1, VT2);
- return getMachineNode(Opcode, dl, VTs, None);
-}
-
-MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
- EVT VT1, EVT VT2, SDValue Op1) {
- SDVTList VTs = getVTList(VT1, VT2);
- SDValue Ops[] = { Op1 };
- return getMachineNode(Opcode, dl, VTs, Ops);
-}
-
-MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
EVT VT1, EVT VT2, SDValue Op1,
SDValue Op2) {
SDVTList VTs = getVTList(VT1, VT2);
@@ -6110,13 +6327,6 @@ MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
}
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
- EVT VT1, EVT VT2, EVT VT3, EVT VT4,
- ArrayRef<SDValue> Ops) {
- SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
- return getMachineNode(Opcode, dl, VTs, Ops);
-}
-
-MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
ArrayRef<EVT> ResultTys,
ArrayRef<SDValue> Ops) {
SDVTList VTs = getVTList(ResultTys);
@@ -6135,7 +6345,7 @@ MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
AddNodeIDNode(ID, ~Opcode, VTs, Ops);
IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
- return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
+ return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
}
}
@@ -6255,6 +6465,9 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
"Cannot replace with this method!");
assert(From != To.getNode() && "Cannot replace uses of with self");
+ // Preserve Debug Values
+ TransferDbgValues(FromN, To);
+
// Iterate over all the existing uses of From. New uses will be added
// to the beginning of the use list, which we avoid visiting.
// This specifically avoids visiting uses of From that arise while the
@@ -6285,8 +6498,6 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
AddModifiedNodeToCSEMaps(User);
}
- // Preserve Debug Values
- TransferDbgValues(FromN, To);
// If we just RAUW'd the root, take note.
if (FromN == getRoot())
@@ -6689,6 +6900,40 @@ bool llvm::isBitwiseNot(SDValue V) {
return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1));
}
+ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) {
+ if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
+ return CN;
+
+ if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
+ BitVector UndefElements;
+ ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
+
+ // BuildVectors can truncate their operands. Ignore that case here.
+ // FIXME: We blindly ignore splats which include undef which is overly
+ // pessimistic.
+ if (CN && UndefElements.none() &&
+ CN->getValueType(0) == N.getValueType().getScalarType())
+ return CN;
+ }
+
+ return nullptr;
+}
+
+ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) {
+ if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
+ return CN;
+
+ if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
+ BitVector UndefElements;
+ ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
+
+ if (CN && UndefElements.none())
+ return CN;
+ }
+
+ return nullptr;
+}
+
HandleSDNode::~HandleSDNode() {
DropOperands();
}
@@ -6710,11 +6955,11 @@ AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
: SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
- SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
- MMO->isNonTemporal(), MMO->isInvariant());
- assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
- assert(isNonTemporal() == MMO->isNonTemporal() &&
- "Non-temporal encoding error!");
+ MemSDNodeBits.IsVolatile = MMO->isVolatile();
+ MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
+ MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
+ MemSDNodeBits.IsInvariant = MMO->isInvariant();
+
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
@@ -6939,8 +7184,8 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
for (; i < ResNE; ++i)
Scalars.push_back(getUNDEF(EltVT));
- return getNode(ISD::BUILD_VECTOR, dl,
- EVT::getVectorVT(*getContext(), EltVT, ResNE), Scalars);
+ EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
+ return getBuildVector(VecVT, dl, Scalars);
}
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
@@ -6962,13 +7207,13 @@ bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
if (Loc.getOpcode() == ISD::FrameIndex) {
if (BaseLoc.getOpcode() != ISD::FrameIndex)
return false;
- const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo();
+ const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
- int FS = MFI->getObjectSize(FI);
- int BFS = MFI->getObjectSize(BFI);
+ int FS = MFI.getObjectSize(FI);
+ int BFS = MFI.getObjectSize(BFI);
if (FS != BFS || FS != (int)Bytes) return false;
- return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
+ return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes);
}
// Handle X + C.
@@ -7033,7 +7278,7 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
}
if (FrameIdx != (1 << 31)) {
- const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo();
+ const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
FrameOffset);
return FIInfoAlign;
@@ -7124,7 +7369,7 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
// false.
unsigned int nOps = getNumOperands();
assert(nOps > 0 && "isConstantSplat has 0-size build vector");
- unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
+ unsigned EltBitSize = VT.getScalarSizeInBits();
for (unsigned j = 0; j < nOps; ++j) {
unsigned i = isBigEndian ? nOps-1-j : j;
@@ -7265,6 +7510,16 @@ SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
return nullptr;
}
+SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
+ if (isa<ConstantFPSDNode>(N))
+ return N.getNode();
+
+ if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
+ return N.getNode();
+
+ return nullptr;
+}
+
#ifndef NDEBUG
static void checkForCyclesHelper(const SDNode *N,
SmallPtrSetImpl<const SDNode*> &Visited,
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.