diff options
Diffstat (limited to 'gnu/llvm/lib/IR/AutoUpgrade.cpp')
| -rw-r--r-- | gnu/llvm/lib/IR/AutoUpgrade.cpp | 576 |
1 files changed, 343 insertions, 233 deletions
diff --git a/gnu/llvm/lib/IR/AutoUpgrade.cpp b/gnu/llvm/lib/IR/AutoUpgrade.cpp index f098ad9725b..27064154221 100644 --- a/gnu/llvm/lib/IR/AutoUpgrade.cpp +++ b/gnu/llvm/lib/IR/AutoUpgrade.cpp @@ -71,7 +71,27 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { // like to use this information to remove upgrade code for some older // intrinsics. It is currently undecided how we will determine that future // point. - if (Name=="ssse3.pabs.b.128" || // Added in 6.0 + if (Name == "addcarryx.u32" || // Added in 8.0 + Name == "addcarryx.u64" || // Added in 8.0 + Name == "addcarry.u32" || // Added in 8.0 + Name == "addcarry.u64" || // Added in 8.0 + Name == "subborrow.u32" || // Added in 8.0 + Name == "subborrow.u64" || // Added in 8.0 + Name.startswith("sse2.padds.") || // Added in 8.0 + Name.startswith("sse2.psubs.") || // Added in 8.0 + Name.startswith("sse2.paddus.") || // Added in 8.0 + Name.startswith("sse2.psubus.") || // Added in 8.0 + Name.startswith("avx2.padds.") || // Added in 8.0 + Name.startswith("avx2.psubs.") || // Added in 8.0 + Name.startswith("avx2.paddus.") || // Added in 8.0 + Name.startswith("avx2.psubus.") || // Added in 8.0 + Name.startswith("avx512.padds.") || // Added in 8.0 + Name.startswith("avx512.psubs.") || // Added in 8.0 + Name.startswith("avx512.mask.padds.") || // Added in 8.0 + Name.startswith("avx512.mask.psubs.") || // Added in 8.0 + Name.startswith("avx512.mask.paddus.") || // Added in 8.0 + Name.startswith("avx512.mask.psubus.") || // Added in 8.0 + Name=="ssse3.pabs.b.128" || // Added in 6.0 Name=="ssse3.pabs.w.128" || // Added in 6.0 Name=="ssse3.pabs.d.128" || // Added in 6.0 Name.startswith("fma4.vfmadd.s") || // Added in 7.0 @@ -265,6 +285,12 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0 Name.startswith("avx512.mask.vpshld.") || // Added in 7.0 Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0 + Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0 + Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0 + Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0 + Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0 + Name.startswith("avx512.vpshld.") || // Added in 8.0 + Name.startswith("avx512.vpshrd.") || // Added in 8.0 Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0 Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0 Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0 @@ -272,10 +298,8 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0 Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0 Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0 - Name.startswith("avx512.mask.prorv.") || // Added in 7.0 - Name.startswith("avx512.mask.pror.") || // Added in 7.0 - Name.startswith("avx512.mask.prolv.") || // Added in 7.0 - Name.startswith("avx512.mask.prol.") || // Added in 7.0 + Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0 + Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0 Name == "sse.cvtsi2ss" || // Added in 7.0 Name == "sse.cvtsi642ss" || // Added in 7.0 Name == "sse2.cvtsi2sd" || // Added in 7.0 @@ -340,6 +364,13 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("avx512.cvtmask2") || // Added in 5.0 (Name.startswith("xop.vpcom") && // Added in 3.2 F->arg_size() == 2) || + Name.startswith("xop.vprot") || // Added in 8.0 + Name.startswith("avx512.prol") || // Added in 8.0 + Name.startswith("avx512.pror") || // Added in 8.0 + Name.startswith("avx512.mask.prorv.") || // Added in 8.0 + Name.startswith("avx512.mask.pror.") || // Added in 8.0 + Name.startswith("avx512.mask.prolv.") || // Added in 8.0 + Name.startswith("avx512.mask.prol.") || // Added in 8.0 Name.startswith("avx512.ptestm") || //Added in 6.0 Name.startswith("avx512.ptestnm") || //Added in 6.0 Name.startswith("sse2.pavg") || // Added in 6.0 @@ -363,6 +394,17 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, return true; } + if (Name == "rdtscp") { // Added in 8.0 + // If this intrinsic has 0 operands, it's the new version. + if (F->getFunctionType()->getNumParams() == 0) + return false; + + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::x86_rdtscp); + return true; + } + // SSE4.1 ptest functions may have an old signature. if (Name.startswith("sse41.ptest")) { // Added in 3.2 if (Name.substr(11) == "c") @@ -427,6 +469,11 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, } } + if (Name == "seh.recoverfp") { + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); + return true; + } + return false; } @@ -456,7 +503,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { // the end of the name. Change name from llvm.arm.neon.vclz.* to // llvm.ctlz.* FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); - NewFn = Function::Create(fType, F->getLinkage(), + NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), "llvm.ctlz." + Name.substr(14), F->getParent()); return true; } @@ -472,7 +519,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { // Can't use Intrinsic::getDeclaration here as the return types might // then only be structurally equal. FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false); - NewFn = Function::Create(fType, F->getLinkage(), + NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), "llvm." + Name + ".p0i8", F->getParent()); return true; } @@ -899,6 +946,148 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, return EmitX86Select(Builder, Mask, Align, Passthru); } +static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI, + bool ZeroMask, bool IndexForm) { + Type *Ty = CI.getType(); + unsigned VecWidth = Ty->getPrimitiveSizeInBits(); + unsigned EltWidth = Ty->getScalarSizeInBits(); + bool IsFloat = Ty->isFPOrFPVectorTy(); + Intrinsic::ID IID; + if (VecWidth == 128 && EltWidth == 32 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_ps_128; + else if (VecWidth == 128 && EltWidth == 32 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_d_128; + else if (VecWidth == 128 && EltWidth == 64 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_pd_128; + else if (VecWidth == 128 && EltWidth == 64 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_q_128; + else if (VecWidth == 256 && EltWidth == 32 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_ps_256; + else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_d_256; + else if (VecWidth == 256 && EltWidth == 64 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_pd_256; + else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_q_256; + else if (VecWidth == 512 && EltWidth == 32 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_ps_512; + else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_d_512; + else if (VecWidth == 512 && EltWidth == 64 && IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_pd_512; + else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) + IID = Intrinsic::x86_avx512_vpermi2var_q_512; + else if (VecWidth == 128 && EltWidth == 16) + IID = Intrinsic::x86_avx512_vpermi2var_hi_128; + else if (VecWidth == 256 && EltWidth == 16) + IID = Intrinsic::x86_avx512_vpermi2var_hi_256; + else if (VecWidth == 512 && EltWidth == 16) + IID = Intrinsic::x86_avx512_vpermi2var_hi_512; + else if (VecWidth == 128 && EltWidth == 8) + IID = Intrinsic::x86_avx512_vpermi2var_qi_128; + else if (VecWidth == 256 && EltWidth == 8) + IID = Intrinsic::x86_avx512_vpermi2var_qi_256; + else if (VecWidth == 512 && EltWidth == 8) + IID = Intrinsic::x86_avx512_vpermi2var_qi_512; + else + llvm_unreachable("Unexpected intrinsic"); + + Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), + CI.getArgOperand(2) }; + + // If this isn't index form we need to swap operand 0 and 1. + if (!IndexForm) + std::swap(Args[0], Args[1]); + + Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), + Args); + Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) + : Builder.CreateBitCast(CI.getArgOperand(1), + Ty); + return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru); +} + +static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI, + bool IsSigned, bool IsAddition) { + Type *Ty = CI.getType(); + Value *Op0 = CI.getOperand(0); + Value *Op1 = CI.getOperand(1); + + Intrinsic::ID IID = + IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat) + : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat); + Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); + Value *Res = Builder.CreateCall(Intrin, {Op0, Op1}); + + if (CI.getNumArgOperands() == 4) { // For masked intrinsics. + Value *VecSrc = CI.getOperand(2); + Value *Mask = CI.getOperand(3); + Res = EmitX86Select(Builder, Mask, Res, VecSrc); + } + return Res; +} + +static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI, + bool IsRotateRight) { + Type *Ty = CI.getType(); + Value *Src = CI.getArgOperand(0); + Value *Amt = CI.getArgOperand(1); + + // Amount may be scalar immediate, in which case create a splat vector. + // Funnel shifts amounts are treated as modulo and types are all power-of-2 so + // we only care about the lowest log2 bits anyway. + if (Amt->getType() != Ty) { + unsigned NumElts = Ty->getVectorNumElements(); + Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); + Amt = Builder.CreateVectorSplat(NumElts, Amt); + } + + Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; + Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); + Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt}); + + if (CI.getNumArgOperands() == 4) { // For masked intrinsics. + Value *VecSrc = CI.getOperand(2); + Value *Mask = CI.getOperand(3); + Res = EmitX86Select(Builder, Mask, Res, VecSrc); + } + return Res; +} + +static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI, + bool IsShiftRight, bool ZeroMask) { + Type *Ty = CI.getType(); + Value *Op0 = CI.getArgOperand(0); + Value *Op1 = CI.getArgOperand(1); + Value *Amt = CI.getArgOperand(2); + + if (IsShiftRight) + std::swap(Op0, Op1); + + // Amount may be scalar immediate, in which case create a splat vector. + // Funnel shifts amounts are treated as modulo and types are all power-of-2 so + // we only care about the lowest log2 bits anyway. + if (Amt->getType() != Ty) { + unsigned NumElts = Ty->getVectorNumElements(); + Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); + Amt = Builder.CreateVectorSplat(NumElts, Amt); + } + + Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; + Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); + Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt}); + + unsigned NumArgs = CI.getNumArgOperands(); + if (NumArgs >= 4) { // For masked intrinsics. + Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : + ZeroMask ? ConstantAggregateZero::get(CI.getType()) : + CI.getArgOperand(0); + Value *Mask = CI.getOperand(NumArgs - 1); + Res = EmitX86Select(Builder, Mask, Res, VecSrc); + } + return Res; +} + static Value *UpgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned) { @@ -1265,106 +1454,13 @@ static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, IID = Intrinsic::x86_avx512_dbpsadbw_512; else llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("vpshld.")) { - if (VecWidth == 128 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshld_q_128; - else if (VecWidth == 128 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshld_d_128; - else if (VecWidth == 128 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshld_w_128; - else if (VecWidth == 256 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshld_q_256; - else if (VecWidth == 256 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshld_d_256; - else if (VecWidth == 256 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshld_w_256; - else if (VecWidth == 512 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshld_q_512; - else if (VecWidth == 512 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshld_d_512; - else if (VecWidth == 512 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshld_w_512; - else - llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("vpshrd.")) { - if (VecWidth == 128 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshrd_q_128; - else if (VecWidth == 128 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshrd_d_128; - else if (VecWidth == 128 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshrd_w_128; - else if (VecWidth == 256 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshrd_q_256; - else if (VecWidth == 256 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshrd_d_256; - else if (VecWidth == 256 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshrd_w_256; - else if (VecWidth == 512 && Name[7] == 'q') - IID = Intrinsic::x86_avx512_vpshrd_q_512; - else if (VecWidth == 512 && Name[7] == 'd') - IID = Intrinsic::x86_avx512_vpshrd_d_512; - else if (VecWidth == 512 && Name[7] == 'w') - IID = Intrinsic::x86_avx512_vpshrd_w_512; - else - llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("prorv.")) { - if (VecWidth == 128 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prorv_d_128; - else if (VecWidth == 256 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prorv_d_256; - else if (VecWidth == 512 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prorv_d_512; - else if (VecWidth == 128 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prorv_q_128; - else if (VecWidth == 256 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prorv_q_256; - else if (VecWidth == 512 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prorv_q_512; - else - llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("prolv.")) { - if (VecWidth == 128 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prolv_d_128; - else if (VecWidth == 256 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prolv_d_256; - else if (VecWidth == 512 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prolv_d_512; - else if (VecWidth == 128 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prolv_q_128; - else if (VecWidth == 256 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prolv_q_256; - else if (VecWidth == 512 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prolv_q_512; - else - llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("pror.")) { - if (VecWidth == 128 && EltWidth == 32) - IID = Intrinsic::x86_avx512_pror_d_128; - else if (VecWidth == 256 && EltWidth == 32) - IID = Intrinsic::x86_avx512_pror_d_256; - else if (VecWidth == 512 && EltWidth == 32) - IID = Intrinsic::x86_avx512_pror_d_512; - else if (VecWidth == 128 && EltWidth == 64) - IID = Intrinsic::x86_avx512_pror_q_128; - else if (VecWidth == 256 && EltWidth == 64) - IID = Intrinsic::x86_avx512_pror_q_256; - else if (VecWidth == 512 && EltWidth == 64) - IID = Intrinsic::x86_avx512_pror_q_512; - else - llvm_unreachable("Unexpected intrinsic"); - } else if (Name.startswith("prol.")) { - if (VecWidth == 128 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prol_d_128; - else if (VecWidth == 256 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prol_d_256; - else if (VecWidth == 512 && EltWidth == 32) - IID = Intrinsic::x86_avx512_prol_d_512; - else if (VecWidth == 128 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prol_q_128; - else if (VecWidth == 256 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prol_q_256; - else if (VecWidth == 512 && EltWidth == 64) - IID = Intrinsic::x86_avx512_prol_q_512; + } else if (Name.startswith("pmultishift.qb.")) { + if (VecWidth == 128) + IID = Intrinsic::x86_avx512_pmultishift_qb_128; + else if (VecWidth == 256) + IID = Intrinsic::x86_avx512_pmultishift_qb_256; + else if (VecWidth == 512) + IID = Intrinsic::x86_avx512_pmultishift_qb_512; else llvm_unreachable("Unexpected intrinsic"); } else @@ -1654,46 +1750,44 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { C = ConstantInt::getNullValue(Builder.getInt16Ty()); Rep = Builder.CreateICmpEQ(Rep, C); Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty()); - } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) { + } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" || + Name == "sse.sub.ss" || Name == "sse2.sub.sd" || + Name == "sse.mul.ss" || Name == "sse2.mul.sd" || + Name == "sse.div.ss" || Name == "sse2.div.sd")) { Type *I32Ty = Type::getInt32Ty(C); Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), ConstantInt::get(I32Ty, 0)); Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), ConstantInt::get(I32Ty, 0)); - Rep = Builder.CreateInsertElement(CI->getArgOperand(0), - Builder.CreateFAdd(Elt0, Elt1), - ConstantInt::get(I32Ty, 0)); - } else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) { - Type *I32Ty = Type::getInt32Ty(C); - Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), - ConstantInt::get(I32Ty, 0)); - Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), - ConstantInt::get(I32Ty, 0)); - Rep = Builder.CreateInsertElement(CI->getArgOperand(0), - Builder.CreateFSub(Elt0, Elt1), - ConstantInt::get(I32Ty, 0)); - } else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) { - Type *I32Ty = Type::getInt32Ty(C); - Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), - ConstantInt::get(I32Ty, 0)); - Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), - ConstantInt::get(I32Ty, 0)); - Rep = Builder.CreateInsertElement(CI->getArgOperand(0), - Builder.CreateFMul(Elt0, Elt1), - ConstantInt::get(I32Ty, 0)); - } else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) { - Type *I32Ty = Type::getInt32Ty(C); - Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), - ConstantInt::get(I32Ty, 0)); - Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), - ConstantInt::get(I32Ty, 0)); - Rep = Builder.CreateInsertElement(CI->getArgOperand(0), - Builder.CreateFDiv(Elt0, Elt1), + Value *EltOp; + if (Name.contains(".add.")) + EltOp = Builder.CreateFAdd(Elt0, Elt1); + else if (Name.contains(".sub.")) + EltOp = Builder.CreateFSub(Elt0, Elt1); + else if (Name.contains(".mul.")) + EltOp = Builder.CreateFMul(Elt0, Elt1); + else + EltOp = Builder.CreateFDiv(Elt0, Elt1); + Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, ConstantInt::get(I32Ty, 0)); } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) { // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt." bool CmpEq = Name[16] == 'e'; Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); + } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) { + Type *OpTy = CI->getArgOperand(0)->getType(); + unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); + Intrinsic::ID IID; + switch (VecWidth) { + default: llvm_unreachable("Unexpected intrinsic"); + case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break; + case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break; + case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break; + } + + Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), + { CI->getOperand(0), CI->getArgOperand(1) }); + Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) { Type *OpTy = CI->getArgOperand(0)->getType(); unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); @@ -1948,6 +2042,23 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); Rep = Builder.CreateOr(Sel0, Sel1); + } else if (IsX86 && (Name.startswith("xop.vprot") || + Name.startswith("avx512.prol") || + Name.startswith("avx512.mask.prol"))) { + Rep = upgradeX86Rotate(Builder, *CI, false); + } else if (IsX86 && (Name.startswith("avx512.pror") || + Name.startswith("avx512.mask.pror"))) { + Rep = upgradeX86Rotate(Builder, *CI, true); + } else if (IsX86 && (Name.startswith("avx512.vpshld.") || + Name.startswith("avx512.mask.vpshld") || + Name.startswith("avx512.maskz.vpshld"))) { + bool ZeroMask = Name[11] == 'z'; + Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); + } else if (IsX86 && (Name.startswith("avx512.vpshrd.") || + Name.startswith("avx512.mask.vpshrd") || + Name.startswith("avx512.maskz.vpshrd"))) { + bool ZeroMask = Name[11] == 'z'; + Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); } else if (IsX86 && Name == "sse42.crc32.64.8") { Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_sse42_crc32_32_8); @@ -2059,6 +2170,24 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (CI->getNumArgOperands() == 3) Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); + } else if (IsX86 && (Name.startswith("sse2.padds.") || + Name.startswith("sse2.psubs.") || + Name.startswith("avx2.padds.") || + Name.startswith("avx2.psubs.") || + Name.startswith("avx512.padds.") || + Name.startswith("avx512.psubs.") || + Name.startswith("avx512.mask.padds.") || + Name.startswith("avx512.mask.psubs."))) { + bool IsAdd = Name.contains(".padds"); + Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd); + } else if (IsX86 && (Name.startswith("sse2.paddus.") || + Name.startswith("sse2.psubus.") || + Name.startswith("avx2.paddus.") || + Name.startswith("avx2.psubus.") || + Name.startswith("avx512.mask.paddus.") || + Name.startswith("avx512.mask.psubus."))) { + bool IsAdd = Name.contains(".paddus"); + Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd); } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) { Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), CI->getArgOperand(1), @@ -2376,24 +2505,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.pand.")) { - Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1)); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, - CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.pandn.")) { - Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)), - CI->getArgOperand(1)); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, - CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.por.")) { - Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1)); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, - CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.pxor.")) { - Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1)); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, - CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.and.")) { + } else if (IsX86 && (Name.startswith("avx512.mask.and.") || + Name.startswith("avx512.mask.pand."))) { VectorType *FTy = cast<VectorType>(CI->getType()); VectorType *ITy = VectorType::getInteger(FTy); Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), @@ -2401,7 +2514,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateBitCast(Rep, FTy); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.andn.")) { + } else if (IsX86 && (Name.startswith("avx512.mask.andn.") || + Name.startswith("avx512.mask.pandn."))) { VectorType *FTy = cast<VectorType>(CI->getType()); VectorType *ITy = VectorType::getInteger(FTy); Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); @@ -2410,7 +2524,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateBitCast(Rep, FTy); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.or.")) { + } else if (IsX86 && (Name.startswith("avx512.mask.or.") || + Name.startswith("avx512.mask.por."))) { VectorType *FTy = cast<VectorType>(CI->getType()); VectorType *ITy = VectorType::getInteger(FTy); Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), @@ -2418,7 +2533,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateBitCast(Rep, FTy); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.xor.")) { + } else if (IsX86 && (Name.startswith("avx512.mask.xor.") || + Name.startswith("avx512.mask.pxor."))) { VectorType *FTy = cast<VectorType>(CI->getType()); VectorType *ITy = VectorType::getInteger(FTy); Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), @@ -2502,26 +2618,16 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { } Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.max.p") && + } else if (IsX86 && (Name.startswith("avx512.mask.max.p") || + Name.startswith("avx512.mask.min.p")) && Name.drop_front(18) == ".512") { - Intrinsic::ID IID; - if (Name[17] == 's') - IID = Intrinsic::x86_avx512_max_ps_512; - else - IID = Intrinsic::x86_avx512_max_pd_512; - - Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), - { CI->getArgOperand(0), CI->getArgOperand(1), - CI->getArgOperand(4) }); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, - CI->getArgOperand(2)); - } else if (IsX86 && Name.startswith("avx512.mask.min.p") && - Name.drop_front(18) == ".512") { - Intrinsic::ID IID; - if (Name[17] == 's') - IID = Intrinsic::x86_avx512_min_ps_512; - else - IID = Intrinsic::x86_avx512_min_pd_512; + bool IsDouble = Name[17] == 'd'; + bool IsMin = Name[13] == 'i'; + static const Intrinsic::ID MinMaxTbl[2][2] = { + { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 }, + { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 } + }; + Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble]; Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), { CI->getArgOperand(0), CI->getArgOperand(1), @@ -3065,62 +3171,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.maskz.vpermt2var."))) { bool ZeroMask = Name[11] == 'z'; bool IndexForm = Name[17] == 'i'; - unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); - unsigned EltWidth = CI->getType()->getScalarSizeInBits(); - bool IsFloat = CI->getType()->isFPOrFPVectorTy(); - Intrinsic::ID IID; - if (VecWidth == 128 && EltWidth == 32 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_ps_128; - else if (VecWidth == 128 && EltWidth == 32 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_d_128; - else if (VecWidth == 128 && EltWidth == 64 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_pd_128; - else if (VecWidth == 128 && EltWidth == 64 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_q_128; - else if (VecWidth == 256 && EltWidth == 32 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_ps_256; - else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_d_256; - else if (VecWidth == 256 && EltWidth == 64 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_pd_256; - else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_q_256; - else if (VecWidth == 512 && EltWidth == 32 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_ps_512; - else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_d_512; - else if (VecWidth == 512 && EltWidth == 64 && IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_pd_512; - else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) - IID = Intrinsic::x86_avx512_vpermi2var_q_512; - else if (VecWidth == 128 && EltWidth == 16) - IID = Intrinsic::x86_avx512_vpermi2var_hi_128; - else if (VecWidth == 256 && EltWidth == 16) - IID = Intrinsic::x86_avx512_vpermi2var_hi_256; - else if (VecWidth == 512 && EltWidth == 16) - IID = Intrinsic::x86_avx512_vpermi2var_hi_512; - else if (VecWidth == 128 && EltWidth == 8) - IID = Intrinsic::x86_avx512_vpermi2var_qi_128; - else if (VecWidth == 256 && EltWidth == 8) - IID = Intrinsic::x86_avx512_vpermi2var_qi_256; - else if (VecWidth == 512 && EltWidth == 8) - IID = Intrinsic::x86_avx512_vpermi2var_qi_512; - else - llvm_unreachable("Unexpected intrinsic"); - - Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), - CI->getArgOperand(2) }; - - // If this isn't index form we need to swap operand 0 and 1. - if (!IndexForm) - std::swap(Args[0], Args[1]); - - Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), - Args); - Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) - : Builder.CreateBitCast(CI->getArgOperand(1), - CI->getType()); - Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); + Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") || Name.startswith("avx512.maskz.vpdpbusd.") || Name.startswith("avx512.mask.vpdpbusds.") || @@ -3181,6 +3232,39 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) : CI->getArgOperand(0); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); + } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" || + Name == "addcarry.u32" || Name == "addcarry.u64" || + Name == "subborrow.u32" || Name == "subborrow.u64")) { + Intrinsic::ID IID; + if (Name[0] == 'a' && Name.back() == '2') + IID = Intrinsic::x86_addcarry_32; + else if (Name[0] == 'a' && Name.back() == '4') + IID = Intrinsic::x86_addcarry_64; + else if (Name[0] == 's' && Name.back() == '2') + IID = Intrinsic::x86_subborrow_32; + else if (Name[0] == 's' && Name.back() == '4') + IID = Intrinsic::x86_subborrow_64; + else + llvm_unreachable("Unexpected intrinsic"); + + // Make a call with 3 operands. + Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2)}; + Value *NewCall = Builder.CreateCall( + Intrinsic::getDeclaration(CI->getModule(), IID), + Args); + + // Extract the second result and store it. + Value *Data = Builder.CreateExtractValue(NewCall, 1); + // Cast the pointer to the right type. + Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), + llvm::PointerType::getUnqual(Data->getType())); + Builder.CreateAlignedStore(Data, Ptr, 1); + // Replace the original call result with the first result of the new call. + Value *CF = Builder.CreateExtractValue(NewCall, 0); + + CI->replaceAllUsesWith(CF); + Rep = nullptr; } else if (IsX86 && Name.startswith("avx512.mask.") && upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { // Rep will be updated by the call in the condition. @@ -3356,6 +3440,32 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { break; } + case Intrinsic::x86_rdtscp: { + // This used to take 1 arguments. If we have no arguments, it is already + // upgraded. + if (CI->getNumOperands() == 0) + return; + + NewCall = Builder.CreateCall(NewFn); + // Extract the second result and store it. + Value *Data = Builder.CreateExtractValue(NewCall, 1); + // Cast the pointer to the right type. + Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), + llvm::PointerType::getUnqual(Data->getType())); + Builder.CreateAlignedStore(Data, Ptr, 1); + // Replace the original call result with the first result of the new call. + Value *TSC = Builder.CreateExtractValue(NewCall, 0); + + std::string Name = CI->getName(); + if (!Name.empty()) { + CI->setName(Name + ".old"); + NewCall->setName(Name); + } + CI->replaceAllUsesWith(TSC); + CI->eraseFromParent(); + return; + } + case Intrinsic::x86_sse41_insertps: case Intrinsic::x86_sse41_dppd: case Intrinsic::x86_sse41_dpps: |