reference, declarationdefinition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced

References

lib/IR/AutoUpgrade.cpp
 1647   Function *F = CI->getCalledFunction();
 1648   LLVMContext &C = CI->getContext();
 1650   Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
 1650   Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
 1675       Value *Arg0 = CI->getArgOperand(0);
 1676       Value *Arg1 = CI->getArgOperand(1);
 1690       CI->eraseFromParent();
 1702       Value *Arg0 = CI->getArgOperand(0);
 1703       Value *Arg1 = CI->getArgOperand(1);
 1715       CI->eraseFromParent();
 1720       Value *Arg0 = CI->getArgOperand(0);
 1721       Value *Arg1 = CI->getArgOperand(1);
 1732       CI->eraseFromParent();
 1739       Value *Arg0 = CI->getArgOperand(0);
 1740       Value *Arg1 = CI->getArgOperand(1);
 1748       CI->eraseFromParent();
 1753       Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
 1754       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
 1754       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
 1758       CI->eraseFromParent();
 1765       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
 1765       UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
 1766                          CI->getArgOperand(2), Aligned);
 1769       CI->eraseFromParent();
 1780                                CI->getArgOperand(0), CI->getArgOperand(1));
 1780                                CI->getArgOperand(0), CI->getArgOperand(1));
 1781       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
 1784       if (CI->getOperand(0)->getType()->isIntegerTy(8))
 1786       unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
 1788       Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
 1792       Value *Vec = CI->getArgOperand(0);
 1803                                                          CI->getType()),
 1804                                {CI->getArgOperand(0)});
 1806       if (CI->getNumArgOperands() == 4 &&
 1807           (!isa<ConstantInt>(CI->getArgOperand(3)) ||
 1808            cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
 1812         Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
 1812         Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
 1813         Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
 1818                                                            CI->getType()),
 1819                                  {CI->getArgOperand(0)});
 1821       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 1822                           CI->getArgOperand(1));
 1825       Value *Op0 = CI->getArgOperand(0);
 1826       Value *Op1 = CI->getArgOperand(1);
 1827       Value *Mask = CI->getArgOperand(2);
 1837           CI->getArgOperand(1)->getType()->getVectorNumElements();
 1838       Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
 1839       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 1840                           CI->getArgOperand(1));
 1842       unsigned NumElts = CI->getType()->getScalarSizeInBits();
 1843       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
 1844       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
 1859       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1861       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1862       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1864       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1866       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1867       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1870       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1872       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1873       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1875       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1877       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1878       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1880       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1882       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1883       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1886       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1888       Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1890       Rep = Builder.CreateBitCast(Rep, CI->getType());
 1893       Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
 1894       Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
 1909       Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
 1911       Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
 1922       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
 1927       Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
 1929       Type *OpTy = CI->getArgOperand(0)->getType();
 1940                                { CI->getOperand(0), CI->getArgOperand(1) });
 1940                                { CI->getOperand(0), CI->getArgOperand(1) });
 1941       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
 1943       Type *OpTy = CI->getArgOperand(0)->getType();
 1963                                { CI->getOperand(0), CI->getArgOperand(1) });
 1963                                { CI->getOperand(0), CI->getArgOperand(1) });
 1964       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
 1966       Type *OpTy = CI->getArgOperand(0)->getType();
 1986       Args.push_back(CI->getArgOperand(0));
 1987       Args.push_back(CI->getArgOperand(1));
 1988       Args.push_back(CI->getArgOperand(2));
 1989       if (CI->getNumArgOperands() == 5)
 1990         Args.push_back(CI->getArgOperand(4));
 1994       Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3));
 1998       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 1999       Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
 2001       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2002       Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
 2007       Value *Op = CI->getArgOperand(0);
 2016       Rep = upgradeAbs(Builder, *CI);
 2022       Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);
 2028       Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);
 2034       Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);
 2040       Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
 2045       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
 2050       Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
 2055       Rep = Builder.CreateSIToFP(CI->getArgOperand(1),
 2056                                  CI->getType()->getVectorElementType());
 2057       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
 2059       Rep = Builder.CreateUIToFP(CI->getArgOperand(1),
 2060                                  CI->getType()->getVectorElementType());
 2061       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
 2063       Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
 2064       Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType());
 2065       Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
 2084       Type *DstTy = CI->getType();
 2085       Rep = CI->getArgOperand(0);
 2099       else if (CI->getNumArgOperands() == 4 &&
 2100                (!isa<ConstantInt>(CI->getArgOperand(3)) ||
 2101                 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
 2104         Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
 2106         Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
 2112       if (CI->getNumArgOperands() >= 3)
 2113         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2114                             CI->getArgOperand(1));
 2116       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
 2117                               CI->getArgOperand(1), CI->getArgOperand(2),
 2117                               CI->getArgOperand(1), CI->getArgOperand(2),
 2120       Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
 2121                               CI->getArgOperand(1),CI->getArgOperand(2),
 2121                               CI->getArgOperand(1),CI->getArgOperand(2),
 2124       Type *ResultTy = CI->getType();
 2128       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
 2131       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
 2137       Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
 2139       Type *ResultTy = CI->getArgOperand(1)->getType();
 2143       Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
 2146       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
 2152       Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
 2155       Type *ResultTy = CI->getType();
 2157       Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
 2164       Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
 2164       Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
 2178       if (CI->getNumArgOperands() == 3) {
 2179         Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2202       Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
 2204       Value *Sel = CI->getArgOperand(2);
 2206       Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
 2207       Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
 2212       Rep = upgradeX86Rotate(Builder, *CI, false);
 2215       Rep = upgradeX86Rotate(Builder, *CI, true);
 2220       Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
 2225       Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
 2229       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
 2230       Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
 2231       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
 2235       Type *VecTy = CI->getType();
 2238       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
 2252       VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());
 2253       VectorType *DstTy = cast<VectorType>(CI->getType());
 2262           CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);
 2268       if (CI->getNumArgOperands() == 3)
 2269         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2270                             CI->getArgOperand(1));
 2275       Type *Ty = CI->getArgOperand(1)->getType();
 2276       Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
 2277       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2278                           CI->getArgOperand(1));
 2282       Type *EltTy = CI->getType()->getVectorElementType();
 2285       Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
 2296       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2297       Type *VT = CI->getType();
 2312       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
 2313                                         CI->getArgOperand(1), ShuffleMask);
 2314       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
 2315                           CI->getArgOperand(3));
 2319                         CI->getArgOperand(0)->getType()->getVectorNumElements();
 2320       unsigned NumDstElts = CI->getType()->getVectorNumElements();
 2326       Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
 2327                                         CI->getArgOperand(0),
 2329       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2330                           CI->getArgOperand(1));
 2336       Value *Op = CI->getArgOperand(0);
 2337       unsigned NumElts = CI->getType()->getVectorNumElements();
 2342       if (CI->getNumArgOperands() == 3)
 2343         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2344                             CI->getArgOperand(1));
 2354       Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
 2362       Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
 2364       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
 2365                                       CI->getArgOperand(1),
 2366                                       CI->getArgOperand(2),
 2367                                       CI->getArgOperand(3),
 2368                                       CI->getArgOperand(4),
 2371       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
 2372                                       CI->getArgOperand(1),
 2373                                       CI->getArgOperand(2),
 2374                                       CI->getArgOperand(3),
 2375                                       CI->getArgOperand(4),
 2380       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2381       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
 2386       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2387       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
 2393       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2394       Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
 2399       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2400       Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
 2406       Value *Op0 = CI->getArgOperand(0);
 2407       Value *Op1 = CI->getArgOperand(1);
 2408       unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2409       VectorType *VecTy = cast<VectorType>(CI->getType());
 2420       Value *Op0 = CI->getArgOperand(0);
 2421       Value *Op1 = CI->getArgOperand(1);
 2422       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2423       unsigned DstNumElts = CI->getType()->getVectorNumElements();
 2459       if (CI->getNumArgOperands() == 5)
 2460         Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
 2461                             CI->getArgOperand(3));
 2465       Value *Op0 = CI->getArgOperand(0);
 2466       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2467       unsigned DstNumElts = CI->getType()->getVectorNumElements();
 2482       if (CI->getNumArgOperands() == 4)
 2483         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2484                             CI->getArgOperand(2));
 2489       Value *Op0 = CI->getArgOperand(0);
 2490       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2491       VectorType *VecTy = cast<VectorType>(CI->getType());
 2500       if (CI->getNumArgOperands() == 4)
 2501         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2502                             CI->getArgOperand(2));
 2513       uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2515       unsigned NumElts = CI->getType()->getVectorNumElements();
 2520       Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
 2520       Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
 2521       Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
 2521       Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
 2524       V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
 2525       V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
 2543       Value *Op0 = CI->getArgOperand(0);
 2544       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2545       VectorType *VecTy = cast<VectorType>(CI->getType());
 2560       if (CI->getNumArgOperands() == 4)
 2561         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2562                             CI->getArgOperand(2));
 2565       Value *Op0 = CI->getArgOperand(0);
 2566       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2567       unsigned NumElts = CI->getType()->getVectorNumElements();
 2579       if (CI->getNumArgOperands() == 4)
 2580         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2581                             CI->getArgOperand(2));
 2584       Value *Op0 = CI->getArgOperand(0);
 2585       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
 2586       unsigned NumElts = CI->getType()->getVectorNumElements();
 2598       if (CI->getNumArgOperands() == 4)
 2599         Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2600                             CI->getArgOperand(2));
 2602       Value *Op0 = CI->getArgOperand(0);
 2603       Value *Op1 = CI->getArgOperand(1);
 2604       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 2605       unsigned NumElts = CI->getType()->getVectorNumElements();
 2607       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
 2624       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
 2625                           CI->getArgOperand(3));
 2629       Value *Op0 = CI->getArgOperand(0);
 2630       unsigned NumElts = CI->getType()->getVectorNumElements();
 2631       unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
 2646       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2647                           CI->getArgOperand(1));
 2650       Value *Op0 = CI->getArgOperand(0);
 2651       Value *Op1 = CI->getArgOperand(1);
 2652       int NumElts = CI->getType()->getVectorNumElements();
 2653       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
 2662       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2663                           CI->getArgOperand(2));
 2666       Value *Op0 = CI->getArgOperand(0);
 2667       Value *Op1 = CI->getArgOperand(1);
 2668       int NumElts = CI->getType()->getVectorNumElements();
 2669       int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
 2678       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2679                           CI->getArgOperand(2));
 2682       VectorType *FTy = cast<VectorType>(CI->getType());
 2684       Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 2685                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 2687       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2688                           CI->getArgOperand(2));
 2691       VectorType *FTy = cast<VectorType>(CI->getType());
 2693       Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
 2695                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 2697       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2698                           CI->getArgOperand(2));
 2701       VectorType *FTy = cast<VectorType>(CI->getType());
 2703       Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 2704                              Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 2706       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2707                           CI->getArgOperand(2));
 2710       VectorType *FTy = cast<VectorType>(CI->getType());
 2712       Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 2713                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 2715       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2716                           CI->getArgOperand(2));
 2718       Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
 2718       Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
 2719       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2720                           CI->getArgOperand(2));
 2722       Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
 2722       Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
 2723       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2724                           CI->getArgOperand(2));
 2726       Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
 2726       Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
 2727       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2728                           CI->getArgOperand(2));
 2738                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2738                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2739                                    CI->getArgOperand(4) });
 2741         Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
 2741         Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
 2743       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2744                           CI->getArgOperand(2));
 2754                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2754                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2755                                    CI->getArgOperand(4) });
 2757         Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
 2757         Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
 2759       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2760                           CI->getArgOperand(2));
 2770                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2770                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2771                                    CI->getArgOperand(4) });
 2773         Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
 2773         Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
 2775       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2776                           CI->getArgOperand(2));
 2786                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2786                                  { CI->getArgOperand(0), CI->getArgOperand(1),
 2787                                    CI->getArgOperand(4) });
 2789         Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
 2789         Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
 2791       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2792                           CI->getArgOperand(2));
 2805                                { CI->getArgOperand(0), CI->getArgOperand(1),
 2805                                { CI->getArgOperand(0), CI->getArgOperand(1),
 2806                                  CI->getArgOperand(4) });
 2807       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 2808                           CI->getArgOperand(2));
 2812                                                          CI->getType()),
 2813                                { CI->getArgOperand(0), Builder.getInt1(false) });
 2814       Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 2815                           CI->getArgOperand(1));
 2883       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
 2951       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
 3017       Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
 3019       Rep = upgradeMaskedMove(Builder, *CI);
 3021       Rep = UpgradeMaskToInt(Builder, *CI);
 3027       Value *Ptr = CI->getArgOperand(0);
 3028       VectorType *VTy = cast<VectorType>(CI->getType());
 3044       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3044       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3045                        CI->getArgOperand(2) };
 3060       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
 3066         Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
 3069       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3069       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3070                        CI->getArgOperand(2) };
 3076       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
 3081       Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
 3095       Value *A = CI->getArgOperand(0);
 3096       Value *B = CI->getArgOperand(1);
 3097       Value *C = CI->getArgOperand(2);
 3110       if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
 3111           cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
 3112         Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
 3119         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
 3122         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
 3134         PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
 3137       Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
 3139       Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
 3155       Value *A = CI->getArgOperand(0);
 3156       Value *B = CI->getArgOperand(1);
 3157       Value *C = CI->getArgOperand(2);
 3166       if (CI->getNumArgOperands() == 5 &&
 3167           (!isa<ConstantInt>(CI->getArgOperand(4)) ||
 3168            cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
 3177                                  { A, B, C, CI->getArgOperand(4) });
 3179         Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
 3185       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
 3186                         IsMask3 ? CI->getArgOperand(2) :
 3187                                   CI->getArgOperand(0);
 3189       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
 3193       int NumElts = CI->getType()->getVectorNumElements();
 3195       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3195       Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3196                        CI->getArgOperand(2) };
 3198       Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
 3221       if (CI->getNumArgOperands() == 5 &&
 3222           (!isa<ConstantInt>(CI->getArgOperand(4)) ||
 3223            cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
 3231         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3231         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3232                          CI->getArgOperand(2), CI->getArgOperand(4) };
 3232                          CI->getArgOperand(2), CI->getArgOperand(4) };
 3237                                  {CI->getArgOperand(0), CI->getArgOperand(1),
 3237                                  {CI->getArgOperand(0), CI->getArgOperand(1),
 3238                                   CI->getArgOperand(2), CI->getArgOperand(4)});
 3238                                   CI->getArgOperand(2), CI->getArgOperand(4)});
 3240         int NumElts = CI->getType()->getVectorNumElements();
 3242         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3242         Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3243                          CI->getArgOperand(2) };
 3245         Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
 3261       Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
 3262                         IsMask3 ? CI->getArgOperand(2) :
 3263                                   CI->getArgOperand(0);
 3265       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
 3269       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
 3270       unsigned EltWidth = CI->getType()->getScalarSizeInBits();
 3287       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
 3287       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
 3288                         CI->getArgOperand(2), CI->getArgOperand(3) };
 3288                         CI->getArgOperand(2), CI->getArgOperand(3) };
 3289       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
 3291       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
 3292                                  : CI->getArgOperand(0);
 3293       Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
 3298       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
 3315       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
 3315       Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
 3316                         CI->getArgOperand(2) };
 3317       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
 3319       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
 3320                                  : CI->getArgOperand(0);
 3321       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
 3327       Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
 3334       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
 3351       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3351       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3352                         CI->getArgOperand(2)  };
 3353       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
 3355       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
 3356                                  : CI->getArgOperand(0);
 3357       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
 3364       unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
 3381       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3381       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3382                         CI->getArgOperand(2)  };
 3383       Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
 3385       Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
 3386                                  : CI->getArgOperand(0);
 3387       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
 3404       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3404       Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
 3405                         CI->getArgOperand(2)};
 3407                                 Intrinsic::getDeclaration(CI->getModule(), IID),
 3413       Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
 3419       CI->replaceAllUsesWith(CF);
 3422                upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
 3425       Value *Arg = CI->getArgOperand(0);
 3432       Value *Ptr = CI->getArgOperand(0);
 3433       Value *Val = CI->getArgOperand(1);
 3438       Value *Arg0 = CI->getArgOperand(0);
 3439       Value *Arg1 = CI->getArgOperand(1);
 3446       Value *Arg0 = CI->getArgOperand(0);
 3447       Value *Arg1 = CI->getArgOperand(1);
 3454       Value *Arg = CI->getArgOperand(0);
 3463       Value *Arg = CI->getArgOperand(0);
 3473                                CI->getArgOperand(0), "h2f");
 3479       CI->replaceAllUsesWith(Rep);
 3480     CI->eraseFromParent();
 3484   const auto &DefaultCase = [&NewFn, &CI]() -> void {
 3487         (CI->getCalledFunction()->getName() != NewFn->getName()) &&
 3489     CI->setCalledFunction(NewFn);
 3499     if (CI->isFast())
 3500       Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
 3502       Args.push_back(CI->getOperand(0));
 3503     Args.push_back(CI->getOperand(1));
 3505     cast<Instruction>(NewCall)->copyFastMathFlags(CI);
 3510     if (CI->isFast())
 3511       Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
 3513       Args.push_back(CI->getOperand(0));
 3514     Args.push_back(CI->getOperand(1));
 3516     cast<Instruction>(NewCall)->copyFastMathFlags(CI);
 3533     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
 3534                                  CI->arg_operands().end());
 3540     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
 3545     assert(CI->getNumArgOperands() == 1 &&
 3548         Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
 3552     Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
 3554                                    : CI->getArgOperand(2);
 3556         CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
 3556         CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
 3558         NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
 3558         NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
 3563     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
 3567     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
 3572     assert(CI->getNumArgOperands() == 4);
 3574     if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
 3578             {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
 3578             {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
 3578             {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
 3581     CI->eraseFromParent();
 3586     NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
 3593     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
 3594                                  CI->arg_operands().end());
 3609     Value *Arg0 = CI->getArgOperand(0);
 3614     Value *Arg1 = CI->getArgOperand(1);
 3628     if (CI->getNumOperands() == 0)
 3635     Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
 3641     std::string Name = CI->getName();
 3643       CI->setName(Name + ".old");
 3646     CI->replaceAllUsesWith(TSC);
 3647     CI->eraseFromParent();
 3659     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
 3660                                  CI->arg_operands().end());
 3679     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
 3680                                  CI->arg_operands().end());
 3695     if (CI->getNumArgOperands() != 5) {
 3701     Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
 3701     Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
 3702                       CI->getArgOperand(2), CI->getArgOperand(4)};
 3702                       CI->getArgOperand(2), CI->getArgOperand(4)};
 3706     const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
 3716   std::string Name = CI->getName();
 3718     CI->setName(Name + ".old");
 3721   CI->replaceAllUsesWith(NewCall);
 3722   CI->eraseFromParent();