|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
References
lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp 49 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
57 N->dump(&DAG); dbgs() << "\n";
127 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
128 TLI.getTypeToTransformTo(*DAG.getContext(),
143 if (DAG.getDataLayout().isBigEndian() &&
148 return DAG.getConstant(Val, SDLoc(CN),
149 TLI.getTypeToTransformTo(*DAG.getContext(),
152 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
153 TLI.getTypeToTransformTo(*DAG.getContext(),
160 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
166 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
172 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
174 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
178 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
185 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
195 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
202 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
212 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
219 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
229 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
234 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
244 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
249 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
270 SDValue SignBit = DAG.getNode(
271 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
272 DAG.getConstant(RSize - 1, dl,
273 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
274 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
280 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
281 DAG.getConstant(SizeDiff, dl,
283 DAG.getDataLayout())));
284 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
286 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
288 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
289 DAG.getConstant(-SizeDiff, dl,
291 DAG.getDataLayout())));
295 SDValue Mask = DAG.getNode(
296 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
297 DAG.getConstant(LSize - 1, dl,
298 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
299 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
299 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
300 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
303 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
307 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
312 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
322 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
329 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
339 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
344 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
354 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
359 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
374 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
384 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
389 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
399 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
404 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
414 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
419 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
429 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
438 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
448 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
455 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
465 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
470 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
480 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
487 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
488 DAG.getConstant(SignMask, dl, NVT));
492 SDValue Ops[2] = { DAG.getConstantFP(-0.0, dl, N->getValueType(0)),
495 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
512 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
531 return TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, SDLoc(N)).first;
537 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
542 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
547 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
550 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
554 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
559 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), NVT, Op);
567 return TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, SDLoc(N)).first;
571 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
578 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
590 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
596 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
606 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
613 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
628 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
638 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
643 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
658 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
668 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
673 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
683 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
690 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
700 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
702 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), NVT, N->getOperand(0));
708 return TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
720 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
728 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
740 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
747 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
754 return DAG.getSelect(SDLoc(N),
761 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
767 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
767 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
775 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
779 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
810 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
816 return TLI.makeLibCall(DAG, LC,
817 TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
827 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
835 N->dump(&DAG); dbgs() << "\n";
873 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
883 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), RVT, Op);
891 return TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N)).first;
911 return TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N)).first;
921 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
927 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
932 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
933 DAG.getCondCode(CCCode), NewLHS, NewRHS,
964 SDValue Res = TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl).first;
967 return DAG.getNode(ISD::TRUNCATE, dl, RVT, Res);
977 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
983 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
988 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
990 DAG.getCondCode(CCCode)),
1001 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1012 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1013 DAG.getCondCode(CCCode)),
1026 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
1027 Val, DAG.getIntPtrConstant(0, dl)));
1031 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1036 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1043 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1053 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1060 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1070 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1077 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1087 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1094 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1112 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1124 N->dump(&DAG); dbgs() << "\n";
1181 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1186 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1186 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1189 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1189 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1201 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1203 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1204 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1283 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1358 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1373 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1400 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1401 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1406 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1408 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1409 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1409 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1489 SDValue Call = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1527 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1534 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1534 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1546 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1557 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1559 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1559 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1561 Hi = DAG.getNode(ISD::SINT_TO_FP, dl, NVT, Src);
1565 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1569 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1576 Hi = TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl).first;
1584 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1608 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi,
1609 DAG.getConstantFP(APFloat(APFloat::PPCDoubleDouble(),
1612 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1612 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1627 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1638 N->dump(&DAG); dbgs() << "\n";
1694 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1696 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()),
1698 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1699 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1701 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
1703 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1704 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1716 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1721 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1722 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1733 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1743 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1754 return TLI.makeLibCall(DAG, LC, RVT, N->getOperand(0), CallOptions, dl).first;
1764 return TLI.makeLibCall(DAG, LC, N->getValueType(0), N->getOperand(0),
1776 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1781 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1783 DAG.getCondCode(CCCode)), 0);
1799 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1800 DAG.getCondCode(CCCode)), 0);
1814 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
1823 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
1831 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1844 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1857 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1870 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
1895 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
1913 N->dump(&DAG); dbgs() << "\n";
1940 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
1941 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
1945 return DAG.getBitcast(N->getValueType(0), Convert);
1954 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1961 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
1973 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
1983 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
1992 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1997 return DAG.getSetCC(SDLoc(N), NVT, Op0, Op1, CCCode);
2010 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2013 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2016 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2025 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2043 N->dump(&DAG); dbgs() << "\n";
2112 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2115 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2117 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2118 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2127 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2128 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2134 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2135 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2164 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2175 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2177 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2178 DAG.getConstant(IdxVal - LoElts, DL,
2192 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2197 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2198 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2206 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2211 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2219 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2222 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2230 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2233 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2238 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2243 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2249 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2253 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2264 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2265 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2268 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2270 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2278 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2279 SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), IVT,
2290 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2291 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2299 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2309 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2319 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2320 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2322 return DAG.getNode(
2324 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2324 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2328 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2328 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2334 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2343 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2344 DAG.getVTList(CastVT, MVT::Other),
2348 SDValue ResultCast = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp 38 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
52 N->dump(&DAG); dbgs() << "\n";
212 return DAG.getNode(ISD::AssertSext, SDLoc(N),
219 return DAG.getNode(ISD::AssertZext, SDLoc(N),
224 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
225 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
237 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
252 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
259 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
260 SDValue Res = DAG.getAtomicCmpSwap(
272 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
273 SDValue Res = DAG.getAtomicCmpSwap(
285 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
287 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
296 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
300 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
304 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
313 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
325 if (DAG.getDataLayout().isBigEndian())
328 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
329 EVT::getIntegerVT(*DAG.getContext(),
332 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
341 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
350 EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
354 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
355 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
356 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
357 DAG.getConstant(0, dl, IdxTy));
358 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
364 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
388 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
389 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
389 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
390 DAG.getConstant(DiffBits, dl, ShiftVT));
400 EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
401 return DAG.getNode(ISD::SRL, dl, NVT,
402 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
403 DAG.getConstant(DiffBits, dl, ShiftVT));
409 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
410 TLI.getTypeToTransformTo(*DAG.getContext(),
422 SDValue Result = DAG.getNode(Opc, dl,
423 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
435 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
437 return DAG.getNode(
439 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
446 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
460 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
460 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
462 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
474 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
482 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
483 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
487 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
511 Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other },
517 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
526 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
529 DAG.getValueType(N->getValueType(0).getScalarType()));
533 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
536 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
543 return DAG.getNode(N->getOpcode(), dl, NVT);
547 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
560 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
561 DAG.getValueType(N->getOperand(0).getValueType()));
563 return DAG.getZeroExtendInReg(Res, dl,
571 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
576 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
580 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
590 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
594 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
604 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
612 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
612 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
625 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
635 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
635 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
643 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
689 EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
690 SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
692 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
694 DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
697 DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
698 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
702 DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
703 return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
708 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
710 DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
711 return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
717 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
718 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
720 DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
721 Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
722 Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
752 EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
753 Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
754 DAG.getConstant(DiffSize, dl, ShiftTy));
755 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
758 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
759 DAG.getConstant(DiffSize, dl, ShiftTy));
761 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
779 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
783 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
784 DAG.getValueType(OVT));
786 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
797 return DAG.getSelect(SDLoc(N),
806 return DAG.getNode(ISD::VSELECT, SDLoc(N),
813 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
820 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
829 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
842 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
846 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
854 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
859 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
869 return DAG.getNode(N->getOpcode(), SDLoc(N),
877 return DAG.getNode(N->getOpcode(), SDLoc(N),
885 return DAG.getNode(N->getOpcode(), SDLoc(N),
895 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
904 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
908 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
934 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
936 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
937 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
939 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
946 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
948 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
951 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
953 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
956 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
957 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
958 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
963 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
980 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
984 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
986 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1017 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1017 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1028 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1051 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1052 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1061 EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1062 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1063 DAG.getConstant(Shift, DL, ShiftTy));
1064 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1065 DAG.getConstant(0, DL, Hi.getValueType()),
1069 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1070 Mul, DAG.getValueType(SmallVT));
1071 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1076 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1085 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1085 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1095 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1096 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1101 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1107 if (DAG.getDataLayout().isBigEndian())
1111 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1112 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1114 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1116 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1117 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1118 TLI.getPointerTy(DAG.getDataLayout())));
1119 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1138 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1151 N->dump(&DAG); dbgs() << "\n";
1261 OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1263 OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1293 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1298 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1317 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1329 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1341 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1342 DAG.getConstant(OVT.getSizeInBits(), dl,
1343 TLI.getPointerTy(DAG.getDataLayout())));
1344 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1367 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1380 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1389 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1390 TLI.getVectorIdxTy(DAG.getDataLayout()));
1391 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1398 return SDValue(DAG.UpdateNodeOperands(N,
1406 DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1422 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1434 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1446 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1450 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1457 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1458 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1459 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1463 return SDValue(DAG.UpdateNodeOperands(N,
1475 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1493 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1500 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1512 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1533 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1552 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1557 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1561 return SDValue(DAG.UpdateNodeOperands(N,
1568 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1569 return DAG.getZeroExtendInReg(Op, dl,
1583 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1589 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1595 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1605 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1612 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1640 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1644 SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1645 return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1657 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1670 N->dump(&DAG); dbgs() << "\n";
1736 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1737 SDValue Tmp = DAG.getAtomicCmpSwap(
1745 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1846 Lo = Hi = DAG.getConstant(0, DL, NVT);
1848 Lo = DAG.getConstant(0, DL, NVT);
1849 Hi = DAG.getNode(ISD::SHL, DL,
1850 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1852 Lo = DAG.getConstant(0, DL, NVT);
1855 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1855 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1856 Hi = DAG.getNode(ISD::OR, DL, NVT,
1857 DAG.getNode(ISD::SHL, DL, NVT, InH,
1858 DAG.getConstant(Amt, DL, ShTy)),
1859 DAG.getNode(ISD::SRL, DL, NVT, InL,
1860 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1867 Lo = Hi = DAG.getConstant(0, DL, NVT);
1869 Lo = DAG.getNode(ISD::SRL, DL,
1870 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1871 Hi = DAG.getConstant(0, DL, NVT);
1874 Hi = DAG.getConstant(0, DL, NVT);
1876 Lo = DAG.getNode(ISD::OR, DL, NVT,
1877 DAG.getNode(ISD::SRL, DL, NVT, InL,
1878 DAG.getConstant(Amt, DL, ShTy)),
1879 DAG.getNode(ISD::SHL, DL, NVT, InH,
1880 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1881 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1881 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1888 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1889 DAG.getConstant(NVTBits - 1, DL, ShTy));
1891 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1892 DAG.getConstant(Amt - NVTBits, DL, ShTy));
1893 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1894 DAG.getConstant(NVTBits - 1, DL, ShTy));
1897 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1898 DAG.getConstant(NVTBits - 1, DL, ShTy));
1900 Lo = DAG.getNode(ISD::OR, DL, NVT,
1901 DAG.getNode(ISD::SRL, DL, NVT, InL,
1902 DAG.getConstant(Amt, DL, ShTy)),
1903 DAG.getNode(ISD::SHL, DL, NVT, InH,
1904 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1905 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1905 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1916 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1925 KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1939 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1940 DAG.getConstant(~HighBitMask, dl, ShTy));
1945 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1946 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1949 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1950 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1953 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1954 DAG.getConstant(NVTBits - 1, dl, ShTy));
1955 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1966 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1967 DAG.getConstant(NVTBits - 1, dl, ShTy));
1983 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1983 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1985 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1987 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1988 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1988 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2003 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2014 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2015 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2016 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2017 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2019 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2020 Amt, DAG.getConstant(0, dl, ShTy),
2028 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2029 HiS = DAG.getNode(ISD::OR, dl, NVT,
2030 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2031 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2034 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2035 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2037 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2038 Hi = DAG.getSelect(dl, NVT, isZero, InH,
2039 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2043 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2044 LoS = DAG.getNode(ISD::OR, dl, NVT,
2045 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2048 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2051 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2052 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2054 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2055 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2056 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2060 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2061 LoS = DAG.getNode(ISD::OR, dl, NVT,
2062 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2063 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2066 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2067 DAG.getConstant(NVTBits - 1, dl, ShTy));
2068 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2070 Lo = DAG.getSelect(dl, NVT, isZero, InL,
2071 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2072 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2109 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2113 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2114 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2117 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2120 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2122 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2139 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2141 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2143 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2145 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2147 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2149 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2162 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2165 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2167 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2169 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2171 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2173 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2181 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2186 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2190 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2191 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2194 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2195 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2201 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2201 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2204 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2205 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2208 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2209 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2215 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2216 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2217 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2221 SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2222 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2226 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2227 DAG.getConstant(1, dl, NVT),
2228 DAG.getConstant(0, dl, NVT));
2229 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2231 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2232 DAG.getConstant(1, dl, NVT), Carry1);
2233 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2235 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2236 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2238 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2243 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2245 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2245 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2246 DAG.getConstant(0, dl, NVT));
2248 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2259 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2264 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2266 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2268 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2270 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2285 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2289 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2291 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2324 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2331 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2335 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2337 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2343 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2348 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2363 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2367 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2369 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2378 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2383 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2384 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2409 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2410 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2410 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2413 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2413 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2415 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2416 DAG.getConstant(NVTBits - 1, dl,
2417 TLI.getPointerTy(DAG.getDataLayout())));
2431 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2432 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2432 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2435 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2435 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2437 Hi = DAG.getConstant(0, dl, NVT);
2445 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2446 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2453 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2454 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2459 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2466 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2467 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2477 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2478 DAG.getConstant(0, dl, VT), N0);
2484 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2485 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2486 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2487 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2497 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2498 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2500 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2501 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2503 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2504 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2505 DAG.getConstant(NVT.getSizeInBits(), dl,
2507 Hi = DAG.getConstant(0, dl, NVT);
2516 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2516 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2517 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2518 Hi = DAG.getConstant(0, dl, NVT);
2528 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2529 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2531 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2532 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2534 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2535 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2536 DAG.getConstant(NVT.getSizeInBits(), dl,
2538 Hi = DAG.getConstant(0, dl, NVT);
2544 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2547 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2548 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2550 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2551 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2567 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2583 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2611 SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2639 SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2667 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2677 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2678 DAG.getConstant(LoSize - 1, dl,
2679 TLI.getPointerTy(DAG.getDataLayout())));
2682 Hi = DAG.getConstant(0, dl, NVT);
2686 Hi = DAG.getUNDEF(NVT);
2688 } else if (DAG.getDataLayout().isLittleEndian()) {
2690 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2695 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2699 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2700 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2701 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2707 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2718 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2719 EVT::getIntegerVT(*DAG.getContext(),
2724 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2725 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2727 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2729 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2734 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2739 Lo = DAG.getNode(
2741 DAG.getNode(ISD::SHL, dl, NVT, Hi,
2742 DAG.getConstant(ExcessBits, dl,
2743 TLI.getPointerTy(DAG.getDataLayout()))));
2745 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2747 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2748 TLI.getPointerTy(DAG.getDataLayout())));
2763 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2764 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2770 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2777 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2800 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2802 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2803 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2805 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2806 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2808 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2814 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2815 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2816 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2817 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2819 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2820 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2821 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2822 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2824 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2825 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2826 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2828 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2829 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2830 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2831 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2832 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2834 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2835 DAG.getNode(ISD::ADD, dl, NVT,
2836 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2837 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2844 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
2851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2852 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2853 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2861 SDValue Result = TLI.expandAddSubSat(N, DAG);
2887 Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2891 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2891 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2897 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2898 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2899 SDValue Zero = DAG.getConstant(0, dl, VT);
2900 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2901 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2902 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2907 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2908 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
2919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2926 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2936 EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2959 SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
2960 Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
2962 Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
2993 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2994 SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3000 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3001 DAG.getConstant(Scale, dl, ShiftTy));
3002 SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3003 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3006 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3009 SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3010 DAG.getConstant(Scale - NVTSize, dl,
3012 SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3017 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3018 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3030 SDValue HLHiMask = DAG.getConstant(
3032 SDValue HLLoMask = DAG.getConstant(
3035 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3036 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3037 SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3038 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3039 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3041 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3042 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3043 SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3044 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3045 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3048 SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3049 SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3050 SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3051 SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3052 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3054 SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3055 SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3056 SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3057 SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3058 DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3063 SDValue HHHiMask = DAG.getConstant(
3065 SDValue HHLoMask = DAG.getConstant(
3067 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3068 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3075 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3075 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3076 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3076 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3079 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3079 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3080 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3091 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3108 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3110 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3111 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3112 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3116 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3117 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3119 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3132 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3132 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3150 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3188 if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3198 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3203 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3206 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3206 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3251 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3261 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3266 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3269 Hi = DAG.getNode(
3271 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3271 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3284 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3285 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3285 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3298 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3303 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3304 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3305 TLI.getPointerTy(DAG.getDataLayout())));
3310 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3311 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3311 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3323 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3323 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3341 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3346 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3348 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3349 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3351 DAG.getConstant(NVT.getSizeInBits(), dl,
3352 TLI.getPointerTy(DAG.getDataLayout())));
3353 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3380 SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3381 SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3383 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3384 SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3385 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3386 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3388 SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3389 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3390 SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3393 SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3394 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3395 SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3404 SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3405 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3406 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3407 SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3408 SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3409 Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3415 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3416 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3417 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3429 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3432 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3432 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3432 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3439 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3454 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3456 TargetLowering::CallLoweringInfo CLI(DAG);
3466 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3467 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3468 DAG.getConstant(0, dl, PtrVT),
3481 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3481 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3498 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3508 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3508 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3525 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3530 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3535 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3536 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3549 Hi = DAG.getZeroExtendInReg(Hi, dl,
3550 EVT::getIntegerVT(*DAG.getContext(),
3559 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3560 SDValue Zero = DAG.getConstant(0, dl, VT);
3561 SDValue Swap = DAG.getAtomicCmpSwap(
3574 SDValue Res = TLI.expandVecReduce(N, DAG);
3587 LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3598 N->dump(&DAG); dbgs() << "\n";
3657 NewLHS = DAG.getNode(ISD::AND, dl,
3665 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3666 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3667 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3668 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3702 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3710 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3718 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3719 LHSHi, RHSHi, DAG.getCondCode(CCCode));
3747 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3771 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3772 SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3773 SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3775 DAG.getCondCode(CCCode));
3785 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3786 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3798 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3803 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3804 DAG.getCondCode(CCCode), NewLHS, NewRHS,
3816 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3821 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3823 DAG.getCondCode(CCCode)), 0);
3840 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3840 DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3855 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3856 SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3857 return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3867 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3876 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3887 return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N)).first;
3898 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3911 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3915 if (DAG.getDataLayout().isLittleEndian()) {
3919 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3924 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3928 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3929 Hi = DAG.getTruncStore(
3932 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3943 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3948 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3949 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3950 TLI.getPointerTy(DAG.getDataLayout())));
3951 Hi = DAG.getNode(
3953 DAG.getNode(ISD::SRL, dl, NVT, Lo,
3954 DAG.getConstant(ExcessBits, dl,
3955 TLI.getPointerTy(DAG.getDataLayout()))));
3959 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3963 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3965 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3967 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3969 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3976 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3988 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3992 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3993 SignedConv = TLI.LowerOperation(SignedConv, DAG);
4016 SDValue SignSet = DAG.getSetCC(dl,
4019 DAG.getConstant(0, dl, Hi.getValueType()),
4024 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
4024 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
4025 TLI.getPointerTy(DAG.getDataLayout()));
4028 SDValue Zero = DAG.getIntPtrConstant(0, dl);
4029 SDValue Four = DAG.getIntPtrConstant(4, dl);
4030 if (DAG.getDataLayout().isBigEndian())
4032 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
4035 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
4041 SDValue Fudge = DAG.getExtLoad(
4042 ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
4043 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
4045 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
4054 return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, dl).first;
4059 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4071 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4090 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4091 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4092 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4095 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4100 return DAG.getBuildVector(NOutVT, dl, Ops);
4115 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4121 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4137 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4143 return DAG.getBuildVector(NOutVT, dl, Ops);
4154 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4158 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4160 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4171 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4175 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4177 return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4184 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4206 SDValue Ext = DAG.getNode(
4208 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4208 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4209 Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4213 return DAG.getBuildVector(NOutVT, dl, Ops);
4218 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4244 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4248 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4253 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4261 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4263 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4271 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4272 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4278 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4279 TLI.getVectorIdxTy(DAG.getDataLayout()));
4280 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4286 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4295 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4296 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4316 SDValue Ex = DAG.getNode(
4318 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4318 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4319 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4324 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
lib/CodeGen/SelectionDAG/LegalizeTypes.cpp 77 for (SDNode &Node : DAG.allnodes()) {
195 HandleSDNode Dummy(DAG.getRoot());
200 DAG.setRoot(SDValue());
205 for (SDNode &Node : DAG.allnodes()) {
225 LLVM_DEBUG(dbgs() << "Legalizing node: "; N->dump(&DAG));
293 LLVM_DEBUG(dbgs() << "Analyzing operand: "; Op.dump(&DAG));
367 LLVM_DEBUG(dbgs() << "Legally typed node: "; N->dump(&DAG);
418 DAG.setRoot(Dummy.getValue());
424 DAG.RemoveDeadNodes();
429 for (SDNode &Node : DAG.allnodes()) {
437 Node.dump(&DAG);
446 Node.getOperand(i).dump(&DAG);
463 Node.dump(&DAG); dbgs() << "\n";
515 SDNode *M = DAG.UpdateNodeOperands(N, NewOps);
641 DAG.ReplaceAllUsesOfValueWith(From, To);
672 DAG.ReplaceAllUsesOfValueWith(OldVal, NewVal);
687 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
696 DAG.transferDbgValues(Op, Result);
701 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
712 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
745 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
754 if (DAG.getDataLayout().isBigEndian()) {
755 DAG.transferDbgValues(Op, Hi, 0, Hi.getValueSizeInBits(), false);
756 DAG.transferDbgValues(Op, Lo, Hi.getValueSizeInBits(),
759 DAG.transferDbgValues(Op, Lo, 0, Lo.getValueSizeInBits(), false);
760 DAG.transferDbgValues(Op, Hi, Lo.getValueSizeInBits(),
782 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
825 TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()) &&
842 return DAG.getNode(ISD::BITCAST, SDLoc(Op),
843 EVT::getIntegerVT(*DAG.getContext(), BitWidth), Op);
850 EVT EltNVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
852 return DAG.getNode(ISD::BITCAST, SDLoc(Op),
853 EVT::getVectorVT(*DAG.getContext(), EltNVT, EltCnt), Op);
861 SDValue StackPtr = DAG.CreateStackTemporary(Op.getValueType(), DestVT);
864 DAG.getStore(DAG.getEntryNode(), dl, Op, StackPtr, MachinePointerInfo());
864 DAG.getStore(DAG.getEntryNode(), dl, Op, StackPtr, MachinePointerInfo());
866 return DAG.getLoad(DestVT, dl, Store, StackPtr, MachinePointerInfo());
884 TLI.ReplaceNodeResults(N, Results, DAG);
886 TLI.LowerOperationWrapper(N, Results, DAG);
921 TLI.ReplaceNodeResults(N, Results, DAG);
952 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
953 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
954 DAG.getIntPtrConstant(0, dl));
955 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
956 DAG.getIntPtrConstant(1, dl));
966 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
969 EVT ShiftAmtVT = TLI.getShiftAmountTy(NVT, DAG.getDataLayout(), false);
970 Lo = DAG.getNode(ISD::ZERO_EXTEND, dlLo, NVT, Lo);
971 Hi = DAG.getNode(ISD::ANY_EXTEND, dlHi, NVT, Hi);
972 Hi = DAG.getNode(ISD::SHL, dlHi, NVT, Hi,
973 DAG.getConstant(LVT.getSizeInBits(), dlHi, ShiftAmtVT));
974 return DAG.getNode(ISD::OR, dlHi, NVT, Lo, Hi);
985 return TLI.makeLibCall(DAG, LC, N->getValueType(0), None, CallOptions,
989 return TLI.makeLibCall(DAG, LC, N->getValueType(0), Op, CallOptions,
993 return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, CallOptions,
1000 return TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, CallOptions, dl).first;
1014 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1021 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1022 TLI.getPointerTy(DAG.getDataLayout()));
1024 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1026 TargetLowering::CallLoweringInfo CLI(DAG);
1049 return DAG.getNode(ExtendCode, dl, BoolVT, Bool);
1059 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Op);
1063 TLI.getScalarShiftAmountTy(DAG.getDataLayout(), Op.getValueType());
1066 Hi = DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op,
1067 DAG.getConstant(LoVT.getSizeInBits(), dl, ShiftAmountTy));
1068 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1076 EVT::getIntegerVT(*DAG.getContext(), Op.getValueSizeInBits() / 2);
lib/CodeGen/SelectionDAG/LegalizeTypes.h 63 return TLI.getTypeAction(*DAG.getContext(), VT);
68 return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
77 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
77 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
196 SelectionDAG &getDAG() const { return DAG; }
262 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
263 DAG.getValueType(OldVT));
271 return DAG.getZeroExtendInReg(Op, dl, OldVT.getScalarType());
283 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), Op,
284 DAG.getValueType(OldVT));
285 return DAG.getZeroExtendInReg(Op, DL, OldVT.getScalarType());
lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp 42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
57 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
58 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
62 auto &DL = DAG.getDataLayout();
68 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
69 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
74 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
76 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
77 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
82 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
83 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
89 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
90 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
91 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
93 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
94 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
104 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
113 ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
114 NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
118 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
122 Vals.push_back(DAG.getNode(
124 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
124 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
134 if (DAG.getDataLayout().isBigEndian())
137 Vals.push_back(DAG.getNode(
139 EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
145 if (DAG.getDataLayout().isBigEndian())
157 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(
158 NOutVT.getTypeForEVT(*DAG.getContext()));
159 SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
162 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
165 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
165 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
168 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo);
172 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
173 DAG.getConstant(IncrementSize, dl,
177 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
182 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
215 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
222 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts);
223 OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
226 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
227 EVT::getVectorVT(*DAG.getContext(),
234 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
235 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
237 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
238 DAG.getConstant(1, dl, Idx.getValueType()));
239 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
241 if (DAG.getDataLayout().isBigEndian())
252 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
260 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), Alignment,
265 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
266 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
267 Hi = DAG.getLoad(NVT, dl, Chain, Ptr,
274 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
278 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
288 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
294 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
295 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
299 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
322 if (DAG.getDataLayout().isBigEndian())
327 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
346 EVT NVT = EVT::getVectorVT(*DAG.getContext(),
347 TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
360 DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts));
361 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
373 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
387 if (DAG.getDataLayout().isBigEndian())
393 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
394 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
397 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
414 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
421 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
422 SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
427 if (DAG.getDataLayout().isBigEndian())
431 Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
432 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
433 Idx = DAG.getNode(ISD::ADD, dl,
435 DAG.getConstant(1, dl, Idx.getValueType()));
436 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
439 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
450 SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
453 return DAG.getBuildVector(VT, dl, Ops);
463 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
475 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
478 Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), Alignment,
481 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
482 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
487 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
516 std::tie(CL, CH) = DAG.SplitVector(Res->getOperand(0), dl);
532 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
536 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
539 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
540 Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
550 Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
552 Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
558 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
559 Lo = DAG.getUNDEF(LoVT);
560 Hi = DAG.getUNDEF(HiVT);
lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp 35 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
43 N->dump(&DAG);
202 return DAG.getNode(N->getOpcode(), SDLoc(N),
210 return DAG.getNode(N->getOpcode(), SDLoc(N),
218 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
244 SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
264 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
265 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
270 SDVTList ScalarVTs = DAG.getVTList(
272 SDNode *ScalarNode = DAG.getNode(
281 SDValue OtherVal = DAG.getNode(
302 return DAG.getNode(ISD::BITCAST, SDLoc(N),
312 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
317 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
325 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
332 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
343 return DAG.getNode(ISD::FPOWI, SDLoc(N),
354 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
361 SDValue Result = DAG.getLoad(
364 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
392 Op = DAG.getNode(
394 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
394 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
396 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
403 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
404 LHS, DAG.getValueType(ExtVT));
418 Op = DAG.getNode(
420 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
420 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
425 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
427 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
429 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
441 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
456 Cond = DAG.getNode(
458 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
458 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
491 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
492 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
498 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
499 Cond, DAG.getValueType(MVT::i1));
507 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
509 return DAG.getSelect(SDLoc(N),
516 return DAG.getSelect(SDLoc(N),
523 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
530 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
537 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
558 LHS = DAG.getNode(
560 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
560 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
561 RHS = DAG.getNode(
563 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
563 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
567 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
573 return DAG.getNode(ExtendCode, DL, NVT, Res);
582 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
591 N->dump(&DAG);
671 return DAG.getNode(ISD::BITCAST, SDLoc(N),
681 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
685 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
694 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
702 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
710 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
720 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
721 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
732 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
753 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
761 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
763 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
774 return DAG.getTruncStore(
780 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
790 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
793 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
800 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
807 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
814 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
827 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
838 N->dump(&DAG);
1024 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1025 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1038 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
1040 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
1053 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
1054 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
1062 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1084 if (DAG.getDataLayout().isBigEndian())
1086 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1087 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1095 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1096 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1101 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1102 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1103 if (DAG.getDataLayout().isBigEndian())
1108 if (DAG.getDataLayout().isBigEndian())
1110 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1111 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1118 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1121 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1124 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1139 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1142 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1145 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1155 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1157 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1159 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1160 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1161 TLI.getVectorIdxTy(DAG.getDataLayout())));
1186 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1187 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1193 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1195 DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1195 DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1198 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1199 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1200 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1201 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1205 DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1210 DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1211 DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1214 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1222 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1223 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1238 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1241 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1242 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1253 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1255 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1256 DAG.getValueType(LoVT));
1257 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1258 DAG.getValueType(HiVT));
1272 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1278 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1293 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1293 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1295 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1296 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1305 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1327 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1336 Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1337 Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1341 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1378 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
1379 DAG.getConstant(i, dl, TLI.getVectorIdxTy(
1380 DAG.getDataLayout())));
1385 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1395 Scalars.push_back(DAG.getUNDEF(EltVT));
1398 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1402 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1403 return DAG.getBuildVector(VecVT, dl, Scalars);
1412 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1413 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1420 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1421 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1425 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1426 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1427 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1428 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1440 SDValue OtherVal = DAG.getNode(
1459 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1463 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1464 DAG.getConstant(IdxVal - LoNumElts, dl,
1465 TLI.getVectorIdxTy(DAG.getDataLayout())));
1478 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1480 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1483 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1487 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1488 auto &MF = DAG.getMachineFunction();
1491 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1491 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1495 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1496 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1497 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1498 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1502 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1505 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1509 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1510 DAG.getConstant(IncrementSize, dl,
1514 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1519 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1521 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1523 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1530 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1531 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1532 Hi = DAG.getUNDEF(HiVT);
1540 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1545 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1552 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1554 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1558 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1559 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1565 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1577 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1594 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1599 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1605 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1607 MachineMemOperand *MMO = DAG.getMachineFunction().
1612 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1615 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1619 MMO = DAG.getMachineFunction().getMachineMemOperand(
1624 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1629 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1642 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1660 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1666 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1672 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1678 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1680 MachineMemOperand *MMO = DAG.getMachineFunction().
1686 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1686 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1690 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1690 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1695 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1711 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1719 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1725 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1727 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1728 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1736 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1745 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1748 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1749 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1751 Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1753 Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1755 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1759 Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other },
1761 Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other },
1765 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1769 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1770 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1780 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1798 LLVMContext &Ctx = *DAG.getContext();
1803 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1807 N->dump(&DAG); dbgs() << "\n");
1810 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1812 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1814 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1815 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1901 SVOps.push_back(DAG.getUNDEF(EltVT));
1909 SVOps.push_back(DAG.getNode(
1911 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1911 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1915 Output = DAG.getBuildVector(NewVT, dl, SVOps);
1918 Output = DAG.getUNDEF(NewVT);
1923 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1925 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1934 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
1940 const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment(
1941 NVT.getTypeForEVT(*DAG.getContext()));
1943 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment);
1944 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment);
1962 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1974 N->dump(&DAG);
2091 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2095 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2096 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2097 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2100 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2102 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2104 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2117 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2145 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2146 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2157 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2161 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2163 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2168 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2175 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2176 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2179 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2191 if (DAG.getDataLayout().isBigEndian())
2194 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2212 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2214 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2215 DAG.getConstant(IdxVal - LoElts, dl,
2234 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2235 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2236 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2249 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2251 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2255 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2256 auto &MF = DAG.getMachineFunction();
2259 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2259 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2262 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2267 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2268 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2269 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2272 return DAG.getExtLoad(
2274 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
2284 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2291 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2306 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2310 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2316 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2322 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2324 MachineMemOperand *MMO = DAG.getMachineFunction().
2330 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2330 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2333 MMO = DAG.getMachineFunction().
2340 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2340 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2345 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2352 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2369 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2376 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2386 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2390 MachineMemOperand *MMO = DAG.getMachineFunction().
2395 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2399 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2403 MMO = DAG.getMachineFunction().getMachineMemOperand(
2408 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2413 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2430 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2437 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2447 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2454 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2457 MachineMemOperand *MMO = DAG.getMachineFunction().
2463 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2463 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2466 MMO = DAG.getMachineFunction().
2475 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2475 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2495 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2499 return TLI.scalarizeVectorStore(N, DAG);
2504 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2507 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2511 Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2514 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2518 Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2522 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2538 Elts.push_back(DAG.getNode(
2540 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2540 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2544 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2580 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2594 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2606 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2607 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2609 SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2610 SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2612 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2613 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2620 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2621 DAG.getTargetConstant(
2622 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2623 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2636 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2637 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2639 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2640 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2641 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2646 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2658 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2662 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2664 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2668 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2672 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2673 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2676 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2682 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2691 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2703 N->dump(&DAG);
2873 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2876 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2910 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2914 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2920 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2923 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2929 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2933 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3022 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3029 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3036 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3041 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3060 SDValue EOp1 = DAG.getNode(
3062 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3062 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3063 SDValue EOp2 = DAG.getNode(
3065 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3065 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3066 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3072 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3077 SDValue EOp1 = DAG.getNode(
3079 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3079 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3080 SDValue EOp2 = DAG.getNode(
3082 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3082 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3083 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3090 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3098 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3104 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3150 Op = DAG.getNode(
3152 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3152 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3158 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3166 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3177 Op = DAG.getNode(
3179 DAG.getConstant(Idx, dl,
3180 TLI.getVectorIdxTy(DAG.getDataLayout())));
3186 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3199 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3202 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3214 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3216 *DAG.getContext(), OvVT.getVectorElementType(),
3222 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3224 *DAG.getContext(), ResVT.getVectorElementType(),
3227 SDValue Zero = DAG.getConstant(
3228 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3229 WideLHS = DAG.getNode(
3230 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3232 WideRHS = DAG.getNode(
3233 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3237 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3238 SDNode *WideNode = DAG.getNode(
3247 SDValue Zero = DAG.getConstant(
3248 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3249 SDValue OtherVal = DAG.getNode(
3261 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3266 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3277 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3278 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3285 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3287 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3289 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3302 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3304 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3306 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3307 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3311 SDValue InVal = DAG.getNode(
3313 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3313 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3316 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3317 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3323 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3328 SDValue Val = DAG.getNode(
3330 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3330 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3332 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3334 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3337 return DAG.getBuildVector(WidenVT, DL, Ops);
3345 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3359 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3365 NewOps[1] = DAG.getNode(
3367 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3367 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3368 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3371 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3374 return DAG.getBuildVector(WidenVT, DL, Ops);
3382 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3398 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3406 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3407 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3407 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3410 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3413 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3416 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3425 Ops.push_back(DAG.getUNDEF(WidenSVT));
3427 return DAG.getBuildVector(WidenVT, DL, Ops);
3437 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3438 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3442 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3445 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3449 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3458 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3464 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3469 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3471 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3475 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3476 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3481 return DAG.getNode(N->getOpcode(), SDLoc(N),
3482 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3494 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3512 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3528 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3543 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3546 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3557 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3560 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3562 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3564 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3581 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3586 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3588 return DAG.getBuildVector(WidenVT, dl, NewOps);
3593 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3605 SDValue UndefVal = DAG.getUNDEF(InVT);
3611 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3615 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3634 return DAG.getVectorShuffle(WidenVT, dl,
3651 Ops[Idx++] = DAG.getNode(
3653 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3653 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3655 SDValue UndefVal = DAG.getUNDEF(EltVT);
3658 return DAG.getBuildVector(WidenVT, dl, Ops);
3663 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3682 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3692 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3693 DAG.getConstant(IdxVal + i, dl,
3694 TLI.getVectorIdxTy(DAG.getDataLayout())));
3696 SDValue UndefVal = DAG.getUNDEF(EltVT);
3699 return DAG.getBuildVector(WidenVT, dl, Ops);
3704 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3727 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3738 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3746 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3751 SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3763 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3772 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3779 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3785 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3785 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3796 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3797 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3852 SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3856 LLVMContext &Ctx = *DAG.getContext();
3862 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3866 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3876 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3877 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3878 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3883 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3885 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3899 LLVMContext &Ctx = *DAG.getContext();
3994 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
4001 return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
4005 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4015 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4038 return DAG.getNode(N->getOpcode(), SDLoc(N),
4045 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4051 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4052 return DAG.getUNDEF(WidenVT);
4059 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4077 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4084 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4090 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4108 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4109 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4118 return DAG.getNode(ISD::SETCC, SDLoc(N),
4127 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4139 N->dump(&DAG);
4240 InOp = DAG.getNode(
4241 ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
4242 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4242 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4244 InOp = DAG.getNode(
4246 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4246 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4264 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4266 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4268 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4276 return DAG.UnrollVectorOp(N);
4295 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4300 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4306 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4307 return DAG.getNode(
4309 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4309 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4320 NewOps[1] = DAG.getNode(
4322 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4322 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4323 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4326 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4330 Ops[i] = DAG.getNode(
4332 DAG.getNode(
4334 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4334 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4337 return DAG.getBuildVector(VT, dl, Ops);
4352 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4354 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4355 return DAG.getNode(
4357 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4357 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4370 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4372 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4373 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4374 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4374 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4391 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4415 Ops[Idx++] = DAG.getNode(
4417 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4417 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4419 return DAG.getBuildVector(VT, dl, Ops);
4424 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4430 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4440 return TLI.scalarizeVectorStore(ST, DAG);
4451 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4469 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4475 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4479 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4488 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4506 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4526 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4532 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4543 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4543 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4563 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4566 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4570 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4573 SDValue CC = DAG.getNode(
4575 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4575 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4580 return DAG.getNode(ExtendCode, dl, VT, CC);
4596 NeutralElem = DAG.getConstant(0, dl, ElemVT);
4599 NeutralElem = DAG.getConstant(1, dl, ElemVT);
4603 NeutralElem = DAG.getAllOnesConstant(dl, ElemVT);
4606 NeutralElem = DAG.getConstant(
4610 NeutralElem = DAG.getConstant(
4614 NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT);
4617 NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT);
4620 NeutralElem = DAG.getConstantFP(
4624 NeutralElem = DAG.getConstantFP(
4633 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4634 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4634 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4636 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags());
4647 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4648 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4651 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4653 return DAG.getNode(
4655 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4655 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4766 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4785 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4787 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4795 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4796 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4797 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4805 SDValue UndefVal = DAG.getUNDEF(NewVT);
4809 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4822 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4827 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4829 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4841 Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4844 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4847 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4863 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4879 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4886 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4895 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4901 SDValue UndefVal = DAG.getUNDEF(LdTy);
4909 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4918 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4939 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4944 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4945 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4952 SDValue UndefVal = DAG.getUNDEF(EltVT);
4956 return DAG.getBuildVector(WidenVT, dl, Ops);
4984 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4990 SDValue EOp = DAG.getNode(
4992 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4992 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4993 StChain.push_back(DAG.getStore(
5000 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
5005 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5006 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5010 SDValue EOp = DAG.getNode(
5012 DAG.getConstant(Idx++, dl,
5013 TLI.getVectorIdxTy(DAG.getDataLayout())));
5014 StChain.push_back(DAG.getStore(
5019 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
5054 SDValue EOp = DAG.getNode(
5056 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5056 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5057 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
5062 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
5063 SDValue EOp = DAG.getNode(
5065 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5065 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5066 StChain.push_back(DAG.getTruncStore(
5093 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5094 DAG.getUNDEF(InVT);
5099 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5103 return DAG.getNode(
5105 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5105 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5113 Ops[Idx] = DAG.getNode(
5115 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5115 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5117 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5118 DAG.getUNDEF(EltVT);
5121 return DAG.getBuildVector(NVT, dl, Ops);