|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
References
lib/Target/X86/X86ISelLowering.cpp25767 unsigned EltSizeInBits = VT.getScalarSizeInBits();
25774 assert(VT.isVector() && "Custom lowering only for vector shifts!");
25783 if (SupportedVectorVarShift(VT, Subtarget, Opc))
25788 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
25788 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
25789 VT == MVT::v8i16 || VT == MVT::v16i8)) {
25789 VT == MVT::v8i16 || VT == MVT::v16i8)) {
25791 SDValue Zero = DAG.getConstant(0, dl, VT);
25792 Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt);
25795 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
25797 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
25802 if (VT == MVT::v2i64 && Opc != ISD::SRA) {
25804 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
25805 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
25806 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
25807 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
25808 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
25814 if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
25814 if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
25816 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
25817 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
25818 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
25819 R = DAG.getNode(ISD::XOR, dl, VT, R, M);
25820 R = DAG.getNode(ISD::SUB, dl, VT, R, M);
25834 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
25834 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
25835 (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
25837 unsigned NumElts = VT.getVectorNumElements();
25860 (VT != MVT::v16i16 ||
25861 is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&
25862 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||
25868 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
25870 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
25872 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
25881 return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
25886 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
25886 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
25887 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
25888 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
25890 SDValue Zero = DAG.getConstant(0, dl, VT);
25891 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
25892 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
25893 return DAG.getSelect(dl, VT, ZAmt, R, Res);
25902 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
25902 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
25906 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
25907 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
25910 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
25910 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
25912 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
25912 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
25914 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
25915 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
25916 Res = DAG.getSelect(dl, VT, Amt0, R, Res);
25917 return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
25926 if (VT == MVT::v4i32) {
25929 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
25929 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
25930 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
25930 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
25931 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
25931 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
25932 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
25932 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
25939 SDValue Z = DAG.getConstant(0, dl, VT);
25940 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
25941 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
25942 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
25943 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
25960 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
25960 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
25961 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
25961 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
25962 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
25962 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
25963 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
25963 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
25968 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
25969 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
25970 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
25972 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
25973 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
25974 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
25981 if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||
25982 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||
25983 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||
25984 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||
25985 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {
25986 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&
25986 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&
25989 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());
25993 return DAG.getNode(ISD::TRUNCATE, dl, VT,
26000 (VT == MVT::v16i8 || VT == MVT::v64i8 ||
26000 (VT == MVT::v16i8 || VT == MVT::v64i8 ||
26001 (VT == MVT::v32i8 && Subtarget.hasInt256())) &&
26003 int NumElts = VT.getVectorNumElements();
26015 if (VT == MVT::v16i8 && Subtarget.hasInt256()) {
26020 return DAG.getZExtOrTrunc(R, dl, VT);
26035 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
26036 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
26043 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
26046 if (VT == MVT::v16i8 ||
26047 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
26048 (VT == MVT::v64i8 && Subtarget.hasBWI())) {
26049 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
26052 if (VT.is512BitVector()) {
26056 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
26057 V0 = DAG.getBitcast(VT, V0);
26058 V1 = DAG.getBitcast(VT, V1);
26059 Sel = DAG.getBitcast(VT, Sel);
26060 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
26062 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
26066 V0 = DAG.getBitcast(VT, V0);
26067 V1 = DAG.getBitcast(VT, V1);
26068 Sel = DAG.getBitcast(VT, Sel);
26069 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
26084 Amt = DAG.getBitcast(VT, Amt);
26088 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
26088 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
26089 R = SignBitSelect(VT, Amt, M, R);
26092 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26095 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
26095 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
26096 R = SignBitSelect(VT, Amt, M, R);
26099 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26102 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
26102 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
26103 R = SignBitSelect(VT, Amt, M, R);
26111 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26111 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26112 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26112 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26113 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
26113 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
26114 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
26114 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
26150 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
26154 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {
26156 SDValue Z = DAG.getConstant(0, dl, VT);
26157 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
26158 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
26159 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
26160 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
26169 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
26172 if (VT == MVT::v8i16) {
26182 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
26186 return DAG.getBitcast(VT, DAG.getSelect(dl, ExtVT, Sel, V0, V1));
26192 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
26193 return DAG.getSelect(dl, VT, C, V0, V1);
26201 ISD::OR, dl, VT,
26202 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
26203 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
26205 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
26209 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
26213 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26216 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
26220 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26223 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
26227 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26230 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
26236 if (VT.is256BitVector())