|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
References
tools/clang/lib/CodeGen/CGBuiltin.cpp 7468 Ops.push_back(EmitScalarExpr(E->getArg(i)));
7476 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
7485 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
7486 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
7505 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7506 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
7516 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
7525 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7526 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
7529 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
7529 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
7531 return Builder.CreateFPToUI(Ops[0], InTy);
7532 return Builder.CreateFPToSI(Ops[0], InTy);
7540 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7541 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
7544 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7544 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7546 return Builder.CreateUIToFP(Ops[0], FTy);
7547 return Builder.CreateSIToFP(Ops[0], FTy);
7557 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7560 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
7562 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
7566 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7566 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7568 return Builder.CreateUIToFP(Ops[0], FTy);
7569 return Builder.CreateSIToFP(Ops[0], FTy);
7575 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7576 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7576 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7578 return Builder.CreateFPToUI(Ops[0], Int16Ty);
7579 return Builder.CreateFPToSI(Ops[0], Int16Ty);
7585 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7586 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7586 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7588 return Builder.CreateFPToUI(Ops[0], Int32Ty);
7589 return Builder.CreateFPToSI(Ops[0], Int32Ty);
7595 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7596 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7596 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7598 return Builder.CreateFPToUI(Ops[0], Int64Ty);
7599 return Builder.CreateFPToSI(Ops[0], Int64Ty);
7613 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7633 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
7633 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt");
7634 return Builder.CreateTrunc(Ops[0], Int16Ty);
7644 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7652 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
7652 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
7654 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
7654 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
7656 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
7656 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg");
7657 return Builder.CreateTrunc(Ops[0], Int16Ty);
7665 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7673 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
7673 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
7674 return Builder.CreateTrunc(Ops[0], Int16Ty);
7682 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7687 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
7687 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
7691 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
7691 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
7694 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n");
7738 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7740 Ops[0], ConvertType(E->getCallReturnType(getContext())),
7746 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7748 Ops[0], ConvertType(E->getCallReturnType(getContext())),
7754 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7756 Ops[0], ConvertType(E->getCallReturnType(getContext())),
7762 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7764 Ops[0], ConvertType(E->getCallReturnType(getContext())),
7770 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7772 Ops[0], ConvertType(E->getCallReturnType(getContext())),
7776 Ops.push_back(EmitScalarExpr(E->getArg(0)));
7777 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7777 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7778 Ops[0] =
7779 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
7780 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
7796 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7797 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7797 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7798 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7798 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7799 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7799 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7799 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7800 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
7816 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7817 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
7817 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
7818 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
7818 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
7819 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7819 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7819 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7820 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
7836 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7837 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7837 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7838 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
7838 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
7839 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7839 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7839 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7840 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
7866 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7867 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7867 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7868 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7868 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7869 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
7869 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
7869 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
7870 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
7874 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7875 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7875 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7876 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7876 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7877 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7877 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7877 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7878 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
7878 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
7880 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
7892 Ops.push_back(EmitScalarExpr(E->getArg(2)));
7893 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7893 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7893 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7896 Ops[1] = Builder.CreateBitCast(Ops[1],
7896 Ops[1] = Builder.CreateBitCast(Ops[1],
7898 Ops.push_back(EmitScalarExpr(E->getArg(2)));
7899 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7899 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7899 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7902 Ops[1] = Builder.CreateBitCast(Ops[1],
7902 Ops[1] = Builder.CreateBitCast(Ops[1],
7904 Ops.push_back(EmitScalarExpr(E->getArg(2)));
7905 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7905 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7905 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
7909 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
7909 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
7910 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7914 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
7914 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
7915 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7919 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
7919 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
7920 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7924 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
7924 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
7925 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7929 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
7929 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
7930 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7933 Ops[0] = Builder.CreateBitCast(Ops[0],
7933 Ops[0] = Builder.CreateBitCast(Ops[0],
7935 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7939 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
7939 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
7940 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7944 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
7944 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
7945 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7948 Ops[0] = Builder.CreateBitCast(Ops[0],
7948 Ops[0] = Builder.CreateBitCast(Ops[0],
7950 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7954 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
7954 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
7955 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7958 Ops[0] = Builder.CreateBitCast(Ops[0],
7958 Ops[0] = Builder.CreateBitCast(Ops[0],
7960 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7963 Ops[0] = Builder.CreateBitCast(Ops[0],
7963 Ops[0] = Builder.CreateBitCast(Ops[0],
7965 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7969 Ops[0] = Builder.CreateBitCast(Ops[0],
7969 Ops[0] = Builder.CreateBitCast(Ops[0],
7971 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7975 Ops[0] = Builder.CreateBitCast(Ops[0],
7975 Ops[0] = Builder.CreateBitCast(Ops[0],
7977 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
7980 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7981 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
7981 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
7983 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7984 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
7984 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
7986 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7987 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
7987 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
7989 Ops.push_back(EmitScalarExpr(E->getArg(1)));
7990 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
7990 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
7995 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]});
8002 return Builder.CreateCall(F, {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]});
8006 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
8009 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
8013 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
8016 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
8019 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
8019 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
8024 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
8027 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8028 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
8028 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
8030 Ops, "vqshlu_n");
8037 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8038 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
8038 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
8039 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
8046 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8047 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
8048 Ops[1] = ConstantInt::get(Int64Ty, -SV);
8049 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
8056 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
8056 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
8057 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
8058 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
8059 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
8059 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
8060 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
8060 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
8066 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
8071 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
8081 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
8086 Ops[1] = Builder.CreateAShr(
8087 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
8090 return Builder.CreateAdd(Ops[0], Ops[1]);
8090 return Builder.CreateAdd(Ops[0], Ops[1]);
8098 return Ops[0];
8099 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
8099 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
8101 return Builder.CreateAdd(Ops[0], Ops[1]);
8101 return Builder.CreateAdd(Ops[0], Ops[1]);
8107 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
8107 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
8110 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
8111 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
8113 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
8116 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
8116 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
8117 Ops.pop_back();
8123 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
8128 ProductOps.push_back(Ops[1]);
8130 Ops[1] =
8137 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
8143 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
8143 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
8146 ProductOps.push_back(Ops[1]);
8147 ProductOps.push_back(Ops[2]);
8148 Ops[1] =
8151 Ops.pop_back();
8157 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
8160 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
8164 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
8327 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
8330 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch))
8339 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
8339 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
8340 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
8340 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
8341 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
8341 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
8343 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
8343 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
8343 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
8344 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
8344 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
8344 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
8345 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
8345 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
8345 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
8346 return Builder.CreateBitCast(Ops[0], Ty);
8352 Value *Addend = Ops[0];
8353 Value *Multiplicand = Ops[1];
8354 Value *LaneSource = Ops[2];
8355 Ops[0] = Multiplicand;
8356 Ops[1] = LaneSource;
8357 Ops[2] = Addend;
8363 llvm::Constant *cst = cast<Constant>(Ops[3]);
8365 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
8365 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
8366 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
8366 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
8366 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
8368 Ops.pop_back();
8370 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
8376 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8376 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8377 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
8377 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
8380 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
8380 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
8381 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8381 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8381 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8383 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8383 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8383 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8387 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8387 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8388 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8388 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8392 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
8392 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
8394 cast<ConstantInt>(Ops[3]));
8395 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
8395 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
8395 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
8397 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8397 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8397 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8401 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8401 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8402 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8402 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8404 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8404 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8405 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
8405 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
8405 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
8406 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8406 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8406 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
8414 Ops.push_back(EmitScalarExpr(E->getArg(3)));
8417 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8417 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8417 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
8418 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8418 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8418 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
8424 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
8430 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
8432 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8434 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax");
8441 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
8443 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8445 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin");
8452 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
8463 TmpOps.push_back(Ops[1]);
8466 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
8474 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
8480 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
8484 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
8486 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8488 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm");
8492 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
8494 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8496 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm");
8498 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8500 Ops, "vrecps");
8503 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8505 Ops, "vrecps");
8507 Ops.push_back(EmitScalarExpr(E->getArg(1)));
8509 Ops, "vrecps");
8512 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
8515 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
8518 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
8521 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
8524 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
8526 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8528 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda");
8533 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
8536 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8538 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi");
8541 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8543 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm");
8548 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
8551 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8553 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn");
8558 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
8561 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8563 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn");
8566 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8568 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp");
8573 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
8576 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8578 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx");
8583 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
8586 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8588 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz");
8593 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
8597 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8597 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8599 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
8600 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
8605 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
8605 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
8607 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
8613 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
8613 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
8615 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
8629 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
8629 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
8631 return Builder.CreateFPToUI(Ops[0], Ty);
8632 return Builder.CreateFPToSI(Ops[0], Ty);
8648 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
8664 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
8680 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
8696 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
8701 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
8707 Ops.push_back(EmitScalarExpr(E->getArg(2)));
8708 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8708 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8708 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8709 Ops.pop_back();
8711 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx");
8719 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8719 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8722 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8722 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8723 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8723 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8723 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
8724 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
8724 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
8734 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
8739 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
8742 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8744 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt");
8749 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8749 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8750 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
8755 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
8766 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8767 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8767 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8768 return Builder.CreateTrunc(Ops[0], Int8Ty);
8778 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8779 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8779 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8780 return Builder.CreateTrunc(Ops[0], Int16Ty);
8790 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8791 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8791 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8792 return Builder.CreateTrunc(Ops[0], Int8Ty);
8802 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8803 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8803 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
8804 return Builder.CreateTrunc(Ops[0], Int16Ty);
8811 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8812 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8812 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8813 return Builder.CreateTrunc(Ops[0], Int8Ty);
8820 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8821 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8821 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8822 return Builder.CreateTrunc(Ops[0], Int16Ty);
8829 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8830 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8830 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8831 return Builder.CreateTrunc(Ops[0], Int8Ty);
8838 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8839 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8839 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8840 return Builder.CreateTrunc(Ops[0], Int16Ty);
8847 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8848 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8848 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8849 return Builder.CreateTrunc(Ops[0], Int8Ty);
8856 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8857 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8857 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8858 return Builder.CreateTrunc(Ops[0], Int16Ty);
8865 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8866 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8866 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8867 return Builder.CreateTrunc(Ops[0], Int8Ty);
8874 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8875 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8875 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8876 return Builder.CreateTrunc(Ops[0], Int16Ty);
8883 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8884 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8884 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8885 return Builder.CreateTrunc(Ops[0], HalfTy);
8892 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8893 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8893 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
8894 return Builder.CreateTrunc(Ops[0], HalfTy);
8901 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8902 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8902 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8903 return Builder.CreateTrunc(Ops[0], Int8Ty);
8910 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8911 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8911 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8912 return Builder.CreateTrunc(Ops[0], Int16Ty);
8919 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8920 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8920 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8921 return Builder.CreateTrunc(Ops[0], Int8Ty);
8928 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8929 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8929 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8930 return Builder.CreateTrunc(Ops[0], Int16Ty);
8937 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8938 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8938 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8939 return Builder.CreateTrunc(Ops[0], Int8Ty);
8946 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8947 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8947 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8948 return Builder.CreateTrunc(Ops[0], Int16Ty);
8955 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8956 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8956 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8957 return Builder.CreateTrunc(Ops[0], Int8Ty);
8964 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8965 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8965 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8966 return Builder.CreateTrunc(Ops[0], Int16Ty);
8973 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8974 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8974 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8975 return Builder.CreateTrunc(Ops[0], HalfTy);
8982 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8983 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8983 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
8984 return Builder.CreateTrunc(Ops[0], HalfTy);
8991 Ops.push_back(EmitScalarExpr(E->getArg(0)));
8992 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
8992 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
8993 return Builder.CreateTrunc(Ops[0], HalfTy);
9000 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9001 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
9001 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv");
9002 return Builder.CreateTrunc(Ops[0], HalfTy);
9009 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9010 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
9010 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
9011 return Builder.CreateTrunc(Ops[0], HalfTy);
9018 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9019 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
9019 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv");
9020 return Builder.CreateTrunc(Ops[0], HalfTy);
9023 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
9023 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
9025 return Builder.CreateFMul(Ops[0], RHS);
9032 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9033 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9033 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9034 return Builder.CreateTrunc(Ops[0], Int16Ty);
9041 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9042 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9049 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9050 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9050 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9051 return Builder.CreateTrunc(Ops[0], Int16Ty);
9058 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9059 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9066 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9067 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9067 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9068 return Builder.CreateTrunc(Ops[0], Int16Ty);
9075 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9076 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9083 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9084 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9084 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9085 return Builder.CreateTrunc(Ops[0], Int16Ty);
9092 Ops.push_back(EmitScalarExpr(E->getArg(0)));
9093 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
9099 return EmitNeonCall(Intrin, Ops, "vsri_n");
9105 return EmitNeonCall(Intrin, Ops, "vsli_n");
9109 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9109 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9110 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
9110 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
9110 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
9111 return Builder.CreateAdd(Ops[0], Ops[1]);
9111 return Builder.CreateAdd(Ops[0], Ops[1]);
9116 TmpOps.push_back(Ops[1]);
9117 TmpOps.push_back(Ops[2]);
9120 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
9120 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
9121 return Builder.CreateAdd(Ops[0], tmp);
9125 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
9125 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
9128 return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
9132 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
9132 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
9133 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
9133 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
9134 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9134 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9137 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9137 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9139 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9139 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9142 Ops[0] =
9143 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
9144 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
9144 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
9144 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
9150 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9150 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9153 Ops[0] =
9154 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
9156 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
9156 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
9157 return EmitNeonSplat(Ops[0], CI);
9161 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9161 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9162 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
9162 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
9162 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
9163 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
9164 return Builder.CreateDefaultAlignedStore(Ops[1],
9165 Builder.CreateBitCast(Ops[0], Ty));
9169 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9169 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9172 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
9172 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
9173 Ops[0] = Builder.CreateBitCast(Ops[0],
9173 Ops[0] = Builder.CreateBitCast(Ops[0],
9174 llvm::PointerType::getUnqual(Ops[1]->getType()));
9175 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9175 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9180 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9180 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9183 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
9183 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
9184 Ops[0] = Builder.CreateBitCast(Ops[0],
9184 Ops[0] = Builder.CreateBitCast(Ops[0],
9185 llvm::PointerType::getUnqual(Ops[1]->getType()));
9186 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9186 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9191 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9191 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9194 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
9194 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
9195 Ops[0] = Builder.CreateBitCast(Ops[0],
9195 Ops[0] = Builder.CreateBitCast(Ops[0],
9196 llvm::PointerType::getUnqual(Ops[1]->getType()));
9197 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9197 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9203 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9203 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9206 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
9206 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
9207 Ops[0] = Builder.CreateBitCast(Ops[0],
9207 Ops[0] = Builder.CreateBitCast(Ops[0],
9208 llvm::PointerType::getUnqual(Ops[1]->getType()));
9209 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9209 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9215 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9215 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9218 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
9218 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
9219 Ops[0] = Builder.CreateBitCast(Ops[0],
9219 Ops[0] = Builder.CreateBitCast(Ops[0],
9220 llvm::PointerType::getUnqual(Ops[1]->getType()));
9221 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9221 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9227 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9227 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
9230 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
9230 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
9231 Ops[0] = Builder.CreateBitCast(Ops[0],
9231 Ops[0] = Builder.CreateBitCast(Ops[0],
9232 llvm::PointerType::getUnqual(Ops[1]->getType()));
9233 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9233 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9237 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
9239 Ops.push_back(Ops[1]);
9239 Ops.push_back(Ops[1]);
9240 Ops.erase(Ops.begin()+1);
9240 Ops.erase(Ops.begin()+1);
9241 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9241 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9242 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9242 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9243 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
9243 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
9244 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
9244 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
9245 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
9246 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9246 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9247 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9247 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9251 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
9253 Ops.push_back(Ops[1]);
9253 Ops.push_back(Ops[1]);
9254 Ops.erase(Ops.begin()+1);
9254 Ops.erase(Ops.begin()+1);
9255 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9255 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9256 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9256 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9257 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
9257 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
9258 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
9258 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
9259 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
9259 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
9260 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
9261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9262 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9262 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9266 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
9268 Ops.push_back(Ops[1]);
9268 Ops.push_back(Ops[1]);
9269 Ops.erase(Ops.begin()+1);
9269 Ops.erase(Ops.begin()+1);
9270 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9270 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9271 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9271 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9272 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
9272 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
9273 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
9273 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
9274 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
9274 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
9275 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
9275 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
9276 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
9277 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9277 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9278 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9278 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9282 Ops.push_back(Ops[0]);
9282 Ops.push_back(Ops[0]);
9283 Ops.erase(Ops.begin());
9283 Ops.erase(Ops.begin());
9284 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
9286 Ops, "");
9290 Ops.push_back(Ops[0]);
9290 Ops.push_back(Ops[0]);
9291 Ops.erase(Ops.begin());
9291 Ops.erase(Ops.begin());
9292 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
9292 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
9293 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
9295 Ops, "");
9299 Ops.push_back(Ops[0]);
9299 Ops.push_back(Ops[0]);
9300 Ops.erase(Ops.begin());
9300 Ops.erase(Ops.begin());
9301 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
9303 Ops, "");
9307 Ops.push_back(Ops[0]);
9307 Ops.push_back(Ops[0]);
9308 Ops.erase(Ops.begin());
9308 Ops.erase(Ops.begin());
9309 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
9309 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
9310 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
9312 Ops, "");
9316 Ops.push_back(Ops[0]);
9316 Ops.push_back(Ops[0]);
9317 Ops.erase(Ops.begin());
9317 Ops.erase(Ops.begin());
9318 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
9320 Ops, "");
9324 Ops.push_back(Ops[0]);
9324 Ops.push_back(Ops[0]);
9325 Ops.erase(Ops.begin());
9325 Ops.erase(Ops.begin());
9326 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
9326 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
9327 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
9329 Ops, "");
9333 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9333 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9334 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9334 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9335 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9335 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9344 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
9345 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
9345 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
9352 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9352 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9353 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9353 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9354 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9354 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9362 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
9363 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
9363 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
9370 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9370 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
9371 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9371 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9372 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9372 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
9381 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
9382 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
9382 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
9389 Ops, "vtbl1");
9393 Ops, "vtbl2");
9397 Ops, "vtbl3");
9401 Ops, "vtbl4");
9405 Ops, "vtbx1");
9409 Ops, "vtbx2");
9413 Ops, "vtbx3");
9417 Ops, "vtbx4");
9422 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
9427 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");