lib/Target/ARM/ARMISelLowering.cpp

reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced

//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that ARM uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//

#include "ARMISelLowering.h"
#include "ARMBaseInstrInfo.h"
#include "ARMBaseRegisterInfo.h"
#include "ARMCallingConv.h"
#include "ARMConstantPoolValue.h"
#include "ARMMachineFunctionInfo.h"
#include "ARMPerfectShuffle.h"
#include "ARMRegisterInfo.h"
#include "ARMSelectionDAGInfo.h"
#include "ARMSubtarget.h"
#include "MCTargetDesc/ARMAddressingModes.h"
#include "MCTargetDesc/ARMBaseInfo.h"
#include "Utils/ARMBaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCInstrItineraries.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSchedule.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MachineValueType.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

using namespace llvm;
using namespace llvm::PatternMatch;

#define DEBUG_TYPE "arm-isel"

STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
STATISTIC(NumConstpoolPromoted,
  "Number of constants with their storage promoted into constant pools");

static cl::opt<bool>
ARMInterworking("arm-interworking", cl::Hidden,
  cl::desc("Enable / disable ARM interworking (for debugging only)"),
  cl::init(true));

static cl::opt<bool> EnableConstpoolPromotion(
    "arm-promote-constant", cl::Hidden,
    cl::desc("Enable / disable promotion of unnamed_addr constants into "
             "constant pools"),
    cl::init(false)); // FIXME: set to true by default once PR32780 is fixed
static cl::opt<unsigned> ConstpoolPromotionMaxSize(
    "arm-promote-constant-max-size", cl::Hidden,
    cl::desc("Maximum size of constant to promote into a constant pool"),
    cl::init(64));
static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
    "arm-promote-constant-max-total", cl::Hidden,
    cl::desc("Maximum size of ALL constants to promote into a constant pool"),
    cl::init(128));

// The APCS parameter registers.
static const MCPhysReg GPRArgRegs[] = {
  ARM::R0, ARM::R1, ARM::R2, ARM::R3
};

void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
                                       MVT PromotedBitwiseVT) {
  if (VT != PromotedLdStVT) {
    setOperationAction(ISD::LOAD, VT, Promote);
    AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);

    setOperationAction(ISD::STORE, VT, Promote);
    AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
  }

  MVT ElemTy = VT.getVectorElementType();
  if (ElemTy != MVT::f64)
    setOperationAction(ISD::SETCC, VT, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
  if (ElemTy == MVT::i32) {
    setOperationAction(ISD::SINT_TO_FP, VT, Custom);
    setOperationAction(ISD::UINT_TO_FP, VT, Custom);
    setOperationAction(ISD::FP_TO_SINT, VT, Custom);
    setOperationAction(ISD::FP_TO_UINT, VT, Custom);
  } else {
    setOperationAction(ISD::SINT_TO_FP, VT, Expand);
    setOperationAction(ISD::UINT_TO_FP, VT, Expand);
    setOperationAction(ISD::FP_TO_SINT, VT, Expand);
    setOperationAction(ISD::FP_TO_UINT, VT, Expand);
  }
  setOperationAction(ISD::BUILD_VECTOR,      VT, Custom);
  setOperationAction(ISD::VECTOR_SHUFFLE,    VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS,    VT, Legal);
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
  setOperationAction(ISD::SELECT,            VT, Expand);
  setOperationAction(ISD::SELECT_CC,         VT, Expand);
  setOperationAction(ISD::VSELECT,           VT, Expand);
  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
  if (VT.isInteger()) {
    setOperationAction(ISD::SHL, VT, Custom);
    setOperationAction(ISD::SRA, VT, Custom);
    setOperationAction(ISD::SRL, VT, Custom);
  }

  // Promote all bit-wise operations.
  if (VT.isInteger() && VT != PromotedBitwiseVT) {
    setOperationAction(ISD::AND, VT, Promote);
    AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT);
    setOperationAction(ISD::OR,  VT, Promote);
    AddPromotedToType (ISD::OR,  VT, PromotedBitwiseVT);
    setOperationAction(ISD::XOR, VT, Promote);
    AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT);
  }

  // Neon does not support vector divide/remainder operations.
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::FDIV, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);
  setOperationAction(ISD::FREM, VT, Expand);

  if (!VT.isFloatingPoint() &&
      VT != MVT::v2i64 && VT != MVT::v1i64)
    for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
      setOperationAction(Opcode, VT, Legal);
}

void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &ARM::DPRRegClass);
  addTypeForNEON(VT, MVT::f64, MVT::v2i32);
}

void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &ARM::DPairRegClass);
  addTypeForNEON(VT, MVT::v2f64, MVT::v4i32);
}

void ARMTargetLowering::setAllExpand(MVT VT) {
  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
    setOperationAction(Opc, VT, Expand);

  // We support these really simple operations even on types where all
  // the actual arithmetic has to be broken down into simpler
  // operations or turned into library calls.
  setOperationAction(ISD::BITCAST, VT, Legal);
  setOperationAction(ISD::LOAD, VT, Legal);
  setOperationAction(ISD::STORE, VT, Legal);
  setOperationAction(ISD::UNDEF, VT, Legal);
}

void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
                                       LegalizeAction Action) {
  setLoadExtAction(ISD::EXTLOAD,  From, To, Action);
  setLoadExtAction(ISD::ZEXTLOAD, From, To, Action);
  setLoadExtAction(ISD::SEXTLOAD, From, To, Action);
}

void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
  const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };

  for (auto VT : IntTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::SHL, VT, Custom);
    setOperationAction(ISD::SRA, VT, Custom);
    setOperationAction(ISD::SRL, VT, Custom);
    setOperationAction(ISD::SMIN, VT, Legal);
    setOperationAction(ISD::SMAX, VT, Legal);
    setOperationAction(ISD::UMIN, VT, Legal);
    setOperationAction(ISD::UMAX, VT, Legal);
    setOperationAction(ISD::ABS, VT, Legal);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::MLOAD, VT, Custom);
    setOperationAction(ISD::MSTORE, VT, Legal);
    setOperationAction(ISD::CTLZ, VT, Legal);
    setOperationAction(ISD::CTTZ, VT, Custom);
    setOperationAction(ISD::BITREVERSE, VT, Legal);
    setOperationAction(ISD::BSWAP, VT, Legal);
    setOperationAction(ISD::SADDSAT, VT, Legal);
    setOperationAction(ISD::UADDSAT, VT, Legal);
    setOperationAction(ISD::SSUBSAT, VT, Legal);
    setOperationAction(ISD::USUBSAT, VT, Legal);

    // No native support for these.
    setOperationAction(ISD::UDIV, VT, Expand);
    setOperationAction(ISD::SDIV, VT, Expand);
    setOperationAction(ISD::UREM, VT, Expand);
    setOperationAction(ISD::SREM, VT, Expand);
    setOperationAction(ISD::CTPOP, VT, Expand);

    // Vector reductions
    setOperationAction(ISD::VECREDUCE_ADD, VT, Legal);
    setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal);
    setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal);
    setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal);
    setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal);

    if (!HasMVEFP) {
      setOperationAction(ISD::SINT_TO_FP, VT, Expand);
      setOperationAction(ISD::UINT_TO_FP, VT, Expand);
      setOperationAction(ISD::FP_TO_SINT, VT, Expand);
      setOperationAction(ISD::FP_TO_UINT, VT, Expand);
    }

    // Pre and Post inc are supported on loads and stores
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
    }
  }

  const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
  for (auto VT : FloatTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    if (!HasMVEFP)
      setAllExpand(VT);

    // These are legal or custom whether we have MVE.fp or not
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::MLOAD, VT, Custom);
    setOperationAction(ISD::MSTORE, VT, Legal);

    // Pre and Post inc are supported on loads and stores
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
    }

    if (HasMVEFP) {
      setOperationAction(ISD::FMINNUM, VT, Legal);
      setOperationAction(ISD::FMAXNUM, VT, Legal);
      setOperationAction(ISD::FROUND, VT, Legal);

      // No native support for these.
      setOperationAction(ISD::FDIV, VT, Expand);
      setOperationAction(ISD::FREM, VT, Expand);
      setOperationAction(ISD::FSQRT, VT, Expand);
      setOperationAction(ISD::FSIN, VT, Expand);
      setOperationAction(ISD::FCOS, VT, Expand);
      setOperationAction(ISD::FPOW, VT, Expand);
      setOperationAction(ISD::FLOG, VT, Expand);
      setOperationAction(ISD::FLOG2, VT, Expand);
      setOperationAction(ISD::FLOG10, VT, Expand);
      setOperationAction(ISD::FEXP, VT, Expand);
      setOperationAction(ISD::FEXP2, VT, Expand);
      setOperationAction(ISD::FNEARBYINT, VT, Expand);
    }
  }

  // We 'support' these types up to bitcast/load/store level, regardless of
  // MVE integer-only / float support. Only doing FP data processing on the FP
  // vector types is inhibited at integer-only level.
  const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
  for (auto VT : LongTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    setAllExpand(VT);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
  }
  // We can do bitwise operations on v2i64 vectors
  setOperationAction(ISD::AND, MVT::v2i64, Legal);
  setOperationAction(ISD::OR, MVT::v2i64, Legal);
  setOperationAction(ISD::XOR, MVT::v2i64, Legal);

  // It is legal to extload from v4i8 to v4i16 or v4i32.
  addAllExtLoads(MVT::v8i16, MVT::v8i8, Legal);
  addAllExtLoads(MVT::v4i32, MVT::v4i16, Legal);
  addAllExtLoads(MVT::v4i32, MVT::v4i8, Legal);

  // Some truncating stores are legal too.
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
  setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
  setTruncStoreAction(MVT::v8i16, MVT::v8i8,  Legal);

  // Pre and Post inc on these are legal, given the correct extends
  for (unsigned im = (unsigned)ISD::PRE_INC;
       im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
    setIndexedLoadAction(im, MVT::v8i8, Legal);
    setIndexedStoreAction(im, MVT::v8i8, Legal);
    setIndexedLoadAction(im, MVT::v4i8, Legal);
    setIndexedStoreAction(im, MVT::v4i8, Legal);
    setIndexedLoadAction(im, MVT::v4i16, Legal);
    setIndexedStoreAction(im, MVT::v4i16, Legal);
  }

  // Predicate types
  const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1};
  for (auto VT : pTypes) {
    addRegisterClass(VT, &ARM::VCCRRegClass);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
    setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
    setOperationAction(ISD::LOAD, VT, Custom);
    setOperationAction(ISD::STORE, VT, Custom);
  }
}

ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
                                     const ARMSubtarget &STI)
    : TargetLowering(TM), Subtarget(&STI) {
  RegInfo = Subtarget->getRegisterInfo();
  Itins = Subtarget->getInstrItineraryData();

  setBooleanContents(ZeroOrOneBooleanContent);
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);

  if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() &&
      !Subtarget->isTargetWatchOS()) {
    bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard;
    for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
      setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID),
                            IsHFTarget ? CallingConv::ARM_AAPCS_VFP
                                       : CallingConv::ARM_AAPCS);
  }

  if (Subtarget->isTargetMachO()) {
    // Uses VFP for Thumb libfuncs if available.
    if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
        Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
      static const struct {
        const RTLIB::Libcall Op;
        const char * const Name;
        const ISD::CondCode Cond;
      } LibraryCalls[] = {
        // Single-precision floating-point arithmetic.
        { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID },
        { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID },
        { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID },
        { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID },

        // Double-precision floating-point arithmetic.
        { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID },
        { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID },
        { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID },
        { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID },

        // Single-precision comparisons.
        { RTLIB::OEQ_F32, "__eqsf2vfp",    ISD::SETNE },
        { RTLIB::UNE_F32, "__nesf2vfp",    ISD::SETNE },
        { RTLIB::OLT_F32, "__ltsf2vfp",    ISD::SETNE },
        { RTLIB::OLE_F32, "__lesf2vfp",    ISD::SETNE },
        { RTLIB::OGE_F32, "__gesf2vfp",    ISD::SETNE },
        { RTLIB::OGT_F32, "__gtsf2vfp",    ISD::SETNE },
        { RTLIB::UO_F32,  "__unordsf2vfp", ISD::SETNE },
        { RTLIB::O_F32,   "__unordsf2vfp", ISD::SETEQ },

        // Double-precision comparisons.
        { RTLIB::OEQ_F64, "__eqdf2vfp",    ISD::SETNE },
        { RTLIB::UNE_F64, "__nedf2vfp",    ISD::SETNE },
        { RTLIB::OLT_F64, "__ltdf2vfp",    ISD::SETNE },
        { RTLIB::OLE_F64, "__ledf2vfp",    ISD::SETNE },
        { RTLIB::OGE_F64, "__gedf2vfp",    ISD::SETNE },
        { RTLIB::OGT_F64, "__gtdf2vfp",    ISD::SETNE },
        { RTLIB::UO_F64,  "__unorddf2vfp", ISD::SETNE },
        { RTLIB::O_F64,   "__unorddf2vfp", ISD::SETEQ },

        // Floating-point to integer conversions.
        // i64 conversions are done via library routines even when generating VFP
        // instructions, so use the same ones.
        { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp",    ISD::SETCC_INVALID },
        { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID },
        { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp",    ISD::SETCC_INVALID },
        { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID },

        // Conversions between floating types.
        { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp",  ISD::SETCC_INVALID },
        { RTLIB::FPEXT_F32_F64,   "__extendsfdf2vfp", ISD::SETCC_INVALID },

        // Integer to floating-point conversions.
        // i64 conversions are done via library routines even when generating VFP
        // instructions, so use the same ones.
        // FIXME: There appears to be some naming inconsistency in ARM libgcc:
        // e.g., __floatunsidf vs. __floatunssidfvfp.
        { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp",    ISD::SETCC_INVALID },
        { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID },
        { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp",    ISD::SETCC_INVALID },
        { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID },
      };

      for (const auto &LC : LibraryCalls) {
        setLibcallName(LC.Op, LC.Name);
        if (LC.Cond != ISD::SETCC_INVALID)
          setCmpLibcallCC(LC.Op, LC.Cond);
      }
    }
  }

  // These libcalls are not available in 32-bit.
  setLibcallName(RTLIB::SHL_I128, nullptr);
  setLibcallName(RTLIB::SRL_I128, nullptr);
  setLibcallName(RTLIB::SRA_I128, nullptr);

  // RTLIB
  if (Subtarget->isAAPCS_ABI() &&
      (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
       Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
    static const struct {
      const RTLIB::Libcall Op;
      const char * const Name;
      const CallingConv::ID CC;
      const ISD::CondCode Cond;
    } LibraryCalls[] = {
      // Double-precision floating-point arithmetic helper functions
      // RTABI chapter 4.1.2, Table 2
      { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Double-precision floating-point comparison helper functions
      // RTABI chapter 4.1.2, Table 3
      { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
      { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::UO_F64,  "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::O_F64,   "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },

      // Single-precision floating-point arithmetic helper functions
      // RTABI chapter 4.1.2, Table 4
      { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Single-precision floating-point comparison helper functions
      // RTABI chapter 4.1.2, Table 5
      { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
      { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::UO_F32,  "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
      { RTLIB::O_F32,   "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },

      // Floating-point to integer conversions.
      // RTABI chapter 4.1.2, Table 6
      { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Conversions between floating types.
      // RTABI chapter 4.1.2, Table 7
      { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::FPEXT_F32_F64,   "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Integer to floating-point conversions.
      // RTABI chapter 4.1.2, Table 8
      { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Long long helper functions
      // RTABI chapter 4.2, Table 9
      { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },

      // Integer division functions
      // RTABI chapter 4.3.1
      { RTLIB::SDIV_I8,  "__aeabi_idiv",     CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SDIV_I16, "__aeabi_idiv",     CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SDIV_I32, "__aeabi_idiv",     CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::SDIV_I64, "__aeabi_ldivmod",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UDIV_I8,  "__aeabi_uidiv",    CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UDIV_I16, "__aeabi_uidiv",    CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UDIV_I32, "__aeabi_uidiv",    CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
    };

    for (const auto &LC : LibraryCalls) {
      setLibcallName(LC.Op, LC.Name);
      setLibcallCallingConv(LC.Op, LC.CC);
      if (LC.Cond != ISD::SETCC_INVALID)
        setCmpLibcallCC(LC.Op, LC.Cond);
    }

    // EABI dependent RTLIB
    if (TM.Options.EABIVersion == EABI::EABI4 ||
        TM.Options.EABIVersion == EABI::EABI5) {
      static const struct {
        const RTLIB::Libcall Op;
        const char *const Name;
        const CallingConv::ID CC;
        const ISD::CondCode Cond;
      } MemOpsLibraryCalls[] = {
        // Memory operations
        // RTABI chapter 4.3.4
        { RTLIB::MEMCPY,  "__aeabi_memcpy",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
        { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
        { RTLIB::MEMSET,  "__aeabi_memset",  CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
      };

      for (const auto &LC : MemOpsLibraryCalls) {
        setLibcallName(LC.Op, LC.Name);
        setLibcallCallingConv(LC.Op, LC.CC);
        if (LC.Cond != ISD::SETCC_INVALID)
          setCmpLibcallCC(LC.Op, LC.Cond);
      }
    }
  }

  if (Subtarget->isTargetWindows()) {
    static const struct {
      const RTLIB::Libcall Op;
      const char * const Name;
      const CallingConv::ID CC;
    } LibraryCalls[] = {
      { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
      { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
    };

    for (const auto &LC : LibraryCalls) {
      setLibcallName(LC.Op, LC.Name);
      setLibcallCallingConv(LC.Op, LC.CC);
    }
  }

  // Use divmod compiler-rt calls for iOS 5.0 and later.
  if (Subtarget->isTargetMachO() &&
      !(Subtarget->isTargetIOS() &&
        Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
    setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
    setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
  }

  // The half <-> float conversion functions are always soft-float on
  // non-watchos platforms, but are needed for some targets which use a
  // hard-float calling convention by default.
  if (!Subtarget->isTargetWatchABI()) {
    if (Subtarget->isAAPCS_ABI()) {
      setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
      setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
      setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
    } else {
      setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
      setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
      setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
    }
  }

  // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
  // a __gnu_ prefix (which is the default).
  if (Subtarget->isTargetAEABI()) {
    static const struct {
      const RTLIB::Libcall Op;
      const char * const Name;
      const CallingConv::ID CC;
    } LibraryCalls[] = {
      { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS },
      { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS },
      { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS },
    };

    for (const auto &LC : LibraryCalls) {
      setLibcallName(LC.Op, LC.Name);
      setLibcallCallingConv(LC.Op, LC.CC);
    }
  }

  if (Subtarget->isThumb1Only())
    addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
  else
    addRegisterClass(MVT::i32, &ARM::GPRRegClass);

  if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
      Subtarget->hasFPRegs()) {
    addRegisterClass(MVT::f32, &ARM::SPRRegClass);
    addRegisterClass(MVT::f64, &ARM::DPRRegClass);
    if (!Subtarget->hasVFP2Base())
      setAllExpand(MVT::f32);
    if (!Subtarget->hasFP64())
      setAllExpand(MVT::f64);
  }

  if (Subtarget->hasFullFP16()) {
    addRegisterClass(MVT::f16, &ARM::HPRRegClass);
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
    setOperationAction(ISD::BITCAST, MVT::f16, Custom);

    setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
    setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
  }

  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
      setTruncStoreAction(VT, InnerVT, Expand);
      addAllExtLoads(VT, InnerVT, Expand);
    }

    setOperationAction(ISD::MULHS, VT, Expand);
    setOperationAction(ISD::SMUL_LOHI, VT, Expand);
    setOperationAction(ISD::MULHU, VT, Expand);
    setOperationAction(ISD::UMUL_LOHI, VT, Expand);

    setOperationAction(ISD::BSWAP, VT, Expand);
  }

  setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
  setOperationAction(ISD::ConstantFP, MVT::f64, Custom);

  setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
  setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);

  if (Subtarget->hasMVEIntegerOps())
    addMVEVectorTypes(Subtarget->hasMVEFloatOps());

  // Combine low-overhead loop intrinsics so that we can lower i1 types.
  if (Subtarget->hasLOB()) {
    setTargetDAGCombine(ISD::BRCOND);
    setTargetDAGCombine(ISD::BR_CC);
  }

  if (Subtarget->hasNEON()) {
    addDRTypeForNEON(MVT::v2f32);
    addDRTypeForNEON(MVT::v8i8);
    addDRTypeForNEON(MVT::v4i16);
    addDRTypeForNEON(MVT::v2i32);
    addDRTypeForNEON(MVT::v1i64);

    addQRTypeForNEON(MVT::v4f32);
    addQRTypeForNEON(MVT::v2f64);
    addQRTypeForNEON(MVT::v16i8);
    addQRTypeForNEON(MVT::v8i16);
    addQRTypeForNEON(MVT::v4i32);
    addQRTypeForNEON(MVT::v2i64);

    if (Subtarget->hasFullFP16()) {
      addQRTypeForNEON(MVT::v8f16);
      addDRTypeForNEON(MVT::v4f16);
    }
  }

  if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
    // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
    // none of Neon, MVE or VFP supports any arithmetic operations on it.
    setOperationAction(ISD::FADD, MVT::v2f64, Expand);
    setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
    setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
    // FIXME: Code duplication: FDIV and FREM are expanded always, see
    // ARMTargetLowering::addTypeForNEON method for details.
    setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
    setOperationAction(ISD::FREM, MVT::v2f64, Expand);
    // FIXME: Create unittest.
    // In another words, find a way when "copysign" appears in DAG with vector
    // operands.
    setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
    // FIXME: Code duplication: SETCC has custom operation action, see
    // ARMTargetLowering::addTypeForNEON method for details.
    setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
    // FIXME: Create unittest for FNEG and for FABS.
    setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
    setOperationAction(ISD::FABS, MVT::v2f64, Expand);
    setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
    setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
    setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
    setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
    setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
    setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
    // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR.
    setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
    setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
    setOperationAction(ISD::FMA, MVT::v2f64, Expand);
  }

  if (Subtarget->hasNEON()) {
    // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
    // supported for v4f32.
    setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
    setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
    setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
    setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
    setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
    setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
    setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
    setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);

    // Mark v2f32 intrinsics.
    setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
    setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
    setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
    setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
    setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
    setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
    setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
    setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);

    // Neon does not support some operations on v1i64 and v2i64 types.
    setOperationAction(ISD::MUL, MVT::v1i64, Expand);
    // Custom handling for some quad-vector types to detect VMULL.
    setOperationAction(ISD::MUL, MVT::v8i16, Custom);
    setOperationAction(ISD::MUL, MVT::v4i32, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
    // Custom handling for some vector types to avoid expensive expansions
    setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
    setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
    setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
    setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
    // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
    // a destination type that is wider than the source, and nor does
    // it have a FP_TO_[SU]INT instruction with a narrower destination than
    // source.
    setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
    setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);

    setOperationAction(ISD::FP_ROUND,   MVT::v2f32, Expand);
    setOperationAction(ISD::FP_EXTEND,  MVT::v2f64, Expand);

    // NEON does not have single instruction CTPOP for vectors with element
    // types wider than 8-bits.  However, custom lowering can leverage the
    // v8i8/v16i8 vcnt instruction.
    setOperationAction(ISD::CTPOP,      MVT::v2i32, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v4i32, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v4i16, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v8i16, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v1i64, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v2i64, Custom);

    setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
    setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);

    // NEON does not have single instruction CTTZ for vectors.
    setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
    setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
    setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
    setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);

    setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
    setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
    setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
    setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);

    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);

    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);

    // NEON only has FMA instructions as of VFP4.
    if (!Subtarget->hasVFP4Base()) {
      setOperationAction(ISD::FMA, MVT::v2f32, Expand);
      setOperationAction(ISD::FMA, MVT::v4f32, Expand);
    }

    setTargetDAGCombine(ISD::INTRINSIC_VOID);
    setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
    setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
    setTargetDAGCombine(ISD::SHL);
    setTargetDAGCombine(ISD::SRL);
    setTargetDAGCombine(ISD::SRA);
    setTargetDAGCombine(ISD::FP_TO_SINT);
    setTargetDAGCombine(ISD::FP_TO_UINT);
    setTargetDAGCombine(ISD::FDIV);
    setTargetDAGCombine(ISD::LOAD);

    // It is legal to extload from v4i8 to v4i16 or v4i32.
    for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
                   MVT::v2i32}) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
        setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
        setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
      }
    }
  }

  if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
    setTargetDAGCombine(ISD::BUILD_VECTOR);
    setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
    setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
    setTargetDAGCombine(ISD::STORE);
    setTargetDAGCombine(ISD::SIGN_EXTEND);
    setTargetDAGCombine(ISD::ZERO_EXTEND);
    setTargetDAGCombine(ISD::ANY_EXTEND);
  }

  if (!Subtarget->hasFP64()) {
    // When targeting a floating-point unit with only single-precision
    // operations, f64 is legal for the few double-precision instructions which
    // are present However, no double-precision operations other than moves,
    // loads and stores are provided by the hardware.
    setOperationAction(ISD::FADD,       MVT::f64, Expand);
    setOperationAction(ISD::FSUB,       MVT::f64, Expand);
    setOperationAction(ISD::FMUL,       MVT::f64, Expand);
    setOperationAction(ISD::FMA,        MVT::f64, Expand);
    setOperationAction(ISD::FDIV,       MVT::f64, Expand);
    setOperationAction(ISD::FREM,       MVT::f64, Expand);
    setOperationAction(ISD::FCOPYSIGN,  MVT::f64, Expand);
    setOperationAction(ISD::FGETSIGN,   MVT::f64, Expand);
    setOperationAction(ISD::FNEG,       MVT::f64, Expand);
    setOperationAction(ISD::FABS,       MVT::f64, Expand);
    setOperationAction(ISD::FSQRT,      MVT::f64, Expand);
    setOperationAction(ISD::FSIN,       MVT::f64, Expand);
    setOperationAction(ISD::FCOS,       MVT::f64, Expand);
    setOperationAction(ISD::FPOW,       MVT::f64, Expand);
    setOperationAction(ISD::FLOG,       MVT::f64, Expand);
    setOperationAction(ISD::FLOG2,      MVT::f64, Expand);
    setOperationAction(ISD::FLOG10,     MVT::f64, Expand);
    setOperationAction(ISD::FEXP,       MVT::f64, Expand);
    setOperationAction(ISD::FEXP2,      MVT::f64, Expand);
    setOperationAction(ISD::FCEIL,      MVT::f64, Expand);
    setOperationAction(ISD::FTRUNC,     MVT::f64, Expand);
    setOperationAction(ISD::FRINT,      MVT::f64, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
    setOperationAction(ISD::FFLOOR,     MVT::f64, Expand);
    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
    setOperationAction(ISD::FP_ROUND,   MVT::f32, Custom);
  }

  if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
    setOperationAction(ISD::FP_EXTEND,  MVT::f64, Custom);
    if (Subtarget->hasFullFP16())
      setOperationAction(ISD::FP_ROUND,  MVT::f16, Custom);
  }

  if (!Subtarget->hasFP16())
    setOperationAction(ISD::FP_EXTEND,  MVT::f32, Custom);

  if (!Subtarget->hasFP64())
    setOperationAction(ISD::FP_ROUND,  MVT::f32, Custom);

  computeRegisterProperties(Subtarget->getRegisterInfo());

  // ARM does not have floating-point extending loads.
  for (MVT VT : MVT::fp_valuetypes()) {
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
  }

  // ... or truncating stores
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f16, Expand);

  // ARM does not have i1 sign extending load.
  for (MVT VT : MVT::integer_valuetypes())
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);

  // ARM supports all 4 flavors of integer indexed load / store.
  if (!Subtarget->isThumb1Only()) {
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im,  MVT::i1,  Legal);
      setIndexedLoadAction(im,  MVT::i8,  Legal);
      setIndexedLoadAction(im,  MVT::i16, Legal);
      setIndexedLoadAction(im,  MVT::i32, Legal);
      setIndexedStoreAction(im, MVT::i1,  Legal);
      setIndexedStoreAction(im, MVT::i8,  Legal);
      setIndexedStoreAction(im, MVT::i16, Legal);
      setIndexedStoreAction(im, MVT::i32, Legal);
    }
  } else {
    // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
    setIndexedLoadAction(ISD::POST_INC, MVT::i32,  Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i32,  Legal);
  }

  setOperationAction(ISD::SADDO, MVT::i32, Custom);
  setOperationAction(ISD::UADDO, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
  setOperationAction(ISD::USUBO, MVT::i32, Custom);

  setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
  setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
  if (Subtarget->hasDSP()) {
    setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
    setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
    setOperationAction(ISD::SADDSAT, MVT::i16, Custom);
    setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
  }
  if (Subtarget->hasBaseDSP()) {
    setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
    setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
  }

  // i64 operation support.
  setOperationAction(ISD::MUL,     MVT::i64, Expand);
  setOperationAction(ISD::MULHU,   MVT::i32, Expand);
  if (Subtarget->isThumb1Only()) {
    setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
    setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
  }
  if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
      || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
    setOperationAction(ISD::MULHS, MVT::i32, Expand);

  setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRL,       MVT::i64, Custom);
  setOperationAction(ISD::SRA,       MVT::i64, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);

  // MVE lowers 64 bit shifts to lsll and lsrl
  // assuming that ISD::SRL and SRA of i64 are already marked custom
  if (Subtarget->hasMVEIntegerOps())
    setOperationAction(ISD::SHL, MVT::i64, Custom);

  // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
  if (Subtarget->isThumb1Only()) {
    setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
    setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
    setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
  }

  if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
    setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);

  // ARM does not have ROTL.
  setOperationAction(ISD::ROTL, MVT::i32, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);
  }
  setOperationAction(ISD::CTTZ,  MVT::i32, Custom);
  setOperationAction(ISD::CTPOP, MVT::i32, Expand);
  if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
    setOperationAction(ISD::CTLZ, MVT::i32, Expand);
    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, LibCall);
  }

  // @llvm.readcyclecounter requires the Performance Monitors extension.
  // Default to the 0 expansion on unsupported platforms.
  // FIXME: Technically there are older ARM CPUs that have
  // implementation-specific ways of obtaining this information.
  if (Subtarget->hasPerfMon())
    setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);

  // Only ARMv6 has BSWAP.
  if (!Subtarget->hasV6Ops())
    setOperationAction(ISD::BSWAP, MVT::i32, Expand);

  bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
                                        : Subtarget->hasDivideInARMMode();
  if (!hasDivide) {
    // These are expanded into libcalls if the cpu doesn't have HW divider.
    setOperationAction(ISD::SDIV,  MVT::i32, LibCall);
    setOperationAction(ISD::UDIV,  MVT::i32, LibCall);
  }

  if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
    setOperationAction(ISD::SDIV, MVT::i32, Custom);
    setOperationAction(ISD::UDIV, MVT::i32, Custom);

    setOperationAction(ISD::SDIV, MVT::i64, Custom);
    setOperationAction(ISD::UDIV, MVT::i64, Custom);
  }

  setOperationAction(ISD::SREM,  MVT::i32, Expand);
  setOperationAction(ISD::UREM,  MVT::i32, Expand);

  // Register based DivRem for AEABI (RTABI 4.2)
  if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
      Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
      Subtarget->isTargetWindows()) {
    setOperationAction(ISD::SREM, MVT::i64, Custom);
    setOperationAction(ISD::UREM, MVT::i64, Custom);
    HasStandaloneRem = false;

    if (Subtarget->isTargetWindows()) {
      const struct {
        const RTLIB::Libcall Op;
        const char * const Name;
        const CallingConv::ID CC;
      } LibraryCalls[] = {
        { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS },

        { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS },
      };

      for (const auto &LC : LibraryCalls) {
        setLibcallName(LC.Op, LC.Name);
        setLibcallCallingConv(LC.Op, LC.CC);
      }
    } else {
      const struct {
        const RTLIB::Libcall Op;
        const char * const Name;
        const CallingConv::ID CC;
      } LibraryCalls[] = {
        { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
        { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS },

        { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
        { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS },
      };

      for (const auto &LC : LibraryCalls) {
        setLibcallName(LC.Op, LC.Name);
        setLibcallCallingConv(LC.Op, LC.CC);
      }
    }

    setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
    setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
    setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
    setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
  } else {
    setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
    setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
  }

  if (Subtarget->isTargetWindows() && Subtarget->getTargetTriple().isOSMSVCRT())
    for (auto &VT : {MVT::f32, MVT::f64})
      setOperationAction(ISD::FPOWI, VT, Custom);

  setOperationAction(ISD::GlobalAddress, MVT::i32,   Custom);
  setOperationAction(ISD::ConstantPool,  MVT::i32,   Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
  setOperationAction(ISD::BlockAddress, MVT::i32, Custom);

  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);

  // Use the default implementation.
  setOperationAction(ISD::VASTART,            MVT::Other, Custom);
  setOperationAction(ISD::VAARG,              MVT::Other, Expand);
  setOperationAction(ISD::VACOPY,             MVT::Other, Expand);
  setOperationAction(ISD::VAEND,              MVT::Other, Expand);
  setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);

  if (Subtarget->isTargetWindows())
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
  else
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);

  // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
  // the default expansion.
  InsertFencesForAtomic = false;
  if (Subtarget->hasAnyDataBarrier() &&
      (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
    // ATOMIC_FENCE needs custom lowering; the others should have been expanded
    // to ldrex/strex loops already.
    setOperationAction(ISD::ATOMIC_FENCE,     MVT::Other, Custom);
    if (!Subtarget->isThumb() || !Subtarget->isMClass())
      setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i64, Custom);

    // On v8, we have particularly efficient implementations of atomic fences
    // if they can be combined with nearby atomic loads and stores.
    if (!Subtarget->hasAcquireRelease() ||
        getTargetMachine().getOptLevel() == 0) {
      // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
      InsertFencesForAtomic = true;
    }
  } else {
    // If there's anything we can use as a barrier, go through custom lowering
    // for ATOMIC_FENCE.
    // If target has DMB in thumb, Fences can be inserted.
    if (Subtarget->hasDataBarrier())
      InsertFencesForAtomic = true;

    setOperationAction(ISD::ATOMIC_FENCE,   MVT::Other,
                       Subtarget->hasAnyDataBarrier() ? Custom : Expand);

    // Set them all for expansion, which will force libcalls.
    setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_SWAP,      MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
    // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
    // Unordered/Monotonic case.
    if (!InsertFencesForAtomic) {
      setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
      setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
    }
  }

  setOperationAction(ISD::PREFETCH,         MVT::Other, Custom);

  // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
  if (!Subtarget->hasV6Ops()) {
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8,  Expand);
  }
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);

  if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
      !Subtarget->isThumb1Only()) {
    // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
    // iff target supports vfp2.
    setOperationAction(ISD::BITCAST, MVT::i64, Custom);
    setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
  }

  // We want to custom lower some of our intrinsics.
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
  setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
  if (Subtarget->useSjLjEH())
    setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");

  setOperationAction(ISD::SETCC,     MVT::i32, Expand);
  setOperationAction(ISD::SETCC,     MVT::f32, Expand);
  setOperationAction(ISD::SETCC,     MVT::f64, Expand);
  setOperationAction(ISD::SELECT,    MVT::i32, Custom);
  setOperationAction(ISD::SELECT,    MVT::f32, Custom);
  setOperationAction(ISD::SELECT,    MVT::f64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::SETCC,     MVT::f16, Expand);
    setOperationAction(ISD::SELECT,    MVT::f16, Custom);
    setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
  }

  setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom);

  setOperationAction(ISD::BRCOND,    MVT::Other, Custom);
  setOperationAction(ISD::BR_CC,     MVT::i32,   Custom);
  if (Subtarget->hasFullFP16())
      setOperationAction(ISD::BR_CC, MVT::f16,   Custom);
  setOperationAction(ISD::BR_CC,     MVT::f32,   Custom);
  setOperationAction(ISD::BR_CC,     MVT::f64,   Custom);
  setOperationAction(ISD::BR_JT,     MVT::Other, Custom);

  // We don't support sin/cos/fmod/copysign/pow
  setOperationAction(ISD::FSIN,      MVT::f64, Expand);
  setOperationAction(ISD::FSIN,      MVT::f32, Expand);
  setOperationAction(ISD::FCOS,      MVT::f32, Expand);
  setOperationAction(ISD::FCOS,      MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS,   MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS,   MVT::f32, Expand);
  setOperationAction(ISD::FREM,      MVT::f64, Expand);
  setOperationAction(ISD::FREM,      MVT::f32, Expand);
  if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
      !Subtarget->isThumb1Only()) {
    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
  }
  setOperationAction(ISD::FPOW,      MVT::f64, Expand);
  setOperationAction(ISD::FPOW,      MVT::f32, Expand);

  if (!Subtarget->hasVFP4Base()) {
    setOperationAction(ISD::FMA, MVT::f64, Expand);
    setOperationAction(ISD::FMA, MVT::f32, Expand);
  }

  // Various VFP goodness
  if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
    // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
    if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
      setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
      setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
    }

    // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
    if (!Subtarget->hasFP16()) {
      setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
      setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
    }
  }

  // Use __sincos_stret if available.
  if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
      getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
    setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
    setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
  }

  // FP-ARMv8 implements a lot of rounding-like FP operations.
  if (Subtarget->hasFPARMv8Base()) {
    setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
    setOperationAction(ISD::FCEIL, MVT::f32, Legal);
    setOperationAction(ISD::FROUND, MVT::f32, Legal);
    setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
    setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
    setOperationAction(ISD::FRINT, MVT::f32, Legal);
    setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
    setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
    if (Subtarget->hasNEON()) {
      setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
      setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
    }

    if (Subtarget->hasFP64()) {
      setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
      setOperationAction(ISD::FCEIL, MVT::f64, Legal);
      setOperationAction(ISD::FROUND, MVT::f64, Legal);
      setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
      setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
      setOperationAction(ISD::FRINT, MVT::f64, Legal);
      setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
    }
  }

  // FP16 often need to be promoted to call lib functions
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::FREM, MVT::f16, Promote);
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
    setOperationAction(ISD::FSIN, MVT::f16, Promote);
    setOperationAction(ISD::FCOS, MVT::f16, Promote);
    setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
    setOperationAction(ISD::FPOWI, MVT::f16, Promote);
    setOperationAction(ISD::FPOW, MVT::f16, Promote);
    setOperationAction(ISD::FEXP, MVT::f16, Promote);
    setOperationAction(ISD::FEXP2, MVT::f16, Promote);
    setOperationAction(ISD::FLOG, MVT::f16, Promote);
    setOperationAction(ISD::FLOG10, MVT::f16, Promote);
    setOperationAction(ISD::FLOG2, MVT::f16, Promote);

    setOperationAction(ISD::FROUND, MVT::f16, Legal);
  }

  if (Subtarget->hasNEON()) {
    // vmin and vmax aren't available in a scalar form, so we use
    // a NEON instruction with an undef lane instead.
    setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);

    if (Subtarget->hasFullFP16()) {
      setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMINNUM, MVT::v8f16, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v8f16, Legal);

      setOperationAction(ISD::FMINIMUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
      setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
    }
  }

  // We have target-specific dag combine patterns for the following nodes:
  // ARMISD::VMOVRRD  - No need to call setTargetDAGCombine
  setTargetDAGCombine(ISD::ADD);
  setTargetDAGCombine(ISD::SUB);
  setTargetDAGCombine(ISD::MUL);
  setTargetDAGCombine(ISD::AND);
  setTargetDAGCombine(ISD::OR);
  setTargetDAGCombine(ISD::XOR);

  if (Subtarget->hasV6Ops())
    setTargetDAGCombine(ISD::SRL);
  if (Subtarget->isThumb1Only())
    setTargetDAGCombine(ISD::SHL);

  setStackPointerRegisterToSaveRestore(ARM::SP);

  if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
      !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
    setSchedulingPreference(Sched::RegPressure);
  else
    setSchedulingPreference(Sched::Hybrid);

  //// temporary - rewrite interface to use type
  MaxStoresPerMemset = 8;
  MaxStoresPerMemsetOptSize = 4;
  MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
  MaxStoresPerMemcpyOptSize = 2;
  MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
  MaxStoresPerMemmoveOptSize = 2;

  // On ARM arguments smaller than 4 bytes are extended, so all arguments
  // are at least 4 bytes aligned.
  setMinStackArgumentAlignment(Align(4));

  // Prefer likely predicted branches to selects on out-of-order cores.
  PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();

  setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));

  setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));

  if (Subtarget->isThumb() || Subtarget->isThumb2())
    setTargetDAGCombine(ISD::ABS);
}

bool ARMTargetLowering::useSoftFloat() const {
  return Subtarget->useSoftFloat();
}

// FIXME: It might make sense to define the representative register class as the
// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
// SPR's representative would be DPR_VFP2. This should work well if register
// pressure tracking were modified such that a register use would increment the
// pressure of the register class's representative and all of it's super
// classes' representatives transitively. We have not implemented this because
// of the difficulty prior to coalescing of modeling operand register classes
// due to the common occurrence of cross class copies and subregister insertions
// and extractions.
std::pair<const TargetRegisterClass *, uint8_t>
ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
                                           MVT VT) const {
  const TargetRegisterClass *RRC = nullptr;
  uint8_t Cost = 1;
  switch (VT.SimpleTy) {
  default:
    return TargetLowering::findRepresentativeClass(TRI, VT);
  // Use DPR as representative register class for all floating point
  // and vector types. Since there are 32 SPR registers and 32 DPR registers so
  // the cost is 1 for both f32 and f64.
  case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
  case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
    RRC = &ARM::DPRRegClass;
    // When NEON is used for SP, only half of the register file is available
    // because operations that define both SP and DP results will be constrained
    // to the VFP2 class (D0-D15). We currently model this constraint prior to
    // coalescing by double-counting the SP regs. See the FIXME above.
    if (Subtarget->useNEONForSinglePrecisionFP())
      Cost = 2;
    break;
  case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
  case MVT::v4f32: case MVT::v2f64:
    RRC = &ARM::DPRRegClass;
    Cost = 2;
    break;
  case MVT::v4i64:
    RRC = &ARM::DPRRegClass;
    Cost = 4;
    break;
  case MVT::v8i64:
    RRC = &ARM::DPRRegClass;
    Cost = 8;
    break;
  }
  return std::make_pair(RRC, Cost);
}

const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
  switch ((ARMISD::NodeType)Opcode) {
  case ARMISD::FIRST_NUMBER:  break;
  case ARMISD::Wrapper:       return "ARMISD::Wrapper";
  case ARMISD::WrapperPIC:    return "ARMISD::WrapperPIC";
  case ARMISD::WrapperJT:     return "ARMISD::WrapperJT";
  case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL";
  case ARMISD::CALL:          return "ARMISD::CALL";
  case ARMISD::CALL_PRED:     return "ARMISD::CALL_PRED";
  case ARMISD::CALL_NOLINK:   return "ARMISD::CALL_NOLINK";
  case ARMISD::BRCOND:        return "ARMISD::BRCOND";
  case ARMISD::BR_JT:         return "ARMISD::BR_JT";
  case ARMISD::BR2_JT:        return "ARMISD::BR2_JT";
  case ARMISD::RET_FLAG:      return "ARMISD::RET_FLAG";
  case ARMISD::INTRET_FLAG:   return "ARMISD::INTRET_FLAG";
  case ARMISD::PIC_ADD:       return "ARMISD::PIC_ADD";
  case ARMISD::CMP:           return "ARMISD::CMP";
  case ARMISD::CMN:           return "ARMISD::CMN";
  case ARMISD::CMPZ:          return "ARMISD::CMPZ";
  case ARMISD::CMPFP:         return "ARMISD::CMPFP";
  case ARMISD::CMPFPw0:       return "ARMISD::CMPFPw0";
  case ARMISD::BCC_i64:       return "ARMISD::BCC_i64";
  case ARMISD::FMSTAT:        return "ARMISD::FMSTAT";

  case ARMISD::CMOV:          return "ARMISD::CMOV";
  case ARMISD::SUBS:          return "ARMISD::SUBS";

  case ARMISD::SSAT:          return "ARMISD::SSAT";
  case ARMISD::USAT:          return "ARMISD::USAT";

  case ARMISD::ASRL:          return "ARMISD::ASRL";
  case ARMISD::LSRL:          return "ARMISD::LSRL";
  case ARMISD::LSLL:          return "ARMISD::LSLL";

  case ARMISD::SRL_FLAG:      return "ARMISD::SRL_FLAG";
  case ARMISD::SRA_FLAG:      return "ARMISD::SRA_FLAG";
  case ARMISD::RRX:           return "ARMISD::RRX";

  case ARMISD::ADDC:          return "ARMISD::ADDC";
  case ARMISD::ADDE:          return "ARMISD::ADDE";
  case ARMISD::SUBC:          return "ARMISD::SUBC";
  case ARMISD::SUBE:          return "ARMISD::SUBE";
  case ARMISD::LSLS:          return "ARMISD::LSLS";

  case ARMISD::VMOVRRD:       return "ARMISD::VMOVRRD";
  case ARMISD::VMOVDRR:       return "ARMISD::VMOVDRR";
  case ARMISD::VMOVhr:        return "ARMISD::VMOVhr";
  case ARMISD::VMOVrh:        return "ARMISD::VMOVrh";
  case ARMISD::VMOVSR:        return "ARMISD::VMOVSR";

  case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP";
  case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP";
  case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH";

  case ARMISD::TC_RETURN:     return "ARMISD::TC_RETURN";

  case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER";

  case ARMISD::DYN_ALLOC:     return "ARMISD::DYN_ALLOC";

  case ARMISD::